{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# [iterative-stratification](https://github.com/trent-b/iterative-stratification)\n\niterative-stratification is a project that provides scikit-learn compatible cross validation with stratification for multilabel data.\n\nscikit-learn provides module to split dataset with stratification. But, there is no option to stratify multilabel dataset.\n\niterative-stratification offers 'MultilabelStratifiedKFold', 'MultilabelRepeatedStratifiedKFold', 'MultilabelStratifiedShuffleSplit' methods which are suitable with multilabel dataset. ","metadata":{}},{"cell_type":"code","source":"pip install iterative-stratification","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/plant-pathology-2021-fgvc8/train.csv')\ndf.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(df)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import collections\n\nlabels = df['labels'].to_list()\nc = collections.Counter(labels)\nc","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sep_labels = []\nfor label in labels:\n    sep_labels.extend(label.split(' '))\ncs = collections.Counter(sep_labels)\ncs","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"key = {label:i for i,label in enumerate(cs.keys())}\nkey","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"text_to_category = {label:[] for label in cs.keys()}\nfor idx, item in df.iterrows():\n    for label in text_to_category:\n        if label in item['labels']:\n            text_to_category[label].append(1)\n        else:\n            text_to_category[label].append(0)\n\nfor label in text_to_category:\n    df[label] = text_to_category[label]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.express as px","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.parallel_categories(df[[\"healthy\", \"scab\", \"frog_eye_leaf_spot\", \"complex\",\"rust\",\"powdery_mildew\"]], color=\"healthy\", color_continuous_scale=\"sunset\",\\\n                             title=\"Parallel categories plot of targets\")\nfig.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X,Y = df['image'].to_numpy(), df[[\"healthy\", \"scab\", \"frog_eye_leaf_spot\", \"complex\",\"rust\",\"powdery_mildew\"]].to_numpy(dtype=np.float32)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from iterstrat.ml_stratifiers import MultilabelStratifiedKFold\n\nmsss = MultilabelStratifiedKFold(n_splits=5, shuffle=True, random_state=1234)\n\nfor train_index, test_index in msss.split(X, Y):\n    print(\"TRAIN:\", train_index, \"TEST:\", test_index)\n    X_train, X_test = X[train_index], X[test_index]\n    y_train, y_test = Y[train_index], Y[test_index]\n    \n    kfold_train_df = pd.DataFrame(columns=[\"healthy\", \"scab\", \"frog_eye_leaf_spot\", \"complex\",\"rust\",\"powdery_mildew\"], data=y_train)\n    kfold_test_df = pd.DataFrame(columns=[\"healthy\", \"scab\", \"frog_eye_leaf_spot\", \"complex\",\"rust\",\"powdery_mildew\"], data=y_test)\n    \n    fig_train = px.parallel_categories(kfold_train_df[[\"healthy\", \"scab\", \"frog_eye_leaf_spot\", \"complex\",\"rust\",\"powdery_mildew\"]], color=\"healthy\", color_continuous_scale=\"sunset\",\\\n                             title=\"categories plot of y_train\")\n    fig_test = px.parallel_categories(kfold_test_df[[\"healthy\", \"scab\", \"frog_eye_leaf_spot\", \"complex\",\"rust\",\"powdery_mildew\"]], color=\"healthy\", color_continuous_scale=\"sunset\",\\\n                             title=\"categories plot of y_test\")\n    \n    fig_train.show()\n    fig_test.show()\n    \n    break","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}