{"cells":[{"metadata":{},"cell_type":"markdown","source":"Hello everyone!\n\nToday I'd like to show you that not all of splitting methods are good for our tasks."},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install iterative-stratification\n!pip install scikit-multilearn","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport time\nimport numpy as np\nimport pandas as pd\n\nfrom sklearn.preprocessing import LabelBinarizer, MultiLabelBinarizer\nfrom sklearn.model_selection import train_test_split, StratifiedShuffleSplit\nfrom iterstrat.ml_stratifiers import MultilabelStratifiedShuffleSplit\nfrom skmultilearn.model_selection import IterativeStratification","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DATASET_PATH = \"/kaggle/input/bengaliai-cv19/\"\nKEYS = [\"grapheme_root\", \"vowel_diacritic\", \"consonant_diacritic\"]\nTEST_SIZE = 0.1\nSEED = 69","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"def get_csv(dataset_path, name):\n    return pd.read_csv(os.path.join(dataset_path, f\"{name}.csv\"))\n\n\ndef count(df):\n    return df.groupby(KEYS).size().reset_index().rename(columns={0: \"size\"})\n\n\ndef split(dataset_path, test_size, stratification):\n    df = get_csv(dataset_path, name=\"train\")\n    img_ids = df[\"image_id\"]\n\n    if stratification == \"sklearn_random\":\n        train_set, valid_set = train_test_split(df[KEYS], test_size=test_size,\n                                                random_state=SEED, shuffle=True)\n    elif stratification == \"sklearn_stratified\":\n        splitter = StratifiedShuffleSplit(n_splits=1,\n                                          test_size=test_size,\n                                          random_state=SEED)\n\n        train_indcs, valid_indcs = next(splitter.split(X=img_ids, y=df[KEYS]))\n        train_set = df.loc[df.index.intersection(train_indcs)].copy()\n        valid_set = df.loc[df.index.intersection(valid_indcs)].copy()\n        \n    elif stratification == \"iterstrat\":\n\n        splitter = MultilabelStratifiedShuffleSplit(n_splits=1,\n                                                    test_size=test_size,\n                                                    random_state=SEED)\n\n        train_indcs, valid_indcs = next(splitter.split(X=img_ids, y=df[KEYS]))\n        train_set = df.loc[df.index.intersection(train_indcs)].copy()\n        valid_set = df.loc[df.index.intersection(valid_indcs)].copy()\n\n    elif stratification == \"skmultilearn\":\n        \n        splitter = IterativeStratification(n_splits=2, order=2, \n                                           sample_distribution_per_fold=[\n                                               test_size, 1.0-test_size])\n        \n        train_indcs, valid_indcs = next(splitter.split(X=img_ids, y=df[KEYS]))\n        train_set = df.loc[df.index.intersection(train_indcs)].copy()\n        valid_set = df.loc[df.index.intersection(valid_indcs)].copy()\n        \n    else:\n        raise ValueError(\"Try something else :)\")\n\n    return train_set, valid_set\n\ndef eval(train, valid):\n    \n    train_count, val_count = count(train), count(valid)\n    \n    total = train_count[\"size\"] + val_count[\"size\"]\n    train_part = train_count[\"size\"] / total\n    val_part = val_count[\"size\"] / total\n    relative = val_part / train_part\n    \n    \n    for k, v in {\"Train\": train_part, \"Valid\": val_part, \n                 \"Valid relative to train\": relative}.items():\n        print(\"-------------------------------------------------------------------\")\n        print(k)\n        print(v)\n        print(\",\".join([f\"{m}: {f(v):.2}\" \n                        for m, f in {\"min\": np.min, \"max\": np.max, \n                                     \"mean\": np.mean, \"std\": np.std}.items()]))\n        print(\"-------------------------------------------------------------------\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's try the most famous random train_test_split and look what are the partitions for every class triplet."},{"metadata":{"trusted":true},"cell_type":"code","source":"method = \"sklearn_random\"\n\n\nstart = time.time()\n\ntrain, valid = split(dataset_path=DATASET_PATH, \n                     test_size=TEST_SIZE, \n                     stratification=method)\n\nprint(f\"Dataset split done for {time.time() - start} seconds\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"eval(train, valid)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"As you can see, some validation triplets represent from 3% to 20% of the whole set, which seems to me not very stratified :)\n\nLet's see how other method works."},{"metadata":{"trusted":true},"cell_type":"code","source":"method = \"iterstrat\"\n\nstart = time.time()\n\ntrain, valid = split(dataset_path=DATASET_PATH, \n                     test_size=TEST_SIZE, \n                     stratification=method)\n\nprint(f\"Dataset split done for {time.time() - start} seconds\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"eval(train, valid)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Next method works veeeeery slow and also wrong, that's why I deleted it, but I left the code for you to try."},{"metadata":{"trusted":true},"cell_type":"code","source":"# method = \"skmultilearn\"\n\n# start = time.time()\n\n# valid, train = split(dataset_path=DATASET_PATH, \n#                      test_size=TEST_SIZE, \n#                      stratification=method)\n\n# print(f\"Dataset split done for {time.time() - start} seconds\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"And last method which stratify dataset which as for me suits much better."},{"metadata":{"trusted":true},"cell_type":"code","source":"method = \"sklearn_stratified\"\n\nstart = time.time()\n\ntrain, valid = split(dataset_path=DATASET_PATH, \n                     test_size=TEST_SIZE, \n                     stratification=method)\n\nprint(f\"Dataset split done for {time.time() - start} seconds\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"eval(train, valid)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"As you can see only last method with sklearn.StratifiedShuffleSplit splits train and val in appropriate stratified sizes."}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}