{"cells":[{"metadata":{},"cell_type":"markdown","source":"### Reference:\n- https://www.kaggle.com/jakubwasikowski/stratified-group-k-fold-cross-validation\n- https://www.kaggle.com/bibek777/5fold-stratified-split"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":false,"_kg_hide-output":false,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":false},"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\n\nimport random\nfrom collections import Counter, defaultdict\n\nSEED = 1234","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# https://www.kaggle.com/jakubwasikowski/stratified-group-k-fold-cross-validation\n\ndef stratified_group_k_fold(X, y, groups, k, seed=None):\n    labels_num = np.max(y) + 1\n    y_counts_per_group = defaultdict(lambda: np.zeros(labels_num))\n    y_distr = Counter()\n    for label, g in zip(y, groups):\n        y_counts_per_group[g][label] += 1\n        y_distr[label] += 1\n\n    y_counts_per_fold = defaultdict(lambda: np.zeros(labels_num))\n    groups_per_fold = defaultdict(set)\n\n    def eval_y_counts_per_fold(y_counts, fold):\n        y_counts_per_fold[fold] += y_counts\n        std_per_label = []\n        for label in range(labels_num):\n            label_std = np.std([y_counts_per_fold[i][label] / y_distr[label] for i in range(k)])\n            std_per_label.append(label_std)\n        y_counts_per_fold[fold] -= y_counts\n        return np.mean(std_per_label)\n    \n    groups_and_y_counts = list(y_counts_per_group.items())\n    random.Random(seed).shuffle(groups_and_y_counts)\n\n    for g, y_counts in sorted(groups_and_y_counts, key=lambda x: -np.std(x[1])):\n        best_fold = None\n        min_eval = None\n        for i in range(k):\n            fold_eval = eval_y_counts_per_fold(y_counts, i)\n            if min_eval is None or fold_eval < min_eval:\n                min_eval = fold_eval\n                best_fold = i\n        y_counts_per_fold[best_fold] += y_counts\n        groups_per_fold[best_fold].add(g)\n\n    all_groups = set(groups)\n    for i in range(k):\n        train_groups = all_groups - groups_per_fold[i]\n        test_groups = groups_per_fold[i]\n\n        train_indices = [i for i, g in enumerate(groups) if g in train_groups]\n        test_indices = [i for i, g in enumerate(groups) if g in test_groups]\n\n        yield train_indices, test_indices","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"path = '../input/understanding_cloud_organization'\nos.listdir(path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"n_train = len(os.listdir(f'{path}/train_images'))\nn_test = len(os.listdir(f'{path}/test_images'))\nprint( f'There are {n_train} images in train dataset' )\nprint( f'There are {n_test} images in test dataset' )","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train_df = pd.read_csv(f'{path}/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train_df['class_name'] = train_df['Image_Label'].apply(lambda x: x.split('_')[1])\ntrain_df['image_name'] = train_df['Image_Label'].apply(lambda x: x.split('_')[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# https://www.kaggle.com/bibek777/5fold-stratified-split\n\ntrain_df['exists'] = train_df['EncodedPixels'].notnull().astype(int)\n\nclass_names_dict = {'Fish':1, 'Flower':2, 'Gravel':3, 'Sugar':4}\ntrain_df['class_id'] = train_df['class_name'].map(class_names_dict)\ntrain_df['class_id'] = [row.class_id if row.exists else 0 for row in train_df.itertuples()]\n\ntrain_df = train_df.sort_values( by='Image_Label', ascending=True )\ntrain_df.reset_index(drop=True, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"groups = train_df.image_name.values\nlabels = train_df.class_id.values\n\nsplits = list( stratified_group_k_fold( train_df, labels, groups, k=5, seed=SEED ) )","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"for i, (train_idx, valid_idx) in enumerate(splits):\n    \n    train = train_df.iloc[train_idx, :]  \n    valid = train_df.iloc[valid_idx, :]\n    \n    train_ids = train['image_name'].drop_duplicates().values\n    valid_ids = valid['image_name'].drop_duplicates().values    \n    \n    print( '=========================' )\n    print( 'K=', i+1 )\n    \n    print( '[train]' )\n    print( 'class: ', Counter(train['class_id']) )   \n    print( 'images: ', len(train_ids) )   \n    \n    print( '===' )\n    \n    print( '[valid]' )\n    print( 'class: ', Counter(valid['class_id']) )\n    print( 'images: ', len(valid_ids) )   ","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}