{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Intro"},{"metadata":{},"cell_type":"markdown","source":"This notebook show how to use Keras to build and train a simple CNN with Keras to detect the components of Bengali graphemes. \nIt relies heavily on [this very interesting notebook from Kaushal Shah](https://www.kaggle.com/kaushal2896/bengali-graphemes-starter-eda-multi-output-cnn#Basic-Model) that lays the basics to get started and helped me a lot.  \nMain modifications are :\n- Added Keras callback to be able to track the competition metrics (weighted average of recall)\n- Modified training loop to train over all data before starting a new epoch (data format need to be changed to feather and saved as dataset before starting in order to not increase training time too much, see [this very handy notebook by Maxime Lenormand](https://www.kaggle.com/maxlenormand/cropping-to-character-resizing-images)). This remove loss spike when switching train dataset while training.\n- Giving a bit more structure / modularity by introducing functions for model creation and training\n- Higher image resolution used (80x80), once again see Maxime Lenormand notebook to see preprocessing"},{"metadata":{},"cell_type":"markdown","source":"# Librairies and modules importation"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import gc\nfrom tqdm import tqdm\n\n# OpenCV for image manipulation\nimport cv2\n\n# Librairies for data manipulation and linear algebra\nimport numpy as np \nimport pandas as pd\n\n# Scikit learn modules for data preprocessing and metrics calculation\nfrom sklearn.preprocessing import LabelBinarizer\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import recall_score\n\n# Keras modules for CNN construction and training\nfrom tensorflow import keras\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.models import Model\nfrom keras.layers import Dense,Conv2D,Flatten,MaxPool2D,Dropout,BatchNormalization, Input, AveragePooling2D\nfrom keras.optimizers import Adam\nfrom keras.callbacks import Callback\n\n# Visualization tools\nfrom keras.utils import plot_model\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Targets Loading"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train_target = pd.read_csv('/kaggle/input/bengaliai-cv19/train.csv', index_col='image_id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rootLB = LabelBinarizer()\nvowelLB = LabelBinarizer()\nconsonantLB = LabelBinarizer()\nrootLB.fit(train_target['grapheme_root'])\nvowelLB.fit(train_target['vowel_diacritic'])\nconsonantLB.fit(train_target['consonant_diacritic'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Custom modules"},{"metadata":{},"cell_type":"markdown","source":"## Custom metric callback"},{"metadata":{},"cell_type":"markdown","source":"Creation of a callback that will be called at the end of each epoch to compute the recall for the 3 targets and the final score (weighted average of recall scores)."},{"metadata":{"trusted":true},"cell_type":"code","source":"class validation_recall(keras.callbacks.Callback):\n    \n    def __init__(self, validation_data=()):\n        super(keras.callbacks.Callback, self).__init__()\n        self.X_val, self.y_val = validation_data\n    \n    def on_epoch_end(self, epoch, logs={}):\n\n        root_val = self.y_val[0]\n        vowel_val = self.y_val[1]\n        consonant_val = self.y_val[2]\n        \n        # Predictions on validation set for the 3 targets \n        #(+ one hot encoding)\n        preds = self.model.predict(self.X_val)\n        \n        root_preds = np.zeros_like(preds[0])\n        root_preds[np.arange(len(preds[0])), \n                   preds[0].argmax(axis=1)] = 1\n        \n        vowel_preds = np.zeros_like(preds[1])\n        vowel_preds[np.arange(len(preds[1])), \n                    preds[1].argmax(axis=1)] = 1\n        \n        consonant_preds = np.zeros_like(preds[2])\n        consonant_preds[np.arange(len(preds[2])), \n                        preds[2].argmax(axis=1)] = 1\n        \n        # Recall calculation for the 3 targets\n        root_recall = recall_score(root_val, root_preds, \n                                   average='macro')\n        vowel_recall = recall_score(vowel_val, vowel_preds, \n                                    average='macro')\n        consonant_recall = recall_score(consonant_val, consonant_preds, \n                                        average='macro')\n        final_score = np.average([root_recall, vowel_recall, \n                                  consonant_recall], weights=[2,1,1])\n        \n        # Displaying scores at the end of each epoch\n        print(' - root_recall: {0:.4f}'.format(root_recall)+\\\n              ' - vowel_recall: {0:.4f}'.format(vowel_recall)+\\\n              ' - consonant_recall: {0:.4f}'.format(consonant_recall)+\\\n              ' - final_score: {0:.4f}'.format(final_score))\n\n        # Saving scores in logs history\n        logs['root_recall'] = root_recall\n        logs['vowel_recall'] = vowel_recall\n        logs['consonant_recall'] = consonant_recall\n        logs['final_score'] = final_score","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Custom data loader"},{"metadata":{},"cell_type":"markdown","source":"Keras DataGenerator modified for multi-outputs model taken from [this very interesting notebook](https://www.kaggle.com/kaushal2896/bengali-graphemes-starter-eda-multi-output-cnn#Basic-Model) from Kaushal Shah. It will allow to generate a batch of data to feed the model while training to avoid memory shortage."},{"metadata":{"trusted":true},"cell_type":"code","source":"class MultiOutputDataGenerator(keras.preprocessing.image.ImageDataGenerator):\n\n    def flow(self,\n             x,\n             y=None,\n             batch_size=32,\n             shuffle=True,\n             sample_weight=None,\n             seed=None,\n             save_to_dir=None,\n             save_prefix='',\n             save_format='png',\n             subset=None):\n\n        targets = None\n        target_lengths = {}\n        ordered_outputs = []\n        for output, target in y.items():\n            if targets is None:\n                targets = target\n            else:\n                targets = np.concatenate((targets, target), axis=1)\n            target_lengths[output] = target.shape[1]\n            ordered_outputs.append(output)\n\n\n        for flowx, flowy in super().flow(x, targets, \n                                         batch_size=batch_size,\n                                         shuffle=shuffle):\n            target_dict = {}\n            i = 0\n            for output in ordered_outputs:\n                target_length = target_lengths[output]\n                target_dict[output] = flowy[:, i: i + target_length]\n                i += target_length\n\n            yield flowx, target_dict","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Resizing function"},{"metadata":{},"cell_type":"markdown","source":"Resizing function taken from [this very handy notebook by Maxime Lenormand](https://www.kaggle.com/maxlenormand/cropping-to-character-resizing-images). Train data have been preprocessed before and loaded as a dataset, the function will only be used on test set for preprocessing before inference."},{"metadata":{"trusted":true},"cell_type":"code","source":"HEIGHT = 137\nWIDTH = 236\nCROP_SIZE = 80","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def crop_and_resize_images(df, resized_df, resize_size = CROP_SIZE):\n    cropped_imgs = {}\n    for img_id in tqdm(range(df.shape[0])):\n        img = resized_df[img_id]\n        _, thresh = cv2.threshold(img, 30, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)\n        contours, _ = cv2.findContours(thresh,cv2.RETR_LIST,cv2.CHAIN_APPROX_SIMPLE)[-2:]\n        \n        idx = 0 \n        ls_xmin = []\n        ls_ymin = []\n        ls_xmax = []\n        ls_ymax = []\n        for cnt in contours:\n            idx += 1\n            x,y,w,h = cv2.boundingRect(cnt)\n            ls_xmin.append(x)\n            ls_ymin.append(y)\n            ls_xmax.append(x + w)\n            ls_ymax.append(y + h)\n        xmin = min(ls_xmin)\n        ymin = min(ls_ymin)\n        xmax = max(ls_xmax)\n        ymax = max(ls_ymax)\n\n        roi = img[ymin:ymax,xmin:xmax]\n        resized_roi = cv2.resize(roi, (resize_size, resize_size))\n        cropped_imgs[df.image_id[img_id]] = resized_roi.reshape(-1)\n        \n    resized = pd.DataFrame(cropped_imgs).T.reset_index()\n    resized.columns = resized.columns.astype(str)\n    resized.rename(columns={'index':'image_id'},inplace=True)\n    return resized #out_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model definition"},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_multi_outputs_model():\n    \n    # Input layer (change shape according to your image size)\n    inputs = Input(shape=(80,80,1))\n    \n    # First convolution block\n    x = Conv2D(32, (3, 3), padding=\"same\", activation='relu')(inputs)\n    x = Conv2D(32, (3, 3), padding=\"same\", activation='relu')(x)\n    x = Conv2D(32, (3, 3), padding=\"same\", activation='relu')(x)\n    x = Conv2D(32, (3, 3), padding=\"same\", activation='relu')(x)\n    x = BatchNormalization(momentum=0.9)(x)\n    x = MaxPool2D(pool_size=(2, 2))(x)\n    \n    # Second convolution block\n    x = Conv2D(64, (3, 3), padding=\"same\", activation='relu')(x)\n    x = Conv2D(64, (3, 3), padding=\"same\", activation='relu')(x)\n    x = Conv2D(64, (3, 3), padding=\"same\", activation='relu')(x)\n    x = Conv2D(64, (3, 3), padding=\"same\", activation='relu')(x)\n    x = BatchNormalization(momentum=0.9)(x)\n    x = MaxPool2D(pool_size=(2, 2))(x)\n    \n    # Third convolution block\n    x = Conv2D(128, (3, 3), padding=\"same\", activation='relu')(x)\n    x = Conv2D(128, (3, 3), padding=\"same\", activation='relu')(x)\n    x = Conv2D(128, (3, 3), padding=\"same\", activation='relu')(x)\n    x = Conv2D(128, (3, 3), padding=\"same\", activation='relu')(x)\n    x = BatchNormalization(momentum=0.9)(x)\n    x = MaxPool2D(pool_size=(2, 2))(x)\n    \n    # Fourth convolution block\n    x = Conv2D(256, (3, 3), padding=\"same\", activation='relu')(x)\n    x = Conv2D(256, (3, 3), padding=\"same\", activation='relu')(x)\n    x = Conv2D(256, (3, 3), padding=\"same\", activation='relu')(x)\n    x = Conv2D(256, (3, 3), padding=\"same\", activation='relu')(x)\n    x = BatchNormalization(momentum=0.9)(x)\n    x = AveragePooling2D(pool_size=(3, 3), padding='valid')(x)\n    \n    # Classification layers\n    x = Flatten()(x)\n    x = Dense(512, activation = \"relu\")(x)\n    x = Dropout(rate=0.3)(x)\n    x = Dense(256, activation = \"relu\")(x)\n    x = Dropout(rate=0.3)(x)\n    \n    # Output layers\n    root_out = Dense(168, activation='softmax', name='root_output')(x)\n    vowel_out = Dense(11, activation='softmax', name='vowel_output')(x)\n    consonant_out = Dense(7, activation='softmax', name='consonant_output')(x)\n    \n    # model assembly\n    model = Model(inputs=inputs, outputs=[root_out, vowel_out, consonant_out])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training function :"},{"metadata":{},"cell_type":"markdown","source":"This training function is using the following process :\n* Loading a set of images and targets data (from one of the four images files)\n* Reshape images data to numpy array with correct dimensions (80 x 80)\n* One hot encoding of the 3 targets\n* Separate all sets in two : training set and validation set\n* Creating Keras data generators to load data batch by batch in the model\n* Train the model\n\nThe inner loop train the model for 1 epoch on the four train sets one after the other and the outer loop allow to repeat this operation X times, thus controlling the number of epochs over all data."},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model(model, epochs, batch_size):\n    complete_history = []\n    for j in range(epochs):\n        print('current epoch : ' + str(j+1) + ' / ' + str(epochs))\n        for i in range(4):\n\n            # Images + targets loading and merging\n            train_data = pd.read_feather('/kaggle/input/bengali-graphemes-'\n                                         'croppedresized-100x100-train/'\n                                         f'train_data_{i}.feather')\n            train_data.set_index(keys=['image_id'], drop=True, inplace=True)\n            train_df = pd.merge(train_data, train_target, \n                                on='image_id').drop(['grapheme'], axis=1)\n\n            # Deleting previous variables to free up memory\n            del train_data\n            gc.collect()\n\n            # One hot encoding the 3 targets with sklearn LabelBinarizer\n            #rootLB = LabelBinarizer()\n            #vowelLB = LabelBinarizer()\n            #consonantLB = LabelBinarizer()\n            rootLabels = rootLB.transform(train_df['grapheme_root'])\n            vowelLabels = vowelLB.transform(train_df['vowel_diacritic'])\n            consonantLabels = consonantLB.transform(\n                              train_df['consonant_diacritic'])\n\n            # Reshape images data to numpy array with same dimensions \n            # as images (80 x 80)\n            train_df.drop(['grapheme_root', 'vowel_diacritic', \n                           'consonant_diacritic'], axis=1, inplace=True)\n            train_df = train_df / 255\n            train_images = train_df.values.reshape(-1, 80, 80, 1)\n\n            # Deleting previous variables to free up memory\n            del train_df\n            gc.collect()\n\n            # Creating Train / validation set for images and 3 targets\n            X_train, X_val, y_train_root, y_val_root, y_train_vowel, \\\n            y_val_vowel, y_train_consonant, y_val_consonant \\\n            = train_test_split(train_images, rootLabels, vowelLabels, \n                               consonantLabels, test_size=0.2, \n                               random_state=42)\n\n            # Deleting previous variables to free up memory\n            del train_images\n            del rootLabels, vowelLabels, consonantLabels\n            gc.collect()\n\n            # Creation of Keras Datagenerator configuration \n            # without data augmentation\n            datagen = MultiOutputDataGenerator(\n            featurewise_center=False,  \n            samplewise_center=False,\n            featurewise_std_normalization=False,\n            samplewise_std_normalization=False,\n            zca_whitening=False,\n            rotation_range=0,\n            zoom_range=0,\n            width_shift_range=0,\n            height_shift_range=0,\n            horizontal_flip=False,\n            vertical_flip=False)\n\n            ''' You can use this piece of code (and removing everything below it) to  \n                have insight of how the datagen works and how batch are structured\n\n            i = 0\n            for batch in datagen.flow(X_train, \n                                     {'root': y_train_root, \n                                     'vowel': y_train_vowel, \n                                     'consonant': y_train_consonant},\n                                      batch_size=1, save_to_dir=None):\n                print(i)\n                print('batch infos :')\n                print(f'type : {type(batch)}')\n                print(f'length type : {len(batch)}')\n                print('features infos :')\n                print(f'type : {type(batch[0])}')\n                print(f'shape : {batch[0].shape}')\n                print('Target infos :')\n                print(f' global type : {type(batch[1])}')\n                print(f'global shape : {len(batch[1])}')\n                print(f'first target shape : {batch[1][\"root\"].shape}')\n                print(f'second target shape : {batch[1][\"vowel\"].shape}')\n                print(f'third target shape : {batch[1][\"consonant\"].shape}')\n\n                i += 1\n                if i > 5:\n                    break  # otherwise the generator would loop indefinitely\n\n            '''\n            # creation of data generator for training set\n            train_generator = datagen.flow(X_train,\n                                           {'root_output': y_train_root, \n                                            'vowel_output': y_train_vowel, \n                                            'consonant_output': y_train_consonant},\n                                           batch_size=batch_size)\n\n            # creation of data generator for validation set\n            val_generator = datagen.flow(X_val,\n                                         {'root_output': y_val_root, \n                                          'vowel_output': y_val_vowel, \n                                          'consonant_output': y_val_consonant},\n                                         batch_size=batch_size)\n\n            # Creation of callback for metric calculation\n            # after each epoch\n            my_call = validation_recall(validation_data=(X_val,\n                                                         [y_val_root,\n                                                          y_val_vowel, \n                                                          y_val_consonant]))\n\n            # Model training\n            history = model.fit_generator(\n                    train_generator,\n                    steps_per_epoch=X_train.shape[0] // batch_size,\n                    epochs=1,\n                    validation_data=val_generator,\n                    validation_steps=X_val.shape[0] // batch_size,\n                    callbacks=[my_call])\n\n            # Appending training history to the one of previous step\n            complete_history.append(history.history)\n\n            del train_generator\n            del val_generator\n            del my_call\n            del X_train\n            del X_val\n            del y_train_root\n            del y_val_root\n            del y_train_vowel\n            del y_val_vowel\n            del y_train_consonant\n            del y_val_consonant\n            gc.collect()\n\n    return model, complete_history","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model training"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Model creation\nmodel = create_multi_outputs_model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Model structure\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Model compilation\nmodel.compile(optimizer='adam', loss='categorical_crossentropy', \n              metrics=None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"# Model training\ntrained_model, history = train_model(model=model, epochs=40, \n                                     batch_size=256)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Model saving\ntrained_model.save('multi_outputs_model_V2.h5')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Results"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Creation of dataframe containing results\nresults_df = pd.DataFrame(columns=['val_loss', 'val_root_output_loss', \n                                   'val_vowel_output_loss','val_consonant_output_loss',\n                                   'loss', 'root_output_loss','vowel_output_loss',\n                                   'consonant_output_loss', 'root_recall',\n                                   'vowel_recall', 'consonant_recall', 'final_score'])\nfor hist in history:\n    results_df = results_df.append(pd.DataFrame(hist), ignore_index=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display(results_df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Recall scores on validation set"},{"metadata":{"trusted":true},"cell_type":"code","source":"# displaying recall score on validation set\n\nfig, ax = plt.subplots(figsize=(16,8))\nax = sns.lineplot(data=results_df.loc[:,['root_recall', 'vowel_recall', \n                                         'consonant_recall', 'final_score']],\n                                         linestyle=None)\n\nax.set_xlabel('epochs', labelpad=15, fontsize=15)\nax.set_ylabel('score', labelpad=15, fontsize=15)\nax.set_title('Validation scores', pad=15, fontsize=20)\nplt.xticks(ticks=np.arange(0,results_df.shape[0],4), \n           labels=np.arange(0,int(results_df.shape[0]/4),1).tolist(), rotation=90)\nplt.yticks(np.arange(0,1.1,0.1))\nax.grid(axis='both', linestyle=':', linewidth=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Losses on validation set"},{"metadata":{"trusted":true},"cell_type":"code","source":"# displaying loss on validation set\n\nfig, ax1 = plt.subplots(figsize=(16,8))\nax1 = sns.lineplot(data=results_df.loc[:,['val_loss', 'val_root_output_loss', \n                                          'val_vowel_output_loss',\n                                          'val_consonant_output_loss']], linestyle=None)\nax1.set_xlabel('epochs', labelpad=15, fontsize=15)\nax1.set_ylabel('loss', labelpad=15, fontsize=15)\nax1.set_title('Validation losses', pad=15, fontsize=20)\nplt.xticks(ticks=np.arange(0,results_df.shape[0],4), \n           labels=np.arange(0,int(results_df.shape[0]/4),1).tolist(), rotation=90)\nax1.grid(axis='both', linestyle=':', linewidth=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Losses on training set"},{"metadata":{"trusted":true},"cell_type":"code","source":"# displaying loss on train set\n\nfig, ax2 = plt.subplots(figsize=(16,8))\nax2 = sns.lineplot(data=results_df.loc[:,['loss', 'root_output_loss',\n                                          'vowel_output_loss', \n                                          'consonant_output_loss']], linestyle=None)\nax2.set_xlabel('epochs', labelpad=15, fontsize=15)\nax2.set_ylabel('loss', labelpad=15, fontsize=15)\nax2.set_title('Training losses', pad=15, fontsize=20)\nplt.xticks(ticks=np.arange(0,results_df.shape[0],4), \n           labels=np.arange(0,int(results_df.shape[0]/4),1).tolist(), rotation=90)\nax2.grid(axis='both', linestyle=':', linewidth=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Inference"},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df = pd.DataFrame(columns=['row_id', 'target'])\nfor i in range(4): \n    \n    # Loading test data and reshaping them as array\n    test_df = pd.read_parquet(f'/kaggle/input/bengaliai-cv19/test_image_data_{i}.parquet')\n    resized = test_df.iloc[:, 1:32333].values.reshape(-1, HEIGHT, WIDTH)\n    \n    # Resizing and cropping test data like it was done for training data\n    cropped_df = crop_and_resize_images(test_df, resized, CROP_SIZE)\n    test_index = cropped_df['image_id'].values.tolist()\n    X_test = cropped_df.iloc[:,1:].values.reshape(-1, CROP_SIZE, CROP_SIZE, 1)/255\n    \n    del test_df\n    del resized\n    del cropped_df\n    \n    # make prediction on test images\n    preds = model.predict(X_test)\n    \n    del X_test\n    \n    # Converting predictions to 3 lists (one for each target)\n    root_preds = preds[0].argmax(axis=1).tolist()\n    vowel_preds = preds[1].argmax(axis=1).tolist()\n    consonant_preds = preds[2].argmax(axis=1).tolist()\n\n    # Creating a 3 lines dataframe (root, vowel, consonant) for each row \n    # in predictions lists and appending them together\n    for image_id, root, vowel, consonant in zip(test_index, root_preds, vowel_preds, \n                                                consonant_preds):\n        \n        root_row = {'row_id': image_id+'_grapheme_root', 'target':root}\n        vowel_row = {'row_id': image_id+'_vowel_diacritic', 'target':vowel}\n        consonant_row = {'row_id': image_id+'_vowel_diacritic', 'target':consonant}\n        submission_df = submission_df.append(pd.DataFrame([root_row, vowel_row, \n                                                           consonant_row]), \n                                             ignore_index=True)\n    \n    del preds\n    del root_preds\n    del vowel_preds\n    del consonant_preds\n    gc.collect\n\n# Save prediction as csv for submission\nsubmission_df.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display(submission_df)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}