{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":14774,"databundleVersionId":875431,"sourceType":"competition"},{"sourceId":9900,"sourceType":"datasetVersion","datasetId":6209}],"dockerImageVersionId":28450,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Dependencies","metadata":{}},{"cell_type":"code","source":"import os\nimport shap\nimport random\nimport warnings\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom sklearn.utils import class_weight\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix, cohen_kappa_score\nfrom keras.models import Model\nfrom keras import optimizers, applications\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import EarlyStopping, ReduceLROnPlateau\nfrom keras.layers import Dense, Dropout, GlobalAveragePooling2D, Input\n\n# Set seeds to make the experiment more reproducible.\nfrom tensorflow import set_random_seed\ndef seed_everything(seed=0):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    set_random_seed(seed)\n\nseed = 0\nseed_everything(seed)\n\n%matplotlib inline\nsns.set(style=\"whitegrid\")\nwarnings.filterwarnings(\"ignore\")","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2024-02-08T16:32:36.808147Z","iopub.execute_input":"2024-02-08T16:32:36.808524Z","iopub.status.idle":"2024-02-08T16:32:36.834423Z","shell.execute_reply.started":"2024-02-08T16:32:36.808452Z","shell.execute_reply":"2024-02-08T16:32:36.833481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load data","metadata":{"_kg_hide-output":true}},{"cell_type":"code","source":"train = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')\ntest = pd.read_csv('../input/aptos2019-blindness-detection/test.csv')\nprint('Number of train samples: ', train.shape[0])\nprint('Number of test samples: ', test.shape[0])\n\n# Preprocecss data\ntrain[\"id_code\"] = train[\"id_code\"].apply(lambda x: x + \".png\")\ntest[\"id_code\"] = test[\"id_code\"].apply(lambda x: x + \".png\")\ntrain['diagnosis'] = train['diagnosis'].astype('str')\ndisplay(train.head())","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_kg_hide-input":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","execution":{"iopub.status.busy":"2024-02-08T16:32:36.83648Z","iopub.execute_input":"2024-02-08T16:32:36.836764Z","iopub.status.idle":"2024-02-08T16:32:36.891418Z","shell.execute_reply.started":"2024-02-08T16:32:36.836715Z","shell.execute_reply":"2024-02-08T16:32:36.89062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model parameters","metadata":{}},{"cell_type":"code","source":"# Model parameters\nBATCH_SIZE = 8\nEPOCHS = 40\nWARMUP_EPOCHS = 2\nLEARNING_RATE = 1e-4\nWARMUP_LEARNING_RATE = 1e-3\nHEIGHT = 320\nWIDTH = 320\nCANAL = 3\nN_CLASSES = train['diagnosis'].nunique()\nES_PATIENCE = 5\nRLROP_PATIENCE = 3\nDECAY_DROP = 0.5","metadata":{"execution":{"iopub.status.busy":"2024-02-08T16:32:36.89269Z","iopub.execute_input":"2024-02-08T16:32:36.892904Z","iopub.status.idle":"2024-02-08T16:32:36.898813Z","shell.execute_reply.started":"2024-02-08T16:32:36.892866Z","shell.execute_reply":"2024-02-08T16:32:36.897906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train test split","metadata":{}},{"cell_type":"code","source":"X_train, X_val = train_test_split(train, test_size=0.2, random_state=seed)","metadata":{"execution":{"iopub.status.busy":"2024-02-08T16:32:36.899962Z","iopub.execute_input":"2024-02-08T16:32:36.900208Z","iopub.status.idle":"2024-02-08T16:32:36.911067Z","shell.execute_reply.started":"2024-02-08T16:32:36.900165Z","shell.execute_reply":"2024-02-08T16:32:36.910379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data generator","metadata":{}},{"cell_type":"code","source":"train_datagen=ImageDataGenerator(rescale=1./255, \n                                 rotation_range=360,\n                                 horizontal_flip=True,\n                                 vertical_flip=True)\n\ntrain_generator=train_datagen.flow_from_dataframe(\n    dataframe=X_train,\n    directory=\"../input/aptos2019-blindness-detection/train_images/\",\n    x_col=\"id_code\",\n    y_col=\"diagnosis\",\n    class_mode=\"categorical\",\n    batch_size=BATCH_SIZE,\n    target_size=(HEIGHT, WIDTH),\n    seed=0)\n\nvalidation_datagen = ImageDataGenerator(rescale=1./255)\n\nvalid_generator=validation_datagen.flow_from_dataframe(\n    dataframe=X_val,\n    directory=\"../input/aptos2019-blindness-detection/train_images/\",\n    x_col=\"id_code\",\n    y_col=\"diagnosis\",\n    class_mode=\"categorical\", \n    batch_size=BATCH_SIZE,   \n    target_size=(HEIGHT, WIDTH),\n    seed=0)\n\ntest_datagen = ImageDataGenerator(rescale=1./255)\n\ntest_generator = test_datagen.flow_from_dataframe(  \n        dataframe=test,\n        directory = \"../input/aptos2019-blindness-detection/test_images/\",\n        x_col=\"id_code\",\n        batch_size=1,\n        class_mode=None,\n        shuffle=False,\n        target_size=(HEIGHT, WIDTH),\n        seed=0)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-08T16:32:36.913182Z","iopub.execute_input":"2024-02-08T16:32:36.913426Z","iopub.status.idle":"2024-02-08T16:32:48.081922Z","shell.execute_reply.started":"2024-02-08T16:32:36.913378Z","shell.execute_reply":"2024-02-08T16:32:48.081036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"def create_model(input_shape, n_out):\n    input_tensor = Input(shape=input_shape)\n    base_model = applications.ResNet50(weights=None, \n                                       include_top=False,\n                                       input_tensor=input_tensor)\n    base_model.load_weights('../input/resnet50/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5')\n\n    x = GlobalAveragePooling2D()(base_model.output)\n    x = Dropout(0.5)(x)\n    x = Dense(2048, activation='relu')(x)\n    x = Dropout(0.5)(x)\n    final_output = Dense(n_out, activation='softmax', name='final_output')(x)\n    model = Model(input_tensor, final_output)\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2024-02-08T16:32:48.08419Z","iopub.execute_input":"2024-02-08T16:32:48.084663Z","iopub.status.idle":"2024-02-08T16:32:48.091952Z","shell.execute_reply.started":"2024-02-08T16:32:48.084458Z","shell.execute_reply":"2024-02-08T16:32:48.091076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train top layers","metadata":{}},{"cell_type":"code","source":"model = create_model(input_shape=(HEIGHT, WIDTH, CANAL), n_out=N_CLASSES)\n\nfor layer in model.layers:\n    layer.trainable = False\n\nfor i in range(-5, 0):\n    model.layers[i].trainable = True\n    \nclass_weights = class_weight.compute_class_weight('balanced', np.unique(train['diagnosis'].astype('int').values), train['diagnosis'].astype('int').values)\n\nmetric_list = [\"accuracy\"]\noptimizer = optimizers.Adam(lr=WARMUP_LEARNING_RATE)\nmodel.compile(optimizer=optimizer, loss='categorical_crossentropy',  metrics=metric_list)\nmodel.summary()","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-02-08T16:32:48.093384Z","iopub.execute_input":"2024-02-08T16:32:48.0938Z","iopub.status.idle":"2024-02-08T16:33:01.033778Z","shell.execute_reply.started":"2024-02-08T16:32:48.093741Z","shell.execute_reply":"2024-02-08T16:33:01.03303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"STEP_SIZE_TRAIN = train_generator.n//train_generator.batch_size\nSTEP_SIZE_VALID = valid_generator.n//valid_generator.batch_size\n\nhistory_warmup = model.fit_generator(generator=train_generator,\n                                     steps_per_epoch=STEP_SIZE_TRAIN,\n                                     validation_data=valid_generator,\n                                     validation_steps=STEP_SIZE_VALID,\n                                     epochs=WARMUP_EPOCHS,\n                                     class_weight=class_weights,\n                                     verbose=1).history","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-02-08T16:33:01.03502Z","iopub.execute_input":"2024-02-08T16:33:01.035251Z","iopub.status.idle":"2024-02-08T16:49:01.048594Z","shell.execute_reply.started":"2024-02-08T16:33:01.035211Z","shell.execute_reply":"2024-02-08T16:49:01.047758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Fine-tune the complete model","metadata":{}},{"cell_type":"code","source":"for layer in model.layers:\n    layer.trainable = True\n\nes = EarlyStopping(monitor='val_loss', mode='min', patience=ES_PATIENCE, restore_best_weights=True, verbose=1)\nrlrop = ReduceLROnPlateau(monitor='val_loss', mode='min', patience=RLROP_PATIENCE, factor=DECAY_DROP, min_lr=1e-6, verbose=1)\n\ncallback_list = [es, rlrop]\noptimizer = optimizers.Adam(lr=LEARNING_RATE)\nmodel.compile(optimizer=optimizer, loss='categorical_crossentropy',  metrics=metric_list)\nmodel.summary()","metadata":{"_kg_hide-input":false,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-02-08T16:49:01.053876Z","iopub.execute_input":"2024-02-08T16:49:01.054138Z","iopub.status.idle":"2024-02-08T16:49:01.159077Z","shell.execute_reply.started":"2024-02-08T16:49:01.054096Z","shell.execute_reply":"2024-02-08T16:49:01.156632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history_finetunning = model.fit_generator(generator=train_generator,\n                                          steps_per_epoch=STEP_SIZE_TRAIN,\n                                          validation_data=valid_generator,\n                                          validation_steps=STEP_SIZE_VALID,\n                                          epochs=EPOCHS,\n                                          callbacks=callback_list,\n                                          class_weight=class_weights,\n                                          verbose=1).history","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-02-08T16:49:01.160734Z","iopub.execute_input":"2024-02-08T16:49:01.161064Z","iopub.status.idle":"2024-02-08T19:29:25.920418Z","shell.execute_reply.started":"2024-02-08T16:49:01.160974Z","shell.execute_reply":"2024-02-08T19:29:25.919268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model loss graph ","metadata":{}},{"cell_type":"code","source":"history = {'loss': history_warmup['loss'] + history_finetunning['loss'], \n           'val_loss': history_warmup['val_loss'] + history_finetunning['val_loss'], \n           'acc': history_warmup['acc'] + history_finetunning['acc'], \n           'val_acc': history_warmup['val_acc'] + history_finetunning['val_acc']}\n\nsns.set_style(\"whitegrid\")\nfig, (ax1, ax2) = plt.subplots(2, 1, sharex='col', figsize=(20, 14))\n\nax1.plot(history['loss'], label='Train loss')\nax1.plot(history['val_loss'], label='Validation loss')\nax1.legend(loc='best')\nax1.set_title('Loss')\n\nax2.plot(history['acc'], label='Train accuracy')\nax2.plot(history['val_acc'], label='Validation accuracy')\nax2.legend(loc='best')\nax2.set_title('Accuracy')\n\nplt.xlabel('Epochs')\nsns.despine()\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-08T19:29:25.922216Z","iopub.execute_input":"2024-02-08T19:29:25.922556Z","iopub.status.idle":"2024-02-08T19:29:26.593536Z","shell.execute_reply.started":"2024-02-08T19:29:25.922482Z","shell.execute_reply":"2024-02-08T19:29:26.592525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Evaluation\n\n## Confusion Matrix","metadata":{}},{"cell_type":"code","source":"import numpy as np\n# Create empty arays to keep the predictions and labels\nlastFullTrainPred = np.empty((0, N_CLASSES))\nlastFullTrainLabels = np.empty((0, N_CLASSES))\nlastFullValPred = np.empty((0, N_CLASSES))\nlastFullValLabels = np.empty((0, N_CLASSES))\n\n# Add train predictions and labels\nfor i in range(STEP_SIZE_TRAIN+1):\n    im, lbl = next(train_generator)\n    scores = model.predict(im, batch_size=train_generator.batch_size)\n    lastFullTrainPred = np.append(lastFullTrainPred, scores, axis=0)\n    lastFullTrainLabels = np.append(lastFullTrainLabels, lbl, axis=0)\n\n# Add validation predictions and labels\nfor i in range(STEP_SIZE_VALID+1):\n    im, lbl = next(valid_generator)\n    scores = model.predict(im, batch_size=valid_generator.batch_size)\n    lastFullValPred = np.append(lastFullValPred, scores, axis=0)\n    lastFullValLabels = np.append(lastFullValLabels, lbl, axis=0)\n    \n    \nlastFullComPred = np.concatenate((lastFullTrainPred, lastFullValPred))\nlastFullComLabels = np.concatenate((lastFullTrainLabels, lastFullValLabels))\ncomplete_labels = [np.argmax(label) for label in lastFullComLabels]\n\ntrain_preds = [np.argmax(pred) for pred in lastFullTrainPred]\ntrain_labels = [np.argmax(label) for label in lastFullTrainLabels]\nvalidation_preds = [np.argmax(pred) for pred in lastFullValPred]\nvalidation_labels = [np.argmax(label) for label in lastFullValLabels]","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-02-08T19:29:26.595022Z","iopub.execute_input":"2024-02-08T19:29:26.595342Z","iopub.status.idle":"2024-02-08T19:37:09.472731Z","shell.execute_reply.started":"2024-02-08T19:29:26.595279Z","shell.execute_reply":"2024-02-08T19:37:09.471869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, (ax1, ax2) = plt.subplots(1, 2, sharex='col', figsize=(24, 7))\nlabels = ['0 - No DR', '1 - Mild', '2 - Moderate', '3 - Severe', '4 - Proliferative DR']\ntrain_cnf_matrix = confusion_matrix(train_labels, train_preds)\nvalidation_cnf_matrix = confusion_matrix(validation_labels, validation_preds)\n\ntrain_cnf_matrix_norm = train_cnf_matrix.astype('float') / train_cnf_matrix.sum(axis=1)[:, np.newaxis]\nvalidation_cnf_matrix_norm = validation_cnf_matrix.astype('float') / validation_cnf_matrix.sum(axis=1)[:, np.newaxis]\n\ntrain_df_cm = pd.DataFrame(train_cnf_matrix_norm, index=labels, columns=labels)\nvalidation_df_cm = pd.DataFrame(validation_cnf_matrix_norm, index=labels, columns=labels)\n\nsns.heatmap(train_df_cm, annot=True, fmt='.2f', cmap=\"Blues\", ax=ax1).set_title('Train')\nsns.heatmap(validation_df_cm, annot=True, fmt='.2f', cmap=sns.cubehelix_palette(8), ax=ax2).set_title('Validation')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-08T19:37:09.474001Z","iopub.execute_input":"2024-02-08T19:37:09.474246Z","iopub.status.idle":"2024-02-08T19:37:10.446246Z","shell.execute_reply.started":"2024-02-08T19:37:09.474206Z","shell.execute_reply":"2024-02-08T19:37:10.445036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Quadratic Weighted Kappa","metadata":{}},{"cell_type":"code","source":"print(\"Train Cohen Kappa score: %.3f\" % cohen_kappa_score(train_preds,train_labels, weights='quadratic'))\nprint(\"Validation Cohen Kappa score: %.3f\" % cohen_kappa_score(validation_preds, validation_labels, weights='quadratic'))\nprint(\"Complete set Cohen Kappa score: %.3f\" % cohen_kappa_score(train_preds+validation_preds, train_labels+validation_labels, weights='quadratic'))","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-08T19:37:10.448576Z","iopub.execute_input":"2024-02-08T19:37:10.449183Z","iopub.status.idle":"2024-02-08T19:37:10.483715Z","shell.execute_reply.started":"2024-02-08T19:37:10.448933Z","shell.execute_reply":"2024-02-08T19:37:10.483065Z"},"trusted":true},"execution_count":null,"outputs":[]}]}