{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nprint(os.listdir(\"/kaggle/input/notebookf30568c6d4\"))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nprint(os.listdir(\"/kaggle/input/notebook540950b0f0\"))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nprint(tf.__version__)\nprint(keras.__version__)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport itertools\nimport os\nimport sys\nfrom prettytable import PrettyTable\nimport pickle\nfrom random import shuffle\nfrom sklearn.model_selection import train_test_split\nimport multiprocessing\nfrom multiprocessing.pool import ThreadPool\nfrom tqdm import tqdm_notebook\nprint(multiprocessing.cpu_count(),\" CPU cores\")\n\nimport seaborn as sns\n%matplotlib inline\nimport matplotlib.pyplot as plt\nplt.rcParams[\"axes.grid\"] = False\n\nfrom sklearn.metrics import confusion_matrix, cohen_kappa_score,accuracy_score\nfrom sklearn.preprocessing import MultiLabelBinarizer\n\nfrom PIL import Image\nimport cv2\nimport tensorflow as tf\n\nfrom tensorflow import keras\nfrom tensorflow.keras import applications\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras import optimizers,Model,Sequential\nfrom tensorflow.keras.layers import Input,GlobalAveragePooling2D,Dropout,Dense,Activation,BatchNormalization,GlobalMaxPooling2D,concatenate,Flatten\nfrom tensorflow.keras.callbacks import EarlyStopping,ReduceLROnPlateau,Callback\nfrom tensorflow.keras.initializers import random_normal\nfrom tensorflow.keras.models import load_model\nfrom tensorflow.keras.losses import binary_crossentropy,categorical_crossentropy,mean_squared_error\nfrom tensorflow.keras import backend as K\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file = open('/kaggle/input/notebookf30568c6d4/df_train_train', 'rb')\ndf_train_train = pickle.load(file)\nfile.close()\n\nfile = open('/kaggle/input/notebookf30568c6d4/df_train_test', 'rb')\ndf_train_test = pickle.load(file)\nfile.close()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FACTOR = 4\nBATCH_SIZE = 8 * FACTOR\nIMG_SIZE = 512\nEPOCHS = 20\nWARMUP_EPOCHS = 5\nLEARNING_RATE = 1e-4 * FACTOR\nWARMUP_LEARNING_RATE = 1e-3 * FACTOR\nHEIGHT = 320\nWIDTH = 320\nCANAL = 3\nN_CLASSES = df_train_train['diagnosis'].nunique()\nES_PATIENCE = 5\nRLROP_PATIENCE = 3\nDECAY_DROP = 0.5\nTrain_Cohen_Kappa_Score = 0.832\nTrain_Accuracy_Score = 0.747\nTest_Cohen_Kappa_Score = 0.803\nTest_Accuracy_Score = 0.775\nLR_WARMUP_EPOCHS_1st = 2\nLR_WARMUP_EPOCHS_2nd = 5\nSTEP_SIZE = len(df_train_train) // BATCH_SIZE\nTOTAL_STEPS_1st = WARMUP_EPOCHS * STEP_SIZE\nTOTAL_STEPS_2nd = EPOCHS * STEP_SIZE\nWARMUP_STEPS_1st = LR_WARMUP_EPOCHS_1st * STEP_SIZE\nWARMUP_STEPS_2nd = LR_WARMUP_EPOCHS_2nd * STEP_SIZE","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ref - https://github.com/umbertogriffo/focal-loss-keras/blob/master/losses.py\n\n'''Below Functions create custom loss functions - Categorical Focal Loss, Binary Focal Loss'''\n\ndef binary_focal_loss(gamma=2., alpha=.25):\n    \"\"\"\n    Binary form of focal loss.\n      FL(p_t) = -alpha * (1 - p_t)**gamma * log(p_t)\n      where p = sigmoid(x), p_t = p or 1 - p depending on if the label is 1 or 0, respectively.\n    References:\n        https://arxiv.org/pdf/1708.02002.pdf\n    Usage:\n     model.compile(loss=[binary_focal_loss(alpha=.25, gamma=2)], metrics=[\"accuracy\"], optimizer=adam)\n    \"\"\"\n    def binary_focal_loss_fixed(y_true, y_pred):\n        \"\"\"\n        :param y_true: A tensor of the same shape as `y_pred`\n        :param y_pred:  A tensor resulting from a sigmoid\n        :return: Output tensor.\n        \"\"\"\n        pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))\n        pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))\n\n        epsilon = K.epsilon()\n        # clip to prevent NaN's and Inf's\n        pt_1 = K.clip(pt_1, epsilon, 1. - epsilon)\n        pt_0 = K.clip(pt_0, epsilon, 1. - epsilon)\n\n        return -K.mean(alpha * K.pow(1. - pt_1, gamma) * K.log(pt_1)) \\\n               -K.mean((1 - alpha) * K.pow(pt_0, gamma) * K.log(1. - pt_0))\n\n    return binary_focal_loss_fixed","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def categorical_focal_loss(gamma=2., alpha=.25):\n    \"\"\"\n    Softmax version of focal loss.\n           m\n      FL = ∑  -alpha * (1 - p_o,c)^gamma * y_o,c * log(p_o,c)\n          c=1\n      where m = number of classes, c = class and o = observation\n    Parameters:\n      alpha -- the same as weighing factor in balanced cross entropy\n      gamma -- focusing parameter for modulating factor (1-p)\n    Default value:\n      gamma -- 2.0 as mentioned in the paper\n      alpha -- 0.25 as mentioned in the paper\n    References:\n        Official paper: https://arxiv.org/pdf/1708.02002.pdf\n        https://www.tensorflow.org/api_docs/python/tf/keras/backend/categorical_crossentropy\n    Usage:\n     model.compile(loss=[categorical_focal_loss(alpha=.25, gamma=2)], metrics=[\"accuracy\"], optimizer=adam)\n    \"\"\"\n    def categorical_focal_loss_fixed(y_true, y_pred):\n        \"\"\"\n        :param y_true: A tensor of the same shape as `y_pred`\n        :param y_pred: A tensor resulting from a softmax\n        :return: Output tensor.\n        \"\"\"\n\n        # Scale predictions so that the class probas of each sample sum to 1\n        y_pred /= K.sum(y_pred, axis=-1, keepdims=True)\n\n        # Clip the prediction value to prevent NaN's and Inf's\n        epsilon = K.epsilon()\n        y_pred = K.clip(y_pred, epsilon, 1. - epsilon)\n\n        # Calculate Cross Entropy\n        cross_entropy = -y_true * K.log(y_pred)\n\n        # Calculate Focal Loss\n        loss = alpha * K.pow(1 - y_pred, gamma) * cross_entropy\n\n        # Compute mean loss in mini_batch\n        return K.mean(loss, axis=1)\n\n    return categorical_focal_loss_fixed","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plt_show1():\n    from matplotlib.pyplot import figure\n    img = cv2.imread('/kaggle/input/result/result-1.JPG')\n    gray = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    figure(figsize=(18, 16), dpi=90)\n    plt.axis('off')\n    plt.grid(b=None)\n    plt.imshow(gray)\n    plt.show()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ncomplete_datagen = ImageDataGenerator(rescale=1./255, rotation_range=360,brightness_range=[0.5, 1.5],\n                                     zoom_range=[1, 1.2],zca_whitening=True,horizontal_flip=True,\n                                     vertical_flip=True,fill_mode='constant')\ncomplete_generator = complete_datagen.flow_from_dataframe(dataframe=df_train_train,\n                                                          directory = \"/kaggle/input/notebookf30568c6d4/train_images_resized_preprocessed/\",\n                                                          x_col=\"file_name\",\n                                                          target_size=(HEIGHT, WIDTH),\n                                                          batch_size=1,\n                                                          shuffle=False,\n                                                          class_mode=None)\nSTEP_SIZE_COMPLETE = complete_generator.n//complete_generator.batch_size\nprint(complete_generator.n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_generator = complete_datagen.flow_from_dataframe(dataframe=df_train_test,\n                                                          directory = \"/kaggle/input/notebookf30568c6d4/test_images_resized_preprocessed/\",\n                                                          x_col=\"file_name\",\n                                                          target_size=(HEIGHT, WIDTH),\n                                                          batch_size=1,\n                                                          shuffle=False,\n                                                          class_mode=None)\nSTEP_SIZE_TEST = test_generator.n//test_generator.batch_size\nprint(test_generator.n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = load_model(\"/kaggle/input/notebook540950b0f0/model_main_training.h5\",custom_objects={'categorical_focal_loss_fixed':categorical_focal_loss(alpha=.25, gamma=2),\n                                                            'binary_focal_loss_fixed' : binary_focal_loss(alpha=.25, gamma=2)})\ntrain_preds = model.predict_generator(complete_generator, steps=STEP_SIZE_COMPLETE,verbose = 1)\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f = open(\"/kaggle/working/train_preds\",\"wb\")\npickle.dump(train_preds,f)\nf.close()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_preds = model.predict_generator(test_generator, steps=STEP_SIZE_TEST,verbose = 1)\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f = open(\"/kaggle/working/test_preds\",\"wb\")\npickle.dump(test_preds,f)\nf.close()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_preds[0].shape,train_preds[1].shape,train_preds[2].shape)\n\ntrain_output_regression = np.array(train_preds[0]).reshape(-1,1)\ntrain_output_classification = np.array(np.argmax(train_preds[1],axis = -1)).reshape(-1,1)\ntrain_output_ordinal_regression = np.array(np.sum(train_preds[2],axis = -1)).reshape(-1,1)\n\nprint(train_output_regression.shape,train_output_classification.shape,train_output_ordinal_regression.shape)\nX_train = np.hstack((train_output_regression,train_output_classification,train_output_ordinal_regression))\nprint(X_train.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(test_preds[0].shape,test_preds[1].shape,test_preds[2].shape)\n\ntest_output_regression = np.array(test_preds[0]).reshape(-1,1)\ntest_output_classification = np.array(np.argmax(test_preds[1],axis = -1)).reshape(-1,1)\ntest_output_ordinal_regression = np.array(np.sum(test_preds[2],axis = -1)).reshape(-1,1)\n\nprint(test_output_regression.shape,test_output_classification.shape,test_output_ordinal_regression.shape)\nX_test = np.hstack((test_output_regression,test_output_classification,test_output_ordinal_regression))\nprint(X_test.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_post = Sequential()\nmodel_post.add(Dense(1, activation='linear', input_shape=(3,)))\nmodel_post.compile(optimizer=optimizers.SGD(lr=LEARNING_RATE), loss='mean_squared_error', metrics=['mean_squared_error'])\n# model_post.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_df_train_test_np_array = np.array(df_train_test.diagnosis.values).astype(str).astype(float)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_df_train_train_np_array = np.array(df_train_train.diagnosis.values).astype(str).astype(float)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# history = model_post.fit(X_train,np.array(df_train_train.diagnosis.values),\n#                          batch_size=BATCH_SIZE,\n#                          epochs=50,\n#                          verbose=1,\n#                          validation_data = (X_test,np.array(df_train_test.diagnosis.values).astype(str).astype(int)))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model_post.fit(X_train,X_df_train_train_np_array,\n                         batch_size=BATCH_SIZE,\n                         epochs=50,\n                         verbose=1,\n                         validation_data = (X_test,X_df_train_test_np_array))\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_post.save(\"/kaggle/working/model_post_training.h5\")\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f = open(\"/kaggle/working/history_post_training\",\"wb\")\npickle.dump(history.history,f)\nf.close()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(25,5))\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('Linear Regression Model Loss - Post Training (50 epochs)')\nplt.ylabel('Loss (MSE)')\nplt.xlabel('Epoch')\nplt.legend(['Train', 'Validation'], loc='best')\nplt.xticks(range(1,51))\nplt.yticks(range(0,4))\nplt.gca().ticklabel_format(axis='both', style='plain', useOffset=False)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def classify(x):\n    if x < 0.5:\n        return 0\n    elif x < 1.5:\n        return 1\n    elif x < 2.5:\n        return 2\n    elif x < 3.5:\n        return 3\n    return 4","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels = model_post.predict(X_train,batch_size=BATCH_SIZE,verbose = 1)\ntrain_labels = np.apply_along_axis(classify, 1, train_labels)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_labels = model_post.predict(X_test,batch_size=BATCH_SIZE,verbose = 1)\ntest_labels = np.apply_along_axis(classify, 1, test_labels)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_conf_matrix(true,pred,classes):\n    cf = confusion_matrix(true, pred)\n    \n    df_cm = pd.DataFrame(cf, range(len(classes)), range(len(classes)))\n    plt.figure(figsize=(8,5.5))\n    sns.set(font_scale=1.4)\n    sns.heatmap(df_cm, annot=True, annot_kws={\"size\": 16},xticklabels = classes ,yticklabels = classes,fmt='g')\n    #sns.heatmap(df_cm, annot=True, annot_kws={\"size\": 16})\n    #sns.heatmap(df_cm, annot=True, fmt='.2f', cmap=\"Blues\")\n    # plt.show()\n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = ['0 - No DR', '1 - Mild', '2 - Moderate', '3 - Severe', '4 - Proliferative DR']\nplot_conf_matrix(list(df_train_test['diagnosis'].astype(int)),test_labels,labels)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnf_matrix = confusion_matrix(df_train_test['diagnosis'].astype('int'), test_labels)\ncnf_matrix_norm = cnf_matrix.astype('float') / cnf_matrix.sum(axis=1)[:, np.newaxis]\ndf_cm = pd.DataFrame(cnf_matrix_norm, index=labels, columns=labels)\nplt.figure(figsize=(16, 7))\nplt_show1()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Train_Cohen_Kappa_score = cohen_kappa_score(train_labels, df_train_train['diagnosis'].astype('int'), weights='quadratic')\nTrain_Accuracy_score = accuracy_score(df_train_train['diagnosis'].astype('int'),train_labels)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Train Cohen Kappa score: %.3f\" % Train_Cohen_Kappa_Score)\nprint(\"Train Accuracy score : %.3f\" % Train_Accuracy_Score)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Test_Cohen_Kappa_score = cohen_kappa_score(test_labels, df_train_test['diagnosis'].astype('int'), weights='quadratic')\nTest_Accuracy_score = accuracy_score(df_train_test['diagnosis'].astype('int'),test_labels)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Test Cohen Kappa score: %.3f\" % Test_Cohen_Kappa_Score)\nprint(\"Test Accuracy score : %.3f\" % Test_Accuracy_Score)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_datagen = ImageDataGenerator(rescale=1./255)\ntest_generator = test_datagen.flow_from_directory(\n        \"/kaggle/input/testing2/\",\n        target_size=(320, 320),\n        batch_size=1,\n        class_mode='binary',\n        shuffle=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_preds1 = model.predict(test_generator)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(test_preds1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_output_regression1 = np.array(test_preds1[0]).reshape(-1,1)\ntest_output_classification1 = np.array(np.argmax(test_preds1[1],axis = -1)).reshape(-1,1)\ntest_output_ordinal_regression1 = np.array(np.sum(test_preds1[2],axis = -1)).reshape(-1,1)\n\nprint(test_output_regression1.shape,test_output_classification1.shape,test_output_ordinal_regression1.shape)\nX_test1 = np.hstack((test_output_regression1,test_output_classification1,test_output_ordinal_regression1))\nprint(X_test1.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X1 = model_post.predict(X_test1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X11 = np.apply_along_axis(classify, 1, X1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(X11)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}