{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"### Transfer Learning using DenseNet-169-B5 ####\n\n### Kudos and Thanks to all Kaggle contributors, to make awesome datasets available ####","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# just to see the input dirs\n\n!ls ../input/\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#    for filename in filenames:\n#        print(os.path.join(dirname, filename))\n","metadata":{"execution":{"iopub.status.busy":"2021-11-01T13:56:47.582475Z","iopub.execute_input":"2021-11-01T13:56:47.582835Z","iopub.status.idle":"2021-11-01T13:56:48.399390Z","shell.execute_reply.started":"2021-11-01T13:56:47.582779Z","shell.execute_reply":"2021-11-01T13:56:48.398396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n\nimport json\nimport math\nimport os\n\nimport cv2\nfrom PIL import Image\nimport numpy as np\nfrom keras import layers\nfrom keras.applications import DenseNet121\nfrom keras.callbacks import Callback, ModelCheckpoint\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.models import Sequential\nfrom keras.optimizers import Adam\nimport matplotlib.pyplot as plt\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import cohen_kappa_score, accuracy_score\nimport scipy\nimport tensorflow as tf\nfrom tqdm import tqdm\nfrom keras.applications.densenet import DenseNet121\nimport seaborn as sns\nsns.set()\n\n\nfrom IPython.display import display\n\n\n%matplotlib inline\n\n# Keep iterating and changing. Grid search would be a better option here !\nEPOCHS = 50\nBATCH_SIZE = 16\nSEED = 20031976\nLRATE = 0.00005\nVERBOSE=0\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n#import os\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n#    for filename in filenames:\n#        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-01T13:56:51.462720Z","iopub.execute_input":"2021-11-01T13:56:51.463034Z","iopub.status.idle":"2021-11-01T13:56:51.479546Z","shell.execute_reply.started":"2021-11-01T13:56:51.462987Z","shell.execute_reply":"2021-11-01T13:56:51.478738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.random.seed(SEED)\ntf.set_random_seed(SEED)\n\ntrain_df = pd.read_csv('/kaggle/input/aptos2019-blindness-detection/train.csv')\ntest_df = pd.read_csv('/kaggle/input/aptos2019-blindness-detection/test.csv')\nprint(\"Datasets loaded..\")\n\nprint ('Train DF Shape', train_df.shape)","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2021-11-01T13:57:04.998006Z","iopub.execute_input":"2021-11-01T13:57:04.998346Z","iopub.status.idle":"2021-11-01T13:57:05.031850Z","shell.execute_reply.started":"2021-11-01T13:57:04.998293Z","shell.execute_reply":"2021-11-01T13:57:05.030908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Data\ndisplay(train_df.head(2))\ndisplay(test_df.head(2))\n\n# Print Shape of Data\nprint(\"train_df shape = \",train_df.shape)\nprint(\"test_df shape = \",test_df.shape)\n\n# Distribution of Training Data\ndisplay(train_df['diagnosis'].value_counts())\nsns.countplot(train_df['diagnosis'], color='black')\n","metadata":{"execution":{"iopub.status.busy":"2021-11-01T13:57:08.035409Z","iopub.execute_input":"2021-11-01T13:57:08.035720Z","iopub.status.idle":"2021-11-01T13:57:08.307020Z","shell.execute_reply.started":"2021-11-01T13:57:08.035659Z","shell.execute_reply":"2021-11-01T13:57:08.300723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def display_image(df, rows, columns):\n    fig=plt.figure(figsize=(10, 10))\n\n    for i in range(columns*rows):\n        image_path = df.loc[i,'id_code']\n        image_id = df.loc[i,'diagnosis']\n        img = cv2.imread(f'/kaggle/input/aptos2019-blindness-detection/train_images/{image_path}.png')\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        \n        fig.add_subplot(rows, columns, i+1)\n        plt.title(image_id)\n        plt.imshow(img)\n    \n    plt.tight_layout()\n    \ndef display_single_image(img):\n    fig=plt.figure(figsize=(10, 10))\n    plt.title('Sample Img')\n    plt.imshow(img)\n\ndisplay_image(train_df, 4, 4)\n\n# Now, I'm observing black borders around the retina, which is not of much interest.\n\nprint('Train DF Shape:',train_df.shape)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T13:57:15.117703Z","iopub.execute_input":"2021-11-01T13:57:15.118003Z","iopub.status.idle":"2021-11-01T13:57:23.107353Z","shell.execute_reply.started":"2021-11-01T13:57:15.117954Z","shell.execute_reply":"2021-11-01T13:57:23.106588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_ben_color(path, sigmaX=10):\n    image = cv2.imread(path)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    image = cv2.resize(image, (224, 224))\n    image=cv2.addWeighted ( image,4, cv2.GaussianBlur( image , (0,0) , sigmaX) ,-4 ,128)\n    return image\n\ndef preprocess_image(image_path, desired_size=224):\n    # Add Lighting, to improve quality\n    im = load_ben_color(image_path)\n    return im\n    \n\n# Trail-1 Under sampling by deleting oversized classes (Class:0)\ndef under_sample_make_all_same(df, categories, max_per_category):\n    df = pd.concat([df[df['diagnosis'] == c][:max_per_category] for c in categories])\n    df = df.sample(n=(max_per_category)*len(categories), replace=False, random_state=20031976)\n    df.index = np.arange(len(df))\n    return df\n# train_df = under_sample_make_all_same(train_df,[0,1,2,3,4], 193 ) \n#Under-sample class-0 (1805-805=1000) and Over-sample other classes so each class has 1000 entries\nprint('Train DF Shape:',train_df.shape)\ntrain_df = train_df.drop(train_df[train_df['diagnosis'] == 0].sample(n=805, replace=False).index)\n\nN = train_df.shape[0]\nx_train = np.empty((N, 224, 224, 3), dtype=np.uint8)\n#tqdm\nfor i, image_id in enumerate((train_df['id_code'])):\n    x_train[i, :, :, :] = preprocess_image(\n        f'/kaggle/input/aptos2019-blindness-detection/train_images/{image_id}.png'\n    )\n#     print('Preprocessing Image:',i)\n    \n    \nN = test_df.shape[0]\nx_test = np.empty((N, 224, 224, 3), dtype=np.uint8)\n\nfor i, image_id in enumerate((test_df['id_code'])):\n    x_test[i, :, :, :] = preprocess_image(\n        f'/kaggle/input/aptos2019-blindness-detection/test_images/{image_id}.png'\n    )\n#     print('Preprocessing Test Image:',i)\n    \ny_train = pd.get_dummies(train_df['diagnosis']).values\n","metadata":{"execution":{"iopub.status.busy":"2021-11-01T13:59:14.014860Z","iopub.execute_input":"2021-11-01T13:59:14.015197Z","iopub.status.idle":"2021-11-01T14:04:22.195394Z","shell.execute_reply.started":"2021-11-01T13:59:14.015129Z","shell.execute_reply":"2021-11-01T14:04:22.194620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# View the sample pre-processed images here\ndef display_single_image(img):\n    fig=plt.figure(figsize=(10, 10))\n    plt.title('Sample Img')\n    plt.imshow(img)\n\n# Training Images\nprint('X Train Shape:', x_train.shape)\ndisplay_single_image(x_train[0])\ndisplay_single_image(x_train[1])\n\n\n# Testing Images\nprint('X Test Shape:', x_test.shape)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T14:04:23.031699Z","iopub.execute_input":"2021-11-01T14:04:23.032367Z","iopub.status.idle":"2021-11-01T14:04:23.924642Z","shell.execute_reply.started":"2021-11-01T14:04:23.032316Z","shell.execute_reply":"2021-11-01T14:04:23.923898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's crop the images, to their region of interest\n# Credits to https://www.kaggle.com/taindow/pre-processing-train-and-test-images\ndef crop_image_from_gray(img,tol=7):\n    \n    if img.ndim ==2:\n        mask = img>tol\n        return img[np.ix_(mask.any(1),mask.any(0))]\n    elif img.ndim==3:\n        gray_img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n        mask = gray_img>tol        \n        check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0))].shape[0]\n        if (check_shape == 0):\n            return img\n        else:\n            img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n            img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n            img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n            img = np.stack([img1,img2,img3],axis=-1)\n        return img\n\ndef circle_crop(img):   \n    height, width, depth = img.shape    \n    \n    x = int(width/2)\n    y = int(height/2)\n    r = np.amin((x,y))\n    \n    circle_img = np.zeros((height, width), np.uint8)\n    cv2.circle(circle_img, (x,y), int(r), 1, thickness=-1)\n    img = cv2.bitwise_and(img, img, mask=circle_img)\n    img = crop_image_from_gray(img)\n    \n    return img \n\ncircle_crop_img = circle_crop(x_train[1])\n\ndisplay_single_image(circle_crop_img)\n\n# This has some work, to do. Not continuing any more :)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T14:04:37.319024Z","iopub.execute_input":"2021-11-01T14:04:37.319352Z","iopub.status.idle":"2021-11-01T14:04:37.720780Z","shell.execute_reply.started":"2021-11-01T14:04:37.319294Z","shell.execute_reply":"2021-11-01T14:04:37.719857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"x_train.shape=\",x_train.shape)\nprint(\"y_train.shape=\",y_train.shape)\nprint(\"x_test.shape=\",x_test.shape)\n","metadata":{"execution":{"iopub.status.busy":"2021-11-01T14:04:51.306196Z","iopub.execute_input":"2021-11-01T14:04:51.306534Z","iopub.status.idle":"2021-11-01T14:04:51.313564Z","shell.execute_reply.started":"2021-11-01T14:04:51.306477Z","shell.execute_reply":"2021-11-01T14:04:51.312475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Trail-2 Over sampling by increasing undersized classes\nfrom imblearn.over_sampling import SMOTE, ADASYN\nx_resampled, y_resampled = SMOTE(random_state=SEED).fit_sample(x_train.reshape(x_train.shape[0], -1), train_df['diagnosis'].ravel())\n\nprint(\"x_resampled.shape=\",x_resampled.shape)\nprint(\"y_resampled.shape=\",y_resampled.shape)\n\nx_train = x_resampled.reshape(x_resampled.shape[0], 224, 224, 3)\ny_train = pd.get_dummies(y_resampled).values\n\nprint(\"x_train.shape=\",x_train.shape)\nprint(\"y_train.shape=\",y_train.shape)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T14:13:22.104188Z","iopub.execute_input":"2021-11-01T14:13:22.104495Z","iopub.status.idle":"2021-11-01T14:13:22.140443Z","shell.execute_reply.started":"2021-11-01T14:13:22.104440Z","shell.execute_reply":"2021-11-01T14:13:22.139033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train_multi = np.empty(y_train.shape, dtype=y_train.dtype)\ny_train_multi[:, 4] = y_train[:, 4]\n\nfor i in range(3, -1, -1):\n    y_train_multi[:, i] = np.logical_or(y_train[:, i], y_train_multi[:, i+1])\n\nprint(\"Original y_train:\", y_train.sum(axis=0))\nprint(\"Multilabel version:\", y_train_multi.sum(axis=0))","metadata":{"execution":{"iopub.status.busy":"2021-11-01T14:13:37.595486Z","iopub.execute_input":"2021-11-01T14:13:37.595814Z","iopub.status.idle":"2021-11-01T14:13:37.606488Z","shell.execute_reply.started":"2021-11-01T14:13:37.595757Z","shell.execute_reply":"2021-11-01T14:13:37.605650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Split 85-15 training-validation sets\nx_sptrain, x_spval, y_sptrain, y_spval = train_test_split(\n    x_train, y_train_multi, \n    test_size=0.10, \n    random_state=SEED\n)\nprint(\"train-validation splitted ...\")","metadata":{"execution":{"iopub.status.busy":"2021-11-01T14:13:53.558110Z","iopub.execute_input":"2021-11-01T14:13:53.558399Z","iopub.status.idle":"2021-11-01T14:13:54.005404Z","shell.execute_reply.started":"2021-11-01T14:13:53.558347Z","shell.execute_reply":"2021-11-01T14:13:54.004390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_datagen():\n    return ImageDataGenerator(\n        zoom_range=0.10,        # set range for random zoom\n        fill_mode='constant',   # set mode for filling points outside the input boundaries\n        cval=0.,                # value used for fill_mode = \"constant\"\n        horizontal_flip=True,   # randomly flip images\n        vertical_flip=True,     # randomly flip images\n#         rotation_range=20       # Degree range for random rotations\n    )\n\n# Using original generator\ndata_generator = create_datagen().flow(x_sptrain, y_sptrain, batch_size=BATCH_SIZE, seed=SEED)\nprint(\"Image data augmentated ...\")","metadata":{"execution":{"iopub.status.busy":"2021-11-01T14:14:07.719893Z","iopub.execute_input":"2021-11-01T14:14:07.720313Z","iopub.status.idle":"2021-11-01T14:14:09.435173Z","shell.execute_reply.started":"2021-11-01T14:14:07.720135Z","shell.execute_reply":"2021-11-01T14:14:09.434137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define evaluation metrics\n\nimport keras.backend as K\n\ndef precision(y_true, y_pred):\n    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n    precision = true_positives / (predicted_positives + K.epsilon())\n    return precision\n\ndef recall(y_true, y_pred):\n    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n    recall = true_positives / (possible_positives + K.epsilon())\n    return recall\n\ndef fbeta_score(y_true, y_pred, beta=1):\n    if beta < 0:\n        raise ValueError('The lowest choosable beta is zero (only precision).')\n\n    # If there are no true positives, fix the F score at 0 like sklearn.\n    if K.sum(K.round(K.clip(y_true, 0, 1))) == 0:\n        return 0\n\n    p = precision(y_true, y_pred)\n    r = recall(y_true, y_pred)\n    bb = beta ** 2\n    fbeta_score = (1 + bb) * (p * r) / (bb * p + r + K.epsilon())\n    return fbeta_score\n\ndef fmeasure(y_true, y_pred):\n    return fbeta_score(y_true, y_pred, beta=1)\n\ndef mean_pred(y_true, y_pred):\n    return K.mean(y_pred)\n\ndef f1_score(y_true, y_pred):\n    p = precision(y_true, y_pred)\n    r = recall(y_true, y_pred)\n    return 2*(p*r) / (p+r+K.epsilon())\n\nprint(\"Evaluation metrics defined ...\")\n","metadata":{"execution":{"iopub.status.busy":"2021-11-01T14:14:27.631843Z","iopub.execute_input":"2021-11-01T14:14:27.632152Z","iopub.status.idle":"2021-11-01T14:14:27.647338Z","shell.execute_reply.started":"2021-11-01T14:14:27.632098Z","shell.execute_reply":"2021-11-01T14:14:27.646265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.applications import DenseNet169\n","metadata":{"execution":{"iopub.status.busy":"2021-11-01T14:14:38.404304Z","iopub.execute_input":"2021-11-01T14:14:38.404733Z","iopub.status.idle":"2021-11-01T14:14:38.408925Z","shell.execute_reply.started":"2021-11-01T14:14:38.404549Z","shell.execute_reply":"2021-11-01T14:14:38.408112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# Transfer Learning.. This is so COOL !!!\ndensenet = DenseNet169(\n    weights='/kaggle/input/densenet-keras/DenseNet-BC-169-32-no-top.h5',\n    include_top=False,\n    input_shape=(224,224,3)\n)\n\nmodel = Sequential()\nmodel.add(densenet)\nmodel.add(layers.GlobalAveragePooling2D())\nmodel.add(layers.Dropout(0.2))\nmodel.add(layers.Dense(5, activation='sigmoid'))\n\nmodel.compile(\n    loss='binary_crossentropy',\n    optimizer=Adam(lr=LRATE),\n    metrics=['accuracy',mean_pred, precision, recall, f1_score, fbeta_score, fmeasure]\n)\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-11-01T14:14:41.836781Z","iopub.execute_input":"2021-11-01T14:14:41.837131Z","iopub.status.idle":"2021-11-01T14:15:25.258362Z","shell.execute_reply.started":"2021-11-01T14:14:41.837087Z","shell.execute_reply":"2021-11-01T14:15:25.257580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# callback to keep track of kappa score during training\nclass KappaMetrics(Callback):\n    def on_train_begin(self, logs={}):\n        self.val_kappas = []\n        \n    def on_epoch_end(self, epoch, logs={}):\n        X_val, y_val = self.validation_data[:2]\n        y_val = y_val.sum(axis=1) - 1\n        \n        y_pred = self.model.predict(X_val) > 0.5\n        y_pred = y_pred.astype(int).sum(axis=1) - 1\n\n        _val_kappa = cohen_kappa_score(\n            y_val,\n            y_pred, \n            weights='quadratic'\n        )\n\n        self.val_kappas.append(_val_kappa)\n\n        print(f\"Epoch: {epoch+1} val_kappa: {_val_kappa:.4f}\")\n        \n        if _val_kappa == max(self.val_kappas):\n            print(\"Validation Kappa has improved. Saving model.\")\n            self.model.save('model.h5')\n\n        return\n    \nkappa_score = KappaMetrics()\n\n\n\nhistory = model.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / BATCH_SIZE,\n    epochs=EPOCHS,\n    validation_data=(x_spval, y_spval),\n    callbacks=[kappa_score],\n    verbose=VERBOSE\n)    ","metadata":{"execution":{"iopub.status.busy":"2021-11-01T14:16:17.521406Z","iopub.execute_input":"2021-11-01T14:16:17.521720Z","iopub.status.idle":"2021-11-01T15:33:35.788780Z","shell.execute_reply.started":"2021-11-01T14:16:17.521667Z","shell.execute_reply":"2021-11-01T15:33:35.787598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('history.json', 'w') as f:\n    json.dump(history.history, f)\n\nhistory_df = pd.DataFrame(history.history)\nhistory_df.head(EPOCHS)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T15:34:37.763388Z","iopub.execute_input":"2021-11-01T15:34:37.763796Z","iopub.status.idle":"2021-11-01T15:34:37.821751Z","shell.execute_reply.started":"2021-11-01T15:34:37.763726Z","shell.execute_reply":"2021-11-01T15:34:37.820865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_test = model.predict(x_test) > 0.5\ny_test = y_test.astype(int).sum(axis=1) - 1\n\ntest_df['diagnosis'] = y_test\ntest_df.to_csv('submission.csv',index=False)\ndisplay(test_df.head(5))\n\n","metadata":{"execution":{"iopub.status.busy":"2021-11-01T15:36:04.702857Z","iopub.execute_input":"2021-11-01T15:36:04.703147Z","iopub.status.idle":"2021-11-01T15:36:11.332761Z","shell.execute_reply.started":"2021-11-01T15:36:04.703099Z","shell.execute_reply":"2021-11-01T15:36:11.331954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}