{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import json\nimport math\nimport os\n\nimport cv2\nfrom PIL import Image\nimport numpy as np\nimport seaborn as sns\nfrom keras import layers\nfrom tensorflow.keras import applications \nfrom keras.applications import DenseNet121, MobileNetV2\nfrom keras.callbacks import Callback, ModelCheckpoint\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.models import Sequential\nfrom keras.optimizers import Adam\nimport matplotlib.pyplot as plt\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import cohen_kappa_score, accuracy_score, confusion_matrix\nimport scipy\nimport tensorflow as tf\nfrom tqdm import tqdm\n\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Set random seed for reproducibility.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"np.random.seed(2020)\ntf.set_random_seed(2020)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loading & Exploration","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/valid-and-test-ta/x_train_8.csv')\nvalid_df = pd.read_csv('../input/valid-and-test-ta/x_valid_8.csv')\ntest_df = pd.read_csv('../input/aptos2019-blindness-detection/test.csv')\nprint(train_df.shape)\nprint(valid_df.shape)\nprint(test_df.shape)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['diagnosis'].hist()\ntrain_df['diagnosis'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Mengecek apakah ukuran sudah sesuai\nprint('Number of train samples: ', train_df.shape[0])\nprint('Number of test samples: ', valid_df.shape[0])\n\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['diagnosis'].hist()\ntrain_df['diagnosis'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Displaying some Sample Images","execution_count":null},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"def display_samples(df, columns=4, rows=3):\n    fig=plt.figure(figsize=(5*columns, 4*rows))\n\n    for i in range(columns*rows):\n        image_path = df.loc[i,'id_code']\n        image_id = df.loc[i,'diagnosis']\n        img = cv2.imread(f'../input/aptos2019-blindness-detection/train_images/{image_path}.png')\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        \n        fig.add_subplot(rows, columns, i+1)\n        plt.title(image_id)\n        plt.imshow(img)\n    \n    plt.tight_layout()\n\ndisplay_samples(train_df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Resize Images\n\nWe will resize the images to 224x224, then create a single numpy array to hold the data.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocess_image(image_path, desired_size=224):\n    im = Image.open(image_path)\n    im = im.resize((desired_size, )*2, resample=Image.BILINEAR)\n    \n    return im","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"N = train_df.shape[0]\nx_train = np.empty((N, 224, 224, 3), dtype=np.float32)\n\nfor i, image_id in enumerate(tqdm(train_df['id_code'])):\n    x_train[i, :, :, :] = preprocess_image(\n        f'../input/aptos2019-blindness-detection/train_images/{image_id}.png'\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"N = valid_df.shape[0]\nx_val = np.empty((N, 224, 224, 3), dtype=np.float32)\n\nfor i, image_id in enumerate(tqdm(valid_df['id_code'])):\n    x_val[i, :, :, :] = preprocess_image(\n        f'../input/aptos2019-blindness-detection/train_images/{image_id}.png'\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train = train_df['diagnosis']\ny_val = valid_df['diagnosis']\nprint(x_train.shape)\nprint(y_train.shape)\nprint(x_val.shape)\nprint(y_val.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 32\n\ndef create_datagen():\n    return ImageDataGenerator(\n        zoom_range=0.1,  # set range for random zoom\n        rotation_range = 360,\n        fill_mode='constant',\n        cval=0.,\n        horizontal_flip=True,  # randomly flip images\n        vertical_flip=True,  # randomly flip images\n    )\n\n# Using original generator\ndata_generator = create_datagen().flow(x_train, y_train, batch_size=BATCH_SIZE, seed=2019)\n# Using Mixup\n#mixup_generator = MixupGenerator(x_train, y_train, batch_size=BATCH_SIZE, alpha=0.2, datagen=create_datagen())()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class_weights = {0: 0.40578462,\n                1: 1.98018018,\n                2: 0.73348165,\n                3: 3.78965517,\n                4: 2.47894737}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Quadratic Weighted Kappa\n\nQuadratic Weighted Kappa (QWK, the greek letter $\\kappa$), also known as Cohen's Kappa, is the official evaluation metric. For our kernel, we will use a custom callback to monitor the score, and plot it at the end.\n\n### What is Cohen Kappa?\n\nAccording to the [wikipedia article](https://en.wikipedia.org/wiki/Cohen%27s_kappa), we have\n> The definition of $\\kappa$ is:\n> $$\\kappa \\equiv \\frac{p_o - p_e}{1 - p_e}$$\n> where $p_o$ is the relative observed agreement among raters (identical to accuracy), and $p_e$ is the hypothetical probability of chance agreement, using the observed data to calculate the probabilities of each observer randomly seeing each category.\n\n### How is it computed?\n\nLet's take the example of a binary classification problem. Say we have:","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Creating keras callback for QWK","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class Metrics(Callback):\n    def on_train_begin(self, logs={}):\n        self.val_kappas = []\n\n    def on_epoch_end(self, epoch, logs={}):\n        X_val, y_val = self.validation_data[:2]\n        \n        y_pred = self.model.predict(X_val)\n        y_pred = np.clip(y_pred,0,4)\n        y_pred = y_pred.astype(int)\n\n        _val_kappa = cohen_kappa_score(\n            y_val,\n            y_pred, \n            weights='quadratic'\n        )\n\n        self.val_kappas.append(_val_kappa)\n\n        print(f\"val_kappa: {_val_kappa:.4f}\")\n        \n        if _val_kappa == max(self.val_kappas):\n            print(\"Validation Kappa has improved. Saving model.\")\n            self.model.save('model_8.h5')\n\n        return","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"kappa_metrics = Metrics()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model: MobileNetV2","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"densenet = MobileNetV2(\n    alpha = 1.3,\n    weights='../input/valid-and-test-ta/mobilenet_v2_weights_tf_dim_ordering_tf_kernels_1.3_224_no_top.h5',\n    include_top=False,\n    input_shape=(224,224,3)\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model():\n    model = Sequential()\n    model.add(densenet)\n    model.add(layers.GlobalAveragePooling2D())\n    model.add(layers.Dense(256))\n    model.add(layers.Dense(256))\n    model.add(layers.Dense(1))\n    \n    model.compile(\n        loss='mse',\n        optimizer=Adam(lr=0.0001),\n        metrics=['accuracy']\n    )\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = build_model()\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training & Evaluation","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / BATCH_SIZE,\n    epochs=100,\n    validation_data=(x_val, y_val),\n    callbacks=[kappa_metrics],\n    class_weight=class_weights\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history_df = pd.DataFrame(history.history)\nhistory_df[['loss', 'val_loss']].plot()\nhistory_df[['acc', 'val_acc']].plot()\nhistory_df.to_csv('CV_8.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(kappa_metrics.val_kappas)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Submit","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"model.load_weights('model_8.h5')\npred = model.predict(x_val)\ny_val_pred = pred\ny_val_pred = np.clip(y_val_pred,0,4)\ny_val_pred = y_val_pred.astype(int)\n\nlabels = ['0 - No DR', '1 - Mild', '2 - Moderate', '3 - Severe', '4 - Proliferative DR']\ncnf_matrix = confusion_matrix(valid_df['diagnosis'].astype('int'), y_val_pred)\ndf_cm = pd.DataFrame(cnf_matrix, index=labels, columns=labels)\nplt.figure(figsize=(16, 7))\nsns.heatmap(df_cm, annot=True, cmap=\"Blues\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"kappa_val = cohen_kappa_score(\n            valid_df['diagnosis'].astype('int'),\n            y_val_pred, \n            weights='quadratic'\n        )\nkappa_val","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred = model.predict(x_train)\ny_train_pred = pred\ny_train_pred = np.clip(y_train_pred,0,4)\ny_train_pred = y_train_pred.astype(int)\n\nlabels = ['0 - No DR', '1 - Mild', '2 - Moderate', '3 - Severe', '4 - Proliferative DR']\ncnf_matrix = confusion_matrix(train_df['diagnosis'].astype('int'), y_train_pred)\ndf_cm = pd.DataFrame(cnf_matrix, index=labels, columns=labels)\nplt.figure(figsize=(16, 7))\nsns.heatmap(df_cm, annot=True, cmap=\"Blues\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"kappa_train = cohen_kappa_score(\n            train_df['diagnosis'].astype('int'),\n            y_train_pred, \n            weights='quadratic'\n        )\nkappa_train","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}