{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Summary\n\n### Note\nSome methods and codes in this kernel are adapted from another of my kernel, which contains more comprehensive citations: https://www.kaggle.com/tommzzhou/eda-and-preprocessing\n\n\n### Thanks to\nSomshubra Majumdar for his keras implementation of efficient Net\n\n@ Neuron Engineer for the dataset containing efficient net weights\n\n@ averagemn for the resized version of the previous dataset\n\n\n\n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport tensorflow as tf\nfrom tensorflow import keras\nimport cv2\nfrom functools import partial\nimport scipy as sp\n\n\n# For transfer learning\nimport tensorflow_hub\nfrom tensorflow.keras import applications\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport random\nimport albumentations\n\nfrom tqdm import tqdm\nfrom math import ceil\nimport math\nimport sys\nimport os\nimport gc\n\nfrom tensorflow.keras.activations import elu\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.models import Sequential, load_model\n\nfrom tensorflow.keras.callbacks import Callback, EarlyStopping, ReduceLROnPlateau\nfrom tensorflow.keras.layers import Dense, Conv2D, Flatten, GlobalAveragePooling2D, Dropout\n\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.model_selection import StratifiedKFold, train_test_split\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames[:4]:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Record the starting time of this commit\nimport time\nt_start = time.time()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls ../input/efficient-net-keras/efficientnet-master/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Repository source: https://github.com/qubvel/efficientnet\nsys.path.append(os.path.abspath('../input/efficient-net-keras/efficientnet-master/'))\n\nfrom efficientnet.tfkeras import EfficientNetB5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!which python3\ngpus = tf.config.experimental.list_physical_devices('GPU')\nfor gpu in gpus:\n    print(\"Name:\", gpu.name, \"  Type:\", gpu.device_type)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Basic Ideas\n\n* Write things in modules, more of an OOP style\n* Some **well-designed** train time data augmentation\n* Using keras Efficient Net\n* Strong and usable data generator\n\n\nTo-do List:\n\n[ ] Fine-tune the network with guidance from CS231n\n\n[ ] Making use of additional data\n\n[ ] MixUp augmentations \n\n[ ] Ways to deal with imbalanced data?\n\n[ ] Apply TTA and see if the results are good\n\n[ ] Ensemble models with Green Channels and Ben's method\n\n[ ] Clever way to split train and validation data\n\n\n"},{"metadata":{},"cell_type":"markdown","source":"## Preparation\n\n* Load the files in\n* Check compatability and versions\n* **Set gloabl variables**"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_file_name = \"/kaggle/input/aptos2019-blindness-detection/train.csv\"\ntest_file_name = \"/kaggle/input/aptos2019-blindness-detection/sample_submission.csv\"\nprevious_contest_file_name = \"/kaggle/input/retinopathy-train-2015/rescaled_train_896/trainLabels.csv\"\n\npreivous_contest_file_path = \"/kaggle/input/retinopathy-train-2015/rescaled_train_896/rescaled_train_896/\"\ntrain_file_path = \"/kaggle/input/aptos2019-blindness-detection/train_images/\"\ntest_file_path = \"/kaggle/input/aptos2019-blindness-detection/test_images/\"\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset_csv = pd.read_csv(train_file_name)\ntest_csv = pd.read_csv(test_file_name)\nprevious_contest_csv = pd.read_csv(previous_contest_file_name)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"print(tf.__version__)\nprint(cv2.__version__)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CLASS = {0: \"No DR\", 1: \"Mild\", 2: \"Moderate\", 3: \"Severe\", 4: \"Proliferative DR\"}\nSEED = 24\n\nBATCHSIZE = 16\n\nIMG_CHANNELS = 3\nIMG_WIDTH = 512\n\n# These are used for histogram equalization\nclipLimit=2.0 \ntileGridSize=(8, 8)  \n\n\nSAVED_MODEL_NAME = \"efficientB5.hdf5\"\nCUSTOM_WEIGHT = \"/kaggle/input/efficientn5/efficientB5.hdf5\"\n\nchannels = {\"R\":0, \"G\": 1, \"B\":2}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset_csv.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_csv.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"previous_contest_csv.columns = [\"id_code\", \"diagnosis\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"previous_contest_csv.head(5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Split and Check Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Check the random seed\nprint(SEED)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Split the data\nUsing stratify param ensures that the training and validation data have the same label distribution"},{"metadata":{"trusted":true},"cell_type":"code","source":"x_training, x_validation, y_training, y_validation = train_test_split(dataset_csv[\"id_code\"], dataset_csv[\"diagnosis\"],\n                                                    test_size=0.15,\n                                                    random_state=SEED,\n                                                    stratify=dataset_csv[\"diagnosis\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(type(x_training))\nprint(x_training[:5])\nprint(type(x_validation))\nprint(x_validation[:5])\n\n# Now check for y labels\nprint(type(y_validation))\nprint(y_validation[:5])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Merge the x_training and y_training to a single dataframe \n# This is done for future convenience\n\ntrain_csv = pd.DataFrame(columns = ['id_code', 'diagnosis'])\ntrain_csv[\"id_code\"] = x_training\ntrain_csv[\"diagnosis\"] = y_training\n\nvalid_csv = pd.DataFrame(columns = ['id_code', 'diagnosis'])\nvalid_csv[\"id_code\"] = x_validation\nvalid_csv[\"diagnosis\"] = y_validation\n\n# Should re-index these newly merged dataframes\ntrain_csv.reset_index(inplace = True)\nvalid_csv.reset_index(inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(dataset_csv[\"diagnosis\"]))\nprint(len(train_csv[\"diagnosis\"]))\nprint(len(valid_csv[\"diagnosis\"]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Now check the distributions"},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset_csv[\"diagnosis\"].hist(figsize = (8,4))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv['diagnosis'].hist(figsize = (8,4))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_csv['diagnosis'].hist(figsize = (8,4))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Display some images from the dataset folder"},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_samples(df, columns=4, rows=3, img_dir = train_file_path):\n    fig=plt.figure(figsize=(4*columns, 3*rows))\n    \n    random.seed(SEED)\n    random_indices = random.sample(range(0, len(df)), columns*rows)\n    count = 0\n    \n    for i in random_indices:\n        image_path = df.loc[i,'id_code']\n        image_rating = df.loc[i,'diagnosis']\n        \n        full_path = os.path.join(img_dir, image_path+\".png\")\n        \n        img = cv2.imread(full_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        \n        fig.add_subplot(rows, columns, count+1)\n        count += 1\n        plt.title(image_rating)\n        plt.imshow(img)\n    \n    plt.tight_layout()\n\n# display_samples(train_csv, 6, 2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# display_samples(previous_contest_csv, 6, 2, img_dir = \"/kaggle/input/retinopathy-train-2015/rescaled_train_896/rescaled_train_896/\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Check Pre-processing of Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"def crop_image(img,tol=7):\n    if img.ndim ==2:\n        mask = img>tol\n        return img[np.ix_(mask.any(1),mask.any(0))]\n    elif img.ndim==3:\n        gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n        mask = gray_img>tol\n        \n        check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0))].shape[0]\n        if (check_shape == 0): # image is too dark so that we crop out everything,\n            return img # return original image\n        else:\n            img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n            img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n            img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n    #         print(img1.shape,img2.shape,img3.shape)\n            img = np.stack([img1,img2,img3],axis=-1)\n        return img","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Single Channel Method"},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_single_channel_samples_crop_resize(df, columns=4, rows=3, channel = \"G\", HE = False, img_dir = train_file_path):\n    fig=plt.figure(figsize=(4*columns, 3*rows))\n    random.seed(SEED) # This lines make sure that all the following function calls will\n                    # show the same set of randomly selected images\n    random_indices = random.sample(range(0, len(df)), columns*rows)\n    count = 0\n    \n    for i in random_indices:\n        image_path = df.loc[i,'id_code']\n        image_rating = df.loc[i,'diagnosis']\n        \n        full_path = os.path.join(img_dir, image_path+\".png\")\n        \n        img = cv2.imread(full_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        \n        img = crop_image(img)\n        img = cv2.resize(img, (IMG_WIDTH, IMG_WIDTH))\n        # Apply some pre-processing\n        img = img[:,:,channels[channel]]\n        if HE: #If the histogram equalization is applied\n            clahe = cv2.createCLAHE(clipLimit=clipLimit, tileGridSize=tileGridSize)\n            img = clahe.apply(img) #This is for creating the image with a higher contrast\n        else:\n            pass\n        \n        fig.add_subplot(rows, columns, count+1)\n        count += 1\n        plt.title(image_rating)\n        plt.imshow(img)\n    \n    plt.tight_layout()\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# display_single_channel_samples_crop_resize(train_csv, 6,2, HE = True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Ben's method"},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef display_samples_bens_crop_resize(df, columns=4, rows=3, sigmaX = 15, img_dir = train_file_path):\n    fig=plt.figure(figsize=(4*columns, 3*rows))\n    random.seed(SEED) # This lines make sure that all the following function calls will\n                    # show the same set of randomly selected images\n    random_indices = random.sample(range(0, len(df)), columns*rows)\n    count = 0\n    \n    for i in random_indices:\n        image_path = df.loc[i,'id_code']\n        image_rating = df.loc[i,'diagnosis']\n        \n        full_path = os.path.join(img_dir, image_path+\".png\")\n        \n        img = cv2.imread(full_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        \n        img = crop_image(img)\n        img = cv2.resize(img, (IMG_WIDTH, IMG_WIDTH))\n        \n        # This following line is the key to ben's method\n        img = cv2.addWeighted ( img,4, cv2.GaussianBlur(img , (0,0) , sigmaX) ,-4 ,128)\n        \n        fig.add_subplot(rows, columns, count+1)\n        count += 1\n        plt.title(image_rating)\n        plt.imshow(img)\n    \n    plt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# display_samples_bens_crop_resize(train_csv, 6, 2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Image Generator with Augmentations"},{"metadata":{},"cell_type":"markdown","source":"### Finalize Image Pre-processor\n\nIn the above cases, we used two different pre-processing methods and compared their results. Here we finalize these two into a generator function. Given the image directly retrieved from the dataset, the generator produces, based on user's choice, pre-processed image. \n\n\nFor the pre-processor to work, the image feed in must be in RGB. Note that this function does ** cropping and resizing ** by itself."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Check the values in this dictionary\nprint(channels)\nprint(clipLimit)\nprint(tileGridSize)\n\ndef pre_process(img, method = \"SingleChannel\", channel = \"G\", sigmaX = 15):\n    \n    img = crop_image(img)\n    img = cv2.resize(img, (IMG_WIDTH, IMG_WIDTH))\n    \n    \n    if method == \"Bens\":\n        img = cv2.addWeighted ( img,4, cv2.GaussianBlur(img , (0,0) , sigmaX) ,-4 ,128)\n#         print(\"Bens\")\n    elif method == \"SingleChannel\":\n        single_ch = img[:,:,channels[channel]]\n        clahe = cv2.createCLAHE(clipLimit=clipLimit, tileGridSize=tileGridSize)\n        single_ch = clahe.apply(single_ch)\n        img = cv2.merge([single_ch, single_ch, single_ch])\n#         print(\"Single Channel\")\n    else:\n        print(\"Argument Error: Can't recognize METHOD passed in\")\n        \n    return img","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Augmentations\n\nWe used albumentations provided by the albumentations library. Stronly recommended.\n\n* Random Brightness\n* Rotation\n* Median Blur\n* Gaussian Noise\n* Horizontal and vertical flip\n* Random Gamma and Alpha\n* Hue and Saturation\n\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"from albumentations import (\n    Compose, HorizontalFlip, CLAHE, HueSaturationValue, VerticalFlip,\n    RandomBrightness, RandomContrast, RandomGamma, OneOf,\n    ToFloat, ShiftScaleRotate,GridDistortion, ElasticTransform, JpegCompression, HueSaturationValue,\n    RGBShift, RandomBrightness, RandomContrast, Blur, MotionBlur, MedianBlur,CenterCrop,\n    IAAAdditiveGaussianNoise,GaussNoise,OpticalDistortion,RandomSizedCrop\n)\n\nAUGMENTATIONS_TRAIN = Compose([\n    HorizontalFlip(p = 0.9),\n    VerticalFlip(p = 0.9),\n    RandomContrast(limit = 0.2, p = 0.8), # to do\n\n    OneOf([\n        RandomGamma(gamma_limit = (90,100), p = 0.5),\n        MedianBlur(blur_limit = 3, p = 0.5)\n    ], p = 0.6),\n    \n    OneOf([\n        ShiftScaleRotate(rotate_limit=25, p = 0.8),\n        OpticalDistortion(p = 0.4)\n    ], p = 1),\n\n    OneOf([\n        HueSaturationValue(hue_shift_limit=10, sat_shift_limit=10, val_shift_limit=10, p=0.5),\n        RandomBrightness(),\n    ], p=0.8)\n    \n],p=1)\n\n\nAUGMENTATIONS_TEST = Compose([\n#     ToFloat(max_value=1)\n],p=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Experiment with these image augmentation methods\n\nThe following function will take a few random images from the dataset provided and then do heavy augmentation repeatedly on them. Afterwards, this function will show the results."},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef test_augmentations(df, num, aug_num, augment, img_dir = train_file_path):\n    # For num images, perform aug_num times of augmentations on each\n    \n    total_num = num*aug_num\n    fig=plt.figure(figsize=(6*aug_num, 4*num))\n    \n    # Randomly select num images\n    random_indices = random.sample(range(0, len(df)), num)\n    count = 0\n    \n    for i in random_indices:\n        image_path = df.loc[i,'id_code']\n        image_rating = df.loc[i,'diagnosis']\n        full_path = os.path.join(img_dir, image_path+\".png\")\n        \n        # Read in the image and transform into RGB\n        img = cv2.imread(full_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        img = pre_process(img, method = \"SingleChannel\")\n        \n        for j in range(aug_num):\n            augmented = augment(image = img)\n            fig.add_subplot(num, aug_num, count+1)\n            plt.title(image_rating)\n            plt.imshow(augmented[\"image\"])\n            count += 1\n    \n    plt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_augmentations(train_csv, 4, 6, augment = AUGMENTATIONS_TRAIN)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(BATCHSIZE)\nprint(train_file_path)\nclass DataGenerator(tf.keras.utils.Sequence):\n#     'Generates data for Keras'\n    def __init__(self,\n                 df = train_csv,\n                 augmentation=None, batch_size=BATCHSIZE,\n                 img_size=IMG_WIDTH, n_channels=IMG_CHANNELS, \n                 shuffle=True,\n                image_dir = train_file_path):\n\n        self.dataframe = df\n        self.batch_size = batch_size\n        self.img_size = img_size\n        \n        self.n_channels = n_channels\n        self.shuffle = shuffle\n        self.augment = augmentation\n        \n        # Get all the paths out from the specified dataframe\n        self.img_paths = [filename for filename in self.dataframe[\"id_code\"]]\n#         print(\"self.img_paths: \", self.img_paths[:10])\n        self.img_dir = image_dir\n        \n        self.on_epoch_end()\n        \n    def __len__(self):\n#         'Denotes the number of batches per epoch'\n        return int(np.ceil(len(self.img_paths) / self.batch_size))\n\n    def __getitem__(self, index):\n        'Generate one batch of data'\n        # Generate indexes of the batch\n        indexes = self.indexes[index*self.batch_size:min((index+1)*self.batch_size,len(self.img_paths))]\n\n        # Select the next set of img paths to load the corresponding images and labels\n        list_IDs_im = [self.img_paths[k] for k in indexes]\n\n        # Generate data\n        X, y = self.data_generation(list_IDs_im)\n        \n        # First generate data\n        # Then pre-process and augment them\n        return X.astype(np.float32)/255, y\n    \n    def pre_process_augment(self, img):\n        # Use the written pre-process function here\n        img = pre_process(img, method = \"SingleChannel\")\n        \n        if self.augment == None:\n            return img\n        else: \n            augmented = self.augment(image = img)\n            aug_im = augmented['image']\n    #       print(\"augmented image data type:\", aug_im.dtype)\n            return aug_im\n\n    def on_epoch_end(self):\n        'Updates indexes after each epoch'\n        self.indexes = np.arange(len(self.img_paths))\n        if self.shuffle == True:\n            np.random.shuffle(self.indexes)\n\n    def data_generation(self, list_IDs_im):\n        'Generates data containing batch_size samples' # X : (n_samples, *dim, n_channels)\n        # Initialization\n        X = np.empty((len(list_IDs_im),self.img_size,self.img_size, self.n_channels))\n        y = np.empty((len(list_IDs_im), 1))\n        \n        # Generate data\n        for i, im_path in enumerate(list_IDs_im):\n            full_path = os.path.join(self.img_dir, im_path+\".png\")\n#             print(full_path)\n            img = cv2.imread(full_path)\n            img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n            \n            # Perform image augmentation here\n            aug_im = self.pre_process_augment(img)\n            \n            X[i,] = aug_im\n            y[i] = self.dataframe.loc[self.dataframe[\"id_code\"] == im_path, \"diagnosis\"]\n            \n        return X,y ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%timeit\na = DataGenerator(batch_size=6,shuffle=False, augmentation=AUGMENTATIONS_TRAIN)\ngen_images,gen_labels = a.__getitem__(0)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model Architecture"},{"metadata":{},"cell_type":"markdown","source":"### Instantiate Data Generators"},{"metadata":{"trusted":true},"cell_type":"code","source":"# The train images and validation images come from the same folder, no need to specify\ntrain_generator = DataGenerator(df = train_csv, batch_size = 6, shuffle = True, augmentation = AUGMENTATIONS_TRAIN)\nvalid_generator = DataGenerator(df = valid_csv, batch_size = 6, shuffle = False, augmentation = None)\n\nprevious_generator = DataGenerator(df = previous_contest_csv, batch_size = 6, shuffle = True, augmentation = None, image_dir = preivous_contest_file_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gen_images,gen_labels = train_generator.__getitem__(0)\n\nfor i in range(2):\n    img = gen_images[i]\n    plt.imshow(img)\n    plt.title(gen_labels[i])\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls /kaggle/input/retinopathy-train-2015/rescaled_train_896\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gen_images,gen_labels = previous_generator.__getitem__(0)\n\nfor i in range(2):\n    img = gen_images[i]\n    plt.imshow(img)\n    plt.title(gen_labels[i])\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gen_images,gen_labels = valid_generator.__getitem__(0)\n\nfor i in range(2):\n    img = gen_images[i]\n    plt.imshow(img)\n    plt.title(gen_labels[i])\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Metrics, Custom Loss, and Checkpointing"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(valid_generator.__len__())\nprint(train_generator.__len__())\nprint(previous_generator.__len__())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Cite the below code:"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_preds_and_labels(model, generator):\n    \"\"\"\n    Get predictions and labels from the generator\n    \"\"\"\n    \n    preds = []\n    labels = []\n    \n    for index in range(generator.__len__()):\n        x, y = generator.__getitem__(index)\n        preds.append(model.predict(x))\n        labels.append(y)\n    \n    # Flatten list of numpy arrays\n    return np.concatenate(preds).ravel(), np.concatenate(labels).ravel()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CustomMetrics(Callback):\n    \"\"\"\n    A custom Keras callback for saving the best model\n    according to the Quadratic Weighted Kappa (QWK) metric\n    \"\"\"\n    def on_train_begin(self, logs={}):\n        \"\"\"\n        Initialize list of QWK scores on validation data\n        \"\"\"\n        self.val_kappas = []\n\n    def on_epoch_end(self, epoch, logs={}):\n        \"\"\"\n        Gets QWK score on the validation data\n        \"\"\"\n        # Get predictions and convert to integers\n        y_pred, labels = get_preds_and_labels(model, valid_generator)\n        y_pred = np.rint(y_pred).astype(np.uint8).clip(0, 4)\n        \n        # We can use sklearns implementation of QWK straight out of the box\n        # as long as we specify weights as 'quadratic'\n        \n        _val_kappa = cohen_kappa_score(labels, y_pred, weights='quadratic')\n        self.val_kappas.append(_val_kappa)\n        print(f\"val_kappa: {round(_val_kappa, 4)}\")\n        if _val_kappa == max(self.val_kappas):\n            print(\"Validation Kappa has improved. Saving model.\")\n            self.model.save(SAVED_MODEL_NAME)\n        return","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"\n### Simple Custom Model\n\nThis part has not been completed yet"},{"metadata":{},"cell_type":"markdown","source":"\n\n### Transfer Learning"},{"metadata":{},"cell_type":"markdown","source":"#### Examine the loaded pre-trained Efficient Net\n\nWe can see that the last three layers of the with_top model are removed. The three layers are : avg_pool, top_dropout, probs\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"# # Load in EfficientNetB5\n# effnet = None\n# effnet = EfficientNetB5(weights=None,\n#                         include_top=False,\n#                         input_shape=(IMG_WIDTH, IMG_WIDTH, IMG_CHANNELS))\n# effnet.load_weights('../input/efficientnet-keras-weights-b0b5/efficientnet-b5_imagenet_1000_notop.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# effnet.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Add some structures"},{"metadata":{},"cell_type":"markdown","source":"Citation to this code"},{"metadata":{"trusted":true},"cell_type":"code","source":"# layer = effnet.layers[0]\n# print(type(layer))\n# print(layer.trainable)\n# print(len(effnet.layers))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Freeze some layers\n\nThis is just a trial, see what score we can get"},{"metadata":{"trusted":true},"cell_type":"code","source":"# for layer in effnet.layers[:100]:\n#     layer.trainable = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # Check the freezing is successful\n# layer = effnet.layers[0]\n# print(type(layer))\n# print(layer.trainable)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model = keras.Sequential()\n# model.add(effnet)\n# model.add(GlobalAveragePooling2D())\n# model.add(Dropout(0.4))\n# model.add(Dense(12, activation=elu)) # How to modify this param?\n# model.add(Dense(1, activation=\"linear\"))\n\n\n# model.compile(loss='mse',\n#             optimizer=Adam(lr=0.0001), \n#             metrics=['mse', 'acc'])\n\n# print(model.summary())\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Instantiate some custom callbacks and train the model"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Directly load the model\nprint(CUSTOM_WEIGHT)\nmodel = load_model(CUSTOM_WEIGHT)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"kappa_metrics = CustomMetrics()\n\n# Monitor MSE to avoid overfitting and save best model\nes = EarlyStopping(monitor='val_loss', mode='auto', verbose=1, patience=12)\n\nrlr = ReduceLROnPlateau(monitor='val_loss', \n                        factor=0.6, \n                        patience=6, \n                        verbose=1, \n                        mode='auto', \n                        epsilon=0.0001)\n\n# Begin training\n# No augmentations ==> a bit faster\n\nhistory = model.fit_generator(train_generator,\n                    steps_per_epoch = train_generator.__len__(),\n                    epochs = 2,\n                    workers = 6,\n                    validation_data=valid_generator,\n                    validation_steps = valid_generator.__len__(),\n                    callbacks=[kappa_metrics, es, rlr],\n                   verbose = 1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Download the trained weight"},{"metadata":{"trusted":true},"cell_type":"code","source":"# !ls","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from IPython.display import FileLinks\n# FileLinks('.')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Make predictions with the best performing model"},{"metadata":{},"cell_type":"markdown","source":"#### Visualize the training procedure"},{"metadata":{"trusted":true},"cell_type":"code","source":"# history_df = pd.DataFrame(model.history.history)\n# history_df[['loss', 'val_loss']].plot(figsize=(12,5))\n# plt.title(\"Loss (MSE)\", fontsize=16, weight='bold')\n# plt.xlabel(\"Epoch\")\n# plt.ylabel(\"Loss (MSE)\")\n# history_df[['acc', 'val_acc']].plot(figsize=(12,5))\n# plt.title(\"Accuracy\", fontsize=16, weight='bold')\n# plt.xlabel(\"Epoch\")\n# plt.ylabel(\"% Accuracy\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model.load_weights(SAVED_MODEL_NAME)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### See how the best performing model performs"},{"metadata":{"trusted":true},"cell_type":"code","source":"# # Calculate QWK on train set\n# y_train_preds, train_labels = get_preds_and_labels(model, train_generator)\n# y_train_preds = np.rint(y_train_preds).astype(np.uint8).clip(0, 4)\n\n# # Calculate score\n# train_score = cohen_kappa_score(train_labels, y_train_preds, weights=\"quadratic\")\n\n# Calculate QWK on validation set\ny_val_preds, val_labels = get_preds_and_labels(model, valid_generator)\ny_val_preds = np.rint(y_val_preds).astype(np.uint8).clip(0, 4)\n\n# Calculate score\nval_score = cohen_kappa_score(val_labels, y_val_preds, weights=\"quadratic\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(val_score)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Using threshold optimizer"},{"metadata":{"trusted":true},"cell_type":"code","source":"class OptimizedRounder(object):\n    \"\"\"\n    An optimizer for rounding thresholds\n    to maximize Quadratic Weighted Kappa score\n    \"\"\"\n    def __init__(self):\n        self.coef_ = 0\n\n    def _kappa_loss(self, coef, X, y):\n        \"\"\"\n        Get loss according to\n        using current coefficients\n        \"\"\"\n        X_p = np.copy(X)\n        for i, pred in enumerate(X_p):\n            if pred < coef[0]:\n                X_p[i] = 0\n            elif pred >= coef[0] and pred < coef[1]:\n                X_p[i] = 1\n            elif pred >= coef[1] and pred < coef[2]:\n                X_p[i] = 2\n            elif pred >= coef[2] and pred < coef[3]:\n                X_p[i] = 3\n            else:\n                X_p[i] = 4\n\n        ll = cohen_kappa_score(y, X_p, weights='quadratic')\n        return -ll\n\n    def fit(self, X, y):\n        \"\"\"\n        Optimize rounding thresholds\n        \"\"\"\n        loss_partial = partial(self._kappa_loss, X=X, y=y)\n        initial_coef = [0.5, 1.5, 2.5, 3.5]\n        self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')\n\n    def predict(self, X, coef):\n        \"\"\"\n        Make predictions with specified thresholds\n        \"\"\"\n        X_p = np.copy(X)\n        for i, pred in enumerate(X_p):\n            if pred < coef[0]:\n                X_p[i] = 0\n            elif pred >= coef[0] and pred < coef[1]:\n                X_p[i] = 1\n            elif pred >= coef[1] and pred < coef[2]:\n                X_p[i] = 2\n            elif pred >= coef[2] and pred < coef[3]:\n                X_p[i] = 3\n            else:\n                X_p[i] = 4\n        return X_p\n\n    def coefficients(self):\n        return self.coef_['x']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Optimize on validation data and evaluate again\ny_val_preds, val_labels = get_preds_and_labels(model, valid_generator)\noptR = OptimizedRounder()\noptR.fit(y_val_preds, val_labels)\ncoefficients = optR.coefficients()\nopt_val_predictions = optR.predict(y_val_preds, coefficients)\nnew_val_score = cohen_kappa_score(val_labels, opt_val_predictions, weights=\"quadratic\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Old validation score:\", val_score)\nprint(\"New validation score:\", new_val_score)\nprint(\"New coefficients:\", coefficients)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Predict and submit"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Notice, must use image data generator\nprint(test_file_path)\ntest_generator = DataGenerator(df = test_csv, batch_size = 6, shuffle = False, augmentation = None, image_dir = test_file_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_test = model.predict_generator(test_generator)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(y_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_test = optR.predict(y_test, coefficients).astype(np.uint8)\ntest_csv['diagnosis'] = y_test\ntest_csv.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Have a basic sense of how much time this kernel takes to train and inference"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_csv.head(20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"t_finish = time.time()\n\ntotal_time = round((t_finish-t_start) / 3600, 4)\nprint('Kernel runtime = {} hours ({} minutes)'.format(total_time, \n                                                      int(total_time*60)))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}