{"cells":[{"metadata":{},"cell_type":"markdown","source":"Hi,\n\nI would like to share a notebook that resulted 0.829 on the public learderboard and 0.916 on the private one.\nThis is a single model B5 classifier.\nWe used it as a part of an ensemble in the final submission.\n\nI trained it locally, so you will have to make some modification to rerun it in the cloud. But it shouldn't be too hard."},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"code","source":"import os\nimport math\nimport cv2 as cv2\nfrom PIL import Image\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt \nimport tensorflow as tf\n\n#import keras\nfrom keras import models\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Activation, Flatten\nfrom keras.layers import Conv2D, MaxPooling2D\n\nfrom keras.layers import Dense, Dropout, Activation, Flatten\nfrom keras.layers import Conv2D, MaxPooling2D, GlobalAveragePooling2D\nfrom keras.layers import LeakyReLU\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.preprocessing.image import load_img\nfrom keras.preprocessing.image import img_to_array\nfrom keras.utils import Sequence, to_categorical\n#from keras import utils as np_utils\n\nfrom keras.callbacks import EarlyStopping,ModelCheckpoint,ReduceLROnPlateau,TensorBoard, Callback\nfrom keras.optimizers import Adam, Optimizer\nimport keras.backend as K\n\nfrom sklearn.metrics import cohen_kappa_score, accuracy_score\nfrom sklearn.utils import class_weight\n\nimport imgaug as ia\nimport imgaug.augmenters as iaa\n\n\n\nfrom sklearn.model_selection import KFold, StratifiedKFold\n\nimport random\n\n%matplotlib inline\n\ndef seed_everything(seed=999):\n    \n    print('Random seeds initialized')\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    tf.set_random_seed(seed)\n    ia.seed(seed)\n\nseed_everything()\n\n\ndef normalize(x):\n    return (x-dataset_mean)/dataset_std\n\ndef denormalize(x):\n    return x*dataset_std+dataset_mean","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loading & Exploration"},{"metadata":{},"cell_type":"markdown","source":"## First load the current competition dataset, and the test set"},{"metadata":{"trusted":false},"cell_type":"code","source":"TRAIN_PATH = './input/prep456_cropOnly_origAspect/'\nVAL_PATH = './input/prep456_cropOnly_origAspect/'\nWIDTH = 380\nHEIGHT = WIDTH\ntarget_size=(WIDTH, HEIGHT)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Separate Train and Validation set"},{"metadata":{"_kg_hide-input":true,"trusted":false},"cell_type":"code","source":"\n## Displaying some Sample Images\ndef display_samples_from_df(df, columns=4, rows=3, path=TRAIN_PATH):\n    fig=plt.figure(figsize=(5*columns, 4*rows))\n\n    idx=1\n    \n    for i in range(columns*rows):\n        image_path = df.loc[i,'id_code']\n        image_id = df.loc[i,'diagnosis']\n        \n        img = Image.open(path+image_path)\n        height, width = img.size\n        fig.add_subplot(rows, columns, idx)\n        plt.title(image_id)\n        plt.imshow(img)\n        idx+=1\n    \n    plt.tight_layout()\n    \ndef display_samples_from_list(x, y, columns=4, rows=3):\n    fig=plt.figure(figsize=(5*columns, 4*rows))\n\n    idx=1\n    \n    for i in range(columns*rows):\n        image_path = x[i]\n        image_id = y[i]\n\n        img = Image.fromarray(image_path.astype(\"uint8\"))\n        fig.add_subplot(rows, columns, idx)\n        plt.title(image_id)\n        plt.imshow(img)\n        idx+=1\n    \n    plt.tight_layout() \n\n#Separate validation data\ndef getValSet(validation_method, train_df, foldIdx=0, fold=3):\n    \n    if (validation_method == 'same'):\n        print(\"Calculate Validation set with same distribution\")\n        target=0.2\n        row_list_train = []\n        row_list_val = []\n        for index, row in train_df.iterrows():\n\n            if random.random()<target: \n                row_list_val.append({\"id_code\": row[\"id_code\"], \"diagnosis\":row[\"diagnosis\"]})\n            else:\n                row_list_train.append({\"id_code\": row[\"id_code\"], \"diagnosis\":row[\"diagnosis\"]})\n\n        train_df = pd.DataFrame(row_list_train)\n        val_df = pd.DataFrame(row_list_val)\n\n    if (validation_method == 'kFold'):\n        print(\"Get kFold Validation\")\n        \n        source = train_df.copy()\n        \n        kf = StratifiedKFold(n_splits = fold, shuffle = False, random_state = 2019)\n        split = kf.split(source['id_code'], source['diagnosis'])\n        for _ in range(foldIdx+1): \n            result=next(split) \n\n        val_df =  pd.DataFrame()\n        train_df = pd.DataFrame()\n        val_df = pd.concat([val_df, source.iloc[result[1]]])\n        train_df = pd.concat([train_df, source.iloc[result[0]]])\n        val_df.reset_index(inplace=True)\n        train_df.reset_index(inplace=True)\n        \n    if (validation_method == 'new'):\n        print(\"Get Validation set fron the Train.CSV\")\n        val_df = pd.read_csv('./input/train.csv')\n        val_df['id_code'] = val_df['id_code']+\".png\"\n        val_df['diagnosis'] = val_df['diagnosis'].astype(\"int\")\n\n    train_df['diagnosis'] = train_df['diagnosis'].astype('str')\n    val_df['diagnosis'] = val_df['diagnosis'].astype('str')\n\n    print(train_df['diagnosis'].value_counts())\n    print(val_df['diagnosis'].value_counts())\n    return (train_df, val_df)\n\n#train_df = pd.read_csv('./input/oldtrain.csv')\n#train, val = getValSet('kFold', train_df, foldIdx=2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Augmentation settings"},{"metadata":{"trusted":false},"cell_type":"code","source":"def color(images, random_state, parents, hooks): # without cutting out a circle\n    \n    blank=np.zeros( images[0].shape, np.uint8 )\n    cropfactor=0.95\n    \n    for i, img in enumerate(images):\n        \n        img = cv2.addWeighted ( img,4, cv2.GaussianBlur( img , (0,0), WIDTH/60) ,-4 ,128)\n        cv2.circle(blank , ( img.shape[1]//2 , img.shape[0]//2) , int( img.shape[1]//2*cropfactor ) , ( 1 , 1 , 1 ) , -1 , 8 , 0 )\n        images[i]=img*blank+0*(1-blank)\n\n    return images\n\ndef colorCrop(images, random_state, parents, hooks): # cutout of a circle\n    \n    blank=np.zeros( images[0].shape, np.uint8 )\n    cropfactor=0.95\n    \n    for i, img in enumerate(images):\n        \n        img = cv2.addWeighted ( img,4, cv2.GaussianBlur( img , (0,0), WIDTH/60) ,-4 ,128)      \n        cv2.circle(blank , ( img.shape[1]//2 , img.shape[0]//2) , int( img.shape[1]//2*cropfactor ) , ( 1 , 1 , 1 ) , -1 , 8 , 0 )\n        images[i]=img*blank+0*(1-blank)\n\n    return images\n\naugment_orig_val = iaa.Sequential()\n\n\n\naugment_strong = iaa.Sequential(\n    [\n        iaa.Fliplr(0.5), # horizontally flip 50% of all images\n        iaa.Flipud(0.5), # vertically flip 20% of all images\n\n        iaa.Sometimes(0.5, iaa.Crop(percent=(0, 0.1  ))),\n        \n        iaa.Sometimes(0.5, iaa.Affine(\n            scale={\"x\": (0.8, 1.2), \"y\": (0.8, 1.2)},\n            translate_percent={\"x\": (-0.1, 0.1), \"y\": (-0.1, 0.1)},\n            rotate=(-180, 180),\n        )),\n\n        iaa.SomeOf((0, 3),\n            [\n                iaa.Sometimes(0.1, \n                    iaa.Superpixels(\n                        p_replace=(0, 0.25),\n                        n_segments=(20, 200)\n                    )\n                ),\n                iaa.OneOf([\n                    iaa.GaussianBlur((0, 3.0)),\n                    iaa.AverageBlur(k=(2, 7)),\n                    iaa.MedianBlur(k=(3, 5)),\n                ]),\n                iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)),\n                iaa.Emboss(alpha=(0, 1.0), strength=(1, 2.0)),\n                iaa.Sometimes(0.2, iaa.OneOf([\n                    iaa.EdgeDetect(alpha=(0, 0.7)),\n                    iaa.DirectedEdgeDetect(\n                        alpha=(0, 0.7), direction=(0.0, 1.0)\n                    ),\n                ])),\n                iaa.AdditiveGaussianNoise(\n                    scale=(0.1, 0.03*255), per_channel=0.5\n                ),\n                iaa.OneOf([\n                    iaa.Dropout((0.01, 0.15), per_channel=0.5),\n                    iaa.CoarseDropout(\n                        (0.03, 0.15), size_percent=(0.03, 0.05)),\n                ]),\n                iaa.Add((-10, 10)),\n                iaa.Multiply((0.8, 1.5)),\n                iaa.ContrastNormalization((0.5, 2.0), per_channel=0.5),\n                iaa.Grayscale(alpha=(0.0, 1.0)),\n                iaa.Sometimes(0.1, \n                    iaa.ElasticTransformation(alpha=(0.0, 2), sigma=0.25)\n                ),\n\n                iaa.Sometimes(0.1, iaa.PiecewiseAffine(scale=(0.01, 0.05)))\n            ],\n            # do all of the above augmentations in random order\n            random_order=True\n        )\n    ]) \n\n\n#augment = augment_origcolor_v3\n#x, y = validation_generator.__getitem__(0)\n#img = denormalize(x)\n#img = img.astype('uint8')\n#img = augment(images=img)\n#display_samples_from_list(img, y)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"\n# Data Generator"},{"metadata":{"trusted":false},"cell_type":"code","source":"def decoder(pred):\n    if pred < 0.5: return 0\n    elif pred < 1.5: return 1\n    elif pred < 2.5: return 2\n    elif pred < 3.5: return 3\n    else: return 4\n\ndef decode(predictions):\n    return np.array([decoder(pred) for pred in predictions])\n\ndef tightCrop(img):    #cuts out the middle of a circle\n    \n    r=img.shape[1]//2\n    a=int(math.sqrt(2)*r*1.05)\n\n    img = img[img.shape[0]//2-a//2:img.shape[0]//2+a//2, r-a//2:r+a//2, :]    \n    img = cv2.resize(img, (WIDTH, HEIGHT))\n    return img\n\ndef tightCrop43(img):    #cuts out a rectangle from the middle of a circle\n    \n    r=img.shape[1]//2\n    x=int(math.sqrt(4/25*r**2)*1.05)\n    sidea=4*x\n    sideb=3*x\n    \n    startx = (img.shape[1]-sidea)//2\n    starty = (img.shape[0]-sideb)//2\n\n    img = img[starty:starty+sideb, startx:startx+sidea, :]    \n    return img\n\ndef middleCrop(img): # crops out the middle of a rectangle\n    \n    h, w, _ = img.shape\n    startx = int(w/2-h/2)\n    return img[:, startx:startx+h, :]\n\ndef load_image(p_path, crop=False):\n    \n    try:\n        image = cv2.imread(p_path) #read\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) #color transform\n    except:\n        print(f\"Can't read: {p_path}\")\n        return None\n        \n    if crop==True: image = crop_image_from_gray(image) #crop black border\n        \n    imgType = getImageType(image)\n    if (imgType == 0): image = tightCrop43(image)\n    if (imgType == 1): image = tightCrop43(image)\n    \n    image = cv2.resize(image, (WIDTH, HEIGHT)) #resize\n    return image\n\n# load X,Y from a dataframe\ndef load_df(df, max_load=1000000000, path=TRAIN_PATH, crop=False):\n\n    x = np.zeros((min(max_load, df.shape[0]), WIDTH, HEIGHT, 3), dtype='uint8')\n    y = np.zeros((min(max_load, df.shape[0])), dtype='uint8')\n    \n    for idx,row in df.iterrows():\n        x[idx, :, :, :] = load_image(row['path'], crop)\n        y[idx] = row['diagnosis']\n        \n        if idx>=max_load-1: break\n    \n    print(f'Return shape: {x.shape}')\n    return x,y\n\n# crops images\ndef crop_image_from_gray(img,tol=7):\n    gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n    mask = gray_img>tol\n        \n    check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0))].shape[0]\n    if (check_shape == 0): # image is too dark so that we crop out everything,\n        return img # return original image\n    else:\n        img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n        img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n        img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n        img = np.stack([img1,img2,img3],axis=-1)\n    return img\n    \ndef getImageType(img, limit=20):\n    \n    #print(np.max(img[ img.shape[0]//4, 0,  : ]))\n    #print(np.max(img[ 0, img.shape[0]//4,  : ]))\n    #print(np.max(img[ img.shape[1]//2,  image.shape[0]//2,  : ]))\n    \n    if (img.shape[1]-img.shape[0])<img.shape[0]/50: return 0 # circle\n    if np.max(img[ img.shape[0]//3, 0,  : ])<limit: return 1 # sides touch sides\n    return 2\n\n#image = './input/preptest456_cropOnly_origAspect/0e9d6c70eaf7.png' #circle\n#image = './input/prep456_cropOnly_origAspect/0097f532ac9f.png'\n#image = './input/preptest456_cropOnly_origAspect/0eb6e41a8658.png'\n#image = cv2.imread(image)\n#image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\n#print(\"Image type: \",getImageType(image))\n#image = tightCrop43(image)\n#image = Image.fromarray(image)\n\n#plt.figure(figsize = (8,8))\n#plt.imshow(image) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"class DataSequence(Sequence):\n\n    def __init__(self, df, batch_size, x_col, y_col, augment=True, shuffle=True, augmenter=None, label_smoothing=0.1):\n        self.df = df\n        self.batch_size = batch_size\n        self.x_col = x_col\n        self.y_col = y_col\n        self.augment = augment\n        self.shuffle = shuffle\n        self.n = df['diagnosis'].count()\n        self.num_classes = len(np.unique(df[y_col]))\n        self.myAugmenter = augmenter\n        self.label_smoothing = label_smoothing\n\n    def __len__(self):\n        return int(math.ceil(len(self.df) / float(self.batch_size)))\n\n    def on_epoch_end(self):\n        # shuffle when needed\n        if self.shuffle: self.df = self.df.sample(frac=1).reset_index(drop=True)\n        \n    def get_batch_labels(self, idx):\n        # Fetch a batch of labels\n        y = np.array(self.df[self.y_col][idx * self.batch_size: (idx + 1) * self.batch_size])\n        y = to_categorical(y, 5)\n        \n        y = y * (1-self.label_smoothing) + self.label_smoothing/5\n        \n        return y\n\n    def get_batch_features(self, idx):\n        # Fetch a batch of inputs\n        images = np.array([load_image(im) for im in self.df[self.x_col][idx * self.batch_size: (1 + idx) * self.batch_size]])\n\n        \n        if self.augment==True: trans_images = normalize(np.array(self.myAugmenter(images=images.astype('uint8'))))\n        else: trans_images=normalize(np.array(images))\n        \n        return trans_images\n\n    def __getitem__(self, idx):\n        batch_x = self.get_batch_features(idx)\n        batch_y = self.get_batch_labels(idx)\n        return batch_x, batch_y\n\ndef fitData(df, path = TRAIN_PATH): \n    fit_datagen =  ImageDataGenerator(\n        featurewise_center=True,\n        featurewise_std_normalization=True)\n        \n    x_fit, y_fit = load_df(df, 300, path)\n    fit_datagen.fit(x_fit)\n        \n    global dataset_mean\n    global dataset_std\n    dataset_mean = fit_datagen.mean\n    dataset_std = fit_datagen.std\n            \n    print(fit_datagen.std)\n    print(fit_datagen.mean)          \n\ndef getGenerators(train_df, val_df, batch_size, augmenter, label_smoothing=0.1):\n\n    global dataset_mean\n    global dataset_std\n\n    fitData(train_df, TRAIN_PATH)\n \n    train_generator = DataSequence(\n            train_df, \n            x_col=\"path\",\n            y_col=\"diagnosis\",\n            batch_size=batch_size,\n            augment=True,\n            shuffle=True,\n            augmenter=augmenter[0],\n            label_smoothing=label_smoothing\n            )\n\n    validation_generator = DataSequence(\n            val_df, \n            x_col=\"path\",\n            y_col=\"diagnosis\",\n            batch_size=batch_size,\n            augment=True,\n            shuffle=False,\n            augmenter=augmenter[1],\n            label_smoothing=label_smoothing\n    )\n\n    return train_generator, validation_generator","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train_df = pd.read_csv('./input/oldtrain.csv') # 2015 training dataset\ntrain_df['id_code'] = train_df['id_code']+\".png\"\ntrain_df['diagnosis'] = train_df['diagnosis'].astype(\"int\")\n\ntrain_df, val_df = getValSet(\"new\", train_df) \ntrain_generator, validation_generator = getGenerators(train_df, val_df, TRAIN_PATH, VAL_PATH, 32, (augment_origcolor, augment_orig_val))\n\nprint('Done')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model: EfficientNet"},{"metadata":{"trusted":false},"cell_type":"code","source":"import efficientnet.keras as efn \n\ndef build_model():\n    \n    seed_everything()\n    base = efn.EfficientNetB5(weights='imagenet', include_top=False, input_shape=(WIDTH, HEIGHT, 3), pooling='avg')\n    base.trainable=True\n\n    #dropout_dense_layer = 0.2 # for B0\n    #dropout_dense_layer = 0.3 # for B3\n    dropout_dense_layer = 0.4  # for B5    \n     \n    model = Sequential()\n    model.add(base)\n    model.add(Dropout(dropout_dense_layer))\n    model.add(Dense(5, activation='softmax'))\n    \n    model.compile(\n        loss='categorical_crossentropy',\n        optimizer=Adam(lr=0.001),\n        metrics=['accuracy']\n    )\n    model.summary()\n    return model\n\nmodel = build_model()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Last check before Training"},{"metadata":{"trusted":false},"cell_type":"code","source":"x, y = train_generator.__getitem__(1)\nx, y = validation_generator.__getitem__(0)\n\ndisplay_samples_from_list(denormalize(x), y)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Phase 1 Training"},{"metadata":{"trusted":false},"cell_type":"code","source":"model_name = \"model_efficientb5_cat_base.h5\"\ncheckpoint = ModelCheckpoint(model_name, monitor='val_loss', verbose=1, save_best_only=True, mode='auto')\nreduce_lr_on_plateau = ReduceLROnPlateau(monitor = 'loss',patience = 5, factor=0.3, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"model.layers[0].trainable = False\nmodel.compile(loss='categorical_crossentropy', optimizer=Adam(lr=0.001), metrics=['accuracy'] )\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"history = model.fit_generator(generator=train_generator,\n                              steps_per_epoch=train_generator.n/train_generator.batch_size,\n                              validation_data=validation_generator,\n                              validation_steps=validation_generator.n/validation_generator.batch_size,\n                              epochs=5,\n                              workers=6,\n                              callbacks=[checkpoint, reduce_lr_on_plateau])\n\n\nhistory_df = pd.DataFrame(history.history)\nhistory_df[['loss', 'val_loss']].plot()\nhistory_df[['acc', 'val_acc']].plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"model.layers[0].trainable = True\nlimit = 300\nfor i in range(limit): model.layers[0].layers[i].trainable = False\nfor i in range(limit, 570): model.layers[0].layers[i].trainable = True\nmodel.compile(loss='categorical_crossentropy', optimizer=Adam(lr=0.001), metrics=['accuracy'] )\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train_generator, validation_generator = getGenerators(train_df, val_df, TRAIN_PATH, VAL_PATH, 16, (augment_origcolor, augment_orig_val))\nhistory = model.fit_generator(generator=train_generator,\n                              steps_per_epoch=train_generator.n/train_generator.batch_size,\n                              validation_data=validation_generator,\n                              validation_steps=validation_generator.n/validation_generator.batch_size,\n                              epochs=15,\n                              workers=6,\n                              callbacks=[checkpoint, reduce_lr_on_plateau])\n\n\nhistory_df = pd.DataFrame(history.history)\nhistory_df[['loss', 'val_loss']].plot()\nhistory_df[['acc', 'val_acc']].plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"model.save_weights(model_name)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Phase 2 training "},{"metadata":{"trusted":false},"cell_type":"code","source":"from keras.callbacks import Callback\nimport keras.backend as K\nimport numpy as np\n\nclass SGDRScheduler(Callback):\n    '''Cosine annealing learning rate scheduler with periodic restarts.\n    # Usage\n        ```python\n            schedule = SGDRScheduler(min_lr=1e-5,\n                                     max_lr=1e-2,\n                                     steps_per_epoch=np.ceil(epoch_size/batch_size),\n                                     lr_decay=0.9,\n                                     cycle_length=5,\n                                     mult_factor=1.5)\n            model.fit(X_train, Y_train, epochs=100, callbacks=[schedule])\n        ```\n    # Arguments\n        min_lr: The lower bound of the learning rate range for the experiment.\n        max_lr: The upper bound of the learning rate range for the experiment.\n        steps_per_epoch: Number of mini-batches in the dataset. Calculated as `np.ceil(epoch_size/batch_size)`. \n        lr_decay: Reduce the max_lr after the completion of each cycle.\n                  Ex. To reduce the max_lr by 20% after each cycle, set this value to 0.8.\n        cycle_length: Initial number of epochs in a cycle.\n        mult_factor: Scale epochs_to_restart after each full cycle completion.\n    # References\n        Blog post: jeremyjordan.me/nn-learning-rate\n        Original paper: http://arxiv.org/abs/1608.03983\n    '''\n    def __init__(self,\n                 min_lr,\n                 max_lr,\n                 steps_per_epoch,\n                 lr_decay=1,\n                 cycle_length=10,\n                 mult_factor=1.1):\n\n        self.min_lr = min_lr\n        self.max_lr = max_lr\n        self.lr_decay = lr_decay\n\n        self.batch_since_restart = 0\n        self.next_restart = cycle_length\n\n        self.steps_per_epoch = steps_per_epoch\n\n        self.cycle_length = cycle_length\n        self.mult_factor = mult_factor\n\n\n    def clr(self):\n        '''Calculate the learning rate.'''\n        fraction_to_restart = self.batch_since_restart / (self.steps_per_epoch * self.cycle_length)\n        lr = self.min_lr + 0.5 * (self.max_lr - self.min_lr) * (1 + np.cos(fraction_to_restart * np.pi))\n        #print(lr)\n        return lr\n\n    def on_train_begin(self, logs={}):\n        '''Initialize the learning rate to the minimum value at the start of training.'''\n        K.set_value(self.model.optimizer.lr, self.max_lr)\n\n    def on_batch_end(self, batch, logs={}):\n        '''Record previous batch statistics and update the learning rate.'''\n\n        self.batch_since_restart += 1\n        K.set_value(self.model.optimizer.lr, self.clr())\n\n    def on_epoch_end(self, epoch, logs={}):\n        '''Check for end of current cycle, apply restarts when necessary.'''\n        if epoch + 1 == self.next_restart:\n            self.batch_since_restart = 0\n            self.cycle_length = np.ceil(self.cycle_length * self.mult_factor)\n            self.next_restart += self.cycle_length\n            self.max_lr *= self.lr_decay\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def train(load_model='model_efficientB5_cat_base.h5', model_name = \"model_efficientB5_cat_ph2.h5\", epochs=50):\n    \n    schedule = SGDRScheduler(min_lr=1e-5,\n                                     max_lr=3e-2,\n                                     steps_per_epoch=np.ceil(train_generator.n/train_generator.batch_size),\n                                     cycle_length=5)    \n    \n    #model = build_model()\n    seed_everything()\n    \n    checkpoint = ModelCheckpoint(model_name, monitor='val_loss', verbose=1, save_best_only=True, mode='auto')\n    reduce_lr_on_plateau = ReduceLROnPlateau(monitor = 'loss',patience = 4, factor=0.5, verbose=1)\n\n    if load_model != None: \n        print(\"Loading model: \", load_model)\n        model.load_weights(load_model)\n        \n    history = model.fit_generator(generator=train_generator,\n                                  steps_per_epoch=train_generator.n/train_generator.batch_size,\n                                  validation_data=validation_generator,\n                                  validation_steps=validation_generator.n/validation_generator.batch_size,\n                                  epochs=epochs,\n                                  verbose=1,\n                                  workers=6,\n                                  class_weight=class_weights,\n                                  callbacks=[checkpoint, reduce_lr_on_plateau])\n\n\n    history_df = pd.DataFrame(history.history)\n    #history_df[['loss', 'val_loss']].plot()\n    #history_df[['acc', 'val_acc']].plot()\n    print('Minimum validation loss: ', history_df['val_loss'].min())\n    return history_df['val_loss'].min()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"origtrain_df = pd.read_csv('./input/train.csv')\norigtrain_df['path'] = TRAIN_PATH+origtrain_df.id_code+'.png'\n#origtrain_df.id_code += '.png'\nKfold = 3\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"TEST_PATH = './input/preptest456_cropOnly_origAspect/'\npseudo = pd.read_csv('./input/linear-stacking-blended-with-best-lb-as-reg-v2-submission.csv')\npseudo['path'] = TEST_PATH+pseudo.id_code+'.png'\npseudo.head()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"newtrain = pd.concat([origtrain_df, pseudo], sort=True)\n#newtrain.id_code += '.png'\nnewtrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"val_df = pd.read_csv('./input/cleanvalidation.csv')\nval_df['path'] = TRAIN_PATH+val_df.id_code+'.png'\nval_df.count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train_df = pd.merge(newtrain, val_df, on='id_code', indicator=True, how='outer').query('_merge==\"left_only\"')\ntrain_df.drop(columns=['diagnosis_y', 'path_y', '_merge'], inplace=True)\ntrain_df.rename(columns={'diagnosis_x':'diagnosis', 'path_x':'path'}, inplace=True)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"#train_df, val_df = getValSet( 'kFold',newtrain, foldIdx=0, fold=5)\ntrain_generator, validation_generator = getGenerators(\n        train_df, \n        val_df, \n        batch_size=16, \n        augmenter=(augment_strong, augment_orig_val)\n)\nclass_weights = class_weight.compute_class_weight('balanced',\n                                                 np.unique(train_df.diagnosis),\n                                                 train_df.diagnosis)\nclass_weights","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"x, y = train_generator.__getitem__(0)\n#x, y = validation_generator.__getitem__(2)\n\ndisplay_samples_from_list(denormalize(x), y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"import tensorflow as tf\n\ndef kappa_loss(y_pred, y_true, y_pow=2, eps=1e-10, N=5, bsize=train_generator.batch_size, name='kappa'):\n    \"\"\"A continuous differentiable approximation of discrete kappa loss.\n        Args:\n            y_pred: 2D tensor or array, [batch_size, num_classes]\n            y_true: 2D tensor or array,[batch_size, num_classes]\n            y_pow: int,  e.g. y_pow=2\n            N: typically num_classes of the model\n            bsize: batch_size of the training or validation ops\n            eps: a float, prevents divide by zero\n            name: Optional scope/name for op_scope.\n        Returns:\n            A tensor with the kappa loss.\"\"\"\n    \n    with tf.name_scope(name):\n        y_true = tf.to_float(y_true)\n        repeat_op = tf.to_float(tf.tile(tf.reshape(tf.range(0, N), [N, 1]), [1, N]))\n        repeat_op_sq = tf.square((repeat_op - tf.transpose(repeat_op)))\n        weights = repeat_op_sq / tf.to_float((N - 1) ** 2)\n    \n        pred_ = y_pred ** y_pow\n        try:\n            pred_norm = pred_ / (eps + tf.reshape(tf.reduce_sum(pred_, 1), [-1, 1]))\n        except Exception:\n            pred_norm = pred_ / (eps + tf.reshape(tf.reduce_sum(pred_, 1), [bsize, 1]))\n    \n        hist_rater_a = tf.reduce_sum(pred_norm, 0)\n        hist_rater_b = tf.reduce_sum(y_true, 0)\n    \n        conf_mat = tf.matmul(tf.transpose(pred_norm), y_true)\n    \n        nom = tf.reduce_sum(weights * conf_mat)\n        denom = tf.reduce_sum(weights * tf.matmul(\n            tf.reshape(hist_rater_a, [N, 1]), tf.reshape(hist_rater_b, [1, N])) /\n                              tf.to_float(bsize))\n    \n        return nom / (denom + eps)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"model.layers[0].trainable = True\nlimit = 300\nfor i in range(limit): model.layers[0].layers[i].trainable = False\nfor i in range(limit, 570): model.layers[0].layers[i].trainable = True\nmodel.compile(loss=kappa_loss, optimizer=Adam(0.001), metrics=['accuracy'] )\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train(load_model='model_efficientB5_cat_base.h5', model_name = \"model_efficientB5_cat_ph2.h5\", epochs=40) ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Evaluation"},{"metadata":{"trusted":false},"cell_type":"code","source":"x_val, y_val = load_df(val_df)\nx_val = normalize(x_val)\n\nx_eval = x_val\ny_eval = y_val\ndisplay_samples_from_list(denormalize(x_eval), y_val)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"y_true = np.array(y_eval).astype('int')\n\nmodel.load_weights('./model_efficientB5_cat_ph2.h5')\npredictions = model.predict(x_eval)\npredictions = np.array(predictions).squeeze()\n\ny_pred = np.argmax(predictions, axis=1)\n\nprint(y_pred[:30])\nprint(y_true[:30])\n\nprint(\"Validation accuracy: \", np.mean(np.equal(y_true, y_pred)))\nprint(\"Cohen Kappa score: %.3f\" % cohen_kappa_score(y_pred, y_true, weights='quadratic'))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Confusion Matrix"},{"metadata":{"trusted":false},"cell_type":"code","source":"from sklearn.metrics import classification_report, confusion_matrix\n\nprint('Confusion Matrix')\nprint(confusion_matrix(y_true, y_pred))\nprint('Classification Report')\ntarget_names = ['0', '1', '2', '3', '4']\nprint(classification_report(y_true, y_pred, target_names=target_names))\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Evaluation with TTT"},{"metadata":{"trusted":false},"cell_type":"code","source":"eval_augment = iaa.Sequential(\n    [\n\n        iaa.Fliplr(0.3), # horizontally flip 50% of all images\n        iaa.Sometimes(0.1, iaa.Crop(percent=(0, 0.1))),\n\n        iaa.Sometimes(0.5, iaa.Affine(\n            scale={\"x\": (0.95, 1.05), \"y\": (0.95, 1.05)},\n            translate_percent={\"x\": (-0.01, 0.01), \"y\": (-0.01, 0.01)},\n            rotate=(-5, 5))),\n\n        iaa.SomeOf((0, 2),\n            [\n                iaa.Sharpen(alpha=(0, 0.5), lightness=(0.85, 1.15)),\n                iaa.Emboss(alpha=(0, 0.5), strength=(0.8, 1.0)),\n                iaa.Sometimes(0.3, iaa.EdgeDetect(alpha=(0, 0.3))),\n                iaa.AdditiveGaussianNoise(),\n                iaa.Multiply((0.9, 1.3)),\n                iaa.ContrastNormalization((0.7, 1.2)),\n\n            ], random_order=True\n        )\n    ]) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"predictions = []\nseed_everything()\ntta_steps = 3\npredictions = np.zeros((tta_steps, y_eval.shape[0]))\nfor tta in range(tta_steps):\n    preds = np.zeros((x_eval.shape[0]))\n       \n    x_trans = eval_augment(images=denormalize(x_eval).astype('uint8'))\n    preds = model.predict(normalize(x_trans))\n    predictions[tta, :] = np.argmax(preds, axis=1)\n    print(f'TTA round {tta+1} done')\n  \n    \npredictions = np.mean(predictions, axis=0)\ny_pred = (predictions).astype(\"int\")\ny_true = np.array(y_eval).astype('int')\n\nprint(y_pred[:30])\nprint(y_true[:30])\n\nprint(\"Validation accuracy with TTT: \", np.mean(np.equal(y_true, y_pred)))\nprint(\"Cohen Kappa score: \", cohen_kappa_score(y_pred, y_true, weights='quadratic'))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Write results to CSV"},{"metadata":{"trusted":false},"cell_type":"code","source":"submit_df = pd.read_csv('./input/sample_submission.csv')\nTEST_PATH = './input/test_images/'\nsubmit_df['path'] = TEST_PATH+submit_df.id_code+'.png'\nshow_df = submit_df.copy()\nshow_df['id_code'] += '.png'\n\nx_test, y_test = load_df(show_df, path=TEST_PATH, crop=True)\ndisplay_samples_from_list(x_test, y_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"#show_df.sample(frac=1).reset_index(drop=True)\nfitData(train_df, path = TRAIN_PATH)\n#dataset_mean = [152.86581, 78.63777, 20.671019]\n#dataset_std = [43.88956  25.214554 18.860243]\nseed_everything()\n\ntta_steps = 1\npredictions = []\npredictions = np.zeros((tta_steps, x_test.shape[0]))\nfor tta in range(tta_steps):\n    preds = np.zeros((x_test.shape[0]))\n       \n    if tta_steps>1: x_trans = eval_augment(images=x_test.astype('uint8'))\n    else: x_trans = x_test.astype('uint8')\n    \n    preds = model.predict(normalize(x_trans))\n    predictions[tta, :] = np.argmax(preds, axis=1)\n    print(f'TTA round {tta+1} done')\n\npredictions = np.mean(predictions, axis=0)\ny_pred = predictions.astype('int')\n#y_pred = predictions #raw","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"submit_df['diagnosis'] = y_pred.squeeze()\nsubmit_df = submit_df.drop(columns=['path'])\nsubmit_df.to_csv('submission.csv',index=False)\n\nsubmit_df.hist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"submit_df.diagnosis.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"ref = pd.read_csv('./submission829.csv')\n#ref = pd.read_csv('./submission_classifier_0798.csv')\n\nref.hist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"ref.diagnosis.value_counts()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.3"}},"nbformat":4,"nbformat_minor":1}