{"cells":[{"metadata":{"_uuid":"f0d95b709e9c79a2fd7ab085be1b90e4044d1688"},"cell_type":"markdown","source":"### One Cycle Policy with Keras\nHighly inspired by following paper:\n- [A disciplined approach to neural network hyper-parameters: Part 1 -- learning rate, batch size, momentum, and weight decay](http://arxiv.org/abs/1803.09820) (Leslie N. Smith)\n\nI have implemented the One Cycle Policy algorithm developed by Leslie N. Smith into the Keras Callback class. Leslie Smith suggests in this paper a slight modification of cyclical learning rate policy for super convergence using one cycle that is smaller than the total number of iterations/epochs and allow the learning rate todecrease several orders of magnitude less than the initial learning rate for the remaining miterations. In his experiments this policy allows the accuracy to plateau before the training ends. This approach (among others) helped me to improve my score."},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"from sklearn.utils import shuffle\nimport pandas as pd\nimport os\n# Save train labels to dataframe\ndf = pd.read_csv(\"../input/train_labels.csv\")\n\n# Save test labels to dataframe\ndf_test = pd.read_csv('../input/sample_submission.csv')\n\ndf = shuffle(df)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"trusted":true},"cell_type":"code","source":"# For demonstration only\ndf = df[:10000]","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_uuid":"ef9a6aee05dec5ec4ba97fd5e8d2724ff4f743ae","scrolled":true,"trusted":true},"cell_type":"code","source":"# Split data set  to train and validation sets\nfrom sklearn.model_selection import train_test_split\n\n# Use stratify= df['label'] to get balance ratio 1/1 in train and validation sets\ndf_train, df_val = train_test_split(df, test_size=0.1, stratify= df['label'])\n\n# Check balancing\nprint(\"Train data: \" + str(len(df_train[df_train[\"label\"] == 1]) + len(df_train[df_train[\"label\"] == 0])))\nprint(\"True positive in train data: \" +  str(len(df_train[df_train[\"label\"] == 1])))\nprint(\"True negative in train data: \" +  str(len(df_train[df_train[\"label\"] == 0])))\nprint(\"Valid data: \" + str(len(df_val[df_val[\"label\"] == 1]) + len(df_val[df_val[\"label\"] == 0])))\nprint(\"True positive in validation data: \" +  str(len(df_val[df_val[\"label\"] == 1])))\nprint(\"True negative in validation data: \" +  str(len(df_val[df_val[\"label\"] == 0])))","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_uuid":"89f0b8d6439c876464b9e44062852a9eb747426f","trusted":true},"cell_type":"code","source":"# Train List\ntrain_list = df_train['id'].tolist()\ntrain_list = ['../input/train/'+ name + \".tif\" for name in train_list]\n\n# Validation List\nval_list = df_val['id'].tolist()\nval_list = ['../input/train/'+ name + \".tif\" for name in val_list]\n\n# Test list\ntest_list = df_test['id'].tolist()\ntest_list = ['../input/test/'+ name + \".tif\" for name in test_list]\n\n# Names library\nid_label_map = {k:v for k,v in zip(df.id.values, df.label.values)}","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_uuid":"5d5639a7f9ebeb8fa044f5743bb6237d73f7fe09","trusted":true},"cell_type":"code","source":"# Functions for generators\ndef get_id_from_file_path(file_path):\n    return file_path.split(os.path.sep)[-1].replace('.tif', '')\n\ndef chunker(seq, size):\n    return (seq[pos:pos + size] for pos in range(0, len(seq), size))","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_uuid":"4e6a540ef8ae82acefbbe97671de4f2fbbb9dc44","trusted":true},"cell_type":"code","source":"# Augmentation\nfrom imgaug import augmenters as iaa\nimport imgaug as ia\nimport numpy as np \nimport cv2\n\ndef augmentation():\n    sometimes = lambda aug: iaa.Sometimes(0.5, aug)\n    seq = iaa.Sequential(\n        [\n            # apply the following augmenters to most images\n            iaa.Fliplr(0.5), # horizontally flip 50% of all images\n            iaa.Flipud(0.5), # vertically flip 20% of all images\n            \n            # crop images by -10% to 10% of their height/width\n            sometimes(iaa.CropAndPad(\n                percent=(-0.2, 0.2),\n                pad_mode=ia.ALL,\n                pad_cval=(0, 255))),\n            \n            sometimes(iaa.Affine(\n                scale={\"x\": (0.8, 1.2), \"y\": (0.8, 1.2)}, # scale images to 80-120% of their size, individually per axis\n                translate_percent={\"x\": (-0.2, 0.2), \"y\": (-0.2, 0.2)}, # translate by -20 to +20 percent (per axis)\n                rotate=(-45, 45), # rotate by -45 to +45 degrees\n                order=[0, 1], # use nearest neighbour or bilinear interpolation (fast)\n                cval=(0, 255), # if mode is constant, use a cval between 0 and 255\n                mode=ia.ALL # use any of scikit-image's warping modes (see 2nd image from the top for examples)\n            )),\n            # execute 0 to 5 of the following (less important) augmenters per image\n            # don't execute all of them, as that would often be way too strong\n            \n            iaa.SomeOf((0, 5),\n                [    \n                    iaa.Sharpen(alpha=(0, 1.0), lightness=(0.9, 1.1)), # sharpen images\n                    iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)), # emboss images\n                    iaa.AdditiveGaussianNoise(scale=(0, 0.05*255)), # add gaussian noise to images\n                    iaa.AddToHueAndSaturation((-1, 1)) # change hue and saturation\n                ],\n                random_order=True\n            )\n        ],\n        random_order=True\n    )\n    return seq","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_uuid":"94bfd61b2dfcd0f4616e47ede87ab9ddb1a2f83e","trusted":true},"cell_type":"code","source":"# Import Pretrained Models\nimport keras\nfrom keras.applications.densenet import DenseNet201, preprocess_input\nfrom keras.layers import Dense, Input, Dropout, MaxPooling2D, Concatenate, GlobalAveragePooling2D, GlobalMaxPooling2D, Flatten, Concatenate\nfrom keras.models import Model\n\ninputTensor = Input((96,96,3))\nmodel_DenseNet201 = keras.applications.DenseNet201(include_top=False,input_shape = (96,96,3), weights='imagenet')","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_uuid":"0a58666bdbd7cb0aa548868a07d34eed44b076d1","trusted":true},"cell_type":"code","source":"# Concatenate Pretrained Models\n######\n#models = [model_DenseNet201, model_ResNet50]\nmodels = [model_DenseNet201]\n######\n\noutputTensors = [m(inputTensor) for m in models]\nif len(models) > 1:\n    output = Concatenate()(outputTensors) \nelse:\n    output = outputTensors[0]","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_uuid":"96e7734847ae13f7be430691cfa8f0a7b96643c8","trusted":true},"cell_type":"code","source":"# Classifier\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.layers import Activation\nfrom keras.models import Sequential\nfrom keras import applications\nfrom keras.layers import Dense, Input, Dropout, MaxPooling2D, Concatenate, GlobalAveragePooling2D, GlobalMaxPooling2D, Flatten, Concatenate\n\nout1 = GlobalAveragePooling2D()(output)\nout2 = GlobalMaxPooling2D()(output)\nout = Concatenate(axis=-1)([out1, out2])\nout = BatchNormalization()(out)\nout = Dropout(0.6)(out)\nout = Dense(1, activation=\"sigmoid\")(out)\nmodel = Model(inputTensor,out)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_uuid":"5b8619e486a52a3d7589e7ee1eef1d87ace6c856","trusted":true},"cell_type":"code","source":"# Read, convert and resize iamges\nimport cv2 as cv\nimport os\n\ndef read_image(path):\n    assert os.path.isfile(path), \"File not found\"\n    im = cv.imread(path)\n    # Convert from cv2 standard of BGR to our convention of RGB.\n    im = cv.cvtColor(im, cv.COLOR_BGR2RGB)\n    return im","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_uuid":"f5eb4e06ec6fdcd61241ae2a4ca4576620e7759c","scrolled":false,"trusted":true},"cell_type":"code","source":"# Train Generator\ndef train_gen(list_files, id_label_map, batch_size, augment = False):  \n    while True:\n        \n        shuffle(list_files)\n        \n        for batch in chunker(list_files, batch_size):\n            \n            X = [read_image(x) for x in batch]\n            Y = [id_label_map[get_id_from_file_path(x)] for x in batch]      \n            \n            if augment:\n                X = augmentation().augment_images(X)   \n            \n            X = [preprocess_input(x) for x in X]  \n            yield np.array(X), np.array(Y)\n\n# Validation Generator \n\ndef val_gen(list_files, batch_size):\n    \n    while True:\n        for batch in chunker(list_files, batch_size):\n                \n            X = [read_image(x) for x in batch]\n            X = [preprocess_input(x) for x in X]        \n            yield np.array(X)    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* #### Here is the implementation of One Cycle Policy in the Keras Callback Class"},{"metadata":{"_uuid":"6f0cdeea4e204cbe8c4518de1e9502b317e6f97a","trusted":true},"cell_type":"code","source":"# Implement One Cycle Policy Algorithm in the Keras Callback Class\n\nfrom sklearn.metrics import log_loss, roc_auc_score, accuracy_score\nfrom keras.losses import binary_crossentropy\nfrom keras.metrics import binary_accuracy\nfrom keras import backend as K\nfrom keras.callbacks import *\n\nclass CyclicLR(keras.callbacks.Callback):\n    \n    def __init__(self,base_lr, max_lr, step_size, base_m, max_m, cyclical_momentum):\n \n        self.base_lr = base_lr\n        self.max_lr = max_lr\n        self.base_m = base_m\n        self.max_m = max_m\n        self.cyclical_momentum = cyclical_momentum\n        self.step_size = step_size\n        \n        self.clr_iterations = 0.\n        self.cm_iterations = 0.\n        self.trn_iterations = 0.\n        self.history = {}\n        \n    def clr(self):\n        \n        cycle = np.floor(1+self.clr_iterations/(2*self.step_size))\n        \n        if cycle == 2:\n            x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)          \n            return self.base_lr-(self.base_lr-self.base_lr/100)*np.maximum(0,(1-x))\n        \n        else:\n            x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)\n            return self.base_lr + (self.max_lr-self.base_lr)*np.maximum(0,(1-x))\n    \n    def cm(self):\n        \n        cycle = np.floor(1+self.clr_iterations/(2*self.step_size))\n        \n        if cycle == 2:\n            \n            x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1) \n            return self.max_m\n        \n        else:\n            x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)\n            return self.max_m - (self.max_m-self.base_m)*np.maximum(0,(1-x))\n        \n        \n    def on_train_begin(self, logs={}):\n        logs = logs or {}\n\n        if self.clr_iterations == 0:\n            K.set_value(self.model.optimizer.lr, self.base_lr)\n        else:\n            K.set_value(self.model.optimizer.lr, self.clr())\n            \n        if self.cyclical_momentum == True:\n            if self.clr_iterations == 0:\n                K.set_value(self.model.optimizer.momentum, self.cm())\n            else:\n                K.set_value(self.model.optimizer.momentum, self.cm())\n            \n            \n    def on_batch_begin(self, batch, logs=None):\n        \n        logs = logs or {}\n        self.trn_iterations += 1\n        self.clr_iterations += 1\n\n        self.history.setdefault('lr', []).append(K.get_value(self.model.optimizer.lr))\n        self.history.setdefault('iterations', []).append(self.trn_iterations)\n        \n        if self.cyclical_momentum == True:\n            self.history.setdefault('momentum', []).append(K.get_value(self.model.optimizer.momentum))\n\n        for k, v in logs.items():\n            self.history.setdefault(k, []).append(v)\n        \n        K.set_value(self.model.optimizer.lr, self.clr())\n        \n        if self.cyclical_momentum == True:\n            K.set_value(self.model.optimizer.momentum, self.cm())\n            ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e0c409e4875937c8e4eacb532e507f70dedede9f","trusted":true},"cell_type":"code","source":"# Define Ony Cycle Policy parameters and train model\n########################################################################################\nfrom keras.optimizers import Adam, SGD\nfrom keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping\n\n# CLR parameters\n\nbatch_size = 64\nepochs = 10\nmax_lr = 0.0005\nbase_lr = max_lr/10\nmax_m = 0.98\nbase_m = 0.85\n\ncyclical_momentum = True\naugment = True\ncycles = 2.35\n\niterations = round(len(train_list)/batch_size*epochs)\niterations = list(range(0,iterations+1))\nstep_size = len(iterations)/(cycles)\n\n\nmodel.compile(loss='binary_crossentropy', optimizer=SGD(0.0000001), metrics=['accuracy'])\n\nclr =  CyclicLR(base_lr=base_lr,\n                max_lr=max_lr,\n                step_size=step_size,\n                max_m=max_m,\n                base_m=base_m,\n                cyclical_momentum=cyclical_momentum)\n    \ncallbacks = [clr,\n            ModelCheckpoint(filepath='best_model.h5', monitor='val_loss',mode='min',verbose=1,save_best_only=True)]\n\nhistory = model.fit_generator(train_gen(train_list, id_label_map, batch_size, augment = augment),\n                              validation_data=train_gen(val_list, id_label_map, batch_size, augment = True),\n                              epochs = epochs,\n                              steps_per_epoch = len(train_list) // batch_size + 1,\n                              validation_steps = len(val_list) // batch_size + 1,\n                              callbacks=callbacks,\n                              verbose = 1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f257ba2df488ab25d49a27bae5bf4803cd1d28fd","trusted":true},"cell_type":"code","source":"# Plot Learning Rate\nimport matplotlib.pyplot as plt\nplt.plot(clr.history['iterations'], clr.history['lr'])\nplt.xlabel('Training Iterations')\nplt.ylabel('Learning Rate')\nplt.title(\"One Cycle Policy\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7d104e360220301c1cc96e95dec25faf346b762b","trusted":false},"cell_type":"code","source":"# Plot momentum\nimport matplotlib.pyplot as plt\nplt.plot(clr.history['iterations'], clr.history['momentum'])\nplt.xlabel('Training Iterations')\nplt.ylabel('Momentum')\nplt.title(\"One Cycle Policy\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fdf50e57ace1eeb2e6e5dff125fcc2c72eef685d","trusted":false},"cell_type":"code","source":"# Plot losses\nval_loss = history.history['val_loss']\nloss = history.history['loss']\nplt.plot(range(len(val_loss)),val_loss,'c',label='Validation loss')\nplt.plot(range(len(loss)),loss,'m',label='Train loss')\n\nplt.title('Training and validation losses')\nplt.legend()\nplt.xlabel('epochs')\nplt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}