{"cells":[{"metadata":{},"cell_type":"markdown","source":"The purpose of this notebook is to illustrate sample code for Resnet50 with Cyclical Learning rate on Tensorflow. This kernel just uses top 100 classes. For actual competition we would need to model all classes with neccessary methods to handle class imbalance","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nimport tensorflow as tf\nimport cv2\nimport skimage.io\nfrom skimage.transform import resize\nfrom imgaug import augmenters as iaa\nfrom sklearn import preprocessing\nfrom sklearn.preprocessing import LabelBinarizer,LabelEncoder\nfrom sklearn.utils.class_weight import compute_class_weight\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.layers import Input, Add, Dense, Dropout, Activation, ZeroPadding2D, BatchNormalization, Flatten, Conv2D, AveragePooling2D, MaxPooling2D, GlobalMaxPooling2D,GlobalAveragePooling2D,Concatenate, ReLU, LeakyReLU,Reshape, Lambda\nfrom tensorflow.keras.callbacks import ModelCheckpoint, LearningRateScheduler, EarlyStopping, ReduceLROnPlateau\nfrom tensorflow.keras.optimizers import Adam,SGD\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.models import Sequential, load_model, Model\nfrom tensorflow.keras.callbacks import LearningRateScheduler\nfrom tensorflow.keras.applications.resnet50 import ResNet50 \nfrom tensorflow.keras.preprocessing import image\nfrom tensorflow.keras.applications.vgg16 import preprocess_input\nfrom tensorflow.keras.applications.vgg16 import decode_predictions\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras import metrics\nfrom tensorflow.keras.preprocessing import image\nfrom tensorflow.keras.applications.imagenet_utils import preprocess_input\nfrom tqdm import tqdm\nimport imgaug as ia\nfrom imgaug import augmenters as iaa\nfrom PIL import Image\nimport keras.backend as K\nK.set_image_data_format('channels_last')\nK.set_learning_phase(1)\n\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Path retrieval**\n\n\nTaken from the public kernel...thanks for sharing this. It was quite helpful\nhttps://www.kaggle.com/derinformatiker/landmark-retrieval-all-paths","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/landmark-retrieval-2020/train.csv\")\ndef get_paths(sub):\n    index = [\"0\",\"1\",\"2\",\"3\",\"4\",\"5\",\"6\",\"7\",\"8\",\"9\",\"a\",\"b\",\"c\",\"d\",\"e\",\"f\"]\n\n    paths = []\n\n    for a in index:\n        for b in index:\n            for c in index:\n                try:\n                    paths.extend([f\"../input/landmark-retrieval-2020/{sub}/{a}/{b}/{c}/\" + x for x in os.listdir(f\"../input/landmark-retrieval-2020/{sub}/{a}/{b}/{c}\")])\n                except:\n                    pass\n\n    return paths","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_path = train\n\nrows = []\nfor i in tqdm(range(len(train))):\n    row = train.iloc[i]\n    path  = list(row[\"id\"])[:3]\n    temp = row[\"id\"]\n    row[\"id\"] = f\"../input/landmark-retrieval-2020/train/{path[0]}/{path[1]}/{path[2]}/{temp}.jpg\"\n    rows.append(row[\"id\"])\n    \nrows = pd.DataFrame(rows)\ntrain_path[\"id\"] = rows","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Setting up some basic model specs**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 128\nseed = 42\nshape = (224, 224, 3) ##desired shape of the image for resizing purposes\nval_sample = 0.1 # 10 % as validation sample\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_labels = pd.read_csv('../input/landmark-retrieval-2020/train.csv')\ntrain_labels.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"k =train[['id','landmark_id']].groupby(['landmark_id']).agg({'id':'count'})\nk.rename(columns={'id':'Count_class'}, inplace=True)\nk.reset_index(level=(0), inplace=True)\nfreq_ct_df = pd.DataFrame(k)\nfreq_ct_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_labels = pd.merge(train,freq_ct_df, on = ['landmark_id'], how='left')\ntrain_labels.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"freq_ct_df.sort_values(by=['Count_class'],ascending=False,inplace=True)\nfreq_ct_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Due to Kernel runtime limits, for illustrating the example let's run this for top 100 landmark categories**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"freq_ct_df_top100 = freq_ct_df.iloc[:100]\ntop100_class = freq_ct_df_top100['landmark_id'].tolist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"top100class_train = train_path[train_path['landmark_id'].isin (top100_class) ]\ntop100class_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def getTrainParams():\n    data = top100class_train.copy()\n    le = preprocessing.LabelEncoder()\n    data['label'] = le.fit_transform(data['landmark_id'])\n    lbls = top100class_train['landmark_id'].tolist()\n    lb = LabelBinarizer()\n    labels = lb.fit_transform(lbls)\n    \n    return np.array(top100class_train['id'].tolist()),np.array(labels),le","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Credit to Michal Haltuf from whose kernel I had first learnt Data Generators two years back.\nYou can check out his kernel here: https://www.kaggle.com/rejpalcz/cnn-128x128x4-keras-from-scratch-lb-0-328","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class Landmark2020_DataGenerator(tf.keras.utils.Sequence):\n    def __init__(self, paths, labels, batch_size, shape, shuffle = False, use_cache = False, augment = False):\n        self.paths, self.labels = paths, labels\n        self.batch_size = batch_size\n        self.shape = shape\n        self.shuffle = shuffle\n        self.use_cache = use_cache\n        self.augment = augment\n        if use_cache == True:\n            self.cache = np.zeros((paths.shape[0], shape[0], shape[1], shape[2]), dtype=np.float16)\n            self.is_cached = np.zeros((paths.shape[0]))\n        self.on_epoch_end()\n    \n    def __len__(self):\n        return int(np.ceil(len(self.paths) / float(self.batch_size)))\n    \n    def __getitem__(self, idx):\n        indexes = self.indexes[idx * self.batch_size : (idx+1) * self.batch_size]\n\n        paths = self.paths[indexes]\n        X = np.zeros((paths.shape[0], self.shape[0], self.shape[1], self.shape[2]))\n        # Generate data\n        if self.use_cache == True:\n            X = self.cache[indexes]\n            for i, path in enumerate(paths[np.where(self.is_cached[indexes] == 0)]):\n                image = self.__load_image(path)\n                self.is_cached[indexes[i]] = 1\n                self.cache[indexes[i]] = image\n                X[i] = image\n        else:\n            for i, path in enumerate(paths):\n                X[i] = self.__load_image(path)\n\n        y = self.labels[indexes]\n                \n        if self.augment == True:\n            seq = iaa.Sequential([\n                iaa.OneOf([\n                    iaa.Fliplr(0.5), # horizontal flips\n                    \n                    iaa.ContrastNormalization((0.75, 1.5)),\n                    iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.05*255), per_channel=0.5),\n                    iaa.Multiply((0.8, 1.2), per_channel=0.2),\n                    \n                    iaa.Affine(rotate=0),\n                    #iaa.Affine(rotate=90),\n                    #iaa.Affine(rotate=180),\n                    #iaa.Affine(rotate=270),\n                    iaa.Fliplr(0.5),\n                    #iaa.Flipud(0.5),\n                ])], random_order=True)\n\n            X = np.concatenate((X, seq.augment_images(X), seq.augment_images(X), seq.augment_images(X)), 0)\n            y = np.concatenate((y, y, y, y), 0)\n        \n        return X, y\n    \n    def on_epoch_end(self):\n        \n        # Updates indexes after each epoch\n        self.indexes = np.arange(len(self.paths))\n        if self.shuffle == True:\n            np.random.shuffle(self.indexes)\n\n    def __iter__(self):\n        \"\"\"Create a generator that iterate over the Sequence.\"\"\"\n        for item in (self[i] for i in range(len(self))):\n            yield item\n            \n    def __load_image(self, path):\n        image_norm = skimage.io.imread(path)/255.0\n        \n\n        im = resize(image_norm, (shape[0], shape[1],shape[2]), mode='reflect')\n        return im","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Cyclical LR credits: https://github.com/bckenstler/CLR","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.callbacks import *\n\nclass CyclicLR(Callback):\n    \"\"\"This callback implements a cyclical learning rate policy (CLR).\n    The method cycles the learning rate between two boundaries with\n    some constant frequency, as detailed in this paper (https://arxiv.org/abs/1506.01186).\n    The amplitude of the cycle can be scaled on a per-iteration or \n    per-cycle basis.\n    This class has three built-in policies, as put forth in the paper.\n    \"triangular\":\n        A basic triangular cycle w/ no amplitude scaling.\n    \"triangular2\":\n        A basic triangular cycle that scales initial amplitude by half each cycle.\n    \"exp_range\":\n        A cycle that scales initial amplitude by gamma**(cycle iterations) at each \n        cycle iteration.\n    For more detail, please see paper.\n    \n    # Example\n        ```python\n            clr = CyclicLR(base_lr=0.001, max_lr=0.006,\n                                step_size=2000., mode='triangular')\n            model.fit(X_train, Y_train, callbacks=[clr])\n        ```\n    \n    Class also supports custom scaling functions:\n        ```python\n            clr_fn = lambda x: 0.5*(1+np.sin(x*np.pi/2.))\n            clr = CyclicLR(base_lr=0.001, max_lr=0.006,\n                                step_size=2000., scale_fn=clr_fn,\n                                scale_mode='cycle')\n            model.fit(X_train, Y_train, callbacks=[clr])\n        ```    \n    # Arguments\n        base_lr: initial learning rate which is the\n            lower boundary in the cycle.\n        max_lr: upper boundary in the cycle. Functionally,\n            it defines the cycle amplitude (max_lr - base_lr).\n            The lr at any cycle is the sum of base_lr\n            and some scaling of the amplitude; therefore \n            max_lr may not actually be reached depending on\n            scaling function.\n        step_size: number of training iterations per\n            half cycle. Authors suggest setting step_size\n            2-8 x training iterations in epoch.\n        mode: one of {triangular, triangular2, exp_range}.\n            Default 'triangular'.\n            Values correspond to policies detailed above.\n            If scale_fn is not None, this argument is ignored.\n        gamma: constant in 'exp_range' scaling function:\n            gamma**(cycle iterations)\n        scale_fn: Custom scaling policy defined by a single\n            argument lambda function, where \n            0 <= scale_fn(x) <= 1 for all x >= 0.\n            mode paramater is ignored \n        scale_mode: {'cycle', 'iterations'}.\n            Defines whether scale_fn is evaluated on \n            cycle number or cycle iterations (training\n            iterations since start of cycle). Default is 'cycle'.\n    \"\"\"\n\n    def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',\n                 gamma=1., scale_fn=None, scale_mode='cycle'):\n        super(CyclicLR, self).__init__()\n\n        self.base_lr = base_lr\n        self.max_lr = max_lr\n        self.step_size = step_size\n        self.mode = mode\n        self.gamma = gamma\n        if scale_fn == None:\n            if self.mode == 'triangular':\n                self.scale_fn = lambda x: 1.\n                self.scale_mode = 'cycle'\n            elif self.mode == 'triangular2':\n                self.scale_fn = lambda x: 1/(2.**(x-1))\n                self.scale_mode = 'cycle'\n            elif self.mode == 'exp_range':\n                self.scale_fn = lambda x: gamma**(x)\n                self.scale_mode = 'iterations'\n        else:\n            self.scale_fn = scale_fn\n            self.scale_mode = scale_mode\n        self.clr_iterations = 0.\n        self.trn_iterations = 0.\n        self.history = {}\n\n        self._reset()\n\n    def _reset(self, new_base_lr=None, new_max_lr=None,\n               new_step_size=None):\n        \"\"\"Resets cycle iterations.\n        Optional boundary/step size adjustment.\n        \"\"\"\n        if new_base_lr != None:\n            self.base_lr = new_base_lr\n        if new_max_lr != None:\n            self.max_lr = new_max_lr\n        if new_step_size != None:\n            self.step_size = new_step_size\n        self.clr_iterations = 0.\n        \n    def clr(self):\n        cycle = np.floor(1+self.clr_iterations/(2*self.step_size))\n        x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)\n        if self.scale_mode == 'cycle':\n            return self.base_lr + (self.max_lr-self.base_lr)*np.maximum(0, (1-x))*self.scale_fn(cycle)\n        else:\n            return self.base_lr + (self.max_lr-self.base_lr)*np.maximum(0, (1-x))*self.scale_fn(self.clr_iterations)\n        \n    def on_train_begin(self, logs={}):\n        logs = logs or {}\n\n        if self.clr_iterations == 0:\n            K.set_value(self.model.optimizer.lr, self.base_lr)\n        else:\n            K.set_value(self.model.optimizer.lr, self.clr())        \n            \n    def on_batch_end(self, epoch, logs=None):\n        \n        logs = logs or {}\n        self.trn_iterations += 1\n        self.clr_iterations += 1\n\n        self.history.setdefault('lr', []).append(K.get_value(self.model.optimizer.lr))\n        self.history.setdefault('iterations', []).append(self.trn_iterations)\n\n        for k, v in logs.items():\n            self.history.setdefault(k, []).append(v)\n        \n        K.set_value(self.model.optimizer.lr, self.clr())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_model(input_shape, n_out):\n    inp = Input(input_shape)\n    pretrain_model = ResNet50(include_top=False, weights='imagenet', input_tensor=inp)\n    #x = pretrain_model.get_layer(name=\"block_13_expand_relu\").output\n    x = pretrain_model.output\n    \n    x = GlobalAveragePooling2D()(x)\n    x = Dropout(0.5)(x)\n    x = Dense(n_out, activation=\"sigmoid\")(x)\n    \n    #for layer in pretrain_model.layers[:160]:\n        #layer.trainable = False\n    \n    for layer in pretrain_model.layers:\n        layer.trainable = False\n        \n    return Model(inp, x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.metrics import categorical_accuracy,top_k_categorical_accuracy\ndef top_5_accuracy(y_true, y_pred):\n    return top_k_categorical_accuracy(y_true, y_pred, k=5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nlabls = top100class_train['landmark_id'].nunique()\nmodel = create_model(input_shape=(224,224,3), n_out=nlabls)\nmodel.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['acc',top_5_accuracy])\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"paths, labels,_ = getTrainParams()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"keys = np.arange(paths.shape[0], dtype=np.int)  \nnp.random.seed(seed)\nnp.random.shuffle(keys)\nlastTrainIndex = int((1-val_sample) * paths.shape[0])\n\npathsTrain = paths[0:lastTrainIndex]\nlabelsTrain = labels[0:lastTrainIndex]\n\npathsVal = paths[lastTrainIndex:]\nlabelsVal = labels[lastTrainIndex:]\n\nprint(paths.shape, labels.shape)\nprint(pathsTrain.shape, labelsTrain.shape, pathsVal.shape, labelsVal.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_generator = Landmark2020_DataGenerator(pathsTrain, labelsTrain, batch_size, shape, use_cache=False, augment = False, shuffle = True)\nval_generator = Landmark2020_DataGenerator(pathsVal, labelsVal, batch_size, shape, use_cache=False, shuffle = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clr = CyclicLR(base_lr=0.005, max_lr=0.01,step_size=2000., mode='triangular')\nepochs = 2\nuse_multiprocessing = True \n#workers = 1 ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base_cnn = model.fit_generator(\n    train_generator,\n    steps_per_epoch=len(train_generator),\n    validation_data=val_generator,\n    validation_steps=8,\n    #class_weight = class_weights,\n    epochs=epochs,\n    callbacks = [clr],\n    use_multiprocessing=use_multiprocessing,\n    #workers=workers,\n    verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.save('ResNet50.h5')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**If you found this useful, an upvote would be much appreciated.**","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}