{"cells":[{"metadata":{},"cell_type":"markdown","source":"In this notebook, I have created a training pipeline for the XceptionNet model. The final steps remaining are actually training the model and saving it as a .pb file (as per competition guidelines).","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Also, I've used this github kernel as a reference (34th place submission in last year's competition): https://github.com/jandaldrop/landmark-recognition-challenge/blob/master/landmarks-xception.ipynb ","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import warnings\n\nimport cv2\n\nimport keras\nimport keras.backend as K\n\nfrom keras import Model, Sequential\nfrom keras.layers import Dense, Dropout, Flatten, Input, LeakyReLU\nfrom keras.layers import BatchNormalization, Activation, Conv2D \nfrom keras.layers import GlobalAveragePooling2D, Lambda\nfrom keras.optimizers import Adam, RMSprop\n\nfrom keras.applications.xception import Xception\nfrom keras.applications.xception import preprocess_input\nfrom keras.preprocessing.image import ImageDataGenerator \nfrom keras.utils import to_categorical\nfrom keras.callbacks import ModelCheckpoint\n\nfrom sklearn.preprocessing import LabelEncoder, OneHotEncoder\nfrom sklearn.metrics import accuracy_score\n\nimport matplotlib.pyplot as plt\n\nimport numpy as np\nimport pandas as pd\n\nimport tensorflow as tf\n\n%matplotlib inline\n\nprint('Keras version:', keras.__version__)\n\nwarnings.simplefilter('default')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train_info = pd.read_csv('../input/landmark-retrieval-2020/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Sample file path: ../input/landmark-retrieval-2020/train/0/0/0/0000059611c7d079.jpg\nprefix = '../input/landmark-retrieval-2020/train/'\ntrain_info['file_path'] = train_info['id'].apply(lambda x:prefix+ x[0]+'/'+x[1]+'/'+x[2]+'/'+x+\".jpg\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_image_files = train_info['file_path'].values\n\n# Sample visualization\ntestimg = cv2.cvtColor(cv2.imread(np.random.choice(train_image_files)), cv2.COLOR_BGR2RGB)\nplt.imshow(testimg)\ntestimg.shape\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_encoder = LabelEncoder()\none_hot_encoder = OneHotEncoder(sparse=True)\n\ntrain_info['label'] = label_encoder.fit_transform(train_info['landmark_id'].values)\ntrain_info['one_hot'] = one_hot_encoder.fit_transform(\n                    train_info['label'].values.reshape(-1, 1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_cat =81313\n\nbatch_size = 48\nbatch_size_predict = 128\ninput_shape = (299,299)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_images(info, input_shape = input_shape):\n    input_shape = tuple(input_shape)\n    imgs = np.zeros((len(info), input_shape[0], input_shape[1], 3))\n\n    for i in range(len(info)):\n        fname = train_info.iloc[i]['file_path']\n        try:\n            img = cv2.cvtColor(\n                  cv2.resize(cv2.imread(fname),input_shape),\n                  cv2.COLOR_BGR2RGB)\n        except:\n            warnings.warn('Warning: could not read image: '+ fname +\n                          '. Use black img instead.')\n            img = np.zeros((input_shape[0], input_shape[1], 3))\n        imgs[i,:,:,:] = img\n    \n    return imgs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef load_cropped_images(info, crop_p=0.2, crop='random'):\n    new_res = np.array([int(input_shape[0]*(1+crop_p)), int(input_shape[1]*(1+crop_p))])\n    if crop == 'random':\n        cx0 = np.random.randint(new_res[0] - input_shape[0], size=len(info))\n        cy0 = np.random.randint(new_res[1] - input_shape[1], size=len(info))\n    else:\n        if crop == 'central':\n            cx0, cy0 = (new_res - input_shape) // 2                \n        if crop == 'upper left':\n            cx0, cy0 = 0, 0\n        if crop == 'upper right':\n            cx0, cy0 = new_res[1] - input_shape[1], 0\n        if crop == 'lower left':\n            cx0, cy0 = 0, new_res[0] - input_shape[0]\n        if crop=='lower right':\n            cx0, cy0 = new_res - input_shape        \n        cx0 = np.repeat(np.expand_dims(cx0, 0), len(info))\n        cy0 = np.repeat(np.expand_dims(cy0, 0), len(info))\n\n    cx1 = cx0 + input_shape[0]\n    cy1 = cy0 + input_shape[1]\n    \n    raw_imgs = load_images(info, input_shape=tuple(new_res))\n    \n    cropped_imgs = np.zeros((len(info), input_shape[0], input_shape[1], 3))\n    for ind in range(len(info)):\n        cropped_imgs[ind,:,:,:] = raw_imgs[ind,\n                                           cy0[ind]:cy1[ind],\n                                           cx0[ind]:cx1[ind], :]\n    \n    return cropped_imgs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_image_gen(info_arg, \n                  shuffle=True, \n                  image_aug=True, \n                  eq_dist=False, \n                  n_ref_imgs=16, \n                  crop_prob=0.5, \n                  crop_p=0.5):\n    if image_aug:\n        datagen = ImageDataGenerator(\n            rotation_range=4.,\n            width_shift_range=0.2,\n            height_shift_range=0.2,\n            shear_range=0.2,\n            zoom_range=0.5,\n            channel_shift_range=25,\n            horizontal_flip=True,\n            fill_mode='nearest')\n        \n        if crop_prob > 0:\n            datagen_crop = ImageDataGenerator(\n                rotation_range=4.,\n                shear_range=0.2,\n                zoom_range=0.1,\n                channel_shift_range=20,\n                horizontal_flip=True,\n                fill_mode='nearest')\n        \n    count = len(info_arg)\n    while True:\n        if eq_dist:\n            def sample(df):\n                return df.sample(min(n_ref_imgs, len(df)))\n            info = info_arg.groupby('landmark_id', group_keys=False).apply(sample)\n        else:\n            info = info_arg\n        print('Generate', len(info), 'for the next round.')\n        \n        #shuffle data\n        if shuffle and count >= len(info):\n            info = info.sample(frac=1)\n            count = 0\n            \n        # load images\n        for ind in range(0,len(info), batch_size):\n            count += batch_size\n\n            y = info['landmark_id'].values[ind:(ind+batch_size)]\n            \n            if np.random.rand() < crop_prob:\n                imgs = load_cropped_images(info.iloc[ind:(ind+batch_size)], \n                                           crop_p=crop_p*np.random.rand() + 0.01, \n                                           crop='random')\n                if image_aug:\n                    cflow = datagen_crop.flow(imgs, \n                                              y, \n                                              batch_size=imgs.shape[0], \n                                              shuffle=False)\n                    imgs, y = next(cflow)                    \n            else:\n                imgs = load_images(info.iloc[ind:(ind+batch_size)])\n                if image_aug:\n                    cflow = datagen.flow(imgs, \n                                       y, \n                                       batch_size=imgs.shape[0], \n                                       shuffle=False)\n                    imgs, y = next(cflow)             \n\n            imgs = preprocess_input(imgs)\n    \n            y_l = label_encoder.transform(y[y>=0.])        \n            y_oh = np.zeros((len(y), n_cat))\n            y_oh[y >= 0., :] = one_hot_encoder.transform(y_l.reshape(-1,1)).todense()\n                    \n            yield imgs, y_oh\n            \ntrain_gen = get_image_gen(train_info, \n                          eq_dist=False, \n                          n_ref_imgs=256, \n                          crop_prob=0.5, \n                          crop_p=0.5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Showing an example image,","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"X_example, y_example = next(train_gen)\nplt.imshow(X_example[1,:,:,:]/2. + 0.5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## The NN model\n\nLet's build the actual model!","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"K.clear_session()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_model = Xception(input_shape=list(input_shape) + [3], \n                   weights='imagenet', \n                   include_top=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_model.summary()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print((x_model.layers[85]).name)\nprint((x_model.layers[25]).name)\nprint((x_model.layers[15]).name)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for layer in x_model.layers:\n    layer.trainable = True\n\nfor layer in x_model.layers[:85]:\n    layer.trainable = False   \n    \nx_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gm_exp = tf.Variable(3., dtype=tf.float32)\ndef generalized_mean_pool_2d(X):\n    pool = (tf.reduce_mean(tf.abs(X**(gm_exp)), \n                           axis=[1,2], \n                           keepdims=False)+1.e-8)**(1./gm_exp)\n    return pool","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_feat = Input(x_model.output_shape[1:])\n\nlambda_layer = Lambda(generalized_mean_pool_2d)\nlambda_layer.trainable_weights.extend([gm_exp])\nX = lambda_layer(X_feat)\nX = Dropout(0.05)(X)\nX = Activation('relu')(X)\nX = Dense(n_cat, activation='softmax')(X)\n\ntop_model = Model(inputs=X_feat, outputs=X)\ntop_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_image = Input(list(input_shape) + [3])\n\nX_f = x_model(X_image)\nX_f = top_model(X_f)\n\nmodel = Model(inputs=X_image, outputs=X_f)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Custom loss function","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_custom_loss(rank_weight=1., epsilon=1.e-9):\n    def custom_loss(y_t, y_p):\n        losses = tf.reduce_sum(-y_t*tf.math.log(y_p+epsilon) - (1.-y_t)*tf.math.log(1.-y_p+epsilon), \n                               axis=-1)\n        \n        pred_idx = tf.argmax(y_p, axis=-1)\n        \n        mask = tf.one_hot(pred_idx, \n                          depth=y_p.shape[1], \n                          dtype=tf.bool, \n                          on_value=True, \n                          off_value=False)\n        pred_cat = tf.boolean_mask(y_p, mask)\n        y_t_cat = tf.boolean_mask(y_t, mask)\n        \n        n_pred = tf.shape(pred_cat)[0]\n        _, ranks = tf.nn.top_k(pred_cat, k=n_pred)\n        \n        ranks = tf.cast(n_pred-ranks, tf.float32)/tf.cast(n_pred, tf.float32)*rank_weight\n        rank_losses = ranks*(-y_t_cat*tf.math.log(pred_cat+epsilon)\n                             -(1.-y_t_cat)*tf.math.log(1.-pred_cat+epsilon))        \n        \n        return rank_losses + losses\n    return custom_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def batch_GAP(y_t, y_p):\n    pred_cat = tf.argmax(y_p, axis=-1)    \n    y_t_cat = tf.argmax(y_t, axis=-1) * tf.cast(\n        tf.reduce_sum(y_t, axis=-1), tf.int64)\n    \n    n_pred = tf.shape(pred_cat)[0]\n    is_c = tf.cast(tf.equal(pred_cat, y_t_cat), tf.float32)\n\n    GAP = tf.reduce_mean(\n          tf.cumsum(is_c) * is_c / tf.cast(\n              tf.range(1, n_pred + 1), \n              dtype=tf.float32))\n    \n    return GAP","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef binary_crossentropy_n_cat(y_t, y_p):\n    return keras.metrics.binary_crossentropy(y_t, y_p) * n_cat","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"opt = Adam(lr=0.0001)\nloss = get_custom_loss(1.0)\nmodel.compile(loss=loss, \n              optimizer=opt, \n              metrics=[binary_crossentropy_n_cat, 'accuracy', batch_GAP])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"checkpoint1 = ModelCheckpoint('dd_checkpoint-1.h5', \n                              period=1, \n                              verbose=1, \n                              save_weights_only=True)\ncheckpoint2 = ModelCheckpoint('dd_checkpoint-2.h5', \n                              period=1, \n                              verbose=1, \n                              save_weights_only=True)\ncheckpoint3 = ModelCheckpoint('dd_checkpoint-3-best.h5', \n                              period=1, \n                              verbose=1, \n                              monitor='loss', \n                              save_best_only=True, \n                              save_weights_only=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit_generator(train_gen, \n                    steps_per_epoch=len(train_info) / batch_size / 8, \n                    verbose=2,\n                    epochs=100)\n\n# model.fit_generator(train_gen, \n#                     steps_per_epoch=len(train_info) / batch_size / 8, \n#                     epochs=1, \n#                     callbacks=[checkpoint1, checkpoint2, checkpoint3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Doesn't work right now\n\ntf.saved_model.save(model, \"../input/output/\", signatures=None, options=None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Code for converting to zip file\n\nfrom zipfile import ZipFile\n\nwith ZipFile('submission.zip','w') as zip:           \n    zip.write('../input/baseline-landmark-retrieval-model/baseline_landmark_retrieval_model/saved_model.pb', arcname='saved_model.pb') \n    zip.write('../input/baseline-landmark-retrieval-model/baseline_landmark_retrieval_model/variables/variables.data-00000-of-00001', arcname='variables/variables.data-00000-of-00001') \n    zip.write('../input/baseline-landmark-retrieval-model/baseline_landmark_retrieval_model/variables/variables.index', arcname='variables/variables.index') ","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}