{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd \nimport os\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nimport random\nimport cv2\nimport tensorflow as tf","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-14T09:15:23.918515Z","iopub.execute_input":"2021-11-14T09:15:23.919029Z","iopub.status.idle":"2021-11-14T09:15:28.607933Z","shell.execute_reply.started":"2021-11-14T09:15:23.918942Z","shell.execute_reply":"2021-11-14T09:15:28.607150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = '/kaggle/input/landmark-recognition-2021'\nos.listdir(path)\n\ntrain_images = f'{path}/train'\ndf_train = pd.read_csv(f'{path}/train.csv')\ndf_train['path'] = df_train['id'].apply(lambda f: os.path.join('/kaggle/input/landmark-recognition-2021/train',f[0], f[1], f[2], f + '.jpg'))\n\ntest_images = f'{path}/test'\ndf_test = pd.read_csv(f'{path}/sample_submission.csv')\ndf_test['path'] = df_test['id'].apply(lambda f: os.path.join('/kaggle/input/landmark-recognition-2021/test',f[0], f[1], f[2], f + '.jpg'))\n\n# Defining the amount of classes and images in the training dataset.\nnr_classes = len(df_train[\"landmark_id\"].unique())\nnr_images = len(df_train)\n\nprint(\"Number of classes in training dataset: \", nr_classes)\nprint(\"Number of images in training dataset: \", nr_images)","metadata":{"execution":{"iopub.status.busy":"2021-11-14T09:15:28.609685Z","iopub.execute_input":"2021-11-14T09:15:28.609971Z","iopub.status.idle":"2021-11-14T09:15:35.974393Z","shell.execute_reply.started":"2021-11-14T09:15:28.609934Z","shell.execute_reply":"2021-11-14T09:15:35.972784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Histogram of data distribution, to show the amount of images in each class.\n\nhist = plt.figure(figsize = (10, 10))\nax = plt.hist(df_train[\"landmark_id\"], bins = df_train[\"landmark_id\"].unique())\nplt.ylim([0, 400])\nplt.show()\n\nclasses = ax[0]\nfrom0To5 = len(classes[classes <= 5])\nfrom5To10 = len(classes[classes <= 10] - from0To5)\nprint(\"Number of classes with 0 to 5 images: \", from0To5)\nprint(\"Number of classes with 5 to 10 images: \", from5To10)","metadata":{"execution":{"iopub.status.busy":"2021-11-14T09:15:35.975552Z","iopub.execute_input":"2021-11-14T09:15:35.975793Z","iopub.status.idle":"2021-11-14T09:17:57.114091Z","shell.execute_reply.started":"2021-11-14T09:15:35.975760Z","shell.execute_reply":"2021-11-14T09:17:57.113332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Overall representation of the data distribution\n\nprint('Distribution of images in classes:')\nvalue_counts = df_train['landmark_id'].value_counts()\nprint(value_counts.describe()) ","metadata":{"execution":{"iopub.status.busy":"2021-11-14T09:17:57.116276Z","iopub.execute_input":"2021-11-14T09:17:57.116952Z","iopub.status.idle":"2021-11-14T09:17:57.154691Z","shell.execute_reply.started":"2021-11-14T09:17:57.116915Z","shell.execute_reply":"2021-11-14T09:17:57.153962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Visualization of some samples.\n\ndisplayImages = []\nfor i in range(0,4):\n    randomClass = df_train[df_train['landmark_id'] == value_counts.iloc[[np.random.randint(0, nr_classes)]].index[0]]\n    for j in range(0,4):\n        randomImages = randomClass.iloc[np.random.randint(0, len(randomClass))]\n        displayImages.append(randomImages)\n        \nplt.subplots(4, 4, figsize = (15, 10))\nfor i in range(len(displayImages)):\n    plt.subplot(4, 4, i + 1)\n    plt.axis('Off')\n    img = cv2.imread(displayImages[i][2])\n    plt.imshow(img)\n    plt.title(f'landmark id: {displayImages[i][1]} ', fontsize=8)","metadata":{"execution":{"iopub.status.busy":"2021-11-14T09:17:57.162833Z","iopub.execute_input":"2021-11-14T09:17:57.163113Z","iopub.status.idle":"2021-11-14T09:17:59.385649Z","shell.execute_reply.started":"2021-11-14T09:17:57.163077Z","shell.execute_reply":"2021-11-14T09:17:59.384832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Setting up hyperparameters and splitting training data into train and val\ndef imagePath(imgPath):\n    images = []\n    for imgFile in imgPath:\n        imgPic = cv2.imread(imgFile, 1)\n        images.append(cv2.resize(imgPic, (img_size, img_size)))\n    \n    return images\n\n# Hyperparameters\nepochs = 20\nbatch_size = 32\nimg_size = 128\ntrain_split = 0.7\nval_split = 0.2\nnrClasses = 100\n\n# Setting up dataset for training\nimgList = []\nlabels = []\ntemp_labels = []\n\ni = 0\nfor lbl in df_train['landmark_id'].unique():\n    if i == nrClasses:\n        break\n    if(len(df_train['path'][df_train['landmark_id'] == lbl].value_counts()) > 200 and\n       len(df_train['path'][df_train['landmark_id'] == lbl].value_counts()) < 400): \n        for path in df_train['path'][df_train['landmark_id'] == lbl]: \n            imgList.append(path) \n            labels.append(lbl)\n            temp_labels.append(i)\n        i = i + 1\n\n# Random shuffle dataset, so it is no longer set up in classes\nshuff = list(zip(imgList, temp_labels))\nrandom.shuffle(shuff)\n\nimgList, lbls = zip(*shuff)\n\n# Preparing data to be split into train and val\nimgNr = round(len(imgList) * train_split)\n\ntrainImages = imgList[:imgNr]\ntrainData = imagePath(trainImages)\ntrainLabels = lbls[:imgNr]\n\nprint(\"Number of total images for training: \", len(trainData))\n\n# Setting images and labels to be split into train and val data\nxData = np.array(trainData)\nyData = tf.keras.utils.to_categorical(trainLabels, num_classes = nrClasses)\n\nx_train, x_val, y_train, y_val = train_test_split(xData, yData, test_size = val_split, random_state = 101)\n\nprint(\"Number of train images: \", len(x_train))\nprint(\"Number of val images: \", len(x_val))\n\n# Random Crop for Data Augmentation\ndef random_crop(image):  \n    height, width = image.shape[:2]\n    random_array = np.random.random(size=4);\n    w = int((width*0.5) * (1+random_array[0]*0.5))\n    h = int((height*0.5) * (1+random_array[1]*0.5))\n    x = int(random_array[2] * (width-w))\n    y = int(random_array[3] * (height-h))\n\n    image_crop = image[y:h+y, x:w+x, 0:3]\n    image_crop = cv2.resize(image_crop, image.shape)\n    return image_crop\n\n# Setting up data generator for data augmentation\ndataGenerator = tf.keras.preprocessing.image.ImageDataGenerator(horizontal_flip = False, # False for no flip, True for flip\n                                                                vertical_flip = False, # False for no flip, True for flip\n                                                                rotation_range = 0, # 0 for no rotation, 90 for rotation\n                                                                zoom_range = 0, # 0 for normal, [0.5, 1.5] for augmentation\n                                                                width_shift_range = 0, # 0 for normal, 0.3 for augmentation\n                                                                height_shift_range = 0, # 0 for normal, 0.3 for augmentation\n                                                                shear_range = 0, # 0 for normal, 45 for augmentation\n                                                                #brightness_range=(0.1, 0.9), # comment for no brightness change\n                                                                #preprocessing_function=random_crop, \n                                                                fill_mode = \"nearest\") # wrap, reflect and constant are some other for experimentation\n                                                 \n\nopt = tf.optimizers.SGD(learning_rate = 0.001) # 0.01\nopt2 = tf.optimizers.Adam(learning_rate = 0.001) #0.01\nopt3 = tf.optimizers.Adagrad(learning_rate = 0.001) #0.01","metadata":{"execution":{"iopub.status.busy":"2021-11-14T09:17:59.386898Z","iopub.execute_input":"2021-11-14T09:17:59.387130Z","iopub.status.idle":"2021-11-14T09:23:22.135826Z","shell.execute_reply.started":"2021-11-14T09:17:59.387098Z","shell.execute_reply":"2021-11-14T09:23:22.135036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# CNN model \n# Resnet101 also be used and some other models as well\nResNet50 = tf.keras.applications.resnet.ResNet50(input_shape = (img_size, img_size, 3),\n                                                      include_top = False, #True\n                                                      weights = 'imagenet', \n                                                      pooling = 'avg') # max\n\n\ninputs = ResNet50.input\nflatten = tf.keras.layers.Flatten()(ResNet50.output)\ndropout1 = tf.keras.layers.Dropout(0.2)(flatten)\ndense1 = tf.keras.layers.Dense(units = 4096, activation = \"relu\")(dropout1)\ndropout2 = tf.keras.layers.Dropout(0.2)(dense1)\ndense2 = tf.keras.layers.Dense(units = 4096, activation = \"relu\")(dropout2)\ndropout3 = tf.keras.layers.Dropout(0.2)(dense2)\noutput = tf.keras.layers.Dense(units = nrClasses, activation = \"softmax\")(dropout3) # activation = relu, linear, softmax\nmodel = tf.keras.Model(inputs = inputs, outputs = output)\n\nprint(model.summary())\n","metadata":{"execution":{"iopub.status.busy":"2021-11-14T09:23:22.137747Z","iopub.execute_input":"2021-11-14T09:23:22.138300Z","iopub.status.idle":"2021-11-14T09:23:27.354789Z","shell.execute_reply.started":"2021-11-14T09:23:22.138261Z","shell.execute_reply":"2021-11-14T09:23:27.354100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\"\n# Since the Batch Normalisation cannot be turned off in ResNet so there is a Resnet50\n# The batch Normalisation can be commented out for experiment, also this is used a non pre-trained model.\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.layers import Input, Add, Dense, Dropout, Activation, ZeroPadding2D, BatchNormalization, Flatten, Conv2D, AveragePooling2D, MaxPooling2D, GlobalMaxPooling2D,GlobalAveragePooling2D,Concatenate, ReLU, LeakyReLU,Reshape, Lambda\nfrom tensorflow.keras.callbacks import ModelCheckpoint, LearningRateScheduler, EarlyStopping, ReduceLROnPlateau\nfrom tensorflow.keras.optimizers import Adam,SGD\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.models import Sequential, load_model, Model\nfrom tensorflow.keras.callbacks import LearningRateScheduler\nfrom tensorflow.keras.preprocessing import image\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras import metrics\nfrom tensorflow.keras.preprocessing import image\nfrom tensorflow.keras.applications.imagenet_utils import preprocess_input\nfrom tensorflow.keras.initializers import glorot_uniform\n\ndef identity_block(X, f, filters, stage, block):\n    \n    conv_name_base = 'res' + str(stage) + block + '_branch'\n    bn_name_base = 'bn' + str(stage) + block + '_branch'\n    \n    F1, F2, F3 = filters\n    \n    X_shortcut = X\n        \n    X = Conv2D(filters = F1, kernel_size = (1, 1), strides = (1,1), padding = 'valid', name = conv_name_base + '2a', kernel_initializer = glorot_uniform(seed=0))(X)\n    X = BatchNormalization(axis = 3, name = bn_name_base + '2a')(X)\n    X = Activation('relu')(X)\n        \n    X = Conv2D(filters = F2, kernel_size = (f, f), strides = (1,1), padding = 'same', name = conv_name_base + '2b', kernel_initializer = glorot_uniform(seed=0))(X)\n    X = BatchNormalization(axis = 3, name = bn_name_base + '2b')(X)\n    X = Activation('relu')(X)\n\n    X = Conv2D(filters = F3, kernel_size = (1, 1), strides = (1,1), padding = 'valid', name = conv_name_base + '2c', kernel_initializer = glorot_uniform(seed=0))(X)\n    X = BatchNormalization(axis = 3, name = bn_name_base + '2c')(X)\n\n    # Add shortcut value to main path\n    X = Add()([X_shortcut, X])\n    X = Activation('relu')(X)\n        \n    return X\n\ndef convolutional_block(X, f, filters, stage, block, s = 2):\n        \n    conv_name_base = 'res' + str(stage) + block + '_branch'\n    bn_name_base = 'bn' + str(stage) + block + '_branch'\n    F1, F2, F3 = filters\n    X_shortcut = X\n    X = Conv2D(F1, (1, 1), strides = (s,s), name = conv_name_base + '2a', kernel_initializer = glorot_uniform(seed=0))(X)\n    X = BatchNormalization(axis = 3, name = bn_name_base + '2a')(X)\n    X = Activation('relu')(X)\n    X = Conv2D(filters = F2, kernel_size = (f, f), strides = (1,1), padding = 'same', name = conv_name_base + '2b', kernel_initializer = glorot_uniform(seed=0))(X)\n    X = BatchNormalization(axis = 3, name = bn_name_base + '2b')(X)\n    X = Activation('relu')(X)\n    X = Conv2D(filters = F3, kernel_size = (1, 1), strides = (1,1), padding = 'valid', name = conv_name_base + '2c', kernel_initializer = glorot_uniform(seed=0))(X)\n    X = BatchNormalization(axis = 3, name = bn_name_base + '2c')(X)\n    X_shortcut = Conv2D(filters = F3, kernel_size = (1, 1), strides = (s,s), padding = 'valid', name = conv_name_base + '1', kernel_initializer = glorot_uniform(seed=0))(X_shortcut)\n    X_shortcut = BatchNormalization(axis = 3, name = bn_name_base + '1')(X_shortcut)\n    X = Add()([X_shortcut, X])\n    X = Activation('relu')(X)\n   \n    return X\n\ndef ResNet50(input_shape = (128, 128, 3), classes = nrClasses):\n    X_input = Input(input_shape)\n    X = ZeroPadding2D((3, 3))(X_input)\n    X = Conv2D(64, (7, 7), strides = (2, 2), name = 'conv1', kernel_initializer = glorot_uniform(seed=0))(X)\n    X = BatchNormalization(axis = 3, name = 'bn_conv1')(X)\n    X = Activation('relu')(X)\n    X = MaxPooling2D((3, 3), strides=(2, 2))(X)\n    X = convolutional_block(X, f = 3, filters = [64, 64, 256], stage = 2, block='a', s = 1)\n    X = identity_block(X, 3, [64, 64, 256], stage=2, block='b')\n    X = identity_block(X, 3, [64, 64, 256], stage=2, block='c')\n    X = convolutional_block(X, f = 3, filters = [128, 128, 512], stage = 3, block='a', s = 2)\n    X = identity_block(X, 3, [128, 128, 512], stage=3, block='b')\n    X = identity_block(X, 3, [128, 128, 512], stage=3, block='c')\n    X = identity_block(X, 3, [128, 128, 512], stage=3, block='d')\n    X = convolutional_block(X, f = 3, filters = [256, 256, 1024], stage = 4, block='a', s = 2)\n    X = identity_block(X, 3, [256, 256, 1024], stage=4, block='b')\n    X = identity_block(X, 3, [256, 256, 1024], stage=4, block='c')\n    X = identity_block(X, 3, [256, 256, 1024], stage=4, block='d')\n    X = identity_block(X, 3, [256, 256, 1024], stage=4, block='e')\n    X = identity_block(X, 3, [256, 256, 1024], stage=4, block='f')\n    X = convolutional_block(X, f = 3, filters = [512, 512, 2048], stage = 5, block='a', s = 2)\n    X = identity_block(X, 3, [512, 512, 2048], stage=5, block='b')\n    X = identity_block(X, 3, [512, 512, 2048], stage=5, block='c')\n    X = AveragePooling2D(pool_size=(2, 2),name='avg_pool')(X)\n    X = Flatten()(X)\n    X = Dense(classes, activation='softmax', name='fc' + str(classes), kernel_initializer = glorot_uniform(seed=0))(X)\n    model = Model(inputs = X_input, outputs = X, name='ResNet50')\n    return model\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2021-11-14T09:23:27.357393Z","iopub.execute_input":"2021-11-14T09:23:27.357829Z","iopub.status.idle":"2021-11-14T09:23:27.370056Z","shell.execute_reply.started":"2021-11-14T09:23:27.357791Z","shell.execute_reply":"2021-11-14T09:23:27.369352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Compilation\n#model = ResNet50(input_shape = (128, 128, 3), classes = nrClasses) # should be used when using batch norm editable resnet\nmodel.compile(optimizer = opt2 , loss = \"categorical_crossentropy\", metrics = ['accuracy'])\n#print(model.summary()) # should be used when using batch norm editable resnet\n\nhistory = model.fit(dataGenerator.flow(x_train, y_train, batch_size = batch_size), validation_data = (x_val, y_val), epochs = epochs)","metadata":{"execution":{"iopub.status.busy":"2021-11-14T09:23:27.371129Z","iopub.execute_input":"2021-11-14T09:23:27.373799Z","iopub.status.idle":"2021-11-14T09:38:42.045529Z","shell.execute_reply.started":"2021-11-14T09:23:27.373761Z","shell.execute_reply":"2021-11-14T09:38:42.044831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plotting the performance of the model\nplt.plot(history.history['accuracy'])\nplt.plot(history.history['val_accuracy'])\nplt.title('ResNet50 Accuracy')\nplt.ylabel('Accuracy')\nplt.xlabel('Epoch')\nplt.legend(['Train', 'Val'], loc = 'upper left')\nplt.show()\n\n# Plot of loss\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('ResNet50 Loss')\nplt.ylabel('Loss')\nplt.xlabel('Epoch')\nplt.legend(['Train', 'Val'], loc = 'upper left')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-14T09:38:42.046872Z","iopub.execute_input":"2021-11-14T09:38:42.047220Z","iopub.status.idle":"2021-11-14T09:38:42.459321Z","shell.execute_reply.started":"2021-11-14T09:38:42.047179Z","shell.execute_reply":"2021-11-14T09:38:42.458690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Predictions of the model -> running model on test data\nfrom sklearn.metrics import precision_recall_fscore_support as score\nfrom sklearn import metrics\n\ntestImages = imgList[len(trainImages):]\ntestImages = imagePath(testImages)\ntestLabels = lbls [len(trainLabels):]\n\ntestData = np.array(testImages)\ntestPrediction = model.predict(dataGenerator.flow(testData, batch_size = batch_size))\n\ngoodAcc = []\nbadAcc = []\nconfidence = []\nfor i in testPrediction:\n    confidence.append(max(i))\n    goodAcc.append(np.argmax(i))\n    badAcc.append(np.argmax(i))\n\nprecision, recall, fscore, support = score(testLabels, goodAcc, labels = np.unique(goodAcc))\n\nprint(\"Confusion Matrix\")\nprint(metrics.confusion_matrix(testLabels, goodAcc))\nprint(\"Classification Report\")\nprint(metrics.classification_report(testLabels, goodAcc, digits = 3))\n","metadata":{"execution":{"iopub.status.busy":"2021-11-14T09:38:42.460681Z","iopub.execute_input":"2021-11-14T09:38:42.460952Z","iopub.status.idle":"2021-11-14T09:40:44.015362Z","shell.execute_reply.started":"2021-11-14T09:38:42.460919Z","shell.execute_reply":"2021-11-14T09:40:44.014457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visulaise the best output of the model on the test data\nfor i in range(len(goodAcc)):\n    plt.axis('Off')\n    if (testLabels[i] == goodAcc[i]):\n        print(\"Perfect Label Match\")\n        title = ('True label: ' + str(testLabels[i]) + '_' + 'Predicted label: ' + str(goodAcc[i]) + '_' + 'confidence: ' + str(confidence[i]))\n        plt.title(title, fontsize = 10)\n        plt.imshow(testImages[i])\n        plt.show()\n        \n    elif(confidence[i] < 0.1):\n        print(\"Poor Confidence\")\n        title = ('True label: ' + str(testLabels[i]) + '_' + 'Predicted label: ' + str(goodAcc[i]) + '_' + 'confidence: ' + str(confidence[i]))\n        plt.title(title, fontsize = 10)\n        plt.imshow(testImages[i])\n        plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-14T09:40:44.017670Z","iopub.execute_input":"2021-11-14T09:40:44.017924Z","iopub.status.idle":"2021-11-14T09:40:57.200883Z","shell.execute_reply.started":"2021-11-14T09:40:44.017889Z","shell.execute_reply":"2021-11-14T09:40:57.199248Z"},"trusted":true},"execution_count":null,"outputs":[]}]}