{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**Google Landmark Recognition 2021 VGIS Mini Project**\nThis notebook contains the material of the third exercise for the Research in VGIS course.\nThe notebook will go through the tasks of the exercise.","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nimport random\nimport cv2\nimport tensorflow as tf\n\n#print(os.listdir('/kaggle/input/landmark-recognition-2021'))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-10-15T20:07:58.170575Z","iopub.execute_input":"2021-10-15T20:07:58.170919Z","iopub.status.idle":"2021-10-15T20:08:03.239625Z","shell.execute_reply.started":"2021-10-15T20:07:58.170840Z","shell.execute_reply":"2021-10-15T20:08:03.238849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = '/kaggle/input/landmark-recognition-2021'\nos.listdir(path)\n\ntrain_images = f'{path}/train'\ndf_train = pd.read_csv(f'{path}/train.csv')\ndf_train['path'] = df_train['id'].apply(lambda f: os.path.join('/kaggle/input/landmark-recognition-2021/train',f[0], f[1], f[2], f + '.jpg'))\n\ntest_images = f'{path}/test'\ndf_test = pd.read_csv(f'{path}/sample_submission.csv')\ndf_test['path'] = df_test['id'].apply(lambda f: os.path.join('/kaggle/input/landmark-recognition-2021/test',f[0], f[1], f[2], f + '.jpg'))\n\n# Defining the amount of classes and images in the training dataset.\nnr_classes = len(df_train[\"landmark_id\"].unique())\nnr_images = len(df_train)\n\n#print(\"Number of classes in training dataset: \", nr_classes)\n#print(\"Number of images in training dataset: \", nr_images)","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:08:03.241241Z","iopub.execute_input":"2021-10-15T20:08:03.241484Z","iopub.status.idle":"2021-10-15T20:08:10.182140Z","shell.execute_reply.started":"2021-10-15T20:08:03.241451Z","shell.execute_reply":"2021-10-15T20:08:10.181372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Histogram of data distribution, to show the amount of images in each class.\n# One class goes higher than the histogram top, which is due to the class containing 6272 images.\n#hist = plt.figure(figsize = (10, 10))\n#ax = plt.hist(df_train[\"landmark_id\"], bins = df_train[\"landmark_id\"].unique())\n#plt.ylim([0, 100])\n#plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:08:10.183232Z","iopub.execute_input":"2021-10-15T20:08:10.183478Z","iopub.status.idle":"2021-10-15T20:08:10.187556Z","shell.execute_reply.started":"2021-10-15T20:08:10.183447Z","shell.execute_reply":"2021-10-15T20:08:10.186659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Showing the number of classes containing 5 or less images in a class.\n# I am doing the same for classes contatining between 5 to 10 images.\n#classes = ax[0]\n#from0To5 = len(classes[classes <= 5])\n#from5To10 = len(classes[classes <= 10] - from0To5)\n#print(\"Number of classes with 0 to 5 images: \", from0To5)\n#print(\"Number of classes with 5 to 10 images: \", from5To10)","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:08:10.190242Z","iopub.execute_input":"2021-10-15T20:08:10.190808Z","iopub.status.idle":"2021-10-15T20:08:10.197381Z","shell.execute_reply.started":"2021-10-15T20:08:10.190775Z","shell.execute_reply":"2021-10-15T20:08:10.196749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Here can a overall representation of the data distribution be seen\nValueCounts = df_train['landmark_id'].value_counts()\nValueCounts.describe()","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:08:10.198528Z","iopub.execute_input":"2021-10-15T20:08:10.199412Z","iopub.status.idle":"2021-10-15T20:08:10.249372Z","shell.execute_reply.started":"2021-10-15T20:08:10.199373Z","shell.execute_reply":"2021-10-15T20:08:10.248666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Visualize 4 sample images from 4 random classes.\ndisplayImages = []\nfor i in range(0,4):\n    randomClass = df_train[df_train['landmark_id'] == ValueCounts.iloc[[np.random.randint(0, nr_classes)]].index[0]]\n    for j in range(0,4):\n        randomImages = randomClass.iloc[np.random.randint(0, len(randomClass))]\n        displayImages.append(randomImages)\n        \nplt.subplots(4, 4, figsize = (15, 10))\nfor i in range(len(displayImages)):\n    plt.subplot(4, 4, i + 1)\n    plt.axis('Off')\n    img = cv2.imread(displayImages[i][2])\n    plt.imshow(img)\n    plt.title(f'landmark id: {displayImages[i][1]} ', fontsize=8)","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:08:10.250458Z","iopub.execute_input":"2021-10-15T20:08:10.250745Z","iopub.status.idle":"2021-10-15T20:08:13.006501Z","shell.execute_reply.started":"2021-10-15T20:08:10.250709Z","shell.execute_reply":"2021-10-15T20:08:13.005738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Setting up hyperparameters and splitting training data into train and val\ndef imagePath(imgPath):\n    images = []\n    for imgFile in imgPath:\n        imgPic = cv2.imread(imgFile, 1)\n        images.append(cv2.resize(imgPic, (img_size, img_size)))\n    \n    return images\n\n# Hyperparameters\nepochs = 50\nbatch_size = 32\nimg_size = 128\ntrain_split = 0.7\nval_split = 0.2\nnrClasses = 120\n\n# Setting up dataset for training\nimgList = []\nlabels = []\ntemp_labels = []\n\ni = 0\nfor lbl in df_train['landmark_id'].unique():\n    if i == nrClasses:\n        break\n    if(len(df_train['path'][df_train['landmark_id'] == lbl].value_counts()) > 50 and # Try to change it to 25 to see if higher accuracy is achieved\n       len(df_train['path'][df_train['landmark_id'] == lbl].value_counts()) < 500): \n        for path in df_train['path'][df_train['landmark_id'] == lbl]: \n            imgList.append(path) \n            labels.append(lbl)\n            temp_labels.append(i)\n        i = i + 1\n\n# Random shuffle dataset, so it is no longer set up in classes\nshuff = list(zip(imgList, temp_labels))\nrandom.shuffle(shuff)\n\nimgList, lbls = zip(*shuff)\n\n# Preparing data to be split into train and val\nimgNr = round(len(imgList) * train_split)\n\ntrainImages = imgList[:imgNr]\nprint(\"Images being resized: \", len(trainImages))\ntrainData = imagePath(trainImages)\ntrainLabels = lbls[:imgNr]\n\nprint(\"Number of training images: \", len(trainData))\nprint(\"Number of training labels: \", len(trainLabels))\n\n# Setting images and labels to be split into x_train, y_tran, x_val and y_val\nxData = np.array(trainData) / 255\nyData = tf.keras.utils.to_categorical(trainLabels, num_classes = nrClasses)\n\nx_train, x_val, y_train, y_val = train_test_split(xData, yData, test_size = val_split, random_state = 101)\n\n# Setting up data generator\ndataGenerator = tf.keras.preprocessing.image.ImageDataGenerator(horizontal_flip = False, \n                                                                vertical_flip = False, \n                                                                rotation_range = 0, \n                                                                zoom_range = 0.2, \n                                                                width_shift_range = 0, \n                                                                height_shift_range = 0, \n                                                                shear_range = 0, \n                                                                fill_mode = \"nearest\")\n\nopt = tf.optimizers.Adam(learning_rate = 0.001)\nopt2 = tf.optimizers.Adam(learning_rate = 0.001, beta_1 = 0.9, beta_2 = 0.999, \nepsilon = 1e-08, decay = 0.0001)","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:08:13.007846Z","iopub.execute_input":"2021-10-15T20:08:13.008645Z","iopub.status.idle":"2021-10-15T20:10:31.929996Z","shell.execute_reply.started":"2021-10-15T20:08:13.008609Z","shell.execute_reply":"2021-10-15T20:10:31.929214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Creating the CNN**","metadata":{}},{"cell_type":"code","source":"# This a implementation of ResNet50. It also allows for experimentation with more layers by changing to RenNet101.\nResNet101 = tf.keras.applications.resnet.ResNet101(input_shape = (img_size, img_size, 3),\n                                                      include_top = False,\n                                                      weights = 'imagenet',\n                                                      pooling = 'avg')\n\n\ninputs = ResNet101.input\nflatten = tf.keras.layers.Flatten()(ResNet101.output)\ndropout1 = tf.keras.layers.Dropout(0.2)(flatten)\ndense1 = tf.keras.layers.Dense(units = 4096, activation = \"relu\")(dropout1)\ndropout2 = tf.keras.layers.Dropout(0.2)(dense1)\ndense2 = tf.keras.layers.Dense(units = 4096, activation = \"relu\")(dropout2)\ndropout3 = tf.keras.layers.Dropout(0.2)(dense2)\noutput = tf.keras.layers.Dense(units = nrClasses, activation = \"softmax\")(dropout3)\nmodel = tf.keras.Model(inputs = inputs, outputs = output)\n\nprint(model.summary())","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:10:31.931659Z","iopub.execute_input":"2021-10-15T20:10:31.931958Z","iopub.status.idle":"2021-10-15T20:10:44.777709Z","shell.execute_reply.started":"2021-10-15T20:10:31.931920Z","shell.execute_reply":"2021-10-15T20:10:44.777045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Compile the network\nmodel.compile(optimizer = opt2, loss = \"categorical_crossentropy\", metrics = ['accuracy'])\n\n#es_callback = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=3)\n\nhistory = model.fit(dataGenerator.flow(x_train, y_train, batch_size = batch_size), validation_data = (x_val, y_val), epochs = epochs)","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:10:44.778920Z","iopub.execute_input":"2021-10-15T20:10:44.779167Z","iopub.status.idle":"2021-10-15T20:38:16.042620Z","shell.execute_reply.started":"2021-10-15T20:10:44.779134Z","shell.execute_reply":"2021-10-15T20:38:16.041905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plotting the performance of the model\nplt.plot(history.history['accuracy'])\nplt.plot(history.history['val_accuracy'])\nplt.title('ResNet50 Accuracy')\nplt.ylabel('Accuracy')\nplt.xlabel('Epoch')\nplt.legend(['Train', 'Val'], loc = 'upper left')\nplt.show()\n\n# Plot of loss\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('ResNet50 Loss')\nplt.ylabel('Loss')\nplt.xlabel('Epoch')\nplt.legend(['Train', 'Val'], loc = 'upper left')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:38:16.045208Z","iopub.execute_input":"2021-10-15T20:38:16.045854Z","iopub.status.idle":"2021-10-15T20:38:16.552183Z","shell.execute_reply.started":"2021-10-15T20:38:16.045815Z","shell.execute_reply":"2021-10-15T20:38:16.551258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Predictions of the model -> running model on test data\nfrom sklearn.metrics import precision_recall_fscore_support as score\nfrom sklearn import metrics\n\ntestImages = imgList[len(trainImages):]\ntestImages = imagePath(testImages)\ntestLabels = lbls [len(trainLabels):]\n\ntestData = np.array(testImages) / 255\ntestPrediction = model.predict(dataGenerator.flow(testData, batch_size = batch_size))\n\ngoodAcc = []\nbadAcc = []\nconfidence = []\nfor i in testPrediction:\n    confidence.append(max(i))\n    goodAcc.append(np.argmax(i))\n    badAcc.append(np.argmax(i))\n\nprecision, recall, fscore, support = score(testLabels, goodAcc, labels = np.unique(goodAcc))\n\nprint(metrics.confusion_matrix(testLabels, goodAcc))\nprint(metrics.classification_report(testLabels, goodAcc, digits = 3))\n\nfor i in range(len(goodAcc)):\n    plt.axis('Off')\n    if (testLabels[i] == goodAcc[i]):\n        print(\"Perfect Label Match\")\n        title = ('True label: ' + str(testLabels[i]) + '_' + 'Predicted label: ' + str(goodAcc[i]) + '_' + 'confidence: ' + str(confidence[i]))\n        plt.title(title, fontsize = 10)\n        plt.imshow(testImages[i])\n        plt.show()\n        \n    elif(confidence[i] > 0.98):\n        print(\"High Confidence\")\n        title = ('True label: ' + str(testLabels[i]) + '_' + 'Predicted label: ' + str(goodAcc[i]) + '_' + 'confidence: ' + str(confidence[i]))\n        plt.title(title, fontsize = 10)\n        plt.imshow(testImages[i])\n        plt.show()\n    \n    elif(confidence[i] < 0.05):\n        print(\"Poor Confidence\")\n        title = ('True label: ' + str(testLabels[i]) + '_' + 'Predicted label: ' + str(goodAcc[i]) + '_' + 'confidence: ' + str(confidence[i]))\n        plt.title(title, fontsize = 10)\n        plt.imshow(testImages[i])\n        plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:38:16.556912Z","iopub.execute_input":"2021-10-15T20:38:16.559153Z","iopub.status.idle":"2021-10-15T20:43:14.096325Z","shell.execute_reply.started":"2021-10-15T20:38:16.559083Z","shell.execute_reply":"2021-10-15T20:43:14.095393Z"},"trusted":true},"execution_count":null,"outputs":[]}]}