{"cells":[{"metadata":{},"cell_type":"markdown","source":"**Importing all the needed libraries[](http://)**"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport pandas as pd\nimport plotly.express as px\nimport numpy as np\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport PIL\nfrom PIL import Image, ImageDraw\nimport glob\nimport cv2\nimport random\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras import layers\nfrom keras import Sequential\nfrom keras import models\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.layers import *\nfrom keras.optimizers import RMSprop\nfrom keras.optimizers import Adagrad\nfrom keras.callbacks import EarlyStopping, ReduceLROnPlateau\nfrom keras.applications import VGG19\nfrom keras.models import load_model\nfrom keras.callbacks import ModelCheckpoint\nimport tensorflow as tf\nimport seaborn as sns\nfrom sklearn import preprocessing\nfrom collections import Counter\nfrom sklearn.utils import class_weight\nfrom sklearn.preprocessing import LabelEncoder\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**How many images does the dataset consist of?**"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train_images_count = sum([len(files) for r, d, files in os.walk('../input/landmark-recognition-2020/train')])\nprint('The number of train images is :', train_images_count)\ntest_images_count = sum([len(files) for r, d, files in os.walk('../input/landmark-recognition-2020/test')])\nprint('The number of test images is :', test_images_count)\nprint('The total number of images is :', train_images_count+test_images_count)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Loading the data and appling some needed adjustments**"},{"metadata":{"trusted":true},"cell_type":"code","source":"Base_path = '../input/landmark-recognition-2020/'\nTrain_DIR = f'{Base_path}/train'\nTest_DIR = f'{Base_path}/test'\ntrain = pd.read_csv(f'{Base_path}/train.csv')\nsubmission = pd.read_csv(f'{Base_path}/sample_submission.csv')\nprint('Reading data completed')\nmy_train_data = train\nmy_test_data = submission\n#We will use something called label encoding/decoding.\n#The way this work is assigning each class a known label that can be used later,during the prediction, to know which class is being predicted. \\n we will use the number of classes for indexing\nprint(\"This is how the raw data looks like.\\n\", my_train_data)\n#Encoding\n# le = preprocessing.LabelEncoder()\n# le.fit(my_train_data.landmark_id.values)\n# new_df = le.transform(my_train_data[\"landmark_id\"])\n# my_train_data.landmark_id = new_df\nprint(\"This is how the data looks like after the encoding.\\n\", my_train_data)\n#adding a filename column which will contain the full path to the sample, which will later be used to access the data.\nmy_train_data[\"filename\"] = my_train_data.id.str[0]+\"/\"+my_train_data.id.str[1]+\"/\"+my_train_data.id.str[2]+\"/\"+my_train_data.id+\".jpg\"\nmy_test_data[\"filename\"] = my_test_data.id.str[0]+\"/\"+my_test_data.id.str[1]+\"/\"+my_test_data.id.str[2]+\"/\"+my_test_data.id+\".jpg\"\n#adding a \"label\" column which is basically the same as \"landmark_id\" but as string. This will be needed later for the data generator.\nmy_train_data[\"label\"] = my_train_data.landmark_id.astype(str)\nprint(\"This is how the data looks like after the encoding and adding the needed columns. \\n\",my_train_data)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**How many classes? How many images per class?**"},{"metadata":{"trusted":true},"cell_type":"code","source":"landmark_count=pd.value_counts(my_train_data[\"landmark_id\"])\nlandmark_count=landmark_count.reset_index()\nlandmark_count.rename(columns={\"index\":'landmark_ids','landmark_id':'count'},inplace=True)\nprint(landmark_count)\n# sample = landmark_count[0:50]\n# sample.rename(columns={\"index\":'landmark_ids','landmark_id':'count'},inplace=True)\n# sample.sort_values(by=['count'],ascending=False,inplace=True)\n# sample['landmark_ids']=sample['landmark_ids'].map(str)\n# sample.info()\n# print(sample)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Show a histogram of the number of instances per class**"},{"metadata":{"trusted":true},"cell_type":"code","source":"number_of_classes = len(my_train_data['landmark_id'].unique())\nprint('Number of unique classes in training images:',number_of_classes)\nnb_images_pr_class= pd.DataFrame(my_train_data.landmark_id.value_counts())\nnb_images_pr_class.reset_index(inplace=True)\nnb_images_pr_class.columns = ['landmark_id','count']\nprint(nb_images_pr_class)\n                \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig=plt.figure(figsize=(18, 3))\nn = plt.hist(my_train_data[\"landmark_id\"],bins=my_train_data[\"landmark_id\"].unique())\nplt.title(\"Distribution of labels\")\nplt.xlabel(\"Landmark_id\")\nplt.ylabel(\"Number of images\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"How many classes have less than 5 training samples? And between 5 and 10 training samples?"},{"metadata":{"trusted":true},"cell_type":"code","source":"less_than_five = 0\nbetween_five_and_ten = 0\nfor x in n[0]:\n    if(x<5):\n        less_than_five+=1\n    elif(x<10):\n        between_five_and_ten+=1\n    \nprint('Number of classes that have less than 5 training samples :',less_than_five)\nprint('Number of classes that have between 5 and 10 training samples :',between_five_and_ten)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Show 4 sample images from 4 random classes**"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_list = glob.glob('../input/landmark-recognition-2020/train/*/*/*/*')\nplt.rcParams[\"axes.grid\"] = False\nf, axarr = plt.subplots(2, 2, figsize=(10, 8))\n\ncurr_row = 0\nfor i in range(4):\n    example = cv2.imread(train_list[random.randint(0,len(train_list)-1)])\n    example = example[:,:,::-1]\n    \n    col = i%2\n    axarr[col, curr_row].imshow(example)\n    if col == 1:\n        curr_row += 1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Consider if/how the data distribution will affect the training of a classifier.**"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize = (10, 8))\nplt.title('Landmark ID Distribuition')\nsns.distplot(my_train_data['landmark_id'])\nplt.show()\n\nprint(\"The data distribution will affect the training process negatively, since some classes have a very large number of samples when compared with other classes. \\n For example the largest class contains 6272 images where there are 4749 classes which contain only 2 images, meaning that the largest class will have higher impact \\n when trying to do some predictions after training the model. \\n in other words, when trying to predict a sample from the small classes, 99% of the times, the classifier will predict it as it belongs to the large class, which is refered to as generalization. \") ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Since the dataset is huge, we will try to take only the 100 largest classes and try to classifiy them.**"},{"metadata":{"trusted":true},"cell_type":"code","source":"c = my_train_data.landmark_id.values\ncount = Counter(c).most_common(100)\nprint(len(count), count[-1])\n# only keep 100 classes\nkeep_labels = [i[0] for i in count]\ntrain_keep = my_train_data[my_train_data.landmark_id.isin(keep_labels)]\nprint(train_keep)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**The distribution of the new dataset**"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize = (10, 8))\nplt.title('Landmark ID Distribuition')\nsns.distplot(train_keep['landmark_id'])\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Training the model**"},{"metadata":{},"cell_type":"markdown","source":"1. The first step is to preprocess the data"},{"metadata":{},"cell_type":"markdown","source":"Some parameters which will be used during the training"},{"metadata":{"trusted":true},"cell_type":"code","source":"val_rate = 0.2# The percentage of the validation data\nepochs = 5 # The maximum number of epochs\nbatch_size = 10 # The batch size\n#opt = RMSprop(learning_rate=0.01, momentum = 0.9) # The used optimizer  \n#loss_function = 'categorical_crossentropy' # The loss function\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#First we start by creating the generator object, which will work as the container of our data.\n#This generator object will be split into two, validation and training, where the size of the validating will be specificed using the \"validation_split\" parameter.\n#and the rest will belong to the training.\ngen = ImageDataGenerator(validation_split=val_rate, rescale=1.0/255.0)\n\ntrain_gen = gen.flow_from_dataframe(\n    my_train_data,\n    directory=\"/kaggle/input/landmark-recognition-2020/train/\",\n    x_col=\"filename\",\n    y_col=\"label\", # The argument to this parameter has to be string, and that is why we created the \"label\" column at the beginning.\n    target_size=(256, 256),# Since the images in the dataset have different sizes, they will get resized into a unified size-256,256 to each color channel-\n    color_mode=\"rgb\",\n    class_mode=\"categorical\",#We have to use the \"categorical\" argument since we multiple classes\n    batch_size=batch_size,\n    shuffle=True, # This parameter will shuffle the data while being passed to the model\n    subset=\"training\",# The name of the subset\n    interpolation=\"nearest\", # This parameter is used to interpolate the pixel values when images get scaled to the target size.\n    validate_filenames=False)\n    \nval_gen = gen.flow_from_dataframe(\n    my_train_data,\n    directory=\"/kaggle/input/landmark-recognition-2020/train/\",\n    x_col=\"filename\",\n    y_col=\"label\",\n    target_size=(256, 256),\n    color_mode=\"rgb\",\n    class_mode=\"categorical\",\n    batch_size=batch_size,\n    shuffle=True,\n    subset=\"validation\",\n    interpolation=\"nearest\",\n    validate_filenames=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"samples = 20000\n\ndata = my_train_data.loc[:samples,:]\nclasses = len(data['landmark_id'].unique())\n\nlencoder = LabelEncoder()\nlencoder.fit(data[\"landmark_id\"])\n\nprint(classes)\nmodel = Sequential()\nmodel.add(Input(shape=(256,256,3)))\nmodel.add(BatchNormalization())\nmodel.add(Conv2D(64, kernel_size = (3,3), padding = \"same\"))\nmodel.add(Conv2D(64, kernel_size = (3,3), padding = \"same\"))\nmodel.add(MaxPooling2D())\nmodel.add(Conv2D(128, kernel_size = (3,3), padding = \"same\"))\nmodel.add(Conv2D(128, kernel_size = (3,3), padding = \"same\"))\nmodel.add(MaxPooling2D())\nmodel.add(Conv2D(256, kernel_size = (3,3), padding = \"same\"))\nmodel.add(Conv2D(256, kernel_size = (3,3), padding = \"same\"))\nmodel.add(Conv2D(256, kernel_size = (3,3), padding = \"same\"))\nmodel.add(Conv2D(256, kernel_size = (3,3), padding = \"same\"))\nmodel.add(MaxPooling2D())\nmodel.add(Conv2D(256, kernel_size = (3,3), padding = \"same\"))\nmodel.add(Conv2D(256, kernel_size = (3,3), padding = \"same\"))\nmodel.add(Conv2D(256, kernel_size = (3,3), padding = \"same\"))\nmodel.add(Conv2D(256, kernel_size = (3,3), padding = \"same\"))\nmodel.add(MaxPooling2D())\nmodel.add(Conv2D(256, kernel_size = (3,3), padding = \"same\"))\nmodel.add(Conv2D(256, kernel_size = (3,3), padding = \"same\"))\nmodel.add(Conv2D(256, kernel_size = (3,3), padding = \"same\"))\nmodel.add(Conv2D(256, kernel_size = (3,3), padding = \"same\"))\nmodel.add(MaxPooling2D())\nmodel.add(Conv2D(256, kernel_size = (3,3), padding = \"same\"))\nmodel.add(Conv2D(256, kernel_size = (3,3), padding = \"same\"))\nmodel.add(Conv2D(256, kernel_size = (3,3), padding = \"same\"))\nmodel.add(Conv2D(256, kernel_size = (3,3), padding = \"same\"))\nmodel.add(MaxPooling2D())\nmodel.add(Flatten())\nmodel.add(Dense(4096, activation = \"relu\"))\nmodel.add(Dense(4096, activation = \"relu\"))\nmodel.add(Dense(classes, activation=\"softmax\"))\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We talked before about the problem with having imbalanced data. One of the approaches to solve this problem, is by computing the class weights accordingely then pass them to the network when training. This is explained more in the report."},{"metadata":{"trusted":true},"cell_type":"code","source":"class_weights = class_weight.compute_class_weight('balanced',\n                                                 np.unique(data.landmark_id),\n                                                 data.landmark_id)\nclass_weights = dict(enumerate(class_weights))\nclass_weights\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"opt = Adagrad(learning_rate = 0.001, initial_accumulator_value=0.1, epsilon=1e-07)\nmodel.compile(optimizer=opt,\n             loss=\"categorical_crossentropy\",\n             metrics=[\"accuracy\"])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The final part is to train the model."},{"metadata":{"trusted":true},"cell_type":"code","source":"train_steps = int(len(data)*(1-val_rate))//batch_size\nval_steps = int(len(data)*val_rate)//batch_size\n\n#model_checkpoint = ModelCheckpoint(\"best_model.h5\", save_best_only=True, verbose=1)\nhistory = model.fit(train_gen, steps_per_epoch = train_steps, epochs = epochs,\n                    validation_data = val_gen, validation_steps=val_steps, \n                    class_weight=class_weights\n                   )\n\nmodel.save(\"weightedClasses.h5\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nprint(history.history.keys())\nplt.plot(history.history['categorical_accuracy'])\nplt.plot(history.history['val_categorical_accuracy'])\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()\n# summarize history for loss\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_model = load_model(\"best_model.h5\")\ntest_gen = ImageDataGenerator().flow_from_dataframe(\n    my_test_data,\n    directory=\"/kaggle/input/landmark-recognition-2020/test/\",\n    x_col=\"filename\",\n    y_col=None,\n    weight_col=None,\n    target_size=(256, 256),\n    color_mode=\"rgb\",\n    classes=None,\n    class_mode=None,\n    batch_size=1,\n    shuffle=True,\n    subset=None,\n    interpolation=\"nearest\",\n    validate_filenames=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions_list = best_model.predict_generator(test_gen, verbose=1, steps=len(my_test_data))\ny_pred = np.argmax(predictions_list, axis=-1)\ny_prob = np.max(predictions_list, axis=-1)\nprint(y_pred.shape, y_prob.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_uniq = np.unique(train_keep.landmark_id.values)\n\ny_pred = [y_uniq[Y] for Y in y_pred]\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(y_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(len(my_test_data)):\n    my_test_data.loc[i, \"landmarks\"] = str(y_pred[i])+\" \"+str(y_prob[i])\nmy_test_data = my_test_data.drop(columns=\"filename\")\nmy_test_data.to_csv(\"submission.csv\", index=False)\nmy_test_data","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}