{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nfrom keras.models import Sequential\nfrom keras.preprocessing.image import ImageDataGenerator\nimport keras.layers as L\nfrom keras import regularizers, optimizers\nfrom collections import Counter\nimport keras\nfrom keras import Model\nimport tensorflow as tf\nfrom tensorflow.keras.applications.xception import Xception\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nfrom keras.models import load_model","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:25:44.998855Z","iopub.execute_input":"2022-04-09T01:25:44.999253Z","iopub.status.idle":"2022-04-09T01:25:51.332034Z","shell.execute_reply.started":"2022-04-09T01:25:44.999176Z","shell.execute_reply":"2022-04-09T01:25:51.331048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the labelling csv file \nlabel = pd.read_csv(\"../input/landmark-recognition-2020/train.csv\")\nlabel.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:26:18.881114Z","iopub.execute_input":"2022-04-09T01:26:18.881443Z","iopub.status.idle":"2022-04-09T01:26:20.049096Z","shell.execute_reply.started":"2022-04-09T01:26:18.881412Z","shell.execute_reply":"2022-04-09T01:26:20.047970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"The total number of pictures in the dataset:\", len(label))\nprint(\"The total number of landmarks in the dataset:\", label.landmark_id.nunique())","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:26:20.051139Z","iopub.execute_input":"2022-04-09T01:26:20.052223Z","iopub.status.idle":"2022-04-09T01:26:20.083851Z","shell.execute_reply.started":"2022-04-09T01:26:20.052172Z","shell.execute_reply":"2022-04-09T01:26:20.082078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"counts = label['landmark_id'].value_counts().sort_values(ascending=False)\ncounts[0:5]","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:26:20.559462Z","iopub.execute_input":"2022-04-09T01:26:20.559762Z","iopub.status.idle":"2022-04-09T01:26:20.610047Z","shell.execute_reply.started":"2022-04-09T01:26:20.559731Z","shell.execute_reply":"2022-04-09T01:26:20.609124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Lets see the classes with least amount of images:\nbelow = counts[counts < 10].index.shape[0]\nbelow","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:26:21.027424Z","iopub.execute_input":"2022-04-09T01:26:21.028496Z","iopub.status.idle":"2022-04-09T01:26:21.038229Z","shell.execute_reply.started":"2022-04-09T01:26:21.028443Z","shell.execute_reply":"2022-04-09T01:26:21.037032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We have 41637 classes with less than 10 images. Inclusding these images in the model may create noise\n# as the number of these images are very less. therefore removing these images from our original df\n\nselected_classes = counts[counts >= 20].index\nlabel = label.loc[label.landmark_id.isin(selected_classes)]\nprint(label.shape)","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:26:21.270526Z","iopub.execute_input":"2022-04-09T01:26:21.270836Z","iopub.status.idle":"2022-04-09T01:26:21.356451Z","shell.execute_reply.started":"2022-04-09T01:26:21.270802Z","shell.execute_reply":"2022-04-09T01:26:21.354847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Approach 1:\n# For this project, lets limit our scope to only 1000 images each of 5 top occuring classes.\n# from these 5000 images, we will split training, validation in the ratio 75:25","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:26:21.712499Z","iopub.execute_input":"2022-04-09T01:26:21.713242Z","iopub.status.idle":"2022-04-09T01:26:21.717943Z","shell.execute_reply.started":"2022-04-09T01:26:21.713204Z","shell.execute_reply":"2022-04-09T01:26:21.716822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label['landmark_id'] = label.landmark_id.astype(int)","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:26:21.904776Z","iopub.execute_input":"2022-04-09T01:26:21.905195Z","iopub.status.idle":"2022-04-09T01:26:21.918372Z","shell.execute_reply.started":"2022-04-09T01:26:21.905160Z","shell.execute_reply":"2022-04-09T01:26:21.916862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#selected_df = label.loc[label['landmark_id'].isin([int(x) for x in list(counts[0:5].index)])]\n\n# Now subset only 1000 images from each of the 5 categories:\n#selected_df = selected_df.groupby('landmark_id').apply(lambda x: x.sample(n=1000, random_state =13)).reset_index(drop = True)\n#Counter(selected_df['landmark_id'])","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:26:22.359677Z","iopub.execute_input":"2022-04-09T01:26:22.359990Z","iopub.status.idle":"2022-04-09T01:26:22.364862Z","shell.execute_reply.started":"2022-04-09T01:26:22.359957Z","shell.execute_reply":"2022-04-09T01:26:22.363491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Approach 2:\n# Lets select specified percentage of images from each class after removing the noise:\nselected_df = label.groupby(\"landmark_id\", group_keys=False).apply(lambda x: x.sample(frac = 0.3, random_state = 123))\nlen(selected_df)","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:26:22.587455Z","iopub.execute_input":"2022-04-09T01:26:22.588306Z","iopub.status.idle":"2022-04-09T01:26:43.361049Z","shell.execute_reply.started":"2022-04-09T01:26:22.588268Z","shell.execute_reply":"2022-04-09T01:26:43.359973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# calculate the number of classes from the sratified dataset above\ncount = Counter(selected_df.landmark_id.values)\nnum_classes = len(count)\nnum_classes","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:26:43.363618Z","iopub.execute_input":"2022-04-09T01:26:43.363992Z","iopub.status.idle":"2022-04-09T01:26:43.466827Z","shell.execute_reply.started":"2022-04-09T01:26:43.363933Z","shell.execute_reply":"2022-04-09T01:26:43.465399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Image generator: Lets load these images using image data generator and perform train test split","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:26:43.469653Z","iopub.execute_input":"2022-04-09T01:26:43.470530Z","iopub.status.idle":"2022-04-09T01:26:43.479723Z","shell.execute_reply.started":"2022-04-09T01:26:43.470280Z","shell.execute_reply":"2022-04-09T01:26:43.478286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selected_df['landmark_id'] = selected_df.landmark_id.astype(str)\nselected_df['id'] = selected_df.id.str[0] + '/' + selected_df.id.str[1] + '/' + selected_df.id.str[2]+'/' + selected_df.id + '.jpg'","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:26:43.484160Z","iopub.execute_input":"2022-04-09T01:26:43.484769Z","iopub.status.idle":"2022-04-09T01:26:45.091353Z","shell.execute_reply.started":"2022-04-09T01:26:43.484722Z","shell.execute_reply":"2022-04-09T01:26:45.090368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_split = 0.25\nbatch_size = 32\nimg_width = img_height = 192","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:26:45.093161Z","iopub.execute_input":"2022-04-09T01:26:45.093449Z","iopub.status.idle":"2022-04-09T01:26:45.098960Z","shell.execute_reply.started":"2022-04-09T01:26:45.093406Z","shell.execute_reply":"2022-04-09T01:26:45.098010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#  Use flow_from_dataframe to generate train data set, test, validation data set\n\ndatagen=ImageDataGenerator(validation_split=val_split,rescale=1. / 255)\n\ntrain_generator=datagen.flow_from_dataframe(dataframe=selected_df,\n                                            directory=\"/kaggle/input/landmark-recognition-2020/train/\",\n                                            x_col=\"id\",\n                                            y_col=\"landmark_id\",\n                                            subset=\"training\",\n                                            batch_size=32,\n                                            seed=42,\n                                            shuffle=True,\n                                            class_mode=\"categorical\",\n                                            target_size=(img_height, img_width))\n\nvalid_generator=datagen.flow_from_dataframe(\ndataframe=selected_df,\ndirectory=\"/kaggle/input/landmark-recognition-2020/train/\",\nx_col=\"id\",\ny_col=\"landmark_id\",\nsubset=\"validation\",\nbatch_size=32,\nseed=42,\nshuffle=True,\nclass_mode=\"categorical\",\ntarget_size=(img_height, img_width))","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:26:45.101160Z","iopub.execute_input":"2022-04-09T01:26:45.102007Z","iopub.status.idle":"2022-04-09T01:37:46.066007Z","shell.execute_reply.started":"2022-04-09T01:26:45.101943Z","shell.execute_reply":"2022-04-09T01:37:46.064879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def my_model(input_shape, num_classes, dropout, learning_rate = 0.0002):\n\n    base_model = Xception(input_shape=input_shape,weights='imagenet', include_top=False)\n    #base_model.load_weights(\"https://storage.googleapis.com/tensorflow/keras-applications/xception/xception_weights_tf_dim_ordering_tf_kernels_notop.h5\")\n\n\n    x = base_model.output\n    x = L.Dropout(dropout)(x)\n    x = L.SeparableConv2D(256, kernel_size=(3, 3), activation='relu',kernel_initializer = tf.keras.initializers.he_uniform(seed=1))(x)\n    x = L.BatchNormalization()(x)\n    x = L.SeparableConv2D(128, kernel_size=(3, 3), activation='relu',kernel_initializer = tf.keras.initializers.he_uniform(seed=3))(x)\n    x = L.BatchNormalization()(x)\n    x = L.SeparableConv2D(num_classes,kernel_size = (1,1), depth_multiplier=1, activation = 'relu',\n                kernel_initializer = tf.keras.initializers.he_uniform(seed=0),\n                kernel_regularizer=tf.keras.regularizers.l1_l2(l1=0.1, l2=0.01)\n                )(x)\n    x = L.GlobalMaxPooling2D()(x)\n    x = L.BatchNormalization()(x)\n    x = L.Flatten()(x)\n\n    pred = L.Dense(num_classes, activation = 'softmax')(x)\n\n    for layer in base_model.layers:\n        layer.trainable = False\n\n    model = Model(inputs = base_model.input,outputs = pred,name='model')\n\n    model.compile(loss='categorical_crossentropy',experimental_steps_per_execution=8, optimizer = tf.keras.optimizers.Adagrad(learning_rate=0.01), metrics='categorical_accuracy')\n\n    model.summary()\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:37:46.067736Z","iopub.execute_input":"2022-04-09T01:37:46.068655Z","iopub.status.idle":"2022-04-09T01:37:46.084415Z","shell.execute_reply.started":"2022-04-09T01:37:46.068598Z","shell.execute_reply":"2022-04-09T01:37:46.083137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = my_model(input_shape = (img_width, img_height, 3), num_classes = num_classes, dropout = 0.5)","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:37:46.088115Z","iopub.execute_input":"2022-04-09T01:37:46.088939Z","iopub.status.idle":"2022-04-09T01:37:51.319883Z","shell.execute_reply.started":"2022-04-09T01:37:46.088900Z","shell.execute_reply":"2022-04-09T01:37:51.318776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = 30 # Defining epochs for the model\ntrain_samples  = int(len(selected_df)*(1-val_split))//batch_size\nvalidation_samples  = int(len(selected_df)*val_split)//batch_size\n\nprint(train_samples)\nprint(validation_samples)","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:37:51.321444Z","iopub.execute_input":"2022-04-09T01:37:51.322024Z","iopub.status.idle":"2022-04-09T01:37:51.331641Z","shell.execute_reply.started":"2022-04-09T01:37:51.321978Z","shell.execute_reply":"2022-04-09T01:37:51.330555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define call backs and metrics:\n\ncheckpointer = ModelCheckpoint('basic_cnn.h5', monitor='val_loss', verbose=1, save_best_only=True)\n\n# Early stopping\nearly_stopping = EarlyStopping(monitor='val_loss', verbose=1, patience=10)\n\nMETRICS = [\n    keras.metrics.Accuracy(name= \"accuracy\"),\n    keras.metrics.Precision(name = \"precision\"),\n    keras.metrics.Recall(name = 'recall'),\n    keras.metrics.AUC(name = 'auc'),\n]","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:37:51.335722Z","iopub.execute_input":"2022-04-09T01:37:51.336139Z","iopub.status.idle":"2022-04-09T01:37:51.366627Z","shell.execute_reply.started":"2022-04-09T01:37:51.336093Z","shell.execute_reply":"2022-04-09T01:37:51.365435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit_generator(\n        train_generator,\n        steps_per_epoch=train_samples // batch_size,\n        epochs=epochs,\n        callbacks=[checkpointer, early_stopping],\n        use_multiprocessing=True,\n        verbose=1,\n        validation_data=valid_generator,\n        validation_steps=validation_samples // batch_size,)\n\nmodel.save(\"basic_cnn.h5\")","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:37:51.368177Z","iopub.execute_input":"2022-04-09T01:37:51.368591Z","iopub.status.idle":"2022-04-09T02:11:29.355926Z","shell.execute_reply.started":"2022-04-09T01:37:51.368543Z","shell.execute_reply":"2022-04-09T02:11:29.354335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Fine Tuning the Xception model","metadata":{}},{"cell_type":"code","source":"for layer in model.layers:\n    layer.trainable = True\n    \nmodel.compile(loss='categorical_crossentropy', experimental_steps_per_execution=8, optimizer = tf.keras.optimizers.Adam(learning_rate=0.0001), metrics='categorical_accuracy')\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-04-09T02:11:29.364476Z","iopub.execute_input":"2022-04-09T02:11:29.367619Z","iopub.status.idle":"2022-04-09T02:11:29.528869Z","shell.execute_reply.started":"2022-04-09T02:11:29.367557Z","shell.execute_reply":"2022-04-09T02:11:29.527768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit_generator(\n        train_generator,\n        steps_per_epoch=train_samples // batch_size,\n        epochs=epochs,\n        callbacks=[checkpointer, early_stopping],\n        use_multiprocessing=True,\n        verbose=1,\n        validation_data=valid_generator,\n        validation_steps=validation_samples // batch_size)","metadata":{"execution":{"iopub.status.busy":"2022-04-09T02:11:29.530813Z","iopub.execute_input":"2022-04-09T02:11:29.531175Z","iopub.status.idle":"2022-04-09T02:47:26.503010Z","shell.execute_reply.started":"2022-04-09T02:11:29.531129Z","shell.execute_reply":"2022-04-09T02:47:26.501587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluating the model","metadata":{}},{"cell_type":"code","source":"scores = model.evaluate_generator(valid_generator, validation_samples, use_multiprocessing=True, verbose=1)\nscores\n#print(\"%s%s: %.2f%%\" % (\"evaluate_generator \",model.metrics_names[1], scores[1]*100))","metadata":{"execution":{"iopub.status.busy":"2022-04-09T02:47:26.514192Z","iopub.execute_input":"2022-04-09T02:47:26.516947Z","iopub.status.idle":"2022-04-09T03:07:04.458907Z","shell.execute_reply.started":"2022-04-09T02:47:26.516865Z","shell.execute_reply":"2022-04-09T03:07:04.457340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv(\"/kaggle/input/landmark-recognition-2020/sample_submission.csv\")\nsubmission[\"id\"] = submission.id.str[0]+\"/\"+submission.id.str[1]+\"/\"+submission.id.str[2]+\"/\"+submission.id+\".jpg\"\nbest_model = load_model(\"basic_cnn.h5\")\n\ntest_gen = ImageDataGenerator().flow_from_dataframe(\n    submission,\n    directory=\"/kaggle/input/landmark-recognition-2020/test/\",\n    x_col=\"id\",\n    y_col=None,\n    weight_col=None,\n    target_size=(img_width, img_height),\n    color_mode=\"rgb\",\n    classes=None,\n    class_mode=None,\n    batch_size=1,\n    shuffle=True,\n    subset=None,\n    interpolation=\"nearest\",\n    validate_filenames=False)","metadata":{"execution":{"iopub.status.busy":"2022-04-09T03:07:04.467409Z","iopub.execute_input":"2022-04-09T03:07:04.471220Z","iopub.status.idle":"2022-04-09T03:07:28.356245Z","shell.execute_reply.started":"2022-04-09T03:07:04.471146Z","shell.execute_reply":"2022-04-09T03:07:28.354971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred_one_hot = best_model.predict_generator(test_gen, verbose=1, steps=len(submission))","metadata":{"execution":{"iopub.status.busy":"2022-04-09T03:07:28.357972Z","iopub.execute_input":"2022-04-09T03:07:28.359293Z","iopub.status.idle":"2022-04-09T03:10:52.613685Z","shell.execute_reply.started":"2022-04-09T03:07:28.359228Z","shell.execute_reply":"2022-04-09T03:10:52.612637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = np.argmax(y_pred_one_hot, axis=-1)\ny_prob = np.max(y_pred_one_hot, axis=-1)\nprint(y_pred.shape, y_prob.shape)","metadata":{"execution":{"iopub.status.busy":"2022-04-09T03:10:52.615861Z","iopub.execute_input":"2022-04-09T03:10:52.616213Z","iopub.status.idle":"2022-04-09T03:10:52.897878Z","shell.execute_reply.started":"2022-04-09T03:10:52.616166Z","shell.execute_reply":"2022-04-09T03:10:52.896772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_uniq = np.unique(selected_df.landmark_id.values)\ny_pred = [y_uniq[Y] for Y in y_pred]","metadata":{"execution":{"iopub.status.busy":"2022-04-09T03:10:52.899887Z","iopub.execute_input":"2022-04-09T03:10:52.900437Z","iopub.status.idle":"2022-04-09T03:10:53.378706Z","shell.execute_reply.started":"2022-04-09T03:10:52.900391Z","shell.execute_reply":"2022-04-09T03:10:53.377578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Extracting best and worst classficiations from predictions\ntemp_sub = submission\n\nfor i in range(len(temp_sub)):\n    temp_sub.loc[i, \"landmarks\"] = str(y_pred[i])\n\ntemp_sub.insert(2, \"pred\", y_prob)    \n\nworst_preds = temp_sub.sort_values(by=['pred'])\nbest_preds = temp_sub.sort_values(by=['pred'], ascending=False)","metadata":{"execution":{"iopub.status.busy":"2022-04-09T03:10:53.380361Z","iopub.execute_input":"2022-04-09T03:10:53.380672Z","iopub.status.idle":"2022-04-09T03:10:54.334933Z","shell.execute_reply.started":"2022-04-09T03:10:53.380630Z","shell.execute_reply":"2022-04-09T03:10:54.333907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"worst_preds[0:5]","metadata":{"execution":{"iopub.status.busy":"2022-04-09T03:10:54.336457Z","iopub.execute_input":"2022-04-09T03:10:54.337521Z","iopub.status.idle":"2022-04-09T03:10:54.356340Z","shell.execute_reply.started":"2022-04-09T03:10:54.337466Z","shell.execute_reply":"2022-04-09T03:10:54.355154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(best_preds[best_preds['pred'] > 0.8])","metadata":{"execution":{"iopub.status.busy":"2022-04-09T03:10:54.358315Z","iopub.execute_input":"2022-04-09T03:10:54.359051Z","iopub.status.idle":"2022-04-09T03:10:54.370921Z","shell.execute_reply.started":"2022-04-09T03:10:54.358998Z","shell.execute_reply":"2022-04-09T03:10:54.369534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}