{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport cv2\nimport shutil\nimport numpy as np\nimport pandas as pd\nfrom scipy import spatial\nfrom tqdm import tqdm\nimport tensorflow as tf\nfrom tensorflow.keras import layers\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import train_test_split\npd.options.mode.chained_assignment = None","metadata":{"execution":{"iopub.status.busy":"2022-05-29T14:23:18.790009Z","iopub.execute_input":"2022-05-29T14:23:18.790764Z","iopub.status.idle":"2022-05-29T14:23:20.510851Z","shell.execute_reply.started":"2022-05-29T14:23:18.790726Z","shell.execute_reply":"2022-05-29T14:23:20.50963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Directories and file paths\nTRAIN_DIR = '../input/landmark-recognition-2020/train'\nTRAIN_CSV = '../input/landmark-recognition-2020/train.csv'\ntrain_df = pd.read_csv(TRAIN_CSV)\n\nTRAIN_PATHS = [os.path.join(TRAIN_DIR, f'{img[0]}/{img[1]}/{img[2]}/{img}.jpg') for img in train_df['id']]\ntrain_df['path'] = TRAIN_PATHS\n\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2022-05-29T14:22:18.404865Z","iopub.execute_input":"2022-05-29T14:22:18.405358Z","iopub.status.idle":"2022-05-29T14:22:24.365566Z","shell.execute_reply.started":"2022-05-29T14:22:18.405323Z","shell.execute_reply":"2022-05-29T14:22:24.364371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df_grouped = pd.DataFrame(train_df.landmark_id.value_counts())\ntrain_df_grouped.reset_index(inplace=True)\ntrain_df_grouped.columns = ['landmark_id','count']\n\n# Selected landmarks based on inclass frequency\nselected_landmarks = train_df_grouped[(train_df_grouped['count'] <= 155) & (train_df_grouped['count'] >= 150)]\n\ntrain_df_sub = train_df[train_df['landmark_id'].isin(selected_landmarks['landmark_id'])]\nnew_id = []\ncurrent_id = 0\nprevious_id = int(train_df_sub.head(1)['landmark_id'])\nfor landmark_id in train_df_sub['landmark_id']:\n    if landmark_id == previous_id:\n        new_id.append(current_id)\n    else:\n        current_id += 1\n        new_id.append(current_id)\n        previous_id = landmark_id\n\ntrain_df_sub['new_id'] = new_id\n\nNUM_CLASSES = train_df_sub['landmark_id'].nunique()\n\nprint(f\"Unique classes found: {NUM_CLASSES}\")\ntrain_df_sub","metadata":{"execution":{"iopub.status.busy":"2022-05-29T14:23:24.724522Z","iopub.execute_input":"2022-05-29T14:23:24.725016Z","iopub.status.idle":"2022-05-29T14:23:24.809501Z","shell.execute_reply.started":"2022-05-29T14:23:24.724976Z","shell.execute_reply":"2022-05-29T14:23:24.808796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(train_df_sub[['id', 'path']], train_df_sub['new_id'],\n                                                  train_size = 0.9,\n                                                  random_state = 123,\n                                                  shuffle = True,\n                                                  stratify = train_df_sub['new_id'])\n\n# Held-out test set for inference\n# Further 95/5 split -> 5% of original training set left for test set\nX_train, X_test, y_train, y_test = train_test_split(X_train, y_train,\n                                                   train_size = 0.95,\n                                                   random_state = 123,\n                                                   shuffle = True,\n                                                   stratify = y_train)\n\nassert X_train.shape[0] + X_val.shape[0] + X_test.shape[0] == train_df_sub.shape[0]\n\nprint(f\"Training data shape: {X_train.shape}\")\nprint(f\"Training label shape: {y_train.shape}\")\nprint(f\"Validation data shape: {X_val.shape}\")\nprint(f\"Validation label shape: {y_val.shape}\")\nprint(f\"Test data shape: {X_test.shape}\")\nprint(f\"Test label shape: {y_test.shape}\")","metadata":{"execution":{"iopub.status.busy":"2022-05-29T14:23:43.48954Z","iopub.execute_input":"2022-05-29T14:23:43.490471Z","iopub.status.idle":"2022-05-29T14:23:43.529321Z","shell.execute_reply.started":"2022-05-29T14:23:43.490418Z","shell.execute_reply":"2022-05-29T14:23:43.52821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Unique classes on y_train: {y_train.nunique()}\")\nprint(f\"Unique classes on y_val: {y_val.nunique()}\")\nprint(f\"Unique classes on y_test: {y_test.nunique()}\")","metadata":{"execution":{"iopub.status.busy":"2022-05-29T14:23:53.845885Z","iopub.execute_input":"2022-05-29T14:23:53.846273Z","iopub.status.idle":"2022-05-29T14:23:53.852983Z","shell.execute_reply.started":"2022-05-29T14:23:53.846243Z","shell.execute_reply":"2022-05-29T14:23:53.852115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Classes distribution on training, validation and test sets\nplt.figure(figsize = (10, 3))\nax = sns.histplot(y_train, bins=75, kde = True)\nax.set_title('Distribution of Landmarks on training set')\nplt.tight_layout()\n\nplt.figure(figsize = (10, 3))\nax = sns.histplot(y_val, bins=75, kde = True)\nax.set_title('Distribution of Landmarks on validation set')\nplt.tight_layout()\n\nplt.figure(figsize = (10, 3))\nax = sns.histplot(y_test, bins=75, kde = True)\nax.set_title('Distribution of Landmarks on test set')\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-29T14:24:06.212624Z","iopub.execute_input":"2022-05-29T14:24:06.213043Z","iopub.status.idle":"2022-05-29T14:24:07.561131Z","shell.execute_reply.started":"2022-05-29T14:24:06.21301Z","shell.execute_reply":"2022-05-29T14:24:07.560036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NEW_BASE_DIR = \"/kaggle/working\"\n\n# Training set directory\nfor file, path, landmark in tqdm(zip(X_train['id'], X_train['path'], y_train)):\n    dir = f\"{NEW_BASE_DIR}/train_sub/{str(landmark)}\"\n    os.makedirs(dir, exist_ok = True)\n    fname = f\"{file}.jpg\"\n    shutil.copyfile(src = path, dst = f\"{dir}/{fname}\")\n\n# Validation set directory    \nfor file, path, landmark in tqdm(zip(X_val['id'], X_val['path'], y_val)):\n    dir = f\"{NEW_BASE_DIR}/val_sub/{str(landmark)}\"\n    os.makedirs(dir, exist_ok = True)\n    fname = f\"{file}.jpg\"\n    shutil.copyfile(src = path, dst = f\"{dir}/{fname}\")\n\n# Training set directory\nfor file, path, landmark in tqdm(zip(X_test['id'], X_test['path'], y_test)):\n    dir = f\"{NEW_BASE_DIR}/test_sub/{str(landmark)}\"\n    os.makedirs(dir, exist_ok = True)\n    fname = f\"{file}.jpg\"\n    shutil.copyfile(src = path, dst = f\"{dir}/{fname}\")","metadata":{"execution":{"iopub.status.busy":"2022-05-29T14:24:36.637705Z","iopub.execute_input":"2022-05-29T14:24:36.638142Z","iopub.status.idle":"2022-05-29T14:25:48.650049Z","shell.execute_reply.started":"2022-05-29T14:24:36.638107Z","shell.execute_reply":"2022-05-29T14:25:48.648038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.utils import image_dataset_from_directory\n\nIMG_SIZE = 224\nBATCH_SIZE = 16\n\nprint(\"Building training dataset...\")\n# Training tf.data.Dataset\ntrain_ds = image_dataset_from_directory(f\"{NEW_BASE_DIR}/train_sub\",\n                                        label_mode = 'int',\n                                        shuffle = True,\n                                        image_size = (IMG_SIZE, IMG_SIZE),\n                                        batch_size = BATCH_SIZE)\n\nprint(\"Building validation dataset...\")\n# Validation tf.data.Dataset\nval_ds = image_dataset_from_directory(f\"{NEW_BASE_DIR}/val_sub\",\n                                        label_mode = 'int',\n                                        shuffle = True,\n                                        image_size = (IMG_SIZE, IMG_SIZE),\n                                        batch_size = BATCH_SIZE)\n\nprint(\"Building test dataset...\")\n# Test tf.data.Dataset\ntest_ds = image_dataset_from_directory(f\"{NEW_BASE_DIR}/test_sub\",\n                                        label_mode = 'int',\n                                        shuffle = True,\n                                        image_size = (IMG_SIZE, IMG_SIZE),\n                                        batch_size = BATCH_SIZE)","metadata":{"execution":{"iopub.status.busy":"2022-05-29T14:26:37.135099Z","iopub.execute_input":"2022-05-29T14:26:37.13567Z","iopub.status.idle":"2022-05-29T14:26:38.064728Z","shell.execute_reply.started":"2022-05-29T14:26:37.135626Z","shell.execute_reply":"2022-05-29T14:26:38.063637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for data_batch, labels_batch in train_ds.take(1):\n    ncols = 4\n    nrows = int(data_batch.shape[0]/ncols)\n    fig, ax = plt.subplots(nrows = nrows, ncols = ncols, figsize=(10, 11),\n                           sharex = True, sharey = True)\n    img_counter = 0\n    for image, label in zip(data_batch, labels_batch):\n        axi = ax.flat[img_counter]\n        axi.imshow(image/255.)\n        label = label.numpy()\n#         axi.set_title(np.where(label == 1)[0])\n        axi.set_title(label)\n        img_counter += 1\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-29T14:26:45.586045Z","iopub.execute_input":"2022-05-29T14:26:45.586544Z","iopub.status.idle":"2022-05-29T14:26:48.308063Z","shell.execute_reply.started":"2022-05-29T14:26:45.586508Z","shell.execute_reply":"2022-05-29T14:26:48.306986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_augmentation = tf.keras.Sequential(\n    # [layers.RandomFlip(\"horizontal\"),\n    [layers.RandomTranslation(height_factor = 0.1, width_factor = 0.1),\n     layers.RandomRotation(0.02),\n     layers.RandomZoom(0.2)],\n     name = \"img_augmentation\",\n)\nplt.figure(figsize=(9, 9))\nfor image, label in train_ds.take(1):\n    for i in range(9):\n        ax = plt.subplot(3, 3, i + 1)\n        augmented_image = img_augmentation(image, training = True)\n        plt.imshow(augmented_image[15].numpy().astype(\"uint8\"))\n        plt.axis(\"off\")","metadata":{"execution":{"iopub.status.busy":"2022-05-29T14:26:54.225927Z","iopub.execute_input":"2022-05-29T14:26:54.227361Z","iopub.status.idle":"2022-05-29T14:26:56.607874Z","shell.execute_reply.started":"2022-05-29T14:26:54.227302Z","shell.execute_reply":"2022-05-29T14:26:56.606645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Model\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.applications import EfficientNetB0\n\nMODELS_DIR = f\"{NEW_BASE_DIR}/models\"\n\nos.makedirs(MODELS_DIR, exist_ok = True)\n\n# Model instantiator\ndef build_model(num_classes = None):\n    inputs = keras.Input(shape = (IMG_SIZE, IMG_SIZE, 3))\n    x = img_augmentation(inputs)\n    # EfficientNetB0 backbone\n    model = EfficientNetB0(input_tensor = x,\n                           weights = 'imagenet',\n                           include_top = False,\n                           drop_connect_rate = DROP_CONNECT_RATE)\n    \n    # Freeze pretrained weights\n    model.trainable = False\n    \n    # Rebuild top\n    x = layers.GlobalAveragePooling2D(name = \"avg_pool\")(model.output)\n    x = layers.BatchNormalization()(x)\n    x = layers.Dropout(TOP_DROPOUT_RATE, name = \"top_dropout\")(x)\n    \n    # Embedding\n    embedding = layers.Dense(512, name = \"embedding_512\")(x)\n    outputs = layers.Dense(num_classes, activation = \"softmax\", name = \"softmax\")(embedding)\n    \n    # Compile\n    model = tf.keras.Model(inputs, outputs, name = \"EfficientNetB0\")\n    optimizer = tf.keras.optimizers.Adam(learning_rate = ADAM_LR)\n    model.compile(optimizer = optimizer,\n                 loss = \"sparse_categorical_crossentropy\",\n                 metrics = [\"accuracy\"])\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2022-05-29T14:27:13.942419Z","iopub.execute_input":"2022-05-29T14:27:13.94294Z","iopub.status.idle":"2022-05-29T14:27:13.955276Z","shell.execute_reply.started":"2022-05-29T14:27:13.9429Z","shell.execute_reply":"2022-05-29T14:27:13.953805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_hist(hist):\n    plt.plot(hist.history[\"accuracy\"])\n    plt.plot(hist.history[\"val_accuracy\"])\n    plt.title(\"model accuracy\")\n    plt.ylabel(\"accuracy\")\n    plt.xlabel(\"epoch\")\n    plt.legend([\"train\", \"validation\"], loc=\"upper left\")\n    plt.show()\n\nDROP_CONNECT_RATE = 0.2 # Dropout rate for stochastic depth on EfficientNet\nTOP_DROPOUT_RATE = 0.2  # Top dropout\nINIT_LR = 5e-3          # Initial learning rate\nEPOCHS = 20\n# Adam optimizer learning rate schedule\nADAM_LR = tf.keras.optimizers.schedules.ExponentialDecay(\n    INIT_LR,\n    decay_steps=100,\n    decay_rate=0.96,\n    staircase=True)\n\nmodel = build_model(num_classes = NUM_CLASSES)","metadata":{"execution":{"iopub.status.busy":"2022-05-29T14:27:37.512175Z","iopub.execute_input":"2022-05-29T14:27:37.512592Z","iopub.status.idle":"2022-05-29T14:27:39.997638Z","shell.execute_reply.started":"2022-05-29T14:27:37.51256Z","shell.execute_reply":"2022-05-29T14:27:39.996645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training embedding layer\nmodel_file_path = os.path.join(MODELS_DIR, \"EfficientNetB0_softmax.keras\")\ncallbacks = [\n    keras.callbacks.ModelCheckpoint(model_file_path,\n                                    save_best_only=True,\n                                    monitor = \"val_accuracy\"),\n    keras.callbacks.EarlyStopping(patience = 2,\n                                  monitor = \"val_accuracy\")]\n\nhist = model.fit(train_ds,\n                 epochs = EPOCHS,\n                 validation_data = val_ds,\n                 shuffle = 'batch',\n                 callbacks = callbacks)\n\nplot_hist(hist)","metadata":{"execution":{"iopub.status.busy":"2022-05-29T14:27:51.917551Z","iopub.execute_input":"2022-05-29T14:27:51.918096Z","iopub.status.idle":"2022-05-29T15:58:29.208192Z","shell.execute_reply.started":"2022-05-29T14:27:51.918059Z","shell.execute_reply":"2022-05-29T15:58:29.206673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Evaluating best model\nmodel = keras.models.load_model(model_file_path)\nprint(\"Predictions on validation set...\")\nprint(f\"Validation accuracy: {model.evaluate(val_ds)[1]*100:.2f} %\")\nprint(\"Predictions on test set...\")\nprint(f\"Test accuracy: {model.evaluate(test_ds)[1]*100:.2f} %\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_image(path, resize = False, reshape = False, target_size = None):\n    img = cv2.imread(path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    if resize:\n        img = cv2.resize(img, dsize = (target_size, target_size))\n    if reshape:\n        img = tf.reshape(img, [1, target_size, target_size, 3])\n    return img\n\n# Get landmark samples\ndef get_landmark(landmark_id, samples = 16):\n    nrows = samples // 4\n    random_imgs = np.random.choice(train_df_sub[train_df_sub['new_id'] == landmark_id].index, samples, replace = False)\n    plt.figure(figsize = (12, 10))\n    for i, img in enumerate(train_df_sub.loc[random_imgs, :].values):\n        ax = plt.subplot(nrows, 4, i + 1)\n        plt.imshow(get_image(img[2]))\n        plt.title(f\"{img[0]}\")\n        plt.suptitle(f\"Samples of landmark {landmark_id}\", fontsize = 14, y = 0.94, weight = \"bold\")\n        plt.axis(\"off\")\n\n# Get image embeddings\ndef get_embeddings(model, image_paths, input_size, as_df = True):\n    embeddings = {}\n    embeddings['images_paths'] = []\n    embeddings['embedded_images'] = []\n    \n    target_dir = os.path.split(os.path.split(image_paths[0])[0])[0]\n    \n    print(f\"Retrieving embeddings for {target_dir} with {model.name}...\")\n    for image_path in tqdm(image_paths):\n        embeddings['images_paths'].append(image_path)\n        embedded_image = model.predict(get_image(image_path,\n                                                 resize = True,\n                                                 reshape = True,\n                                                 target_size = input_size))\n        embeddings['embedded_images'].append(embedded_image)\n    \n    if as_df:\n        embeddings = pd.DataFrame(embeddings)\n    \n    return embeddings\n\n# Get similarities between query key pair\ndef get_similarities(query, key):\n    '''\n    Get cosine similarity matrix between query and key pairs\n    Arguments:\n    query, key: embedded images\n    '''\n    query_array = np.stack(query.tolist()).reshape(query.shape[0],\n                                                   query[0].shape[1])\n    key_array = np.stack(key.tolist()).reshape(key.shape[0],\n                                               key[0].shape[1])\n    \n    # Initializing similarity matrix\n    similarity = np.zeros((query_array.shape[0], key_array.shape[0]))\n    \n    # Getting pairwise similarities\n    print(f\"Getting pairwise {query_array.shape[0]} query: {key_array.shape[0]} key similarities...\")\n    for query_index in tqdm(range(query_array.shape[0])):\n        similarity[query_index] = 1 - spatial.distance.cdist(query_array[np.newaxis, query_index, :],\n                                                             key_array,\n                                                             'cosine')[0]\n    return similarity\n\n# Plot top ranked images\ndef plot_similar(similar_imgs, img_paths):\n    '''\n    Plot top N similar samples from similarity index\n    '''\n    plt.figure(figsize = (18, 6))\n    nrows = similar_imgs.shape[0]//5\n    for i, img in enumerate(similar_imgs):\n        ax = plt.subplot(nrows, 5, i + 1)\n        plt.imshow(get_image(img_paths[img]))\n        plt.title(f\"Landmark id: {os.path.split(os.path.split(img_paths[img])[0])[1]}\")\n        plt.axis(\"off\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Embedding models\nembedding_layer = 'embedding_512'\nembedding_model = tf.keras.Model(inputs = model.input,\n                                 outputs = model.get_layer(embedding_layer).output,\n                                 name = \"EfficientNetB0_embed512\")\n# Retrieving embeddings\ntrain_img_paths = train_ds.file_paths\nval_img_paths = val_ds.file_paths\n\ntrain_embeddings = get_embeddings(model = embedding_model,\n                                 image_paths = train_img_paths,\n                                 input_size = IMG_SIZE)\n\nval_embeddings = get_embeddings(model = embedding_model,\n                                 image_paths = val_img_paths,\n                                 input_size = IMG_SIZE)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_embeddings.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_train_similarity = get_similarities(val_embeddings['embedded_images'],\n                                        train_embeddings['embedded_images'])\nval_train_similarity.shape","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}