{"cells":[{"metadata":{},"cell_type":"markdown","source":"- This Notebook is an attempt to do a simple image search using autoencoders.\n- The image reconstructed is still blurry.","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"%matplotlib inline\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nimport re\nimport math, random\n\nfrom tqdm import tqdm\nfrom glob import glob\nimport gc\n\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import train_test_split\n\nimport cv2\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom IPython.display import display\n\nplt.rcParams[\"figure.figsize\"] = (8, 8)\nplt.rcParams['axes.titlesize'] = 8\n\nfrom kaggle_datasets import KaggleDatasets\n\nimport tensorflow as tf\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras import Input, Model\nimport tensorflow.keras.layers as L\nimport tensorflow.keras.backend as K\n\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint, EarlyStopping\nfrom tensorflow.keras.metrics import categorical_accuracy\n\n\nfrom time import time, strftime, gmtime\n\nstart = time()\nprint(start)\n\nimport datetime\nprint(str(datetime.datetime.now()))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/landmark-retrieval-2020/train.csv')\nprint(train.shape)\ntrain","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"landmark_counts = train['landmark_id'].value_counts().reset_index().rename(columns = {'landmark_id': 'count', 'index': 'landmark_id'})\nlandmark_counts = landmark_counts.sort_values('count')\nlandmark_counts","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"__Taking only the topp 10 classes__","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"top_100 = landmark_counts.tail(100).reset_index(drop = True)\ntop_100 = train.loc[train['landmark_id'].isin(top_100['landmark_id'])].reset_index(drop = True)\ntop_100","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> __Create a dataframe from top 100 classes with each class containing 300 images - simplicity purpose__","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.DataFrame()\n\nfor each in tqdm(top_100['landmark_id'].unique()):\n    temp = top_100.loc[top_100['landmark_id'] == each]\n    temp = temp.sample(300)\n    df = pd.concat([df, temp], ignore_index = True)\nprint(df.shape)\ndf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_classes = df['landmark_id'].nunique()\nnum_classes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lbl = LabelEncoder()\n\ndf['landmark_id'] = lbl.fit_transform(df['landmark_id'])\ndf","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"__Adding Image path to the df as a column__\n- Thanks to this notebook for the below code https://www.kaggle.com/derinformatiker/landmark-retrieval-all-paths","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_paths(sub):\n    index = [\"0\",\"1\",\"2\",\"3\",\"4\",\"5\",\"6\",\"7\",\"8\",\"9\",\"a\",\"b\",\"c\",\"d\",\"e\",\"f\"]\n\n    paths = []\n\n    for a in index:\n        for b in index:\n            for c in index:\n                try:\n                    paths.extend([f\"{sub}/{a}/{b}/{c}/\" + x for x in os.listdir(f\"/kaggle/input/landmark-retrieval-2020/{sub}/{a}/{b}/{c}\")])\n                except:\n                    pass\n\n    return paths","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_path = df.copy()\n\nrows = []\nfor i in tqdm(range(len(df))):\n    row = df.iloc[i]\n    path  = list(row[\"id\"])[:3]\n    temp = row[\"id\"]\n    row[\"id\"] = f\"train/{path[0]}/{path[1]}/{path[2]}/{temp}.jpg\"\n    rows.append(row[\"id\"])\n    \nrows = pd.DataFrame(rows)\ndf_path[\"id\"] = rows","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(df_path.shape)\ndf_path.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_images_random(data, nrows, ncols, title = None):\n    plt.suptitle(title, fontsize = 16)\n    plt.figure(figsize = (16, 16))\n    plt.rcParams[\"axes.grid\"] = False\n    for i, img_id  in enumerate(np.random.choice(data['id'], nrows * ncols)):\n        try:\n            img = cv2.imread('/kaggle/input/landmark-retrieval-2020/' + img_id)\n            img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n            img = cv2.resize(img, (224, 224))\n            plt.subplot(nrows, ncols, i + 1)\n            plt.imshow(img)\n        except:\n            pass","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_images_random(df_path, 3, 4, 'Images from Train folder')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dim = [224, 224]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection. No parameters necessary if TPU_NAME environment variable is set. On Kaggle this is always the case.\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy() # default distribution strategy in Tensorflow. Works on CPU and single GPU.\n\nAUTO = tf.data.experimental.AUTOTUNE\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)\n\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\nprint(BATCH_SIZE)\n\nGCS_DS_PATH = KaggleDatasets().get_gcs_path()\nprint(GCS_DS_PATH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Check the GCS path bucket - helps alot\n!gsutil ls $GCS_DS_PATH","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def format_path(pt):\n    return os.path.join(GCS_DS_PATH, pt)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_paths = df_path['id'].apply(format_path).values\n\nfrom tensorflow.keras.utils import to_categorical\n\ntrain_targets = to_categorical(df_path['landmark_id'].values, num_classes = num_classes)\n\ntrain_paths[:2], train_targets.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_path, valid_path, train_label, valid_label = train_test_split(train_paths, train_targets, test_size = 0.05, random_state = 2019)\nprint(train_path.shape, train_label.shape, valid_path.shape, valid_label.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(filename, label = None, image_size = dim):\n    bits = tf.io.read_file(filename)\n    image = tf.image.decode_jpeg(bits, channels = 3)\n    image = tf.cast(image, tf.float32) / 255.0\n    image = tf.image.resize(image, image_size)\n    \n    if label is None:\n        return image\n    else:\n        return image, image\n\ndef data_augment(image, label = None):\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_flip_up_down(image)\n    \n    if label is None:\n        return image\n    else:\n        return image, image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((train_path, train_label))\n    .map(decode_image, num_parallel_calls = AUTO)\n    .cache()\n    .repeat()\n    .shuffle(1024)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTO)\n)\n\nvalid_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((valid_path, valid_label))\n    .map(decode_image, num_parallel_calls = AUTO)\n    .batch(BATCH_SIZE)\n    .cache()\n    .prefetch(AUTO)\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset, valid_dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"__Visualize images in the train and valid datasets__","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"image_batch, label_batch = next(iter(train_dataset))\n\nplt.figure(figsize = (10, 10))\nfor i in range(9):\n    ax = plt.subplot(3, 3, i + 1)\n    plt.imshow(image_batch[i].numpy())\n    label = label_batch[i]\n    plt.title(np.argmax(label))\n    plt.axis(\"off\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_batch, label_batch = next(iter(valid_dataset))\n\nplt.figure(figsize = (10, 10))\nfor i in range(9):\n    ax = plt.subplot(3, 3, i + 1)\n    plt.imshow(image_batch[i].numpy())\n    label = label_batch[i]\n    plt.title(np.argmax(label))\n    plt.axis(\"off\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"LR = 1e-4\nEPOCHS = 80\nch_dim = -1\nhidden = 16\nch = 3","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"__Simple Autoencoder__","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    #Encoder\n    inputs = Input(shape = (*dim, ch))\n\n    x = L.Conv2D(128, (3, 3), strides = 2, padding = 'same')(inputs)\n    x = L.LeakyReLU(alpha = 0.2)(x)\n    x = L.BatchNormalization(axis = ch_dim)(x)\n    \n    x = L.Conv2D(64, (3, 3), strides = 2, padding = 'same')(x)\n    x = L.LeakyReLU(alpha = 0.2)(x)\n    x = L.BatchNormalization(axis = ch_dim)(x)\n    \n    x = L.Conv2D(32, (3, 3), strides = 2, padding = 'same')(x)\n    x = L.LeakyReLU(alpha = 0.2)(x)\n    x = L.BatchNormalization(axis = ch_dim)(x)\n\n    enc_size = K.int_shape(x)\n    #print(enc_size)\n\n    x = L.Flatten()(x)\n    encoder_output = L.Dense(hidden, name = 'Encoder')(x)\n\n    encoder_model = Model(inputs = inputs, outputs = encoder_output, name = 'encoder_model')\n\n    #Decoder\n    x = L.Dense(np.prod(enc_size[1: ]))(encoder_output)\n\n    x = L.Reshape((enc_size[1], enc_size[2], enc_size[3]))(x)\n\n    x = L.Conv2DTranspose(32, (3, 3), strides = 2, padding = 'same')(x)\n    x = L.LeakyReLU(alpha = 0.2)(x)\n    x = L.BatchNormalization(axis = ch_dim)(x)\n    \n    x = L.Conv2DTranspose(64, (3, 3), strides = 2, padding = 'same')(x)\n    x = L.LeakyReLU(alpha = 0.2)(x)\n    x = L.BatchNormalization(axis = ch_dim)(x)\n    \n    x = L.Conv2DTranspose(128, (3, 3), strides = 2, padding = 'same')(x)\n    x = L.LeakyReLU(alpha = 0.2)(x)\n    x = L.BatchNormalization(axis = ch_dim)(x)\n\n    x = L.Conv2D(ch, (3, 3), padding = 'same')(x)\n    out = L.Activation('sigmoid', name = 'Decoder')(x)\n\n    autoencoder = Model(inputs = inputs, outputs = out, name = 'autoencoder')\n\n    opt = Adam(lr = LR, decay = LR / EPOCHS)\n    autoencoder.compile(loss = tf.keras.losses.MeanSquaredError(), optimizer = opt)\n\n    autoencoder.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"encoder_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"STEPS_PER_EPOCH = train_label.shape[0] // BATCH_SIZE\n\ncheckpoint = ModelCheckpoint('auto_tpu_model.h5', monitor = 'val_loss', save_best_only = True, verbose = 1)\n\nreduce_lr = ReduceLROnPlateau(monitor = 'loss', factor = 0.1, patience = 5, min_lr = 0.0001, verbose = 1)\n\nearly = EarlyStopping(monitor = 'val_loss', patience = 10, verbose = 1, mode = 'auto')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = autoencoder.fit(train_dataset,  epochs = EPOCHS, batch_size = BATCH_SIZE,\n                    steps_per_epoch = STEPS_PER_EPOCH,\n                    validation_data = valid_dataset,\n                   verbose = 1, callbacks = [checkpoint, early, reduce_lr]\n                   )\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"encoder_model.save('./EncoderModel_tpu.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history.history.keys()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.DataFrame(history.history).plot(y = ['loss', 'val_loss'], logy = False)\nplt.xlabel(\"Epochs\")\nplt.ylabel(\"Loss\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"__Visualize Autoencoder outputs__","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def visualize_autoencoder(img, encoder, decoder):\n    \"\"\"Draws original, encoded and decoded images\"\"\"\n    # img[None] will have shape of (1, 32, 32, 3) which is the same as the model input\n    encoder_output = encoder.predict(img[None])\n    recon = decoder.predict(img[None])\n\n    plt.figure(figsize = (12, 12))\n    plt.subplot(1,3,1)\n    plt.title(\"Original\")\n    plt.imshow(img)\n\n    plt.subplot(1,3,2)\n    plt.title(\"Encoder Output\")\n    plt.imshow(encoder_output.reshape([encoder_output.shape[-1] // 2, -1]))\n\n    plt.subplot(1,3,3)\n    plt.title(\"Reconstructed\")\n    plt.imshow(recon.squeeze())\n    plt.show()\n\nimage_batch, _ = next(iter(train_dataset))\n\nfor i in range(5):\n    visualize_autoencoder(image_batch[i], encoder_model, autoencoder)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"__Create test and index dataframes with thier respective paths__","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"test_paths = get_paths('test')\ntest_df = pd.DataFrame(test_paths, columns = ['id'])\ntest_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"index_paths = get_paths('index')\nindex_df = pd.DataFrame(index_paths, columns = ['id'])\nindex_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_paths = test_df['id'].apply(format_path).values\ntest_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices(test_paths)\n    .map(decode_image, num_parallel_calls = AUTO)\n    .batch(BATCH_SIZE)\n)\ntest_dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_batch = next(iter(test_dataset))\n\nplt.figure(figsize = (10, 10))\nfor i in range(9):\n    ax = plt.subplot(3, 3, i + 1)\n    plt.imshow(test_batch[i].numpy())\n    plt.title('Test Images')\n    plt.axis(\"off\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"__Test Image Embeddings obtained by making predictions using Encoder Model__","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"test_emb = encoder_model.predict(test_dataset, verbose = 1)\nnp.save('./test_embs.npy', test_emb)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"index_paths = index_df['id'].apply(format_path).values\n\nindex_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices(index_paths)\n    .map(decode_image, num_parallel_calls = AUTO)\n    .batch(BATCH_SIZE)\n)\nindex_dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"index_batch = next(iter(index_dataset))\n\nplt.figure(figsize = (10, 10))\nfor i in range(9):\n    ax = plt.subplot(3, 3, i + 1)\n    plt.imshow(index_batch[i].numpy())\n    plt.title('Index Images')\n    plt.axis(\"off\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"__Index Image Embeddings obtained by making predictions using Encoder Model__","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"index_emb = encoder_model.predict(index_dataset, verbose = 1)\nnp.save('./index_embs.npy', index_emb)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_emb.shape, index_emb.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"__Finding the embedding distances__","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def euclidean(a, b):\n    #compute and return the euclidean distance between two vectors\n    return np.linalg.norm(a - b)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dist = []\ntest_ret = {}\nknn = 10\nfor i, test_img in enumerate(test_emb):\n    for index_img in index_emb:\n        dist.append(euclidean(test_img, index_img))\n    d = {i: np.argsort(dist)[:knn]}\n    test_ret.update(d)\n    dist = []\nlen(test_ret), test_ret","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_predictions(keys, values, nrows = 2, ncols = 5):\n    plt.title('Test Image {}'.format(keys), fontsize = 12)\n    plt.rcParams[\"axes.grid\"] = False\n    img = cv2.imread('/kaggle/input/landmark-retrieval-2020/' + test_df['id'].iloc[keys])\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    img = cv2.resize(img, (224, 224))\n    plt.imshow(img)\n    plt.figure(figsize = (16, 16))\n    for i, ind  in enumerate(values):\n        img = cv2.imread('/kaggle/input/landmark-retrieval-2020/' + index_df['id'].iloc[ind])\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        img = cv2.resize(img, (224, 224))\n        plt.subplot(nrows, ncols, i + 1)\n        plt.imshow(img)\n        plt.title('Index Images {}'.format(ind), fontsize = 16)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"keys = random.sample(test_ret.keys(), 5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_predictions(keys[0], test_ret[keys[0]])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_predictions(keys[1], test_ret[keys[1]])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_predictions(keys[2], test_ret[keys[2]])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_predictions(keys[3], test_ret[keys[3]])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_predictions(keys[4], test_ret[keys[4]])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"finish = time()\nprint(strftime(\"%H:%M:%S\", gmtime(finish - start)))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}