{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# EfficientNet Tuned for Landmark Classification\nBaseline image classifier: additional layers are added on top of the pretrained **EfficientnetB0** model. The model is being trained on augmented data until validation loss stops decreasing.","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nimport glob\nimport gc\nimport tensorflow as tf\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\nimport matplotlib.pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-08-24T12:16:19.634988Z","iopub.execute_input":"2021-08-24T12:16:19.635416Z","iopub.status.idle":"2021-08-24T12:16:26.761439Z","shell.execute_reply.started":"2021-08-24T12:16:19.635323Z","shell.execute_reply":"2021-08-24T12:16:26.760444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plots display settings\nplt.rcParams['figure.figsize'] = 12, 8\nplt.rcParams.update({'font.size': 14})","metadata":{"execution":{"iopub.status.busy":"2021-08-24T12:16:26.762865Z","iopub.execute_input":"2021-08-24T12:16:26.763165Z","iopub.status.idle":"2021-08-24T12:16:26.767691Z","shell.execute_reply.started":"2021-08-24T12:16:26.763135Z","shell.execute_reply":"2021-08-24T12:16:26.766643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# DataFrame display settings\npd.set_option('display.max_columns', None)\npd.set_option('display.max_rows', None)\npd.options.display.float_format = '{:.4f}'.format","metadata":{"execution":{"iopub.status.busy":"2021-08-24T12:16:26.769834Z","iopub.execute_input":"2021-08-24T12:16:26.770114Z","iopub.status.idle":"2021-08-24T12:16:26.779308Z","shell.execute_reply.started":"2021-08-24T12:16:26.770087Z","shell.execute_reply":"2021-08-24T12:16:26.778109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# To avoid errors during notebook submission\nINPUT_DIR = os.path.join('..', 'input')\nDATASET_DIR = os.path.join(INPUT_DIR, 'landmark-recognition-2021')\n\nTRAIN_METADATA = os.path.join(DATASET_DIR, 'train.csv')\nTRAIN_DIRECTORY = os.path.join(DATASET_DIR, 'train')\nTEST_DIRECTORY = os.path.join(DATASET_DIR, 'test')","metadata":{"execution":{"iopub.status.busy":"2021-08-24T12:16:26.781158Z","iopub.execute_input":"2021-08-24T12:16:26.781460Z","iopub.status.idle":"2021-08-24T12:16:26.790855Z","shell.execute_reply.started":"2021-08-24T12:16:26.781433Z","shell.execute_reply":"2021-08-24T12:16:26.789759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Pretrained image classification model EfficientNetB7\n# from tf.keras.applications with global average pooling as a final layer\n# In this notebook the model is loaded from a public dataset on Kaggle\n# at https://www.kaggle.com/ekaterinadranitsyna/keras-applications-models\nIMG_MODEL = '../input/keras-applications-models/EfficientNetB0.h5'","metadata":{"execution":{"iopub.status.busy":"2021-08-24T12:16:26.792145Z","iopub.execute_input":"2021-08-24T12:16:26.792438Z","iopub.status.idle":"2021-08-24T12:16:26.801777Z","shell.execute_reply.started":"2021-08-24T12:16:26.792411Z","shell.execute_reply":"2021-08-24T12:16:26.800941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# TensorFlow settings\nAUTOTUNE = tf.data.experimental.AUTOTUNE\nIMG_SIZE = 224\nBATCH_SIZE = 256\nDROPOUT_RATE = 0.1\nLEARNING_RATE = 1e-3\nEPOCHS = 1000\nPATIENCE = 3\n\nVAL_SIZE = 0.15\nTOP_CLASSES = 10","metadata":{"execution":{"iopub.status.busy":"2021-08-24T12:16:26.803053Z","iopub.execute_input":"2021-08-24T12:16:26.803354Z","iopub.status.idle":"2021-08-24T12:16:26.812877Z","shell.execute_reply.started":"2021-08-24T12:16:26.803316Z","shell.execute_reply":"2021-08-24T12:16:26.812015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Functions","metadata":{}},{"cell_type":"code","source":"def id_to_path(image_id: str, directory: str) -> str:\n    \"\"\"Function transforms image ID into path to the image.\n    :param image_id: String id from 'train.csv'\n    :param directory: Path to the directory with images\n    :return: Path to the image file\n    \"\"\"\n    subdir_1 = image_id[0]\n    subdir_2 = image_id[1]\n    subdir_3 = image_id[2]\n    path = f'{directory}/{subdir_1}/{subdir_2}/{subdir_3}/{image_id}.jpg'\n    return path\n\n\ndef path_to_id(path: str) -> str:\n    \"\"\"Function transforms path to the image into image ID.\n    :param path: Path to image file\n    :return: String representing image ID\n    \"\"\"\n    return path[-20:-4]\n\n\ndef samples_distribution(value_counts: pd.Series) -> None:\n    \"\"\"Function displays distribution of images per class\n    and prints out basic statistics.\n    :param value_counts: Series objects where index represent class IDs, values - number of samples\n    :return: None\n    \"\"\"\n    mean_images = round(value_counts.mean(), 0)\n    median_images = round(value_counts.median(), 0)\n    print(f'Total number of classes: {len(value_counts)}')\n    print(f'{value_counts.min()} - {value_counts.max()} samples per class')\n    print(f'Mean value: {mean_images} samples\\n'\n          f'Median value: {median_images} samples')\n\n    images_per_class.hist(bins=20, log=True)\n    plt.vlines(mean_images, ymin=0, ymax=80_000, colors='red', label='Mean number')\n    plt.vlines(median_images, ymin=0, ymax=80_000, colors='green', label='Median number')\n    plt.title('Train samples per class')\n    plt.xlabel('Number of images')\n    plt.ylabel('Frequency')\n    plt.show()\n\n    \n@tf.function\ndef get_image(path: str) -> tf.Tensor:\n    \"\"\"Function loads image from a file and preprocesses it.\n    :param path: Path to image file\n    :return: Tensor with preprocessed image\n    \"\"\"\n    image = tf.image.decode_jpeg(tf.io.read_file(path), channels=3)\n    image = tf.cast(tf.image.resize_with_pad(image, IMG_SIZE, IMG_SIZE), dtype=tf.int32)\n    return tf.keras.applications.efficientnet.preprocess_input(image)\n\n\n@tf.function\ndef process_dataset(path: str, label: int) -> tuple:\n    \"\"\"Function loads image from a file and preprocesses it.\n    :param path: Path to image file\n    :param label: Class label\n    :return: tf.Tensor with preprocessed image, numeric label\n    \"\"\"\n    return get_image(path), label\n\n\n@tf.function\ndef get_dataset(x, y=None) -> tf.data.Dataset:\n    \"\"\"Function creates batched optimized dataset for the model\n    out of an array of file paths and (optionally) class labels.\n    :param x: Input data for the model (array of file paths)\n    :param y: Target values for the model (array of class indexes)\n    :return TensorFlow Dataset object\n    \"\"\"\n    if y is not None:\n        ds = tf.data.Dataset.from_tensor_slices((x, y))\n        return ds.map(process_dataset, num_parallel_calls=AUTOTUNE) \\\n            .batch(BATCH_SIZE).prefetch(buffer_size=AUTOTUNE)\n    else:\n        ds = tf.data.Dataset.from_tensor_slices(x)\n        return ds.map(get_image, num_parallel_calls=AUTOTUNE) \\\n            .batch(BATCH_SIZE).prefetch(buffer_size=AUTOTUNE)\n    \n    \ndef plot_history(hist):\n    \"\"\"Function plots a chart with training and validation metrics.\n    :param hist: Tensorflow history object from model.fit()\n    \"\"\"\n    # Losses and metrics\n    loss = hist.history['loss']\n    val_loss = hist.history['val_loss']\n    acc = hist.history['sparse_categorical_accuracy']\n    val_acc = hist.history['val_sparse_categorical_accuracy']\n\n    # Epochs to plot along x axis\n    x_axis = range(1, len(loss) + 1)\n\n    fig, (ax1, ax2) = plt.subplots(nrows=2, ncols=1, sharex=True)\n\n    ax1.plot(x_axis, loss, 'bo', label='Training')\n    ax1.plot(x_axis, val_loss, 'ro', label='Validation')\n    ax1.set_title('Training and validation loss')\n    ax1.set_ylabel('Loss')\n    ax1.legend()\n\n    ax2.plot(x_axis, acc, 'bo', label='Training')\n    ax2.plot(x_axis, val_acc, 'ro', label='Validation')\n    ax2.set_title('Training and validation accuracy')\n    ax2.set_xlabel('Epochs')\n    ax2.set_ylabel('Accuracy')\n    ax2.legend()\n\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-24T12:16:26.814117Z","iopub.execute_input":"2021-08-24T12:16:26.814388Z","iopub.status.idle":"2021-08-24T12:16:26.835544Z","shell.execute_reply.started":"2021-08-24T12:16:26.814361Z","shell.execute_reply":"2021-08-24T12:16:26.834399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data processing","metadata":{}},{"cell_type":"code","source":"# Find paths to all test images\ntest_paths = glob.glob(f'{TEST_DIRECTORY}/*/*/*/*.jpg')\n\n# DataFrame for predicted values\ntest_prediction = pd.DataFrame()\ntest_prediction['path'] = test_paths\ntest_prediction['id'] = test_prediction['path'].apply(path_to_id)\n\nn_test_samples = len(test_prediction)\nprint(f'Test data: {n_test_samples} images')\ntest_prediction.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-24T12:16:26.838068Z","iopub.execute_input":"2021-08-24T12:16:26.838656Z","iopub.status.idle":"2021-08-24T12:16:37.977125Z","shell.execute_reply.started":"2021-08-24T12:16:26.838585Z","shell.execute_reply":"2021-08-24T12:16:37.976146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_train = pd.read_csv(TRAIN_METADATA)\ndata_train['path'] = data_train['id'].apply(lambda x: id_to_path(x, TRAIN_DIRECTORY))\nn_train_samples = len(data_train)\nprint(f'Train data: {n_train_samples} images')\ndata_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-24T12:16:37.979449Z","iopub.execute_input":"2021-08-24T12:16:37.979889Z","iopub.status.idle":"2021-08-24T12:16:41.050043Z","shell.execute_reply.started":"2021-08-24T12:16:37.979843Z","shell.execute_reply":"2021-08-24T12:16:41.049068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# To save time when saving the notebook - check total number of train images.\nif n_train_samples == 1_580_470:\n    # If the number of available images is limited\n    full_size_training = False\n    EPOCHS = 10\n    print('WARNING: Training will be limited to a portion of the train data.')\nelse:\n    # Full size training and inference mode\n    full_size_training = True\n    print('INFO: Full size training and inference will be executed.')","metadata":{"execution":{"iopub.status.busy":"2021-08-24T12:16:41.051533Z","iopub.execute_input":"2021-08-24T12:16:41.051921Z","iopub.status.idle":"2021-08-24T12:16:41.057804Z","shell.execute_reply.started":"2021-08-24T12:16:41.051887Z","shell.execute_reply":"2021-08-24T12:16:41.056708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visualize class imbalance\nimages_per_class = data_train['landmark_id'].value_counts()\nn_classes = len(images_per_class)\nsamples_distribution(images_per_class)","metadata":{"execution":{"iopub.status.busy":"2021-08-24T12:16:41.059386Z","iopub.execute_input":"2021-08-24T12:16:41.059816Z","iopub.status.idle":"2021-08-24T12:16:42.079885Z","shell.execute_reply.started":"2021-08-24T12:16:41.059772Z","shell.execute_reply":"2021-08-24T12:16:42.078950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Missing classes in the train set\nlargest_id = data_train[\"landmark_id\"].max()\nprint(f'Largest landmark ID: {largest_id}')\nprint(f'Number of missing classes: {largest_id - n_classes}')","metadata":{"execution":{"iopub.status.busy":"2021-08-24T12:16:42.081173Z","iopub.execute_input":"2021-08-24T12:16:42.081468Z","iopub.status.idle":"2021-08-24T12:16:42.089807Z","shell.execute_reply.started":"2021-08-24T12:16:42.081438Z","shell.execute_reply":"2021-08-24T12:16:42.088859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This trick is to save the notebook faster.\n# It would not apply in real inference execution.\nif not full_size_training:\n    selected_classes = data_train['landmark_id'].value_counts().head(TOP_CLASSES).index\n    data_train = data_train.loc[data_train['landmark_id'].isin(selected_classes), :]\n    n_classes = TOP_CLASSES\n    print(f'Limited train data to {TOP_CLASSES} most frequent landmark IDs: {len(data_train)} samples.')","metadata":{"execution":{"iopub.status.busy":"2021-08-24T12:16:42.090869Z","iopub.execute_input":"2021-08-24T12:16:42.091162Z","iopub.status.idle":"2021-08-24T12:16:42.417354Z","shell.execute_reply.started":"2021-08-24T12:16:42.091132Z","shell.execute_reply":"2021-08-24T12:16:42.416122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Assign new integer IDs to all classes starting from 0 without any gaps\nencoder = LabelEncoder()\ndata_train['class_id'] = encoder.fit_transform(data_train['landmark_id'])\ndata_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-24T12:16:42.420422Z","iopub.execute_input":"2021-08-24T12:16:42.420721Z","iopub.status.idle":"2021-08-24T12:16:42.500736Z","shell.execute_reply.started":"2021-08-24T12:16:42.420692Z","shell.execute_reply":"2021-08-24T12:16:42.499680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Select train and validation images so that all classes are present in both sets\ntrain_paths, valid_paths, train_labels, valid_labels = train_test_split(\n    data_train['path'], data_train['class_id'], test_size=VAL_SIZE,\n    shuffle=True, stratify=data_train['class_id'])\nprint(f'Train set: {len(train_paths)} samples\\n'\n      f'Validation set: {len(valid_paths)} samples')","metadata":{"execution":{"iopub.status.busy":"2021-08-24T12:16:42.501920Z","iopub.execute_input":"2021-08-24T12:16:42.502185Z","iopub.status.idle":"2021-08-24T12:16:45.532562Z","shell.execute_reply.started":"2021-08-24T12:16:42.502158Z","shell.execute_reply":"2021-08-24T12:16:45.531473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create tensorflow datasets\ntrain_ds = get_dataset(train_paths, train_labels)\nvalid_ds = get_dataset(valid_paths, valid_labels)","metadata":{"execution":{"iopub.status.busy":"2021-08-24T12:16:45.534575Z","iopub.execute_input":"2021-08-24T12:16:45.535001Z","iopub.status.idle":"2021-08-24T12:16:47.020507Z","shell.execute_reply.started":"2021-08-24T12:16:45.534955Z","shell.execute_reply":"2021-08-24T12:16:47.019705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model architecture","metadata":{}},{"cell_type":"code","source":"# Block for data augmentation inside the model\naugmentation = tf.keras.models.Sequential(\n    [\n        tf.keras.layers.experimental.preprocessing.RandomRotation(factor=(-0.15, 0.15)),\n        tf.keras.layers.experimental.preprocessing.RandomTranslation(\n            height_factor=(-0.1, 0.1), width_factor=(-0.1, 0.1)),\n        tf.keras.layers.experimental.preprocessing.RandomContrast(factor=(0., 0.1)),\n        tf.keras.layers.experimental.preprocessing.RandomCrop(IMG_SIZE, IMG_SIZE),\n        tf.keras.layers.experimental.preprocessing.RandomZoom(\n            height_factor=(-0.15, 0.15), width_factor=None)\n    ],\n    name='image_augmentation',\n)\n\n# Pretrained image classification model\nimage_model = tf.keras.models.load_model(IMG_MODEL)\n\n# Freeze weights in the original model\nimage_model.trainable = False\n\nmodel = tf.keras.models.Sequential(\n    [\n        tf.keras.layers.Input(shape=(IMG_SIZE, IMG_SIZE, 3)),\n        augmentation,  # This block works only in training mode\n        image_model,  # Pretrained model with all layers frozen\n        tf.keras.layers.BatchNormalization(),\n        tf.keras.layers.Dropout(DROPOUT_RATE, name='top_dropout'),\n        tf.keras.layers.Dense(n_classes, activation='softmax', name='class_proba')\n    ]\n)\n\n# Compile the model\nmodel.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=LEARNING_RATE),\n              loss=tf.keras.losses.SparseCategoricalCrossentropy(),\n              metrics=[tf.keras.metrics.SparseCategoricalAccuracy()])\n\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-08-24T12:16:47.021634Z","iopub.execute_input":"2021-08-24T12:16:47.022025Z","iopub.status.idle":"2021-08-24T12:16:51.633139Z","shell.execute_reply.started":"2021-08-24T12:16:47.021993Z","shell.execute_reply":"2021-08-24T12:16:51.631146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model training","metadata":{}},{"cell_type":"code","source":"print('Started training the model with original layers frozen.')\nearly_stop = tf.keras.callbacks.EarlyStopping(\n    monitor='val_loss', patience=PATIENCE, restore_best_weights=True)\n\nhistory = model.fit(train_ds, validation_data=valid_ds,\n                    epochs=EPOCHS, callbacks=[early_stop],\n                    use_multiprocessing=True,\n                    workers=-1)","metadata":{"execution":{"iopub.status.busy":"2021-08-24T12:16:51.635639Z","iopub.execute_input":"2021-08-24T12:16:51.635938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history(history)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Prediction for the test set","metadata":{}},{"cell_type":"code","source":"print(f'Started inference for {n_test_samples} images.')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The number of test images could be much larger and cause memory errors.\n# To avoid it process the test data in chunks of size 5,000.\nstart_idx = 0\nend_idx = 4_999\n\n# To add predicted classes and probabilities for all chunks of images\nchunks = []\n\nwhile start_idx < n_test_samples:\n    print(f'Processing rows: {start_idx} - {end_idx}')\n    test_ds = get_dataset(test_prediction.loc[start_idx:end_idx, 'path'])\n    pred_proba = model.predict(test_ds,\n                               use_multiprocessing=True,\n                               workers=-1)\n\n    cur_chunk = pd.DataFrame()\n    cur_chunk['class_id'] = tf.math.argmax(pred_proba, axis=1)\n    cur_chunk['landmark_id'] = encoder.inverse_transform(cur_chunk['class_id'])\n    cur_chunk['proba'] = np.max(pred_proba, axis=1)\n\n    chunks.append(cur_chunk)\n\n    start_idx += 5_000\n    end_idx += 5_000\n\n    if end_idx >= n_test_samples:\n        end_idx = n_test_samples - 1\n\n    gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Concatenate all chunks and add them to the combined dataframe\ntest_prediction = pd.concat(\n    [\n        test_prediction,\n        pd.concat(chunks, ignore_index=True)\n    ],\n    axis=1\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Sort all collected data by probabilities\ntest_prediction.sort_values(by='proba', ascending=False, inplace=True)\n\n# Combine predicted class and probability into a single string\ntest_prediction['landmark_id'] = test_prediction['landmark_id'].astype('str')\ntest_prediction['landmarks'] = test_prediction[['landmark_id', 'proba']].values.tolist()\ntest_prediction['landmarks'] = test_prediction['landmarks'].apply(lambda x: f'{x[0]} {round(x[1], 3)}')\ntest_prediction.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save results to file\ntest_prediction[['id', 'landmarks']].to_csv('submission.csv', index=False)\ntest_prediction[['id', 'landmarks']].head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}