{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Set up the notebook with imports and constants"},{"metadata":{},"cell_type":"markdown","source":"Install downloaded efficientnet package. Will work without internet access in the notebook"},{"metadata":{"trusted":true},"cell_type":"code","source":"#pip download efficientnet -d ./efficientnet\n#import os\n#from zipfile import ZipFile\n#\n#dirName = \"./\"\n#zipName = \"packages.zip\"\n\n## Create a ZipFile Object\n#with ZipFile(zipName, 'w') as zipObj:\n#    # Iterate over all the files in directory\n#    for folderName, subfolders, filenames in os.walk(dirName):\n#        for filename in filenames:\n#            if (filename != zipName):\n#                # create complete filepath of file in directory\n#                filePath = os.path.join(folderName, filename)\n#                # Add file to zip\n#                zipObj.write(filePath)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"! pip install efficientnet --no-index --find-links=file:///kaggle/input/vgis9-2020-packages/efficientnet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"! [ -f /kaggle/input/vgis2020model/bestmodel.h5 ] && cp /kaggle/input/vgis2020model/bestmodel.h5 /kaggle/working/bestmodel.h5","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport sys\nfrom pathlib import Path\nimport random\nimport pickle\n\n%matplotlib inline\nimport matplotlib.pyplot as plt\nfrom matplotlib.image import imread\nimport cv2\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n\nimport tensorflow as tf\nfrom keras_preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.callbacks import ModelCheckpoint\nimport efficientnet.tfkeras as efn\n\nfrom tqdm import tqdm\n\ninput_dir = Path('../input')\ndataset_dir = input_dir / 'landmark-recognition-2020'\n\ntest_image_dir = dataset_dir / 'test'\ntrain_image_dir = dataset_dir / 'train'\ntrain_label_path = dataset_dir / 'train.csv'\nbestmodel_path = Path('/kaggle/working/bestmodel.h5')\n    \nERROR = 1\nWARN = 2\nINFO = 3\nDEBUG = 4\nSPAM = 5\n\nVERBOSITY = INFO\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Variables"},{"metadata":{"trusted":true},"cell_type":"code","source":"validation_ratio = 0.2\nbatch_size = 16\nmax_epochs = 6\n\ntop_n = 1000\nimg_size = (256,256)\nseed = 496\n\nforce_retrain = False","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Setting up some helper functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_img_path(df, prepend=\"\"):\n    return prepend + df.id.str[0] + \"/\" + df.id.str[1] + \"/\" + df.id.str[2] + \"/\" + df.id + \".jpg\" ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_history(history):\n    loss_list = [s for s in history.history.keys() if 'loss' in s and 'val' not in s]\n    val_loss_list = [s for s in history.history.keys() if 'loss' in s and 'val' in s]\n    acc_list = [s for s in history.history.keys() if 'acc' in s and 'val' not in s]\n    val_acc_list = [s for s in history.history.keys() if 'acc' in s and 'val' in s]\n    \n    if len(loss_list) == 0:\n        print('Loss is missing in history')\n        return \n    \n    ## As loss always exists\n    epochs = range(1,len(history.history[loss_list[0]]) + 1)\n    \n    ## Loss\n    plt.figure(1)\n    for l in loss_list:\n        plt.plot(epochs, history.history[l], 'b', label='Training loss (' + str(str(format(history.history[l][-1],'.5f'))+')'))\n    for l in val_loss_list:\n        plt.plot(epochs, history.history[l], 'g', label='Validation loss (' + str(str(format(history.history[l][-1],'.5f'))+')'))\n    \n    plt.title('Loss')\n    plt.xlabel('Epochs')\n    plt.ylabel('Loss')\n    plt.legend()\n    \n    ## Accuracy\n    plt.figure(2)\n    for l in acc_list:\n        plt.plot(epochs, history.history[l], 'b', label='Training accuracy (' + str(format(history.history[l][-1],'.5f'))+')')\n    for l in val_acc_list:    \n        plt.plot(epochs, history.history[l], 'g', label='Validation accuracy (' + str(format(history.history[l][-1],'.5f'))+')')\n\n    plt.title('Accuracy')\n    plt.xlabel('Epochs')\n    plt.ylabel('Accuracy')\n    plt.legend()\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data loading / pre-processing"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_labels = pd.read_csv(train_label_path)\ntrain_labels.head(5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data exploration"},{"metadata":{"trusted":true},"cell_type":"code","source":"def check_for_test():\n    testdf = pd.read_csv('../input/landmark-recognition-2020/sample_submission.csv')\n    test_images  = test_image_dir.glob(\"**/*.jpg\")\n\n    test_img_arr = []\n    for img in test_images:\n        test_img_arr.append(img.stem)\n    \n    x = True\n    for _id in testdf.id.values:\n        if _id not in test_img_arr:\n            x = False\n            print(f\"{_id} missing from folder\")\n\n    for img in test_img_arr:\n        if img not in testdf.id.values:\n            x = False\n            print(f\"{_img} missing from csv\")\n    return x\n\n# x = \"are\" if check_for_test() else \"aren't\"\n# print(f\"All test images {x} listed in sample_submission.csv\")\n## All test images are listed in sample.csv. Will use that","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class_count = len(train_labels[\"landmark_id\"].unique())\ntest_df = pd.read_csv(dataset_dir/\"sample_submission.csv\")\n\ntest_image_count = len(test_df.id.values)\ntrain_image_count = len(train_labels.id.values)\n\nprint(f'''Dataset info:\n      \\tUnique classes: {class_count:}\n      \\tImages  : {test_image_count + train_image_count :9,d}\n      \\t  test  : {test_image_count :9,d}\n      \\t  train : {train_image_count :9,d}\n      ''')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Make a dataframe sorted by amount of images \ndf_by_samples = pd.DataFrame(train_labels['landmark_id'].value_counts())\ndf_by_samples.reset_index(inplace=True)\ndf_by_samples.columns=['landmark_id','count']\n\n\nlt_5_cnt = len(df_by_samples.loc[df_by_samples['count'] < 5])\ngt_5_lt_10_cnt = len(df_by_samples.loc[(df_by_samples['count'] > 5) & (df_by_samples['count'] < 10)])\nlt_100_cnt = len(df_by_samples.loc[df_by_samples['count'] < 100]) \nprint(f\"\"\"Classes with:\n    <5 samples   : {lt_5_cnt}\n    >5<10 samples: {gt_5_lt_10_cnt}\n    <500 samples : {lt_100_cnt}\"\"\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Plotting a bar graph \"histogram\""},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_bars(data, edges, col=None):\n\n    if col is None:\n        col = data\n    else:\n        col = data[col]\n\n    bins = {}\n    for idx in range(len(edges)-1):\n        if idx == len(edges)-2:\n            key = f\">{edges[idx]}\"\n        else:\n            key = f\">{edges[idx]} <={edges[idx+1]}\"\n        bins[key] = len(data.loc[(col > edges[idx]) & (col <= edges[idx+1])])\n\n    \n    fig = plt.figure(figsize=(10,3.5))\n    \n    plt.bar(bins.keys(), bins.values(), width=0.4)\n\n    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_bars(df_by_samples, [0,5,10,50,100,7000], 'count')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Plotting random classes"},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_n_img(dataset, n :int, drop_dupes=True, title=None):\n    \n    if drop_dupes:\n        ids = dataset.drop_duplicates(subset=['landmark_id']).sample(n)\n    \n    else:\n        ids = dataset.sample(n)\n    \n    paths = get_img_path(ids, str(train_image_dir.resolve())+'/').values\n    grid_size = int(np.ceil(np.sqrt(len(paths))))\n    \n    fig = plt.figure(figsize=(grid_size*3,grid_size*3))\n    \n    axes = []\n    for idx in range(grid_size*grid_size):\n        if idx == n:\n            break\n        axes.append(fig.add_subplot(grid_size, grid_size, idx+1))\n        plt.imshow(imread(paths[idx]))\n        if title is not None:\n            plt.title(title)\n    \n    fig.tight_layout()\n    plt.show()\n\ndef plot_img_from_class(dataset, class_id :int, n :int):\n    \"\"\"Plots n images from a given class    \n    \"\"\"\n    class_subset = dataset.loc[dataset['landmark_id'] == class_id]\n    \n    plot_n_img(class_subset, n, False, str(class_id))\n    \n    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_n_img(train_labels, 16)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Because there's so many classes with few samples, which could cause an issue for training, we'll take a subset of the dataset, using only the top 1000 classes. "},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ndf_by_samples = df_by_samples.drop(df_by_samples.index[top_n:])\nfull_train = train_labels.copy() # Make copy for later testing\ntrain_labels = train_labels[train_labels.landmark_id.isin(df_by_samples['landmark_id'])]\nprint(df_by_samples.tail(1))\nprint(train_labels.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"As can be seen, taking the top classes results in classes having at least 59 samples per class, while still leaving us with over half a million images"},{"metadata":{},"cell_type":"markdown","source":"## Split data into training and validation sets"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_labels['path'] = get_img_path(train_labels)\ntrain_labels['label'] = train_labels.landmark_id.astype(str)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_genny(data, x_col, y_col, base_dir :str, target_size=(256,256), batch_size=32, validation_ratio=0.0, subset=None, seed=496):\n    gen = ImageDataGenerator(validation_split=validation_ratio)\n    #gen = ImageDataGenerator(validation_split=validation_ratio, horizontal_flip=True)  # Introduce random flips\n    #gen = ImageDataGenerator(validation_split=validation_ratio, zoom_range=0.1)  # 25% random zoom\n    \n    class_mode = \"categorical\" if validation_ratio > 0 else None\n    \n    genny = gen.flow_from_dataframe(\n        data,\n        directory = base_dir,\n        x_col=x_col,\n        y_col=y_col,\n        target_size=target_size,\n        batch_size=batch_size,\n        subset=subset,\n        class_mode=class_mode,\n        validate_filenames=False,\n        seed=seed\n    )\n    return genny","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# The flow_from_dataframe() shuffles the data after splitting it, meaning the training and validation set will contain different classes, so we shuffle the data before\ntrain_labels = train_labels.sample(frac=1, random_state=seed).reset_index(drop=True)\n\ntrain_gen = get_genny(train_labels, \"path\", \"label\", str(train_image_dir), img_size, batch_size, validation_ratio, \"training\")\nvalid_gen = get_genny(train_labels, \"path\", \"label\", str(train_image_dir), img_size, batch_size, validation_ratio, \"validation\")\n\n\n\nprint(f\"Split training set into a training and validation set\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if not bestmodel_path.exists() or force_retrain:\n    model = tf.keras.Sequential([\n        efn.EfficientNetB2(\n            input_shape=(256, 256, 3),\n            weights='imagenet',\n            include_top=False\n        ),\n        tf.keras.layers.GlobalAveragePooling2D(),\n        tf.keras.layers.Dense(top_n, activation='softmax')\n    ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if not bestmodel_path.exists() or force_retrain:\n    modified_adam = tf.keras.optimizers.Adam(learning_rate=0.005)\n    normal_adam = tf.keras.optimizers.Adam()\n    model.compile(\n        #optimizer=modified_adam,\n        optimizer=normal_adam,\n        loss = 'categorical_crossentropy',\n        metrics = ['categorical_accuracy']\n    )\n    # I'm using the adam optimizer for a few reasons. It's very popular, and that tends to be for a reason, and it attempts to combine the best of both wordls of momentum and RMSProp\n    # I'm using categorical_crossentropy as there's a lot of classes\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_count = len(train_labels)\n\ntrain_steps = int(image_count * (1-validation_ratio) // batch_size)\nvalid_steps = int(image_count * validation_ratio // batch_size)\n\nif not bestmodel_path.exists() or force_retrain:\n    print(f\"Fitting model over {max_epochs} epochs with {train_steps} training steps and {valid_steps} validation steps.\")\n    \n    model_checkpoint = ModelCheckpoint(\"bestmodel.h5\", save_best_only=True, verbose=1)\n\n    hist = model.fit(train_gen,\n                    steps_per_epoch=train_steps,\n                    epochs=max_epochs,\n                    validation_data=valid_gen,\n                    validation_steps=valid_steps,\n                    callbacks=[model_checkpoint]\n    )\n    plot_history(hist)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Checking the classifier on validation set"},{"metadata":{"trusted":true},"cell_type":"code","source":"def one_hot_to_labels(pred, class_map=None):\n    \"\"\"Convert from one-hot to predictions to labels with probability\"\"\"\n    \n    pred_idx = np.argmax(pred, axis=-1) # Get the index of the one-hot bit in the last axis\n\n    if class_map is None:\n        class_map = np.unique(train_labels.landmark_id.values)\n    \n    pred_labels = [class_map[idx] for idx in pred_idx]\n    pred_prob = np.max(pred, axis=-1)\n    \n    return pred_labels, pred_prob\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_model = tf.keras.models.load_model(\"bestmodel.h5\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Get a general evaluation of the trained models performance on the validation set"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_gen = get_genny(train_labels, \"path\", \"label\", str(train_image_dir), img_size, 1, validation_ratio, \"validation\") # Validation set but with batch-size 1\n#scores = best_model.evaluate(x=test_gen)\n#print(f\"Validation set classifies with a loss of: {scores[0]} and a categorical_accuracy of {scores[1]}]\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Get predictions on the validation set to allow more exploration of the results"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_gen.reset()\n\nclass_map = {idx: name for name, idx in test_gen.class_indices.items()} # Flip the mapping to get the names from idx\n\n\nresults_pickle = Path('../input/vgis2020-pickles/results.p')\nquick_run = False\n\n\nif results_pickle.is_file() and quick_run:\n    with results_pickle.open('rb') as f:\n        results = pickle.load(f)\nelse:\n    results = []\n    for step in tqdm(range(len(test_gen))):\n        X, y = next(test_gen)\n        pred = best_model.predict(X)\n    \n        pred_idx = np.argmax(pred)\n        true_idx = np.argmax(y)\n        pred_prob = np.max(pred)\n    \n        results.append([class_map[true_idx], class_map[pred_idx], pred_prob])\n\n    with open('results.p', 'wb') as f:\n        pickle.dump(results, f)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"errs = [x for x in results if x[0] != x[1]]\nerrs = pd.DataFrame(errs, columns = ['target', 'predicted', 'probability'])\n\nprint(f\"Testing on the validation set gives {(len(errs) / len(results)) * 100:0.2f}% incorrectly classified landmarks\")\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results_ranked = pd.DataFrame(results, columns = ['target', 'predicted', 'probability'])\n\nresults_ranked = results_ranked['target'].value_counts().to_frame()\nresults_ranked.reset_index(level=0, inplace=True)\nresults_ranked.columns = ['class', 'count']\n\n\nclass_err = pd.DataFrame(errs, columns = ['target', 'predicted', 'probability'])\n\nclass_err = class_err['target'].value_counts().to_frame()\nclass_err.reset_index(level=0, inplace=True)\nclass_err.columns = ['class','count']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"The top 5 worst classified classes were {class_err.head(5).iloc[:,0].values} with {class_err.head(5).iloc[:,1].values} misclassifications respectively\")\nprint(f\"A few pictures from the worst prediced class {class_err.iloc[0,0]} have been plotted, as well as some from classes it was mistaken as\")\n\nworst_class = class_err.iloc[0,0]\nmistaken_as = errs.loc[errs['target'] == class_err.iloc[0,0]]['predicted'].drop_duplicates().sample(3).values\n\n\nplot_img_from_class(train_labels, int(worst_class), 2)\n\nfor mistake in mistaken_as:\n    plot_img_from_class(train_labels, int(mistake), 1)\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> "},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"The top 5 correctly classified classes were {results_ranked.head(5).iloc[:,0].values} with {results_ranked.head(5).iloc[:,1].values} classifications respectively\")\nprint(f\"A few pictures from the classes have been plotted\")\n\n\ntop_classes_list = results_ranked.head(5).iloc[:,0].values\n\nfor class_id in top_classes_list:\n    plot_img_from_class(train_labels, int(class_id), 2)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submission Generation\n\nHere we will run the test images through the trained model and generate a submission.csv\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df = pd.read_csv(dataset_dir / \"sample_submission.csv\")\nsub_df[\"path\"] = get_img_path(sub_df)\n\ntest_gen = get_genny(sub_df, \"path\", None, str(test_image_dir), img_size, 1)\npredictions = best_model.predict(test_gen, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Convert from one hot encoding back to categorical labels with probablities"},{"metadata":{"trusted":true},"cell_type":"code","source":"predicted_labels, prediction_prob = one_hot_to_labels(predictions)\npredicted_labels = np.argmax(predictions, axis=-1) # Get the index of the one-hot bit in the last axis\n\nclasses = np.unique(train_labels.landmark_id.values)\nprint(classes.shape)\nprint(predicted_labels.shape)\n\npredicted_labels = [classes[idx] for idx in predicted_labels] \nprediction_prob = np.max(predictions, axis=-1)\n\nprint(f\"{predicted_labels[0]}: {prediction_prob[0]}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Save predictions as submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"result = [str(predicted_labels[idx]) + \" \" + str(prediction_prob[idx]) for idx in range(len(predicted_labels))]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df[\"landmarks\"] = result\nsub_df.drop(columns=\"path\")\n\nsub_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}