{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport cv2\n#import tensorflow as tf\nimport matplotlib.pyplot as plt\nfrom keras.preprocessing.image import ImageDataGenerator\n\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\n# Data parameter\ninput_dir = os.path.join('..', 'input')\noutput_dir = os.path.join('..', 'output')\n\ndataset_dir = os.path.join(input_dir, 'landmark-recognition-2020')\ntrain_dir = os.path.join(dataset_dir, 'train')\ntrain_labelmap_dir = os.path.join(dataset_dir, 'train.csv')\ntest_dir = os.path.join(dataset_dir, 'test')\n\n\ntrain_df = pd.read_csv(train_labelmap_dir)\nnum_data = len(train_df)\n\nprint(\"Shape of train_data :\", train_df.shape)\nload_pretrain_model = True\nversion_to_load = '23'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### helper function "},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_history(history):\n    loss_list = [s for s in history.history.keys() if 'loss' in s and 'val' not in s]\n    val_loss_list = [s for s in history.history.keys() if 'loss' in s and 'val' in s]\n    acc_list = [s for s in history.history.keys() if 'acc' in s and 'val' not in s]\n    val_acc_list = [s for s in history.history.keys() if 'acc' in s and 'val' in s]\n    \n    if len(loss_list) == 0:\n        print('Loss is missing in history')\n        return \n    \n    ## As loss always exists\n    epochs = range(1,len(history.history[loss_list[0]]) + 1)\n    \n    ## Loss\n    plt.figure(1)\n    for l in loss_list:\n        plt.plot(epochs, history.history[l], 'b', label='Training loss (' + str(str(format(history.history[l][-1],'.5f'))+')'))\n    for l in val_loss_list:\n        plt.plot(epochs, history.history[l], 'g', label='Validation loss (' + str(str(format(history.history[l][-1],'.5f'))+')'))\n    \n    plt.title('Loss')\n    plt.xlabel('Epochs')\n    plt.ylabel('Loss')\n    plt.legend()\n    \n    ## Accuracy\n    plt.figure(2)\n    for l in acc_list:\n        plt.plot(epochs, history.history[l], 'b', label='Training accuracy (' + str(format(history.history[l][-1],'.5f'))+')')\n    for l in val_acc_list:    \n        plt.plot(epochs, history.history[l], 'g', label='Validation accuracy (' + str(format(history.history[l][-1],'.5f'))+')')\n\n    plt.title('Accuracy')\n    plt.xlabel('Epochs')\n    plt.ylabel('Accuracy')\n    plt.legend()\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"print('Number of classes:')\nlandmark = train_df.landmark_id.value_counts()\nprint(landmark.size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"landmark_df = pd.DataFrame({'landmark_id':landmark.index, 'frequency':landmark.values})#.head(30)\n\nlandmark_df.reset_index(inplace=True)\n\nprint(landmark_df)\n\nprint(landmark_df['frequency'].describe())\n\nplt.hist(landmark_df['frequency'], 100, range = (0, 950), label = 'test')\nplt.xlabel(\"Amount of images\")\nplt.ylabel(\"Occurences\")\n\nprint(\"Amount of classes with less than 5 trainning samples:\", (landmark_df['frequency'].between(0,4)).sum())\nprint(\"Amount of classes with between 5 and 10 training samples:\", (landmark_df['frequency'].between(5,10)).sum())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(landmark_df[0:10])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visualize different images"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nlencoder = LabelEncoder()\nlencoder.fit(train_df[\"landmark_id\"])\n\ndef encode_label(lbl):\n    return lencoder.transform(lbl)\n    \ndef decode_label(lbl):\n    return lencoder.inverse_transform(lbl)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Visualize random images from the dataset\n\ndef get_image_from_number(num):\n    fname, label = train_df.loc[num,:]\n    fname = fname + \".jpg\"\n    f1 = fname[0]\n    f2 = fname[1]\n    f3 = fname[2]\n    path = os.path.join(f1,f2,f3,fname)\n    im = cv2.imread(os.path.join(train_dir,path))\n    return im, label\n\n\nfig=plt.figure(figsize=(16, 8))\n\ncolumns = 4\nrows = 2\nfor i in range(1, columns*rows +1):\n    n = np.random.randint(num_data)\n    img, lbl = get_image_from_number(n)\n    fig.add_subplot(rows, columns, i)\n    plt.imshow(img)\n    plt.title(\"Label = \" + str(lbl))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[\"filename\"] = train_df.id.str[0]+\"/\"+train_df.id.str[1]+\"/\"+train_df.id.str[2]+\"/\"+train_df.id+\".jpg\"\ntrain_df[\"label\"] = train_df.landmark_id.astype(str)\nprint(train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import Counter\n\nno_classes_keep = 1000\n\nc = train_df.landmark_id.values\ncount = Counter(c).most_common(no_classes_keep)\nprint(len(count), count[-1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"seed_value = 131\nmax_sample_per_class = 1000\nkeep_labels = [i[0] for i in count]\nprint(len(keep_labels))\ntrain_keep = pd.DataFrame()#columns=['id', 'landmark_id', 'filename', 'label'])\nfor label in keep_labels:\n    if len(train_df[train_df.landmark_id.isin([label])]) < max_sample_per_class:\n        max_sample_per_class = len(train_df[train_df.landmark_id.isin([label])])\n    train_keep_label = train_df[train_df.landmark_id.isin([label])].sample(n=max_sample_per_class)\n    #train_keep_label.remove()\n    #print(len(train_keep_label))\n    train_keep = train_keep.append(train_keep_label, ignore_index=True)\n#train_keep = train_df[train_df.landmark_id.isin(keep_labels)]\n#train_keep\nprint(len(train_keep))\n#train_keep.hist()\n\n#shuffle the rows of train_keep\ntrain_keep = train_keep.sample(frac=1, random_state=seed_value).reset_index(drop=True)\nprint(train_keep)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_split = 0.2\nbatch_size = 50\n\ndatagen = ImageDataGenerator(validation_split=val_split, horizontal_flip=True)# zoom_range=0.15)\n\ntrain_datagen = datagen.flow_from_dataframe(\n    train_keep, # Pandas dataframe containing the filepaths relative to directory (or absolute paths if directory is None) and classes label\n    directory=train_dir + \"/\",\n    x_col=\"filename\",\n    y_col=\"label\",\n    weight_col=None,\n    target_size=(224, 224),\n    color_mode=\"rgb\",\n    classes=None,\n    class_mode=\"categorical\",\n    batch_size=batch_size,\n    shuffle=True,\n    seed=seed_value,\n    subset=\"training\",\n    interpolation=\"nearest\",\n    validate_filenames=False)\n\nval_datagen = datagen.flow_from_dataframe(\n    train_keep, # Pandas dataframe containing the filepaths relative to directory (or absolute paths if directory is None) and classes label\n    directory=train_dir + \"/\",\n    x_col=\"filename\",\n    y_col=\"label\",\n    weight_col=None,\n    target_size=(224, 224),\n    color_mode=\"rgb\",\n    classes=None,\n    class_mode=\"categorical\",\n    batch_size=batch_size,\n    shuffle=False,\n    seed=seed_value,\n    subset=\"validation\",\n    interpolation=\"nearest\",\n    validate_filenames=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training model"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.applications import VGG19\nfrom tensorflow.keras.applications import EfficientNetB2\nfrom keras.layers import *\nfrom keras import Sequential\nfrom keras.callbacks import ModelCheckpoint\nfrom keras.models import load_model\nfrom keras.optimizers import Adam","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if load_pretrain_model:\n    model_path = '../input/rvgis-model-version-' + version_to_load + '/model_efficientnetb2.h5'\n    model = load_model(model_path)\n    model.summary()\n    model.save(\"model_efficientnetb2.h5\")\nelse:\n    vgg_model = VGG19(input_shape=(224, 224, 3), weights='imagenet', include_top=False)\n    effNetB2_model = EfficientNetB2(input_shape=(224, 224, 3), weights='imagenet', include_top=False)\n\n    #effNetB2_model.summary()\n\n    model = Sequential([\n        EfficientNetB2(\n        input_shape=(224, 224, 3),\n            weights='imagenet',\n            include_top=False\n        ),\n        GlobalAveragePooling2D(),\n    #    Flatten(),\n    #    Dense(4096, activation='relu'),\n    #    Dense(4096, activation='relu'),\n        Dense(no_classes_keep, activation='softmax')\n    ])\n\n    #model = Sequential()\n\n    #for layer in vgg_model.layers:\n        #if layer == vgg_model.layers[-21]:\n        #    model.add(BatchNormalization())\n    #    model.add(layer)\n    #model.add(GlobalAveragePooling2D())\n    #model.add(Flatten())\n    #model.add(Dense(4096, activation = \"relu\"))\n    #model.add(Dense(4096, activation = \"relu\"))\n    #model.add(Dense(2048, activation = \"relu\"))\n    #model.add(Dense(1024, activation = \"relu\"))\n    #model.add(Dense(512, activation = \"relu\"))\n    #model.add(Dense(256, activation = \"relu\"))\n    #model.add(Dropout(0.5))\n    #model.add(Dense(no_classes_keep, activation=\"softmax\"))\n\n    #for layer in model.layers[19:]:\n    #    layer.trainable = False\n    modified_adam = Adam(learning_rate=0.001)\n\n    model.summary()\n\n    model.compile(\n        optimizer='adam',\n        #optimizer=modified_adam,\n        loss = 'categorical_crossentropy',\n        metrics=['categorical_accuracy']\n    )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train model"},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs = 5\n\ntrain_steps = int(len(train_keep)*(1-val_split))//batch_size\nval_steps = int(len(train_keep)*val_split)//batch_size","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if not load_pretrain_model:\n    model_checkpoint = ModelCheckpoint(\"model_efficientnetb2.h5\", save_best_only=True, verbose=1)\n    history = model.fit(train_datagen, steps_per_epoch=train_steps, epochs=epochs,validation_data=val_datagen, validation_steps=val_steps, callbacks=[model_checkpoint])\n\n    model.save(\"model.h5\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if not load_pretrain_model:\n    plot_history(history)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Validate model"},{"metadata":{"trusted":true},"cell_type":"code","source":"predict = model.predict(val_datagen, val_steps)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(predict)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"good_preds = []\nbad_preds = []\n\nval_filenames = val_datagen.filenames\nlabel_map = (val_datagen.class_indices)\n#label_categories = to_categorical(np.asarray(labels)) \ncla = np.argmax(predict, axis=-1)\nlabel_map = list(map(int, label_map.keys()))\nval_label = val_datagen.labels\n\nfor idx, res in enumerate(predict):\n    #print(\"image_id: \", val_filenames[idx], \", class predict: \", label_map[cla[idx]], \"class: \", label_map[val_label[idx]])\n    \n    if label_map[cla[idx]] != label_map[val_label[idx]]:\n        bad_preds.append([val_filenames[idx], label_map[cla[idx]], label_map[val_label[idx]], res[cla[idx]]])\n    else:\n        good_preds.append([val_filenames[idx], label_map[cla[idx]], label_map[val_label[idx]], res[cla[idx]]])\nprint(\"wrong predictions: \", len(bad_preds), \" right predictions: \", len(good_preds), \" acc: \", np.round(100*(len(predict)-len(bad_preds))/len(predict),2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### plot some of the best predictions\n\nfig=plt.figure(figsize=(16, 8))\n\ngood_preds = np.array(good_preds)\ngood_preds = np.array(sorted(good_preds, key = lambda x: x[3], reverse=True))\n#print(good_preds.shape)\n\ncolumns = 4\nrows = 1\nfor i in range(1, columns*rows +1):\n    n = good_preds[i,0]\n    #print(n)\n    img = cv2.imread(os.path.join(train_dir,n))\n    lbl = good_preds[i,2]\n    fig.add_subplot(rows, columns, i)\n    plt.imshow(img)\n    lbl2 = good_preds[i,1]\n    plt.title(\"Label = \" + str(lbl) + \"\\nClassified:\" + str(lbl2) + \"\\nConfidence:\" + str(good_preds[i,3]))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### plot the worst predictions\n\nfig=plt.figure(figsize=(16, 8))\n\nbad_preds = np.array(bad_preds)\nbad_preds = np.array(sorted(bad_preds, key = lambda x: x[3], reverse=True))\n#print(bad_preds.shape)\n\ncolumns = 4\nrows = 1\nfor i in range(1, columns*rows +1):\n    n = bad_preds[i,0]\n    #print(n)\n    img = cv2.imread(os.path.join(train_dir,n))\n    lbl = bad_preds[i,2]\n    fig.add_subplot(rows, columns, i)\n    plt.imshow(img)\n    lbl2 = bad_preds[i,1]\n    plt.title(\"Label = \" + str(lbl) + \"\\nClassified:\" + str(lbl2) + \"\\nConfidence:\" + str(good_preds[i,3]))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Explaining the predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_val = train_keep.landmark_id.value_counts()\ntrain_keep_df = pd.DataFrame({'landmark_id':train_val.index, 'frequency':train_val.values})#.head(30)\ntrain_keep_df.reset_index(inplace=True)\n#print(train_keep_df)\nall_preds = np.concatenate((good_preds, bad_preds), axis=0)\nval_img_per_class = Counter(all_preds[:,2])\nbad_val_labels = Counter(bad_preds[:,2])\ngood_val_labels =  Counter(good_preds[:,2])\n\nprint(\"Top 5 training classes with most data:\")\nfor i in range(5):\n    print(\"label:\", train_keep_df.landmark_id[i], \"has\", train_keep_df.frequency[i], \"instances in training set\" )\n\ntrain_keep_df.set_index(\"landmark_id\", inplace = True)\nprint(\"\\nTop 5 classes with the worst prediction\")\n\nbad_label_pos = 0\nfor i in range(5):\n    if i == 0:\n        label = bad_preds[bad_label_pos, 2]\n        pre_label = label\n        bad_label_pos += 1\n    else:\n        label = bad_preds[bad_label_pos, 2]\n        while pre_label == label and bad_label_pos < len(bad_preds):\n            bad_label_pos += 1\n            label = bad_preds[bad_label_pos, 2]\n        pre_label = label    \n    #label = bad_preds[i, 2]\n    #print(label)\n    label_counts = train_keep_df.loc[int(label)]\n    #print(label_counts)\n    #print(\"label:\", label, \"has\", label_counts[\"frequency\"], \"images in the class and \",  )\n    print(\"label:\", label, \"has\", val_img_per_class[label], \" validation images images in the class \\nand \",  bad_val_labels[label], \" images classified wrong\")\n    \ngood_label_pos = 0 \nprint(\"\\nTop 5 classes with the best prediction\")\nfor i in range(5):\n    if i == 0:\n        label = good_preds[good_label_pos, 2]\n        pre_label = label\n        good_label_pos += 1\n    else:\n        label = good_preds[good_label_pos, 2]\n        while pre_label == label and good_label_pos < len(good_preds):\n            good_label_pos += 1\n            label = good_preds[bad_label_pos, 2]\n        pre_label = label\n    #print(label)\n    label_counts = train_keep_df.loc[int(label)]\n    #print(label_counts)\n    print(\"label:\", label, \"has\", val_img_per_class[label], \" validation images images in the class \\nand \",  good_val_labels[label], \" images classified correct\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Run model on test data"},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv(\"/kaggle/input/landmark-recognition-2020/sample_submission.csv\")\nsub[\"filename\"] = sub.id.str[0]+\"/\"+sub.id.str[1]+\"/\"+sub.id.str[2]+\"/\"+sub.id+\".jpg\"\nsub","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_model = load_model(\"model_efficientnetb2.h5\")\n\ntest_gen = ImageDataGenerator().flow_from_dataframe(\n    sub,\n    directory=\"/kaggle/input/landmark-recognition-2020/test/\",\n    x_col=\"filename\",\n    y_col=None,\n    weight_col=None,\n    target_size=(224, 224),\n    color_mode=\"rgb\",\n    classes=None,\n    class_mode=None,\n    batch_size=1,\n    shuffle=False,\n    subset=None,\n    interpolation=\"nearest\",\n    validate_filenames=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred_one_hot = best_model.predict_generator(test_gen, verbose=1, steps=len(sub))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred = np.argmax(y_pred_one_hot, axis=-1)\ny_prob = np.max(y_pred_one_hot, axis=-1)\nprint(y_pred.shape, y_prob.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_uniq = np.unique(train_keep.landmark_id.values)\n\ny_pred = [y_uniq[Y] for Y in y_pred]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(len(sub)):\n    sub.loc[i, \"landmarks\"] = str(y_pred[i])+\" \"+str(y_prob[i])\nsub = sub.drop(columns=\"filename\")\nsub.to_csv(\"submission.csv\", index=False)\nsub","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}