{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-02T12:56:55.945518Z","iopub.execute_input":"2022-05-02T12:56:55.945805Z","iopub.status.idle":"2022-05-02T12:56:55.950333Z","shell.execute_reply.started":"2022-05-02T12:56:55.945774Z","shell.execute_reply":"2022-05-02T12:56:55.949446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.utils import np_utils\nfrom tensorflow.keras.models import Model, Sequential, load_model\nfrom tensorflow.keras.layers import Dense, Conv2D, MaxPooling2D, Flatten, Reshape, Dropout\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.applications.resnet50 import ResNet50\nfrom sklearn.metrics import confusion_matrix, classification_report\nimport itertools\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:56:55.989007Z","iopub.execute_input":"2022-05-02T12:56:55.98919Z","iopub.status.idle":"2022-05-02T12:57:02.201428Z","shell.execute_reply.started":"2022-05-02T12:56:55.989168Z","shell.execute_reply":"2022-05-02T12:57:02.200705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.applications.vgg16 import VGG16","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:02.202826Z","iopub.execute_input":"2022-05-02T12:57:02.203055Z","iopub.status.idle":"2022-05-02T12:57:02.209037Z","shell.execute_reply.started":"2022-05-02T12:57:02.203023Z","shell.execute_reply":"2022-05-02T12:57:02.208365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.applications.vgg19 import VGG19","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:02.210006Z","iopub.execute_input":"2022-05-02T12:57:02.210348Z","iopub.status.idle":"2022-05-02T12:57:02.232526Z","shell.execute_reply.started":"2022-05-02T12:57:02.210316Z","shell.execute_reply":"2022-05-02T12:57:02.231597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.utils import to_categorical","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:02.234952Z","iopub.execute_input":"2022-05-02T12:57:02.235218Z","iopub.status.idle":"2022-05-02T12:57:02.241345Z","shell.execute_reply.started":"2022-05-02T12:57:02.235186Z","shell.execute_reply":"2022-05-02T12:57:02.240654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Constants\nimg_rows = 224\nimg_cols = 224\ninput_shape = (img_rows,img_cols,3)\nepochs = 10\nbatch_size = 64\n","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:02.243426Z","iopub.execute_input":"2022-05-02T12:57:02.244042Z","iopub.status.idle":"2022-05-02T12:57:02.248874Z","shell.execute_reply.started":"2022-05-02T12:57:02.244006Z","shell.execute_reply":"2022-05-02T12:57:02.248171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get ResNet-50 Model\ndef getResNet50Model(lastFourTrainable=False):\n    resnet_model = ResNet50(weights='imagenet', input_shape=input_shape, include_top=True)\n    # Make all layers non-trainable\n    for layer in resnet_model.layers[:]:\n        layer.trainable = False\n    # Add fully connected layer which have 1024 neuron to ResNet-50 model\n    output = resnet_model.get_layer('avg_pool').output\n    output = Flatten(name='new_flatten')(output)\n    output = Dense(units=1024, activation='relu', name='new_fc')(output)\n    predictions = Dense(units=50, activation='softmax')(output)\n    resnet_model = Model(resnet_model.input, predictions)\n    # Make last 4 layers trainable if lastFourTrainable == True\n    if lastFourTrainable == True:\n        resnet_model.get_layer('conv5_block3_2_bn').trainable = True\n        resnet_model.get_layer('conv5_block3_3_conv').trainable = True\n        resnet_model.get_layer('conv5_block3_3_bn').trainable = True\n        resnet_model.get_layer('new_fc').trainable = True\n    # Compile ResNet-50 model\n    resnet_model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n    resnet_model.summary()\n    return resnet_model","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:02.250309Z","iopub.execute_input":"2022-05-02T12:57:02.250559Z","iopub.status.idle":"2022-05-02T12:57:02.259049Z","shell.execute_reply.started":"2022-05-02T12:57:02.250526Z","shell.execute_reply":"2022-05-02T12:57:02.25823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"traindf = pd.read_csv(\"../input/landmark-recognition-2021/train.csv\")\ntraindf.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:02.260541Z","iopub.execute_input":"2022-05-02T12:57:02.26081Z","iopub.status.idle":"2022-05-02T12:57:03.683276Z","shell.execute_reply.started":"2022-05-02T12:57:02.260777Z","shell.execute_reply":"2022-05-02T12:57:03.68256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"traindf.shape","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:03.684445Z","iopub.execute_input":"2022-05-02T12:57:03.684678Z","iopub.status.idle":"2022-05-02T12:57:03.690606Z","shell.execute_reply.started":"2022-05-02T12:57:03.684646Z","shell.execute_reply":"2022-05-02T12:57:03.689767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"landmark_unique = traindf['landmark_id'].unique()\nlen(landmark_unique)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:03.692514Z","iopub.execute_input":"2022-05-02T12:57:03.692816Z","iopub.status.idle":"2022-05-02T12:57:03.720401Z","shell.execute_reply.started":"2022-05-02T12:57:03.692753Z","shell.execute_reply":"2022-05-02T12:57:03.71982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"landmark_unique[0:50]","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:03.723535Z","iopub.execute_input":"2022-05-02T12:57:03.723714Z","iopub.status.idle":"2022-05-02T12:57:03.729183Z","shell.execute_reply.started":"2022-05-02T12:57:03.72369Z","shell.execute_reply":"2022-05-02T12:57:03.728372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_ids = []\nlabels = []\ntemp_labels = []\ni=0\nfor id_ in landmark_unique[0:50]:\n    for iid in traindf['id'][traindf['landmark_id'] == id_]:\n        image_ids.append(iid)\n        labels.append(id_)\n        temp_labels.append(i)\n    i = i+1\nlen(image_ids)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:03.730648Z","iopub.execute_input":"2022-05-02T12:57:03.731017Z","iopub.status.idle":"2022-05-02T12:57:03.849227Z","shell.execute_reply.started":"2022-05-02T12:57:03.730982Z","shell.execute_reply":"2022-05-02T12:57:03.848511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mainpath = '../input/landmark-recognition-2021/train'\nimage_path = []\nimages_pixels = []\n\nfor i in range(0,len(image_ids)):\n    first_dir = os.path.join(mainpath,image_ids[i][0])\n    second_dir = os.path.join(first_dir,image_ids[i][1])\n    third_dir = os.path.join(second_dir,image_ids[i][2])\n    finalpath = os.path.join(third_dir,image_ids[i]+'.jpg')\n    \n    img_pix = cv2.imread(finalpath,1)\n    images_pixels.append(cv2.resize(img_pix, (224,224)))\n    \n    image_path.append(finalpath)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:03.851942Z","iopub.execute_input":"2022-05-02T12:57:03.852131Z","iopub.status.idle":"2022-05-02T12:57:24.881752Z","shell.execute_reply.started":"2022-05-02T12:57:03.852107Z","shell.execute_reply":"2022-05-02T12:57:24.881008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Images: ', len(image_path))\nprint('Image labels: ', len(labels))","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:24.8832Z","iopub.execute_input":"2022-05-02T12:57:24.883455Z","iopub.status.idle":"2022-05-02T12:57:24.889339Z","shell.execute_reply.started":"2022-05-02T12:57:24.88342Z","shell.execute_reply":"2022-05-02T12:57:24.888584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(temp_labels)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:24.890786Z","iopub.execute_input":"2022-05-02T12:57:24.891359Z","iopub.status.idle":"2022-05-02T12:57:24.898341Z","shell.execute_reply.started":"2022-05-02T12:57:24.891255Z","shell.execute_reply":"2022-05-02T12:57:24.897492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:24.899907Z","iopub.execute_input":"2022-05-02T12:57:24.900225Z","iopub.status.idle":"2022-05-02T12:57:24.906349Z","shell.execute_reply.started":"2022-05-02T12:57:24.900194Z","shell.execute_reply":"2022-05-02T12:57:24.905502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.gcf()\nfig.set_size_inches(16, 16)\n\nnext_pix_ = image_path\n\nfor i, img_path in enumerate(next_pix_[0:16]):\n    \n    sp = plt.subplot(5, 4, i + 1)\n    sp.axis('Off')\n\n    img = cv2.imread(img_path)\n    plt.imshow(img)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:24.907246Z","iopub.execute_input":"2022-05-02T12:57:24.907418Z","iopub.status.idle":"2022-05-02T12:57:26.350521Z","shell.execute_reply.started":"2022-05-02T12:57:24.907397Z","shell.execute_reply":"2022-05-02T12:57:26.349777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.gcf()\nfig.set_size_inches(16, 16)\n\nnext_pix = image_path\nrandom.shuffle(next_pix)\n\nfor i, img_path in enumerate(next_pix[0:12]):\n    \n    sp = plt.subplot(4, 4, i + 1)\n    sp.axis('Off')\n\n    img = cv2.imread(img_path)\n    plt.imshow(img)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:26.351555Z","iopub.execute_input":"2022-05-02T12:57:26.351785Z","iopub.status.idle":"2022-05-02T12:57:27.651145Z","shell.execute_reply.started":"2022-05-02T12:57:26.351754Z","shell.execute_reply":"2022-05-02T12:57:27.650556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shuf = list(zip(images_pixels,temp_labels))\nrandom.shuffle(shuf)\n\ntrain_data, labels_data = zip(*shuf)\nprint('Images: ', len(train_data))\nprint('Image labels: ', len(labels_data))","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:27.652398Z","iopub.execute_input":"2022-05-02T12:57:27.652831Z","iopub.status.idle":"2022-05-02T12:57:27.66095Z","shell.execute_reply.started":"2022-05-02T12:57:27.652795Z","shell.execute_reply":"2022-05-02T12:57:27.660332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(labels_data)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:27.662303Z","iopub.execute_input":"2022-05-02T12:57:27.662806Z","iopub.status.idle":"2022-05-02T12:57:27.670058Z","shell.execute_reply.started":"2022-05-02T12:57:27.662764Z","shell.execute_reply":"2022-05-02T12:57:27.66913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''# Function for plotting the Confusion Matrix\ndef plotConfusionMatrix(cm, class_names, normalize=True, title='Confusion matrix', cmap=plt.cm.Blues):\n    plt.figure(figsize=(10,10))\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(class_names))\n    plt.xticks(tick_marks, class_names, rotation=45)\n    plt.yticks(tick_marks, class_names)\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n        cm = np.around(cm, decimals=2)\n        cm[np.isnan(cm)] = 0.0\n        print(\"Normalized confusion matrix\")\n    else:\n        print('Confusion matrix, without normalization')\n\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, cm[i, j],\n                horizontalalignment=\"center\",\n                color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n    plt.show()\n    '''","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:27.671602Z","iopub.execute_input":"2022-05-02T12:57:27.672194Z","iopub.status.idle":"2022-05-02T12:57:27.680011Z","shell.execute_reply.started":"2022-05-02T12:57:27.672159Z","shell.execute_reply":"2022-05-02T12:57:27.679235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n# Function for train the given model and get Confusion Matrix\ndef trainModelAndGetConfusionMatrix(model,train_generator,valid_generator,test_generator,epochs,batch_size):\n    # Fit the model\n    model.fit_generator(train_generator,\n                      epochs=epochs,\n                      steps_per_epoch=len(train_generator) // batch_size,\n                      validation_data=valid_generator,\n                      validation_steps=len(valid_generator) // batch_size)\n    # Evaluate the model\n    loss_and_metrics = model.evaluate_generator(test_generator, steps=len(test_generator) // batch_size+1)\n    print(\"Test Loss: \", loss_and_metrics[0])\n    print(\"Test Accuracy: \", loss_and_metrics[1])\n    test_generator.reset()\n    # Get Confusion Matrix and plot it\n    Y_pred = model.predict_generator(test_generator, steps=len(test_generator) // batch_size+1)\n    y_pred = np.argmax(Y_pred, axis=1)\n    class_names = []\n    for key in train_generator.class_indices:\n        class_names.append(key)\n    cr = classification_report(test_generator.classes, y_pred, target_names=class_names)\n    print(\"Classification Report\\n\" + cr) # Print classification report\n    cm = confusion_matrix(test_generator.classes, y_pred)\n    plotConfusionMatrix(cm, class_names, normalize=False, title='Confusion Matrix')\n    plotConfusionMatrix(cm, class_names, normalize=True, title='Confusion Matrix')\n    return model\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:27.681587Z","iopub.execute_input":"2022-05-02T12:57:27.682126Z","iopub.status.idle":"2022-05-02T12:57:27.692017Z","shell.execute_reply.started":"2022-05-02T12:57:27.682091Z","shell.execute_reply":"2022-05-02T12:57:27.691261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = np.array(train_data) #/ 255\n","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:27.693643Z","iopub.execute_input":"2022-05-02T12:57:27.694303Z","iopub.status.idle":"2022-05-02T12:57:27.752251Z","shell.execute_reply.started":"2022-05-02T12:57:27.694268Z","shell.execute_reply":"2022-05-02T12:57:27.751503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a = np.array(labels_data)\na_pd = pd.get_dummies(a).astype('float32').values ","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:27.754344Z","iopub.execute_input":"2022-05-02T12:57:27.754867Z","iopub.status.idle":"2022-05-02T12:57:27.764906Z","shell.execute_reply.started":"2022-05-02T12:57:27.75483Z","shell.execute_reply":"2022-05-02T12:57:27.764246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(labels_data[0:6])","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:27.76644Z","iopub.execute_input":"2022-05-02T12:57:27.766681Z","iopub.status.idle":"2022-05-02T12:57:27.771031Z","shell.execute_reply.started":"2022-05-02T12:57:27.766648Z","shell.execute_reply":"2022-05-02T12:57:27.770193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(a_pd)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:27.772406Z","iopub.execute_input":"2022-05-02T12:57:27.772701Z","iopub.status.idle":"2022-05-02T12:57:27.78083Z","shell.execute_reply.started":"2022-05-02T12:57:27.772667Z","shell.execute_reply":"2022-05-02T12:57:27.779943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_data=a_pd","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:27.782262Z","iopub.execute_input":"2022-05-02T12:57:27.782519Z","iopub.status.idle":"2022-05-02T12:57:27.787263Z","shell.execute_reply.started":"2022-05-02T12:57:27.782487Z","shell.execute_reply":"2022-05-02T12:57:27.786444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(labels_data[0:5])","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:27.788707Z","iopub.execute_input":"2022-05-02T12:57:27.788981Z","iopub.status.idle":"2022-05-02T12:57:27.799988Z","shell.execute_reply.started":"2022-05-02T12:57:27.788947Z","shell.execute_reply":"2022-05-02T12:57:27.799217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_data.shape","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:27.806612Z","iopub.execute_input":"2022-05-02T12:57:27.806877Z","iopub.status.idle":"2022-05-02T12:57:27.811936Z","shell.execute_reply.started":"2022-05-02T12:57:27.80685Z","shell.execute_reply":"2022-05-02T12:57:27.811147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_train, X_test, Y_train, Y_test = train_test_split(train_data, labels_data, test_size = 0.3, random_state=101)\n\nprint(\"X train data : \", len(X_train))\nprint(\"X label data : \", len(X_test))\nprint(\"Y test data : \", len(Y_train))\nprint(\"Y label data : \", len(Y_test))","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:27.813636Z","iopub.execute_input":"2022-05-02T12:57:27.81413Z","iopub.status.idle":"2022-05-02T12:57:27.887044Z","shell.execute_reply.started":"2022-05-02T12:57:27.814092Z","shell.execute_reply":"2022-05-02T12:57:27.885541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_sample=X_train.shape[0]\n#20% of the enteries has to be in validation set\nvalidation_freq=int(num_sample*0.2)\n#Generating random sample of indices equal to validation_freq\nvalidationlist = random.sample(range(0, num_sample), validation_freq)\ntraininglist=list(set(range(0,num_sample))-set(validationlist))\nprint(\"No interesection between validationlist and traininglist:\",set(traininglist).intersection(validationlist))","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:27.888986Z","iopub.execute_input":"2022-05-02T12:57:27.889243Z","iopub.status.idle":"2022-05-02T12:57:27.896821Z","shell.execute_reply.started":"2022-05-02T12:57:27.889209Z","shell.execute_reply":"2022-05-02T12:57:27.896068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_x_final=[]\ntrain_y_final=[]\nfor val in traininglist:\n    train_x_final.append(X_train[val])\n    train_y_final.append(Y_train[val])\ntrain_x_final=np.array(train_x_final)\ntrain_y_final=np.array(train_y_final)\nprint(\"Training data shape\",train_x_final.shape)\nprint(\"Training label shape\",train_y_final.shape)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:27.898431Z","iopub.execute_input":"2022-05-02T12:57:27.898941Z","iopub.status.idle":"2022-05-02T12:57:27.938161Z","shell.execute_reply.started":"2022-05-02T12:57:27.898907Z","shell.execute_reply":"2022-05-02T12:57:27.937314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation_x_final=[]\nvalidation_y_final=[]\nfor val in validationlist:\n    validation_x_final.append(X_train[val])\n    validation_y_final.append(Y_train[val])\nvalidation_x_final=np.array(validation_x_final)\nvalidation_y_final=np.array(validation_y_final)\nprint(\"Validation data shape\",validation_x_final.shape)\nprint(\"Validation label shape\",validation_y_final.shape)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:27.939664Z","iopub.execute_input":"2022-05-02T12:57:27.939939Z","iopub.status.idle":"2022-05-02T12:57:27.956272Z","shell.execute_reply.started":"2022-05-02T12:57:27.9399Z","shell.execute_reply":"2022-05-02T12:57:27.95535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot the training results\ndef plot_hist(history,title):\n    plt.subplot(121)\n    plt.plot(history.history['accuracy'])\n    plt.plot(history.history['val_accuracy'])\n    plt.title(str(title)+' accuracy')\n    plt.ylabel('Accuracy')\n    plt.xlabel('Epoch')\n    plt.legend(['Train', 'Test'], loc='upper left')\n\n    # Plot training & validation loss values\n    plt.subplot(122)\n    plt.plot(history.history['loss'])\n    plt.plot(history.history['val_loss'])\n    plt.title(str(title)+' loss')\n    plt.ylabel('Loss')\n    plt.xlabel('Epoch')\n    plt.legend(['Train', 'Test'], loc='upper left')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:27.957573Z","iopub.execute_input":"2022-05-02T12:57:27.958021Z","iopub.status.idle":"2022-05-02T12:57:27.965228Z","shell.execute_reply.started":"2022-05-02T12:57:27.957984Z","shell.execute_reply":"2022-05-02T12:57:27.964411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get VGG-16 Model\ndef getVGG16Model(lastFourTrainable=False):\n    vgg_model = VGG16(weights='imagenet', input_shape=input_shape, include_top=True)\n    # Make all layers untrainable\n    for layer in vgg_model.layers[:]:\n        layer.trainable = False\n    # Add fully connected layer which have 1024 neuron to VGG-16 model\n    output = vgg_model.get_layer('fc2').output\n    output = Flatten(name='new_flatten')(output)\n    output = Dense(units=1024, activation='relu', name='new_fc')(output)\n    output = Dense(units=50, activation='softmax')(output)\n    vgg_model = Model(vgg_model.input, output)\n    # Make last 4 layers trainable if lastFourTrainable == True\n    if lastFourTrainable == True:\n        vgg_model.get_layer('block5_conv3').trainable = True\n        vgg_model.get_layer('fc1').trainable = True\n        vgg_model.get_layer('fc2').trainable = True\n        vgg_model.get_layer('new_fc').trainable = True\n    # Compile VGG-16 model\n    vgg_model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n    vgg_model.summary()\n\n    return vgg_model","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:27.966768Z","iopub.execute_input":"2022-05-02T12:57:27.967022Z","iopub.status.idle":"2022-05-02T12:57:27.975839Z","shell.execute_reply.started":"2022-05-02T12:57:27.966988Z","shell.execute_reply":"2022-05-02T12:57:27.97493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get VGG-19 Model\ndef getVGG19Model(lastFourTrainable=False):\n    vgg_model_19 = VGG19(weights='imagenet', input_shape=input_shape, include_top=True)\n    # Make all layers untrainable\n    for layer in vgg_model_19.layers[:]:\n        layer.trainable = False\n    # Add fully connected layer which have 1024 neuron to VGG-16 model\n    output = vgg_model_19.get_layer('fc2').output\n    output = Flatten(name='new_flatten')(output)\n    output = Dense(units=1024, activation='relu', name='new_fc')(output)\n    output = Dense(units=50, activation='softmax')(output)\n    vgg_model_19 = Model(vgg_model_19.input, output)\n    # Make last 4 layers trainable if lastFourTrainable == True\n    if lastFourTrainable == True:\n        vgg_model_19.get_layer('block5_conv3').trainable = True\n        vgg_model_19.get_layer('fc1').trainable = True\n        vgg_model_19.get_layer('fc2').trainable = True\n        vgg_model_19.get_layer('new_fc').trainable = True\n    # Compile VGG-16 model\n    vgg_model_19.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n    vgg_model_19.summary()\n\n    return vgg_model_19","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:27.977409Z","iopub.execute_input":"2022-05-02T12:57:27.977667Z","iopub.status.idle":"2022-05-02T12:57:27.987701Z","shell.execute_reply.started":"2022-05-02T12:57:27.977634Z","shell.execute_reply":"2022-05-02T12:57:27.986961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vgg_model19_a = getVGG19Model(lastFourTrainable=False)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:27.98906Z","iopub.execute_input":"2022-05-02T12:57:27.98954Z","iopub.status.idle":"2022-05-02T12:57:34.190617Z","shell.execute_reply.started":"2022-05-02T12:57:27.989489Z","shell.execute_reply":"2022-05-02T12:57:34.189949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vgg_model19_b = getVGG19Model(lastFourTrainable=True)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:34.191714Z","iopub.execute_input":"2022-05-02T12:57:34.191969Z","iopub.status.idle":"2022-05-02T12:57:36.209154Z","shell.execute_reply.started":"2022-05-02T12:57:34.191934Z","shell.execute_reply":"2022-05-02T12:57:36.208403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get ResNet-50 Model with lastFourTrainable=False\nresnet_model_a = getResNet50Model(lastFourTrainable=False)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:36.21049Z","iopub.execute_input":"2022-05-02T12:57:36.210811Z","iopub.status.idle":"2022-05-02T12:57:38.314086Z","shell.execute_reply.started":"2022-05-02T12:57:36.210701Z","shell.execute_reply":"2022-05-02T12:57:38.313401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def score_train(model,test_x,test_y):\n    # Score trained model.\n    train_scores = model.evaluate(test_x, test_y, verbose=1)\n    print('Test loss:', train_scores[0])\n    print('Test accuracy:', train_scores[1])","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:38.315127Z","iopub.execute_input":"2022-05-02T12:57:38.31558Z","iopub.status.idle":"2022-05-02T12:57:38.323675Z","shell.execute_reply.started":"2022-05-02T12:57:38.315539Z","shell.execute_reply":"2022-05-02T12:57:38.323009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.optimizers import Adam","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:38.324928Z","iopub.execute_input":"2022-05-02T12:57:38.325282Z","iopub.status.idle":"2022-05-02T12:57:38.531542Z","shell.execute_reply.started":"2022-05-02T12:57:38.325237Z","shell.execute_reply":"2022-05-02T12:57:38.530804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(validation_x_final.shape)\nprint(validation_y_final.shape)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:38.532948Z","iopub.execute_input":"2022-05-02T12:57:38.533325Z","iopub.status.idle":"2022-05-02T12:57:38.541125Z","shell.execute_reply.started":"2022-05-02T12:57:38.533285Z","shell.execute_reply":"2022-05-02T12:57:38.54031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:38.542554Z","iopub.execute_input":"2022-05-02T12:57:38.543609Z","iopub.status.idle":"2022-05-02T12:57:38.549296Z","shell.execute_reply.started":"2022-05-02T12:57:38.543572Z","shell.execute_reply":"2022-05-02T12:57:38.548441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_x_final_resnet=tensorflow.keras.applications.resnet.preprocess_input(train_x_final)\nvalidation_x_final_resnet=tensorflow.keras.applications.resnet.preprocess_input(validation_x_final)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:38.550652Z","iopub.execute_input":"2022-05-02T12:57:38.551085Z","iopub.status.idle":"2022-05-02T12:57:38.829341Z","shell.execute_reply.started":"2022-05-02T12:57:38.551047Z","shell.execute_reply":"2022-05-02T12:57:38.828482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_x_final_resnet=tensorflow.keras.applications.resnet.preprocess_input(X_test)\n","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:38.830973Z","iopub.execute_input":"2022-05-02T12:57:38.831268Z","iopub.status.idle":"2022-05-02T12:57:38.954171Z","shell.execute_reply.started":"2022-05-02T12:57:38.831217Z","shell.execute_reply":"2022-05-02T12:57:38.953417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"RESNET-50-a","metadata":{}},{"cell_type":"code","source":"# Train ResNet-50 Model \n#resnet_model_a.compile(optimizer=Adam(learning_rate=0.001),loss='categorical_crossentropy',metrics=['accuracy'])\nhistory1 = resnet_model_a.fit(train_x_final_resnet, train_y_final, epochs=30,validation_data=(validation_x_final_resnet, validation_y_final))\n#resnet_model_a = trainModelAndGetConfusionMatrix(resnet_model_a,train_x_final,validation_x_final,X_test,10,64)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:57:38.955609Z","iopub.execute_input":"2022-05-02T12:57:38.955896Z","iopub.status.idle":"2022-05-02T12:59:04.712747Z","shell.execute_reply.started":"2022-05-02T12:57:38.95586Z","shell.execute_reply":"2022-05-02T12:59:04.711927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_hist(history1,'Resnet-50')","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:59:04.717583Z","iopub.execute_input":"2022-05-02T12:59:04.71782Z","iopub.status.idle":"2022-05-02T12:59:04.998972Z","shell.execute_reply.started":"2022-05-02T12:59:04.717794Z","shell.execute_reply":"2022-05-02T12:59:04.998234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score_train(resnet_model_a,test_x_final_resnet,Y_test)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:59:05.000081Z","iopub.execute_input":"2022-05-02T12:59:05.00085Z","iopub.status.idle":"2022-05-02T12:59:07.546858Z","shell.execute_reply.started":"2022-05-02T12:59:05.00081Z","shell.execute_reply":"2022-05-02T12:59:07.546106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Resnet-50-b","metadata":{}},{"cell_type":"code","source":"resnet_model_b = getResNet50Model(lastFourTrainable=True)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:59:07.552692Z","iopub.execute_input":"2022-05-02T12:59:07.553031Z","iopub.status.idle":"2022-05-02T12:59:08.99134Z","shell.execute_reply.started":"2022-05-02T12:59:07.553Z","shell.execute_reply":"2022-05-02T12:59:08.990635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train ResNet-50 Model \n#resnet_model_a.compile(optimizer=Adam(learning_rate=0.001),loss='categorical_crossentropy',metrics=['accuracy'])\nhistory = resnet_model_b.fit(train_x_final_resnet, train_y_final, epochs=30,validation_data=(validation_x_final_resnet, validation_y_final))\n#resnet_model_a = trainModelAndGetConfusionMatrix(resnet_model_a,train_x_final,validation_x_final,X_test,10,64)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T12:59:08.99247Z","iopub.execute_input":"2022-05-02T12:59:08.992708Z","iopub.status.idle":"2022-05-02T13:00:34.532191Z","shell.execute_reply.started":"2022-05-02T12:59:08.992673Z","shell.execute_reply":"2022-05-02T13:00:34.531423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_hist(history,'Resnet-50-b')","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:00:34.536857Z","iopub.execute_input":"2022-05-02T13:00:34.537068Z","iopub.status.idle":"2022-05-02T13:00:34.808628Z","shell.execute_reply.started":"2022-05-02T13:00:34.537044Z","shell.execute_reply":"2022-05-02T13:00:34.807804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score_train(resnet_model_b,test_x_final_resnet,Y_test)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:00:34.809987Z","iopub.execute_input":"2022-05-02T13:00:34.810222Z","iopub.status.idle":"2022-05-02T13:00:37.709525Z","shell.execute_reply.started":"2022-05-02T13:00:34.81019Z","shell.execute_reply":"2022-05-02T13:00:37.708626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"VGG-16-a","metadata":{}},{"cell_type":"code","source":"train_x_final_vgg16=tensorflow.keras.applications.vgg16.preprocess_input(train_x_final)\nvalidation_x_final_vgg16=tensorflow.keras.applications.vgg16.preprocess_input(validation_x_final)\n","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:00:37.711304Z","iopub.execute_input":"2022-05-02T13:00:37.711627Z","iopub.status.idle":"2022-05-02T13:00:37.981479Z","shell.execute_reply.started":"2022-05-02T13:00:37.711588Z","shell.execute_reply":"2022-05-02T13:00:37.980708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_x_final_vgg16=tensorflow.keras.applications.vgg16.preprocess_input(X_test)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:00:37.982828Z","iopub.execute_input":"2022-05-02T13:00:37.983162Z","iopub.status.idle":"2022-05-02T13:00:38.104943Z","shell.execute_reply.started":"2022-05-02T13:00:37.983125Z","shell.execute_reply":"2022-05-02T13:00:38.104177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vgga=getVGG16Model(lastFourTrainable=False)\n","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:00:38.106078Z","iopub.execute_input":"2022-05-02T13:00:38.106704Z","iopub.status.idle":"2022-05-02T13:00:45.080103Z","shell.execute_reply.started":"2022-05-02T13:00:38.10666Z","shell.execute_reply":"2022-05-02T13:00:45.079395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = vgga.fit(train_x_final_vgg16, train_y_final,validation_data=(validation_x_final_vgg16,validation_y_final), epochs=30)\n","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:00:45.081342Z","iopub.execute_input":"2022-05-02T13:00:45.08157Z","iopub.status.idle":"2022-05-02T13:02:08.410305Z","shell.execute_reply.started":"2022-05-02T13:00:45.081538Z","shell.execute_reply":"2022-05-02T13:02:08.409544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_hist(history,'VGG-16')","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:02:08.412124Z","iopub.execute_input":"2022-05-02T13:02:08.41246Z","iopub.status.idle":"2022-05-02T13:02:08.678365Z","shell.execute_reply.started":"2022-05-02T13:02:08.412421Z","shell.execute_reply":"2022-05-02T13:02:08.677737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score_train(vgga,test_x_final_vgg16,Y_test)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:02:08.679565Z","iopub.execute_input":"2022-05-02T13:02:08.679813Z","iopub.status.idle":"2022-05-02T13:02:12.363277Z","shell.execute_reply.started":"2022-05-02T13:02:08.679784Z","shell.execute_reply":"2022-05-02T13:02:12.362596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vggb=getVGG16Model(lastFourTrainable=True)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:02:12.364882Z","iopub.execute_input":"2022-05-02T13:02:12.365179Z","iopub.status.idle":"2022-05-02T13:02:14.315982Z","shell.execute_reply.started":"2022-05-02T13:02:12.365139Z","shell.execute_reply":"2022-05-02T13:02:14.315286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history_vgg16_b = vggb.fit(train_x_final_vgg16, train_y_final,validation_data=(validation_x_final_vgg16,validation_y_final), epochs=30)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:02:14.317222Z","iopub.execute_input":"2022-05-02T13:02:14.317462Z","iopub.status.idle":"2022-05-02T13:03:17.603217Z","shell.execute_reply.started":"2022-05-02T13:02:14.317427Z","shell.execute_reply":"2022-05-02T13:03:17.602484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_hist(history_vgg16_b,'Vgg-16-b')","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:03:17.604799Z","iopub.execute_input":"2022-05-02T13:03:17.605034Z","iopub.status.idle":"2022-05-02T13:03:17.881184Z","shell.execute_reply.started":"2022-05-02T13:03:17.605001Z","shell.execute_reply":"2022-05-02T13:03:17.880547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score_train(vggb,test_x_final_vgg16,Y_test)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:03:17.882372Z","iopub.execute_input":"2022-05-02T13:03:17.883176Z","iopub.status.idle":"2022-05-02T13:03:19.299702Z","shell.execute_reply.started":"2022-05-02T13:03:17.883138Z","shell.execute_reply":"2022-05-02T13:03:19.299003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"VGG-19-a","metadata":{}},{"cell_type":"code","source":"train_x_final_vgg19=tensorflow.keras.applications.vgg19.preprocess_input(train_x_final)\nvalidation_x_final_vgg19=tensorflow.keras.applications.vgg19.preprocess_input(validation_x_final)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:03:19.30114Z","iopub.execute_input":"2022-05-02T13:03:19.301403Z","iopub.status.idle":"2022-05-02T13:03:19.827875Z","shell.execute_reply.started":"2022-05-02T13:03:19.301367Z","shell.execute_reply":"2022-05-02T13:03:19.827091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_x_final_vgg19=tensorflow.keras.applications.vgg19.preprocess_input(X_test)\n","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:03:19.831004Z","iopub.execute_input":"2022-05-02T13:03:19.831224Z","iopub.status.idle":"2022-05-02T13:03:20.073631Z","shell.execute_reply.started":"2022-05-02T13:03:19.831198Z","shell.execute_reply":"2022-05-02T13:03:20.072865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history_vgg19_a = vgg_model19_a.fit(train_x_final_vgg19, train_y_final,validation_data=(validation_x_final_vgg19,validation_y_final), epochs=30)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:03:20.075062Z","iopub.execute_input":"2022-05-02T13:03:20.075305Z","iopub.status.idle":"2022-05-02T13:04:43.742397Z","shell.execute_reply.started":"2022-05-02T13:03:20.075262Z","shell.execute_reply":"2022-05-02T13:04:43.741452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score_train(vgg_model19_a,test_x_final_vgg19,Y_test)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:04:43.745227Z","iopub.execute_input":"2022-05-02T13:04:43.74557Z","iopub.status.idle":"2022-05-02T13:04:45.885463Z","shell.execute_reply.started":"2022-05-02T13:04:43.745533Z","shell.execute_reply":"2022-05-02T13:04:45.884788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_hist(history_vgg19_a,'vgg-19-a')","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:04:45.886906Z","iopub.execute_input":"2022-05-02T13:04:45.88774Z","iopub.status.idle":"2022-05-02T13:04:46.15622Z","shell.execute_reply.started":"2022-05-02T13:04:45.887685Z","shell.execute_reply":"2022-05-02T13:04:46.15556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"VGG-19-b","metadata":{}},{"cell_type":"code","source":"history_vgg19_b = vgg_model19_b.fit(train_x_final_vgg19, train_y_final,validation_data=(validation_x_final_vgg19,validation_y_final), epochs=30)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:04:46.157412Z","iopub.execute_input":"2022-05-02T13:04:46.158872Z","iopub.status.idle":"2022-05-02T13:06:09.763437Z","shell.execute_reply.started":"2022-05-02T13:04:46.158832Z","shell.execute_reply":"2022-05-02T13:06:09.762679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_hist(history_vgg19_b,'VGG-19-b')","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:06:09.765319Z","iopub.execute_input":"2022-05-02T13:06:09.765578Z","iopub.status.idle":"2022-05-02T13:06:10.036264Z","shell.execute_reply.started":"2022-05-02T13:06:09.765543Z","shell.execute_reply":"2022-05-02T13:06:10.035591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score_train(vgg_model19_b,test_x_final_vgg19,Y_test)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:06:10.037553Z","iopub.execute_input":"2022-05-02T13:06:10.037814Z","iopub.status.idle":"2022-05-02T13:06:11.631183Z","shell.execute_reply.started":"2022-05-02T13:06:10.037779Z","shell.execute_reply":"2022-05-02T13:06:11.630331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"DenseNet","metadata":{}},{"cell_type":"code","source":"from keras.applications.densenet import DenseNet121","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:06:11.632948Z","iopub.execute_input":"2022-05-02T13:06:11.633247Z","iopub.status.idle":"2022-05-02T13:06:11.637421Z","shell.execute_reply.started":"2022-05-02T13:06:11.633206Z","shell.execute_reply":"2022-05-02T13:06:11.63651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get DenseNet-121 Model\ndef getDenseNet121Model(lastFourTrainable=False):\n    densenet_model = DenseNet121(weights='imagenet', input_shape=input_shape, include_top=True)\n    # Make all layers non-trainable\n    for layer in densenet_model.layers[:]:\n        layer.trainable = False\n    # Add fully connected layer which have 1024 neuron to ResNet-50 model\n    output = densenet_model.get_layer('avg_pool').output\n    output = Flatten(name='new_flatten')(output)\n    output = Dense(units=1024, activation='relu', name='new_fc')(output)\n    predictions = Dense(units=50, activation='softmax')(output)\n    densenet_model = Model(densenet_model.input, predictions)\n    # Make last 4 layers trainable if lastFourTrainable == True\n    if lastFourTrainable == True:\n        densenet_model.get_layer('conv5_block3_2_bn').trainable = True\n        densenet_model.get_layer('conv5_block3_3_conv').trainable = True\n        densenet_model.get_layer('conv5_block3_3_bn').trainable = True\n        densenet_model.get_layer('new_fc').trainable = True\n    # Compile ResNet-50 model\n    densenet_model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n    densenet_model.summary()\n    return densenet_model","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:06:11.639065Z","iopub.execute_input":"2022-05-02T13:06:11.639318Z","iopub.status.idle":"2022-05-02T13:06:11.649007Z","shell.execute_reply.started":"2022-05-02T13:06:11.639285Z","shell.execute_reply":"2022-05-02T13:06:11.648028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get DenseNet-121 Model with lastFourTrainable=False\ndensenet_model_a = getDenseNet121Model(lastFourTrainable=False)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:06:11.650185Z","iopub.execute_input":"2022-05-02T13:06:11.650973Z","iopub.status.idle":"2022-05-02T13:06:15.297126Z","shell.execute_reply.started":"2022-05-02T13:06:11.650935Z","shell.execute_reply":"2022-05-02T13:06:15.296399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_x_final_densenet=tensorflow.keras.applications.densenet.preprocess_input(train_x_final)\nvalidation_x_final_densenet=tensorflow.keras.applications.densenet.preprocess_input(validation_x_final)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:06:15.298318Z","iopub.execute_input":"2022-05-02T13:06:15.298801Z","iopub.status.idle":"2022-05-02T13:06:15.928679Z","shell.execute_reply.started":"2022-05-02T13:06:15.298763Z","shell.execute_reply":"2022-05-02T13:06:15.927927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_x_final_densenet=tensorflow.keras.applications.densenet.preprocess_input(X_test)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:06:15.930578Z","iopub.execute_input":"2022-05-02T13:06:15.931022Z","iopub.status.idle":"2022-05-02T13:06:16.301251Z","shell.execute_reply.started":"2022-05-02T13:06:15.930985Z","shell.execute_reply":"2022-05-02T13:06:16.300503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train DENSETNET-121 Model \n#resnet_model_a.compile(optimizer=Adam(learning_rate=0.001),loss='categorical_crossentropy',metrics=['accuracy'])\nhistory_densenet_model_a = densenet_model_a.fit(train_x_final_densenet, train_y_final, epochs=30,validation_data=(validation_x_final_densenet, validation_y_final))\n#resnet_model_a = trainModelAndGetConfusionMatrix(resnet_model_a,train_x_final,validation_x_final,X_test,10,64)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:06:16.302662Z","iopub.execute_input":"2022-05-02T13:06:16.302932Z","iopub.status.idle":"2022-05-02T13:07:44.143122Z","shell.execute_reply.started":"2022-05-02T13:06:16.302897Z","shell.execute_reply":"2022-05-02T13:07:44.142293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_hist(history_densenet_model_a,'Densenet')\nscore_train(densenet_model_a,test_x_final_densenet,Y_test)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:07:44.145143Z","iopub.execute_input":"2022-05-02T13:07:44.145385Z","iopub.status.idle":"2022-05-02T13:07:47.584407Z","shell.execute_reply.started":"2022-05-02T13:07:44.145353Z","shell.execute_reply":"2022-05-02T13:07:47.583267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## KNN- Image Retrieval","metadata":{}},{"cell_type":"markdown","source":"Using resnet-50-a for feature extraction and KNN","metadata":{}},{"cell_type":"code","source":"from keras.models import Model","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:07:47.586178Z","iopub.execute_input":"2022-05-02T13:07:47.586416Z","iopub.status.idle":"2022-05-02T13:07:47.590495Z","shell.execute_reply.started":"2022-05-02T13:07:47.586382Z","shell.execute_reply":"2022-05-02T13:07:47.589492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_model_vgg_model19_a= Model(inputs=vgg_model19_a.input, outputs=vgg_model19_a.get_layer('new_fc').output)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:07:47.591941Z","iopub.execute_input":"2022-05-02T13:07:47.592358Z","iopub.status.idle":"2022-05-02T13:07:47.604716Z","shell.execute_reply.started":"2022-05-02T13:07:47.592323Z","shell.execute_reply":"2022-05-02T13:07:47.60403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_vects=feature_model_vgg_model19_a.predict(train_x_final_vgg19)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:07:47.606014Z","iopub.execute_input":"2022-05-02T13:07:47.606384Z","iopub.status.idle":"2022-05-02T13:07:50.140482Z","shell.execute_reply.started":"2022-05-02T13:07:47.606249Z","shell.execute_reply":"2022-05-02T13:07:50.139643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_vects.shape","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:07:50.14201Z","iopub.execute_input":"2022-05-02T13:07:50.142257Z","iopub.status.idle":"2022-05-02T13:07:50.148031Z","shell.execute_reply.started":"2022-05-02T13:07:50.142223Z","shell.execute_reply":"2022-05-02T13:07:50.14665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.neighbors import KNeighborsClassifier as KNeighborsClassifier","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:07:50.149257Z","iopub.execute_input":"2022-05-02T13:07:50.149667Z","iopub.status.idle":"2022-05-02T13:07:50.266463Z","shell.execute_reply.started":"2022-05-02T13:07:50.14963Z","shell.execute_reply":"2022-05-02T13:07:50.265864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import GridSearchCV","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:07:50.273557Z","iopub.execute_input":"2022-05-02T13:07:50.273767Z","iopub.status.idle":"2022-05-02T13:07:50.277416Z","shell.execute_reply.started":"2022-05-02T13:07:50.273739Z","shell.execute_reply":"2022-05-02T13:07:50.276582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_integers=np.argmax(train_y_final, axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:07:50.278673Z","iopub.execute_input":"2022-05-02T13:07:50.279441Z","iopub.status.idle":"2022-05-02T13:07:50.286545Z","shell.execute_reply.started":"2022-05-02T13:07:50.279405Z","shell.execute_reply":"2022-05-02T13:07:50.285847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_integers.shape","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:07:50.287856Z","iopub.execute_input":"2022-05-02T13:07:50.288345Z","iopub.status.idle":"2022-05-02T13:07:50.296824Z","shell.execute_reply.started":"2022-05-02T13:07:50.28831Z","shell.execute_reply":"2022-05-02T13:07:50.296031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"knn = KNeighborsClassifier()\nknn.fit(feature_vects,labels_integers)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:07:50.299014Z","iopub.execute_input":"2022-05-02T13:07:50.29981Z","iopub.status.idle":"2022-05-02T13:07:50.310077Z","shell.execute_reply.started":"2022-05-02T13:07:50.299751Z","shell.execute_reply":"2022-05-02T13:07:50.309365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_vects=feature_model_vgg_model19_a.predict(test_x_final_vgg19)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:07:50.311316Z","iopub.execute_input":"2022-05-02T13:07:50.311619Z","iopub.status.idle":"2022-05-02T13:07:51.861592Z","shell.execute_reply.started":"2022-05-02T13:07:50.311585Z","shell.execute_reply":"2022-05-02T13:07:51.860857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_vects.shape","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:07:51.863206Z","iopub.execute_input":"2022-05-02T13:07:51.863467Z","iopub.status.idle":"2022-05-02T13:07:51.870134Z","shell.execute_reply.started":"2022-05-02T13:07:51.863433Z","shell.execute_reply":"2022-05-02T13:07:51.869203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_test.shape","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:07:51.871775Z","iopub.execute_input":"2022-05-02T13:07:51.872019Z","iopub.status.idle":"2022-05-02T13:07:51.881245Z","shell.execute_reply.started":"2022-05-02T13:07:51.871986Z","shell.execute_reply":"2022-05-02T13:07:51.880517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_label_integers=np.argmax(Y_test, axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:07:51.882543Z","iopub.execute_input":"2022-05-02T13:07:51.882772Z","iopub.status.idle":"2022-05-02T13:07:51.888891Z","shell.execute_reply.started":"2022-05-02T13:07:51.882736Z","shell.execute_reply":"2022-05-02T13:07:51.888073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_label_integers.shape","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:07:51.890217Z","iopub.execute_input":"2022-05-02T13:07:51.890537Z","iopub.status.idle":"2022-05-02T13:07:51.898368Z","shell.execute_reply.started":"2022-05-02T13:07:51.890501Z","shell.execute_reply":"2022-05-02T13:07:51.897578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(knn.score(test_vects,test_label_integers))\n#ypred=knn.predict(fin_test_img)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:07:51.899935Z","iopub.execute_input":"2022-05-02T13:07:51.900269Z","iopub.status.idle":"2022-05-02T13:07:51.949489Z","shell.execute_reply.started":"2022-05-02T13:07:51.900198Z","shell.execute_reply":"2022-05-02T13:07:51.948779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"RETREIVAL of FIRST 7 IMAGES","metadata":{}},{"cell_type":"code","source":"knn7 = KNeighborsClassifier(n_neighbors = 7)\nknn7.fit(feature_vects,labels_integers)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:07:51.9505Z","iopub.execute_input":"2022-05-02T13:07:51.950949Z","iopub.status.idle":"2022-05-02T13:07:51.958529Z","shell.execute_reply.started":"2022-05-02T13:07:51.950915Z","shell.execute_reply":"2022-05-02T13:07:51.957709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"indices=knn7.kneighbors(test_vects, return_distance=False)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:07:51.95954Z","iopub.execute_input":"2022-05-02T13:07:51.96039Z","iopub.status.idle":"2022-05-02T13:07:51.984288Z","shell.execute_reply.started":"2022-05-02T13:07:51.960338Z","shell.execute_reply":"2022-05-02T13:07:51.983573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"indices.shape","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:07:51.985262Z","iopub.execute_input":"2022-05-02T13:07:51.985639Z","iopub.status.idle":"2022-05-02T13:07:51.991085Z","shell.execute_reply.started":"2022-05-02T13:07:51.985607Z","shell.execute_reply":"2022-05-02T13:07:51.990403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.max(indices)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:07:51.992387Z","iopub.execute_input":"2022-05-02T13:07:51.992833Z","iopub.status.idle":"2022-05-02T13:07:52.001376Z","shell.execute_reply.started":"2022-05-02T13:07:51.992799Z","shell.execute_reply":"2022-05-02T13:07:52.000596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.min(indices)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:07:52.002605Z","iopub.execute_input":"2022-05-02T13:07:52.003111Z","iopub.status.idle":"2022-05-02T13:07:52.010447Z","shell.execute_reply.started":"2022-05-02T13:07:52.003057Z","shell.execute_reply":"2022-05-02T13:07:52.009776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_neighbors(orig, neighbors):\n    f, axarr = plt.subplots(4, 2)\n    for i, ax in enumerate(axarr.flatten()):\n        if i == 0:\n            ax.set_title(\"Query image\")\n            ax.imshow(orig)\n        else:\n            ax.set_title(f\"Neighbor {i}\")\n            ax.imshow(train_x_final[neighbors[i-1]])\n        ax.set_yticklabels([])\n        ax.set_xticklabels([])\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:07:52.011996Z","iopub.execute_input":"2022-05-02T13:07:52.012971Z","iopub.status.idle":"2022-05-02T13:07:52.022304Z","shell.execute_reply.started":"2022-05-02T13:07:52.012933Z","shell.execute_reply":"2022-05-02T13:07:52.021502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"indices=knn7.kneighbors(test_vects, return_distance=False)\nfor i in range(0,5):\n    show_neighbors(X_test[i],indices[i])","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:07:52.023592Z","iopub.execute_input":"2022-05-02T13:07:52.02399Z","iopub.status.idle":"2022-05-02T13:07:55.756241Z","shell.execute_reply.started":"2022-05-02T13:07:52.023957Z","shell.execute_reply":"2022-05-02T13:07:55.755579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ypred=knn7.predict(test_vects)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:07:55.757615Z","iopub.execute_input":"2022-05-02T13:07:55.758082Z","iopub.status.idle":"2022-05-02T13:07:55.798451Z","shell.execute_reply.started":"2022-05-02T13:07:55.758045Z","shell.execute_reply":"2022-05-02T13:07:55.797737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn import metrics \nknn_classification_report = metrics.classification_report(test_label_integers,ypred,output_dict=True)\nprint(metrics.classification_report(test_label_integers,ypred))","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:07:55.8027Z","iopub.execute_input":"2022-05-02T13:07:55.804895Z","iopub.status.idle":"2022-05-02T13:07:55.832613Z","shell.execute_reply.started":"2022-05-02T13:07:55.804849Z","shell.execute_reply":"2022-05-02T13:07:55.831942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type(knn_classification_report)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:07:55.833958Z","iopub.execute_input":"2022-05-02T13:07:55.834393Z","iopub.status.idle":"2022-05-02T13:07:55.841119Z","shell.execute_reply.started":"2022-05-02T13:07:55.834358Z","shell.execute_reply":"2022-05-02T13:07:55.840052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame(knn_classification_report).transpose()","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:07:55.842466Z","iopub.execute_input":"2022-05-02T13:07:55.842716Z","iopub.status.idle":"2022-05-02T13:07:55.854743Z","shell.execute_reply.started":"2022-05-02T13:07:55.842684Z","shell.execute_reply":"2022-05-02T13:07:55.853799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:07:55.856292Z","iopub.execute_input":"2022-05-02T13:07:55.856524Z","iopub.status.idle":"2022-05-02T13:07:55.885748Z","shell.execute_reply.started":"2022-05-02T13:07:55.856493Z","shell.execute_reply":"2022-05-02T13:07:55.884791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1. Define utility functions","metadata":{}},{"cell_type":"code","source":"import os\nimport pickle\nimport random\n\nimport numpy as np\nimport tensorflow as tf\nfrom tqdm  import tqdm\nfrom PIL import Image, ImageFile\nfrom scipy.io import savemat, loadmat\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:07:55.887018Z","iopub.execute_input":"2022-05-02T13:07:55.887453Z","iopub.status.idle":"2022-05-02T13:07:55.91761Z","shell.execute_reply.started":"2022-05-02T13:07:55.887419Z","shell.execute_reply":"2022-05-02T13:07:55.916996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Adapted from: https://github.com/filipradenovic/revisitop/blob/master/python/example_process_images.py\n\ndef pil_loader(path):\n    # to avoid crashing for truncated (corrupted images)\n    ImageFile.LOAD_TRUNCATED_IMAGES = True\n    # open path as file to avoid ResourceWarning \n    # (https://github.com/python-pillow/Pillow/issues/835)\n    with open(path, 'rb') as f:\n        img = Image.open(f)\n        return img.convert('RGB')","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:07:55.918796Z","iopub.execute_input":"2022-05-02T13:07:55.919028Z","iopub.status.idle":"2022-05-02T13:07:55.923426Z","shell.execute_reply.started":"2022-05-02T13:07:55.918999Z","shell.execute_reply":"2022-05-02T13:07:55.922755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Adapted from: https://github.com/filipradenovic/revisitop/blob/master/python/example_process_images.py\n\ndef extract_features(test_dataset, cfg, model):\n    \"\"\"\n    Generates file with serialized model outputs for each image from test_dataset.\n    \n    Arguments\n    ---------\n    test_dataset   : name of dataset of interest (roxford5k | rparis6k)\n    cfg            : unserialized dataset config, containing annotation metadata\n    model          : loaded Tensorflow baseline model object\n    \"\"\"\n    \n    print('>> Processing query images...', flush=True)\n    Q = []\n    for i in tqdm(np.arange(cfg['nq'])):\n        qim = pil_loader(cfg['qim_fname'](cfg, i)).crop(cfg['gnd'][i]['bbx'])\n        image_data = np.array(qim)\n        image_tensor = tf.convert_to_tensor(image_data)\n        Q.append(model(image_tensor)['global_descriptor'].numpy())\n    Q = np.array(Q, dtype=np.float32)\n    Q = Q.transpose()\n    \n    print('>> Processing index images...', flush=True)\n    X = []\n    for i in tqdm(np.arange(cfg['n'])):\n        im = pil_loader(cfg['im_fname'](cfg, i))\n        image_data = np.array(im)\n        image_tensor = tf.convert_to_tensor(image_data)\n        X.append(model(image_tensor)['global_descriptor'].numpy())\n    X = np.array(X, dtype=np.float32)\n    X = X.transpose()\n\n    feature_dict = {'X': X, 'Q': Q}\n    mat_save_path = \"{}_delg_baseline.mat\".format(test_dataset)\n    print('>> Saving model outputs to: {}'.format(mat_save_path))\n    savemat(mat_save_path, feature_dict)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:07:55.924703Z","iopub.execute_input":"2022-05-02T13:07:55.9251Z","iopub.status.idle":"2022-05-02T13:07:55.936457Z","shell.execute_reply.started":"2022-05-02T13:07:55.925068Z","shell.execute_reply":"2022-05-02T13:07:55.9357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Adapted from: https://github.com/filipradenovic/revisitop/blob/master/python/example_evaluate.py\n\ndef run_evaluation(test_dataset, cfg, features_dir):\n    ks = [1, 5, 10]\n    gnd = cfg['gnd']\n    features = loadmat(os.path.join(features_dir, '{}_delg_baseline.mat'.format(test_dataset)))\n\n    Q = features['Q']\n    X = features['X']\n    sim = np.dot(X.T, Q)\n    ranks = np.argsort(-sim, axis=0)\n\n    # search for easy\n    gnd_t = []\n    for i in range(len(gnd)):\n        g = {}\n        g['ok'] = np.concatenate([gnd[i]['easy']])\n        g['junk'] = np.concatenate([gnd[i]['junk'], gnd[i]['hard']])\n        gnd_t.append(g)\n    mapE, apsE, mprE, prsE = compute_map(ranks, gnd_t, ks)\n\n    # search for easy & hard\n    gnd_t = []\n    for i in range(len(gnd)):\n        g = {}\n        g['ok'] = np.concatenate([gnd[i]['easy'], gnd[i]['hard']])\n        g['junk'] = np.concatenate([gnd[i]['junk']])\n        gnd_t.append(g)\n    mapM, apsM, mprM, prsM = compute_map(ranks, gnd_t, ks)\n\n    # search for hard\n    gnd_t = []\n    for i in range(len(gnd)):\n        g = {}\n        g['ok'] = np.concatenate([gnd[i]['hard']])\n        g['junk'] = np.concatenate([gnd[i]['junk'], gnd[i]['easy']])\n        gnd_t.append(g)\n    mapH, apsH, mprH, prsH = compute_map(ranks, gnd_t, ks)\n\n    print('>> {}: mAP E: {}, M: {}, H: {}'.format(test_dataset, np.around(mapE*100, decimals=2), np.around(mapM*100, decimals=2), np.around(mapH*100, decimals=2)))\n    print('>> {}: mP@k{} E: {}, M: {}, H: {}'.format(test_dataset, np.array(ks), np.around(mprE*100, decimals=2), np.around(mprM*100, decimals=2), np.around(mprH*100, decimals=2)))","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:07:55.938981Z","iopub.execute_input":"2022-05-02T13:07:55.939194Z","iopub.status.idle":"2022-05-02T13:07:55.955781Z","shell.execute_reply.started":"2022-05-02T13:07:55.939152Z","shell.execute_reply":"2022-05-02T13:07:55.955089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport pickle\n\nDATASETS = ['roxford5k', 'rparis6k', 'revisitop1m']\n\ndef configdataset(dataset, dir_main):\n\n    dataset = dataset.lower()\n\n    if dataset not in DATASETS:    \n        raise ValueError('Unknown dataset: {}!'.format(dataset))\n\n    if dataset == 'roxford5k' or dataset == 'rparis6k':\n        # loading imlist, qimlist, and gnd, in cfg as a dict\n        gnd_fname = os.path.join(dir_main, dataset, 'gnd_{}.pkl'.format(dataset))\n        with open(gnd_fname, 'rb') as f:\n            cfg = pickle.load(f)\n        cfg['gnd_fname'] = gnd_fname\n        cfg['ext'] = '.jpg'\n        cfg['qext'] = '.jpg'\n\n    elif dataset == 'revisitop1m':\n        # loading imlist from a .txt file\n        cfg = {}\n        cfg['imlist_fname'] = os.path.join(dir_main, dataset, '{}.txt'.format(dataset))\n        cfg['imlist'] = read_imlist(cfg['imlist_fname'])\n        cfg['qimlist'] = []\n        cfg['ext'] = ''\n        cfg['qext'] = ''\n\n    cfg['dir_data'] = os.path.join(dir_main, dataset)\n    cfg['dir_images'] = os.path.join(cfg['dir_data'], 'jpg')\n\n    cfg['n'] = len(cfg['imlist'])\n    cfg['nq'] = len(cfg['qimlist'])\n\n    cfg['im_fname'] = config_imname\n    cfg['qim_fname'] = config_qimname\n\n    cfg['dataset'] = dataset\n\n    return cfg\n\ndef config_imname(cfg, i):\n    return os.path.join(cfg['dir_images'], cfg['imlist'][i] + cfg['ext'])\n\ndef config_qimname(cfg, i):\n    return os.path.join(cfg['dir_images'], cfg['qimlist'][i] + cfg['qext'])\n\ndef read_imlist(imlist_fn):\n    with open(imlist_fn, 'r') as file:\n        imlist = file.read().splitlines()\n    return imlist","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:07:55.956922Z","iopub.execute_input":"2022-05-02T13:07:55.95731Z","iopub.status.idle":"2022-05-02T13:07:55.970914Z","shell.execute_reply.started":"2022-05-02T13:07:55.957275Z","shell.execute_reply":"2022-05-02T13:07:55.970144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\n\ndef compute_ap(ranks, nres):\n    \"\"\"\n    Computes average precision for given ranked indexes.\n    \n    Arguments\n    ---------\n    ranks : zerro-based ranks of positive images\n    nres  : number of positive images\n    \n    Returns\n    -------\n    ap    : average precision\n    \"\"\"\n\n    # number of images ranked by the system\n    nimgranks = len(ranks)\n\n    # accumulate trapezoids in PR-plot\n    ap = 0\n\n    recall_step = 1. / nres\n\n    for j in np.arange(nimgranks):\n        rank = ranks[j]\n\n        if rank == 0:\n            precision_0 = 1.\n        else:\n            precision_0 = float(j) / rank\n\n        precision_1 = float(j + 1) / (rank + 1)\n\n        ap += (precision_0 + precision_1) * recall_step / 2.\n\n    return ap\n\ndef compute_map(ranks, gnd, kappas=[]):\n    \"\"\"\n    Computes the mAP for a given set of returned results.\n         Usage: \n           map = compute_map (ranks, gnd) \n                 computes mean average precsion (map) only\n        \n           map, aps, pr, prs = compute_map (ranks, gnd, kappas) \n                 computes mean average precision (map), average precision (aps) for each query\n                 computes mean precision at kappas (pr), precision at kappas (prs) for each query\n        \n         Notes:\n         1) ranks starts from 0, ranks.shape = db_size X #queries\n         2) The junk results (e.g., the query itself) should be declared in the gnd stuct array\n         3) If there are no positive images for some query, that query is excluded from the evaluation\n    \"\"\"\n\n    map = 0.\n    nq = len(gnd) # number of queries\n    aps = np.zeros(nq)\n    pr = np.zeros(len(kappas))\n    prs = np.zeros((nq, len(kappas)))\n    nempty = 0\n\n    for i in np.arange(nq):\n        qgnd = np.array(gnd[i]['ok'])\n\n        # no positive images, skip from the average\n        if qgnd.shape[0] == 0:\n            aps[i] = float('nan')\n            prs[i, :] = float('nan')\n            nempty += 1\n            continue\n\n        try:\n            qgndj = np.array(gnd[i]['junk'])\n        except:\n            qgndj = np.empty(0)\n\n        # sorted positions of positive and junk images (0 based)\n        pos  = np.arange(ranks.shape[0])[np.in1d(ranks[:,i], qgnd)]\n        junk = np.arange(ranks.shape[0])[np.in1d(ranks[:,i], qgndj)]\n\n        k = 0;\n        ij = 0;\n        if len(junk):\n            # decrease positions of positives based on the number of\n            # junk images appearing before them\n            ip = 0\n            while (ip < len(pos)):\n                while (ij < len(junk) and pos[ip] > junk[ij]):\n                    k += 1\n                    ij += 1\n                pos[ip] = pos[ip] - k\n                ip += 1\n\n        # compute ap\n        ap = compute_ap(pos, len(qgnd))\n        map = map + ap\n        aps[i] = ap\n\n        # compute precision @ k\n        pos += 1 # get it to 1-based\n        for j in np.arange(len(kappas)):\n            kq = min(max(pos), kappas[j]); \n            prs[i, j] = (pos <= kq).sum() / kq\n        pr = pr + prs[i, :]\n\n    map = map / (nq - nempty)\n    pr = pr / (nq - nempty)\n\n    return map, aps, pr, prs","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:07:55.972166Z","iopub.execute_input":"2022-05-02T13:07:55.972954Z","iopub.status.idle":"2022-05-02T13:07:55.990052Z","shell.execute_reply.started":"2022-05-02T13:07:55.972916Z","shell.execute_reply":"2022-05-02T13:07:55.989264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2. Feature extraction","metadata":{}},{"cell_type":"code","source":"data_root = '/kaggle/input/roxfordparis'","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:07:55.991088Z","iopub.execute_input":"2022-05-02T13:07:55.99209Z","iopub.status.idle":"2022-05-02T13:07:56.001583Z","shell.execute_reply.started":"2022-05-02T13:07:55.992056Z","shell.execute_reply":"2022-05-02T13:07:56.000787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model = feature_model_vgg_model19_a","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:07:56.00264Z","iopub.execute_input":"2022-05-02T13:07:56.003233Z","iopub.status.idle":"2022-05-02T13:07:56.009479Z","shell.execute_reply.started":"2022-05-02T13:07:56.003194Z","shell.execute_reply":"2022-05-02T13:07:56.008781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Extract features for roxford5k & rparis6k datasets\n\nshould_extract_features = True\n\ndatasets = ['roxford5k', 'rparis6k']\nfor test_dataset in datasets:\n    if should_extract_features:\n        print('Processing dataset: {}'.format(test_dataset), flush=True)\n        cfg = configdataset(test_dataset, data_root)\n        extract_features(test_dataset, cfg, model)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:55:03.380543Z","iopub.execute_input":"2022-05-02T13:55:03.380828Z","iopub.status.idle":"2022-05-02T13:55:03.445868Z","shell.execute_reply.started":"2022-05-02T13:55:03.380795Z","shell.execute_reply":"2022-05-02T13:55:03.444915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model Evaluation","metadata":{}},{"cell_type":"code","source":"# Evaluate on roxford5k & rparis6k datasets\n\ndatasets = ['roxford5k', 'rparis6k']\nfor test_dataset in datasets:\n    print('Evaluating on dataset: {}'.format(test_dataset), flush=True)\n    cfg = configdataset(test_dataset, data_root)\n    run_evaluation(test_dataset, cfg, '/kaggle/input/delg-baseline-roxfordparis-output')\n    print()","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:55:09.76386Z","iopub.execute_input":"2022-05-02T13:55:09.764111Z","iopub.status.idle":"2022-05-02T13:55:10.826933Z","shell.execute_reply.started":"2022-05-02T13:55:09.764084Z","shell.execute_reply":"2022-05-02T13:55:10.826136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The final mAP metrics are:\n- roxford5k - Easy: 90.92, Medium: 76.23, Hard: 55.54\n- rparis5k - Easy: 94.06, Medium: 87.25, Hard: 74.2\n\nIt's interesting to noticed that these metrics are higher than ones presented in [original DELG paper](https://arxiv.org/pdf/2001.05027.pdf) the baseline model is based on (check rows corresponding to DELG in columns marked as `ROxf` and `RPar`). That suggests that the baseline model is more powerful than the one presented in the paper, which means it is a considerate challenge to come up with model that improves baseline score on the Leaderboard.","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}