{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":29762,"databundleVersionId":2541532,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-03-09T11:34:50.375175Z","iopub.execute_input":"2023-03-09T11:34:50.375807Z","iopub.status.idle":"2023-03-09T11:34:50.403872Z","shell.execute_reply.started":"2023-03-09T11:34:50.375716Z","shell.execute_reply":"2023-03-09T11:34:50.403155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.utils import np_utils\nfrom tensorflow.keras.models import Model, Sequential, load_model\nfrom tensorflow.keras.layers import Dense, Conv2D, MaxPooling2D, Flatten, Reshape, Dropout\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.applications.resnet50 import ResNet50\nfrom sklearn.metrics import confusion_matrix, classification_report\nimport itertools\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:34:50.406359Z","iopub.execute_input":"2023-03-09T11:34:50.4069Z","iopub.status.idle":"2023-03-09T11:34:56.221123Z","shell.execute_reply.started":"2023-03-09T11:34:50.406862Z","shell.execute_reply":"2023-03-09T11:34:56.220327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.applications.vgg16 import VGG16","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:34:56.223392Z","iopub.execute_input":"2023-03-09T11:34:56.223937Z","iopub.status.idle":"2023-03-09T11:34:56.230554Z","shell.execute_reply.started":"2023-03-09T11:34:56.223897Z","shell.execute_reply":"2023-03-09T11:34:56.229007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.applications.vgg19 import VGG19","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:34:56.232675Z","iopub.execute_input":"2023-03-09T11:34:56.233137Z","iopub.status.idle":"2023-03-09T11:34:56.242203Z","shell.execute_reply.started":"2023-03-09T11:34:56.233101Z","shell.execute_reply":"2023-03-09T11:34:56.241436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.utils import to_categorical","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:34:56.243152Z","iopub.execute_input":"2023-03-09T11:34:56.243372Z","iopub.status.idle":"2023-03-09T11:34:56.251044Z","shell.execute_reply.started":"2023-03-09T11:34:56.243348Z","shell.execute_reply":"2023-03-09T11:34:56.2503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Constants\nimg_rows = 224\nimg_cols = 224\ninput_shape = (img_rows,img_cols,3)\nepochs = 10\nbatch_size = 64\n","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:34:56.252435Z","iopub.execute_input":"2023-03-09T11:34:56.252772Z","iopub.status.idle":"2023-03-09T11:34:56.258689Z","shell.execute_reply.started":"2023-03-09T11:34:56.252738Z","shell.execute_reply":"2023-03-09T11:34:56.257957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get ResNet-50 Model\ndef getResNet50Model(lastFourTrainable=False):\n    resnet_model = ResNet50(weights='imagenet', input_shape=input_shape, include_top=True)\n    # Make all layers non-trainable\n    for layer in resnet_model.layers[:]:\n        layer.trainable = False\n    # Add fully connected layer which have 1024 neuron to ResNet-50 model\n    output = resnet_model.get_layer('avg_pool').output\n    output = Flatten(name='new_flatten')(output)\n    output = Dense(units=1024, activation='relu', name='new_fc')(output)\n    predictions = Dense(units=50, activation='softmax')(output)\n    resnet_model = Model(resnet_model.input, predictions)\n    # Make last 4 layers trainable if lastFourTrainable == True\n    if lastFourTrainable == True:\n        resnet_model.get_layer('conv5_block3_2_bn').trainable = True\n        resnet_model.get_layer('conv5_block3_3_conv').trainable = True\n        resnet_model.get_layer('conv5_block3_3_bn').trainable = True\n        resnet_model.get_layer('new_fc').trainable = True\n    # Compile ResNet-50 model\n    resnet_model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n    resnet_model.summary()\n    return resnet_model","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:34:56.260013Z","iopub.execute_input":"2023-03-09T11:34:56.260267Z","iopub.status.idle":"2023-03-09T11:34:56.268706Z","shell.execute_reply.started":"2023-03-09T11:34:56.260234Z","shell.execute_reply":"2023-03-09T11:34:56.267808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"traindf = pd.read_csv(\"../input/landmark-recognition-2021/train.csv\")\ntraindf.head()","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:34:56.270328Z","iopub.execute_input":"2023-03-09T11:34:56.2707Z","iopub.status.idle":"2023-03-09T11:34:57.984061Z","shell.execute_reply.started":"2023-03-09T11:34:56.270666Z","shell.execute_reply":"2023-03-09T11:34:57.983318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"traindf.shape","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:34:57.985272Z","iopub.execute_input":"2023-03-09T11:34:57.987884Z","iopub.status.idle":"2023-03-09T11:34:57.993523Z","shell.execute_reply.started":"2023-03-09T11:34:57.987855Z","shell.execute_reply":"2023-03-09T11:34:57.992517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"landmark_unique = traindf['landmark_id'].unique()\nlen(landmark_unique)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:34:57.997705Z","iopub.execute_input":"2023-03-09T11:34:57.997917Z","iopub.status.idle":"2023-03-09T11:34:58.027416Z","shell.execute_reply.started":"2023-03-09T11:34:57.997891Z","shell.execute_reply":"2023-03-09T11:34:58.026618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"landmark_unique[0:50]","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:34:58.028926Z","iopub.execute_input":"2023-03-09T11:34:58.0292Z","iopub.status.idle":"2023-03-09T11:34:58.038315Z","shell.execute_reply.started":"2023-03-09T11:34:58.029155Z","shell.execute_reply":"2023-03-09T11:34:58.037512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_ids = []\nlabels = []\ntemp_labels = []\ni=0\nfor id_ in landmark_unique[0:50]:\n    for iid in traindf['id'][traindf['landmark_id'] == id_]:\n        image_ids.append(iid)\n        labels.append(id_)\n        temp_labels.append(i)\n    i = i+1\nlen(image_ids)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:34:58.039558Z","iopub.execute_input":"2023-03-09T11:34:58.040349Z","iopub.status.idle":"2023-03-09T11:34:58.160737Z","shell.execute_reply.started":"2023-03-09T11:34:58.0403Z","shell.execute_reply":"2023-03-09T11:34:58.160048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mainpath = '../input/landmark-recognition-2021/train'\nimage_path = []\nimages_pixels = []\n\nfor i in range(0,len(image_ids)):\n    first_dir = os.path.join(mainpath,image_ids[i][0])\n    second_dir = os.path.join(first_dir,image_ids[i][1])\n    third_dir = os.path.join(second_dir,image_ids[i][2])\n    finalpath = os.path.join(third_dir,image_ids[i]+'.jpg')\n    \n    img_pix = cv2.imread(finalpath,1)\n    images_pixels.append(cv2.resize(img_pix, (224,224)))\n    \n    image_path.append(finalpath)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:34:58.162239Z","iopub.execute_input":"2023-03-09T11:34:58.162644Z","iopub.status.idle":"2023-03-09T11:35:18.75798Z","shell.execute_reply.started":"2023-03-09T11:34:58.162607Z","shell.execute_reply":"2023-03-09T11:35:18.757184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Images: ', len(image_path))\nprint('Image labels: ', len(labels))","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:35:18.760433Z","iopub.execute_input":"2023-03-09T11:35:18.760976Z","iopub.status.idle":"2023-03-09T11:35:18.767293Z","shell.execute_reply.started":"2023-03-09T11:35:18.760936Z","shell.execute_reply":"2023-03-09T11:35:18.766534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(temp_labels)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:35:18.768869Z","iopub.execute_input":"2023-03-09T11:35:18.769291Z","iopub.status.idle":"2023-03-09T11:35:18.799156Z","shell.execute_reply.started":"2023-03-09T11:35:18.769254Z","shell.execute_reply":"2023-03-09T11:35:18.798315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:35:18.800824Z","iopub.execute_input":"2023-03-09T11:35:18.801111Z","iopub.status.idle":"2023-03-09T11:35:18.806352Z","shell.execute_reply.started":"2023-03-09T11:35:18.801076Z","shell.execute_reply":"2023-03-09T11:35:18.805513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.gcf()\nfig.set_size_inches(16, 16)\n\nnext_pix_ = image_path\n\nfor i, img_path in enumerate(next_pix_[0:16]):\n    \n    sp = plt.subplot(5, 4, i + 1)\n    sp.axis('Off')\n\n    img = cv2.imread(img_path)\n    plt.imshow(img)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:35:18.808238Z","iopub.execute_input":"2023-03-09T11:35:18.808534Z","iopub.status.idle":"2023-03-09T11:35:20.459369Z","shell.execute_reply.started":"2023-03-09T11:35:18.8085Z","shell.execute_reply":"2023-03-09T11:35:20.458731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.gcf()\nfig.set_size_inches(16, 16)\n\nnext_pix = image_path\nrandom.shuffle(next_pix)\n\nfor i, img_path in enumerate(next_pix[0:12]):\n    \n    sp = plt.subplot(4, 4, i + 1)\n    sp.axis('Off')\n\n    img = cv2.imread(img_path)\n    plt.imshow(img)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:35:20.460262Z","iopub.execute_input":"2023-03-09T11:35:20.46046Z","iopub.status.idle":"2023-03-09T11:35:21.843829Z","shell.execute_reply.started":"2023-03-09T11:35:20.460433Z","shell.execute_reply":"2023-03-09T11:35:21.843198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shuf = list(zip(images_pixels,temp_labels))\nrandom.shuffle(shuf)\n\ntrain_data, labels_data = zip(*shuf)\nprint('Images: ', len(train_data))\nprint('Image labels: ', len(labels_data))","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:35:21.844973Z","iopub.execute_input":"2023-03-09T11:35:21.845714Z","iopub.status.idle":"2023-03-09T11:35:21.854139Z","shell.execute_reply.started":"2023-03-09T11:35:21.845666Z","shell.execute_reply":"2023-03-09T11:35:21.853513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(labels_data)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:35:21.855229Z","iopub.execute_input":"2023-03-09T11:35:21.855879Z","iopub.status.idle":"2023-03-09T11:35:21.861874Z","shell.execute_reply.started":"2023-03-09T11:35:21.855846Z","shell.execute_reply":"2023-03-09T11:35:21.861137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''# Function for plotting the Confusion Matrix\ndef plotConfusionMatrix(cm, class_names, normalize=True, title='Confusion matrix', cmap=plt.cm.Blues):\n    plt.figure(figsize=(10,10))\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(class_names))\n    plt.xticks(tick_marks, class_names, rotation=45)\n    plt.yticks(tick_marks, class_names)\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n        cm = np.around(cm, decimals=2)\n        cm[np.isnan(cm)] = 0.0\n        print(\"Normalized confusion matrix\")\n    else:\n        print('Confusion matrix, without normalization')\n\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, cm[i, j],\n                horizontalalignment=\"center\",\n                color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n    plt.show()\n    '''","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:35:21.862965Z","iopub.execute_input":"2023-03-09T11:35:21.863582Z","iopub.status.idle":"2023-03-09T11:35:21.87222Z","shell.execute_reply.started":"2023-03-09T11:35:21.863538Z","shell.execute_reply":"2023-03-09T11:35:21.871343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n# Function for train the given model and get Confusion Matrix\ndef trainModelAndGetConfusionMatrix(model,train_generator,valid_generator,test_generator,epochs,batch_size):\n    # Fit the model\n    model.fit_generator(train_generator,\n                      epochs=epochs,\n                      steps_per_epoch=len(train_generator) // batch_size,\n                      validation_data=valid_generator,\n                      validation_steps=len(valid_generator) // batch_size)\n    # Evaluate the model\n    loss_and_metrics = model.evaluate_generator(test_generator, steps=len(test_generator) // batch_size+1)\n    print(\"Test Loss: \", loss_and_metrics[0])\n    print(\"Test Accuracy: \", loss_and_metrics[1])\n    test_generator.reset()\n    # Get Confusion Matrix and plot it\n    Y_pred = model.predict_generator(test_generator, steps=len(test_generator) // batch_size+1)\n    y_pred = np.argmax(Y_pred, axis=1)\n    class_names = []\n    for key in train_generator.class_indices:\n        class_names.append(key)\n    cr = classification_report(test_generator.classes, y_pred, target_names=class_names)\n    print(\"Classification Report\\n\" + cr) # Print classification report\n    cm = confusion_matrix(test_generator.classes, y_pred)\n    plotConfusionMatrix(cm, class_names, normalize=False, title='Confusion Matrix')\n    plotConfusionMatrix(cm, class_names, normalize=True, title='Confusion Matrix')\n    return model\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:35:21.873486Z","iopub.execute_input":"2023-03-09T11:35:21.873794Z","iopub.status.idle":"2023-03-09T11:35:21.885139Z","shell.execute_reply.started":"2023-03-09T11:35:21.87376Z","shell.execute_reply":"2023-03-09T11:35:21.884245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = np.array(train_data) #/ 255\n","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:35:21.88678Z","iopub.execute_input":"2023-03-09T11:35:21.887245Z","iopub.status.idle":"2023-03-09T11:35:21.950549Z","shell.execute_reply.started":"2023-03-09T11:35:21.887211Z","shell.execute_reply":"2023-03-09T11:35:21.949762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a = np.array(labels_data)\na_pd = pd.get_dummies(a).astype('float32').values ","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:35:21.952028Z","iopub.execute_input":"2023-03-09T11:35:21.952308Z","iopub.status.idle":"2023-03-09T11:35:21.961841Z","shell.execute_reply.started":"2023-03-09T11:35:21.952272Z","shell.execute_reply":"2023-03-09T11:35:21.961002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(labels_data[0:6])","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:35:21.963221Z","iopub.execute_input":"2023-03-09T11:35:21.963517Z","iopub.status.idle":"2023-03-09T11:35:21.969272Z","shell.execute_reply.started":"2023-03-09T11:35:21.963482Z","shell.execute_reply":"2023-03-09T11:35:21.968388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(a_pd)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:35:21.970757Z","iopub.execute_input":"2023-03-09T11:35:21.971158Z","iopub.status.idle":"2023-03-09T11:35:21.978391Z","shell.execute_reply.started":"2023-03-09T11:35:21.971125Z","shell.execute_reply":"2023-03-09T11:35:21.977529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_data=a_pd","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:35:21.979882Z","iopub.execute_input":"2023-03-09T11:35:21.980149Z","iopub.status.idle":"2023-03-09T11:35:21.986152Z","shell.execute_reply.started":"2023-03-09T11:35:21.980117Z","shell.execute_reply":"2023-03-09T11:35:21.985235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(labels_data[0:5])","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:35:21.995313Z","iopub.execute_input":"2023-03-09T11:35:21.99552Z","iopub.status.idle":"2023-03-09T11:35:22.001417Z","shell.execute_reply.started":"2023-03-09T11:35:21.995494Z","shell.execute_reply":"2023-03-09T11:35:22.000534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_data.shape","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:35:22.002812Z","iopub.execute_input":"2023-03-09T11:35:22.003896Z","iopub.status.idle":"2023-03-09T11:35:22.011915Z","shell.execute_reply.started":"2023-03-09T11:35:22.00386Z","shell.execute_reply":"2023-03-09T11:35:22.010998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_train, X_test, Y_train, Y_test = train_test_split(train_data, labels_data, test_size = 0.3, random_state=101)\n\nprint(\"X train data : \", len(X_train))\nprint(\"X label data : \", len(X_test))\nprint(\"Y test data : \", len(Y_train))\nprint(\"Y label data : \", len(Y_test))","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:35:22.013378Z","iopub.execute_input":"2023-03-09T11:35:22.013743Z","iopub.status.idle":"2023-03-09T11:35:22.087108Z","shell.execute_reply.started":"2023-03-09T11:35:22.013709Z","shell.execute_reply":"2023-03-09T11:35:22.086136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_sample=X_train.shape[0]\n#20% of the enteries has to be in validation set\nvalidation_freq=int(num_sample*0.2)\n#Generating random sample of indices equal to validation_freq\nvalidationlist = random.sample(range(0, num_sample), validation_freq)\ntraininglist=list(set(range(0,num_sample))-set(validationlist))\nprint(\"No interesection between validationlist and traininglist:\",set(traininglist).intersection(validationlist))","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:35:22.088757Z","iopub.execute_input":"2023-03-09T11:35:22.08903Z","iopub.status.idle":"2023-03-09T11:35:22.096109Z","shell.execute_reply.started":"2023-03-09T11:35:22.088993Z","shell.execute_reply":"2023-03-09T11:35:22.095123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_x_final=[]\ntrain_y_final=[]\nfor val in traininglist:\n    train_x_final.append(X_train[val])\n    train_y_final.append(Y_train[val])\ntrain_x_final=np.array(train_x_final)\ntrain_y_final=np.array(train_y_final)\nprint(\"Training data shape\",train_x_final.shape)\nprint(\"Training label shape\",train_y_final.shape)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:35:22.097617Z","iopub.execute_input":"2023-03-09T11:35:22.098032Z","iopub.status.idle":"2023-03-09T11:35:22.138788Z","shell.execute_reply.started":"2023-03-09T11:35:22.097995Z","shell.execute_reply":"2023-03-09T11:35:22.137845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation_x_final=[]\nvalidation_y_final=[]\nfor val in validationlist:\n    validation_x_final.append(X_train[val])\n    validation_y_final.append(Y_train[val])\nvalidation_x_final=np.array(validation_x_final)\nvalidation_y_final=np.array(validation_y_final)\nprint(\"Validation data shape\",validation_x_final.shape)\nprint(\"Validation label shape\",validation_y_final.shape)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:35:22.140353Z","iopub.execute_input":"2023-03-09T11:35:22.140627Z","iopub.status.idle":"2023-03-09T11:35:22.156646Z","shell.execute_reply.started":"2023-03-09T11:35:22.140591Z","shell.execute_reply":"2023-03-09T11:35:22.1559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot the training results\ndef plot_hist(history,title):\n    plt.subplot(121)\n    plt.plot(history.history['accuracy'])\n    plt.plot(history.history['val_accuracy'])\n    plt.title(str(title)+' accuracy')\n    plt.ylabel('Accuracy')\n    plt.xlabel('Epoch')\n    plt.legend(['Train', 'Test'], loc='upper left')\n\n    # Plot training & validation loss values\n    plt.subplot(122)\n    plt.plot(history.history['loss'])\n    plt.plot(history.history['val_loss'])\n    plt.title(str(title)+' loss')\n    plt.ylabel('Loss')\n    plt.xlabel('Epoch')\n    plt.legend(['Train', 'Test'], loc='upper left')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:35:22.157893Z","iopub.execute_input":"2023-03-09T11:35:22.158327Z","iopub.status.idle":"2023-03-09T11:35:22.164983Z","shell.execute_reply.started":"2023-03-09T11:35:22.158291Z","shell.execute_reply":"2023-03-09T11:35:22.164065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get VGG-16 Model\ndef getVGG16Model(lastFourTrainable=False):\n    vgg_model = VGG16(weights='imagenet', input_shape=input_shape, include_top=True)\n    # Make all layers untrainable\n    for layer in vgg_model.layers[:]:\n        layer.trainable = False\n    # Add fully connected layer which have 1024 neuron to VGG-16 model\n    output = vgg_model.get_layer('fc2').output\n    output = Flatten(name='new_flatten')(output)\n    output = Dense(units=1024, activation='relu', name='new_fc')(output)\n    output = Dense(units=50, activation='softmax')(output)\n    vgg_model = Model(vgg_model.input, output)\n    # Make last 4 layers trainable if lastFourTrainable == True\n    if lastFourTrainable == True:\n        vgg_model.get_layer('block5_conv3').trainable = True\n        vgg_model.get_layer('fc1').trainable = True\n        vgg_model.get_layer('fc2').trainable = True\n        vgg_model.get_layer('new_fc').trainable = True\n    # Compile VGG-16 model\n    vgg_model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n    vgg_model.summary()\n\n    return vgg_model","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:35:22.166401Z","iopub.execute_input":"2023-03-09T11:35:22.166845Z","iopub.status.idle":"2023-03-09T11:35:22.177078Z","shell.execute_reply.started":"2023-03-09T11:35:22.166811Z","shell.execute_reply":"2023-03-09T11:35:22.175972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get VGG-19 Model\ndef getVGG19Model(lastFourTrainable=False):\n    vgg_model_19 = VGG19(weights='imagenet', input_shape=input_shape, include_top=True)\n    # Make all layers untrainable\n    for layer in vgg_model_19.layers[:]:\n        layer.trainable = False\n    # Add fully connected layer which have 1024 neuron to VGG-16 model\n    output = vgg_model_19.get_layer('fc2').output\n    output = Flatten(name='new_flatten')(output)\n    output = Dense(units=1024, activation='relu', name='new_fc')(output)\n    output = Dense(units=50, activation='softmax')(output)\n    vgg_model_19 = Model(vgg_model_19.input, output)\n    # Make last 4 layers trainable if lastFourTrainable == True\n    if lastFourTrainable == True:\n        vgg_model_19.get_layer('block5_conv3').trainable = True\n        vgg_model_19.get_layer('fc1').trainable = True\n        vgg_model_19.get_layer('fc2').trainable = True\n        vgg_model_19.get_layer('new_fc').trainable = True\n    # Compile VGG-16 model\n    vgg_model_19.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n    vgg_model_19.summary()\n\n    return vgg_model_19","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:35:22.178447Z","iopub.execute_input":"2023-03-09T11:35:22.17886Z","iopub.status.idle":"2023-03-09T11:35:22.188206Z","shell.execute_reply.started":"2023-03-09T11:35:22.178826Z","shell.execute_reply":"2023-03-09T11:35:22.187523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vgg_model19_a = getVGG19Model(lastFourTrainable=False)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:35:22.189449Z","iopub.execute_input":"2023-03-09T11:35:22.189826Z","iopub.status.idle":"2023-03-09T11:35:50.777446Z","shell.execute_reply.started":"2023-03-09T11:35:22.189789Z","shell.execute_reply":"2023-03-09T11:35:50.77675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vgg_model19_b = getVGG19Model(lastFourTrainable=True)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:35:50.778753Z","iopub.execute_input":"2023-03-09T11:35:50.778986Z","iopub.status.idle":"2023-03-09T11:35:52.89299Z","shell.execute_reply.started":"2023-03-09T11:35:50.778954Z","shell.execute_reply":"2023-03-09T11:35:52.892287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get ResNet-50 Model with lastFourTrainable=False\nresnet_model_a = getResNet50Model(lastFourTrainable=False)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:35:52.894205Z","iopub.execute_input":"2023-03-09T11:35:52.894448Z","iopub.status.idle":"2023-03-09T11:35:59.22186Z","shell.execute_reply.started":"2023-03-09T11:35:52.894414Z","shell.execute_reply":"2023-03-09T11:35:59.221125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def score_train(model,test_x,test_y):\n    # Score trained model.\n    train_scores = model.evaluate(test_x, test_y, verbose=1)\n    print('Test loss:', train_scores[0])\n    print('Test accuracy:', train_scores[1])","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:35:59.223277Z","iopub.execute_input":"2023-03-09T11:35:59.223527Z","iopub.status.idle":"2023-03-09T11:35:59.229068Z","shell.execute_reply.started":"2023-03-09T11:35:59.223493Z","shell.execute_reply":"2023-03-09T11:35:59.228373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.optimizers import Adam","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:35:59.230436Z","iopub.execute_input":"2023-03-09T11:35:59.230694Z","iopub.status.idle":"2023-03-09T11:36:00.026807Z","shell.execute_reply.started":"2023-03-09T11:35:59.230653Z","shell.execute_reply":"2023-03-09T11:36:00.026016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(validation_x_final.shape)\nprint(validation_y_final.shape)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:36:00.027866Z","iopub.execute_input":"2023-03-09T11:36:00.029697Z","iopub.status.idle":"2023-03-09T11:36:00.039381Z","shell.execute_reply.started":"2023-03-09T11:36:00.029649Z","shell.execute_reply":"2023-03-09T11:36:00.03867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:36:00.040589Z","iopub.execute_input":"2023-03-09T11:36:00.04127Z","iopub.status.idle":"2023-03-09T11:36:00.045998Z","shell.execute_reply.started":"2023-03-09T11:36:00.041235Z","shell.execute_reply":"2023-03-09T11:36:00.045102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_x_final_resnet=tensorflow.keras.applications.resnet.preprocess_input(train_x_final)\nvalidation_x_final_resnet=tensorflow.keras.applications.resnet.preprocess_input(validation_x_final)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:36:00.047336Z","iopub.execute_input":"2023-03-09T11:36:00.048303Z","iopub.status.idle":"2023-03-09T11:36:00.328655Z","shell.execute_reply.started":"2023-03-09T11:36:00.048266Z","shell.execute_reply":"2023-03-09T11:36:00.327892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_x_final_resnet=tensorflow.keras.applications.resnet.preprocess_input(X_test)\n","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:36:00.329926Z","iopub.execute_input":"2023-03-09T11:36:00.330275Z","iopub.status.idle":"2023-03-09T11:36:00.453998Z","shell.execute_reply.started":"2023-03-09T11:36:00.330239Z","shell.execute_reply":"2023-03-09T11:36:00.453252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"RESNET-50-a","metadata":{}},{"cell_type":"code","source":"# Train ResNet-50 Model \n#resnet_model_a.compile(optimizer=Adam(learning_rate=0.001),loss='categorical_crossentropy',metrics=['accuracy'])\nhistory1 = resnet_model_a.fit(train_x_final_resnet, train_y_final, epochs=30,validation_data=(validation_x_final_resnet, validation_y_final))\n#resnet_model_a = trainModelAndGetConfusionMatrix(resnet_model_a,train_x_final,validation_x_final,X_test,10,64)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:36:00.455155Z","iopub.execute_input":"2023-03-09T11:36:00.455488Z","iopub.status.idle":"2023-03-09T11:36:56.072907Z","shell.execute_reply.started":"2023-03-09T11:36:00.455428Z","shell.execute_reply":"2023-03-09T11:36:56.072183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_hist(history1,'Resnet-50')","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:36:56.080397Z","iopub.execute_input":"2023-03-09T11:36:56.080622Z","iopub.status.idle":"2023-03-09T11:36:56.366108Z","shell.execute_reply.started":"2023-03-09T11:36:56.080594Z","shell.execute_reply":"2023-03-09T11:36:56.365326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score_train(resnet_model_a,test_x_final_resnet,Y_test)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:36:56.367349Z","iopub.execute_input":"2023-03-09T11:36:56.36808Z","iopub.status.idle":"2023-03-09T11:36:59.012682Z","shell.execute_reply.started":"2023-03-09T11:36:56.368042Z","shell.execute_reply":"2023-03-09T11:36:59.011894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Resnet-50-b","metadata":{}},{"cell_type":"code","source":"resnet_model_b = getResNet50Model(lastFourTrainable=True)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:36:59.015855Z","iopub.execute_input":"2023-03-09T11:36:59.016065Z","iopub.status.idle":"2023-03-09T11:37:00.486544Z","shell.execute_reply.started":"2023-03-09T11:36:59.016038Z","shell.execute_reply":"2023-03-09T11:37:00.485792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train ResNet-50 Model \n#resnet_model_a.compile(optimizer=Adam(learning_rate=0.001),loss='categorical_crossentropy',metrics=['accuracy'])\nhistory = resnet_model_b.fit(train_x_final_resnet, train_y_final, epochs=30,validation_data=(validation_x_final_resnet, validation_y_final))\n#resnet_model_a = trainModelAndGetConfusionMatrix(resnet_model_a,train_x_final,validation_x_final,X_test,10,64)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:37:00.48789Z","iopub.execute_input":"2023-03-09T11:37:00.488122Z","iopub.status.idle":"2023-03-09T11:37:51.638255Z","shell.execute_reply.started":"2023-03-09T11:37:00.488088Z","shell.execute_reply":"2023-03-09T11:37:51.637516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_hist(history,'Resnet-50-b')","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:37:51.644775Z","iopub.execute_input":"2023-03-09T11:37:51.645001Z","iopub.status.idle":"2023-03-09T11:37:51.935415Z","shell.execute_reply.started":"2023-03-09T11:37:51.644966Z","shell.execute_reply":"2023-03-09T11:37:51.934714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score_train(resnet_model_b,test_x_final_resnet,Y_test)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:37:51.936755Z","iopub.execute_input":"2023-03-09T11:37:51.937025Z","iopub.status.idle":"2023-03-09T11:37:55.094908Z","shell.execute_reply.started":"2023-03-09T11:37:51.936996Z","shell.execute_reply":"2023-03-09T11:37:55.09409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"VGG-16-a","metadata":{}},{"cell_type":"code","source":"train_x_final_vgg16=tensorflow.keras.applications.vgg16.preprocess_input(train_x_final)\nvalidation_x_final_vgg16=tensorflow.keras.applications.vgg16.preprocess_input(validation_x_final)\n","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:37:55.096402Z","iopub.execute_input":"2023-03-09T11:37:55.096934Z","iopub.status.idle":"2023-03-09T11:37:55.365565Z","shell.execute_reply.started":"2023-03-09T11:37:55.096894Z","shell.execute_reply":"2023-03-09T11:37:55.364781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_x_final_vgg16=tensorflow.keras.applications.vgg16.preprocess_input(X_test)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:37:55.366992Z","iopub.execute_input":"2023-03-09T11:37:55.367259Z","iopub.status.idle":"2023-03-09T11:37:55.490382Z","shell.execute_reply.started":"2023-03-09T11:37:55.367223Z","shell.execute_reply":"2023-03-09T11:37:55.489602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vgga=getVGG16Model(lastFourTrainable=False)\n","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:37:55.491863Z","iopub.execute_input":"2023-03-09T11:37:55.492331Z","iopub.status.idle":"2023-03-09T11:38:19.879726Z","shell.execute_reply.started":"2023-03-09T11:37:55.492293Z","shell.execute_reply":"2023-03-09T11:38:19.878976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = vgga.fit(train_x_final_vgg16, train_y_final,validation_data=(validation_x_final_vgg16,validation_y_final), epochs=30)\n","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:38:19.88087Z","iopub.execute_input":"2023-03-09T11:38:19.88111Z","iopub.status.idle":"2023-03-09T11:39:20.113766Z","shell.execute_reply.started":"2023-03-09T11:38:19.881076Z","shell.execute_reply":"2023-03-09T11:39:20.112993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_hist(history,'VGG-16')","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:39:20.1194Z","iopub.execute_input":"2023-03-09T11:39:20.119627Z","iopub.status.idle":"2023-03-09T11:39:20.405048Z","shell.execute_reply.started":"2023-03-09T11:39:20.1196Z","shell.execute_reply":"2023-03-09T11:39:20.404329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score_train(vgga,test_x_final_vgg16,Y_test)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:39:20.406377Z","iopub.execute_input":"2023-03-09T11:39:20.406763Z","iopub.status.idle":"2023-03-09T11:39:24.353227Z","shell.execute_reply.started":"2023-03-09T11:39:20.406726Z","shell.execute_reply":"2023-03-09T11:39:24.352515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vggb=getVGG16Model(lastFourTrainable=True)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:39:24.354745Z","iopub.execute_input":"2023-03-09T11:39:24.355547Z","iopub.status.idle":"2023-03-09T11:39:26.537173Z","shell.execute_reply.started":"2023-03-09T11:39:24.355507Z","shell.execute_reply":"2023-03-09T11:39:26.536427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history_vgg16_b = vggb.fit(train_x_final_vgg16, train_y_final,validation_data=(validation_x_final_vgg16,validation_y_final), epochs=30)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:39:26.538576Z","iopub.execute_input":"2023-03-09T11:39:26.538824Z","iopub.status.idle":"2023-03-09T11:40:30.020055Z","shell.execute_reply.started":"2023-03-09T11:39:26.538789Z","shell.execute_reply":"2023-03-09T11:40:30.019296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_hist(history_vgg16_b,'Vgg-16-b')","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:40:30.026342Z","iopub.execute_input":"2023-03-09T11:40:30.026573Z","iopub.status.idle":"2023-03-09T11:40:30.334327Z","shell.execute_reply.started":"2023-03-09T11:40:30.026546Z","shell.execute_reply":"2023-03-09T11:40:30.333623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score_train(vggb,test_x_final_vgg16,Y_test)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:40:30.335521Z","iopub.execute_input":"2023-03-09T11:40:30.335845Z","iopub.status.idle":"2023-03-09T11:40:31.806496Z","shell.execute_reply.started":"2023-03-09T11:40:30.335809Z","shell.execute_reply":"2023-03-09T11:40:31.805628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"VGG-19-a","metadata":{}},{"cell_type":"code","source":"train_x_final_vgg19=tensorflow.keras.applications.vgg19.preprocess_input(train_x_final)\nvalidation_x_final_vgg19=tensorflow.keras.applications.vgg19.preprocess_input(validation_x_final)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:40:31.808232Z","iopub.execute_input":"2023-03-09T11:40:31.808511Z","iopub.status.idle":"2023-03-09T11:40:32.348139Z","shell.execute_reply.started":"2023-03-09T11:40:31.808458Z","shell.execute_reply":"2023-03-09T11:40:32.347376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_x_final_vgg19=tensorflow.keras.applications.vgg19.preprocess_input(X_test)\n","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:40:32.349422Z","iopub.execute_input":"2023-03-09T11:40:32.349688Z","iopub.status.idle":"2023-03-09T11:40:32.689019Z","shell.execute_reply.started":"2023-03-09T11:40:32.349646Z","shell.execute_reply":"2023-03-09T11:40:32.688269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history_vgg19_a = vgg_model19_a.fit(train_x_final_vgg19, train_y_final,validation_data=(validation_x_final_vgg19,validation_y_final), epochs=30)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:40:32.690266Z","iopub.execute_input":"2023-03-09T11:40:32.690535Z","iopub.status.idle":"2023-03-09T11:41:38.780231Z","shell.execute_reply.started":"2023-03-09T11:40:32.690499Z","shell.execute_reply":"2023-03-09T11:41:38.779515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score_train(vgg_model19_a,test_x_final_vgg19,Y_test)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:41:38.787057Z","iopub.execute_input":"2023-03-09T11:41:38.787262Z","iopub.status.idle":"2023-03-09T11:41:40.464146Z","shell.execute_reply.started":"2023-03-09T11:41:38.787236Z","shell.execute_reply":"2023-03-09T11:41:40.463349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_hist(history_vgg19_a,'vgg-19-a')","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:41:40.465758Z","iopub.execute_input":"2023-03-09T11:41:40.466013Z","iopub.status.idle":"2023-03-09T11:41:40.776775Z","shell.execute_reply.started":"2023-03-09T11:41:40.465979Z","shell.execute_reply":"2023-03-09T11:41:40.77587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"VGG-19-b","metadata":{}},{"cell_type":"code","source":"history_vgg19_b = vgg_model19_b.fit(train_x_final_vgg19, train_y_final,validation_data=(validation_x_final_vgg19,validation_y_final), epochs=30)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:41:40.777823Z","iopub.execute_input":"2023-03-09T11:41:40.778191Z","iopub.status.idle":"2023-03-09T11:42:54.292898Z","shell.execute_reply.started":"2023-03-09T11:41:40.778156Z","shell.execute_reply":"2023-03-09T11:42:54.292148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_hist(history_vgg19_b,'VGG-19-b')","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:42:54.300201Z","iopub.execute_input":"2023-03-09T11:42:54.300561Z","iopub.status.idle":"2023-03-09T11:42:54.624909Z","shell.execute_reply.started":"2023-03-09T11:42:54.300522Z","shell.execute_reply":"2023-03-09T11:42:54.624189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score_train(vgg_model19_b,test_x_final_vgg19,Y_test)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:42:54.62624Z","iopub.execute_input":"2023-03-09T11:42:54.627622Z","iopub.status.idle":"2023-03-09T11:42:56.721987Z","shell.execute_reply.started":"2023-03-09T11:42:54.627582Z","shell.execute_reply":"2023-03-09T11:42:56.721173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"DenseNet","metadata":{}},{"cell_type":"code","source":"from keras.applications.densenet import DenseNet121","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:42:56.723449Z","iopub.execute_input":"2023-03-09T11:42:56.723791Z","iopub.status.idle":"2023-03-09T11:42:56.72801Z","shell.execute_reply.started":"2023-03-09T11:42:56.723754Z","shell.execute_reply":"2023-03-09T11:42:56.727279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get DenseNet-121 Model\ndef getDenseNet121Model(lastFourTrainable=False):\n    densenet_model = DenseNet121(weights='imagenet', input_shape=input_shape, include_top=True)\n    # Make all layers non-trainable\n    for layer in densenet_model.layers[:]:\n        layer.trainable = False\n    # Add fully connected layer which have 1024 neuron to ResNet-50 model\n    output = densenet_model.get_layer('avg_pool').output\n    output = Flatten(name='new_flatten')(output)\n    output = Dense(units=1024, activation='relu', name='new_fc')(output)\n    predictions = Dense(units=50, activation='softmax')(output)\n    densenet_model = Model(densenet_model.input, predictions)\n    # Make last 4 layers trainable if lastFourTrainable == True\n    if lastFourTrainable == True:\n        densenet_model.get_layer('conv5_block3_2_bn').trainable = True\n        densenet_model.get_layer('conv5_block3_3_conv').trainable = True\n        densenet_model.get_layer('conv5_block3_3_bn').trainable = True\n        densenet_model.get_layer('new_fc').trainable = True\n    # Compile ResNet-50 model\n    densenet_model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n    densenet_model.summary()\n    return densenet_model","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:42:56.729441Z","iopub.execute_input":"2023-03-09T11:42:56.729975Z","iopub.status.idle":"2023-03-09T11:42:56.739082Z","shell.execute_reply.started":"2023-03-09T11:42:56.729939Z","shell.execute_reply":"2023-03-09T11:42:56.738248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get DenseNet-121 Model with lastFourTrainable=False\ndensenet_model_a = getDenseNet121Model(lastFourTrainable=False)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:42:56.740627Z","iopub.execute_input":"2023-03-09T11:42:56.740918Z","iopub.status.idle":"2023-03-09T11:43:01.878844Z","shell.execute_reply.started":"2023-03-09T11:42:56.740883Z","shell.execute_reply":"2023-03-09T11:43:01.878149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_x_final_densenet=tensorflow.keras.applications.densenet.preprocess_input(train_x_final)\nvalidation_x_final_densenet=tensorflow.keras.applications.densenet.preprocess_input(validation_x_final)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:43:01.880199Z","iopub.execute_input":"2023-03-09T11:43:01.880456Z","iopub.status.idle":"2023-03-09T11:43:02.633386Z","shell.execute_reply.started":"2023-03-09T11:43:01.880411Z","shell.execute_reply":"2023-03-09T11:43:02.632602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_x_final_densenet=tensorflow.keras.applications.densenet.preprocess_input(X_test)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:43:02.634745Z","iopub.execute_input":"2023-03-09T11:43:02.634997Z","iopub.status.idle":"2023-03-09T11:43:03.060182Z","shell.execute_reply.started":"2023-03-09T11:43:02.634958Z","shell.execute_reply":"2023-03-09T11:43:03.059387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train DENSETNET-121 Model \n#resnet_model_a.compile(optimizer=Adam(learning_rate=0.001),loss='categorical_crossentropy',metrics=['accuracy'])\nhistory_densenet_model_a = densenet_model_a.fit(train_x_final_densenet, train_y_final, epochs=30,validation_data=(validation_x_final_densenet, validation_y_final))\n#resnet_model_a = trainModelAndGetConfusionMatrix(resnet_model_a,train_x_final,validation_x_final,X_test,10,64)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:43:03.061448Z","iopub.execute_input":"2023-03-09T11:43:03.061746Z","iopub.status.idle":"2023-03-09T11:43:56.920524Z","shell.execute_reply.started":"2023-03-09T11:43:03.061705Z","shell.execute_reply":"2023-03-09T11:43:56.919751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_hist(history_densenet_model_a,'Densenet')\nscore_train(densenet_model_a,test_x_final_densenet,Y_test)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:43:56.926716Z","iopub.execute_input":"2023-03-09T11:43:56.92693Z","iopub.status.idle":"2023-03-09T11:44:00.543583Z","shell.execute_reply.started":"2023-03-09T11:43:56.926903Z","shell.execute_reply":"2023-03-09T11:44:00.542768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## KNN- Image Retrieval","metadata":{}},{"cell_type":"markdown","source":"Using resnet-50-a for feature extraction and KNN","metadata":{}},{"cell_type":"code","source":"from keras.models import Model","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:44:00.545345Z","iopub.execute_input":"2023-03-09T11:44:00.545795Z","iopub.status.idle":"2023-03-09T11:44:00.549602Z","shell.execute_reply.started":"2023-03-09T11:44:00.545759Z","shell.execute_reply":"2023-03-09T11:44:00.548729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_model_vgg_model19_a= Model(inputs=vgg_model19_a.input, outputs=vgg_model19_a.get_layer('new_fc').output)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:44:00.550885Z","iopub.execute_input":"2023-03-09T11:44:00.551309Z","iopub.status.idle":"2023-03-09T11:44:00.565301Z","shell.execute_reply.started":"2023-03-09T11:44:00.551275Z","shell.execute_reply":"2023-03-09T11:44:00.564496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_vects=feature_model_vgg_model19_a.predict(train_x_final_vgg19)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:44:00.566579Z","iopub.execute_input":"2023-03-09T11:44:00.56682Z","iopub.status.idle":"2023-03-09T11:44:04.248301Z","shell.execute_reply.started":"2023-03-09T11:44:00.566789Z","shell.execute_reply":"2023-03-09T11:44:04.247533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_vects.shape","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:44:04.249823Z","iopub.execute_input":"2023-03-09T11:44:04.250067Z","iopub.status.idle":"2023-03-09T11:44:04.258344Z","shell.execute_reply.started":"2023-03-09T11:44:04.250033Z","shell.execute_reply":"2023-03-09T11:44:04.257507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.neighbors import KNeighborsClassifier as KNeighborsClassifier","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:44:04.266114Z","iopub.execute_input":"2023-03-09T11:44:04.266319Z","iopub.status.idle":"2023-03-09T11:44:04.364431Z","shell.execute_reply.started":"2023-03-09T11:44:04.266295Z","shell.execute_reply":"2023-03-09T11:44:04.363699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import GridSearchCV","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:44:04.365624Z","iopub.execute_input":"2023-03-09T11:44:04.367545Z","iopub.status.idle":"2023-03-09T11:44:04.37108Z","shell.execute_reply.started":"2023-03-09T11:44:04.367506Z","shell.execute_reply":"2023-03-09T11:44:04.370404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_integers=np.argmax(train_y_final, axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:44:04.372482Z","iopub.execute_input":"2023-03-09T11:44:04.372938Z","iopub.status.idle":"2023-03-09T11:44:04.382804Z","shell.execute_reply.started":"2023-03-09T11:44:04.372903Z","shell.execute_reply":"2023-03-09T11:44:04.382014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_integers.shape","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:44:04.383809Z","iopub.execute_input":"2023-03-09T11:44:04.384105Z","iopub.status.idle":"2023-03-09T11:44:04.394064Z","shell.execute_reply.started":"2023-03-09T11:44:04.384071Z","shell.execute_reply":"2023-03-09T11:44:04.393284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"knn = KNeighborsClassifier()\nknn.fit(feature_vects,labels_integers)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:44:04.395542Z","iopub.execute_input":"2023-03-09T11:44:04.395823Z","iopub.status.idle":"2023-03-09T11:44:04.406751Z","shell.execute_reply.started":"2023-03-09T11:44:04.395791Z","shell.execute_reply":"2023-03-09T11:44:04.405887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_vects=feature_model_vgg_model19_a.predict(test_x_final_vgg19)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:44:04.40842Z","iopub.execute_input":"2023-03-09T11:44:04.408719Z","iopub.status.idle":"2023-03-09T11:44:06.34944Z","shell.execute_reply.started":"2023-03-09T11:44:04.408686Z","shell.execute_reply":"2023-03-09T11:44:06.34867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_vects.shape","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:44:06.352241Z","iopub.execute_input":"2023-03-09T11:44:06.352535Z","iopub.status.idle":"2023-03-09T11:44:06.357587Z","shell.execute_reply.started":"2023-03-09T11:44:06.3525Z","shell.execute_reply":"2023-03-09T11:44:06.356904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_test.shape","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:44:06.358987Z","iopub.execute_input":"2023-03-09T11:44:06.35945Z","iopub.status.idle":"2023-03-09T11:44:06.36941Z","shell.execute_reply.started":"2023-03-09T11:44:06.359414Z","shell.execute_reply":"2023-03-09T11:44:06.368546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_label_integers=np.argmax(Y_test, axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:44:06.371005Z","iopub.execute_input":"2023-03-09T11:44:06.371529Z","iopub.status.idle":"2023-03-09T11:44:06.377832Z","shell.execute_reply.started":"2023-03-09T11:44:06.371494Z","shell.execute_reply":"2023-03-09T11:44:06.377056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_label_integers.shape","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:44:06.379242Z","iopub.execute_input":"2023-03-09T11:44:06.379833Z","iopub.status.idle":"2023-03-09T11:44:06.388225Z","shell.execute_reply.started":"2023-03-09T11:44:06.379796Z","shell.execute_reply":"2023-03-09T11:44:06.387175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(knn.score(test_vects,test_label_integers))\n#ypred=knn.predict(fin_test_img)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:44:06.38955Z","iopub.execute_input":"2023-03-09T11:44:06.390021Z","iopub.status.idle":"2023-03-09T11:44:06.443406Z","shell.execute_reply.started":"2023-03-09T11:44:06.389986Z","shell.execute_reply":"2023-03-09T11:44:06.442652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"RETREIVAL of FIRST 7 IMAGES","metadata":{}},{"cell_type":"code","source":"knn7 = KNeighborsClassifier(n_neighbors = 7)\nknn7.fit(feature_vects,labels_integers)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:44:06.444465Z","iopub.execute_input":"2023-03-09T11:44:06.44487Z","iopub.status.idle":"2023-03-09T11:44:06.456137Z","shell.execute_reply.started":"2023-03-09T11:44:06.444836Z","shell.execute_reply":"2023-03-09T11:44:06.455249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"indices=knn7.kneighbors(test_vects, return_distance=False)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:44:06.457616Z","iopub.execute_input":"2023-03-09T11:44:06.458086Z","iopub.status.idle":"2023-03-09T11:44:06.480313Z","shell.execute_reply.started":"2023-03-09T11:44:06.458052Z","shell.execute_reply":"2023-03-09T11:44:06.479337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"indices.shape","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:44:06.481697Z","iopub.execute_input":"2023-03-09T11:44:06.482155Z","iopub.status.idle":"2023-03-09T11:44:06.488579Z","shell.execute_reply.started":"2023-03-09T11:44:06.48212Z","shell.execute_reply":"2023-03-09T11:44:06.487753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.max(indices)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:44:06.489757Z","iopub.execute_input":"2023-03-09T11:44:06.490481Z","iopub.status.idle":"2023-03-09T11:44:06.501455Z","shell.execute_reply.started":"2023-03-09T11:44:06.490431Z","shell.execute_reply":"2023-03-09T11:44:06.500384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.min(indices)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:44:06.505498Z","iopub.execute_input":"2023-03-09T11:44:06.509113Z","iopub.status.idle":"2023-03-09T11:44:06.519171Z","shell.execute_reply.started":"2023-03-09T11:44:06.509061Z","shell.execute_reply":"2023-03-09T11:44:06.518299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_neighbors(orig, neighbors):\n    f, axarr = plt.subplots(4, 2)\n    for i, ax in enumerate(axarr.flatten()):\n        if i == 0:\n            ax.set_title(\"Query image\")\n            ax.imshow(orig)\n        else:\n            ax.set_title(f\"Neighbor {i}\")\n            ax.imshow(train_x_final[neighbors[i-1]])\n        ax.set_yticklabels([])\n        ax.set_xticklabels([])\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:44:06.521077Z","iopub.execute_input":"2023-03-09T11:44:06.521686Z","iopub.status.idle":"2023-03-09T11:44:06.537403Z","shell.execute_reply.started":"2023-03-09T11:44:06.521629Z","shell.execute_reply":"2023-03-09T11:44:06.536525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"indices=knn7.kneighbors(test_vects, return_distance=False)\nfor i in range(0,5):\n    show_neighbors(X_test[i],indices[i])\n    \n    ","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:44:06.542355Z","iopub.execute_input":"2023-03-09T11:44:06.543013Z","iopub.status.idle":"2023-03-09T11:44:10.654183Z","shell.execute_reply.started":"2023-03-09T11:44:06.542966Z","shell.execute_reply":"2023-03-09T11:44:10.653362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ypred=knn7.predict(test_vects)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:44:10.655556Z","iopub.execute_input":"2023-03-09T11:44:10.655806Z","iopub.status.idle":"2023-03-09T11:44:10.704191Z","shell.execute_reply.started":"2023-03-09T11:44:10.655775Z","shell.execute_reply":"2023-03-09T11:44:10.703227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn import metrics \nprint(metrics.classification_report(test_label_integers,ypred))","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:44:10.705365Z","iopub.execute_input":"2023-03-09T11:44:10.705762Z","iopub.status.idle":"2023-03-09T11:44:10.728812Z","shell.execute_reply.started":"2023-03-09T11:44:10.705727Z","shell.execute_reply":"2023-03-09T11:44:10.727884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Saliency Map\n![Saliency Map](https://raw.githubusercontent.com/somya-15/image_hosting/main/diag.png)","metadata":{}},{"cell_type":"markdown","source":"Installing Xplique and utils","metadata":{}},{"cell_type":"code","source":"!pip install -q xplique\n\nimport os\nimport cv2\nimport numpy as np\nimport tensorflow as tf\nfrom matplotlib import pyplot as plt\n\n%matplotlib inline\n%config InlineBackend.figure_format='retina'\n\nimport xplique\nfrom xplique.plots import plot_attributions","metadata":{"execution":{"iopub.status.busy":"2023-03-09T11:55:42.402566Z","iopub.execute_input":"2023-03-09T11:55:42.402855Z","iopub.status.idle":"2023-03-09T11:56:08.504246Z","shell.execute_reply.started":"2023-03-09T11:55:42.402825Z","shell.execute_reply":"2023-03-09T11:56:08.503427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Generate explanations\n💡 To explain the logits is to explain the class, to explain the softmax is to explain why this class rather than another. It is thus recommended to remove the softmax activation to explain the logit","metadata":{}},{"cell_type":"code","source":"print(X_train.shape)","metadata":{"execution":{"iopub.status.busy":"2023-03-09T12:18:44.989948Z","iopub.execute_input":"2023-03-09T12:18:44.99021Z","iopub.status.idle":"2023-03-09T12:18:44.994994Z","shell.execute_reply.started":"2023-03-09T12:18:44.99018Z","shell.execute_reply":"2023-03-09T12:18:44.994065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_train.shape","metadata":{"execution":{"iopub.status.busy":"2023-03-09T12:18:16.555644Z","iopub.execute_input":"2023-03-09T12:18:16.555918Z","iopub.status.idle":"2023-03-09T12:18:16.56184Z","shell.execute_reply.started":"2023-03-09T12:18:16.555889Z","shell.execute_reply":"2023-03-09T12:18:16.560972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from xplique.attributions import (Saliency, GradientInput, IntegratedGradients, SmoothGrad, VarGrad,\n                                  SquareGrad, GradCAM, Occlusion, Rise, GuidedBackprop,\n                                  GradCAMPP, Lime, KernelShap, SobolAttributionMethod)\n\n# to explain the logits is to explain the class, \n# to explain the softmax is to explain why this class rather than another\n# it is therefore recommended to explain the logit\nmodel = vgg_model19_a\nmodel.layers[-1].activation = tf.keras.activations.linear\nbatch_size = 64\n\nexplainers = [\n             Saliency(model),\n             GradientInput(model),\n             GuidedBackprop(model),\n             IntegratedGradients(model, steps=50, batch_size=batch_size),\n             SmoothGrad(model, nb_samples=50, batch_size=batch_size),\n             SquareGrad(model, nb_samples=50, batch_size=batch_size),\n             VarGrad(model, nb_samples=50, batch_size=batch_size),\n             GradCAM(model),\n             GradCAMPP(model),\n             Occlusion(model, patch_size=10, patch_stride=10, batch_size=batch_size),\n             SobolAttributionMethod(model, batch_size=batch_size),\n             # Rise(model, nb_samples=4000, batch_size=batch_size),\n             # Lime(model, nb_samples = 1000),\n             # KernelShap(model, nb_samples = 1000)\n]\n\nexplanations_to_test = {}\n\nfor explainer in explainers:\n\n  explainer_name = explainer.__class__.__name__\n  explanations = explainer(X_train, Y_train)\n\n  if len(explanations.shape) > 3:\n    explanations = np.mean(explanations, -1)\n\n  # store the explanations to use the metrics\n  explanations_to_test[explainer_name] = explanations\n  \n  print(f\"Method: {explainer_name}\")\n  plot_attributions(explanations[:5], X_train[:5], cmap='jet', alpha=0.4,\n                    cols=5, clip_percentile=0.5, absolute_value=True)\n  plt.show()\n  print(\"\\n\")","metadata":{"execution":{"iopub.status.busy":"2023-03-09T12:22:54.175538Z","iopub.execute_input":"2023-03-09T12:22:54.175838Z","iopub.status.idle":"2023-03-09T13:51:47.074011Z","shell.execute_reply.started":"2023-03-09T12:22:54.175808Z","shell.execute_reply":"2023-03-09T13:51:47.073372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}