{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Prior notice\n\nWe often converted cells we do want to skip running to so-called 'raw'-cells. This is done by using clicking into a cell, then pressing ESC. Now one can do different actions, such as converting them back to code (press \"Y\"), raw (press \"R\") or markdown cells (press \"M\").\n\nRaw cells do not have any sort of text highlighting and everything is gray. ","metadata":{}},{"cell_type":"markdown","source":"# Imports and initial setup","metadata":{}},{"cell_type":"code","source":"# %%capture\n# !pip install vision-transformer-pytorch\n# !pip install efficientnet_pytorch","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:25.941094Z","iopub.execute_input":"2022-11-09T01:41:25.941432Z","iopub.status.idle":"2022-11-09T01:41:25.945074Z","shell.execute_reply.started":"2022-11-09T01:41:25.941398Z","shell.execute_reply":"2022-11-09T01:41:25.943969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nos.environ['CUDA_LAUNCH_BLOCKING'] = \"1\"","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:25.949197Z","iopub.execute_input":"2022-11-09T01:41:25.950008Z","iopub.status.idle":"2022-11-09T01:41:25.955591Z","shell.execute_reply.started":"2022-11-09T01:41:25.949965Z","shell.execute_reply":"2022-11-09T01:41:25.954816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Imports\nimport torch\nimport torch.nn as nn\n\nimport torchvision.models as models\n\nimport pathlib\nimport numpy as np\nimport pandas as pd\nfrom torchvision import datasets, transforms\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader, random_split\nfrom tqdm import tqdm\nimport sklearn\nimport seaborn as sns","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-11-09T01:41:25.957067Z","iopub.execute_input":"2022-11-09T01:41:25.957476Z","iopub.status.idle":"2022-11-09T01:41:28.149243Z","shell.execute_reply.started":"2022-11-09T01:41:25.957442Z","shell.execute_reply":"2022-11-09T01:41:28.148407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_directory = \"../input/landmark-recognition-2020/\"","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:28.151151Z","iopub.execute_input":"2022-11-09T01:41:28.151517Z","iopub.status.idle":"2022-11-09T01:41:28.155585Z","shell.execute_reply.started":"2022-11-09T01:41:28.151481Z","shell.execute_reply":"2022-11-09T01:41:28.154606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(\"../input/landmark-recognition-2020/train.csv\")\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:28.157531Z","iopub.execute_input":"2022-11-09T01:41:28.157940Z","iopub.status.idle":"2022-11-09T01:41:29.487637Z","shell.execute_reply.started":"2022-11-09T01:41:28.157905Z","shell.execute_reply":"2022-11-09T01:41:29.486805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"raw","source":"temp_df = pd.read_csv(os.path.join(base_directory, 'sample_submission.csv'))\n\ntest_df = temp_df[['id']].copy()\ntest_df['filepath'] = test_df['id'].apply(lambda x: os.path.join(base_directory, 'test', x[0], x[1], x[2], f'{x}.jpg'))","metadata":{"execution":{"iopub.status.busy":"2022-10-30T23:12:12.604322Z","iopub.execute_input":"2022-10-30T23:12:12.604726Z","iopub.status.idle":"2022-10-30T23:12:12.664227Z","shell.execute_reply.started":"2022-10-30T23:12:12.604694Z","shell.execute_reply":"2022-10-30T23:12:12.663180Z"}}},{"cell_type":"code","source":"pd.set_option('display.max_colwidth', None)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:29.488926Z","iopub.execute_input":"2022-11-09T01:41:29.489359Z","iopub.status.idle":"2022-11-09T01:41:29.494888Z","shell.execute_reply.started":"2022-11-09T01:41:29.489298Z","shell.execute_reply":"2022-11-09T01:41:29.494026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data exploration and preprocessing","metadata":{}},{"cell_type":"code","source":"print(f\"The total number of pictures in the dataset: {len(train_df)}\")\nprint(f\"The total number of landmarks in the dataset: {train_df.landmark_id.nunique()}\")","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:29.496471Z","iopub.execute_input":"2022-11-09T01:41:29.497084Z","iopub.status.idle":"2022-11-09T01:41:29.520976Z","shell.execute_reply.started":"2022-11-09T01:41:29.497047Z","shell.execute_reply":"2022-11-09T01:41:29.520018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Add category as label","metadata":{}},{"cell_type":"code","source":"landmark_id_to_url_category_df = pd.read_csv('https://s3.amazonaws.com/google-landmark/metadata/train_label_to_category.csv')\nlandmark_id_to_url_category_df","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:29.522299Z","iopub.execute_input":"2022-11-09T01:41:29.522736Z","iopub.status.idle":"2022-11-09T01:41:30.966052Z","shell.execute_reply.started":"2022-11-09T01:41:29.522685Z","shell.execute_reply":"2022-11-09T01:41:30.965227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#landmark_id_to_url_category_df.where(landmark_id_to_url_category_df[\"landmark_id\"]==126636)\n# have a look at top classes","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:30.967389Z","iopub.execute_input":"2022-11-09T01:41:30.967732Z","iopub.status.idle":"2022-11-09T01:41:30.971770Z","shell.execute_reply.started":"2022-11-09T01:41:30.967694Z","shell.execute_reply":"2022-11-09T01:41:30.970718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from urllib.parse import unquote\n\nurl = \"Corktown,_Toronto\"\nurl = unquote(url)\nurl","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:30.975743Z","iopub.execute_input":"2022-11-09T01:41:30.976227Z","iopub.status.idle":"2022-11-09T01:41:30.984641Z","shell.execute_reply.started":"2022-11-09T01:41:30.976192Z","shell.execute_reply":"2022-11-09T01:41:30.983782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"landmark_id_to_url_category_df['category'] = landmark_id_to_url_category_df['category'].map(lambda x: unquote(x.lstrip('http://commons.wikimedia.org/wiki/Category:')).replace(\"_\", \" \"))\nlandmark_id_to_url_category_df.head(20)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:30.986529Z","iopub.execute_input":"2022-11-09T01:41:30.986847Z","iopub.status.idle":"2022-11-09T01:41:31.414357Z","shell.execute_reply.started":"2022-11-09T01:41:30.986815Z","shell.execute_reply":"2022-11-09T01:41:31.413413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df.merge(landmark_id_to_url_category_df, left_on=['landmark_id'], right_on=['landmark_id'])\ntrain_df.head(20)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:31.415752Z","iopub.execute_input":"2022-11-09T01:41:31.416086Z","iopub.status.idle":"2022-11-09T01:41:31.636644Z","shell.execute_reply.started":"2022-11-09T01:41:31.416050Z","shell.execute_reply":"2022-11-09T01:41:31.635654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"category_count_df = train_df['category'].value_counts()\ncategory_count_df","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:31.638175Z","iopub.execute_input":"2022-11-09T01:41:31.638556Z","iopub.status.idle":"2022-11-09T01:41:31.786056Z","shell.execute_reply.started":"2022-11-09T01:41:31.638517Z","shell.execute_reply":"2022-11-09T01:41:31.785118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_category_df = train_df[train_df[\"category\"].str.match(\"Luitpoldpark in Munich\")]\nexample_category_df","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:31.787381Z","iopub.execute_input":"2022-11-09T01:41:31.787720Z","iopub.status.idle":"2022-11-09T01:41:32.552031Z","shell.execute_reply.started":"2022-11-09T01:41:31.787685Z","shell.execute_reply":"2022-11-09T01:41:32.551218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = train_df.drop_duplicates(subset=[\"landmark_id\"])\ndf = df.drop([\"id\"], axis = 1)\ndf = df.sort_values(by=['landmark_id'], ascending=True)\ndf.head(25)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:32.553223Z","iopub.execute_input":"2022-11-09T01:41:32.553584Z","iopub.status.idle":"2022-11-09T01:41:32.618419Z","shell.execute_reply.started":"2022-11-09T01:41:32.553554Z","shell.execute_reply":"2022-11-09T01:41:32.617485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#show examples of pictures of a single landmark\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport glob\n\nplt.figure(figsize=(20,10))\ncolumns = 5\n\nfor idx, row in example_category_df.iterrows():\n    print(row[\"id\"])\n    individual_image = row[\"id\"] + '.jpg'\n    img = Image.open('../input/landmark-recognition-2020/train/'+individual_image[0]+ \"/\" + individual_image[1] + \"/\" + individual_image[2]+\"/\" + individual_image)\n    plt.subplot(len(example_category_df) / columns + 1, columns, idx + 1)\n    plt.imshow(img)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:32.619827Z","iopub.execute_input":"2022-11-09T01:41:32.620168Z","iopub.status.idle":"2022-11-09T01:41:33.342638Z","shell.execute_reply.started":"2022-11-09T01:41:32.620131Z","shell.execute_reply":"2022-11-09T01:41:33.341871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Remove classes with too few images","metadata":{}},{"cell_type":"code","source":"imagepercatg = train_df['landmark_id'].value_counts().sort_values(ascending=False)\nimagepercatg[0:10]","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:33.343720Z","iopub.execute_input":"2022-11-09T01:41:33.344098Z","iopub.status.idle":"2022-11-09T01:41:33.381248Z","shell.execute_reply.started":"2022-11-09T01:41:33.344050Z","shell.execute_reply":"2022-11-09T01:41:33.380306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classes_with_more_than_500_imgs = imagepercatg[(imagepercatg > 500)]","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:33.382476Z","iopub.execute_input":"2022-11-09T01:41:33.382818Z","iopub.status.idle":"2022-11-09T01:41:33.400818Z","shell.execute_reply.started":"2022-11-09T01:41:33.382782Z","shell.execute_reply":"2022-11-09T01:41:33.400068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"number_imgs_for_classes_with_more_than_500_imgs = classes_with_more_than_500_imgs.sum()\nnumber_imgs_for_classes_with_more_than_500_imgs","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:33.401986Z","iopub.execute_input":"2022-11-09T01:41:33.402511Z","iopub.status.idle":"2022-11-09T01:41:33.408659Z","shell.execute_reply.started":"2022-11-09T01:41:33.402476Z","shell.execute_reply":"2022-11-09T01:41:33.407680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"number_of_classes_with_more_than_500_imgs = classes_with_more_than_500_imgs.count()\nnumber_of_classes_with_more_than_500_imgs","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:33.410432Z","iopub.execute_input":"2022-11-09T01:41:33.410777Z","iopub.status.idle":"2022-11-09T01:41:33.417428Z","shell.execute_reply.started":"2022-11-09T01:41:33.410736Z","shell.execute_reply":"2022-11-09T01:41:33.416457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"min_500_list = classes_with_more_than_500_imgs.index.tolist()\n# sorted(min_500_list)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:33.418783Z","iopub.execute_input":"2022-11-09T01:41:33.419334Z","iopub.status.idle":"2022-11-09T01:41:33.424510Z","shell.execute_reply.started":"2022-11-09T01:41:33.419283Z","shell.execute_reply":"2022-11-09T01:41:33.423781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#delete all classes with less than 500 images\ntrain_df = train_df.loc[train_df['landmark_id'].isin(min_500_list)]\nunique_values = train_df.landmark_id.unique()\nunique_values","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:33.426145Z","iopub.execute_input":"2022-11-09T01:41:33.426821Z","iopub.status.idle":"2022-11-09T01:41:33.497305Z","shell.execute_reply.started":"2022-11-09T01:41:33.426785Z","shell.execute_reply":"2022-11-09T01:41:33.496406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"landmark_to_unique_mapping = dict()\n\nfor i, value in enumerate(unique_values):\n    landmark_to_unique_mapping[value] = i\n    \n# landmark_to_unique_mapping","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:33.498639Z","iopub.execute_input":"2022-11-09T01:41:33.498984Z","iopub.status.idle":"2022-11-09T01:41:33.503226Z","shell.execute_reply.started":"2022-11-09T01:41:33.498948Z","shell.execute_reply":"2022-11-09T01:41:33.502448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[\"old_landmark_ids\"] = train_df.landmark_id\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:33.504460Z","iopub.execute_input":"2022-11-09T01:41:33.504941Z","iopub.status.idle":"2022-11-09T01:41:33.525031Z","shell.execute_reply.started":"2022-11-09T01:41:33.504902Z","shell.execute_reply":"2022-11-09T01:41:33.524191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.landmark_id = train_df.landmark_id.apply(lambda old: landmark_to_unique_mapping[old])\ntrain_df.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:33.526253Z","iopub.execute_input":"2022-11-09T01:41:33.526618Z","iopub.status.idle":"2022-11-09T01:41:33.563218Z","shell.execute_reply.started":"2022-11-09T01:41:33.526584Z","shell.execute_reply":"2022-11-09T01:41:33.562296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df.drop([\"old_landmark_ids\"],axis=1)\ntrain_df.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:33.564536Z","iopub.execute_input":"2022-11-09T01:41:33.564860Z","iopub.status.idle":"2022-11-09T01:41:33.577600Z","shell.execute_reply.started":"2022-11-09T01:41:33.564826Z","shell.execute_reply":"2022-11-09T01:41:33.576721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"freq_df = train_df.copy()\nfreq_df[\"freq\"] = train_df.groupby('landmark_id')['landmark_id'].transform('count')\nfreq_df = freq_df.drop([\"id\"], axis = 1)\nfreq_df = freq_df.sort_values(by=['freq'], ascending=False)\nfreq_df = freq_df.drop_duplicates()\nfreq_df.head(50)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:33.578965Z","iopub.execute_input":"2022-11-09T01:41:33.579310Z","iopub.status.idle":"2022-11-09T01:41:33.610142Z","shell.execute_reply.started":"2022-11-09T01:41:33.579275Z","shell.execute_reply":"2022-11-09T01:41:33.609180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"landmark_id = str(train_df.iloc[0].landmark_id)\nindividual_image = train_df.iloc[0].id + '.jpg'\n\nprint(len(train_df.iloc[0].id))\nprint(individual_image, landmark_id )\n\nfrom PIL import Image\nimport matplotlib.pyplot as plt\n\nimg = Image.open('../input/landmark-recognition-2020/train/'+individual_image[0]+ \"/\" + individual_image[1] + \"/\" + individual_image[2]+\"/\" + individual_image)\nplt.imshow(img)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:33.614882Z","iopub.execute_input":"2022-11-09T01:41:33.615121Z","iopub.status.idle":"2022-11-09T01:41:33.822098Z","shell.execute_reply.started":"2022-11-09T01:41:33.615096Z","shell.execute_reply":"2022-11-09T01:41:33.821181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"raw","source":"#Calculate mean and std for the images\nimport numpy as np\nimport cv2\n \nfrom pathlib import Path\n \nimageFilesDir = Path(base_directory, \"train\")\nfiles = list(imageFilesDir.rglob('*.jpg'))\n \nlen(files)\n \nmean = np.array([0.,0.,0.])\nstdTemp = np.array([0.,0.,0.])\nstd = np.array([0.,0.,0.])\n \nnumSamples = len(files)\n \nfor i in tqdm(range(numSamples)):\n    im = cv2.imread(str(files[i]))\n    im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)\n    im = im.astype(float) / 255.\n     \n    for j in range(3):\n        mean[j] += np.mean(im[:,:,j])\n       \nmean = (mean/numSamples)\n \nprint(mean)\n\nfor i in tqdm(range(numSamples)):\n    im = cv2.imread(str(files[i]))\n    im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)\n    im = im.astype(float) / 255.\n    for j in range(3):\n        stdTemp[j] += ((im[:,:,j] - mean[j])**2).sum()/(im.shape[0]*im.shape[1])\n \nstd = np.sqrt(stdTemp/numSamples)\n \nprint(std)","metadata":{"execution":{"iopub.status.busy":"2022-10-26T01:41:57.488831Z","iopub.execute_input":"2022-10-26T01:41:57.489374Z","iopub.status.idle":"2022-10-26T01:42:11.264441Z","shell.execute_reply.started":"2022-10-26T01:41:57.489327Z","shell.execute_reply":"2022-10-26T01:42:11.262081Z"}}},{"cell_type":"code","source":"mean = (0.485, 0.456, 0.406)\nstd =  (0.229,0.225,0.224)\nIMG_SIZE = 128\n# IMG_SIZE = 384 #needed to use pretrained Vision Transformer \n# transformations = transforms.Compose([transforms.Resize((IMG_SIZE,IMG_SIZE),interpolation=Image.NEAREST),\n#                                       transforms.ToTensor(),\n#                                       transforms.Normalize(mean,std)\n#                                      ]\n#                                     )    ","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:33.824432Z","iopub.execute_input":"2022-11-09T01:41:33.824725Z","iopub.status.idle":"2022-11-09T01:41:33.829986Z","shell.execute_reply.started":"2022-11-09T01:41:33.824693Z","shell.execute_reply":"2022-11-09T01:41:33.828868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_transform = transforms.Compose(\n        [\n            transforms.Resize((IMG_SIZE, IMG_SIZE)),\n#             transforms.Resize(256),\n#             transforms.RandomCrop(224),\n#             transforms.CenterCrop(224),\n            transforms.RandomHorizontalFlip(),\n            transforms.RandomVerticalFlip(),\n            # transforms.GaussianBlur(kernel_size=(5, 9), sigma=(0.1, 5)),\n#             transforms.RandomRotation(degrees=(30, 70)),\n            transforms.ToTensor(),\n            # transforms.RandomErasing(), #problems because it only works on tensors and caused problems with images \n            transforms.Normalize(mean=mean, std=std),\n        ]\n    )\n\ntest_val_transform = transforms.Compose(\n        [\n            transforms.Resize((IMG_SIZE, IMG_SIZE)),\n#             transforms.Resize(256),\n#             transforms.CenterCrop(224),\n            transforms.ToTensor(),\n            transforms.Normalize(mean=mean, std=std),\n        ]\n    )","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:33.831420Z","iopub.execute_input":"2022-11-09T01:41:33.831790Z","iopub.status.idle":"2022-11-09T01:41:33.839296Z","shell.execute_reply.started":"2022-11-09T01:41:33.831740Z","shell.execute_reply":"2022-11-09T01:41:33.838414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class get_landmarks(Dataset):\n    def __init__(self,paths =train_df, training_image_path=\"../input/landmark-recognition-2020/train/\", \n                 my_transform=None, paths_is_df = True):\n        if paths_is_df:\n            self.training_df = paths\n        else: \n            self.training_df = pd.read_csv(paths)\n        self.image_path = training_image_path\n        self.transform = my_transform\n        \n    def __len__(self):\n        return len(self.training_df)\n    \n    def __getitem__(self,index):\n        \n        individual_image = self.training_df.iloc[index].id + \".jpg\"\n        landmark_id = self.training_df.iloc[index].landmark_id\n        full_path = self.image_path +individual_image[0]+ \"/\" + individual_image[1] + \"/\" + individual_image[2]+ \"/\" + individual_image\n#         print(full_path)\n        \n        img = Image.open(full_path)\n        \n        if self.transform:\n            img = self.transform(img)\n        \n        return img, landmark_id","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:33.840708Z","iopub.execute_input":"2022-11-09T01:41:33.841094Z","iopub.status.idle":"2022-11-09T01:41:33.852951Z","shell.execute_reply.started":"2022-11-09T01:41:33.841059Z","shell.execute_reply":"2022-11-09T01:41:33.852118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"raw","source":"# count number of images\nnumber_of_images = 0\nfor image in pathlib.Path(base_directory).rglob(\"*/*.jpg\"):\n    number_of_images += 1\ntotal = number_of_images\nprint(f\"Number of images: {total}\")","metadata":{"execution":{"iopub.status.busy":"2022-10-26T01:47:20.769766Z","iopub.execute_input":"2022-10-26T01:47:20.770496Z","iopub.status.idle":"2022-10-26T01:54:32.470888Z","shell.execute_reply.started":"2022-10-26T01:47:20.770457Z","shell.execute_reply":"2022-10-26T01:54:32.470076Z"}}},{"cell_type":"raw","source":"# check all the different resolutions of all images\nfrom collections import Counter\nimport PIL\nfrom PIL import Image\n\nresolutions = Counter()\nwidths = Counter()\nheights = Counter()\n\n# loading the image\nfor image_path in tqdm(pathlib.Path(base_directory).rglob(\"*/*.jpg\")):\n    img = PIL.Image.open(image_path)\n    resolutions[img.size] += 1\n    widths[img.width] += 1\n    heights[img.height] += 1\n    \nprint(f\"Number of different resolutions {len(resolutions)}\")    \nprint(f\"Resolutions: {resolutions.most_common()}\")\nprint(f\"Widths: {resolutions.most_common()}\")\nprint(f\"Heights: {resolutions.most_common()}\")\nprint(f\"Min Width: {min(widths.keys())}\")\nprint(f\"Min Heights: {min(heights.keys())}\")\n\n","metadata":{"execution":{"iopub.status.busy":"2022-10-26T04:09:55.582588Z","iopub.execute_input":"2022-10-26T04:09:55.583019Z","iopub.status.idle":"2022-10-26T04:10:22.362288Z","shell.execute_reply.started":"2022-10-26T04:09:55.582985Z","shell.execute_reply":"2022-10-26T04:10:22.360801Z"}}},{"cell_type":"code","source":"dataset = get_landmarks(my_transform=None)\n\nx, y = dataset[0]\n\nprint(y)\nplt.imshow(x)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:33.854265Z","iopub.execute_input":"2022-11-09T01:41:33.854628Z","iopub.status.idle":"2022-11-09T01:41:34.059233Z","shell.execute_reply.started":"2022-11-09T01:41:33.854593Z","shell.execute_reply":"2022-11-09T01:41:34.058199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset.training_df.landmark_id.max()","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:34.060597Z","iopub.execute_input":"2022-11-09T01:41:34.060921Z","iopub.status.idle":"2022-11-09T01:41:34.067503Z","shell.execute_reply.started":"2022-11-09T01:41:34.060885Z","shell.execute_reply":"2022-11-09T01:41:34.066577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://stackoverflow.com/questions/51782021/how-to-use-different-data-augmentation-for-subsets-in-pytorch\nclass DatasetFromSubset(Dataset):\n    def __init__(self, subset, transform=None):\n        self.subset = subset\n        self.transform = transform\n\n    def __getitem__(self, index):\n        x, y = self.subset[index]\n        if self.transform:\n            x = self.transform(x)\n        return x, y\n\n    def __len__(self):\n        return len(self.subset)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:34.069026Z","iopub.execute_input":"2022-11-09T01:41:34.069759Z","iopub.status.idle":"2022-11-09T01:41:34.079163Z","shell.execute_reply.started":"2022-11-09T01:41:34.069722Z","shell.execute_reply":"2022-11-09T01:41:34.078392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ratio = 0.8\ntest_ratio = 0.1\nval_ratio = 0.1\ntrain_size = round(len(dataset) * train_ratio)\ntest_size = round(len(dataset) * test_ratio)\nval_size = round(len(dataset) * val_ratio)\nsum_size = train_size + test_size + val_size\nprint(f\"Using {sum_size}/{len(dataset)} ({sum_size/len(dataset):.2%}) of the dataset for train, validation and test.\")","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:34.080599Z","iopub.execute_input":"2022-11-09T01:41:34.080941Z","iopub.status.idle":"2022-11-09T01:41:34.090698Z","shell.execute_reply.started":"2022-11-09T01:41:34.080908Z","shell.execute_reply":"2022-11-09T01:41:34.089812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds, test_ds, val_ds, _ = random_split(\n    dataset,\n    [train_size, test_size, val_size, len(dataset)-sum_size],\n    generator=torch.Generator().manual_seed(42),\n)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:34.091943Z","iopub.execute_input":"2022-11-09T01:41:34.092289Z","iopub.status.idle":"2022-11-09T01:41:34.117767Z","shell.execute_reply.started":"2022-11-09T01:41:34.092255Z","shell.execute_reply":"2022-11-09T01:41:34.117052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds = DatasetFromSubset(\n    train_ds, transform=train_transform\n    )\ntest_ds = DatasetFromSubset(\n    test_ds, transform=test_val_transform,\n    )\nval_ds = DatasetFromSubset(\n    val_ds, transform=test_val_transform,\n)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:34.118950Z","iopub.execute_input":"2022-11-09T01:41:34.119415Z","iopub.status.idle":"2022-11-09T01:41:34.123648Z","shell.execute_reply.started":"2022-11-09T01:41:34.119378Z","shell.execute_reply":"2022-11-09T01:41:34.122659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x, y = test_ds[0]\n\nprint(y)\nplt.imshow(x.permute(1,2,0))","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:34.124947Z","iopub.execute_input":"2022-11-09T01:41:34.125636Z","iopub.status.idle":"2022-11-09T01:41:34.304268Z","shell.execute_reply.started":"2022-11-09T01:41:34.125548Z","shell.execute_reply":"2022-11-09T01:41:34.303523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Outdated train/test/val split due to missing labels in test due to competition\n\nCells are put in raw format (ESC + R), to be able to execute them, you need to put them back in code format. Press ESC + Y to do so.","metadata":{}},{"cell_type":"raw","source":"train_dataset = get_landmarks(my_transform=train_transform)\ntest_val_dataset = get_landmarks(training_image_path=\"../input/landmark-recognition-2020/test\", my_transform=test_val_transform) # doesn't work\n\nx, y = train_dataset[0]\nx, y = test_val_dataset[0]\n\nprint(y)\nplt.imshow(x.permute(1,2,0))","metadata":{"_kg_hide-input":false}},{"cell_type":"raw","source":"train_dataset = get_landmarks(my_transform=train_transform)\ntest_val_dataset = get_landmarks(paths=test_df, training_image_path=\"../input/landmark-recognition-2020/test\", my_transform=test_val_transform, paths_is_df=True)\n\nx, y = train_dataset[0]\nx, y = test_val_dataset[0]\n\nprint(y)\nplt.imshow(x.permute(1,2,0))","metadata":{}},{"cell_type":"raw","source":"x, y = test_val_dataset[0]\n\nprint(y)\nplt.imshow(x.permute(1,2,0))","metadata":{}},{"cell_type":"raw","source":"test_ratio = 0.5\nval_ratio = 0.5\n\ntest_size = round(len(dataset.imgs) * test_ratio)\nval_size = round(len(dataset.imgs) * val_ratio)\n\ntest_size + val_size","metadata":{"execution":{"iopub.status.busy":"2022-10-26T04:11:19.223291Z","iopub.status.idle":"2022-10-26T04:11:19.223754Z"}}},{"cell_type":"raw","source":"validation_dataset, test_dataset = random_split(\n    test_val_dataset,\n    [val_size/(test_size+val_size), test_size/(test_size+val_size)],\n    generator=torch.Generator().manual_seed(42),\n)","metadata":{"execution":{"iopub.status.busy":"2022-10-26T04:11:19.225536Z","iopub.status.idle":"2022-10-26T04:11:19.226046Z"}}},{"cell_type":"markdown","source":"## New Train/Val/Test split","metadata":{}},{"cell_type":"code","source":"#reducing size of dataset for testing purposes/compututional speed up\nfull_train_dataset, full_validation_dataset, full_test_dataset = train_ds, test_ds, val_ds\nsubset_ratio = 1 #put this to <1 to reduce the size of your data set\n\ntrain_size = round(len(full_train_dataset) * subset_ratio)\nval_size = round(len(full_validation_dataset) * subset_ratio)\ntest_size = round(len(full_test_dataset) * subset_ratio)\nleft = round(len(full_test_dataset) * subset_ratio)\n\ntrain_dataset, _ = random_split(\n    full_train_dataset,\n    [train_size, len(full_train_dataset)-train_size],\n    generator=torch.Generator().manual_seed(42),\n)\n\nvalidation_dataset, _ = random_split(\n    full_validation_dataset,\n    [val_size, len(full_validation_dataset)-val_size],\n    generator=torch.Generator().manual_seed(42),\n)\n\ntest_dataset, _ = random_split(\n    full_test_dataset,\n    [test_size, len(full_test_dataset)-test_size],\n    generator=torch.Generator().manual_seed(42),\n)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:34.305569Z","iopub.execute_input":"2022-11-09T01:41:34.306081Z","iopub.status.idle":"2022-11-09T01:41:34.317038Z","shell.execute_reply.started":"2022-11-09T01:41:34.306042Z","shell.execute_reply":"2022-11-09T01:41:34.316205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Length Train: {len(train_dataset)}\")\nprint(f\"Length Validation: {len(validation_dataset)}\")\nprint(f\"Length Test: {len(test_dataset)}\")","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:34.318313Z","iopub.execute_input":"2022-11-09T01:41:34.318816Z","iopub.status.idle":"2022-11-09T01:41:34.326818Z","shell.execute_reply.started":"2022-11-09T01:41:34.318782Z","shell.execute_reply":"2022-11-09T01:41:34.325851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size =128\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nprint(f\"Length of Train Data : {len(train_ds)}\")\nprint(f\"Length of Validation Data : {len(val_ds)}\")\nprint(f\"Length of Test Data : {len(test_ds)}\")\n\n# load the train, validation and test into batches.\ntrain_dl = DataLoader(\n    train_dataset,\n    batch_size,\n    shuffle=True,\n)  # pin_memory=True)\nval_dl = DataLoader(\n    validation_dataset,\n    batch_size,\n)  # pin_memory=True)\ntest_dl = DataLoader(\n    test_dataset,\n    batch_size,\n)  # pin_memory=True)\n\nimg, label = train_ds[0]\nprint(train_ds[0])\nprint(f\"Shape of {img.shape} with label {label}\")\nimg, label = test_ds[0]\nprint(f\"Shape of {img.shape} with label {label}\")\nimg, label = val_ds[0]\nprint(f\"Shape of {img.shape} with label {label}\")","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:34.328261Z","iopub.execute_input":"2022-11-09T01:41:34.328610Z","iopub.status.idle":"2022-11-09T01:41:34.495163Z","shell.execute_reply.started":"2022-11-09T01:41:34.328575Z","shell.execute_reply":"2022-11-09T01:41:34.494273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"raw","source":"labels = [label for img, label in tqdm(train_dataset)]\n\nlen(labels)","metadata":{"execution":{"iopub.status.busy":"2022-10-30T00:53:47.834044Z","iopub.execute_input":"2022-10-30T00:53:47.834402Z","iopub.status.idle":"2022-10-30T01:04:22.252736Z","shell.execute_reply.started":"2022-10-30T00:53:47.834375Z","shell.execute_reply":"2022-10-30T01:04:22.251889Z"}}},{"cell_type":"raw","source":"#calculate weights for all the classes due to imbalance\nclass_weights=sklearn.utils.class_weight.compute_class_weight(class_weight='balanced',classes=np.unique(labels), y=labels)\nclass_weights=torch.tensor(class_weights,dtype=torch.float)\n\nclass_weights\n#train 80% on 51 classes: class_weights = tensor([1.7697, 0.9261, 1.0013, 1.1458, 1.1976, 1.0483, 0.5100, 1.3289, 1.7697,\n#         1.7396, 1.0561, 1.2632, 1.2999, 1.2097, 1.5748, 1.1896, 1.0056, 1.1916,\n#         1.4895, 1.6783, 1.4651, 1.6323, 0.5089, 1.7524, 1.7145, 1.7064, 1.5115,\n#         0.7754, 1.7396, 1.0735, 0.4019, 0.1440, 0.9131, 1.6982, 0.9285, 1.7697,\n#         1.0437, 1.3289, 1.2285, 0.9862, 1.3618, 1.1224, 0.9688, 0.8199, 1.1916,\n#         1.5995, 1.4186, 1.4651, 1.5376, 1.6212, 0.8471])","metadata":{"execution":{"iopub.status.busy":"2022-10-30T01:04:22.254365Z","iopub.execute_input":"2022-10-30T01:04:22.254738Z","iopub.status.idle":"2022-10-30T01:04:22.276871Z","shell.execute_reply.started":"2022-10-30T01:04:22.254703Z","shell.execute_reply":"2022-10-30T01:04:22.276060Z"}}},{"cell_type":"raw","source":"weights_name = './class_weights.pt'\ntorch.save(class_weights,weights_name)","metadata":{"execution":{"iopub.status.busy":"2022-10-30T01:04:22.279735Z","iopub.execute_input":"2022-10-30T01:04:22.280010Z","iopub.status.idle":"2022-10-30T01:04:22.287281Z","shell.execute_reply.started":"2022-10-30T01:04:22.279983Z","shell.execute_reply":"2022-10-30T01:04:22.286426Z"}}},{"cell_type":"code","source":"# class_weights = torch.load('./class_weights.pt')","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:34.496482Z","iopub.execute_input":"2022-11-09T01:41:34.496940Z","iopub.status.idle":"2022-11-09T01:41:34.500834Z","shell.execute_reply.started":"2022-11-09T01:41:34.496904Z","shell.execute_reply":"2022-11-09T01:41:34.499932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# class_weights","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:34.502187Z","iopub.execute_input":"2022-11-09T01:41:34.502751Z","iopub.status.idle":"2022-11-09T01:41:34.510099Z","shell.execute_reply.started":"2022-11-09T01:41:34.502715Z","shell.execute_reply":"2022-11-09T01:41:34.509149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train and evaluate methods","metadata":{}},{"cell_type":"code","source":"# from  torch.cuda.amp import autocast\n\n# 3# Train my neural network.\ndef train(\n    model,\n    train_dl,\n    val_dl,\n    device,\n    lr=0.001,\n    epochs=5,\n    criterion=nn.CrossEntropyLoss(),\n    skip_train_accuracy=True,\n    saving_name=\"model\",\n    path_to_folder=\"./\",\n    best_val_acc=0,\n    weight = None,\n):\n        optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n#     with autocast():\n        for epoch in range(epochs):\n            for data in tqdm(train_dl):\n                X, y = data\n#                 X, y = X.to(device), y.to(device)\n                X, y = X.to(device).half(), y.to(device)\n                optimizer.zero_grad()  # clear gradient information.\n                if weight is not None:\n                    criterion.weight = weight.half()\n                criterion.to(device)\n                \n                with torch.cuda.amp.autocast():\n                    output = model(X)\n                    loss = criterion(output, y)\n                \n                loss.backward()  # do pack-propagation step\n                optimizer.step()  # tell optimizer that you finished batch/iteration.\n\n            print(f\"Loss: {loss.data}\")\n            if not skip_train_accuracy:\n                print(f\"Accuracy: {evaluate(model, train_dl, device)}\")\n            val_acc = evaluate(model, val_dl, device)\n            print(f\"Validation accuracy: {val_acc}\")\n            if val_acc > best_val_acc:\n                best_val_acc = val_acc\n                filename = f\"{path_to_folder}{saving_name}_{val_acc:.3f}.pt\"\n                torch.save(model.state_dict(), filename)\n        return best_val_acc\n\n\n# Evaluate the trained network.\ndef evaluate(model, test_dl, device):\n    total = 0\n    correct = 0\n    with torch.no_grad():  # No need for keeping track of necessary changes to the gradient.\n        for data in tqdm(test_dl):\n            X, y = data\n            X, y = X.to(device), y.to(device)\n            output = model(X)\n            for idx, val in enumerate(output):\n                if torch.argmax(val) == y[idx]:\n                    correct += 1\n                total += 1\n        # print('\\nAccuracy:', round(correct/total, 3))\n    return round(correct / total, 3)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:34.512780Z","iopub.execute_input":"2022-11-09T01:41:34.513047Z","iopub.status.idle":"2022-11-09T01:41:34.524279Z","shell.execute_reply.started":"2022-11-09T01:41:34.513017Z","shell.execute_reply":"2022-11-09T01:41:34.523316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Models - Training and testing","metadata":{}},{"cell_type":"code","source":"!pip3 install torchinfo\n\nfrom torchinfo import summary","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:34.525495Z","iopub.execute_input":"2022-11-09T01:41:34.526001Z","iopub.status.idle":"2022-11-09T01:41:42.262236Z","shell.execute_reply.started":"2022-11-09T01:41:34.525955Z","shell.execute_reply":"2022-11-09T01:41:42.261098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_val_acc = 0","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:42.264002Z","iopub.execute_input":"2022-11-09T01:41:42.264403Z","iopub.status.idle":"2022-11-09T01:41:42.269151Z","shell.execute_reply.started":"2022-11-09T01:41:42.264359Z","shell.execute_reply":"2022-11-09T01:41:42.267827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_classes = len(train_df['landmark_id'].unique())\nnum_classes","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:42.270639Z","iopub.execute_input":"2022-11-09T01:41:42.271310Z","iopub.status.idle":"2022-11-09T01:41:42.283357Z","shell.execute_reply.started":"2022-11-09T01:41:42.271270Z","shell.execute_reply":"2022-11-09T01:41:42.282363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"No of GPUs: {torch.cuda.device_count()}\")\nprint(f\"Device: {device}\")","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:42.284794Z","iopub.execute_input":"2022-11-09T01:41:42.285181Z","iopub.status.idle":"2022-11-09T01:41:42.291731Z","shell.execute_reply.started":"2022-11-09T01:41:42.285147Z","shell.execute_reply":"2022-11-09T01:41:42.290919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## ResNet 152","metadata":{}},{"cell_type":"code","source":"!pip install cnn_finetune\n\nfrom cnn_finetune import make_model","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:42.293253Z","iopub.execute_input":"2022-11-09T01:41:42.293869Z","iopub.status.idle":"2022-11-09T01:41:53.105649Z","shell.execute_reply.started":"2022-11-09T01:41:42.293832Z","shell.execute_reply":"2022-11-09T01:41:53.104678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model = models.resnet152(pretrained=False)\n# model.fc = nn.Linear(in_features=2048, out_features=num_classes, bias=True)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:53.107084Z","iopub.execute_input":"2022-11-09T01:41:53.107462Z","iopub.status.idle":"2022-11-09T01:41:54.198027Z","shell.execute_reply.started":"2022-11-09T01:41:53.107418Z","shell.execute_reply":"2022-11-09T01:41:54.197146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = make_model('resnet152', num_classes=num_classes, pretrained=True)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:41:54.199233Z","iopub.execute_input":"2022-11-09T01:41:54.199643Z","iopub.status.idle":"2022-11-09T01:42:07.390713Z","shell.execute_reply.started":"2022-11-09T01:41:54.199597Z","shell.execute_reply":"2022-11-09T01:42:07.389805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"load = True\nPATH = \"../input/resnet/resnet152_0.948.pt\" #this is actually the ResNet152, I forgot to change \"saving_name\" in the train function below before when c&p it\n\nif load:\n    best_val_acc = float(PATH[-8:-3])    \n    model.load_state_dict(torch.load(PATH))\n    model.to(device)\n\nbest_val_acc","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:42:07.392062Z","iopub.execute_input":"2022-11-09T01:42:07.392409Z","iopub.status.idle":"2022-11-09T01:42:11.893028Z","shell.execute_reply.started":"2022-11-09T01:42:07.392370Z","shell.execute_reply":"2022-11-09T01:42:11.892236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fail","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:42:11.894371Z","iopub.execute_input":"2022-11-09T01:42:11.894720Z","iopub.status.idle":"2022-11-09T01:42:11.898596Z","shell.execute_reply.started":"2022-11-09T01:42:11.894683Z","shell.execute_reply":"2022-11-09T01:42:11.897614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"summary(model, input_size=(batch_size, 3, IMG_SIZE, IMG_SIZE))","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:42:11.899957Z","iopub.execute_input":"2022-11-09T01:42:11.900365Z","iopub.status.idle":"2022-11-09T01:42:13.401195Z","shell.execute_reply.started":"2022-11-09T01:42:11.900294Z","shell.execute_reply":"2022-11-09T01:42:13.400264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if torch.cuda.device_count() > 1:\n#     model = nn.DataParallel(model) #use multiple GPUs","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:42:13.402458Z","iopub.execute_input":"2022-11-09T01:42:13.402818Z","iopub.status.idle":"2022-11-09T01:42:13.408743Z","shell.execute_reply.started":"2022-11-09T01:42:13.402780Z","shell.execute_reply":"2022-11-09T01:42:13.405592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"raw","source":"num_epochs = 3\nlr = 1e-4\nmodel.to(device)\n# class_weights.to(device)\n\nbest_val_acc = train(\n    model=model,\n    lr=lr,\n    epochs=num_epochs,\n    train_dl=train_dl,\n    val_dl=val_dl,\n    device=device,\n    saving_name=\"resnet152\",\n    best_val_acc=best_val_acc,\n    weight=None,\n)","metadata":{"execution":{"iopub.status.busy":"2022-11-03T19:38:23.421054Z","iopub.status.idle":"2022-11-03T19:38:23.421564Z"}}},{"cell_type":"code","source":"model.to(device)\n\nprint(f\"\\nTraining accuracy {evaluate(model, train_dl, device)}\")\nprint(f\"\\nValidation accuracy {evaluate(model, val_dl, device)}\")\nprint(f\"\\nTest accuracy {evaluate(model, test_dl, device)}\")","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:42:13.409901Z","iopub.execute_input":"2022-11-09T01:42:13.410244Z","iopub.status.idle":"2022-11-09T01:55:21.231666Z","shell.execute_reply.started":"2022-11-09T01:42:13.410209Z","shell.execute_reply":"2022-11-09T01:55:21.230743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_save = []\nfor data in tqdm(test_dl):\n             with torch.no_grad(): \n                X, y = data\n#                 X, y = X.to(device), y.to(device)\n                X, y = X.to(device).half(), y.to(device)\n                #optimizer.zero_grad()  # clear gradient information.\n                #if weight is not None:\n                #    criterion.weight = weight.half()\n                #criterion.to(device)\n                \n                with torch.cuda.amp.autocast():\n                    output = model(X)\n                #for date in output:\n                    for date, label in zip(output, y):\n                        y_save.append((torch.argmax(date).item(), label.item()))","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:55:21.233028Z","iopub.execute_input":"2022-11-09T01:55:21.233380Z","iopub.status.idle":"2022-11-09T01:56:14.986437Z","shell.execute_reply.started":"2022-11-09T01:55:21.233340Z","shell.execute_reply":"2022-11-09T01:56:14.985590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_save[:50]","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:56:14.987809Z","iopub.execute_input":"2022-11-09T01:56:14.988402Z","iopub.status.idle":"2022-11-09T01:56:14.996954Z","shell.execute_reply.started":"2022-11-09T01:56:14.988360Z","shell.execute_reply":"2022-11-09T01:56:14.995701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\nranges = []\nfor i in range(0, 51):\n    ranges.append(i)\nreal = []\npred = []\n\nfor tup in y_save:\n    pred.append(tup[0])\n    real.append(tup[1])\ncnf_matrix = confusion_matrix(y_true=real, y_pred=pred, normalize='pred')\ncorrect = 0\nfalse = 0\n\n\nfor i in ranges:\n    for j in ranges:\n        \n        cnf_matrix[i][j] = int(cnf_matrix[i][j]*100)\n        #print(type(cnf_matrix[i]))\n        #x = int(cnf_matrix[i][j])\n        #cnf_matrix[i][j] = x\nhui = []\nfor arr in cnf_matrix:\n    hui.append(arr.astype(int))\nprint(\"correct: {}\".format(correct))\nprint(\"false: {}\".format(false))\n#print(false/(correct + false))\nprint(hui)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:57:00.478105Z","iopub.execute_input":"2022-11-09T01:57:00.478470Z","iopub.status.idle":"2022-11-09T01:57:00.571310Z","shell.execute_reply.started":"2022-11-09T01:57:00.478432Z","shell.execute_reply":"2022-11-09T01:57:00.570439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(20,20), dpi=400) \nsns.heatmap(hui, annot=True, fmt='d', ax=ax, xticklabels=ranges, yticklabels=ranges)\nplt.savefig('heatmap.png')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-09T02:14:02.158208Z","iopub.execute_input":"2022-11-09T02:14:02.158593Z","iopub.status.idle":"2022-11-09T02:14:22.548021Z","shell.execute_reply.started":"2022-11-09T02:14:02.158556Z","shell.execute_reply":"2022-11-09T02:14:22.547205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## EfficientNet","metadata":{}},{"cell_type":"raw","source":"!wget https://raw.githubusercontent.com/abhuse/pytorch-efficientnet/master/efficientnet_v2.py","metadata":{"execution":{"iopub.status.busy":"2022-11-03T19:38:23.424731Z","iopub.status.idle":"2022-11-03T19:38:23.425322Z"}}},{"cell_type":"raw","source":"import torch\nfrom efficientnet_v2 import EfficientNetV2\n\nmodel = EfficientNetV2('s',\n                        in_channels=3,\n                        n_classes=num_classes,\n                        pretrained=True)","metadata":{"execution":{"iopub.status.busy":"2022-11-03T19:38:23.426524Z","iopub.status.idle":"2022-11-03T19:38:23.427106Z"}}},{"cell_type":"raw","source":"summary(model, input_size=(batch_size, 3, IMG_SIZE, IMG_SIZE))","metadata":{"execution":{"iopub.status.busy":"2022-11-03T19:38:23.428270Z","iopub.status.idle":"2022-11-03T19:38:23.428989Z"}}},{"cell_type":"raw","source":"if torch.cuda.device_count() > 1:\n    model = nn.DataParallel(model) #use multiple GPUs","metadata":{"execution":{"iopub.status.busy":"2022-11-03T19:38:23.430088Z","iopub.status.idle":"2022-11-03T19:38:23.430688Z"}}},{"cell_type":"raw","source":"num_epochs = 4\nlr = 0.001\nmodel.to(device)\n# class_weights.to(device)\n\nbest_val_acc = train(\n    model=model,\n    lr=lr,\n    epochs=num_epochs,\n    train_dl=train_dl,\n    val_dl=val_dl,\n    device=device,\n    saving_name=\"EfficientNetV2-s\",\n    best_val_acc=best_val_acc,\n    weight=None,\n)","metadata":{"execution":{"iopub.status.busy":"2022-11-03T19:38:23.433014Z","iopub.status.idle":"2022-11-03T19:38:23.433824Z"}}},{"cell_type":"raw","source":"print(f\"\\nTraining accuracy {evaluate(model, train_dl, device)}\")\nprint(f\"\\nValidation accuracy {evaluate(model, val_dl, device)}\")\nprint(f\"\\nTest accuracy {evaluate(model, test_dl, device)}\")","metadata":{"execution":{"iopub.status.busy":"2022-11-03T19:38:23.434991Z","iopub.status.idle":"2022-11-03T19:38:23.435781Z"}}},{"cell_type":"markdown","source":"## Vision Transformer","metadata":{}},{"cell_type":"raw","source":"from vision_transformer_pytorch import VisionTransformer\nmodel = VisionTransformer.from_pretrained('ViT-B_16', num_classes=num_classes)\nmodel.image_size","metadata":{"execution":{"iopub.status.busy":"2022-10-30T00:16:37.755473Z","iopub.execute_input":"2022-10-30T00:16:37.755816Z","iopub.status.idle":"2022-10-30T00:16:44.186335Z","shell.execute_reply.started":"2022-10-30T00:16:37.755783Z","shell.execute_reply":"2022-10-30T00:16:44.185305Z"}}},{"cell_type":"raw","source":"# if torch.cuda.device_count() > 1:\n#     model = nn.DataParallel(model) #use multiple GPUs","metadata":{"execution":{"iopub.status.busy":"2022-11-03T19:38:23.436968Z","iopub.status.idle":"2022-11-03T19:38:23.437761Z"}}},{"cell_type":"raw","source":"model","metadata":{"execution":{"iopub.status.busy":"2022-11-03T19:38:23.438940Z","iopub.status.idle":"2022-11-03T19:38:23.439726Z"}}},{"cell_type":"raw","source":"num_epochs = 4\nlr = 0.001\nmodel.to(device)\n# class_weights.to(device)\n\nbest_val_acc = train(\n    model=model,\n    lr=lr,\n    epochs=num_epochs,\n    train_dl=train_dl,\n    val_dl=val_dl,\n    device=device,\n    saving_name=\"simple_vit\",\n    best_val_acc=best_val_acc,\n    weight=None,\n)","metadata":{"execution":{"iopub.status.busy":"2022-11-03T19:38:23.440954Z","iopub.status.idle":"2022-11-03T19:38:23.441755Z"}}},{"cell_type":"raw","source":"print(f\"\\nTraining accuracy {evaluate(model, train_dl, device)}\")\nprint(f\"\\nValidation accuracy {evaluate(model, val_dl, device)}\")\nprint(f\"\\nTest accuracy {evaluate(model, test_dl, device)}\")","metadata":{"execution":{"iopub.status.busy":"2022-11-03T19:38:23.442932Z","iopub.status.idle":"2022-11-03T19:38:23.443726Z"}}},{"cell_type":"markdown","source":"## ResNet 50","metadata":{}},{"cell_type":"raw","source":"model = models.resnet50(pretrained=True)","metadata":{"execution":{"iopub.status.busy":"2022-11-03T19:38:23.444901Z","iopub.status.idle":"2022-11-03T19:38:23.445694Z"}}},{"cell_type":"raw","source":"model","metadata":{"execution":{"iopub.status.busy":"2022-11-03T19:38:23.446871Z","iopub.status.idle":"2022-11-03T19:38:23.447654Z"}}},{"cell_type":"raw","source":"model.fc = nn.Linear(in_features=2048, out_features=num_classes, bias=True)","metadata":{"execution":{"iopub.status.busy":"2022-11-03T19:38:23.448826Z","iopub.status.idle":"2022-11-03T19:38:23.449609Z"}}},{"cell_type":"raw","source":"model = model.to(device=device)","metadata":{"execution":{"iopub.status.busy":"2022-11-03T19:38:23.450765Z","iopub.status.idle":"2022-11-03T19:38:23.451573Z"}}},{"cell_type":"raw","source":"# if torch.cuda.device_count() > 1:\n#     model = nn.DataParallel(model) #use multiple GPUs","metadata":{"execution":{"iopub.status.busy":"2022-11-03T19:38:23.452743Z","iopub.status.idle":"2022-11-03T19:38:23.453515Z"}}},{"cell_type":"raw","source":"num_epochs = 4\nlr = 0.00005\nmodel.to(device)\n# class_weights.to(device)\n\nbest_val_acc = train(\n    model=model,\n    lr=lr,\n    epochs=num_epochs,\n    train_dl=train_dl,\n    val_dl=val_dl,\n    device=device,\n    saving_name=\"resnet50\",\n    best_val_acc=best_val_acc,\n    weight=None,\n)","metadata":{"execution":{"iopub.status.busy":"2022-11-03T19:38:23.454683Z","iopub.status.idle":"2022-11-03T19:38:23.455525Z"}}},{"cell_type":"raw","source":"print(f\"\\nTraining accuracy {evaluate(model, train_dl, device)}\")\nprint(f\"\\nValidation accuracy {evaluate(model, val_dl, device)}\")\nprint(f\"\\nTest accuracy {evaluate(model, test_dl, device)}\")","metadata":{"execution":{"iopub.status.busy":"2022-11-03T19:38:23.456812Z","iopub.status.idle":"2022-11-03T19:38:23.457587Z"}}},{"cell_type":"markdown","source":"## Demo","metadata":{}},{"cell_type":"code","source":"#Adding a personal image for testing purposes\npersonal_img_path = \"../input/Personal-Golden-Gate-Bridge-Jannis/IMG-20221102-WA0030.jpg\"\nimg = Image.open(personal_img_path)\nplt.imshow(img)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:57:12.176019Z","iopub.execute_input":"2022-11-09T01:57:12.176453Z","iopub.status.idle":"2022-11-09T01:57:12.420964Z","shell.execute_reply.started":"2022-11-09T01:57:12.176418Z","shell.execute_reply":"2022-11-09T01:57:12.420052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Getting some random, likely unrelated pictures from the internet depicting some of the most frequent classes\n!mkdir test\n!wget --output-document test/img1.jpg https://upload.wikimedia.org/wikipedia/commons/b/bf/Golden_Gate_Bridge_as_seen_from_Battery_East.jpg    \n!wget --output-document test/img2.jpg https://www.inside-guide-to-san-francisco-tourism.com/images/golden-gate-bridge-headlands-view-1.jpg    \n!wget --output-document test/img3.jpg https://upload.wikimedia.org/wikipedia/commons/d/dd/Khachkars.jpg\n!wget --output-document test/img4.jpg https://caucasusholidays.am/sites/default/files/Noraduz-Cemetery-2.3.jpg    \n!wget --output-document test/img5.jpg https://upload.wikimedia.org/wikipedia/commons/thumb/d/d5/Noratus_Khachkars.jpg/315px-Noratus_Khachkars.jpg    \n!wget --output-document test/img6.jpg https://live.staticflickr.com/613/21984578165_a567fa76c1_b.jpg    \n!wget --output-document test/img7.jpg https://upload.wikimedia.org/wikipedia/commons/0/08/Museum_of_Folk_Architecture_and_Ethnography_in_Pyrohiv_-_old_house_-_2388.jpg\n!wget --output-document test/img8.jpg https://upload.wikimedia.org/wikipedia/commons/d/dc/Museum_of_Folk_Architecture_and_Ethnography_in_Pyrohiv_-_wooden_church_from_Kanora.jpg\n!wget --output-document test/img9.jpg https://upload.wikimedia.org/wikipedia/commons/thumb/4/4d/Museum_of_Folk_Architecture_and_Ethnography_in_Pyrohiv_-_Pylypets_house_with_watermill_-_2420.jpg/1280px-Museum_of_Folk_Architecture_and_Ethnography_in_Pyrohiv_-_Pylypets_house_with_watermill_-_2420.jpg\n!wget --output-document test/img10.jpg https://www.niagarafallsstatepark.com/~/media/parks/niagara-falls/homepage/banner-niagara1.jpg\n!wget --output-document test/img11.jpg https://cdn.britannica.com/30/94430-050-D0FC51CD/Niagara-Falls.jpg","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:57:12.422270Z","iopub.execute_input":"2022-11-09T01:57:12.422715Z","iopub.status.idle":"2022-11-09T01:57:34.360821Z","shell.execute_reply.started":"2022-11-09T01:57:12.422681Z","shell.execute_reply":"2022-11-09T01:57:34.359795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Showing all images for the demonstration\nimage_paths = []\nimage_paths.append(personal_img_path)\n\nimgs = []\n\nfor i in range(11):\n    image_paths.append(f\"./test/img{i+1}.jpg\")\n    \nplt.figure(figsize=(25,25), dpi=250)\ncolumns = 3\n\nfor i, image_path in enumerate(image_paths):\n    img = Image.open(image_path)\n    imgs.append(img)\n#     plt.subplot(len(image_paths) / columns + 1, columns, i + 1)\n#     plt.imshow(img)\n    \nfor i, img in enumerate(imgs):\n    plt.subplot(len(imgs) / columns + 1, columns, i + 1)\n    plt.imshow(img)\n    \nimgs","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:57:34.362499Z","iopub.execute_input":"2022-11-09T01:57:34.362860Z","iopub.status.idle":"2022-11-09T01:57:46.172642Z","shell.execute_reply.started":"2022-11-09T01:57:34.362820Z","shell.execute_reply":"2022-11-09T01:57:46.171388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Showing the problem of using tensor.unsqueeze_(0) on our images by only getting the same prediction over and over again\nindices = []\ntransformed_real_imgs = []\n\nfor img in imgs:\n    transformed_image = test_val_transform(img).to(device)\n    \n    img = transforms.ToPILImage()(transformed_image)\n    transformed_real_imgs.append(img)\n    \n    transformed_image.unsqueeze_(0)\n    output = model(transformed_image)\n    for idx, val in enumerate(output):\n        print(torch.argmax(val))\n\n    index = output.cpu().data.numpy().argmax()\n#     index = torch.argmax(output)\n\n    indices.append(index)\n\ncolumns = 4\nplt.figure(figsize=(20,20), dpi=250)\ncolumns = 3\n        \nfor i, img in enumerate(transformed_real_imgs):\n    ax = plt.subplot(len(transformed_real_imgs) / columns + 1, columns, i + 1)\n    ax.title.set_text(f'Prediction: {indices[i]} / {train_df.loc[train_df[\"landmark_id\"] == indices[i]].category.drop_duplicates().values[0]}')\n    plt.imshow(img)\n","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:57:46.174369Z","iopub.execute_input":"2022-11-09T01:57:46.175056Z","iopub.status.idle":"2022-11-09T01:57:50.651953Z","shell.execute_reply.started":"2022-11-09T01:57:46.174999Z","shell.execute_reply":"2022-11-09T01:57:50.651185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Showing how the model actually works perfectly fine on the unseen test data\nimport cv2\n\ntest_imgs = []\ntest_indices = []\n\nplt.figure(figsize=(20,20), dpi=250)\ncolumns = 3\n\nfor data in tqdm(test_dl):\n    x, Y = data\n    for tensor in x:\n#         img = transforms.ToPILImage(tensor)\n        img = transforms.ToPILImage()(tensor)\n        test_imgs.append(img)\n        \n    x, Y = x.to(device), Y.to(device)\n    output = model(x)\n    for idx, val in enumerate(output):\n        max_val = val.cpu().data.numpy().argmax()\n        test_indices.append((max_val, Y[idx].item()))\n    break\n        \nfor i, img in enumerate(test_imgs[:12]):\n    ax = plt.subplot(len(test_imgs[:12]) / columns + 1, columns, i + 1)\n    ax.title.set_text(f'Pred/Label: {test_indices[i]} / {train_df.loc[train_df[\"landmark_id\"] == test_indices[i][0]].category.drop_duplicates().values[0]}')\n    plt.imshow(img)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:57:50.653260Z","iopub.execute_input":"2022-11-09T01:57:50.653967Z","iopub.status.idle":"2022-11-09T01:57:55.634267Z","shell.execute_reply.started":"2022-11-09T01:57:50.653913Z","shell.execute_reply":"2022-11-09T01:57:55.633302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Showing how the model cannot predict the same picture from the test set we just saw before correctly predicted when using unsqueeze\nimport cv2\n\ntest_imgs = []\ntest_indices = []\n\nplt.figure(figsize=(20,20), dpi=80)\ncolumns = 3\n\nfor data in tqdm(test_dl):\n    x, Y = data\n    for i, tensor in enumerate(x):\n#         img = transforms.ToPILImage(tensor)\n        img = transforms.ToPILImage()(tensor)\n        test_imgs.append(img)\n        tensor = tensor.to(device)\n        \n#         tensor.unsqueeze_(0)\n#         output = model(tensor)\n\n#         output = model(tensor[None, ...])\n        \n#         x = x.to(device)\n#         output = model(x[i:i+1])\n        \n\n        index = output.cpu().data.numpy().argmax()\n        test_indices.append((max_val, Y[i].item()))\n        break\n    break\n        \nfor i, img in enumerate(test_imgs[:12]):\n    ax = plt.subplot(len(test_imgs[:12]) / columns + 1, columns, i + 1)\n    ax.title.set_text(f'Pred/Label: {test_indices[i]} / {train_df.loc[train_df[\"landmark_id\"] == test_indices[i][0]].category.drop_duplicates().values[0]}')\n    plt.imshow(img)\n    \ntest_imgs","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:57:55.635785Z","iopub.execute_input":"2022-11-09T01:57:55.636101Z","iopub.status.idle":"2022-11-09T01:57:57.017580Z","shell.execute_reply.started":"2022-11-09T01:57:55.636065Z","shell.execute_reply":"2022-11-09T01:57:57.016735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Fixing the unsqueeze problem by replacing it with the ImageFolder, creating a small batch for our manually picked \n#images and now comparing the performance of the model on those pictures\nimport torchvision\n\ndata = torchvision.datasets.ImageFolder(root=\"./\", transform=test_val_transform)\ndata_loader = torch.utils.data.DataLoader(dataset=data, batch_size=12, shuffle=False, num_workers=0)\n\nimgs = []\nindices = []\n\n\nplt.figure(figsize=(20,20), dpi=200)\ncolumns = 3\n\nfor data in tqdm(data_loader):\n    x, Y = data\n    for tensor in x:\n#         img = transforms.ToPILImage(tensor)\n        img = transforms.ToPILImage()(tensor)\n        imgs.append(img)\n        \n    x, Y = x.to(device), Y.to(device)\n    output = model(x)\n    for idx, val in enumerate(output):\n        max_val = val.cpu().data.numpy().argmax()\n        indices.append(max_val)\n        \nfor i, img in enumerate(imgs[:12]):\n    ax = plt.subplot(len(imgs[:12]) / columns + 1, columns, i + 1)\n    ax.title.set_text(f'Pred: {indices[i]} / {train_df.loc[train_df[\"landmark_id\"] == indices[i]].category.drop_duplicates().values[0]}')\n    plt.imshow(img)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T01:57:57.018927Z","iopub.execute_input":"2022-11-09T01:57:57.019279Z","iopub.status.idle":"2022-11-09T01:58:00.159021Z","shell.execute_reply.started":"2022-11-09T01:57:57.019233Z","shell.execute_reply":"2022-11-09T01:58:00.158190Z"},"trusted":true},"execution_count":null,"outputs":[]}]}