{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Training file Hotel Id assignment #\n*more info here*","metadata":{}},{"cell_type":"code","source":"# Global constanst etc:\n\nNMBR_HOTELS = 3116\n#NMBR_HOTELS = 10\nPASSES_PER_EPOCH = 5\nMAX_WINDOWS_FILENAME_CHAR_LENGTH = 260 \nEMBEDDING_SIZE = 1500\nVAL_AFTER_EPOCHS = 5\n\nNMBR_EPOCHS = 5\nLR = 0.0005\nGAMMA = 0.8\nModel_description = \"EfficientNet, pretrained & constant, Linear embedding layer\"\n\nprint(\"\"\"FULL RUN:\nPasses per Epoch: {0};\nEmbedding Size: {1};\n# Epochs: {2};\nLearning Rate: {3};\nGamma: {4};\nModel: {5}.\n\"\"\".format(PASSES_PER_EPOCH, EMBEDDING_SIZE, NMBR_EPOCHS, LR, GAMMA, Model_description))","metadata":{"execution":{"iopub.status.busy":"2022-06-06T08:05:26.346188Z","iopub.execute_input":"2022-06-06T08:05:26.346589Z","iopub.status.idle":"2022-06-06T08:05:26.374634Z","shell.execute_reply.started":"2022-06-06T08:05:26.346513Z","shell.execute_reply":"2022-06-06T08:05:26.37373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Imports, Setup, etc. #","metadata":{}},{"cell_type":"code","source":"# Imports:\n! pip install timm --no-index --find-links=file:///kaggle/input/timm-package/timm\n! python -m pip -qq install --no-index --find-links /kaggle/input/faiss-163/ faiss-cpu==1.6.3 && \\\necho \"Successfully installed FAISS package\"","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-06-06T08:05:26.37639Z","iopub.execute_input":"2022-06-06T08:05:26.376877Z","iopub.status.idle":"2022-06-06T08:05:47.404434Z","shell.execute_reply.started":"2022-06-06T08:05:26.376839Z","shell.execute_reply":"2022-06-06T08:05:47.403424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport math as m\nfrom datetime import datetime\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport torch\nimport torch.nn as nn\nimport torchvision\nimport random\nimport numpy.random as rnd\nimport os\nimport matplotlib.pyplot as plt\nfrom PIL import Image as pil_image\nimport albumentations.pytorch as APT\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision.models as models\nimport timm\nimport cv2\n\n\n# Don't know what this does, so I'll just keep it there just in case...\n#import os\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n#    for filename in filenames:\n#        pass\n        #print(os.path.join(dirname, filename))\n\n# File & Folder names (not all used atm):        \nPROJECT_FOLDER = \"../input/hotel-id-to-combat-human-trafficking-2022-fgvc9/\"\nDATA_FOLDER = PROJECT_FOLDER+\"/train_images\" # I added this one but we do not use it currently... (includes all pictures shuffled and padded.)\nIMAGE_FOLDER = DATA_FOLDER + \"images/\"\nOUTPUT_FOLDER = \"\"\n\n# Make random-ness deterministic\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    \nseed_everything(42)","metadata":{"execution":{"iopub.status.busy":"2022-06-06T08:05:47.406325Z","iopub.execute_input":"2022-06-06T08:05:47.406922Z","iopub.status.idle":"2022-06-06T08:05:56.336671Z","shell.execute_reply.started":"2022-06-06T08:05:47.406879Z","shell.execute_reply":"2022-06-06T08:05:56.335727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Image Processing\n##### Includes creating datasets & Loaders, and data transformations","metadata":{}},{"cell_type":"markdown","source":"### Dataset class v2\n##### used in rest of code","metadata":{}},{"cell_type":"code","source":"class Hotel_Dataset_Merlijn_Triplets(Dataset):\n    \"\"\"Dataset to load Hotel images.\"\"\"\n    \n    def __init__(self, root_dir=\"../input/hotelid-2022-train-images-256x256/\", data_path=\"images/\", hotel_file_path=\"train.csv\", \n                 train_partition = 0.9, is_trainSet = True, transform = None, nmbr_hotels = None):\n        # Setting variables:\n        \n        self.transform = transform\n        self.is_trainSet = is_trainSet\n        self.data_path = os.path.join(root_dir,data_path)\n        \n        self.image_hotelId_dict = dict({})\n        self.csv_data = pd.read_csv(os.path.join(root_dir, hotel_file_path))[\"hotel_id\"]\n        \n        # Read all data:\n        for image_file in os.listdir(self.data_path):\n            if image_file[-4:] == \".jpg\":\n                image_nmbr = int(image_file[:-4])\n                if image_nmbr in self.csv_data:\n                    hotel_id = self.csv_data[image_nmbr]\n\n                    if hotel_id in self.image_hotelId_dict:\n                        self.image_hotelId_dict[hotel_id].append(image_nmbr)\n                    else:\n                        self.image_hotelId_dict[hotel_id] = [image_nmbr]\n        \n        if nmbr_hotels == None:\n            self.nmbr_hotels = len(self.image_hotelId_dict)\n        else:\n            self.nmbr_hotels = nmbr_hotels\n        self.data = np.arange(self.nmbr_hotels*PASSES_PER_EPOCH)\n        \n        # Re-write into labels, create val & train set\n        self.Label_to_Id = np.zeros(len(self.image_hotelId_dict), dtype = int)\n        self.image_label_dict_train = dict({})\n        self.image_label_dict_val = dict({})\n        self.image_label_dict_total = dict({})\n        \n        for (label, (idx,images)) in enumerate(self.image_hotelId_dict.items()):\n            self.Label_to_Id[label] = idx\n            max_train_index = max(m.floor(len(images)*train_partition),1)\n            self.image_label_dict_train[label] = images[:max_train_index]\n            self.image_label_dict_val[label] = images[max_train_index:]\n            if len(self.image_label_dict_val[label]) == 0:\n                self.image_label_dict_val[label] = self.image_label_dict_train[label]\n            self.image_label_dict_total[label] = images\n                \n    \n    def label_to_id(self, label):\n        return self.Label_to_Id[int(label)]\n    \n    def labelList_to_id(self, labels):\n        ids = []\n        for label in labels:\n            ids.append(self.label_to_id(label))\n        return ids\n    \n    def __len__(self):\n        return len(self.data)\n    \n    def __size__(self):\n        return np.shape(self.data)\n    \n    def __get_label_list__(self):\n        return self.Label_to_Id\n    \n    def __images_to_filepaths__(self, ims):\n        paths = []\n        for im in ims:\n            file = ( '0000000000'+str(im)+\".jpg\") [-13:]\n            paths.append (os.path.join(self.data_path, file) )\n        return paths\n    \n    def __get_all_files__(self):\n        return self.image_label_dict_total\n    \n    def __get_labels__(self, images):\n        all_labels = []\n        for image in images:\n            all_labels.append(self.label_image_dict_total[image])\n        return all_labels\n        \n    def __getitem__(self, label): # AKA: get a (random!) triplet from given class.\n        \n        if self.is_trainSet:\n            image_dict = self.image_label_dict_train\n        else:\n            image_dict = self.image_label_dict_val\n        \n        label = label % self.nmbr_hotels\n        \n        # Get image nmbrs \n        im1, im2 = rnd.choice(image_dict[label], 2) # our two positives images\n        neg_label = label\n        while neg_label == label:\n            neg_label = rnd.randint(0,self.nmbr_hotels) # get a random different class\n        im3 = rnd.choice(image_dict[neg_label],1)[0] # our negative images\n        \n        # Get file paths:\n        im1, im2, im3 = self.__images_to_filepaths__([im1,im2,im3])\n        \n        # rewrite to tensors:\n        if self.transform:\n            #im1, im2, im3 = self.transform(pil_image.open(im1)), self.transform(pil_image.open(im2)), self.transform(pil_image.open(im3))\n            im1, im2, im3 = self.transform(image=np.array(pil_image.open(im1)))[\"image\"], self.transform(image=np.array(pil_image.open(im2)))[\"image\"], self.transform(image=np.array(pil_image.open(im3)))[\"image\"]\n        \n        return [im1, im2, im3, (label, neg_label)]\n\ntest = Hotel_Dataset_Merlijn_Triplets()","metadata":{"execution":{"iopub.status.busy":"2022-06-06T08:05:56.338285Z","iopub.execute_input":"2022-06-06T08:05:56.339315Z","iopub.status.idle":"2022-06-06T08:05:57.445408Z","shell.execute_reply.started":"2022-06-06T08:05:56.339274Z","shell.execute_reply":"2022-06-06T08:05:57.444588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Defining data transformations:","metadata":{}},{"cell_type":"code","source":"class AddRandomMaskTransform:\n    \n    def __init__(self):\n        self.masks_path = r'../input/hotel-id-to-combat-human-trafficking-2022-fgvc9/train_masks'\n        self.max_covered_area_ratio=0.5\n    \n    def __call__(self, source_img: pil_image):\n        import math\n        source_width, source_height = source_img.size\n        source_area = source_width*source_height\n        \n        # Retrieve a random mask\n        random_mask_file_name = random.choice(os.listdir(self.masks_path))\n        random_mask_path = os.path.join(self.masks_path, random_mask_file_name)\n        random_mask = pil_image.open(random_mask_path)\n\n        # The training masks provided by the competition often seem to be at least as large as some of the images\n        # of the hotels. Resize the mask to constrain its surface area to max_covered_area_ratio, while retaining\n        # its aspect ratio.\n        original_mask_width, original_mask_height = random_mask.size\n        original_mask_area = original_mask_width*original_mask_height\n        max_mask_area = source_area*self.max_covered_area_ratio\n        mask_width = math.floor(math.sqrt((max_mask_area*original_mask_width)/original_mask_height))\n        mask_height = math.floor(math.sqrt((max_mask_area*original_mask_height)/original_mask_width))\n        mask = random_mask.resize((mask_width, mask_height))\n\n        # Paste the mask over a random position in the image\n        max_x_coord = max(0, source_width - mask_width)\n        max_y_coord = max(0, source_height - mask_height)\n        random_x_coord = random.randint(0, max_x_coord)\n        random_y_coord = random.randint(0, max_y_coord)\n        source_img.paste(mask, (random_x_coord, random_y_coord), mask)\n        # Mind that most masks have a transparent border that offsets them from (0,0), preventing\n        # most of them from ever showing up all the way at the left or the top of the image.\n        return source_img","metadata":{"execution":{"iopub.status.busy":"2022-06-06T08:05:57.447467Z","iopub.execute_input":"2022-06-06T08:05:57.447747Z","iopub.status.idle":"2022-06-06T08:05:57.458402Z","shell.execute_reply.started":"2022-06-06T08:05:57.447722Z","shell.execute_reply":"2022-06-06T08:05:57.457416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMAGE_SIZE_WIDTH = 256\nIMAGE_SIZE_HEIGHT = 256\nIMG_SIZE = IMAGE_SIZE_WIDTH\n\nimport albumentations as A\nimport albumentations.pytorch as APT\nimport cv2 \n\n# used for training dataset - augmentations and occlusions\ntrain_transform = A.Compose([\n    A.HorizontalFlip(p=0.75),\n    A.VerticalFlip(p=0.25),\n    A.ShiftScaleRotate(p=0.5, border_mode=cv2.BORDER_CONSTANT),\n    A.OpticalDistortion(p=0.25),\n    A.Perspective(p=0.25),\n    A.CoarseDropout(p=0.5, min_holes=1, max_holes=6, \n                    min_height=IMG_SIZE//16, max_height=IMG_SIZE//4,\n                    min_width=IMG_SIZE//16,  max_width=IMG_SIZE//4), # normal coarse dropout\n    \n    A.CoarseDropout(p=0.75, max_holes=1, \n                    min_height=IMG_SIZE//4, max_height=IMG_SIZE//2,\n                    min_width=IMG_SIZE//4,  max_width=IMG_SIZE//2, \n                    fill_value=(255,0,0)),# simulating occlusions in test data\n\n    #A.RandomBrightnessContrast(p=0.75),\n    A.ToFloat(),\n    APT.transforms.ToTensorV2(),\n])\n\n# used for validation dataset - only occlusions\nval_transform = A.Compose([\n    A.CoarseDropout(p=0.75, max_holes=1, \n                    min_height=IMG_SIZE//4, max_height=IMG_SIZE//2,\n                    min_width=IMG_SIZE//4,  max_width=IMG_SIZE//2, \n                    fill_value=(255,0,0)),# simulating occlusions\n    A.ToFloat(),\n    APT.transforms.ToTensorV2(),\n])\n\n# no augmentations\nbase_transform = A.Compose([\n    A.ToFloat(),\n    APT.transforms.ToTensorV2(),\n])","metadata":{"execution":{"iopub.status.busy":"2022-06-06T08:05:57.460452Z","iopub.execute_input":"2022-06-06T08:05:57.460872Z","iopub.status.idle":"2022-06-06T08:05:57.473772Z","shell.execute_reply.started":"2022-06-06T08:05:57.460772Z","shell.execute_reply":"2022-06-06T08:05:57.47293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Hotel_Dataset_Merlijn(Dataset):\n    \"\"\"Dataset to load Hotel images.\"\"\"\n    \n    def __init__(self, root_dir=\"../input/hotel-id-to-combat-human-trafficking-2022-fgvc9/\", data_path=\"train_images/\",\n                 labels_known = True, max_images=10, transform=None, label_list=None, save_names=False):\n        # Setting variables:\n        \n        self.transform = transform\n        self.max_images = max_images\n        self.labels_known = labels_known\n        \n        # Directory paths:\n        self.root_dir = root_dir\n        self.data_path = data_path\n        self.full_data_path = os.path.join(root_dir, data_path)\n        dirs = os.listdir(self.full_data_path)\n\n        # Initialise as empty dataset:\n        data_size = NMBR_HOTELS*self.max_images\n        if labels_known:\n            self.data = np.chararray((data_size,2), itemsize=MAX_WINDOWS_FILENAME_CHAR_LENGTH)\n            self.label_to_id_arr = np.zeros( NMBR_HOTELS, dtype=np.int32 )\n        else:\n            self.data = np.chararray(data_size, itemsize=MAX_WINDOWS_FILENAME_CHAR_LENGTH)\n            if type(label_list) != None:\n                self.label_to_id_arr = label_list\n            else:\n                print(\"Warning: no lable list provided for test dataset!\")\n            \n        # Sampling Train or Val data (with labels known):\n        \n        if labels_known:\n            for (dir_nmbr, this_dir) in enumerate(dirs):\n                if dir_nmbr < NMBR_HOTELS:\n                    self.label_to_id_arr[dir_nmbr] = int(this_dir)\n\n                    # Sample pictures:\n                    path_to_dir = os.path.join(self.full_data_path, this_dir)\n                    files = os.listdir(path_to_dir)\n                    sampled_pics = rnd.choice(files, size=self.max_images) #Note: these could be duplicates...\n\n                    #Save as data:\n                    data = np.dstack( (sampled_pics, np.repeat(dir_nmbr, self.max_images)))\n                    (index_start, index_end) = ( (dir_nmbr)*self.max_images, (dir_nmbr+1)*self.max_images)\n                    self.data[index_start: index_end] = data\n            \n        # Sampling Test data:\n        \n        else:\n            self.data = os.listdir(self.full_data_path)\n    \n    def label_to_id(self, label):\n        return self.label_to_id_arr[int(label)]\n    \n    def __len__(self):\n        return len(self.data)\n    \n    def __size__(self):\n        return np.shape(self.data)\n    \n    def __get_label_list__(self):\n        return self.label_to_id_arr\n    \n    def __getitem__(self, idx):\n        #note: has to be changed for no label types, still!\n        \n        if self.labels_known:\n            record = self.data[idx].decode()\n            label = int(record[1])\n            hotel_id = self.label_to_id(label)\n            image_path = os.path.join(self.full_data_path, str(hotel_id), record[0])\n            image = pil_image.open(image_path)\n            if self.transform:\n                image = self.transform(image=np.array(image))[\"image\"]\n            return (image, label)\n        \n        else:\n            record = self.data[idx]\n            image_path = os.path.join(self.full_data_path, record)\n            image = pil_image.open(image_path)\n\n            if self.transform:\n                image = self.transform(image=np.array(image))[\"image\"]\n            return (image, record)","metadata":{"execution":{"iopub.status.busy":"2022-06-06T08:08:13.087486Z","iopub.execute_input":"2022-06-06T08:08:13.087992Z","iopub.status.idle":"2022-06-06T08:08:13.107157Z","shell.execute_reply.started":"2022-06-06T08:08:13.087956Z","shell.execute_reply":"2022-06-06T08:08:13.106266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Creating Datasets & Loaders","metadata":{}},{"cell_type":"code","source":"# Method 1: both sets have each hotel represented, but duplicates are possible:\n\n#dataset_train = Hotel_Dataset_Merlijn(max_images = 1, transform=train_transform)\n#dataset_val = Hotel_Dataset_Merlijn(max_images = 1, transform=train_transform)\n# Not sure if we cannot just use the same one twice, but never mind...\n\n\n# Method 2: only duplicates in dataset may occur, no guarantee about labels\nnmbr_images, val_perc = 10, 0.8\ndataset_train = Hotel_Dataset_Merlijn_Triplets(transform=base_transform, nmbr_hotels = NMBR_HOTELS)\ndataset_val = Hotel_Dataset_Merlijn_Triplets(is_trainSet = False, transform = train_transform, nmbr_hotels = NMBR_HOTELS)\n\nlabels = dataset_train.__get_label_list__()\ndataset_test = Hotel_Dataset_Merlijn(data_path=\"test_images/\", transform=base_transform, labels_known = False, label_list = labels)\n     \ntrain_loader = DataLoader(dataset_train, batch_size = 128, shuffle=True) # these are now the classes that get taken: that should be fixed...\nval_loader = DataLoader(dataset_val, batch_size = 128, shuffle=True)\ntest_loader = DataLoader(dataset_test, batch_size = 1) #this seems sloppy, but makes predict-code nicer now...\n\n# Method 3\n# dataset_train = Hotel_Dataset_Merlijn_Bram(max_images = 20, training=True, transform=train_transform)\n# dataset_val = Hotel_Dataset_Merlijn_Bram(max_images = 20, training=False, transform=test_transform)\n# dataset_test = Hotel_Dataset_Merlijn_Bram(max_images = 20, training=False, transform=test_transform)\n\n# train_loader = DataLoader(dataset_train, batch_size = 5, shuffle=True)#len(dataset_train))\n# val_loader = DataLoader(dataset_test, batch_size = 5, shuffle=True)\n# test_loader = DataLoader(dataset_test, batch_size = 1, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2022-06-06T08:09:35.832179Z","iopub.execute_input":"2022-06-06T08:09:35.832644Z","iopub.status.idle":"2022-06-06T08:09:36.675275Z","shell.execute_reply.started":"2022-06-06T08:09:35.832602Z","shell.execute_reply":"2022-06-06T08:09:36.674281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Visualising/testing data stuff","metadata":{}},{"cell_type":"code","source":"# Test/visualise datasets:\n\n# Function to visualise datasets:\ndef show_images(ds, n_images=10, labels_known = True):\n    # Only works properly for n_images>10...\n    xlen, ylen = 10, m.ceil(n_images/10)\n    fig, ax = plt.subplots(ylen,xlen, figsize=(22,8))\n    \n    \n    for x in range(xlen):\n        for y in range(ylen):\n            if not labels_known:\n                item = ds.__getitem__(y*10+x)\n            else:\n                (item, label) = ds.__getitem__(y*10+x)\n            ax[y,x].imshow(np.transpose(item, (1,2,0))) # gives error: convert to PIL\n\ndef visualise_data():\n    print(\"\"\"Sizes Datasets:\n        Train: {0}\n        Validate: {1}\n        Test: {2}\"\"\")#.format(dataset_train.__size__(), dataset_val.__size__(), dataset_test.__size__()))\n    print(dataset_train[0])\n\n    show_images(dataset_train,20, labels_known = True) # This gives an error, but I am too lazy to fix it now...\n#visualise_data()\n\n","metadata":{"execution":{"iopub.status.busy":"2022-06-06T08:05:58.237342Z","iopub.execute_input":"2022-06-06T08:05:58.238005Z","iopub.status.idle":"2022-06-06T08:05:58.246134Z","shell.execute_reply.started":"2022-06-06T08:05:58.237964Z","shell.execute_reply":"2022-06-06T08:05:58.245333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Auxiliariy functions","metadata":{}},{"cell_type":"code","source":"def save_checkpoint(model, scheduler, optimizer, epoch, name, loss=None, score=None):\n    checkpoint = {\"epoch\": epoch,\n                  \"model\": model.state_dict(),\n                  #\"scheduler\": scheduler.state_dict(),\n                  \"optimizer\": optimizer.state_dict(),\n                  \"loss\": loss,\n                  \"score\": score,\n                  }\n\n    torch.save(checkpoint, f\"{OUTPUT_FOLDER}checkpoint-{name}.pt\")","metadata":{"execution":{"iopub.status.busy":"2022-06-06T08:05:58.248515Z","iopub.execute_input":"2022-06-06T08:05:58.248875Z","iopub.status.idle":"2022-06-06T08:05:58.259984Z","shell.execute_reply.started":"2022-06-06T08:05:58.248839Z","shell.execute_reply":"2022-06-06T08:05:58.259176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_all_embeddings(model, dataloader, transform):\n    # Create array with all embeddings:\n    print (\"Creating all embeddings:\")\n    nmbr_dirs_to_report = 100\n    all_embeddings = []\n    all_classes = []\n    model.eval()\n    with torch.no_grad():\n        all_files_dict = dataloader.__get_all_files__()\n        for (i,(hotel, image_list)) in enumerate(all_files_dict.items()):\n            if i <= NMBR_HOTELS:\n                all_images = []\n                for (j,image) in enumerate(image_list):\n                    if j < 100: # Filtering out our giant hotel-set just for testing...\n                        image = dataloader.__images_to_filepaths__([image])[0]\n                        all_images.append( transform(image=np.array(pil_image.open(image)))[\"image\"] )\n                all_images = torch.stack(all_images).to(device)\n                embeddings, _p = model(all_images)\n                embeddings = embeddings.to('cpu')\n\n                for emb in embeddings:\n                    all_embeddings.append(emb.detach().numpy())\n                    all_classes.append(hotel)\n    \n    return (np.stack(all_embeddings), np.array(all_classes))\n\ndef get_closest_labels(embedding, all_embeddings, all_labels, n=5, get_ids = False):\n    \n    index = faiss.IndexFlatL2(EMBEDDING_SIZE)\n    index.add(all_embeddings)\n    _ds, closest_images = index.search(embedding, 100)\n    closest_images = closest_images[:,1:] #filter itself!\n    all_top_5s = []\n    for ci in closest_images:\n        closest_classes = all_labels[ci]\n        unique_classes, ind = np.unique(closest_classes, return_index=True)\n        top_5 = unique_classes[np.argsort(ind)][:5]\n        if get_ids:\n            top_5 = dataset_train.__get_label_list__()[top_5]\n        all_top_5s.append(top_5)\n    \n    return all_top_5s\n    \ndef get_accuracy(top5s, target):\n    accuracy = 0\n    for i,t5 in enumerate(top5s):\n        is_correct = (t5 == target[i]).astype(float)\n        accuracy += np.max(is_correct * ( (0.5) ** np.arange(5) ) )\n    return accuracy/len(target)","metadata":{"execution":{"iopub.status.busy":"2022-06-06T08:05:58.264412Z","iopub.execute_input":"2022-06-06T08:05:58.264861Z","iopub.status.idle":"2022-06-06T08:05:58.278811Z","shell.execute_reply.started":"2022-06-06T08:05:58.264833Z","shell.execute_reply":"2022-06-06T08:05:58.277669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TripletLoss(nn.Module): # As taken from https://towardsdatascience.com/a-hands-on-introduction-to-image-retrieval-in-deep-learning-with-pytorch-651cd6dba61e\n    def __init__(self, margin=1.0):\n        super(TripletLoss, self).__init__()\n        self.margin = margin\n        \n    def calc_euclidean(self, x1, x2):\n        return (x1 - x2).pow(2).sum(1)\n    \n    def val_score(self, x1, x2, x3):\n        return self.forward(x1,x2,x3) # Does this just work?\n    \n    # Distances in embedding space is calculated in euclidean\n    def forward(self, anchor, positive, negative):\n        distance_positive = self.calc_euclidean(anchor, positive)\n        distance_negative = self.calc_euclidean(anchor, negative)\n        losses = torch.relu(distance_positive - distance_negative + self.margin)\n        return losses.mean()","metadata":{"execution":{"iopub.status.busy":"2022-06-06T08:05:58.280207Z","iopub.execute_input":"2022-06-06T08:05:58.280629Z","iopub.status.idle":"2022-06-06T08:05:58.290337Z","shell.execute_reply.started":"2022-06-06T08:05:58.280594Z","shell.execute_reply":"2022-06-06T08:05:58.289611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model","metadata":{}},{"cell_type":"code","source":"# Define model:\n\nclass Hotel_Model (nn.Module):\n    \n    def __init__(self, n_classes = NMBR_HOTELS, embedding_size = EMBEDDING_SIZE):\n        super(Hotel_Model, self).__init__()\n        \n        # use (soon-to-be) pre-trained model for heavy lifting:\n        #self.main_model = timm.create_model(\"efficientnet_b0\", num_classes=EMBEDDING_SIZE, pretrained=False)\n        \n        self.main_model = timm.create_model(\"efficientnet_b0\", num_classes=1000, pretrained=False)\n        self.main_model.load_state_dict(torch.load(\"../input/timm-pretrained-efficientnet/efficientnet/efficientnet_b0_ra-3dd342df.pth\"))\n        for param in self.main_model.parameters():\n            param.requires_grad = False\n        self.main_model.eval()\n        \n        # Use own torch-layers for last embedding step and classification:\n        in_features = self.main_model.get_classifier().in_features    \n        self.main_model.classifier = nn.Identity() # turn off classifier in pre-trained model\n        self.embedding = nn.Sequential(\n            nn.ReLU(),\n            nn.Linear(in_features, embedding_size),\n            nn.ReLU())\n        \n        # Note: classification is not used for loss, only for gauging accuracy!\n        self.classifier = nn.Linear(embedding_size, n_classes) \n        \n    \n    def forward(self,x):\n        x = self.main_model(x)\n        #x = x.view(x.size(0), -1) #re-scale\n        x = self.embedding(x)\n        return x, x #self.classifier(x)\n    \nmodel_test = Hotel_Model()","metadata":{"execution":{"iopub.status.busy":"2022-06-06T08:05:58.291712Z","iopub.execute_input":"2022-06-06T08:05:58.292091Z","iopub.status.idle":"2022-06-06T08:05:58.935854Z","shell.execute_reply.started":"2022-06-06T08:05:58.292055Z","shell.execute_reply":"2022-06-06T08:05:58.935026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training Cycle","metadata":{}},{"cell_type":"code","source":"def train_epoch(train_loader, optimizer, loss_fn, model):\n    \"\"\"Training and adjusting model for one epoch.\"\"\"\n    \n    for i, data in enumerate(train_loader): #loop over all data in current batch (in loader!)        \n        # Initialisation:\n        optimizer.zero_grad()\n        x1, x2, x3,  (_label, _neg_label) = data\n        x1, x2, x3 = x1.to(device), x2.to(device), x3.to(device)\n        \n        # Run trough model\n        e1, _p1 = model(x1)\n        e2, _p2 = model(x2)\n        e3, _p3 = model(x3)\n        \n        e1, e2, e3 = e1.to('cpu'), e2.to('cpu'), e3.to('cpu')\n        \n        # Compute loss & alter model\n        loss = loss_fn(e1, e2, e3)\n        loss.backward()\n        optimizer.step()\n        \n        # Log progress\n        #prediction, labels = prediction.detach().numpy(), labels.detach().numpy()\n        #accuracy = np.mean(labels == np.argmax(prediction, axis = 1))\n        \n        print(\"Training batch {} done! (Loss = {})\".format(i+1, loss))\n        \n        # Gather data and stuff: TBW\n    \n    print(\"All training completed!\")\n    \n    # Clear all arrays from memory:\n    #data, inputs, labels, embedding, prediction, loss = [],[],[],[],[],[]\n    \ndef val_epoch(val_loader, loss_fn, model):\n    \"Compute validation data for one epoch\"\n    # get embeddings of all images:\n    all_embeddings, all_labels = get_all_embeddings(model, dataset_train, val_transform)\n    index = faiss.IndexFlatL2(EMBEDDING_SIZE)\n    print(all_embeddings.shape, EMBEDDING_SIZE)\n    index.add(all_embeddings)\n\n    # Just like train_epoch, but without adjusting the model...\n    total_vloss = 0.0\n    avg_accuracy = 0.0\n    nmbr_images = 0.0\n    \n    for i, vdata in enumerate(val_loader):\n        x1, _x2, _x3, (label, _neg_label) = vdata\n        x1 = x1.to(device)\n        \n        emb, p1 = model(x1)\n        emb, label = emb.to('cpu').detach().numpy(), label.detach().numpy()\n        closest_image_labels = get_closest_labels(emb, all_embeddings, all_labels)\n        accuracy = get_accuracy(closest_image_labels, label)\n        avg_accuracy += accuracy\n        print( \"Validation batch {0} done! Acc: {1}\".format((i+1), accuracy) )\n    \n    vdata, inputs, labels, emb, prediction = [],[],[],[],[]\n    return (avg_accuracy / (i+1) )","metadata":{"execution":{"iopub.status.busy":"2022-06-06T08:05:58.937166Z","iopub.execute_input":"2022-06-06T08:05:58.937525Z","iopub.status.idle":"2022-06-06T08:05:58.94942Z","shell.execute_reply.started":"2022-06-06T08:05:58.937489Z","shell.execute_reply":"2022-06-06T08:05:58.948696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_complete(train_loader, val_loader, model, optimizer, scheduler, loss_fn, nmbr_epochs = 5):\n    \"\"\"Trains and Validates model given loaders and initial model.\"\"\"\n\n    timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')\n    best_vloss = 10**9\n    val_epochs = VAL_AFTER_EPOCHS\n    for epoch in range(nmbr_epochs):\n        print(\"Starting epoch {}\".format(epoch+1))\n        \n        # Train one epoch:\n        model.train()\n        train_epoch(train_loader, optimizer, loss_fn, model)\n        scheduler.step()\n        \n        if (epoch+1)%val_epochs == 0:\n            # Validate epoch:\n            print(\"Start Validation...\")\n            model.eval()\n            with torch.no_grad():\n                vloss = val_epoch(val_loader,loss_fn, model)\n                #vloss = \"Not validating!\"\n            print(\"Validation complete: Total Accuracy = {}. Save model & start new epoch:\".format(vloss))\n        \n        # Save current model:\n        save_checkpoint(model, \"test\", optimizer, epoch, \"TestTraining\")     ","metadata":{"execution":{"iopub.status.busy":"2022-06-06T08:05:58.950828Z","iopub.execute_input":"2022-06-06T08:05:58.951529Z","iopub.status.idle":"2022-06-06T08:05:58.96568Z","shell.execute_reply.started":"2022-06-06T08:05:58.951495Z","shell.execute_reply":"2022-06-06T08:05:58.964569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport faiss\n\na1 = np.array([0.,0.,0.,0.,0.])\na2 = np.array([1.,2.,3.,4.,4.])\na3 = np.array([1.,2.,3.,3.,3.])\na4 = np.array([1.,2.,2.,2.,2.])\n\nembeddings = np.array([a3,a1,a4,a2])\nembeddings = embeddings.astype(np.float32)\nprint(\"Embeddings: \\n{}\".format(embeddings))\n\ntarget = np.array([1.,2.,3.,4.,5.])\ntarget = target.astype(np.float32)\nprint(\"Target: \\n{}\".format(target))\n\nn = 3\n\ndef find_nearest_n_neighbours(target, embeddings, n: int):\n    index = faiss.IndexFlatL2(len(target))\n    index.add(embeddings)\n    \n    distances, indices = index.search(np.array([target]), n)\n    #print(\"Indices: \\n{}\".format(indices))\n    #print(\"Distances: \\n{}\".format(distances))\n    \nfind_nearest_n_neighbours(target,embeddings,n)","metadata":{"execution":{"iopub.status.busy":"2022-06-06T08:05:58.967103Z","iopub.execute_input":"2022-06-06T08:05:58.967654Z","iopub.status.idle":"2022-06-06T08:05:59.036879Z","shell.execute_reply.started":"2022-06-06T08:05:58.967619Z","shell.execute_reply":"2022-06-06T08:05:59.036048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict(test_loader, model, labels_to_id, train_embeddings, train_labels):\n    \"\"\"Function to predict the 5 most likely hotels\"\"\"\n    \n    index = faiss.IndexFlatL2(EMBEDDING_SIZE)\n    index.add(train_embeddings)\n    \n    nmbr_cases = len(test_loader)\n    predictions_top_5 = np.chararray((nmbr_cases ,2), itemsize=MAX_WINDOWS_FILENAME_CHAR_LENGTH)\n    \n    for i,data in enumerate(test_loader): #Should just be one batch...\n        (image, name) = data\n        image = image.to(device)\n        predictions_top_5[i,0] = name[0] # Don't know why we need the [0]...\n        this_embedding, _p = model(image)\n        this_embedding = this_embedding.to('cpu').detach().numpy()\n        closest_images = get_closest_labels(this_embedding, train_embeddings, train_labels, n=5, get_ids = True)[0]\n        closest_images_str = ' '.join(map(str, closest_images))\n        predictions_top_5[i,1] = closest_images_str\n        \n    return predictions_top_5\n","metadata":{"execution":{"iopub.status.busy":"2022-06-06T08:06:38.19651Z","iopub.execute_input":"2022-06-06T08:06:38.197061Z","iopub.status.idle":"2022-06-06T08:06:38.207971Z","shell.execute_reply.started":"2022-06-06T08:06:38.197013Z","shell.execute_reply":"2022-06-06T08:06:38.206621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Running the code & Making submission:","metadata":{}},{"cell_type":"code","source":"all_images_tensor = []","metadata":{"execution":{"iopub.status.busy":"2022-06-06T08:05:59.049979Z","iopub.execute_input":"2022-06-06T08:05:59.050542Z","iopub.status.idle":"2022-06-06T08:05:59.059034Z","shell.execute_reply.started":"2022-06-06T08:05:59.050503Z","shell.execute_reply":"2022-06-06T08:05:59.057951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Specify device:\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n\n# Define model & Datasets\nmodel = Hotel_Model().to(device)\n\n# Optimizer & Loss function: Placeholders for now:\n\noptimizer = torch.optim.Adagrad(model.parameters(), lr=LR)\nscheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma = GAMMA )\nloss_fn = TripletLoss()","metadata":{"execution":{"iopub.status.busy":"2022-06-06T08:05:59.062534Z","iopub.execute_input":"2022-06-06T08:05:59.063072Z","iopub.status.idle":"2022-06-06T08:06:04.02819Z","shell.execute_reply.started":"2022-06-06T08:05:59.063003Z","shell.execute_reply":"2022-06-06T08:06:04.027379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train model:\n\nnmbr_epochs = NMBR_EPOCHS\ntrain_complete(train_loader, val_loader, model, optimizer, scheduler, loss_fn, nmbr_epochs)\n\nmodel.train(False)\nwith torch.no_grad():\n    all_embeddings, all_labels = get_all_embeddings(model, dataset_train, train_transform)\n    # Make prediction:\n\n    final_preds = predict(test_loader, model, dataset_train.labelList_to_id, all_embeddings, all_labels ) # predictions as np-array\n    final_preds = pd.DataFrame(final_preds).stack().str.decode('utf-8').unstack() # as panda dataFrame of strings\n    final_preds.columns = ['image_id','hotel_id'] # rename header\n\n    print(final_preds)\n    final_preds.to_csv(OUTPUT_FOLDER + 'submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-06-06T08:15:52.391095Z","iopub.execute_input":"2022-06-06T08:15:52.39147Z","iopub.status.idle":"2022-06-06T08:15:54.041889Z","shell.execute_reply.started":"2022-06-06T08:15:52.391442Z","shell.execute_reply":"2022-06-06T08:15:54.041045Z"},"trusted":true},"execution_count":null,"outputs":[]}]}