{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**Hello all! This is a baseline model posted by the hosts for the competition. The model does image retrieval for k most similar images using Deep Metric learning. We have used the Pytorch-metric-learning library for this model (https://github.com/KevinMusgrave/pytorch-metric-learning). Competitors can use this model as a baseline and improve on it using the different mining functions available in the pytorch metric learning library.**","metadata":{}},{"cell_type":"code","source":"#install the pytorch metric learning library\n!ls ../input/pytorchmetriclearning\n!pip install pytorch_metric_learning --no-index --find-links=file:///kaggle/input/pytorchmetriclearning/ ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#install the faiss library\n!pip install faiss-gpu --no-index --find-links=file:///kaggle/input/faiss-gpu/ ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\nimport os\nimport torch\nimport torch.nn as nn\nimport pandas as pd\nimport numpy as np\nfrom torchvision import models , datasets\nimport torch.optim as optim\nimport faiss\nfrom torchvision.datasets.folder import default_loader\nimport torchvision.transforms as transforms\nfrom pytorch_metric_learning import miners, losses, samplers , distances\nfrom pytorch_metric_learning.utils import accuracy_calculator\nfrom PIL import ImageFile\nimport copy\nimport csv","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#copy the resnet50 model for offline use\n!mkdir -p /root/.cache/torch/hub/checkpoints\n!cp ../input/resnet50/resnet50.pth /root/.cache/torch/hub/checkpoints/resnet50-19c8e357.pth","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Arguments\nBATCH_SIZE = 128\nWORKERS=15\nOUTPUT_SIZE=256\nnum_epochs = 20\ntrainfile_path = '/kaggle/input/hotel-id-2021-fgvc8/train.csv'\nlr = 0.00001\nrandom.seed(224)\noutput_file = 'submission.csv'   \nmodel_path = '/kaggle/input/batchallmodel/model_batchall_best.pth'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Function for reading the images from train folder\ndef load_images(data_path):\n    image_set = []\n    \n    for path, subdirs, files in os.walk(data_path):\n        for name in files:\n            if name.endswith('.jpg'):\n                image_set.append(os.path.join(path,name))\n\n    print('Total images', len(image_set))\n    return image_set\n\ntrain_image_set =load_images('/kaggle/input/hotel-id-2021-fgvc8/train_images')\n\n#separating into training & validation sets\nsplit_index = int(len(train_image_set) * 0.9) \ntrain_images = train_image_set[:split_index]\nval_images = train_image_set[split_index:]\n\nprint('Training images -',len(train_images),'validation images -',len(val_images))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Custom Dataloader for Pytorch**  \n The custom LoadDataset class is used to override the Pytorch Dataset creater since our training metadata needs extraction of the image ids and their labels.  \n This class takes the training set file to create the dataset with data,labels mapping. It has a custom collate function since some of the images (being crowdsourced) do not load properly and such images will be eliminated from the training process.  \n The __getitem__ function can replace an image which is not loaded with a different image in the batch using a random value for \"index\". We have not used it here.","metadata":{}},{"cell_type":"code","source":"\ndef _extract_ids(im_path):\n    path_splits = im_path.split(os.sep)\n    img_id = path_splits[-1]\n    \n    return img_id\n\n# custom collate functions for pytorch for images that are not loaded\n# This will take a batch and eliminate all tensors that are set to None by the \"_default_loader\" function below.\ndef collate_fn(batch):\n    batch = list(filter(lambda x: x is not None, batch))\n    return torch.utils.data.dataloader.default_collate(batch)\n\nclass LoadDataset(torch.utils.data.Dataset):\n    def __init__(self, paths, csv_file, classes=None, transform=None):\n        self.paths = paths\n        self.mapping_file = pd.read_csv(csv_file) if csv_file is not None else None\n        if classes is None:\n            self.classes, self.class_to_idx = self._find_classes()\n        else:\n            self.classes = classes\n            self.class_to_idx = {classes[i]: i for i in range(len(classes))}\n       \n        self.samples = self._make_dataset()\n        self.targets = [s[1] for s in self.samples]\n       \n        self.transform = transform\n    \n    #function will create a class to index mapping for the classes in the training set. The validation and test sets will use the same mappings for their images.\n    def _find_classes(self):\n        classes = set()\n        for hotel_id in self.mapping_file['hotel_id'].tolist():\n            classes.add(hotel_id)\n        classes = list(classes)\n        classes.sort()\n        class_to_idx = {classes[i]: i for i in range(len(classes))}\n        return classes, class_to_idx\n    \n    def _make_dataset(self):\n        samples = []\n        num_missing = 0\n        for path in self.paths:\n            img_id = _extract_ids(path)\n            if self.mapping_file is not None:\n                row = self.mapping_file[self.mapping_file['image']==img_id]\n                hotel_id = row['hotel_id'].values[0]\n                if hotel_id in self.class_to_idx:\n                    item = (path, self.class_to_idx[hotel_id])\n                    samples.append(item)\n            else:\n                samples.append(path)\n\n        return samples\n    \n    #if image cannot be loaded, set it to None\n    def _default_loader(self,path):\n        try:\n           with open(path, 'rb') as f:\n               with ImageFile.Image.open(f) as img:\n                   return img.convert('RGB')\n        except:\n           return None\n     \n    def __getitem__(self, index):\n        if self.mapping_file is not None:\n            path, target = self.samples[index]\n        else:\n            path = self.samples[index]\n            target = index\n        image = self._default_loader(path)\n        if image is not None:\n            if self.transform is not None:\n                image = self.transform(image)\n            return image, target, path\n        else:\n            return None\n    \n    def __len__(self):\n        return len(self.samples)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"*For our data augmentation process*, we use Pytorch data tranforms to randomly select images and apply Color Jitter to them , flip them and make crops of size 224x224. For the test set we only take center crops of size 224 to match the training size.\nThe normalization has been done with the Imagenet mean values. We use these mean values since the CNN model we are using is a ResNet-50 trained on Imagenet.\n","metadata":{}},{"cell_type":"code","source":"#Data augmentation for training data \ntrain_transforms =  [transforms.RandomApply([\n                                             transforms.ColorJitter(brightness=0.5, contrast=0.3, saturation=0.4, hue=0.2),\n                                             transforms.RandomRotation(30),\n                                             transforms.RandomHorizontalFlip(p=0.3)\n                                            ],p=0.2),\n                     transforms.RandomResizedCrop(224),\n                     transforms.ToTensor(), \n                     transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                          std=[0.229, 0.224, 0.225]),\n                     ]\n\n#No augmentation for test set, center crop of 224 to match the training data\ntest_transforms =  [transforms.Resize((256,256)),\n                    transforms.CenterCrop(224),\n                    transforms.ToTensor(), \n                    transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                          std=[0.229, 0.224, 0.225])\n                    ]\n\ntrain_folder = LoadDataset(train_images, trainfile_path,transform=transforms.Compose(train_transforms))\n\n#val_query_folder = LoadDataset(val_images, trainfile_path,classes=train_folder.classes,\n#                                              transform=transforms.Compose(test_transforms))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Creating batch sampler with 8 image samples per class**  \nThe Pytorch MPerClassSampler creates a bacth sampling of 8 images per class in each batch.\nThe Pytorch Dataloader takes the training set to divide into batches of size 128 with the sampler. Since the sampler function performs the shuffling of the images, the shuffle for the DataLoader is set to false.  \nFor the training purpose we have put aside 10% of the images for the validation set.\n","metadata":{}},{"cell_type":"code","source":"\nsampler = samplers.MPerClassSampler(train_folder.targets, m=8,batch_size=BATCH_SIZE, length_before_new_iter=len(train_folder))\ntrain_loader = torch.utils.data.DataLoader(train_folder, batch_size=BATCH_SIZE, shuffle=False, sampler=sampler, num_workers=WORKERS, pin_memory=True ,collate_fn=collate_fn)\n#val_query_loader = torch.utils.data.DataLoader(val_query_folder, batch_size=BATCH_SIZE, shuffle=False, num_workers=WORKERS, pin_memory=True,collate_fn=collate_fn)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**#Model for training using pretrained Resnet-50**  \nA pre-trained Resnet-50 model is used for training the baseline model for the competition.The final layer from the Resnet-50 model is replaced by a fully connected layer to output 256 dimensional embeddings.","metadata":{}},{"cell_type":"code","source":"\nclass Model(nn.Module):\n    def __init__(self, output_dim):\n        super().__init__()\n        self.resnet_model = models.resnet50(pretrained=True)  \n        self.num_features = self.resnet_model.fc.in_features\n        self.pool = self.resnet_model.avgpool\n        self.resnet_model = nn.Sequential(*list(self.resnet_model.children())[:-2])\n        self.final_embedding = nn.Linear(self.num_features,output_dim)\n        \n    def forward(self,x):\n        ft = self.resnet_model(x) \n        embedding = torch.squeeze(self.pool(ft))\n        output = self.final_embedding(embedding)\n        return output\n\n# Custom pytorch metric learning accuracy calculation for retreiving k nearest embeddings and knn_labels.\n# The original Pytorch metric learning library does not provide an accuracy measure to calculate the accuracy of the embeddings retrieved \n# by our Image Retrieval method. So here we write a custom class to override the Pytorch Metric learning library \"AccuracyCalculator\" class\n# to return the retrieval accuracies for the 1,10,100 nearest(most similar) embeddings found for the query images in the validation set. \n# The calculate_knn_labels function gives us the class labels for the 100 nearest embeddings.\n\nclass AccCalculator(accuracy_calculator.AccuracyCalculator):\n   \n    def calculate_knn_labels(self, knn_labels, query_labels, **kwargs):\n        return knn_labels\n    \n    def retrieval_at_k(self, k, knn_labels, query_labels):\n        curr_knn_labels = knn_labels[:, :k]\n        accuracy_per_sample = np.apply_along_axis(any, axis=1, arr=(curr_knn_labels == query_labels[:, None]))\n        return accuracy_calculator.maybe_get_avg_of_avgs(accuracy_per_sample, query_labels, self.avg_of_avgs)\n    \n    def calculate_retrieval_at_1(self, knn_labels, query_labels, **kwargs):\n        return self.retrieval_at_k(1, knn_labels, query_labels)\n    \n    def calculate_retrieval_at_10(self, knn_labels, query_labels, **kwargs):\n        return self.retrieval_at_k(10, knn_labels, query_labels)\n    \n    def calculate_retrieval_at_100(self, knn_labels, query_labels, **kwargs):\n        return self.retrieval_at_k(100, knn_labels, query_labels)\n\n    def requires_knn(self):\n        return super().requires_knn() + [\"knn_labels\"]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pytorch-metric-learning batch-all functions.  \nThis is the part which uses Deep Metric Learning methods to learn the relative distance between image pairs. For this, the model uses a mining function -The Triplet Miner, which will find the triplets i.e the 3 images -  \n1.an sample anchor image (an image in the batch)  \n2.a positive image which belongs to the same class and has features similar to the anchor.  \n3. a negative image from a different class which is most dissimilar to the anchor.  \n\nUsing these three images,the Triplet Loss function then optimizes the model to maximize the distance between dissimilar images and minimize the distance between similar images from the same class. The margin value is used to define the difference in the distance between the positive sample and the  negative sample with respect to the anchor.  \nThe \"all\" in the function parameters indicates that the functions uses all the triplet pairs mined. ","metadata":{}},{"cell_type":"code","source":"\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n#Define our model for training\nmodel = Model(OUTPUT_SIZE)\nmodel = model.to(device)\n\noptimizer = optim.Adam(model.parameters(), lr=lr)\n\n\ndistance = distances.CosineSimilarity()\nloss = losses.TripletMarginLoss(margin=0.2,distance=distance,triplets_per_anchor=\"all\")\nminer = miners.TripletMarginMiner(margin=0.2,distance=distance, type_of_triplets=\"all\")\nacc_calculator = AccCalculator(include=(\"knn_labels\",),\n                             k=15)","metadata":{"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(model,loss,miner,device,train_loader,optimizer,epoch):\n    model.train()                                                      #set the model to training mode\n    for batch_idx , (data,labels, _) in enumerate(train_loader):\n        try:\n            data , labels = data.to(device) , labels.to(device)\n            optimizer.zero_grad()                                     # we set the gradient to zero before doing backprop since the training process accummulates the gradients.\n            embeddings = model(data)\n            miner_pairs = miner(embeddings, labels)                    #the miner uses the feature embeddings returned by the resnet model to find the triplets.\n            loss_value = loss(embeddings,labels,miner_pairs)           #these triplets are then given to the Triplet loss function which the model will use to optimize \n            loss_value.backward()                                      #perform backprop\n            optimizer.step()\n            if batch_idx %100 ==0:\n               print(\"Epoch {} Iteration {}: Loss = {}, Number of triplets = {}\".format(epoch, batch_idx, loss_value.item(), miner.num_triplets))\n        except Exception as e:\n            print(e)\n            print(\"batch_idx=\",batch_idx,\"data=\",data)\n\n    return outputs\n\n# The embed function is for evaluating and retrieving the embeddings for the validation/test set.\n# These embeddings are then used to find the most similar embeddings in the training set to determine the class label.\ndef embed(data_loader, model):\n    outputs = {\n        'embeddings': torch.Tensor([]),\n        'labels': torch.Tensor([]),\n    }\n    \n    model.eval()                      #set the trained model to evaluation mode.\n    with torch.no_grad():\n        for i, (inputs, labels, _) in enumerate(data_loader):\n            inputs , labels = inputs.to(device) , labels.to(device)\n            embeddings = model(inputs)\n            outputs['embeddings'] = torch.cat((outputs['embeddings'], embeddings.detach().cpu()))\n            outputs['labels'] = torch.cat((outputs['labels'], labels.detach().cpu()))\n            if i%20 ==0:\n                print(\"Completed: \",i)\n    outputs = {key: value.numpy() for key, value in outputs.items()}\n    return outputs\n\n#Calculate the retrieval accuracies of the validation embeddings using the training embeddings.\ndef get_accuracies(acc_calculator, ref_embeddings, query_embeddings, ref_labels, query_labels,\n                   embeddings_come_from_same_source=False):\n    #normalizing the embeddings\n    faiss.normalize_L2(ref_embeddings)\n    faiss.normalize_L2(query_embeddings)\n    \n    accuracies = acc_calculator.get_accuracy(query_embeddings,\n                                         ref_embeddings,\n                                         query_labels,\n                                         ref_labels,\n                                         embeddings_come_from_same_source)\n\n    return accuracies['knn_labels']\n\n\ndef test(train_loader, query_loader,model,device,outputs,acc_calculator):\n    if training:\n        outputs['train']= embed(train_loader,model)\n        outputs['validation'] = embed(query_loader,model)\n   \n    return get_accuracies(\n                acc_calculator,\n                ref_embeddings=outputs['train']['embeddings'],\n                query_embeddings=outputs['validation']['embeddings'],\n                ref_labels=outputs['train']['labels'],\n                query_labels=outputs['validation']['labels']\n            )\n\n# Function which calculates the map@k which is similar to the metrics used for this competition.\n# The y_true is the labels in the validation and the y_pred are the labels returned by the  \"get_accuracies\" function above.\n# Used against the validation set only.\ndef map_at_k(y_true, y_pred, k):\n    \"\"\"\n    y_true: list of ground truths\n    y_pred: list of y_predictions\n    k: value to set\n    \"\"\"\n    avg_precision = []\n    count = 0\n    idx = 0\n    for true, pred in zip(y_true, y_pred):\n        assert len(pred) >= k, f\"Length of each prediction must be equal or greater than {k}!\"\n        idx += 1\n        for i in range(k):\n            if int(true) == int(pred[i]):\n                count += 1\n                avg_precision.append(1 / (i + 1))\n                break\n        else:\n            avg_precision.append(0)\n\n    return (sum(avg_precision) / len(avg_precision))\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#For submission to the competition we have used a model trained on our local machine for about 40 epochs\n#load the model trained on local machine to evaluate\nsaved_model = Model(OUTPUT_SIZE)\nsaved_model.load_state_dict(torch.load(model_path,device))\nsaved_model = saved_model.to(device)\nsaved_model.eval()","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#loading the test images\ntest_image_set = load_images('/kaggle/input/hotel-id-2021-fgvc8/test_images')\ntest_query_folder = LoadDataset(test_image_set,None ,classes=train_folder.classes,\n                                              transform=transforms.Compose(test_transforms))\ntest_query_loader = torch.utils.data.DataLoader(test_query_folder, batch_size=BATCH_SIZE, shuffle=False, num_workers=WORKERS, pin_memory=True)\n\noutputs = {\n        'train': {},\n        'test': {}\n        }","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#evaluate the saved model to get the test embeddings \noutputs['train']= torch.load('../input/embeddings/embeddings.csv')\noutputs['test'] = embed(test_query_loader,saved_model)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Retrieve the class labels for the k nearest embeddings\ntest_knn_labels = get_accuracies(\n                acc_calculator,\n                ref_embeddings=outputs['train']['embeddings'],\n                query_embeddings=outputs['test']['embeddings'],\n                ref_labels=outputs['train']['labels'],\n                query_labels=outputs['test']['labels']\n            )","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#function to search for k=10 similar images for the test set.This can be done with either the faiss library or the CosineSimilarity function in Pytorch.\ndef getOutput(query_data,knn_labels,filename):\n    hotel_id_to_class =  {v:k for k, v in query_data.class_to_idx.items()}\n   \n    data = []\n    for img ,labels in zip(query_data.paths,knn_labels):\n        img_name = img.split(os.sep)[-1]\n        #find the unique classes predicted\n        pred_classes = []\n        for i in labels:\n            if i not in pred_classes:\n                pred_classes.append(i)\n        \n        labels = ' '.join(['%s' % hotel_id_to_class[i] for i in pred_classes[:5]])\n        data.append([img_name,labels])\n    df = pd.DataFrame(data,columns=['image','hotel_id'])\n    df.to_csv(filename,index=False)\n\n\ngetOutput(test_query_folder,test_knn_labels.numpy(),output_file)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}