{"cells":[{"metadata":{"_uuid":"224c263766392273f0de6d4fedc07c35b65b2017"},"cell_type":"markdown","source":"## Loading requited libraries"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"%matplotlib inline\n\nimport os\n\nimport torch\nimport torch.nn.functional as F\nimport torch.nn as nn\n\nfrom torch import optim\nfrom torch.autograd import Variable\n\nimport torchvision\nimport torchvision.datasets as dset\nimport torchvision.models as models\nimport torchvision.transforms as transforms\nimport torchvision.utils\nfrom torch.utils.data import DataLoader,Dataset\n\nimport matplotlib.pyplot as plt\n\nimport pandas as pd\nimport numpy as np\n\nimport random\n\nfrom PIL import Image\nimport PIL.ImageOps    ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a521b6d9f592d546830150e292f29918ee019868"},"cell_type":"markdown","source":"## Class to generate embedding vector using restnet18 "},{"metadata":{"trusted":true,"_uuid":"98453b3791320f2ae125664a6ccc0c3ac8327e43"},"cell_type":"code","source":"class Img2Vec():\n\n    def __init__(self, cuda=False, model='resnet-18', layer='default', layer_output_size=512):\n        \"\"\" Img2Vec\n        :param cuda: If set to True, will run forward pass on GPU\n        :param model: String name of requested model\n        :param layer: String or Int depending on model.  See more docs: https://github.com/christiansafka/img2vec.git\n        :param layer_output_size: Int depicting the output size of the requested layer\n        \"\"\"\n        self.device = torch.device(\"cuda\" if cuda else \"cpu\")\n        self.layer_output_size = layer_output_size\n        self.model_name = model\n        \n        self.model, self.extraction_layer = self._get_model_and_layer(model, layer)\n\n        self.model = self.model.to(self.device)\n\n        self.model.eval()\n\n        self.scaler = transforms.Resize((224, 224))\n        self.normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                              std=[0.229, 0.224, 0.225])\n        self.to_tensor = transforms.ToTensor()\n\n    def get_vec(self, file_name, tensor=False,bbox = [0,224,0,224]):\n        \"\"\" Get vector embedding from PIL image\n        :param img: PIL Image\n        :param tensor: If True, get_vec will return a FloatTensor instead of Numpy array\n        :returns: Numpy ndarrayim\n        \"\"\"\n        image = Image.open(os.path.join(file_name)).crop(bbox).convert('RGB')\n        image = self.normalize(self.to_tensor(self.scaler(image))).unsqueeze(0).to(self.device)\n\n        if self.model_name == 'alexnet':\n            my_embedding = torch.zeros(1, self.layer_output_size)\n        else:\n            my_embedding = torch.zeros(1, self.layer_output_size, 1, 1)\n\n        def copy_data(m, i, o):\n            my_embedding.copy_(o.data)\n\n        h = self.extraction_layer.register_forward_hook(copy_data)\n        h_x = self.model(image)\n        h.remove()\n\n        if tensor:\n            return my_embedding\n        else:\n            if self.model_name == 'alexnet':\n                return my_embedding.numpy()[0, :]\n            else:\n                return my_embedding.numpy()[0, :, 0, 0]\n\n    def _get_model_and_layer(self, model_name, layer):\n        \"\"\" Internal method for getting layer from model\n        :param model_name: model name such as 'resnet-18'\n        :param layer: layer as a string for resnet-18 or int for alexnet\n        :returns: pytorch model, selected layer\n        \"\"\"\n        if model_name == 'resnet-18':\n            model = models.resnet18(pretrained=True)\n            if layer == 'default':\n                layer = model._modules.get('avgpool')\n                self.layer_output_size = 512\n            else:\n                layer = model._modules.get(layer)\n\n            return model, layer\n\n        elif model_name == 'alexnet':\n            model = models.alexnet(pretrained=True)\n            if layer == 'default':\n                layer = model.classifier[-2]\n                self.layer_output_size = 4096\n            else:\n                layer = model.classifier[-layer]\n\n            return model, layer\n\n        else:\n            raise KeyError('Model %s was not found' % model_name)\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5fd07b52e1c67ff361173fb8d046623424b315e0"},"cell_type":"markdown","source":"## Functions to calculate the competition evaluation metrics"},{"metadata":{"trusted":true,"_uuid":"7a8a6b6f32bb44f6f91617b1bc7f2b642e21fc8b"},"cell_type":"code","source":"def apk(actual, predicted, k=10):\n    \"\"\"\n    Computes the average precision at k.\n    This function computes the average prescision at k between two lists of\n    items.\n    Parameters\n    ----------\n    actual : True identity\n    predicted : list\n                A list of lists of predicted elements\n                (order matters in the lists)\n    k : int, optional\n        The maximum number of predicted elements\n    Returns\n    -------\n    score : double\n            The average precision at k over the input lists\n    \"\"\"\n    if len(predicted)>k:\n        predicted = predicted[:k]\n\n    score = 0.0\n    num_hits = 0.0\n    for i,p in enumerate(predicted):\n        score = 0.0\n        if p == actual :\n            score = 1.0/(i+1.0)\n            break\n    return score\n\ndef mapk(actual, predicted, k=10):\n    \"\"\"\n    Computes the mean average precision at k.\n    This function computes the mean average prescision at k between two lists\n    of lists of items.\n    Parameters\n    ----------\n    actual : True identity\n    predicted : list\n                A list of lists of predicted elements\n                (order matters in the lists)\n    k : int, optional\n        The maximum number of predicted elements\n    Returns\n    -------\n    score : double\n            The mean average precision at k over the input lists\n    \"\"\"\n    return np.mean([apk(a,p,k) for a,p in zip(actual, predicted)])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d8cd0bef1e7275c82be7f9bc9314225652299fc3"},"cell_type":"markdown","source":"## Image visualizing functions"},{"metadata":{"trusted":true,"_uuid":"d054ee94aa1341bc7c1556a894f8b5b09a44330d"},"cell_type":"code","source":"def imshow(img,text=None,should_save=False):\n    fig = plt.figure(figsize=(20, 80))\n    if text:\n        plt.text(75, 8, text, style='italic',fontweight='bold',\n            bbox={'facecolor':'white', 'alpha':0.8, 'pad':10})\n    plt.imshow(img.permute(1, 2, 0)) \n\ndef show_plot(iteration,loss):\n    plt.plot(iteration,loss)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7f24d2a60701e4aec5de76a42f5930088ed17aff"},"cell_type":"markdown","source":"## Contractive Loss class for siamese network"},{"metadata":{"trusted":true,"_uuid":"a58050712f4b3af7fc18d545be3dbfc6fa4230f5"},"cell_type":"code","source":"class ContrastiveLoss(torch.nn.Module):\n    \"\"\"\n    Contrastive loss function.\n    Based on: http://yann.lecun.com/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf\n    \"\"\"\n\n    def __init__(self, margin=2.0):\n        super(ContrastiveLoss, self).__init__()\n        self.margin = margin\n\n    def forward(self, output1, output2, label):\n        euclidean_distance = F.pairwise_distance(output1, output2)\n        loss_contrastive = torch.mean((1-label) * torch.pow(euclidean_distance, 2) +\n                                      (label) * torch.pow(torch.clamp(self.margin - euclidean_distance, min=0.0), 2))\n\n\n        return loss_contrastive","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a761257e09a43cfee0482dd17dc3dfa8cd8ab4b3"},"cell_type":"markdown","source":"## Siamese data set class "},{"metadata":{"trusted":true,"_uuid":"32da580aacb044855a6a3ab9633989108fb4395e"},"cell_type":"code","source":"class SiameseNetworkEmbeddingDataset(Dataset):\n    \n    def __init__(self,df,img2vec=None,genEmbed = False):\n        self.img2vec = img2vec\n        self.df = df\n        self.genEmbed = genEmbed\n        \n    def __getitem__(self,idx):\n        # not selecting 'new_whale' for anchor image.\n        img0_idx = random.choice(self.df[self.df.Id != 'new_whale'].index.values)\n        \n        # we need to make sure approx 50% of images are in the same class\n        should_get_same_class = random.randint(0,1)\n        if should_get_same_class:\n            img1_idx = random.choice(self.df[self.df.Id == self.df.Id[img0_idx]].index.values) \n        else:\n            img1_idx = random.choice(self.df[self.df.Id != self.df.Id[img0_idx]].index.values)\n            \n        img0_embedding = self.df.loc[img0_idx,'embedding']\n        img1_embedding = self.df.loc[img1_idx,'embedding']\n        \n        if self.genEmbed :\n            print(\"a\")\n            img0_embedding = img2vec.get_vec(self.df.loc[img0_idx,'Image'], tensor=False) \n            img1_embedding = img2vec.get_vec(self.df.loc[img1_idx,'Image'], tensor=False)\n        \n        return img0_embedding, img1_embedding , torch.from_numpy(np.array([int(self.df.Id[img0_idx] != self.df.Id[img1_idx])],dtype=np.float32))\n    \n    def __len__(self):\n        return(self.df.shape[0])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"40b1e7605ee490e7523f971772cfc020bfb671b0"},"cell_type":"markdown","source":"## Siamese nural network"},{"metadata":{"trusted":true,"_uuid":"3751f5fc9635af1e5bca76f57b1c1de45f9d5ebb"},"cell_type":"code","source":"class SiameseNetworkEmbedding(nn.Module):\n    def __init__(self):\n        super(SiameseNetworkEmbedding, self).__init__()\n\n        self.fc1 = nn.Sequential(\n            nn.Linear(512, 512),\n            nn.ReLU(inplace=True),\n\n            nn.Linear(512, 5))\n\n    def forward_once(self, x):\n        output = self.fc1(x)\n        return output\n\n    def forward(self, input1, input2):\n        output1 = self.forward_once(input1)\n        output2 = self.forward_once(input2)\n        return output1, output2","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a1f58dde266170bc3b3c18199b153719b117a548"},"cell_type":"markdown","source":"## Function to create embedding using pretrained model\nThis is used to reduce the time to generate embedding on the fly. But random transformation is not able to apply with this methodology"},{"metadata":{"trusted":true,"_uuid":"680e31f0cae5b2b6304c5ae8eb9a9adfaad132db"},"cell_type":"code","source":"def createEmbeddingFiles() :\n    img2vec = Img2Vec()\n    train_full = pd.read_csv(\"../input/humpback-whale-identification/train.csv\")\n    test_df = pd.read_csv(\"../input/humpback-whale-identification/sample_submission.csv\")\n    train_full = pd.read_csv(\"../input/humpback-whale-identification/train.csv\")\n\n    id_counts = train_full.Id.value_counts()\n\n    valid_df = train_full.loc[train_full.Id.isin(id_counts[id_counts>5].index.values),:].sample(frac=0.3)\n\n    train_df = train_full.loc[~train_full.index.isin(valid_df.index.values),:]\n\n    test_df = pd.read_csv(\"../input/humpback-whale-identification/sample_submission.csv\")\n\n    bbox_df = pd.read_csv(\"../input/metadata/bounding_boxes.csv\")\n\n    def getEmbedding(file_path,x):\n        file_name = os.path.join(file_path,x)\n        bbox = bbox_df.loc[bbox_df.Image==x,:].values[0,1:]\n        return(img2vec.get_vec(file_name,tensor=False,bbox=bbox).squeeze())\n\n    train_df_embed = train_df.assign(embedding = train_df['Image'].apply(lambda x : getEmbedding('../input/humpback-whale-identification/train/',x)))\n    valid_df_embed = valid_df.assign(embedding = valid_df['Image'].apply(lambda x : getEmbedding('../input/humpback-whale-identification/train/',x)))\n    test_df_embed = test_df.assign(embedding = test_df['Image'].apply(lambda x : getEmbedding('../input/humpback-whale-identification/test/',x)))\n\n    pickle.dump(train_df_embed,open( \"train_df_embed.p\",'wb'))\n    pickle.dump(valid_df_embed,open( \"valid_df_embed.p\",'wb'))\n    pickle.dump(test_df_embed,open( \"test_df_embed.p\",'wb'))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"add0b4aa0587dfee555d2aa58f0163b7e7dfee9f"},"cell_type":"markdown","source":"## Importing already created embedding files\nI have already created the embedding vectors and added to the dataset"},{"metadata":{"trusted":true,"_uuid":"ad4ce5db75e331690ad499909d00e0331d1611af"},"cell_type":"code","source":"import pickle\ntrain_df = pickle.load( open( \"../input/humpback-whale-identification-embedding/train_df_embed.p\",'rb'))\nvalid_df = pickle.load( open( \"../input/humpback-whale-identification-embedding/valid_df_embed.p\",'rb'))\ntest_df = pickle.load( open( \"../input/humpback-whale-identification-embedding/test_df_embed.p\",'rb'))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"776ca916bb495f59a6f4646d6a82abf74018449d"},"cell_type":"markdown","source":"## Setting up the training the siamese network"},{"metadata":{"trusted":true,"_uuid":"c59d644473d4eb17baa714edad01932438a56182"},"cell_type":"code","source":"train_dataset = SiameseNetworkEmbeddingDataset(train_df)\n\ntrain_dataloader = DataLoader(train_dataset,\n                        shuffle=True,\n                        num_workers=0,\n                        batch_size=64)\n\nnet = SiameseNetworkEmbedding().cuda()\ncriterion = ContrastiveLoss()\noptimizer = optim.Adam(net.parameters(),lr = 0.0005 )\n\ncounter = []\nloss_history = [] \niteration_number= 0","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c065eedecb7d6c9dd952b77e796dffb7c22f5f8b"},"cell_type":"markdown","source":"## Training the siamese network"},{"metadata":{"trusted":true,"_uuid":"616edb6c7e750800b81a60451bb34e5b7a7af456"},"cell_type":"code","source":"net.train()\nfor epoch in range(0,50):\n    for i, data in enumerate(train_dataloader,0):\n        img0, img1 , label = data\n        img0, img1 , label = img0.cuda(), img1.cuda() , label.cuda()\n        optimizer.zero_grad()\n        output1,output2 = net(img0,img1)\n        loss_contrastive = criterion(output1,output2,label)\n        loss_contrastive.backward()\n        optimizer.step()\n        if i %100 == 0 :\n            print(\"Epoch number {} \\t Iteration number {} \\t Current loss {}\\n\".format(epoch,iteration_number,loss_contrastive.item()))\n            iteration_number +=10\n            counter.append(iteration_number)\n            loss_history.append(loss_contrastive.item())\nshow_plot(counter,loss_history)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"48acaddcc5e0214f7f4d583d1eb2817799715659"},"cell_type":"markdown","source":"## Evaluation"},{"metadata":{"trusted":true,"_uuid":"ffaffca574644f641d656b65c1ccc657b7fb3251"},"cell_type":"code","source":"net.eval()\n\ntrain_df = train_df.assign(pred=train_df.loc[:,'embedding'].apply(lambda x : (net.forward_once(torch.from_numpy(x).squeeze().cuda()).detach().cpu().numpy())))\nvalid_df = valid_df.assign(pred = valid_df.loc[:,'embedding'].apply(lambda x : (net.forward_once(torch.from_numpy(x).squeeze().cuda()).detach().cpu().numpy())))\ntest_df = test_df.assign(pred = test_df.loc[:,'embedding'].apply(lambda x : (net.forward_once(torch.from_numpy(x).squeeze().cuda()).detach().cpu().numpy())))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cad662fc0fefe8510bbd0a27d504a054a80f1808"},"cell_type":"markdown","source":"## Calculating the distance matrix"},{"metadata":{"trusted":true,"_uuid":"d4df1f23350a745e4c6b6f5b125859ddaa6eafa1"},"cell_type":"code","source":"from sklearn.metrics.pairwise import euclidean_distances\ndistance_mat_valid = pd.DataFrame(euclidean_distances(np.stack(train_df.pred.values), np.stack(valid_df.pred.values)),columns = valid_df.Image.values,index=train_df.Image.values)\ndistance_mat_test = pd.DataFrame(euclidean_distances(np.stack(train_df.pred.values), np.stack(test_df.pred.values)),columns = test_df.Image.values,index=train_df.Image.values)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8a0e0dac93287066aacfda84ad89b8cce47ac16b"},"cell_type":"markdown","source":"## Getting the top five Ids"},{"metadata":{"trusted":true,"_uuid":"e45f6ee721968f0b2cb470ffbd7133694542bdb5"},"cell_type":"code","source":"Id_Df = train_df[['Image','Id']].set_index('Image')\n\ndef getTopFiveIdValid(x):\n    sortedIds = Id_Df.loc[distance_mat_valid.loc[:,x].sort_values().index.values,'Id'].values\n    topFiveIds = sortedIds[np.sort(np.unique(sortedIds, return_index=True)[1])[:5]]\n    return(topFiveIds)\n\ndef getTopFiveIdTest(x):\n    sortedIds = Id_Df.loc[distance_mat_test.loc[:,x].sort_values().index.values,'Id'].values\n    topFiveIds = ' '.join(sortedIds[np.sort(np.unique(sortedIds, return_index=True)[1])[:5]])\n    return(topFiveIds)\n\ntest_df = test_df.assign(Id = test_df.loc[:,'Image'].apply(getTopFiveIdTest))\nvalid_df = valid_df.assign(topFiveIds =  valid_df.loc[:,'Image'].apply(getTopFiveIdValid))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"659efe51e2e0db1645b360220e3e90f41665c7e0"},"cell_type":"markdown","source":"## Evaluation metric"},{"metadata":{"trusted":true,"_uuid":"ab45d84653ffbed4d7f9aaafb95ddd0f103e950e"},"cell_type":"code","source":"mapk(valid_df.Id, valid_df.topFiveIds, k=5)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b0d2db26814b84d0d5d2b47399ddafadbfd31603"},"cell_type":"markdown","source":"## Creating the submission file"},{"metadata":{"trusted":true,"_uuid":"b13faebde3408f8d2f98ff0112e02551a1145372"},"cell_type":"code","source":"test_df[['Image','Id']].to_csv('submission_3.csv',index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}