{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Public LB: \n- 0.114 (12621.2s - GPU)(epoch=1(/folds),ntree=10)(February 2, 2022)  \n- 0.121 (35601.9s - GPU)(epoch=5(/folds),ntree=100)(February 3, 2022)\n\nI'm using **EfficientNet_b4** & **ContrastiveLoss** & **CrossBatchMemory(2000)** & **αQE(α=1,K=1)**.  \n\nIn this model, no distinction is made between different types of animals. \n\n**Increasing the value of \"epoch\" or increasing the value of \"ntree\" in Annoy will improve the performance.**  \n\nI'm a beginner, so I'm sharing my notebook for study.  \nThe code is messed up, but if you run it from above, you will get submisson.csv.  \nIf there are any mistakes, I would appreciate it if you could point them out.  \n\nI referred to [this notebook](https://www.kaggle.com/moeinshariatnia/contrastive-loss-pretraining-in-depth-explanation) .    \nI recommend this notebook for its detailed description of metric learning.","metadata":{}},{"cell_type":"markdown","source":"# Settings","metadata":{}},{"cell_type":"code","source":"mkdir /kaggle/working/weights","metadata":{"execution":{"iopub.status.busy":"2022-02-02T10:56:44.46255Z","iopub.execute_input":"2022-02-02T10:56:44.463162Z","iopub.status.idle":"2022-02-02T10:56:45.148361Z","shell.execute_reply.started":"2022-02-02T10:56:44.463067Z","shell.execute_reply":"2022-02-02T10:56:45.147447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    SEED = 0\n    size = 224\n    batch_size = 16\n    num_workers = 2\n    efficientnet_feature = 1792\n    model_name = \"efficientnet_b4\"\n    pretrained = True\n    \n    dropout = None\n    linear = None\n    margin = 1\n    scheduler = \"ReduceLROnPlateau\"\n    step = \"epoch\"\n\n    learning_rate = 1e-3\n    factor =0.5\n    patience = 2\n    epochs = 5\n","metadata":{"execution":{"iopub.status.busy":"2022-02-02T10:58:07.738519Z","iopub.execute_input":"2022-02-02T10:58:07.739153Z","iopub.status.idle":"2022-02-02T10:58:07.744642Z","shell.execute_reply.started":"2022-02-02T10:58:07.73911Z","shell.execute_reply":"2022-02-02T10:58:07.743728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport random\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\n\n\nimport albumentations\n!pip install timm\nimport timm\n\nfrom tqdm.autonotebook import tqdm\n\n!pip install pytorch-metric-learning\nfrom pytorch_metric_learning import losses\n\nimport gc\nfrom albumentations.pytorch.transforms import ToTensorV2\nfrom torchvision import transforms\nfrom torch.utils.data import Dataset\nfrom sklearn.model_selection import KFold\nimport cv2\n\nfrom annoy import AnnoyIndex","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fix_seed(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    #gpu randomseed fixed\n    torch.backends.cudnn.deterministic = True\n   \nfix_seed(CFG.SEED)","metadata":{"execution":{"iopub.status.busy":"2022-02-02T10:58:23.542317Z","iopub.execute_input":"2022-02-02T10:58:23.54265Z","iopub.status.idle":"2022-02-02T10:58:23.551335Z","shell.execute_reply.started":"2022-02-02T10:58:23.542606Z","shell.execute_reply":"2022-02-02T10:58:23.550576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nprint(device)","metadata":{"execution":{"iopub.status.busy":"2022-02-02T10:58:23.55377Z","iopub.execute_input":"2022-02-02T10:58:23.55448Z","iopub.status.idle":"2022-02-02T10:58:23.559728Z","shell.execute_reply.started":"2022-02-02T10:58:23.554452Z","shell.execute_reply":"2022-02-02T10:58:23.559031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# data","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/happy-whale-and-dolphin/train.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-02-02T10:58:23.56209Z","iopub.execute_input":"2022-02-02T10:58:23.56259Z","iopub.status.idle":"2022-02-02T10:58:23.624271Z","shell.execute_reply.started":"2022-02-02T10:58:23.562547Z","shell.execute_reply":"2022-02-02T10:58:23.623555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-02T10:58:23.625676Z","iopub.execute_input":"2022-02-02T10:58:23.625933Z","iopub.status.idle":"2022-02-02T10:58:23.635642Z","shell.execute_reply.started":"2022-02-02T10:58:23.625898Z","shell.execute_reply":"2022-02-02T10:58:23.63496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[\"path\"] = \"/kaggle/input/happy-whale-and-dolphin/train_images/\" + df[\"image\"].astype(str)\ndf[\"index\"] = df.index","metadata":{"execution":{"iopub.status.busy":"2022-02-02T10:58:23.637083Z","iopub.execute_input":"2022-02-02T10:58:23.637616Z","iopub.status.idle":"2022-02-02T10:58:23.655136Z","shell.execute_reply.started":"2022-02-02T10:58:23.637579Z","shell.execute_reply":"2022-02-02T10:58:23.654471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-02T10:58:23.656442Z","iopub.execute_input":"2022-02-02T10:58:23.6567Z","iopub.status.idle":"2022-02-02T10:58:23.666872Z","shell.execute_reply.started":"2022-02-02T10:58:23.656652Z","shell.execute_reply":"2022-02-02T10:58:23.666073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_id_idx = pd.DataFrame({\"individual_id\":df[\"individual_id\"].unique(),\n              \"idx\":range(len(df[\"individual_id\"].unique()))})","metadata":{"execution":{"iopub.status.busy":"2022-02-02T10:58:23.691981Z","iopub.execute_input":"2022-02-02T10:58:23.692524Z","iopub.status.idle":"2022-02-02T10:58:23.710534Z","shell.execute_reply.started":"2022-02-02T10:58:23.692467Z","shell.execute_reply":"2022-02-02T10:58:23.709912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Augmentation","metadata":{}},{"cell_type":"code","source":"train_transforms = albumentations.Compose([\n            albumentations.Resize(CFG.size,CFG.size),\n            albumentations.HorizontalFlip(p=0.5),\n            albumentations.Rotate(limit=10, p=0.8),\n            albumentations.RandomBrightness(limit=(0.09,0.6),p=0.5),\n            albumentations.Normalize(),\n            ToTensorV2(p=1.0),\n            ])\n    \ndef train_albumentations_transform(image, transform=train_transforms):\n    if transform:\n        image_np = np.array(image)\n        augmented = transform(image=image_np)\n    return augmented\n\ntrain_data_transform = transforms.Compose([\n    transforms.Lambda(train_albumentations_transform),\n])","metadata":{"execution":{"iopub.status.busy":"2022-02-02T10:58:23.668538Z","iopub.execute_input":"2022-02-02T10:58:23.66883Z","iopub.status.idle":"2022-02-02T10:58:23.680673Z","shell.execute_reply.started":"2022-02-02T10:58:23.668793Z","shell.execute_reply":"2022-02-02T10:58:23.679961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_transforms = albumentations.Compose([\n            albumentations.Resize(CFG.size,CFG.size, always_apply=True),\n            albumentations.Normalize(),\n            ToTensorV2(p=1.0),\n            ])\n    \ndef valid_albumentations_transform(image, transform=valid_transforms):\n    if transform:\n        image_np = np.array(image)\n        augmented = transform(image=image_np)\n    return augmented\n\nvalid_data_transform = transforms.Compose([\n    transforms.Lambda(valid_albumentations_transform),\n])\n","metadata":{"execution":{"iopub.status.busy":"2022-02-02T10:58:23.681838Z","iopub.execute_input":"2022-02-02T10:58:23.682631Z","iopub.status.idle":"2022-02-02T10:58:23.688786Z","shell.execute_reply.started":"2022-02-02T10:58:23.682599Z","shell.execute_reply":"2022-02-02T10:58:23.687875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Functions","metadata":{}},{"cell_type":"code","source":"class ContrasiveDataset(Dataset):\n    def __init__(self, df, df_id_idx, transforms):\n        self.transforms=transforms\n        self.individual_id = list(df[\"individual_id\"].unique())\n        self.individual_id_to_imgs = {individual_id: df[df[\"individual_id\"] == individual_id].path.values for individual_id in self.individual_id}\n        self.df = df\n        self.df_id_idx = df_id_idx\n\n    def __getitem__(self, idx):\n        id = self.individual_id[idx]\n\n        if random.random()  > 0.5:\n            same = True\n            same_id_images = self.individual_id_to_imgs[id]\n            if len(same_id_images)==1:\n                img1 = same_id_images[0]\n                img2 = same_id_images[0]\n            else:\n                img1, img2 = np.random.choice(same_id_images, size=2, replace=False if len(same_id_images) > 1 else True)\n        else:\n            same = False\n            img1 = np.random.choice(self.individual_id_to_imgs[id], size=1)[0]\n\n            while True:\n                different_label = np.random.choice(self.individual_id, size=1)[0]\n\n                if different_label != id:\n                    break\n            \n            img2 = np.random.choice(self.individual_id_to_imgs[different_label], size=1)[0]\n\n        img1_tensor, img2_tensor= self.process_imgs(img1, img2)\n\n        return {\"images1\":img1_tensor,\n                \"images2\":img2_tensor,\n                \"same\":torch.tensor(same).float(),\n                \"label1\":df_id_idx[df_id_idx[\"individual_id\"]==id][\"idx\"].values[0],\n                \"label2\":df_id_idx[df_id_idx[\"individual_id\"]==id][\"idx\"].values[0] if same else df_id_idx[df_id_idx[\"individual_id\"]==different_label][\"idx\"].values[0],\n                \"image1_name\":img1,\n                \"image2_name\":img2}\n\n\n    def read_transform_one(self, img):\n        img = cv2.imread(img)[..., :: -1]\n\n        if self.transforms is not None:\n            img = self.transforms(img)[\"image\"]\n        return img\n\n    def process_imgs(self, img1, img2):\n        img1 = self.read_transform_one(img1)\n        img2 = self.read_transform_one(img2)\n        return img1, img2\n    \n    def __len__(self):\n        return len(self.individual_id)\n","metadata":{"execution":{"iopub.status.busy":"2022-02-02T10:58:23.711702Z","iopub.execute_input":"2022-02-02T10:58:23.711944Z","iopub.status.idle":"2022-02-02T10:58:23.727624Z","shell.execute_reply.started":"2022-02-02T10:58:23.711911Z","shell.execute_reply":"2022-02-02T10:58:23.72676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Model(nn.Module):\n    def __init__(self,\n                 model_name=CFG.model_name,\n                 pretrained=True,\n                 dropout=0.2,\n                 linear=128):\n\n                 super().__init__()\n                 model = timm.create_model(model_name,\n                                           pretrained=pretrained,\n                                           num_classes=0)\n                 self.num_features=model.num_features\n                 self.linear = None\n                 if linear is not None and linear >0:\n                     self.linear = nn.Linear(self.num_features, linear)\n                 self.backbone = nn.Sequential(model,\n                                              self.linear if self.linear is not None else nn.Identity(),\n                                              nn.ReLU() if self.linear is not None else nn.Identity(),\n                                              nn.Dropout(0.2) if dropout is not None else nn.Identity())\n\n    def forward(self, batch):\n        images_1 = self.backbone(batch[\"images1\"].to(device))\n        images_2 = self.backbone(batch[\"images2\"].to(device))\n        \n        return images_1, images_2","metadata":{"execution":{"iopub.status.busy":"2022-02-02T10:58:23.729163Z","iopub.execute_input":"2022-02-02T10:58:23.729489Z","iopub.status.idle":"2022-02-02T10:58:23.741857Z","shell.execute_reply.started":"2022-02-02T10:58:23.729453Z","shell.execute_reply":"2022-02-02T10:58:23.741137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AvgMeter:\n    def __init__(self, name=\"Metric\"):\n        self.name = name\n        self.reset()\n\n    def reset(self):\n        self.avg, self.sum, self.count = [0]*3\n    \n    def update(self, val, count=1):\n        self.count += count\n        self.sum += val * count\n        self.avg = self.sum / self.count\n    \n    def __repr__(self):\n        text = \"{}:{:.4f}\".format(self.name,self.avg)\n        return text\n    \ndef get_lr(optimizer):\n    for param_group in optimizer.param_groups:\n        return param_group[\"lr\"]","metadata":{"execution":{"iopub.status.busy":"2022-02-02T10:58:23.743541Z","iopub.execute_input":"2022-02-02T10:58:23.743909Z","iopub.status.idle":"2022-02-02T10:58:23.751346Z","shell.execute_reply.started":"2022-02-02T10:58:23.743855Z","shell.execute_reply":"2022-02-02T10:58:23.750401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def one_epoch(model,\n              criterion,\n              loader,\n              optimizer=None,\n              lr_scheduler=None,\n              mode=\"train\",\n              step=\"batch\"):\n    loss_meter = AvgMeter()\n    distances = None\n    labels = None\n    tqdm_object = tqdm(enumerate(loader), total=len(loader))\n    for i, batch in tqdm_object:\n        images1_f, images2_f = model(batch)\n        \n        embeddings = torch.cat((images1_f, images2_f))\n        labels = torch.cat((batch[\"label1\"], batch[\"label2\"]))\n        loss = criterion(embeddings, labels).to(device)\n\n        if mode == \"train\":\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n            if step ==  \"batch\":\n                lr_scheduler.step()\n\n        count = len(labels)\n        loss_meter.update(loss.item(), count)\n\n        if mode == \"train\":\n            tqdm_object.set_postfix(train_loss=loss_meter.avg, lr=get_lr(optimizer))\n        else:\n            tqdm_object.set_postfix(validloss=loss_meter.avg)\n    \n    return loss_meter","metadata":{"execution":{"iopub.status.busy":"2022-02-02T10:58:23.752589Z","iopub.execute_input":"2022-02-02T10:58:23.753471Z","iopub.status.idle":"2022-02-02T10:58:23.763989Z","shell.execute_reply.started":"2022-02-02T10:58:23.753436Z","shell.execute_reply":"2022-02-02T10:58:23.76321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_eval(epochs, model, train_loader, valid_loader,\n               criterion, optimizer, lr_scheduler=None, fold=None, li=None):\n    best_loss = float(\"inf\")\n\n    for epoch in range(epochs):\n        print(\"*\"*30)\n        print(\"Epoch{}\".format(epoch+1))\n        current_lr = get_lr(optimizer)\n\n        model.train()\n        train_loss = one_epoch(model,\n                                        criterion,\n                                        train_loader,\n                                        optimizer=optimizer,\n                                        lr_scheduler=lr_scheduler,\n                                        mode=\"train\",\n                                        step=CFG.step)\n        \n        model.eval()\n        with torch.no_grad():\n            valid_loss = one_epoch(model,\n                                            criterion,\n                                            valid_loader,\n                                            optimizer=None,\n                                            mode=\"valid\")\n        li[fold].append(float(valid_loss.avg))\n            \n        if valid_loss.avg < best_loss:\n            best_loss = valid_loss.avg\n\n            if fold is not None:\n                weight_name = \"/kaggle/working/weights/best\" + str(fold) + \".pt\"\n            else:\n                weight_name = \"/kaggle/working/weights/best.pt\"\n\n            torch.save(model.state_dict(), weight_name)\n            print(\"Saved best model!\")\n        if isinstance(lr_scheduler, torch.optim.lr_scheduler.ReduceLROnPlateau):\n            lr_scheduler.step(train_loss.avg)","metadata":{"execution":{"iopub.status.busy":"2022-02-02T10:58:23.765498Z","iopub.execute_input":"2022-02-02T10:58:23.765766Z","iopub.status.idle":"2022-02-02T10:58:23.776499Z","shell.execute_reply.started":"2022-02-02T10:58:23.76573Z","shell.execute_reply":"2022-02-02T10:58:23.775812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{}},{"cell_type":"code","source":"n_splits = 5\n\nlosses_list = [[] for i in range(n_splits)]\n\nkf = KFold(n_splits=n_splits, shuffle=True, random_state=CFG.SEED)\nids = df[\"individual_id\"].unique()\nfor fold, (train_ids, valid_ids) in enumerate(kf.split(ids)):\n    train_df= df[df[\"index\"].isin(train_ids)].reset_index(drop=True)\n    valid_df = df[df[\"index\"].isin(valid_ids)].reset_index(drop=True)\n    train_dataset = ContrasiveDataset(train_df,df_id_idx, train_data_transform)\n    valid_dataset = ContrasiveDataset(valid_df,df_id_idx, valid_data_transform)\n    train_loader = torch.utils.data.DataLoader(train_dataset,\n                                           batch_size=CFG.batch_size,\n                                           num_workers=CFG.num_workers,\n                                           pin_memory=True,\n                                           shuffle=True)\n    valid_loader = torch.utils.data.DataLoader(valid_dataset,\n                                           batch_size=CFG.batch_size,\n                                           num_workers=CFG.num_workers,\n                                           pin_memory=True,\n                                           shuffle=False)\n    \n    model = Model(CFG.model_name, CFG.pretrained, CFG.dropout, CFG.linear)\n    model.to(device)\n    optimizer = torch.optim.Adam(model.parameters(), lr=CFG.learning_rate)\n\n    if CFG.scheduler ==\"ReduceLROnPlateau\":\n        lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,\n                                                                mode=\"min\",\n                                                                factor=CFG.factor,\n                                                                patience=CFG.patience)\n    CFG.step = \"epoch\"\n\n    criterion = losses.ContrastiveLoss(pos_margin=0, neg_margin=CFG.margin)\n    criterion = losses.CrossBatchMemory(criterion, CFG.efficientnet_feature, memory_size=2000)\n\n    train_eval(CFG.epochs,\n            model,\n            train_loader,\n            valid_loader,\n            criterion,\n            optimizer,\n            lr_scheduler,\n            fold,\n            losses_list)\n    \n    print(losses_list)\n\n    gc.collect()\n    torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2022-02-02T10:58:23.777994Z","iopub.execute_input":"2022-02-02T10:58:23.778622Z","iopub.status.idle":"2022-02-02T12:34:52.642764Z","shell.execute_reply.started":"2022-02-02T10:58:23.778585Z","shell.execute_reply":"2022-02-02T12:34:52.64187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    del model, Model, train_loader, valid_loader, train_dataset, valid_dataset, optimizer, criterion, train_eval, lr_scheduler, labels, one_epoch, train_df, valid_df, train_labels, valid_labels, AvgMeter, get_lr, ContrasiveDataset, train_transforms, train_albumentations_transform, train_data_transform\nexcept:\n    print(\"some variable do not exist\")","metadata":{"execution":{"iopub.status.busy":"2022-02-02T12:35:01.109915Z","iopub.execute_input":"2022-02-02T12:35:01.110173Z","iopub.status.idle":"2022-02-02T12:35:01.115758Z","shell.execute_reply.started":"2022-02-02T12:35:01.110145Z","shell.execute_reply":"2022-02-02T12:35:01.11505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()\ntorch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2022-02-02T12:35:02.636002Z","iopub.execute_input":"2022-02-02T12:35:02.636484Z","iopub.status.idle":"2022-02-02T12:35:02.77744Z","shell.execute_reply.started":"2022-02-02T12:35:02.636444Z","shell.execute_reply":"2022-02-02T12:35:02.776669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Making Embeddings","metadata":{}},{"cell_type":"code","source":"df_2 = pd.read_csv(\"/kaggle/input/happy-whale-and-dolphin/sample_submission.csv\")\ndf_2 = df_2.drop('predictions', axis=1)\ndf_2[\"path\"] = \"/kaggle/input/happy-whale-and-dolphin/test_images/\" + df_2[\"image\"].astype(str)","metadata":{"execution":{"iopub.status.busy":"2022-02-02T13:57:04.675632Z","iopub.execute_input":"2022-02-02T13:57:04.676335Z","iopub.status.idle":"2022-02-02T13:57:04.72099Z","shell.execute_reply.started":"2022-02-02T13:57:04.676289Z","shell.execute_reply":"2022-02-02T13:57:04.720304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_list = df.values.tolist()\ndf_2_list = df_2.values.tolist()","metadata":{"execution":{"iopub.status.busy":"2022-02-02T13:57:26.924523Z","iopub.execute_input":"2022-02-02T13:57:26.924803Z","iopub.status.idle":"2022-02-02T13:57:26.963426Z","shell.execute_reply.started":"2022-02-02T13:57:26.924771Z","shell.execute_reply":"2022-02-02T13:57:26.96259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TrainDataset(Dataset):\n    def __init__(self, List, transforms=None):\n        self.transforms=transforms\n        self.List = List\n\n    def __len__(self):\n        return len(self.List)\n\n    def __getitem__(self, idx):\n        img = cv2.imread(self.List[idx][3])[...,::-1]\n        if self.transforms is not None:\n            img = self.transforms(img)[\"image\"]\n        return {\"images1\":img}","metadata":{"execution":{"iopub.status.busy":"2022-02-02T12:35:48.976442Z","iopub.execute_input":"2022-02-02T12:35:48.976721Z","iopub.status.idle":"2022-02-02T12:35:48.982585Z","shell.execute_reply.started":"2022-02-02T12:35:48.976686Z","shell.execute_reply":"2022-02-02T12:35:48.981881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = TrainDataset(df_list, valid_data_transform)\ntrain_loader = torch.utils.data.DataLoader(train_dataset,\n                                          batch_size=CFG.batch_size,\n                                          num_workers=os.cpu_count(),\n                                          pin_memory=True,\n                                          shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2022-02-02T12:36:10.551603Z","iopub.execute_input":"2022-02-02T12:36:10.552225Z","iopub.status.idle":"2022-02-02T12:36:10.556774Z","shell.execute_reply.started":"2022-02-02T12:36:10.552185Z","shell.execute_reply":"2022-02-02T12:36:10.55603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Model(nn.Module):\n    def __init__(self,\n                 model_name=CFG.model_name,\n                 pretrained=True,\n                 dropout=0.2,\n                 linear=128):\n\n                 super().__init__()\n                 model = timm.create_model(model_name,\n                                           pretrained=pretrained,\n                                           num_classes=0)\n                 self.num_features=model.num_features\n                 self.linear = None\n                 if linear is not None and linear >0:\n                     self.linear = nn.Linear(self.num_features, linear)\n                 self.backbone = nn.Sequential(model,\n                                              self.linear if self.linear is not None else nn.Identity(),\n                                              nn.ReLU() if self.linear is not None else nn.Identity(),\n                                              nn.Dropout(0.2) if dropout is not None else nn.Identity())\n    def forward(self, batch):\n        images = self.backbone(batch[\"images1\"].to(device))\n        return images","metadata":{"execution":{"iopub.status.busy":"2022-02-02T12:36:32.203653Z","iopub.execute_input":"2022-02-02T12:36:32.20394Z","iopub.status.idle":"2022-02-02T12:36:32.212005Z","shell.execute_reply.started":"2022-02-02T12:36:32.203911Z","shell.execute_reply":"2022-02-02T12:36:32.21134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = []\nfor i in range(5):\n    model = Model(CFG.model_name, CFG.pretrained, CFG.dropout, CFG.linear)\n    models.append(model)\n    model_path = \"/kaggle/working/weights/best\"+str(i)+\".pt\"\n    models[i].load_state_dict(torch.load(model_path))\n    models[i].to(device)","metadata":{"execution":{"iopub.status.busy":"2022-02-02T12:36:57.597068Z","iopub.execute_input":"2022-02-02T12:36:57.597327Z","iopub.status.idle":"2022-02-02T12:37:01.284277Z","shell.execute_reply.started":"2022-02-02T12:36:57.597289Z","shell.execute_reply":"2022-02-02T12:37:01.283538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calc_norm(array):\n    array = array/np.linalg.norm(array)\n    return array.tolist()","metadata":{"execution":{"iopub.status.busy":"2022-02-02T12:37:05.259586Z","iopub.execute_input":"2022-02-02T12:37:05.260301Z","iopub.status.idle":"2022-02-02T12:37:05.264453Z","shell.execute_reply.started":"2022-02-02T12:37:05.260262Z","shell.execute_reply":"2022-02-02T12:37:05.263614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_trees = 100\nindex = AnnoyIndex(CFG.efficientnet_feature, metric=\"euclidean\")\nfor i in range(len(models)):\n    models[i].eval()\n\nlabel_index = 0\n\nfor batch in train_loader:\n    with torch.inference_mode():\n        for i in range(len(models)):\n            if i == 0:\n                outputs = models[i](batch).cpu()\n            else:\n                outputs += models[i](batch).cpu()\n        outputs = outputs/float(len(models))\n\n    for i, feature in enumerate(outputs):\n        feature=feature.numpy().copy()\n        feature = calc_norm(feature)\n        index.add_item(label_index, feature)\n        label_index += 1\n\nindex.build(n_trees, n_jobs=-1)\nindex.save(\"/kaggle/working/feature.ann\")","metadata":{"execution":{"iopub.status.busy":"2022-02-02T12:37:59.989828Z","iopub.execute_input":"2022-02-02T12:37:59.990601Z","iopub.status.idle":"2022-02-02T13:49:26.113334Z","shell.execute_reply.started":"2022-02-02T12:37:59.990548Z","shell.execute_reply":"2022-02-02T13:49:26.112535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TestDataset(Dataset):\n    def __init__(self, List, transforms=None):\n        self.transforms=transforms\n        self.List = List\n\n    def __len__(self):\n        return len(self.List)\n\n    def __getitem__(self, idx):\n        img = cv2.imread(self.List[idx][1])[...,::-1]\n        if self.transforms is not None:\n            img = self.transforms(img)[\"image\"]\n        return {\"images1\":img}","metadata":{"execution":{"iopub.status.busy":"2022-02-02T13:57:47.272414Z","iopub.execute_input":"2022-02-02T13:57:47.272683Z","iopub.status.idle":"2022-02-02T13:57:47.278569Z","shell.execute_reply.started":"2022-02-02T13:57:47.272636Z","shell.execute_reply":"2022-02-02T13:57:47.27791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset = TestDataset(df_2_list, valid_data_transform)\ntest_loader = torch.utils.data.DataLoader(test_dataset,\n                                          batch_size=CFG.batch_size,\n                                          num_workers=os.cpu_count(),\n                                          pin_memory=True,\n                                          shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2022-02-02T13:58:18.572528Z","iopub.execute_input":"2022-02-02T13:58:18.573215Z","iopub.status.idle":"2022-02-02T13:58:18.579403Z","shell.execute_reply.started":"2022-02-02T13:58:18.573164Z","shell.execute_reply":"2022-02-02T13:58:18.578482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_trees = 3\nindex_test = AnnoyIndex(CFG.efficientnet_feature, metric=\"euclidean\")\nfor i in range(len(models)):\n    models[i].eval()\nlabel_index = 0\n\nfor batch in test_loader:\n    with torch.inference_mode():\n        for i in range(len(models)):\n            if i == 0:\n                outputs = models[i](batch).cpu()\n            else:\n                outputs += models[i](batch).cpu()\n        outputs = outputs/float(len(models))\n    for i, feature in enumerate(outputs):\n        feature=feature.numpy().copy()\n        feature = calc_norm(feature)\n        index_test.add_item(label_index, feature)\n        label_index += 1\n\nindex_test.build(n_trees, n_jobs=-1)","metadata":{"execution":{"iopub.status.busy":"2022-02-02T13:58:22.029861Z","iopub.execute_input":"2022-02-02T13:58:22.030346Z","iopub.status.idle":"2022-02-02T14:37:24.375335Z","shell.execute_reply.started":"2022-02-02T13:58:22.030309Z","shell.execute_reply":"2022-02-02T14:37:24.374535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def query_expansion(query_vec, similar_vec):\n    query_vec = np.array(query_vec)\n    similar_vec = np.array(similar_vec)\n    similarity = np.dot(query_vec, similar_vec)\n    similarity = float(similarity)\n\n    vec = (1*query_vec + similarity*similar_vec)/(1+similarity)\n    vec = calc_norm(vec)\n    return vec","metadata":{"execution":{"iopub.status.busy":"2022-02-02T14:37:24.377641Z","iopub.execute_input":"2022-02-02T14:37:24.377953Z","iopub.status.idle":"2022-02-02T14:37:24.383153Z","shell.execute_reply.started":"2022-02-02T14:37:24.377911Z","shell.execute_reply":"2022-02-02T14:37:24.382428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!touch submission.csv\n!echo \"image,predictions\">> submission.csv","metadata":{"execution":{"iopub.status.busy":"2022-02-02T14:38:34.407327Z","iopub.execute_input":"2022-02-02T14:38:34.408051Z","iopub.status.idle":"2022-02-02T14:38:35.827187Z","shell.execute_reply.started":"2022-02-02T14:38:34.408005Z","shell.execute_reply":"2022-02-02T14:38:35.826256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(\"submission.csv\", \"a\") as f:\n    for i in range(len(df_2)):\n        test_query_vec = index_test.get_item_vector(i)\n        result = index.get_nns_by_vector(test_query_vec,1)#****\n        tmp_index_vec = index.get_item_vector(result[0])#****\n\n        test_query_vec = query_expansion(test_query_vec, tmp_index_vec)\n\n        result = index.get_nns_by_vector(test_query_vec,4)\n        results =str(df_2.iloc[i][\"image\"])+\",\"\n        for k in range(5):\n            if k == 4:\n                results = results+ \"new_individual\" +\"\\n\"\n            else:\n                results = results+ str(df_list[result[k]][2]) + \" \"\n        f.writelines(\"{}\".format(results))","metadata":{"execution":{"iopub.status.busy":"2022-02-02T14:39:35.918352Z","iopub.execute_input":"2022-02-02T14:39:35.918851Z","iopub.status.idle":"2022-02-02T14:41:08.394997Z","shell.execute_reply.started":"2022-02-02T14:39:35.918807Z","shell.execute_reply":"2022-02-02T14:41:08.394128Z"},"trusted":true},"execution_count":null,"outputs":[]}]}