{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# <span><h1 style = \"font-family: garamond; font-size: 40px; font-style: normal; letter-spcaing: 3px; background-color: #f6f5f5; color :#346efe; border-radius: 100px 100px; text-align:center\">Torch inference notebook</h1></span>","metadata":{}},{"cell_type":"markdown","source":"<br>\n<h2 style = \"font-size:16px\" \n\nThis is the inference notebook made for training with  https://www.kaggle.com/vladvdv/pytorch-train-notebook-arcface-gem-pooling/notebook  \n    \nModifications for version 30:\n* replaced supervized KNeighborsClassifier with unsupervized NearestNeighbors   \n* corrected gridsearch for determining optim \"new_individual\" threhsold* (there are used the same training data as the ones the model was trained, for training the NearestNeighbors algorithm, and then the same validation data that the model was trained to predict on the NearestNeighbors algorithm.  \n   \nTo do:\n* Implement all folds model blending\n\nModifications for version 31:\n* Change original dataset to fins dataset\n* Change the prediction function (design general function for max, avg and custom blend)","metadata":{}},{"cell_type":"code","source":"import pickle\nimport os\nimport gc\nimport cv2\nimport math\nimport copy\nimport time\nimport random\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.cuda import amp\nimport joblib\nfrom tqdm import tqdm\nfrom collections import defaultdict\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import StratifiedKFold\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nimport sys\nsys.path.append(\"../input/timm-pytorch-image-models\")\nimport timm\nimport seaborn as sns\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.neighbors import NearestNeighbors","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-04-10T16:25:46.38066Z","iopub.execute_input":"2022-04-10T16:25:46.381221Z","iopub.status.idle":"2022-04-10T16:25:46.391154Z","shell.execute_reply.started":"2022-04-10T16:25:46.38118Z","shell.execute_reply":"2022-04-10T16:25:46.390309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CONFIG = {\"seed\": 21, # choose your lucky seed\n          \"img_size\": 512, # training image size\n          \"model_name\": \"tf_efficientnet_b7_ns\", # training model arhitecture\n          \"num_classes\": 15587, # total individuals in training data\n          \"test_batch_size\": 4, # choose acording to the training arhitecture and image size \n          \"device\": torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\"), # gpu\n          \"test_mode\":False, # selects just the first 200 samples from the test data, usefull for debuging purposes\n          \"n_fold\":5,\n          # ArcFace Hyperparameters\n          \"s\": 30.0, \n          \"m\": 0.30,\n          \"ls_eps\": 0.0,\n          \"easy_margin\": False,\n          \"rotate_h\": False,\n          \"public_blend\": True\n          }","metadata":{"execution":{"iopub.status.busy":"2022-04-10T16:25:46.392856Z","iopub.execute_input":"2022-04-10T16:25:46.393484Z","iopub.status.idle":"2022-04-10T16:25:46.40364Z","shell.execute_reply.started":"2022-04-10T16:25:46.393445Z","shell.execute_reply":"2022-04-10T16:25:46.402983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed=42):\n    '''Sets the seed of the entire notebook so results are the same every time we run.\n    This is for REPRODUCIBILITY.'''\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    # When running on the CuDNN backend, two further options must be set\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    # Set a fixed value for the hash seed\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    \nset_seed(CONFIG['seed'])","metadata":{"execution":{"iopub.status.busy":"2022-04-10T16:25:46.405584Z","iopub.execute_input":"2022-04-10T16:25:46.405933Z","iopub.status.idle":"2022-04-10T16:25:46.415129Z","shell.execute_reply.started":"2022-04-10T16:25:46.405895Z","shell.execute_reply":"2022-04-10T16:25:46.414459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_test_file_path(id):\n    return f\"{TEST_DIR}/{id}\"\n\ndef get_train_file_path(id):\n    return f\"{TRAIN_DIR}/{id}\"\n\nTEST_DIR = '../input/convert-backfintfrecords/happy-whale-and-dolphin-backfin/test_images'\nTRAIN_DIR = '../input/convert-backfintfrecords/happy-whale-and-dolphin-backfin/train_images'\nweights_path = \"../input/dummymodel17/Loss7.7886_epoch19.bin\"\n\nif CONFIG[\"test_mode\"]==True:\n    df_test = pd.read_csv(\"../input/finsmetadata/sample_submission.csv\")[:2000]\n    df_train = pd.read_csv(\"../input/finsmetadata/train.csv\")[:2000]\nelse:\n    df_test = pd.read_csv(\"../input/finsmetadata/sample_submission.csv\")\n    df_train = pd.read_csv(\"../input/finsmetadata/train.csv\")  \n\n\n\ndf_test['file_path'] = df_test['image'].apply(get_test_file_path)\ndf_train['file_path'] = df_train['image'].apply(get_train_file_path)\ntrain_labels = np.array(df_train['individual_id'].values)\n#split into train and valid like in the training notebook for validating NearestNeighbors approach \ntrainFold = 0 # this model was trained on fold 0\nskf = StratifiedKFold(n_splits=CONFIG['n_fold'])\nfor fold, ( _, val_) in enumerate(skf.split(X=df_train, y=train_labels)):\n      df_train.loc[val_ , \"kfold\"] = fold\ndf_train_cnn = df_train[df_train.kfold != trainFold].reset_index(drop=True)\ndf_valid_cnn = df_train[df_train.kfold == trainFold].reset_index(drop=True)\n\n\n#hardcode dummy label for input in ArcMargin forward function\ndf_test['dummy_labels'] = 0\ndf_train_cnn['dummy_labels'] = 0\ndf_valid_cnn['dummy_labels'] = 0\n\n","metadata":{"execution":{"iopub.status.busy":"2022-04-10T16:56:41.291645Z","iopub.execute_input":"2022-04-10T16:56:41.292232Z","iopub.status.idle":"2022-04-10T16:56:41.949297Z","shell.execute_reply.started":"2022-04-10T16:56:41.29219Z","shell.execute_reply":"2022-04-10T16:56:41.948493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.iloc[0]['file_path']","metadata":{"execution":{"iopub.status.busy":"2022-04-10T16:56:45.38765Z","iopub.execute_input":"2022-04-10T16:56:45.388013Z","iopub.status.idle":"2022-04-10T16:56:45.394627Z","shell.execute_reply.started":"2022-04-10T16:56:45.387973Z","shell.execute_reply":"2022-04-10T16:56:45.393858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class HappyWhaleDataset(Dataset):\n    def __init__(self, df, transforms=None):\n        self.df = df\n        self.file_names = df['file_path'].values\n        self.labels = df['dummy_labels'].values\n        self.transforms = transforms\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        img_path = self.file_names[index]\n        img = cv2.imread(img_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        label = self.labels[index]\n        \n        if self.transforms:\n            img = self.transforms(image=img)[\"image\"]\n        return {\n            'image': img,\n            'label': torch.tensor(label, dtype=torch.long)\n        }\n        ","metadata":{"execution":{"iopub.status.busy":"2022-04-10T16:56:47.489711Z","iopub.execute_input":"2022-04-10T16:56:47.490422Z","iopub.status.idle":"2022-04-10T16:56:47.498585Z","shell.execute_reply.started":"2022-04-10T16:56:47.490386Z","shell.execute_reply":"2022-04-10T16:56:47.497896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class GeM(nn.Module):\n    def __init__(self, p=3, eps=1e-6):\n        super(GeM, self).__init__()\n        self.p = nn.Parameter(torch.ones(1)*p)\n        self.eps = eps\n\n    def forward(self, x):\n        return self.gem(x, p=self.p, eps=self.eps)\n        \n    def gem(self, x, p=3, eps=1e-6):\n        return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1./p)\n        \n    def __repr__(self):\n        return self.__class__.__name__ + \\\n                '(' + 'p=' + '{:.4f}'.format(self.p.data.tolist()[0]) + \\\n                ', ' + 'eps=' + str(self.eps) + ')'","metadata":{"execution":{"iopub.status.busy":"2022-04-10T16:56:50.819822Z","iopub.execute_input":"2022-04-10T16:56:50.820515Z","iopub.status.idle":"2022-04-10T16:56:50.828346Z","shell.execute_reply.started":"2022-04-10T16:56:50.820478Z","shell.execute_reply":"2022-04-10T16:56:50.827616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ArcMarginProduct(nn.Module):\n    r\"\"\"Implement of large margin arc distance: :\n        Args:\n            in_features: size of each input sample\n            out_features: size of each output sample\n            s: norm of input feature\n            m: margin\n            cos(theta + m)\n        \"\"\"\n    def __init__(self, in_features, out_features, s=30.0, \n                 m=0.50, easy_margin=False, ls_eps=0.0):\n        super(ArcMarginProduct, self).__init__()\n        self.in_features = in_features\n        self.out_features = out_features\n        self.s = s\n        self.m = m\n        self.ls_eps = ls_eps  # label smoothing\n        self.weight = nn.Parameter(torch.FloatTensor(out_features, in_features))\n        nn.init.xavier_uniform_(self.weight)\n\n        self.easy_margin = easy_margin\n        self.cos_m = math.cos(m)\n        self.sin_m = math.sin(m)\n        self.th = math.cos(math.pi - m)\n        self.mm = math.sin(math.pi - m) * m\n\n    def forward(self, input, label):\n        # --------------------------- cos(theta) & phi(theta) ---------------------\n        cosine = F.linear(F.normalize(input), F.normalize(self.weight))\n        sine = torch.sqrt(1.0 - torch.pow(cosine, 2))\n        phi = cosine * self.cos_m - sine * self.sin_m\n        if self.easy_margin:\n            phi = torch.where(cosine > 0, phi, cosine)\n        else:\n            phi = torch.where(cosine > self.th, phi, cosine - self.mm)\n        # --------------------------- convert label to one-hot ---------------------\n        # one_hot = torch.zeros(cosine.size(), requires_grad=True, device='cuda')\n        one_hot = torch.zeros(cosine.size(), device=CONFIG['device'])\n        one_hot.scatter_(1, label.view(-1, 1).long(), 1)\n        if self.ls_eps > 0:\n            one_hot = (1 - self.ls_eps) * one_hot + self.ls_eps / self.out_features\n        # -------------torch.where(out_i = {x_i if condition_i else y_i) ------------\n        output = (one_hot * phi) + ((1.0 - one_hot) * cosine)\n        output *= self.s\n\n        return output","metadata":{"execution":{"iopub.status.busy":"2022-04-10T16:56:51.810207Z","iopub.execute_input":"2022-04-10T16:56:51.811049Z","iopub.status.idle":"2022-04-10T16:56:51.825261Z","shell.execute_reply.started":"2022-04-10T16:56:51.810995Z","shell.execute_reply":"2022-04-10T16:56:51.824305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class HappyWhaleModel(nn.Module):\n    def __init__(self, model_name, pretrained=True):\n        super(HappyWhaleModel, self).__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained)\n        in_features = self.model.classifier.in_features\n        self.model.classifier = nn.Identity()\n        self.model.global_pool = nn.Identity()\n        self.pooling = GeM()\n        self.drop = nn.Dropout(p=0.2, inplace=False)\n        self.fc = nn.Linear(in_features,2048)\n        self.arc = ArcMarginProduct(2048, \n                           CONFIG[\"num_classes\"],\n                           s=CONFIG[\"s\"], \n                           m=CONFIG[\"m\"], \n                           easy_margin=CONFIG[\"ls_eps\"], \n                           ls_eps=CONFIG[\"ls_eps\"])\n    def forward(self, images, labels):\n        features = self.model(images)\n        pooled_features = self.pooling(features).flatten(1)\n        pooled_drop = self.drop(pooled_features)\n        emb = self.fc(pooled_drop)\n        output = self.arc(emb,labels)\n        return output,emb","metadata":{"execution":{"iopub.status.busy":"2022-04-10T16:56:52.901192Z","iopub.execute_input":"2022-04-10T16:56:52.901691Z","iopub.status.idle":"2022-04-10T16:56:52.911314Z","shell.execute_reply.started":"2022-04-10T16:56:52.901652Z","shell.execute_reply":"2022-04-10T16:56:52.910583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_transforms = {\n    \"test\": A.Compose([\n        A.Resize(CONFIG['img_size'], CONFIG['img_size']),\n        A.Normalize(\n                mean=[0.485, 0.456, 0.406], \n                std=[0.229, 0.224, 0.225], \n                max_pixel_value=255.0, \n                p=1.0\n            ),\n        ToTensorV2()], p=1.)\n    \n}","metadata":{"execution":{"iopub.status.busy":"2022-04-10T16:56:54.325694Z","iopub.execute_input":"2022-04-10T16:56:54.326216Z","iopub.status.idle":"2022-04-10T16:56:54.332474Z","shell.execute_reply.started":"2022-04-10T16:56:54.32618Z","shell.execute_reply":"2022-04-10T16:56:54.331477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@torch.inference_mode()\ndef inference(model, dataloader, device):\n    model.eval()  \n    outputList=[]\n    bar = tqdm(enumerate(dataloader), total=len(dataloader))\n    for step, data in bar:        \n        images = data['image'].to(device, dtype=torch.float)\n        labels = data['label'].to(device, dtype=torch.long)\n        _, outputs = model(images,labels)\n        outputList.extend(outputs.cpu().detach().numpy())\n    return outputList","metadata":{"execution":{"iopub.status.busy":"2022-04-10T16:56:56.484842Z","iopub.execute_input":"2022-04-10T16:56:56.485508Z","iopub.status.idle":"2022-04-10T16:56:56.491215Z","shell.execute_reply.started":"2022-04-10T16:56:56.485472Z","shell.execute_reply":"2022-04-10T16:56:56.490515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = HappyWhaleModel(CONFIG['model_name'])\nmodel.to(CONFIG['device']);\nmodel.load_state_dict(torch.load(weights_path))\n#predict first on train dataset to extract embeddings\ntrain_dataset = HappyWhaleDataset(df_train_cnn, transforms=data_transforms[\"test\"])\ntrain_loader = DataLoader(train_dataset, batch_size=CONFIG['test_batch_size'], \n                          num_workers=4, shuffle=False, pin_memory=True)\n\nvalid_dataset = HappyWhaleDataset(df_valid_cnn, transforms=data_transforms[\"test\"])\nvalid_loader = DataLoader(valid_dataset, batch_size=CONFIG['test_batch_size'], \n                          num_workers=4, shuffle=False, pin_memory=True)\n\ntest_dataset = HappyWhaleDataset(df_test, transforms=data_transforms[\"test\"])\ntest_loader = DataLoader(test_dataset, batch_size=CONFIG['test_batch_size'], \n                          num_workers=4, shuffle=False, pin_memory=True)","metadata":{"execution":{"iopub.status.busy":"2022-04-10T16:56:59.088333Z","iopub.execute_input":"2022-04-10T16:56:59.089122Z","iopub.status.idle":"2022-04-10T16:57:01.902245Z","shell.execute_reply.started":"2022-04-10T16:56:59.089082Z","shell.execute_reply":"2022-04-10T16:57:01.901494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def map_per_image(label, predictions):\n    \"\"\"Computes the precision score of one image.\n\n    Parameters\n    ----------\n    label : string\n            The true label of the image\n    predictions : list\n            A list of predicted elements (order does matter, 5 predictions allowed per image)\n\n    Returns\n    -------\n    score : double\n    \"\"\"    \n    try:\n        return 1 / (predictions[:5].index(label) + 1)\n    except ValueError:\n        return 0.0\n\ndef map_per_set(labels, predictions):\n    \"\"\"Computes the average over multiple images.\n\n    Parameters\n    ----------\n    labels : list\n             A list of the true labels. (Only one true label per images allowed!)\n    predictions : list of list\n             A list of predicted elements (order does matter, 5 predictions allowed per image)\n\n    Returns\n    -------\n    score : double\n    \"\"\"\n    return np.mean([map_per_image(l, p) for l,p in zip(labels, predictions)])","metadata":{"execution":{"iopub.status.busy":"2022-04-10T16:57:08.776329Z","iopub.execute_input":"2022-04-10T16:57:08.77689Z","iopub.status.idle":"2022-04-10T16:57:08.783335Z","shell.execute_reply.started":"2022-04-10T16:57:08.776854Z","shell.execute_reply":"2022-04-10T16:57:08.782338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def PredictGrid(train_cnn_predictions,valid_cnn_predictions, train_labels, valid_labels,new_individual_thres):\n    neigh = NearestNeighbors(n_neighbors=CONFIG[\"neigh\"],metric=\"cosine\")\n    neigh.fit(train_cnn_predictions)\n    \n    distances,idxs = neigh.kneighbors(valid_cnn_predictions, return_distance=True)\n    conf = 1-distances\n    preds=[]\n\n    for j in range(len(idxs)):\n        preds.append(list(train_labels[idxs[j]]))\n\n        \n\n    allTop5Preds=[]\n    valid_labels_list=[]\n    for i in range(len(preds)):\n        valid_labels_list.append((valid_labels[i]))\n\n        predictTop = preds[i][:5]\n        Top5Conf = conf[i][:5]\n\n        if Top5Conf[0] < new_individual_thres:\n           \n            tempList=['new_individual',predictTop[0],predictTop[1],predictTop[2],predictTop[3]]\n            allTop5Preds.append(tempList)   \n           \n        elif Top5Conf[1] < new_individual_thres:\n   \n            tempList=[predictTop[0],'new_individual',predictTop[1],predictTop[2],predictTop[3]]\n            allTop5Preds.append(tempList)    \n           \n        elif Top5Conf[2] < new_individual_thres:\n\n            tempList=[predictTop[0],predictTop[1],'new_individual',predictTop[2],predictTop[3]]\n            allTop5Preds.append(tempList)    \n           \n        elif Top5Conf[3] < new_individual_thres:\n           \n            tempList=[predictTop[0],predictTop[1],predictTop[2],'new_individual',predictTop[3]]        \n            allTop5Preds.append(tempList)  \n           \n        elif Top5Conf[4] < new_individual_thres:\n\n            tempList=[predictTop[0],predictTop[1],predictTop[2],predictTop[3],'new_individual']        \n            allTop5Preds.append(tempList)        \n           \n        else:\n            allTop5Preds.append(predictTop)\n\n        if (('new_individual' in allTop5Preds[-1]) and (valid_labels_list[i] not in train_labels)):\n            allTop5Preds[-1] = [valid_labels_list[i] if x=='new_individual' else x for x in allTop5Preds[-1]]\n\n    score = map_per_set(valid_labels_list,allTop5Preds)\n\n    return score","metadata":{"execution":{"iopub.status.busy":"2022-04-10T16:57:11.322581Z","iopub.execute_input":"2022-04-10T16:57:11.322868Z","iopub.status.idle":"2022-04-10T16:57:11.336228Z","shell.execute_reply.started":"2022-04-10T16:57:11.322834Z","shell.execute_reply":"2022-04-10T16:57:11.335413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**We are training a NearestNeighbors algorithm on the data that had been use as trainset in the train kernel and test the results on the validation data from the original kernel** (https://www.kaggle.com/vladvdv/pytorch-train-notebook-arcface-gem-pooling/notebook  )","metadata":{}},{"cell_type":"code","source":"df_train_cnn_predictions = np.array(inference(model, train_loader, CONFIG['device']))\ndf_valid_cnn_predictions = np.array(inference(model, valid_loader, CONFIG['device']))\ntrain_cnn_labels = np.array(df_train_cnn['individual_id'].values)\nvalid_cnn_labels = np.array(df_valid_cnn['individual_id'].values)\n","metadata":{"execution":{"iopub.status.busy":"2022-04-10T16:26:42.518994Z","iopub.execute_input":"2022-04-10T16:26:42.519669Z","iopub.status.idle":"2022-04-10T16:48:57.511247Z","shell.execute_reply.started":"2022-04-10T16:26:42.519632Z","shell.execute_reply":"2022-04-10T16:48:57.509667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_cnn_predictions =  np.array(inference(model, test_loader, CONFIG['device']))\n\nallTrainData = np.concatenate((df_train_cnn_predictions,df_valid_cnn_predictions))\nallTrainingLabels = np.concatenate((train_cnn_labels,valid_cnn_labels)) ","metadata":{"execution":{"iopub.status.busy":"2022-04-10T16:57:14.957471Z","iopub.execute_input":"2022-04-10T16:57:14.958178Z","iopub.status.idle":"2022-04-10T17:12:08.236621Z","shell.execute_reply.started":"2022-04-10T16:57:14.958126Z","shell.execute_reply":"2022-04-10T17:12:08.235834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def GetDuplicates(elem, lst):\n    if elem in lst:\n        counter = 0\n        elem_pos = []\n        for i in lst:\n            if i == elem:\n                elem_pos.append(counter)\n            counter = counter + 1\n        return(elem_pos)    \n    \ndef GetSubmission(train_data_list,test_data_list, train_labels_list,neighbors=5,metric='cosine', new_individual_thres=0.6,blendType = \"avg\"):\n    df = pd.read_csv(r\"../input/finsmetadata/sample_submission.csv\")     \n    dfResults =pd.DataFrame(columns=set(train_labels_list[0]), index = df['image'])\n    \n    for it in range(len(train_data_list)):\n        train_data = train_data_list[it]\n        test_data = test_data_list[it]\n        train_labels = train_labels_list[it]\n        \n        neigh = NearestNeighbors(n_neighbors=neighbors,metric=metric)\n        neigh.fit(train_data)\n        distances,idxs = neigh.kneighbors(test_data, return_distance=True)\n        conf = 1-distances\n\n        preds=[]\n        \n        for i in range(len(idxs)):\n            preds.append(train_labels[idxs[i]])\n        \n        #remove duplicate predictions\n        allPredsCleared = []\n        allConfCleared = []\n        for x in range(len(preds)):\n            indexesToRemove=[]    \n            for i in range(len(preds[x])):\n                duplList = GetDuplicates(preds[x][i],preds[x])\n                if len(duplList)>1:\n                    for elem in duplList[1:]:\n                        indexesToRemove.append(elem)\n                \n                \n            indexesToRemove=list(set(indexesToRemove))\n            \n            # print(indexesToRemove)\n            \n            predsCleared = [i for j, i in enumerate(preds[x]) if j not in indexesToRemove]\n            confCleared  = [i for j, i in enumerate(conf[x]) if j not in indexesToRemove]    \n            allPredsCleared.append(predsCleared)\n            allConfCleared.append(confCleared)\n    \n    \n        # if prediction is nan create new record, otherwise append to existing one\n        for i in range(len(allPredsCleared)):\n            for j in range(len(allPredsCleared[i])):\n                if np.isnan(dfResults.loc[df.iloc[i]['image']][allPredsCleared[i][j]]).any()==True:\n                    dfResults.loc[df.iloc[i]['image']][allPredsCleared[i][j]] = [allConfCleared[i][j]]\n                else:\n                    dfResults.loc[df.iloc[i]['image']][allPredsCleared[i][j]].append(conf[i][j])\n    \n    #get mean predictions\n    dictList=[]\n    for imageName in (dfResults.index):\n        results= dfResults.loc[imageName]\n        tempDict={}\n        for k in range(len(results)):\n            if np.isnan(results[k]).any()!=True:\n                if(blendType==\"avg\"):\n                    blendConf=np.mean(results[k]) \n                if(blendType==\"max\"):\n                    blendConf=np.max(results[k])\n                if(blendType==\"custom1\"):\n                    occurance = len(results[k])\n                    halfFoldsNr = round(math.ceil(len(train_data_list)/2))\n                    blendConf = np.max(results[k]) + ((occurance - halfFoldsNr)/10)\n                if(blendType==\"custom2\"):\n                    occurance = len(results[k])\n                    halfFoldsNr = round(math.ceil(len(train_data_list)/2))\n                    blendConf = np.mean(results[k]) + ((occurance - halfFoldsNr)/10)\n                tempDict[results.index[k]] = blendConf\n        dictList.append(tempDict)\n\n\n    # calculate where to insert \"new_individual\"\n    best5SubsAll=[]\n    best5ConfAll=[]\n    for i in range(len(dictList)):\n        best5Subs=sorted(dictList[i], key=dictList[i].get,reverse=True)[:5]\n        if len(best5Subs)<5:\n            elementsToAppend= 5-len(best5Subs)\n            for j in range(elementsToAppend):\n                best5Subs.append(best5Subs[0])\n        best5Conf=[dictList[i][best5Subs[0]], dictList[i][best5Subs[1]], dictList[i][best5Subs[2]],dictList[i][best5Subs[3]],dictList[i][best5Subs[4]]]\n        best5ConfAll.append(best5Conf)\n        best5SubsAll.append(best5Subs)\n    predictTopDecoded={}\n    \n    for i in range(len(best5SubsAll)):\n        predictTop = best5SubsAll[i]\n        topValues = best5ConfAll[i]\n        if (usePublic_new_individ == False):\n            if topValues[0] < new_individual_thres:\n                \n                tempList=['new_individual',predictTop[0],predictTop[1],predictTop[2],predictTop[3]]\n                predictTopDecoded[df.iloc[i]['image']] = tempList  \n                \n            elif topValues[1] < new_individual_thres:\n        \n                tempList=[predictTop[0],'new_individual',predictTop[1],predictTop[2],predictTop[3]]\n                predictTopDecoded[df.iloc[i]['image']] = tempList     \n                \n            elif topValues[2] < new_individual_thres:\n        \n                tempList=[predictTop[0],predictTop[1],'new_individual',predictTop[2],predictTop[3]]\n                predictTopDecoded[df.iloc[i]['image']] = tempList   \n                \n            elif topValues[3] < new_individual_thres:\n                \n                tempList=[predictTop[0],predictTop[1],predictTop[2],'new_individual',predictTop[3]]        \n                predictTopDecoded[df.iloc[i]['image']] = tempList  \n                \n            elif topValues[4] < new_individual_thres:\n        \n                tempList=[predictTop[0],predictTop[1],predictTop[2],predictTop[3],'new_individual']        \n                predictTopDecoded[df.iloc[i]['image']] = tempList         \n                \n            else:\n              predictTopDecoded[df.iloc[i]['image']] = predictTop  \n              \n        if (usePublic_new_individ == True):\n            if topValues[0] < new_individual_thres:\n                \n                tempList=['new_individual',predictTop[0],predictTop[1],predictTop[2],predictTop[3]]\n                predictTopDecoded[df.iloc[i]['image']] = tempList             \n            else:\n                tempList=[predictTop[0],'new_individual',predictTop[1],predictTop[2],predictTop[3]]\n                predictTopDecoded[df.iloc[i]['image']] = tempList  \n\n                 \n    for x in tqdm(predictTopDecoded):\n        predictTopDecoded[x] = ' '.join(predictTopDecoded[x])\n    \n    predictions = pd.Series(predictTopDecoded).reset_index()\n    predictions.columns = ['image','predictions']\n    return predictions ","metadata":{"execution":{"iopub.status.busy":"2022-04-10T18:07:38.512567Z","iopub.execute_input":"2022-04-10T18:07:38.512902Z","iopub.status.idle":"2022-04-10T18:07:38.551353Z","shell.execute_reply.started":"2022-04-10T18:07:38.512869Z","shell.execute_reply":"2022-04-10T18:07:38.550582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def BlendWithPublic(predictions):\n    public_predictions = pd.read_csv(PUBLIC_SUBMISSION_CSV_PATH)\n    ids_without_backfin = np.load(IDS_WITHOUT_BACKFIN_PATH, allow_pickle=True)\n\n    ids2 = public_predictions[\"image\"][~public_predictions[\"image\"].isin(predictions[\"image\"])]\n\n    predictions = pd.concat(\n        [\n            predictions[~(predictions[\"image\"].isin(ids_without_backfin))],\n            public_predictions[public_predictions[\"image\"].isin(ids_without_backfin)],\n            public_predictions[public_predictions[\"image\"].isin(ids2)],\n        ]\n    )\n    predictions = predictions.drop_duplicates()\n\n    predictions.to_csv(SUBMISSION_CSV_PATH, index=False)    ","metadata":{"execution":{"iopub.status.busy":"2022-04-10T18:07:44.255069Z","iopub.execute_input":"2022-04-10T18:07:44.255614Z","iopub.status.idle":"2022-04-10T18:07:44.265733Z","shell.execute_reply.started":"2022-04-10T18:07:44.255574Z","shell.execute_reply":"2022-04-10T18:07:44.264737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SUBMISSION_CSV_PATH = \"submission.csv\"\nPUBLIC_SUBMISSION_CSV_PATH = \"../input/finsmetadata/submissionToMix.csv\"\nIDS_WITHOUT_BACKFIN_PATH = \"../input/finsmetadata/ids_without_backfin.npy\"\nusePublic_new_individ = True\npredictions = GetSubmission([allTrainData],[test_cnn_predictions], [allTrainingLabels],neighbors=100,metric='cosine', new_individual_thres=0.65, blendType=\"max\")\nBlendWithPublic(predictions)","metadata":{"execution":{"iopub.status.busy":"2022-04-10T18:07:47.118137Z","iopub.execute_input":"2022-04-10T18:07:47.118426Z","iopub.status.idle":"2022-04-10T18:24:04.475247Z","shell.execute_reply.started":"2022-04-10T18:07:47.118394Z","shell.execute_reply":"2022-04-10T18:24:04.474278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}