{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":" Thanks to previous Notebooks  \n https://www.kaggle.com/cdeotte/part-2-rapids-tfidfvectorizer-cv-0-700  \n https://www.kaggle.com/muhammad4hmed/b3-tfidf-knn-boom-p  \n https://www.kaggle.com/parthdhameliya77/pytorch-eca-nfnet-l0-image-tfidf-inference\n ","metadata":{}},{"cell_type":"markdown","source":"# 1. Pip Installation & Importing Libraries","metadata":{}},{"cell_type":"code","source":"!pip install ../input/efficientnetkerasapplications/Keras_Applications-1.0.8-py3-none-any.whl\n!pip install ../input/efficientnetkerasapplications/efficientnet-1.1.1-py3-none-any.whl\nimport sys\nsys.path.append('../input/timm-pytorch-image-models/pytorch-image-models-master')\n\nimport numpy as np\nimport pandas as pd\nimport gc\nimport matplotlib.pyplot as plt\nimport cudf\nimport cuml\nimport cupy\nfrom cuml.feature_extraction.text import TfidfVectorizer\nfrom cuml import PCA\nfrom cuml.neighbors import NearestNeighbors\nimport tensorflow as tf\nimport efficientnet.tfkeras as efn\nimport math\nfrom shutil import copyfile\ncopyfile(src = \"../input/bert-and-tokenization/tokenization.py\", dst = \"../working/tokenization.py\")\nimport tokenization\nimport tensorflow_hub as hub\n\nimport os\nimport cv2\nimport random\nfrom tqdm import tqdm\n\n# import albumentations\nfrom albumentations.pytorch.transforms import ToTensorV2\nimport albumentations as A \n\nimport torch\nimport timm\nimport torch\nimport torch.nn as nn\nfrom torch.nn import functional as F\nfrom torch.utils.data import Dataset,DataLoader\nimport fasttext as ft","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:23:52.048603Z","iopub.execute_input":"2022-08-01T10:23:52.048993Z","iopub.status.idle":"2022-08-01T10:25:00.259524Z","shell.execute_reply.started":"2022-08-01T10:23:52.048912Z","shell.execute_reply":"2022-08-01T10:25:00.258328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Basic Setup Function","metadata":{}},{"cell_type":"code","source":"# For tf.dataset\nAUTO = tf.data.experimental.AUTOTUNE\n\n# Configuration\nBATCH_SIZE = 8\nIMAGE_SIZE = [512, 512]\n# Seed\nSEED = 42\n# Verbosity\nVERBOSE = 1\n# Number of classes\nN_CLASSES = 11011","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:25:00.263831Z","iopub.execute_input":"2022-08-01T10:25:00.264125Z","iopub.status.idle":"2022-08-01T10:25:00.269152Z","shell.execute_reply.started":"2022-08-01T10:25:00.264092Z","shell.execute_reply":"2022-08-01T10:25:00.268278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# RESTRICT TENSORFLOW TO 2GB OF GPU RAM\n# SO THAT WE HAVE 14GB RAM FOR RAPIDS\nLIMIT = 2.0\ngpus = tf.config.experimental.list_physical_devices('GPU')\nif gpus:\n    try:\n        tf.config.experimental.set_virtual_device_configuration(\n            gpus[0],\n            [tf.config.experimental.VirtualDeviceConfiguration(memory_limit=1024*LIMIT)])\n        logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n        #print(len(gpus), \"Physical GPUs,\", len(logical_gpus), \"Logical GPUs\")\n    except RuntimeError as e:\n        print(e)\nprint('We will restrict TensorFlow to max %iGB GPU RAM'%LIMIT)\nprint('then RAPIDS can use %iGB GPU RAM'%(16-LIMIT))","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:25:00.271433Z","iopub.execute_input":"2022-08-01T10:25:00.272108Z","iopub.status.idle":"2022-08-01T10:25:10.301234Z","shell.execute_reply.started":"2022-08-01T10:25:00.272058Z","shell.execute_reply":"2022-08-01T10:25:10.300302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GET_CV = False\n# Flag to check ram allocations (debug)\nCHECK_SUB = False\n\ndf = cudf.read_csv('../input/shopee-product-matching/test.csv')\n# If we are comitting, replace train set for test set and dont get cv\nif len(df) > 3:\n    GET_CV = False\ndel df\n\n# Function to get our f1 score\ndef f1_score(y_true, y_pred):\n    y_true = y_true.apply(lambda x: set(x.split()))\n    y_pred = y_pred.apply(lambda x: set(x.split()))\n    intersection = np.array([len(x[0] & x[1]) for x in zip(y_true, y_pred)])\n    len_y_pred = y_pred.apply(lambda x: len(x)).values\n    len_y_true = y_true.apply(lambda x: len(x)).values\n    f1 = 2 * intersection / (len_y_pred + len_y_true)\n    return f1\n\n\n# Function to combine predictions\ndef combine_predictions(row):\n    x = np.concatenate([row['image_predictions_effn'], row['text_predictions_bert'], row['text_tfidf'], row['oof_hash']])\n    return ' '.join( np.unique(x) )\n\n# Function to read out dataset\ndef read_dataset():\n    if GET_CV:\n        df = pd.read_csv('../input/shopee-product-matching/train.csv')\n        tmp = df.groupby(['label_group'])['posting_id'].unique().to_dict()\n        df['matches'] = df['label_group'].map(tmp)\n        df['matches'] = df['matches'].apply(lambda x: ' '.join(x))\n        if CHECK_SUB:\n            df = pd.concat([df, df], axis = 0)\n            df.reset_index(drop = True, inplace = True)\n        df_cu = cudf.DataFrame(df)\n        image_paths = '../input/shopee-product-matching/train_images/' + df['image']\n    else:\n        df = pd.read_csv('../input/shopee-product-matching/test.csv')\n        df_cu = cudf.DataFrame(df)\n        image_paths = '../input/shopee-product-matching/test_images/' + df['image']\n        \n    return df, df_cu, image_paths\n\n\n# Function to decode our images\ndef decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels = 3)\n    image = tf.image.resize(image, IMAGE_SIZE)\n    image = tf.cast(image, tf.float32) / 255.0\n    return image\n\n# Function to read our test image and return image\ndef read_image(image):\n    image = tf.io.read_file(image)\n    image = decode_image(image)\n    return image\n\n# Function to get our dataset that read images\ndef get_dataset(image):\n    dataset = tf.data.Dataset.from_tensor_slices(image)\n    dataset = dataset.map(read_image, num_parallel_calls = AUTO)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\n","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:25:10.303750Z","iopub.execute_input":"2022-08-01T10:25:10.304455Z","iopub.status.idle":"2022-08-01T10:25:10.350800Z","shell.execute_reply.started":"2022-08-01T10:25:10.304390Z","shell.execute_reply":"2022-08-01T10:25:10.349865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Mish_func(torch.autograd.Function):\n    \n    \"\"\"from: https://github.com/tyunist/memory_efficient_mish_swish/blob/master/mish.py\"\"\"\n    \n    @staticmethod\n    def forward(ctx, i):\n        result = i * torch.tanh(F.softplus(i))\n        ctx.save_for_backward(i)\n        return result\n\n    @staticmethod\n    def backward(ctx, grad_output):\n        i = ctx.saved_variables[0]\n  \n        v = 1. + i.exp()\n        h = v.log() \n        grad_gh = 1./h.cosh().pow_(2)\n        \n        # Note that grad_hv * grad_vx = sigmoid(x)\n        #grad_hv = 1./v  \n        #grad_vx = i.exp()\n        \n        grad_hx = i.sigmoid()\n\n        grad_gx = grad_gh *  grad_hx #grad_hv * grad_vx \n        \n        grad_f =  torch.tanh(F.softplus(i)) + i * grad_gx \n        \n        return grad_output * grad_f \n    \nclass Mish(nn.Module):\n    def __init__(self, **kwargs):\n        super().__init__()\n        pass\n    def forward(self, input_tensor):\n        return Mish_func.apply(input_tensor)\n    \n    \ndef replace_activations(model, existing_layer, new_layer):\n    \n    \"\"\"A function for replacing existing activation layers\"\"\"\n    \n    for name, module in reversed(model._modules.items()):\n        if len(list(module.children())) > 0:\n            model._modules[name] = replace_activations(module, existing_layer, new_layer)\n\n        if type(module) == existing_layer:\n            layer_old = module\n            layer_new = new_layer\n            model._modules[name] = layer_new\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:25:10.352140Z","iopub.execute_input":"2022-08-01T10:25:10.352513Z","iopub.status.idle":"2022-08-01T10:25:10.364385Z","shell.execute_reply.started":"2022-08-01T10:25:10.352473Z","shell.execute_reply":"2022-08-01T10:25:10.363333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Arcmarginproduct class keras layer\nclass ArcMarginProduct(tf.keras.layers.Layer):\n    '''\n    Implements large margin arc distance.\n\n    Reference:\n        https://arxiv.org/pdf/1801.07698.pdf\n        https://github.com/lyakaap/Landmark2019-1st-and-3rd-Place-Solution/\n            blob/master/src/modeling/metric_learning.py\n    '''\n    def __init__(self, n_classes, s=30, m=0.50, easy_margin=False,\n                 ls_eps=0.0, **kwargs):\n\n        super(ArcMarginProduct, self).__init__(**kwargs)\n\n        self.n_classes = n_classes\n        self.s = s\n        self.m = m\n        self.ls_eps = ls_eps\n        self.easy_margin = easy_margin\n        self.cos_m = tf.math.cos(m)\n        self.sin_m = tf.math.sin(m)\n        self.th = tf.math.cos(math.pi - m)\n        self.mm = tf.math.sin(math.pi - m) * m\n        \n    def get_config(self):\n\n        config = super().get_config().copy()\n        config.update({\n            'n_classes': self.n_classes,\n            's': self.s,\n            'm': self.m,\n            'ls_eps': self.ls_eps,\n            'easy_margin': self.easy_margin,\n        })\n        return config\n\n    def build(self, input_shape):\n        super(ArcMarginProduct, self).build(input_shape[0])\n\n        self.W = self.add_weight(\n            name='W',\n            shape=(int(input_shape[0][-1]), self.n_classes),\n            initializer='glorot_uniform',\n            dtype='float32',\n            trainable=True,\n            regularizer=None)\n    \n    def call(self, inputs):\n        X, y = inputs\n        y = tf.cast(y, dtype=tf.int32)\n        cosine = tf.matmul(\n            tf.math.l2_normalize(X, axis=1),\n            tf.math.l2_normalize(self.W, axis=0)\n        )\n        sine = tf.math.sqrt(1.0 - tf.math.pow(cosine, 2))\n        phi = cosine * self.cos_m - sine * self.sin_m\n        if self.easy_margin:\n            phi = tf.where(cosine > 0, phi, cosine)\n        else:\n            phi = tf.where(cosine > self.th, phi, cosine - self.mm)\n        one_hot = tf.cast(\n            tf.one_hot(y, depth=self.n_classes),\n            dtype=cosine.dtype\n        )\n        if self.ls_eps > 0:\n            one_hot = (1 - self.ls_eps) * one_hot + self.ls_eps / self.n_classes\n\n        output = (one_hot * phi) + ((1.0 - one_hot) * cosine)\n        output *= self.s\n        return output\n    \n    \n    ","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:25:10.365896Z","iopub.execute_input":"2022-08-01T10:25:10.366587Z","iopub.status.idle":"2022-08-01T10:25:10.384989Z","shell.execute_reply.started":"2022-08-01T10:25:10.366548Z","shell.execute_reply":"2022-08-01T10:25:10.384048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    \n    img_size = 512\n    batch_size = 12\n    seed = 2020\n    \n    device = 'cuda'\n    classes = 11014\n    \n    model_name = 'eca_nfnet_l0'\n    model_path = '../input/shopee-pytorch-models/arcface_512x512_nfnet_l0 (mish).pt'\n    \n#     model_name_2 =  'tf_efficientnet_b4'\n#     model_path_2 = '../input/utils-shopee/arcface_512x512_tf_efficientnet_b4_LR.pt'\n   \n    model_name_2 = 'tf_efficientnet_b5_ns'\n    model_path_2 = '../input/shopee-pytorch-models/arcface_512x512_eff_b5_.pt'\n    \n    model_name_3 = 'resnext50_32x4d'\n    model_path_3 = '../input/shopee-pytorch-models/arcface_512x512_resnext32x4d.pt'\n    \n    scale = 30 \n    margin = 0.5\n    \n\ndef read_dataset():\n    df = pd.read_csv('../input/shopee-product-matching/test.csv')\n    df_cu = cudf.DataFrame(df)\n    image_paths = '../input/shopee-product-matching/test_images/' + df['image']\n    return df, df_cu, image_paths\n\n\ndef seed_torch(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    \nseed_torch(CFG.seed)\n\n\ndef get_test_transforms():\n\n    return A.Compose(\n        [\n            A.Resize(CFG.img_size,CFG.img_size,always_apply=True),\n            A.Normalize(),\n        ToTensorV2(p=1.0)\n        ]\n    )","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:25:10.387662Z","iopub.execute_input":"2022-08-01T10:25:10.388331Z","iopub.status.idle":"2022-08-01T10:25:10.404254Z","shell.execute_reply.started":"2022-08-01T10:25:10.388286Z","shell.execute_reply":"2022-08-01T10:25:10.403421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ShopeeDataset(Dataset):\n    def __init__(self, image_paths, transforms=None):\n\n        self.image_paths = image_paths\n        self.augmentations = transforms\n\n    def __len__(self):\n        return self.image_paths.shape[0]\n\n    def __getitem__(self, index):\n        image_path = self.image_paths[index]\n        \n        image = cv2.imread(image_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        \n        if self.augmentations:\n            augmented = self.augmentations(image=image)\n            image = augmented['image']       \n    \n        return image,torch.tensor(1)\n\n\nclass ArcMarginProduct_Image(nn.Module):\n    def __init__(self, in_features, out_features, scale=30.0, margin=0.50, easy_margin=False, ls_eps=0.0):\n        super(ArcMarginProduct_Image, self).__init__()\n        self.in_features = in_features\n        self.out_features = out_features\n        self.scale = scale\n        self.margin = margin\n        self.ls_eps = ls_eps  # label smoothing\n        self.weight = nn.Parameter(torch.FloatTensor(out_features, in_features))\n        nn.init.xavier_uniform_(self.weight)\n\n        self.easy_margin = easy_margin\n        self.cos_m = math.cos(margin)\n        self.sin_m = math.sin(margin)\n        self.th = math.cos(math.pi - margin)\n        self.mm = math.sin(math.pi - margin) * margin\n        \n    def forward(self, input, label):\n        # --------------------------- cos(theta) & phi(theta) ---------------------------\n        cosine = F.linear(F.normalize(input), F.normalize(self.weight))\n        sine = torch.sqrt(1.0 - torch.pow(cosine, 2))\n        phi = cosine * self.cos_m - sine * self.sin_m\n        if self.easy_margin:\n            phi = torch.where(cosine > 0, phi, cosine)\n        else:\n            phi = torch.where(cosine > self.th, phi, cosine - self.mm)\n        # --------------------------- convert label to one-hot ---------------------------\n        # one_hot = torch.zeros(cosine.size(), requires_grad=True, device='cuda')\n        one_hot = torch.zeros(cosine.size(), device='cuda')\n        one_hot.scatter_(1, label.view(-1, 1).long(), 1)\n        if self.ls_eps > 0:\n            one_hot = (1 - self.ls_eps) * one_hot + self.ls_eps / self.out_features\n        # -------------torch.where(out_i = {x_i if condition_i else y_i) -------------\n        output = (one_hot * phi) + ((1.0 - one_hot) * cosine)\n        output *= self.scale\n        \n        return output\n    \n\n\nclass ShopeeModel(nn.Module):\n\n    def __init__(\n        self,\n        n_classes = CFG.classes,\n        model_name = CFG.model_name,\n        #model_name_2 = CFG.model_name_2,\n        fc_dim = 512,\n        margin = CFG.margin,\n        scale = CFG.scale,\n        use_fc = True,\n        pretrained = False):\n\n\n        super(ShopeeModel,self).__init__()\n        print('Building Model Backbone for {} model'.format(model_name))\n        \n        self.backbone = timm.create_model(model_name, pretrained=pretrained)\n\n        if model_name == 'resnext50_32x4d':\n            final_in_features = self.backbone.fc.in_features\n            self.backbone.fc = nn.Identity()\n            self.backbone.global_pool = nn.Identity()\n\n        elif model_name == 'efficientnet_b3':\n            final_in_features = self.backbone.classifier.in_features\n            self.backbone.classifier = nn.Identity()\n            self.backbone.global_pool = nn.Identity()\n\n        elif model_name == 'tf_efficientnet_b5_ns':\n            final_in_features = self.backbone.classifier.in_features\n            self.backbone.classifier = nn.Identity()\n            self.backbone.global_pool = nn.Identity()\n            \n        elif model_name == 'tf_efficientnet_b4':\n            final_in_features = self.backbone.classifier.in_features\n            self.backbone.classifier = nn.Identity()\n            self.backbone.global_pool = nn.Identity()\n            \n        elif model_name == 'eca_nfnet_l0':\n            final_in_features = self.backbone.head.fc.in_features\n            self.backbone.head.fc = nn.Identity()\n            self.backbone.head.global_pool = nn.Identity()\n\n        self.pooling =  nn.AdaptiveAvgPool2d(1)\n\n        self.use_fc = use_fc\n\n        self.dropout = nn.Dropout(p=0.0)\n        self.fc = nn.Linear(final_in_features, fc_dim)\n        self.bn = nn.BatchNorm1d(fc_dim)\n        self._init_params()\n        final_in_features = fc_dim\n\n        self.final = ArcMarginProduct_Image(\n            final_in_features,\n            n_classes,\n            scale = scale,\n            margin = margin,\n            easy_margin = False,\n            ls_eps = 0.0\n        )\n\n    def _init_params(self):\n        nn.init.xavier_normal_(self.fc.weight)\n        nn.init.constant_(self.fc.bias, 0)\n        nn.init.constant_(self.bn.weight, 1)\n        nn.init.constant_(self.bn.bias, 0)\n        \n    def forward(self, image, label):\n        feature = self.extract_feat(image)\n        #logits = self.final(feature,label)\n        return feature\n\n    def extract_feat(self, x):\n        batch_size = x.shape[0]\n        x = self.backbone(x)\n        x = self.pooling(x).view(batch_size, -1)\n\n        if self.use_fc:\n            x = self.dropout(x)\n            x = self.fc(x)\n            x = self.bn(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:25:10.407657Z","iopub.execute_input":"2022-08-01T10:25:10.407989Z","iopub.status.idle":"2022-08-01T10:25:10.437456Z","shell.execute_reply.started":"2022-08-01T10:25:10.407959Z","shell.execute_reply":"2022-08-01T10:25:10.436429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Image Embedding","metadata":{}},{"cell_type":"code","source":"# Function to get 50 nearest neighbors of each image and apply a distance threshold to maximize cv\ndef get_neighbors(df, embeddings, KNN = 50, image = True):\n    model = NearestNeighbors(n_neighbors = KNN)\n    model.fit(embeddings)\n    distances, indices = model.kneighbors(embeddings)\n    \n    # Iterate through different thresholds to maximize cv, run this in interactive mode, then replace else clause with a solid threshold\n    if GET_CV:\n        if image:\n            thresholds = list(np.arange(4.5, 5.0, 0.1))\n        else:\n            thresholds = list(np.arange(31, 35, 1))\n        scores = []\n        for threshold in thresholds:\n            predictions = []\n            for k in range(embeddings.shape[0]):\n                idx = np.where(distances[k,] < threshold)[0]\n                ids = indices[k,idx]\n                posting_ids = ' '.join(df['posting_id'].iloc[ids].values)\n                predictions.append(posting_ids)\n            df['pred_matches'] = predictions\n            df['f1'] = f1_score(df['matches'], df['pred_matches'])\n            score = df['f1'].mean()\n            print(f'Our f1 score for threshold {threshold} is {score}')\n            scores.append(score)\n        thresholds_scores = pd.DataFrame({'thresholds': thresholds, 'scores': scores})\n        max_score = thresholds_scores[thresholds_scores['scores'] == thresholds_scores['scores'].max()]\n        best_threshold = max_score['thresholds'].values[0]\n        best_score = max_score['scores'].values[0]\n        print(f'Our best score is {best_score} and has a threshold {best_threshold}')\n        \n        # Use threshold\n        predictions = []\n        for k in range(embeddings.shape[0]):\n            ids = np.array([])\n            # Because we are predicting the test set that have 70K images and different label groups, confidence should be smaller\n            if image:\n                idx = np.where(distances[k,] < 3.3)[0]\n                ids = indices[k,idx]\n            else:\n                idx = np.where(distances[k,] < 20.0)[0]\n                ids = indices[k,idx]\n                if (len(idx)>1):\n                    arr = distances[k,np.where(distances[k,]<20)[0]][1:]\n                    mean = np.mean(arr)\n                    standard_deviation = np.std(arr)\n                    if(standard_deviation>0):\n                        distance_from_mean = abs(arr - mean)\n                        max_deviations = 2\n                        not_outlier = distance_from_mean < max_deviations * standard_deviation\n                        max_dist = arr[not_outlier][-1]\n                        idx = np.where(distances[k,] <= max_dist)[0]\n                        ids = indices[k,idx]\n            posting_ids = df['posting_id'].iloc[ids].values\n            predictions.append(posting_ids)\n    \n    # Because we are predicting the test set that have 70K images and different label groups, confidence should be smaller\n    else:\n        predictions = []\n        for k in tqdm(range(embeddings.shape[0])):\n            ids = np.array([])\n            if image:\n#                 idx = np.where(distances[k,] < 3.3)[0]\n                idx = np.where(distances[k,] < 4.5)[0]\n                ids = indices[k,idx]\n            else:\n                #idx = np.where(distances[k,] < 16.0)[0]\n                idx = np.where(distances[k,] < 15.0)[0]\n\n#                 idx = np.where(distances[k,] < 0.75)[0]    \n                ids = indices[k,idx]\n                if (len(idx)>1):\n                    arr = distances[k,np.where(distances[k,]<15.0)[0]][1:]\n                    #arr = distances[k,np.where(distances[k,]<0.75)[0]][1:]\n                    mean = np.mean(arr)\n                    standard_deviation = np.std(arr)\n                    if(standard_deviation>0):\n                        distance_from_mean = abs(arr - mean)\n                        max_deviations = 2\n                        not_outlier = distance_from_mean < max_deviations * standard_deviation\n                        max_dist = arr[not_outlier][-1]\n                        idx = np.where(distances[k,] <= max_dist)[0]\n                        ids = indices[k,idx]\n            \n            posting_ids = df['posting_id'].iloc[ids].values\n            predictions.append(posting_ids)\n        \n    del model, distances, indices\n    gc.collect()\n    return df, predictions","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:25:10.439551Z","iopub.execute_input":"2022-08-01T10:25:10.439929Z","iopub.status.idle":"2022-08-01T10:25:10.463259Z","shell.execute_reply.started":"2022-08-01T10:25:10.439889Z","shell.execute_reply":"2022-08-01T10:25:10.462333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_image_neighbors(df, embeddings, KNN=100, threshold=4.5, metric='minkowski'):\n    \n    if metric == 'cosine':\n        model = NearestNeighbors(n_neighbors = KNN, metric = metric)\n    else:\n        model = NearestNeighbors(n_neighbors = KNN)\n    model.fit(embeddings)\n    distances, indices = model.kneighbors(embeddings)\n    \n    predictions = []\n    for k in tqdm(range(embeddings.shape[0])):\n        idx = np.where(distances[k,] < threshold)[0]\n        ids = indices[k,idx]\n        posting_ids = df['posting_id'].iloc[ids].values\n        predictions.append(posting_ids)\n\n    return df, predictions","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:25:10.464707Z","iopub.execute_input":"2022-08-01T10:25:10.465073Z","iopub.status.idle":"2022-08-01T10:25:10.478881Z","shell.execute_reply.started":"2022-08-01T10:25:10.465034Z","shell.execute_reply":"2022-08-01T10:25:10.478036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_image_predictions(df, embeddings,threshold = 0.0):\n    \n    if len(df) > 3:\n        KNN = 50\n    else : \n        KNN = 3\n    \n    model = NearestNeighbors(n_neighbors = KNN, metric = 'cosine')\n    model.fit(embeddings)\n    distances, indices = model.kneighbors(embeddings)\n    \n    predictions = []\n    for k in tqdm(range(embeddings.shape[0])):\n        idx = np.where(distances[k,] < threshold)[0]\n        ids = indices[k,idx]\n        posting_ids = df['posting_id'].iloc[ids].values\n        predictions.append(posting_ids)\n        \n    del model, distances, indices\n    gc.collect()\n    return predictions","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:25:10.480313Z","iopub.execute_input":"2022-08-01T10:25:10.480710Z","iopub.status.idle":"2022-08-01T10:25:10.489679Z","shell.execute_reply.started":"2022-08-01T10:25:10.480672Z","shell.execute_reply":"2022-08-01T10:25:10.488856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3.1 EfficientNetB3","metadata":{}},{"cell_type":"code","source":"# Function to get the embeddings of our images with the fine-tuned model\ndef get_image_embeddings_effn(image_paths):\n    embeds = []\n    \n    margin = ArcMarginProduct(\n            n_classes = N_CLASSES, \n            s = 30, \n            m = 0.5, \n            name='head/arc_margin', \n            dtype='float32'\n            )\n\n    inp = tf.keras.layers.Input(shape = (*IMAGE_SIZE, 3), name = 'inp1')\n    label = tf.keras.layers.Input(shape = (), name = 'inp2')\n    \n    # using efficientnetB3 model\n    x = efn.EfficientNetB3(weights = None, include_top = False)(inp)\n    \n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n    x = margin([x, label])\n    output = tf.keras.layers.Softmax(dtype='float32')(x)\n    model = tf.keras.models.Model(inputs = [inp, label], outputs = [output])\n    \n    model.load_weights('../input/efficientnet/EfficientNetB3_512_42.h5')\n    \n    model = tf.keras.models.Model(inputs = model.input[0], outputs = model.layers[-4].output)\n    chunk = 5000\n    iterator = np.arange(np.ceil(len(df) / chunk))\n    for j in iterator:\n        a = int(j * chunk)\n        b = int((j + 1) * chunk)\n        image_dataset = get_dataset(image_paths[a:b])\n        image_embeddings = model.predict(image_dataset)\n        embeds.append(image_embeddings)\n    del model\n    image_embeddings = np.concatenate(embeds)\n    print(f'Our image embeddings shape is {image_embeddings.shape}')\n    del embeds\n    gc.collect()\n    return image_embeddings\n\n","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:25:10.491121Z","iopub.execute_input":"2022-08-01T10:25:10.491545Z","iopub.status.idle":"2022-08-01T10:25:10.504609Z","shell.execute_reply.started":"2022-08-01T10:25:10.491505Z","shell.execute_reply":"2022-08-01T10:25:10.503731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df, df_cu, image_paths = read_dataset()\nimage_embeddings = get_image_embeddings_effn(image_paths)\ngc.collect()             # Garbage Collector interface","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:25:10.505939Z","iopub.execute_input":"2022-08-01T10:25:10.506331Z","iopub.status.idle":"2022-08-01T10:25:22.696771Z","shell.execute_reply.started":"2022-08-01T10:25:10.506293Z","shell.execute_reply":"2022-08-01T10:25:22.695913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"extra_effn = True\nif GET_CV:\n    #df, image_predictions_effn = get_neighbors(df, image_embeddings, KNN = 25, image = True)\n    _, image_predictions_effn = get_image_neighbors(df, image_embeddings, KNN = 50,\n                                                     metric='minkowski')\nelse:\n    #df, image_predictions_effn = get_neighbors(df, image_embeddings, KNN = 2, image = True)\n    _, image_predictions_effn = get_image_neighbors(df, image_embeddings, KNN = 2,\n                                                     metric='minkowski')\n    if extra_effn:\n        image_predictions_effn_extra = get_image_predictions(df, image_embeddings,\n                                               #threshold = 0.3\n                                               #threshold = 0.18\n                                               threshold = 0.2\n                                              )\n        \n        \n        ","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:25:22.698297Z","iopub.execute_input":"2022-08-01T10:25:22.698695Z","iopub.status.idle":"2022-08-01T10:25:23.864394Z","shell.execute_reply.started":"2022-08-01T10:25:22.698653Z","shell.execute_reply":"2022-08-01T10:25:23.863460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3.2 Arcface","metadata":{}},{"cell_type":"code","source":"def get_image_embeddings_arcface(image_paths,\n                                 models,\n                                 models_path,\n                                 model_name = CFG.model_name,\n                                 dual_embed = False,\n                                 tri_embed = False):\n    \n    if dual_embed or tri_embed:\n        img_embed = []\n        \n        for index, model_name in enumerate(models):\n            embeds = []\n\n            model = ShopeeModel(model_name = model_name)\n            model.eval()\n\n            if model_name == 'eca_nfnet_l0':\n                model = replace_activations(model, torch.nn.SiLU, Mish())\n\n            #model.load_state_dict(torch.load(CFG.model_path))\n            model.load_state_dict(torch.load(models_path[index]))\n            model = model.to(CFG.device)\n\n\n            image_dataset = ShopeeDataset(image_paths=image_paths,transforms=get_test_transforms())\n            image_loader = torch.utils.data.DataLoader(\n                image_dataset,\n                batch_size=CFG.batch_size,\n                pin_memory=True,\n                drop_last=False,\n                num_workers=4\n            )\n\n            with torch.no_grad():\n                for img,label in tqdm(image_loader): \n                    img = img.cuda()\n                    label = label.cuda()\n                    feat = model(img,label)\n                    image_embeddings = feat.detach().cpu().numpy()\n                    embeds.append(image_embeddings)\n\n\n            del model\n\n            image_embeddings = np.concatenate(embeds)\n            img_embed.append(image_embeddings)\n            print(f'Our image embeddings shape is {image_embeddings.shape}')\n            del embeds\n            gc.collect()\n            \n        return img_embed\n\n\n    else:\n        embeds = []\n\n        model = ShopeeModel(model_name = model_name)\n        model.eval()\n\n        if model_name == 'eca_nfnet_l0':\n            model = replace_activations(model, torch.nn.SiLU, Mish())\n\n        model.load_state_dict(torch.load(CFG.model_path))\n        model = model.to(CFG.device)\n\n\n        image_dataset = ShopeeDataset(image_paths=image_paths,transforms=get_test_transforms())\n        image_loader = torch.utils.data.DataLoader(\n            image_dataset,\n            batch_size=CFG.batch_size,\n            pin_memory=True,\n            drop_last=False,\n            num_workers=4\n        )\n\n        with torch.no_grad():\n            for img,label in tqdm(image_loader): \n                img = img.cuda()\n                label = label.cuda()\n                feat = model(img,label)\n                image_embeddings = feat.detach().cpu().numpy()\n                embeds.append(image_embeddings)\n\n\n        del model\n\n        image_embeddings = np.concatenate(embeds)\n        print(f'Our image embeddings shape is {image_embeddings.shape}')\n        del embeds\n        gc.collect()\n        return image_embeddings","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:25:23.865826Z","iopub.execute_input":"2022-08-01T10:25:23.866183Z","iopub.status.idle":"2022-08-01T10:25:23.882021Z","shell.execute_reply.started":"2022-08-01T10:25:23.866144Z","shell.execute_reply":"2022-08-01T10:25:23.880950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_image,df_image_cu,image_paths = read_dataset()\ndf_image.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:25:23.883602Z","iopub.execute_input":"2022-08-01T10:25:23.883965Z","iopub.status.idle":"2022-08-01T10:25:23.920093Z","shell.execute_reply.started":"2022-08-01T10:25:23.883927Z","shell.execute_reply":"2022-08-01T10:25:23.919255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dual_embed = True\ntri_embed = False\nextra_arcface = True\n\nif tri_embed:\n    models_= [CFG.model_name, CFG.model_name_2, CFG.model_name_3 ]\n    models_path_= [CFG.model_path, CFG.model_path_2, CFG.model_path_3]\nelif dual_embed:\n    models_= [CFG.model_name, CFG.model_name_2 ]\n    models_path_= [CFG.model_path, CFG.model_path_2]\n    \nif dual_embed or tri_embed:\n    image_embeddings_arcface = get_image_embeddings_arcface(image_paths.values,\n                                                            models = models_,\n                                                            models_path = models_path_,\n                                                            dual_embed = dual_embed,\n                                                            tri_embed  = tri_embed\n                                                           )\n    image_embeddings_arcface_1 = image_embeddings_arcface[0]\n    image_embeddings_arcface_2 = image_embeddings_arcface[1]\n    \n    \n    image_predictions_arcface_1 = get_image_predictions(df_image, image_embeddings_arcface[0],\n                                               #threshold = 0.3\n                                               #threshold = 0.18\n                                               threshold = 0.2\n                                              )\n    image_predictions_arcface_2 = get_image_predictions(df_image, image_embeddings_arcface[1],\n                                               #threshold = 0.3\n                                               #threshold = 0.18\n                                                threshold = 0.2\n                                              )\n    if tri_embed:\n        image_embeddings_arcface_3 = image_embeddings_arcface[2]\n        image_predictions_arcface_3 = get_image_predictions(df_image, image_embeddings_arcface[2],\n                                               #threshold = 0.3\n                                               #threshold = 0.18\n                                                threshold = 0.2\n                                              )\n        \n    if extra_arcface:\n        _, image_predictions_arcface_1_extra = get_image_neighbors(df_image, image_embeddings_arcface[0], KNN = 2,\n                                                     metric='minkowski')\n        _, image_predictions_arcface_2_extra = get_image_neighbors(df_image, image_embeddings_arcface[1], KNN = 2,\n                                                     metric='minkowski')\n    \n\nelse:\n    image_embeddings_arcface = get_image_embeddings_arcface(image_paths.values)\n    image_predictions_arcface = get_image_predictions(df_image, image_embeddings_arcface,\n                                               #threshold = 0.3\n                                               #threshold = 0.18\n                                                threshold = 0.2\n                                              )","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:25:23.922227Z","iopub.execute_input":"2022-08-01T10:25:23.922763Z","iopub.status.idle":"2022-08-01T10:25:30.545829Z","shell.execute_reply.started":"2022-08-01T10:25:23.922725Z","shell.execute_reply":"2022-08-01T10:25:30.544908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. Text Embedding","metadata":{}},{"cell_type":"markdown","source":"## 4.1 Bert","metadata":{}},{"cell_type":"code","source":"# Return tokens, masks and segments from a text array or series\ndef bert_encode(texts, tokenizer, max_len=512):\n    all_tokens = []\n    all_masks = []\n    all_segments = []\n    \n    for text in texts:\n        text = tokenizer.tokenize(text)\n            \n        text = text[:max_len-2]\n        input_sequence = [\"[CLS]\"] + text + [\"[SEP]\"]\n        pad_len = max_len - len(input_sequence)\n        \n        tokens = tokenizer.convert_tokens_to_ids(input_sequence)\n        tokens += [0] * pad_len\n        pad_masks = [1] * len(input_sequence) + [0] * pad_len\n        segment_ids = [0] * max_len\n        \n        all_tokens.append(tokens)\n        all_masks.append(pad_masks)\n        all_segments.append(segment_ids)\n    \n    return np.array(all_tokens), np.array(all_masks), np.array(all_segments)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:25:30.547464Z","iopub.execute_input":"2022-08-01T10:25:30.548113Z","iopub.status.idle":"2022-08-01T10:25:30.556124Z","shell.execute_reply.started":"2022-08-01T10:25:30.548069Z","shell.execute_reply":"2022-08-01T10:25:30.555360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to get our text title embeddings using a pre-trained bert model\ndef get_text_embeddings_bert(df, max_len = 70):\n    embeds = []\n    \n    module_url = \"../input/bert-en-uncased-l24-h1024-a16-1\"\n\n    bert_layer = hub.KerasLayer(module_url, trainable = True)\n    \n    vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()\n    do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()\n    tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case)\n    text = bert_encode(df['title'].values, tokenizer, max_len = max_len)\n    \n    margin = ArcMarginProduct(\n            n_classes = 11014, \n            s = 30, \n            m = 0.5, \n            name='head/arc_margin', \n            dtype='float32'\n            )\n    input_word_ids = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32, name=\"input_word_ids\")\n    input_mask = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32, name=\"input_mask\")\n    segment_ids = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32, name=\"segment_ids\")\n    label = tf.keras.layers.Input(shape = (), name = 'label')\n\n    _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])\n    clf_output = sequence_output[:, 0, :]\n    x = margin([clf_output, label])\n    output = tf.keras.layers.Softmax(dtype='float32')(x)\n    model = tf.keras.models.Model(inputs = [input_word_ids, input_mask, segment_ids, label], outputs = [output])\n    \n    model.load_weights('../input/bert-and-tokenization/Bert_123.h5')\n    model = tf.keras.models.Model(inputs = model.input[0:3], outputs = model.layers[-4].output)\n    \n    chunk = 5000\n    iterator = np.arange(np.ceil(len(df) / chunk))\n    for j in iterator:\n        a = int(j * chunk)\n        b = int((j + 1) * chunk)\n        text_chunk = ((text[0][a:b], text[1][a:b], text[2][a:b]))\n        text_embeddings = model.predict(text_chunk, batch_size = BATCH_SIZE)\n        embeds.append(text_embeddings)\n    del model\n    text_embeddings = np.concatenate(embeds)\n    print(f'Our text embeddings shape is {text_embeddings.shape}')\n    del embeds\n    gc.collect()\n    return text_embeddings","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:25:30.558877Z","iopub.execute_input":"2022-08-01T10:25:30.559224Z","iopub.status.idle":"2022-08-01T10:25:30.575832Z","shell.execute_reply.started":"2022-08-01T10:25:30.559196Z","shell.execute_reply":"2022-08-01T10:25:30.574882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df, df_cu, image_paths = read_dataset()\ntext_embeddings = get_text_embeddings_bert(df)\ngc.collect()             # Garbage Collector interface","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:25:30.577296Z","iopub.execute_input":"2022-08-01T10:25:30.577872Z","iopub.status.idle":"2022-08-01T10:26:11.026192Z","shell.execute_reply.started":"2022-08-01T10:25:30.577836Z","shell.execute_reply":"2022-08-01T10:26:11.025329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if GET_CV:\n    df, text_predictions_bert = get_neighbors(df, text_embeddings, KNN = 25, image = False)\nelse:\n    df, text_predictions_bert = get_neighbors(df, text_embeddings, KNN = 2, image = False)\n        ","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:26:11.027734Z","iopub.execute_input":"2022-08-01T10:26:11.028115Z","iopub.status.idle":"2022-08-01T10:26:11.435832Z","shell.execute_reply.started":"2022-08-01T10:26:11.028077Z","shell.execute_reply":"2022-08-01T10:26:11.434765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4.2 Tfidf ","metadata":{}},{"cell_type":"code","source":"from cuml.feature_extraction.text import TfidfVectorizer\n\nmodel = TfidfVectorizer(stop_words=None, binary=True, max_features=25000)\ntext_embeddings2 = model.fit_transform(df_cu.title).toarray()\n\ndel model\nprint('text embeddings shape',text_embeddings2.shape)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:26:11.437405Z","iopub.execute_input":"2022-08-01T10:26:11.437792Z","iopub.status.idle":"2022-08-01T10:26:25.576097Z","shell.execute_reply.started":"2022-08-01T10:26:11.437752Z","shell.execute_reply":"2022-08-01T10:26:25.575073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_text_predictions(df_cu, text_embeddings2, max_features=25_000, threshold=0.75, PRINT_CHUNK=True):\n\n    preds = []\n    CHUNK = 1024*4\n\n    print('Finding similar titles...')\n    CTS = len(df_cu)//CHUNK\n    if len(df_cu)%CHUNK!=0: CTS += 1\n\n    for j in range( CTS ):\n\n        a = j*CHUNK\n        b = (j+1)*CHUNK\n        b = min(b,len(df_cu))\n        if PRINT_CHUNK:\n            print('chunk',a,'to',b)\n\n        # COSINE SIMILARITY DISTANCE\n        # cts = np.dot( text_embeddings, text_embeddings[a:b].T).T\n        cts = cupy.matmul(text_embeddings2, text_embeddings2[a:b].T).T\n\n        for k in range(b-a):\n            # IDX = np.where(cts[k,]>0.7)[0]\n            IDX = cupy.where(cts[k,]>0.75)[0]\n    #         IDX = cupy.where(cts[k,]>0.7)[0]\n            o = df_cu.iloc[cupy.asnumpy(IDX)].posting_id.to_pandas().values\n            preds.append(o)\n\n    del text_embeddings2\n    \n    return preds","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:26:25.579633Z","iopub.execute_input":"2022-08-01T10:26:25.579915Z","iopub.status.idle":"2022-08-01T10:26:25.589495Z","shell.execute_reply.started":"2022-08-01T10:26:25.579886Z","shell.execute_reply":"2022-08-01T10:26:25.587502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_cu['text_tfidf'] = get_text_predictions(df_cu,\n                                           text_embeddings2,\n                                          threshold=0.75\n                                          )","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:26:25.591142Z","iopub.execute_input":"2022-08-01T10:26:25.591563Z","iopub.status.idle":"2022-08-01T10:26:25.853294Z","shell.execute_reply.started":"2022-08-01T10:26:25.591506Z","shell.execute_reply":"2022-08-01T10:26:25.852279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5. Neighbor Searching","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 6. Submission","metadata":{}},{"cell_type":"code","source":"# Function to combine predictions\ndef combine_predictions(row):\n    x = np.concatenate([row['image_predictions_effn'], row['text_predictions_bert'], row['text_tfidf'], row['oof_hash']])\n    return ' '.join( np.unique(x) )","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:26:25.854730Z","iopub.execute_input":"2022-08-01T10:26:25.855074Z","iopub.status.idle":"2022-08-01T10:26:25.861088Z","shell.execute_reply.started":"2022-08-01T10:26:25.855038Z","shell.execute_reply":"2022-08-01T10:26:25.860272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to combine predictions\ndef combine_predictions_oof(row):\n    x = np.concatenate([row['image_predictions_effn'], row['text_predictions_bert'],\n                        row['text_tfidf'],\n                        row['oof_hash'],\n                        row['image_arcface']\n                       ])\n    return ' '.join( np.unique(x) )\n\n# Function to combine predictions\ndef combine_predictions_dual_embed(row):\n    x = np.concatenate([row['image_predictions_effn'], row['text_predictions_bert'],\n                        row['text_tfidf'],\n                        row['oof_hash'],\n                        row['image_arcface_1'],\n                        row['image_arcface_2']\n                       ])\n    return ' '.join( np.unique(x) )\n\ndef combine_predictions_tri_embed(row):\n    x = np.concatenate([row['image_predictions_effn'], row['text_predictions_bert'],\n                        row['text_tfidf'],\n                        row['oof_hash'],\n                        row['image_arcface_1'],\n                        row['image_arcface_2'],\n                        row['image_arcface_3'],\n                        \n                       ])\n    return ' '.join( np.unique(x) )\n\ndef combine_predictions_effn_and_dua_embed_extras(row):\n    x = np.concatenate([row['image_predictions_effn'],\n                        row['image_predictions_effn_extra'],\n                        row['text_predictions_bert'],\n                        row['text_tfidf'],\n                        row['oof_hash'],\n                        row['image_arcface_1'],\n                        row['image_arcface_2'],\n                        row['image_arcface_1_extra'],\n                        row['image_arcface_2_extra'],\n                        \n                       ])\n    return ' '.join( np.unique(x) )","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:26:25.862573Z","iopub.execute_input":"2022-08-01T10:26:25.863183Z","iopub.status.idle":"2022-08-01T10:26:25.874708Z","shell.execute_reply.started":"2022-08-01T10:26:25.863144Z","shell.execute_reply":"2022-08-01T10:26:25.873714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_cu","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:26:25.875836Z","iopub.execute_input":"2022-08-01T10:26:25.876108Z","iopub.status.idle":"2022-08-01T10:26:25.911819Z","shell.execute_reply.started":"2022-08-01T10:26:25.876081Z","shell.execute_reply":"2022-08-01T10:26:25.911029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Concatenate image predctions with text predictions\ntmp = df.groupby('image_phash').posting_id.agg('unique').to_dict()\ndf['oof_hash'] = df.image_phash.map(tmp)\n\nif GET_CV:\n    df['image_predictions_effn'] = image_predictions_effn\n    df['text_prediction_berts'] = text_predictions_bert\n    df['text_tfidf'] = df_cu['text_tfidf'].to_pandas().values\n    df['pred_matches'] = df.apply(combine_predictions, axis = 1)\n    df['f1'] = f1_score(df['matches'], df['pred_matches'])\n    score = df['f1'].mean()\n    print(f'Our final f1 cv score is {score}')\n    df['matches'] = df['pred_matches']\n    df[['posting_id', 'matches']].to_csv('submission.csv', index = False)\nelse:\n    \n        df['image_predictions_effn'] = image_predictions_effn\n        df['text_tfidf'] = df_cu['text_tfidf'].to_pandas().values\n        df['text_predictions_bert'] = text_predictions_bert\n        if dual_embed or tri_embed:\n            \n            df['image_arcface_1']=image_predictions_arcface_1\n            df['image_arcface_2']=image_predictions_arcface_2\n            \n            if tri_embed:\n                df['image_arcface_3']=image_predictions_arcface_3\n                df['matches'] = df.apply(combine_predictions_tri_embed, axis = 1)\n                \n            elif extra_effn and extra_arcface:\n                df['image_predictions_effn_extra'] = image_predictions_effn_extra\n                df['image_arcface_1_extra']=image_predictions_arcface_1_extra\n                df['image_arcface_2_extra']=image_predictions_arcface_2_extra\n                df['matches'] = df.apply(combine_predictions_effn_and_dua_embed_extras, axis = 1)\n                print(\"extra effn and exrta arcface for dual embed situation\")\n                \n            else:\n                df['matches'] = df.apply(combine_predictions_dual_embed, axis = 1)\n        else:\n            df['image_arcface']=image_predictions_arcface\n            df['matches'] = df.apply(combine_predictions_oof, axis = 1)\n\n        \n        df[['posting_id', 'matches']].to_csv('submission.csv', index = False)\n        \n   ","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:26:25.912973Z","iopub.execute_input":"2022-08-01T10:26:25.913248Z","iopub.status.idle":"2022-08-01T10:26:25.952406Z","shell.execute_reply.started":"2022-08-01T10:26:25.913220Z","shell.execute_reply":"2022-08-01T10:26:25.951559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:26:25.954176Z","iopub.execute_input":"2022-08-01T10:26:25.954719Z","iopub.status.idle":"2022-08-01T10:26:25.984178Z","shell.execute_reply.started":"2022-08-01T10:26:25.954678Z","shell.execute_reply":"2022-08-01T10:26:25.983282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv('submission.csv')\nsub.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:26:25.985506Z","iopub.execute_input":"2022-08-01T10:26:25.986029Z","iopub.status.idle":"2022-08-01T10:26:25.999844Z","shell.execute_reply.started":"2022-08-01T10:26:25.985989Z","shell.execute_reply":"2022-08-01T10:26:25.998938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}