{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Pairwise inference\nIn this notebook, we demonstrate how to use pairwise model to predict in this competition. Please note that the inference time is much longer than pointwise method or using cosine similarity. \n\n1. I used a bert-small model pretrained with pairwise-mlm.\n2. Training with pairwise examples with negative samples randomly sampled.\n3. **Inference and predict for all the pairs for test dataset.**\n\n* [Pretrain](https://www.kaggle.com/code/yuanzhezhou/ai4code-pairwise-bertsmall-pretrain/notebook)\n* [Training](https://www.kaggle.com/yuanzhezhou/ai4code-pairwise-bertsmall-training)\n* [Inference](https://www.kaggle.com/yuanzhezhou/ai4code-pairwise-bertsmall-inference)","metadata":{}},{"cell_type":"code","source":"import json\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nfrom scipy import sparse\nfrom tqdm import tqdm\n\npd.options.display.width = 180\npd.options.display.max_colwidth = 120\n\nBERT_PATH = \"../input/huggingface-bert-variants/distilbert-base-uncased/distilbert-base-uncased\"\n\ndata_dir = Path('../input/AI4Code')","metadata":{"papermill":{"duration":0.122804,"end_time":"2022-05-12T10:15:14.04297","exception":false,"start_time":"2022-05-12T10:15:13.920166","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-14T15:59:46.528575Z","iopub.execute_input":"2022-07-14T15:59:46.531164Z","iopub.status.idle":"2022-07-14T15:59:46.654678Z","shell.execute_reply.started":"2022-07-14T15:59:46.531061Z","shell.execute_reply":"2022-07-14T15:59:46.653859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUM_TRAIN = 200\n\n\ndef read_notebook(path):\n    return (\n        pd.read_json(\n            path,\n            dtype={'cell_type': 'category', 'source': 'str'})\n        .assign(id=path.stem)\n        .rename_axis('cell_id')\n    )\n\n\npaths_train = list((data_dir / 'train').glob('*.json'))[:NUM_TRAIN]\nnotebooks_train = [\n    read_notebook(path) for path in tqdm(paths_train, desc='Train NBs')\n]\ndf = (\n    pd.concat(notebooks_train)\n    .set_index('id', append=True)\n    .swaplevel()\n    .sort_index(level='id', sort_remaining=False)\n)\n\ndf","metadata":{"papermill":{"duration":82.291505,"end_time":"2022-05-12T10:16:36.365197","exception":false,"start_time":"2022-05-12T10:15:14.073692","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-14T15:59:46.656122Z","iopub.execute_input":"2022-07-14T15:59:46.656540Z","iopub.status.idle":"2022-07-14T15:59:51.826749Z","shell.execute_reply.started":"2022-07-14T15:59:46.656499Z","shell.execute_reply":"2022-07-14T15:59:51.825671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get an example notebook\nnb_id = df.index.unique('id')[6]\nprint('Notebook:', nb_id)\n\nprint(\"The disordered notebook:\")\nnb = df.loc[nb_id, :]\ndisplay(nb)\nprint()","metadata":{"papermill":{"duration":0.270693,"end_time":"2022-05-12T10:16:36.882443","exception":false,"start_time":"2022-05-12T10:16:36.61175","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-14T15:59:51.830016Z","iopub.execute_input":"2022-07-14T15:59:51.830514Z","iopub.status.idle":"2022-07-14T15:59:51.852103Z","shell.execute_reply.started":"2022-07-14T15:59:51.830476Z","shell.execute_reply":"2022-07-14T15:59:51.851161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_orders = pd.read_csv(\n    data_dir / 'train_orders.csv',\n    index_col='id',\n    squeeze=True,\n).str.split()  # Split the string representation of cell_ids into a list\n\ndf_orders","metadata":{"papermill":{"duration":2.835076,"end_time":"2022-05-12T10:16:39.9675","exception":false,"start_time":"2022-05-12T10:16:37.132424","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-14T15:59:51.854665Z","iopub.execute_input":"2022-07-14T15:59:51.855123Z","iopub.status.idle":"2022-07-14T15:59:54.282902Z","shell.execute_reply.started":"2022-07-14T15:59:51.855089Z","shell.execute_reply":"2022-07-14T15:59:54.282086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(df_orders.loc[\"002ba502bdac45\"])","metadata":{"papermill":{"duration":0.257536,"end_time":"2022-05-12T10:16:40.472139","exception":false,"start_time":"2022-05-12T10:16:40.214603","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-14T15:59:54.284390Z","iopub.execute_input":"2022-07-14T15:59:54.284639Z","iopub.status.idle":"2022-07-14T15:59:54.292017Z","shell.execute_reply.started":"2022-07-14T15:59:54.284602Z","shell.execute_reply":"2022-07-14T15:59:54.291237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cell_order = df_orders.loc[nb_id]\n\nprint(\"The ordered notebook:\")\nnb.loc[cell_order, :]","metadata":{"papermill":{"duration":0.265934,"end_time":"2022-05-12T10:16:40.98571","exception":false,"start_time":"2022-05-12T10:16:40.719776","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-14T15:59:54.293440Z","iopub.execute_input":"2022-07-14T15:59:54.293677Z","iopub.status.idle":"2022-07-14T15:59:54.310751Z","shell.execute_reply.started":"2022-07-14T15:59:54.293646Z","shell.execute_reply":"2022-07-14T15:59:54.309977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_ranks(base, derived):\n    return [base.index(d) for d in derived]\n\ncell_ranks = get_ranks(cell_order, list(nb.index))\nnb.insert(0, 'rank', cell_ranks)\n\nnb","metadata":{"papermill":{"duration":0.265625,"end_time":"2022-05-12T10:16:41.501618","exception":false,"start_time":"2022-05-12T10:16:41.235993","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-14T15:59:54.312448Z","iopub.execute_input":"2022-07-14T15:59:54.312692Z","iopub.status.idle":"2022-07-14T15:59:54.329286Z","shell.execute_reply.started":"2022-07-14T15:59:54.312661Z","shell.execute_reply":"2022-07-14T15:59:54.328535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_orders_ = df_orders.to_frame().join(\n    df.reset_index('cell_id').groupby('id')['cell_id'].apply(list),\n    how='right',\n)\n\nranks = {}\nfor id_, cell_order, cell_id in df_orders_.itertuples():\n    ranks[id_] = {'cell_id': cell_id, 'rank': get_ranks(cell_order, cell_id)}\n\ndf_ranks = (\n    pd.DataFrame\n    .from_dict(ranks, orient='index')\n    .rename_axis('id')\n    .apply(pd.Series.explode)\n    .set_index('cell_id', append=True)\n)\n\ndf_ranks","metadata":{"papermill":{"duration":2.967892,"end_time":"2022-05-12T10:16:44.752979","exception":false,"start_time":"2022-05-12T10:16:41.785087","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-14T15:59:54.330148Z","iopub.execute_input":"2022-07-14T15:59:54.330365Z","iopub.status.idle":"2022-07-14T15:59:54.426717Z","shell.execute_reply.started":"2022-07-14T15:59:54.330337Z","shell.execute_reply":"2022-07-14T15:59:54.426040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ancestors = pd.read_csv(data_dir / 'train_ancestors.csv', index_col='id')\ndf_ancestors","metadata":{"papermill":{"duration":0.44203,"end_time":"2022-05-12T10:16:45.446006","exception":false,"start_time":"2022-05-12T10:16:45.003976","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-14T15:59:54.427871Z","iopub.execute_input":"2022-07-14T15:59:54.428169Z","iopub.status.idle":"2022-07-14T15:59:54.653538Z","shell.execute_reply.started":"2022-07-14T15:59:54.428134Z","shell.execute_reply":"2022-07-14T15:59:54.652869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df.reset_index().merge(df_ranks, on=[\"id\", \"cell_id\"]).merge(df_ancestors, on=[\"id\"])\ndf","metadata":{"papermill":{"duration":1.007951,"end_time":"2022-05-12T10:16:46.70626","exception":false,"start_time":"2022-05-12T10:16:45.698309","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-14T15:59:54.656553Z","iopub.execute_input":"2022-07-14T15:59:54.656758Z","iopub.status.idle":"2022-07-14T15:59:54.723117Z","shell.execute_reply.started":"2022-07-14T15:59:54.656734Z","shell.execute_reply":"2022-07-14T15:59:54.722305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[\"pct_rank\"] = df[\"rank\"] / df.groupby(\"id\")[\"cell_id\"].transform(\"count\")\n\ndf[\"pct_rank\"].hist(bins=10)","metadata":{"papermill":{"duration":0.862186,"end_time":"2022-05-12T10:16:47.820945","exception":false,"start_time":"2022-05-12T10:16:46.958759","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-14T15:59:54.724442Z","iopub.execute_input":"2022-07-14T15:59:54.724860Z","iopub.status.idle":"2022-07-14T15:59:54.966668Z","shell.execute_reply.started":"2022-07-14T15:59:54.724823Z","shell.execute_reply":"2022-07-14T15:59:54.966013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dict_cellid_source = dict(zip(df['cell_id'].values, df['source'].values))","metadata":{"execution":{"iopub.status.busy":"2022-07-14T15:59:54.967965Z","iopub.execute_input":"2022-07-14T15:59:54.968220Z","iopub.status.idle":"2022-07-14T15:59:54.974960Z","shell.execute_reply.started":"2022-07-14T15:59:54.968170Z","shell.execute_reply":"2022-07-14T15:59:54.974046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport re\n# import fasttext\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics.pairwise import cosine_similarity\nfrom nltk.stem import WordNetLemmatizer\nfrom pathlib import Path\nimport nltk\nnltk.download('wordnet')\n\nstemmer = WordNetLemmatizer()\n\ndef preprocess_text(document):\n        # Remove all the special characters\n        document = re.sub(r'\\W', ' ', str(document))\n\n        # remove all single characters\n        document = re.sub(r'\\s+[a-zA-Z]\\s+', ' ', document)\n\n        # Remove single characters from the start\n        document = re.sub(r'\\^[a-zA-Z]\\s+', ' ', document)\n\n        # Substituting multiple spaces with single space\n        document = re.sub(r'\\s+', ' ', document, flags=re.I)\n\n        # Removing prefixed 'b'\n        document = re.sub(r'^b\\s+', '', document)\n\n        # Converting to Lowercase\n        document = document.lower()\n        #return document\n\n        # Lemmatization\n        tokens = document.split()\n        tokens = [stemmer.lemmatize(word) for word in tokens]\n        tokens = [word for word in tokens if len(word) > 3]\n\n        preprocessed_text = ' '.join(tokens)\n        return preprocessed_text\n\n    \ndef preprocess_df(df):\n    \"\"\"\n    This function is for processing sorce of notebook\n    returns preprocessed dataframe\n    \"\"\"\n    return [preprocess_text(message) for message in df.source]\n\ndf.source = df.source.apply(preprocess_text)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T15:59:54.976550Z","iopub.execute_input":"2022-07-14T15:59:54.977264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import GroupShuffleSplit\n\nNVALID = 0.1  # size of validation set\n\nsplitter = GroupShuffleSplit(n_splits=1, test_size=NVALID, random_state=0)\n\ntrain_ind, val_ind = next(splitter.split(df, groups=df[\"ancestor_id\"]))\n\ntrain_df = df.loc[train_ind].reset_index(drop=True)\nval_df = df.loc[val_ind].reset_index(drop=True)","metadata":{"papermill":{"duration":1.895199,"end_time":"2022-05-12T10:16:49.969199","exception":false,"start_time":"2022-05-12T10:16:48.074","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm.notebook import tqdm\n\ndef generate_triplet(df, mode='train'):\n  triplets = []\n  ids = df.id.unique()\n  random_drop = np.random.random(size=10000)>0.9\n  count = 0\n\n  for id, df_tmp in tqdm(df.groupby('id')):\n    df_tmp_markdown = df_tmp[df_tmp['cell_type']=='markdown']\n\n    df_tmp_code = df_tmp[df_tmp['cell_type']=='code']\n    df_tmp_code_rank = df_tmp_code['rank'].values\n    df_tmp_code_cell_id = df_tmp_code['cell_id'].values\n\n    for cell_id, rank in df_tmp_markdown[['cell_id', 'rank']].values:\n      labels = np.array([(r==(rank+1)) for r in df_tmp_code_rank]).astype('int')\n\n      for cid, label in zip(df_tmp_code_cell_id, labels):\n        count += 1\n        if label==1:\n          triplets.append( [cell_id, cid, label] )\n          # triplets.append( [cid, cell_id, label] )\n        elif mode == 'test':\n          triplets.append( [cell_id, cid, label] )\n          # triplets.append( [cid, cell_id, label] )\n        elif random_drop[count%10000]:\n          triplets.append( [cell_id, cid, label] )\n          # triplets.append( [cid, cell_id, label] )\n    \n  return triplets\n\ntriplets = generate_triplet(train_df)\nval_triplets = generate_triplet(val_df, mode = 'test')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_df.head()","metadata":{"papermill":{"duration":0.27219,"end_time":"2022-05-12T10:16:50.495502","exception":false,"start_time":"2022-05-12T10:16:50.223312","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from bisect import bisect\n\n\ndef count_inversions(a):\n    inversions = 0\n    sorted_so_far = []\n    for i, u in enumerate(a):\n        j = bisect(sorted_so_far, u)\n        inversions += i - j\n        sorted_so_far.insert(j, u)\n    return inversions\n\n\ndef kendall_tau(ground_truth, predictions):\n    total_inversions = 0\n    total_2max = 0  # twice the maximum possible inversions across all instances\n    for gt, pred in zip(ground_truth, predictions):\n        ranks = [gt.index(x) for x in pred]  # rank predicted order in terms of ground truth\n        total_inversions += count_inversions(ranks)\n        n = len(gt)\n        total_2max += n * (n - 1)\n    return 1 - 4 * total_inversions / total_2max","metadata":{"papermill":{"duration":0.262837,"end_time":"2022-05-12T10:16:51.011588","exception":false,"start_time":"2022-05-12T10:16:50.748751","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import RobertaTokenizer, RobertaConfig, RobertaModel\nimport torch.nn.functional as F\nimport torch.nn as nn\nimport torch\nfrom transformers import AutoModelWithLMHead, AutoTokenizer, AutoModel\n\nMAX_LEN = 128\n\n    \nclass MarkdownModel(nn.Module):\n    def __init__(self):\n        super(MarkdownModel, self).__init__()\n        self.distill_bert = AutoModel.from_pretrained(\"../input/mymodelpairbertsmallpretrained/models/checkpoint-18000\")\n        self.top = nn.Linear(512, 1)\n\n        self.dropout = nn.Dropout(0.2)\n        \n    def forward(self, ids, mask):\n        x = self.distill_bert(ids, mask)[0]\n        x = self.dropout(x)\n        x = self.top(x[:, 0, :])\n        x = torch.sigmoid(x) \n        return x","metadata":{"papermill":{"duration":7.145711,"end_time":"2022-05-12T10:17:00.757077","exception":false,"start_time":"2022-05-12T10:16:53.611366","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import DataLoader, Dataset\n\n\n\nclass MarkdownDataset(Dataset):\n    \n    def __init__(self, df, max_len, mode='train'):\n        super().__init__()\n        self.df = df\n        self.max_len = max_len\n        self.tokenizer = AutoTokenizer.from_pretrained(\"../input/mymodelpairbertsmallpretrained/my_own_tokenizer\", do_lower_case=True)\n        self.mode=mode\n\n    def __getitem__(self, index):\n        row = self.df[index]\n\n        label = row[-1]\n\n        txt = dict_cellid_source[row[0]] + '[SEP]' + dict_cellid_source[row[1]]\n\n        inputs = self.tokenizer.encode_plus(\n            txt,\n            None,\n            add_special_tokens=True,\n            max_length=self.max_len,\n            padding=\"max_length\",\n            return_token_type_ids=True,\n            truncation=True\n        )\n        ids = torch.LongTensor(inputs['input_ids'])\n        mask = torch.LongTensor(inputs['attention_mask'])\n\n        return ids, mask, torch.FloatTensor([label])\n\n\n\n\n    def __len__(self):\n        return len(self.df)\n\n\n# train_ds = MarkdownDataset(triplets, max_len=MAX_LEN, mode='test')\n# val_ds = MarkdownDataset(val_triplets, max_len=MAX_LEN, mode='test')\n\n\n# train_ds[1]","metadata":{"papermill":{"duration":0.474499,"end_time":"2022-05-12T10:17:01.487031","exception":false,"start_time":"2022-05-12T10:17:01.012532","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def adjust_lr(optimizer, epoch):\n    if epoch < 1:\n        lr = 5e-5\n    elif epoch < 2:\n        lr = 1e-3\n    elif epoch < 5:\n        lr = 1e-4\n    else:\n        lr = 1e-5\n\n    for p in optimizer.param_groups:\n        p['lr'] = lr\n    return lr\n    \ndef get_optimizer(net):\n    optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, net.parameters()), lr=3e-4, betas=(0.9, 0.999),\n                                 eps=1e-08)\n    return optimizer\n\nBS = 128\nNW = 8\n\n# train_loader = DataLoader(train_ds, batch_size=BS, shuffle=True, num_workers=NW,\n#                           pin_memory=False, drop_last=True)\n# val_loader = DataLoader(val_ds, batch_size=BS * 8, shuffle=False, num_workers=NW,\n#                           pin_memory=False, drop_last=False)","metadata":{"papermill":{"duration":0.265988,"end_time":"2022-05-12T10:17:02.580374","exception":false,"start_time":"2022-05-12T10:17:02.314386","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_data(data):\n    return tuple(d.cuda() for d in data[:-1]), data[-1].cuda()\n\n\ndef validate(model, val_loader, mode='train'):\n    model.eval()\n    \n    tbar = tqdm(val_loader, file=sys.stdout)\n    \n    preds = np.zeros(len(val_loader.dataset), dtype='float32')\n    labels = []\n    count = 0\n\n    with torch.no_grad():\n        for idx, data in enumerate(tbar):\n            inputs, target = read_data(data)\n\n            pred = model(inputs[0], inputs[1]).detach().cpu().numpy().ravel()\n\n            preds[count:count+len(pred)] = pred\n            count += len(pred)\n            \n            if mode=='test':\n              labels.append(target.detach().cpu().numpy().ravel())\n    if mode=='test':\n      return preds\n    else:\n      return np.concatenate(labels), np.concatenate(preds)\n","metadata":{"papermill":{"duration":987.160977,"end_time":"2022-05-12T10:33:30.548236","exception":false,"start_time":"2022-05-12T10:17:03.387259","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"paths_test = list((data_dir / 'test').glob('*.json'))\nnotebooks_test = [\n    read_notebook(path) for path in tqdm(paths_test, desc='Test NBs')\n]\ntest_df = (\n    pd.concat(notebooks_test)\n    .set_index('id', append=True)\n    .swaplevel()\n    .sort_index(level='id', sort_remaining=False)\n).reset_index()","metadata":{"papermill":{"duration":3.156008,"end_time":"2022-05-12T10:33:49.98707","exception":false,"start_time":"2022-05-12T10:33:46.831062","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.source = test_df.source.apply(preprocess_text)\ndict_cellid_source = dict(zip(test_df['cell_id'].values, test_df['source'].values))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df[\"rank\"] = test_df.groupby([\"id\", \"cell_type\"]).cumcount()\ntest_df[\"pred\"] = test_df.groupby([\"id\", \"cell_type\"])[\"rank\"].rank(pct=False)","metadata":{"papermill":{"duration":3.580422,"end_time":"2022-05-12T10:33:56.648552","exception":false,"start_time":"2022-05-12T10:33:53.06813","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_triplets = generate_triplet(test_df, mode = 'test')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df[\"pct_rank\"] = 0\ntest_ds = MarkdownDataset(test_triplets, max_len=MAX_LEN)\ntest_loader = DataLoader(test_ds, batch_size=BS * 4, shuffle=False, num_workers=NW,\n                          pin_memory=False, drop_last=False)\n\n\nimport gc \ngc.collect()\nlen(test_ds), test_ds[0]","metadata":{"papermill":{"duration":3.130783,"end_time":"2022-05-12T10:34:03.120038","exception":false,"start_time":"2022-05-12T10:33:59.989255","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys \n\nmodel = MarkdownModel()\nmodel = model.cuda()\nmodel.load_state_dict(torch.load('../input/mymodelbertsmallpretrained120000/my_own_model.bin'))\ny_test = validate(model, test_loader, mode='test')\n\n","metadata":{"papermill":{"duration":4.00212,"end_time":"2022-05-12T10:34:10.223448","exception":false,"start_time":"2022-05-12T10:34:06.221328","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_copy = y_test","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_vals = []\ncount = 0\nfor id, df_tmp in tqdm(test_df.groupby('id')):\n  df_tmp_mark = df_tmp[df_tmp['cell_type']=='markdown']\n  df_tmp_code = df_tmp[df_tmp['cell_type']!='markdown']\n  df_tmp_code_rank = df_tmp_code['rank'].rank().values\n  N_code = len(df_tmp_code_rank)\n  N_mark = len(df_tmp_mark)\n\n  preds_tmp = preds_copy[count:count+N_mark * N_code]\n\n  count += N_mark * N_code\n\n  for i in range(N_mark):\n    pred = preds_tmp[i*N_code:i*N_code+N_code] \n\n    softmax = np.exp((pred-np.mean(pred)) *20)/np.sum(np.exp((pred-np.mean(pred)) *20)) \n\n    rank = np.sum(softmax * df_tmp_code_rank)\n    pred_vals.append(rank)\n\ndel model\ndel test_triplets[:]\ndel dict_cellid_source\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.loc[test_df[\"cell_type\"] == \"markdown\", \"pred\"] = pred_vals\n","metadata":{"papermill":{"duration":3.164567,"end_time":"2022-05-12T10:34:16.415308","exception":false,"start_time":"2022-05-12T10:34:13.250741","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df = test_df.sort_values(\"pred\").groupby(\"id\")[\"cell_id\"].apply(lambda x: \" \".join(x)).reset_index()\nsub_df.rename(columns={\"cell_id\": \"cell_order\"}, inplace=True)\nsub_df.head()","metadata":{"papermill":{"duration":3.093752,"end_time":"2022-05-12T10:34:22.827853","exception":false,"start_time":"2022-05-12T10:34:19.734101","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.to_csv(\"submission.csv\", index=False)","metadata":{"papermill":{"duration":3.551878,"end_time":"2022-05-12T10:34:29.528868","exception":false,"start_time":"2022-05-12T10:34:25.97699","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":3.030555,"end_time":"2022-05-12T10:34:35.897014","exception":false,"start_time":"2022-05-12T10:34:32.866459","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Please upvote if you find it helpful! :D","metadata":{}}]}