{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Pairwise inference\nIn this notebook, we demonstrate how to use pairwise model to predict in this competition. Please note that the inference time is much longer than pointwise method or using cosine similarity. \n\n1. I used a bert-small model pretrained with pairwise-mlm.\n2. Training with pairwise examples with negative samples randomly sampled.\n3. **Inference and predict for all the pairs for test dataset.**\n\n* [Pretrain](https://www.kaggle.com/code/yuanzhezhou/ai4code-pairwise-bertsmall-pretrain/notebook)\n* [Training](https://www.kaggle.com/yuanzhezhou/ai4code-pairwise-bertsmall-training)\n* [Inference](https://www.kaggle.com/yuanzhezhou/ai4code-pairwise-bertsmall-inference)","metadata":{}},{"cell_type":"code","source":"import json\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nfrom scipy import sparse\nfrom tqdm import tqdm\n\"\"\"\n\npd.options.display.width = 180\npd.options.display.max_colwidth = 120\n\n#BERT_PATH = \"../input/huggingface-bert-variants/distilbert-base-uncased/distilbert-base-uncased\"\n\"\"\"\ndata_dir = Path('../input/AI4Code')","metadata":{"papermill":{"duration":0.122804,"end_time":"2022-05-12T10:15:14.04297","exception":false,"start_time":"2022-05-12T10:15:13.920166","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-31T00:02:50.990501Z","iopub.execute_input":"2022-07-31T00:02:50.990768Z","iopub.status.idle":"2022-07-31T00:02:50.996528Z","shell.execute_reply.started":"2022-07-31T00:02:50.990738Z","shell.execute_reply":"2022-07-31T00:02:50.995244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUM_TRAIN = 200\n\n\ndef read_notebook(path):\n    return (\n        pd.read_json(\n            path,\n            dtype={'cell_type': 'category', 'source': 'str'})\n        .assign(id=path.stem)\n        .rename_axis('cell_id')\n    )\n\n\n\"\"\"\npaths_train = list((data_dir / 'train').glob('*.json'))[:NUM_TRAIN]\nnotebooks_train = [\n    read_notebook(path) for path in tqdm(paths_train, desc='Train NBs')\n]\ndf = (\n    pd.concat(notebooks_train)\n    .set_index('id', append=True)\n    .swaplevel()\n    .sort_index(level='id', sort_remaining=False)\n)\n\ndf\n\"\"\"","metadata":{"papermill":{"duration":82.291505,"end_time":"2022-05-12T10:16:36.365197","exception":false,"start_time":"2022-05-12T10:15:14.073692","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-31T00:02:51.025757Z","iopub.execute_input":"2022-07-31T00:02:51.026091Z","iopub.status.idle":"2022-07-31T00:02:51.034019Z","shell.execute_reply.started":"2022-07-31T00:02:51.026057Z","shell.execute_reply":"2022-07-31T00:02:51.033187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n# Get an example notebook\nnb_id = df.index.unique('id')[6]\nprint('Notebook:', nb_id)\n\nprint(\"The disordered notebook:\")\nnb = df.loc[nb_id, :]\ndisplay(nb)\nprint()\n\"\"\"","metadata":{"papermill":{"duration":0.270693,"end_time":"2022-05-12T10:16:36.882443","exception":false,"start_time":"2022-05-12T10:16:36.61175","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-31T00:02:51.050235Z","iopub.execute_input":"2022-07-31T00:02:51.050739Z","iopub.status.idle":"2022-07-31T00:02:51.056608Z","shell.execute_reply.started":"2022-07-31T00:02:51.050702Z","shell.execute_reply":"2022-07-31T00:02:51.055525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\ndf_orders = pd.read_csv(\n    data_dir / 'train_orders.csv',\n    index_col='id',\n    squeeze=True,\n).str.split()  # Split the string representation of cell_ids into a list\n\ndf_orders\n\"\"\"","metadata":{"papermill":{"duration":2.835076,"end_time":"2022-05-12T10:16:39.9675","exception":false,"start_time":"2022-05-12T10:16:37.132424","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-31T00:02:51.077873Z","iopub.execute_input":"2022-07-31T00:02:51.078350Z","iopub.status.idle":"2022-07-31T00:02:51.084551Z","shell.execute_reply.started":"2022-07-31T00:02:51.078302Z","shell.execute_reply":"2022-07-31T00:02:51.083688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#len(df_orders.loc[\"002ba502bdac45\"])","metadata":{"papermill":{"duration":0.257536,"end_time":"2022-05-12T10:16:40.472139","exception":false,"start_time":"2022-05-12T10:16:40.214603","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-31T00:02:51.101849Z","iopub.execute_input":"2022-07-31T00:02:51.102233Z","iopub.status.idle":"2022-07-31T00:02:51.109111Z","shell.execute_reply.started":"2022-07-31T00:02:51.102203Z","shell.execute_reply":"2022-07-31T00:02:51.107318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\ncell_order = df_orders.loc[nb_id]\n\nprint(\"The ordered notebook:\")\nnb.loc[cell_order, :]\n\"\"\"","metadata":{"papermill":{"duration":0.265934,"end_time":"2022-05-12T10:16:40.98571","exception":false,"start_time":"2022-05-12T10:16:40.719776","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-31T00:02:51.126597Z","iopub.execute_input":"2022-07-31T00:02:51.127023Z","iopub.status.idle":"2022-07-31T00:02:51.132806Z","shell.execute_reply.started":"2022-07-31T00:02:51.126959Z","shell.execute_reply":"2022-07-31T00:02:51.132096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_ranks(base, derived):\n    return [base.index(d) for d in derived]\n\n\"\"\"\ncell_ranks = get_ranks(cell_order, list(nb.index))\nnb.insert(0, 'rank', cell_ranks)\n\nnb\n\"\"\"","metadata":{"papermill":{"duration":0.265625,"end_time":"2022-05-12T10:16:41.501618","exception":false,"start_time":"2022-05-12T10:16:41.235993","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-31T00:02:51.150227Z","iopub.execute_input":"2022-07-31T00:02:51.150766Z","iopub.status.idle":"2022-07-31T00:02:51.158097Z","shell.execute_reply.started":"2022-07-31T00:02:51.150718Z","shell.execute_reply":"2022-07-31T00:02:51.157043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\ndf_orders_ = df_orders.to_frame().join(\n    df.reset_index('cell_id').groupby('id')['cell_id'].apply(list),\n    how='right',\n)\n\nranks = {}\nfor id_, cell_order, cell_id in df_orders_.itertuples():\n    ranks[id_] = {'cell_id': cell_id, 'rank': get_ranks(cell_order, cell_id)}\n\ndf_ranks = (\n    pd.DataFrame\n    .from_dict(ranks, orient='index')\n    .rename_axis('id')\n    .apply(pd.Series.explode)\n    .set_index('cell_id', append=True)\n)\n\ndf_ranks\n\"\"\"","metadata":{"papermill":{"duration":2.967892,"end_time":"2022-05-12T10:16:44.752979","exception":false,"start_time":"2022-05-12T10:16:41.785087","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-31T00:02:51.174145Z","iopub.execute_input":"2022-07-31T00:02:51.174601Z","iopub.status.idle":"2022-07-31T00:02:51.182483Z","shell.execute_reply.started":"2022-07-31T00:02:51.174565Z","shell.execute_reply":"2022-07-31T00:02:51.181709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df_ancestors = pd.read_csv(data_dir / 'train_ancestors.csv', index_col='id')\n#df_ancestors","metadata":{"papermill":{"duration":0.44203,"end_time":"2022-05-12T10:16:45.446006","exception":false,"start_time":"2022-05-12T10:16:45.003976","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-31T00:02:51.193306Z","iopub.execute_input":"2022-07-31T00:02:51.193630Z","iopub.status.idle":"2022-07-31T00:02:51.197819Z","shell.execute_reply.started":"2022-07-31T00:02:51.193594Z","shell.execute_reply":"2022-07-31T00:02:51.196994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df = df.reset_index().merge(df_ranks, on=[\"id\", \"cell_id\"]).merge(df_ancestors, on=[\"id\"])\n#df","metadata":{"papermill":{"duration":1.007951,"end_time":"2022-05-12T10:16:46.70626","exception":false,"start_time":"2022-05-12T10:16:45.698309","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-31T00:02:51.221894Z","iopub.execute_input":"2022-07-31T00:02:51.222656Z","iopub.status.idle":"2022-07-31T00:02:51.226544Z","shell.execute_reply.started":"2022-07-31T00:02:51.222613Z","shell.execute_reply":"2022-07-31T00:02:51.225356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df[\"pct_rank\"] = df[\"rank\"] / df.groupby(\"id\")[\"cell_id\"].transform(\"count\")\n\n#df[\"pct_rank\"].hist(bins=10)","metadata":{"papermill":{"duration":0.862186,"end_time":"2022-05-12T10:16:47.820945","exception":false,"start_time":"2022-05-12T10:16:46.958759","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-31T00:02:51.253738Z","iopub.execute_input":"2022-07-31T00:02:51.254281Z","iopub.status.idle":"2022-07-31T00:02:51.258137Z","shell.execute_reply.started":"2022-07-31T00:02:51.254245Z","shell.execute_reply":"2022-07-31T00:02:51.257276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#dict_cellid_source = dict(zip(df['cell_id'].values, df['source'].values))","metadata":{"execution":{"iopub.status.busy":"2022-07-31T00:02:51.273786Z","iopub.execute_input":"2022-07-31T00:02:51.274064Z","iopub.status.idle":"2022-07-31T00:02:51.277694Z","shell.execute_reply.started":"2022-07-31T00:02:51.274032Z","shell.execute_reply":"2022-07-31T00:02:51.276911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nimport numpy as np\nimport pandas as pd\nimport os\nimport re\n# import fasttext\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics.pairwise import cosine_similarity\nfrom nltk.stem import WordNetLemmatizer\nfrom pathlib import Path\nimport nltk\nnltk.download('wordnet')\n\nstemmer = WordNetLemmatizer()\n\ndef preprocess_text(document):\n        # Remove all the special characters\n        document = re.sub(r'\\W', ' ', str(document))\n\n        # remove all single characters\n        document = re.sub(r'\\s+[a-zA-Z]\\s+', ' ', document)\n\n        # Remove single characters from the start\n        document = re.sub(r'\\^[a-zA-Z]\\s+', ' ', document)\n\n        # Substituting multiple spaces with single space\n        document = re.sub(r'\\s+', ' ', document, flags=re.I)\n\n        # Removing prefixed 'b'\n        document = re.sub(r'^b\\s+', '', document)\n\n        # Converting to Lowercase\n        document = document.lower()\n        #return document\n\n        # Lemmatization\n        tokens = document.split()\n        tokens = [stemmer.lemmatize(word) for word in tokens]\n        tokens = [word for word in tokens if len(word) > 3]\n\n        preprocessed_text = ' '.join(tokens)\n        return preprocessed_text\n\n    \ndef preprocess_df(df):\n    \n    #This function is for processing sorce of notebook\n    #returns preprocessed dataframe\n\n    return [preprocess_text(message) for message in df.source]\n\n#df.source = df.source.apply(preprocess_text)","metadata":{"execution":{"iopub.status.busy":"2022-07-31T00:02:51.297639Z","iopub.execute_input":"2022-07-31T00:02:51.298342Z","iopub.status.idle":"2022-07-31T00:02:51.305449Z","shell.execute_reply.started":"2022-07-31T00:02:51.298305Z","shell.execute_reply":"2022-07-31T00:02:51.304561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nfrom sklearn.model_selection import GroupShuffleSplit\n\nNVALID = 0.1  # size of validation set\n\nsplitter = GroupShuffleSplit(n_splits=1, test_size=NVALID, random_state=0)\n\ntrain_ind, val_ind = next(splitter.split(df, groups=df[\"ancestor_id\"]))\n\ntrain_df = df.loc[train_ind].reset_index(drop=True)\nval_df = df.loc[val_ind].reset_index(drop=True)\n\"\"\"","metadata":{"papermill":{"duration":1.895199,"end_time":"2022-05-12T10:16:49.969199","exception":false,"start_time":"2022-05-12T10:16:48.074","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-31T00:02:51.317285Z","iopub.execute_input":"2022-07-31T00:02:51.317792Z","iopub.status.idle":"2022-07-31T00:02:51.324067Z","shell.execute_reply.started":"2022-07-31T00:02:51.317759Z","shell.execute_reply":"2022-07-31T00:02:51.323193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm.notebook import tqdm\n\ndef generate_triplet(df, mode='train'):\n  triplets = []\n  ids = df.id.unique()\n  random_drop = np.random.random(size=10000)>0.9\n  count = 0\n\n  for id, df_tmp in tqdm(df.groupby('id')):\n    df_tmp_markdown = df_tmp[df_tmp['cell_type']=='markdown']\n\n    df_tmp_code = df_tmp[df_tmp['cell_type']=='code']\n    df_tmp_code_rank = df_tmp_code['rank'].values\n    df_tmp_code_cell_id = df_tmp_code['cell_id'].values\n\n    for cell_id, rank in df_tmp_markdown[['cell_id', 'rank']].values:\n      labels = np.array([(r==(rank+1)) for r in df_tmp_code_rank]).astype('int')\n\n      for cid, label in zip(df_tmp_code_cell_id, labels):\n        count += 1\n        if label==1:\n          triplets.append( [cell_id, cid, label] )\n          # triplets.append( [cid, cell_id, label] )\n        elif mode == 'test':\n          triplets.append( [cell_id, cid, label] )\n          # triplets.append( [cid, cell_id, label] )\n        elif random_drop[count%10000]:\n          triplets.append( [cell_id, cid, label] )\n          # triplets.append( [cid, cell_id, label] )\n    \n  return triplets\n\n\n#triplets = generate_triplet(train_df)\n#val_triplets = generate_triplet(val_df, mode = 'test')","metadata":{"execution":{"iopub.status.busy":"2022-07-31T00:02:51.337404Z","iopub.execute_input":"2022-07-31T00:02:51.338241Z","iopub.status.idle":"2022-07-31T00:02:51.348832Z","shell.execute_reply.started":"2022-07-31T00:02:51.338188Z","shell.execute_reply":"2022-07-31T00:02:51.348063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#val_df.head()","metadata":{"papermill":{"duration":0.27219,"end_time":"2022-05-12T10:16:50.495502","exception":false,"start_time":"2022-05-12T10:16:50.223312","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-31T00:02:51.357294Z","iopub.execute_input":"2022-07-31T00:02:51.357796Z","iopub.status.idle":"2022-07-31T00:02:51.363519Z","shell.execute_reply.started":"2022-07-31T00:02:51.357766Z","shell.execute_reply":"2022-07-31T00:02:51.362711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from bisect import bisect\n\n\ndef count_inversions(a):\n    inversions = 0\n    sorted_so_far = []\n    for i, u in enumerate(a):\n        j = bisect(sorted_so_far, u)\n        inversions += i - j\n        sorted_so_far.insert(j, u)\n    return inversions\n\n\ndef kendall_tau(ground_truth, predictions):\n    total_inversions = 0\n    total_2max = 0  # twice the maximum possible inversions across all instances\n    for gt, pred in zip(ground_truth, predictions):\n        ranks = [gt.index(x) for x in pred]  # rank predicted order in terms of ground truth\n        total_inversions += count_inversions(ranks)\n        n = len(gt)\n        total_2max += n * (n - 1)\n    return 1 - 4 * total_inversions / total_2max","metadata":{"papermill":{"duration":0.262837,"end_time":"2022-05-12T10:16:51.011588","exception":false,"start_time":"2022-05-12T10:16:50.748751","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-31T00:02:51.373158Z","iopub.execute_input":"2022-07-31T00:02:51.373861Z","iopub.status.idle":"2022-07-31T00:02:51.381929Z","shell.execute_reply.started":"2022-07-31T00:02:51.373818Z","shell.execute_reply":"2022-07-31T00:02:51.381111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import RobertaTokenizer, RobertaConfig, RobertaModel\nimport torch.nn.functional as F\nimport torch.nn as nn\nimport torch\nfrom transformers import AutoModelWithLMHead, AutoTokenizer, AutoModel\n\nMAX_LEN = 128\n\n    \nclass MarkdownModel(nn.Module):\n    def __init__(self):\n        super(MarkdownModel, self).__init__()\n        self.distill_bert = AutoModel.from_pretrained(\"../input/codeberta-small-v1\")\n        self.top = nn.Linear(768, 1)\n\n        self.dropout = nn.Dropout(0.2)\n        \n    def forward(self, ids, mask):\n        x = self.distill_bert(ids, mask)[0]\n        x = self.dropout(x)\n        x = self.top(x[:, 0, :])\n        x = torch.sigmoid(x) \n        return x","metadata":{"papermill":{"duration":7.145711,"end_time":"2022-05-12T10:17:00.757077","exception":false,"start_time":"2022-05-12T10:16:53.611366","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-31T00:02:51.393409Z","iopub.execute_input":"2022-07-31T00:02:51.394054Z","iopub.status.idle":"2022-07-31T00:02:51.403924Z","shell.execute_reply.started":"2022-07-31T00:02:51.394013Z","shell.execute_reply":"2022-07-31T00:02:51.402685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import DataLoader, Dataset\n\n\n\nclass MarkdownDataset(Dataset):\n    \n    def __init__(self, df, max_len, mode='train'):\n        super().__init__()\n        self.df = df\n        self.max_len = max_len\n        self.tokenizer = AutoTokenizer.from_pretrained(\"../input/codeberta-small-v1\", do_lower_case=True)\n        self.mode=mode\n\n    def __getitem__(self, index):\n        row = self.df[index]\n\n        label = row[-1]\n\n        txt = dict_cellid_source[row[0]] + '[SEP]' + dict_cellid_source[row[1]]\n\n        inputs = self.tokenizer.encode_plus(\n            txt,\n            None,\n            add_special_tokens=True,\n            max_length=self.max_len,\n            padding=\"max_length\",\n            return_token_type_ids=True,\n            truncation=True\n        )\n        ids = torch.LongTensor(inputs['input_ids'])\n        mask = torch.LongTensor(inputs['attention_mask'])\n\n        return ids, mask, torch.FloatTensor([label])\n\n\n\n\n    def __len__(self):\n        return len(self.df)\n\n\n# train_ds = MarkdownDataset(triplets, max_len=MAX_LEN, mode='test')\n# val_ds = MarkdownDataset(val_triplets, max_len=MAX_LEN, mode='test')\n\n\n# train_ds[1]","metadata":{"papermill":{"duration":0.474499,"end_time":"2022-05-12T10:17:01.487031","exception":false,"start_time":"2022-05-12T10:17:01.012532","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-31T00:02:51.409390Z","iopub.execute_input":"2022-07-31T00:02:51.409913Z","iopub.status.idle":"2022-07-31T00:02:51.419565Z","shell.execute_reply.started":"2022-07-31T00:02:51.409878Z","shell.execute_reply":"2022-07-31T00:02:51.418863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def adjust_lr(optimizer, epoch):\n    if epoch < 1:\n        lr = 5e-5\n    elif epoch < 2:\n        lr = 1e-3\n    elif epoch < 5:\n        lr = 1e-4\n    else:\n        lr = 1e-5\n\n    for p in optimizer.param_groups:\n        p['lr'] = lr\n    return lr\n    \ndef get_optimizer(net):\n    optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, net.parameters()), lr=3e-4, betas=(0.9, 0.999),\n                                 eps=1e-08)\n    return optimizer\n\nBS = 128\nNW = 8\n\n# train_loader = DataLoader(train_ds, batch_size=BS, shuffle=True, num_workers=NW,\n#                           pin_memory=False, drop_last=True)\n# val_loader = DataLoader(val_ds, batch_size=BS * 8, shuffle=False, num_workers=NW,\n#                           pin_memory=False, drop_last=False)","metadata":{"papermill":{"duration":0.265988,"end_time":"2022-05-12T10:17:02.580374","exception":false,"start_time":"2022-05-12T10:17:02.314386","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-31T00:02:51.425868Z","iopub.execute_input":"2022-07-31T00:02:51.426503Z","iopub.status.idle":"2022-07-31T00:02:51.435035Z","shell.execute_reply.started":"2022-07-31T00:02:51.426469Z","shell.execute_reply":"2022-07-31T00:02:51.434287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_data(data):\n    return tuple(d.cuda() for d in data[:-1]), data[-1].cuda()\n\n\ndef validate(model, val_loader, mode='train'):\n    model.eval()\n    \n    tbar = tqdm(val_loader, file=sys.stdout)\n    \n    preds = np.zeros(len(val_loader.dataset), dtype='float32')\n    labels = []\n    count = 0\n\n    with torch.no_grad():\n        for idx, data in enumerate(tbar):\n            inputs, target = read_data(data)\n\n            pred = model(inputs[0], inputs[1]).detach().cpu().numpy().ravel()\n\n            preds[count:count+len(pred)] = pred\n            count += len(pred)\n            \n            if mode=='test':\n              labels.append(target.detach().cpu().numpy().ravel())\n    if mode=='test':\n      return preds\n    else:\n      return np.concatenate(labels), np.concatenate(preds)\n","metadata":{"papermill":{"duration":987.160977,"end_time":"2022-05-12T10:33:30.548236","exception":false,"start_time":"2022-05-12T10:17:03.387259","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-31T00:02:51.445582Z","iopub.execute_input":"2022-07-31T00:02:51.446067Z","iopub.status.idle":"2022-07-31T00:02:51.456331Z","shell.execute_reply.started":"2022-07-31T00:02:51.446020Z","shell.execute_reply":"2022-07-31T00:02:51.455527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"paths_test = list((data_dir / 'test').glob('*.json'))\nnotebooks_test = [\n    read_notebook(path) for path in tqdm(paths_test, desc='Test NBs')\n]\ntest_df = (\n    pd.concat(notebooks_test)\n    .set_index('id', append=True)\n    .swaplevel()\n    .sort_index(level='id', sort_remaining=False)\n).reset_index()\ntest_df","metadata":{"papermill":{"duration":3.156008,"end_time":"2022-05-12T10:33:49.98707","exception":false,"start_time":"2022-05-12T10:33:46.831062","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-31T00:02:51.465566Z","iopub.execute_input":"2022-07-31T00:02:51.466071Z","iopub.status.idle":"2022-07-31T00:02:51.556504Z","shell.execute_reply.started":"2022-07-31T00:02:51.466031Z","shell.execute_reply":"2022-07-31T00:02:51.555652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.source = test_df.source.apply(preprocess_text)\ndict_cellid_source = dict(zip(test_df['cell_id'].values, test_df['source'].values))","metadata":{"execution":{"iopub.status.busy":"2022-07-31T00:02:51.558626Z","iopub.execute_input":"2022-07-31T00:02:51.559188Z","iopub.status.idle":"2022-07-31T00:02:51.564156Z","shell.execute_reply.started":"2022-07-31T00:02:51.559145Z","shell.execute_reply":"2022-07-31T00:02:51.563063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df[\"rank\"] = test_df.groupby([\"id\", \"cell_type\"]).cumcount()\ntest_df[\"pred\"] = test_df.groupby([\"id\", \"cell_type\"])[\"rank\"].rank(pct=False)\ntest_df.head(30)","metadata":{"papermill":{"duration":3.580422,"end_time":"2022-05-12T10:33:56.648552","exception":false,"start_time":"2022-05-12T10:33:53.06813","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-31T00:02:51.565932Z","iopub.execute_input":"2022-07-31T00:02:51.566743Z","iopub.status.idle":"2022-07-31T00:02:51.601263Z","shell.execute_reply.started":"2022-07-31T00:02:51.566698Z","shell.execute_reply":"2022-07-31T00:02:51.600400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_triplets = generate_triplet(test_df, mode = 'test')","metadata":{"execution":{"iopub.status.busy":"2022-07-31T00:02:51.603387Z","iopub.execute_input":"2022-07-31T00:02:51.604086Z","iopub.status.idle":"2022-07-31T00:02:51.664886Z","shell.execute_reply.started":"2022-07-31T00:02:51.604046Z","shell.execute_reply":"2022-07-31T00:02:51.664020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df[\"pct_rank\"] = 0\ntest_ds = MarkdownDataset(test_triplets, max_len=MAX_LEN)\ntest_loader = DataLoader(test_ds, batch_size=128*4, shuffle=False, num_workers=NW,\n                          pin_memory=False, drop_last=False)\n\n\nimport gc \ngc.collect()\nlen(test_ds), test_ds[0]","metadata":{"papermill":{"duration":3.130783,"end_time":"2022-05-12T10:34:03.120038","exception":false,"start_time":"2022-05-12T10:33:59.989255","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-31T00:02:51.666610Z","iopub.execute_input":"2022-07-31T00:02:51.667143Z","iopub.status.idle":"2022-07-31T00:02:52.229649Z","shell.execute_reply.started":"2022-07-31T00:02:51.667092Z","shell.execute_reply":"2022-07-31T00:02:52.228813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys \n\nmodel = MarkdownModel()\nmodel = model.cuda()\nmodel.load_state_dict(torch.load('../input/model-0/my_own_small_model_75.bin'))\ny_test = validate(model, test_loader, mode='test')\n\n","metadata":{"papermill":{"duration":4.00212,"end_time":"2022-05-12T10:34:10.223448","exception":false,"start_time":"2022-05-12T10:34:06.221328","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-31T00:02:52.230906Z","iopub.execute_input":"2022-07-31T00:02:52.231293Z","iopub.status.idle":"2022-07-31T00:02:56.704590Z","shell.execute_reply.started":"2022-07-31T00:02:52.231249Z","shell.execute_reply":"2022-07-31T00:02:56.703702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_copy = y_test\npreds_copy.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-31T00:02:56.709411Z","iopub.execute_input":"2022-07-31T00:02:56.711105Z","iopub.status.idle":"2022-07-31T00:02:56.722918Z","shell.execute_reply.started":"2022-07-31T00:02:56.710908Z","shell.execute_reply":"2022-07-31T00:02:56.722267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\npred_vals = []\ncount = 0\nfor id, df_tmp in tqdm(test_df.groupby('id')):\n  df_tmp_mark = df_tmp[df_tmp['cell_type']=='markdown']\n  df_tmp_code = df_tmp[df_tmp['cell_type']!='markdown']\n  df_tmp_code_rank = df_tmp_code['rank'].rank().values\n  N_code = len(df_tmp_code_rank)\n  N_mark = len(df_tmp_mark)\n\n  preds_tmp = preds_copy[count:count+N_mark * N_code]\n\n  count += N_mark * N_code\n\n  for i in range(N_mark):\n    pred = preds_tmp[i*N_code:i*N_code+N_code]\n    #ind = np.argmax(pred)\n    \n    softmax = np.exp((pred-np.mean(pred)) *20)/np.sum(np.exp((pred-np.mean(pred)) *20)) \n\n    rank_avg = np.sum(softmax * df_tmp_code_rank)\n    #if rank_avg-df_tmp_code_rank[ind] < 0.1 and rank_avg >= rank_max:\n        #rank = rank_avg-0.1\n    pred_vals.append(rank_avg-0.1)\n\ndel model\ndel test_triplets[:]\ndel dict_cellid_source\ngc.collect()\n","metadata":{"execution":{"iopub.status.busy":"2022-07-31T00:02:56.724450Z","iopub.execute_input":"2022-07-31T00:02:56.727254Z","iopub.status.idle":"2022-07-31T00:02:57.445450Z","shell.execute_reply.started":"2022-07-31T00:02:56.727087Z","shell.execute_reply":"2022-07-31T00:02:57.444681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\npred_vals = []\ncount = 0\nfor id, df_tmp in tqdm(test_df.groupby('id')):\n    df_tmp_mark = df_tmp[df_tmp['cell_type']=='markdown']\n    df_tmp_code = df_tmp[df_tmp['cell_type']!='markdown']\n    df_tmp_code_rank = df_tmp_code['rank'].rank().values\n    N_code = len(df_tmp_code_rank)\n    N_mark = len(df_tmp_mark)\n\n    preds_tmp = preds_copy[count:count+N_mark * N_code]\n\n    count += N_mark * N_code\n\n    for i in range(N_mark):\n        pred = preds_tmp[i*N_code:i*N_code+N_code]\n        out = np.array([pro>0,5 for pro in pred])\n        \n        \n        #softmax = np.exp((pred-np.mean(pred)) *20)/np.sum(np.exp((pred-np.mean(pred)) *20)) \n        rank = np.sum(out * df_tmp_code_rank) / np.count_nonzero(out == 1)\n        \n        #rank = df_tmp_code_rank[ind]-0.5\n        pred_vals.append(rank)\n\n\ndel model\ndel test_triplets[:]\ndel dict_cellid_source\ngc.collect()\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2022-07-31T00:02:57.451051Z","iopub.execute_input":"2022-07-31T00:02:57.453253Z","iopub.status.idle":"2022-07-31T00:02:57.470643Z","shell.execute_reply.started":"2022-07-31T00:02:57.453205Z","shell.execute_reply":"2022-07-31T00:02:57.469854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#import numpy as np\n#a=np.array([1,1,3,1,4,5]).reshape(6,)\n#b=np.array([1 if i>2 else 0 for i in a])\n#c=b*a\n#print(0.1)","metadata":{"execution":{"iopub.status.busy":"2022-07-31T00:02:57.474939Z","iopub.execute_input":"2022-07-31T00:02:57.477146Z","iopub.status.idle":"2022-07-31T00:02:57.484962Z","shell.execute_reply.started":"2022-07-31T00:02:57.477100Z","shell.execute_reply":"2022-07-31T00:02:57.484218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.loc[test_df[\"cell_type\"] == \"markdown\", \"pred\"] = pred_vals\ntest_df.iloc[23:80]","metadata":{"papermill":{"duration":3.164567,"end_time":"2022-05-12T10:34:16.415308","exception":false,"start_time":"2022-05-12T10:34:13.250741","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-31T00:02:57.487377Z","iopub.execute_input":"2022-07-31T00:02:57.490226Z","iopub.status.idle":"2022-07-31T00:02:57.631680Z","shell.execute_reply.started":"2022-07-31T00:02:57.490178Z","shell.execute_reply":"2022-07-31T00:02:57.630840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df = test_df.sort_values(\"pred\").groupby(\"id\")[\"cell_id\"].apply(lambda x: \" \".join(x)).reset_index()\nsub_df.rename(columns={\"cell_id\": \"cell_order\"}, inplace=True)\nsub_df.head()","metadata":{"papermill":{"duration":3.093752,"end_time":"2022-05-12T10:34:22.827853","exception":false,"start_time":"2022-05-12T10:34:19.734101","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-31T00:02:57.636800Z","iopub.execute_input":"2022-07-31T00:02:57.639407Z","iopub.status.idle":"2022-07-31T00:02:57.680879Z","shell.execute_reply.started":"2022-07-31T00:02:57.639359Z","shell.execute_reply":"2022-07-31T00:02:57.680041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.to_csv(\"submission.csv\", index=False)","metadata":{"papermill":{"duration":3.551878,"end_time":"2022-05-12T10:34:29.528868","exception":false,"start_time":"2022-05-12T10:34:25.97699","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-31T00:02:57.686975Z","iopub.execute_input":"2022-07-31T00:02:57.691088Z","iopub.status.idle":"2022-07-31T00:02:57.704154Z","shell.execute_reply.started":"2022-07-31T00:02:57.691036Z","shell.execute_reply":"2022-07-31T00:02:57.703191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Please upvote if you find it helpful! :D","metadata":{}}]}