{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import json\nfrom pathlib import Path\nimport random\nimport os\nimport sys\n\nimport numpy as np\nimport pandas as pd\nfrom scipy import sparse\nfrom tqdm import tqdm\nimport re\nimport string\nfrom transformers import AutoModel, AutoTokenizer\nfrom torch.utils.data import DataLoader, Dataset\nimport torch.nn.functional as F\nimport torch.nn as nn\nimport torch\n\ntotal_max_len = 512\ndata_dir = Path('../input/AI4Code')\nmodel_path = \"../input/graphcodebert-base-model\"\nckpt_path = \"../input/ai4codeckpt/model_4.bin\"\ntokenizer_path = \"../input/graphcodebert-base-tokenizer\"","metadata":{"papermill":{"duration":0.11544,"end_time":"2022-07-18T14:28:22.002783","exception":false,"start_time":"2022-07-18T14:28:21.887343","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-05T08:41:24.660988Z","iopub.execute_input":"2022-08-05T08:41:24.662457Z","iopub.status.idle":"2022-08-05T08:41:26.871011Z","shell.execute_reply.started":"2022-08-05T08:41:24.662336Z","shell.execute_reply":"2022-08-05T08:41:26.869943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    \nseed_everything(42)","metadata":{"papermill":{"duration":0.018661,"end_time":"2022-07-18T14:28:22.034253","exception":false,"start_time":"2022-07-18T14:28:22.015592","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-05T08:41:26.872969Z","iopub.execute_input":"2022-08-05T08:41:26.873782Z","iopub.status.idle":"2022-08-05T08:41:26.884381Z","shell.execute_reply.started":"2022-08-05T08:41:26.873743Z","shell.execute_reply":"2022-08-05T08:41:26.883254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import pipeline\nlang_detection= pipeline(model=\"../input/ai4codehuggingfacemodels/xlm-roberta-base-language-detection\",task=\"text-classification\",device=0)\n# translator_pt = pipeline(model=\"../input/huggingface-models-ai4code/opus-mt-cpp-en\",task=\"translation_pt_to_en\",device=0)\ntranslator_ru = pipeline(model=\"../input/ai4codehuggingfacemodels/wmt19-ru-en\",task=\"translation_ru_to_en\",device=0)\n#translator_jap = pipeline(model=\"../input/ai4codehuggingfacemodels/opus-mt-ja-en\",task=\"translation_ja_to_en\",device=0)\n\n# # translator_tr = pipeline(model=\"../input/huggingface-models-ai4code/opus-mt-tr-en\",task=\"translation_tr_to_en\",device=0)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T08:41:26.886061Z","iopub.execute_input":"2022-08-05T08:41:26.886658Z","iopub.status.idle":"2022-08-05T08:42:15.834101Z","shell.execute_reply.started":"2022-08-05T08:41:26.886621Z","shell.execute_reply":"2022-08-05T08:42:15.833147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_notebook(path):\n    return (\n        pd.read_json(\n            path,\n            dtype={'cell_type': 'category', 'source': 'str'})\n        .assign(id=path.stem)\n        .rename_axis('cell_id')\n    )\n\npaths_test = list((data_dir / 'test').glob('*.json'))\n#paths_test = random.sample(paths_test,1000)\nnotebooks_test = [\n    read_notebook(path) for path in tqdm(paths_test, desc='Test NBs')\n]\ntest_df = (\n    pd.concat(notebooks_test)\n    .set_index('id', append=True)\n    .swaplevel()\n    .sort_index(level='id', sort_remaining=False)\n).reset_index()\n","metadata":{"papermill":{"duration":0.102206,"end_time":"2022-07-18T14:28:22.14876","exception":false,"start_time":"2022-07-18T14:28:22.046554","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-05T08:42:15.836575Z","iopub.execute_input":"2022-08-05T08:42:15.836939Z","iopub.status.idle":"2022-08-05T08:42:15.902694Z","shell.execute_reply.started":"2022-08-05T08:42:15.836900Z","shell.execute_reply":"2022-08-05T08:42:15.901789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def language_detection(df):\n    lang_json = {\"id\":[] , \"lang\" : []}\n    markdown_cells = df[df['cell_type']==\"markdown\"]\n    grouped = markdown_cells.groupby(\"id\").sum().reset_index()\n    num_nbs = len(grouped)\n    for idx , row in tqdm(grouped.iterrows(),total=num_nbs):\n        row_lang = lang_detection(row['source'][:512])[0]['label']\n        lang_json['id'].append(row['id'])\n        lang_json['lang'].append(row_lang)\n    test_df = pd.merge(df,pd.DataFrame(lang_json))\n    return test_df\n\n\n# def translate_markdown(df):\n#     df_markdown = df[df['cell_type']==\"markdown\"]\n#     pt_markdown = df_markdown[df_markdown['lang']==\"pt\"]\n#     pt_markdown['source'] = pt_markdown['source'].apply(lambda x : x[:512])\n#     if len(pt_markdown) != 0:\n#         pt_markdown_trans = translator_pt(pt_markdown['source'].tolist())\n#         trans_json = {\"id\":pt_markdown['id'].tolist(),\"cell_id\":pt_markdown['cell_id'].tolist(),'source':pt_markdown_trans}\n#         translated_markdown = pd.DataFrame(pd.DataFrame(trans_json).set_index([\"id\",\"cell_id\"])[\"source\"].apply(lambda x : x['translation_text'])).reset_index()\n#         test_df_markdown = df[((df['cell_type']=='markdown' )& (df['lang']==\"pt\"))].drop(\"source\",axis=1)\n#         translated_df_markdown = pd.merge(test_df_markdown.reset_index(drop=True),translated_markdown).set_index(test_df_markdown.index)\n#         df.update(translated_df_markdown)\n#     return df\n        \n    \ndef translate(df):\n    md_cells = df[df['cell_type']==\"markdown\"]\n    russian_md_cells = md_cells[md_cells['lang']==\"ru\"]\n    #japanese_md_cells = md_cells[md_cells['lang']==\"ja\"]\n    \n    russian_md_cells['source'] = russian_md_cells['source'].apply(lambda x : x[:512])\n    #japanese_md_cells['source'] = japanese_md_cells['source'].apply(lambda x : x[:512])\n\n    \n  \n    #ru and jp\n    if (len(russian_md_cells)!=0) :#& (len(japanese_md_cells) != 0 ):\n        \n        #ru\n        ru_translator = translator_ru(russian_md_cells['source'].tolist())\n        ru_source_df = pd.DataFrame(ru_translator)\n        ru_source_df.columns = [\"source\"]\n        ru_source_df.set_index(russian_md_cells.index,inplace=True)\n        test_df.loc[(test_df.lang == \"ru\") & (test_df.cell_type ==\"markdown\"),'source'] = ru_source_df['source']\n        \n        \n#         #jp\n#         jp_translator = translator_jap(japanese_md_cells['source'].tolist())\n#         jp_source_df = pd.DataFrame(jp_translator)\n#         jp_source_df.columns = [\"source\"]\n#         jp_source_df.set_index(japanese_md_cells.index,inplace=True)\n#         test_df.loc[(test_df.lang == \"ja\") & (test_df.cell_type ==\"markdown\"),'source'] = jp_source_df['source']\n        \n#     #japanes only\n#     elif (len(russian_md_cells)== 0 )& (len(japanese_md_cells) != 0) :\n#         #jp\n#         jp_translator = translator_jap(japanese_md_cells['source'].tolist())\n#         jp_source_df = pd.DataFrame(jp_translator)\n#         jp_source_df.columns = [\"source\"]\n#         jp_source_df.set_index(japanese_md_cells.index,inplace=True)\n#         test_df.loc[(test_df.lang == \"ja\") & (test_df.cell_type ==\"markdown\"),'source'] = jp_source_df['source']\n\n\n\n        \n#     #russian only\n#     elif (len(russian_md_cells)!= 0) & (len(japanese_md_cells) == 0 ):\n#         ru_translator = translator_ru(russian_md_cells['source'].tolist())\n#         ru_source_df = pd.DataFrame(ru_translator)\n#         ru_source_df.columns = [\"source\"]\n#         ru_source_df.set_index(russian_md_cells.index,inplace=True)\n#         test_df.loc[(test_df.lang == \"ru\") & (test_df.cell_type ==\"markdown\"),'source'] = ru_source_df['source']\n\n        \n#     for idx , row in tqdm(md_cells.iterrows(),total=len_md):\n        \n#         if row['lang'] == \"ru\":\n#             translated_text = translator_ru(row['source'][:128])[0]['translation_text']\n#             test_df.loc[idx]['source']  = translated_text\n            \n#         elif row['lang'] == \"ja\":\n#             translated_text = translator_jap(row['source'][:128])[0]['translation_text']\n#             test_df.loc[idx]['source']  = translated_text\n        ","metadata":{"execution":{"iopub.status.busy":"2022-08-05T08:42:15.905360Z","iopub.execute_input":"2022-08-05T08:42:15.906005Z","iopub.status.idle":"2022-08-05T08:42:15.918006Z","shell.execute_reply.started":"2022-08-05T08:42:15.905966Z","shell.execute_reply":"2022-08-05T08:42:15.916870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntest_df = language_detection(test_df)\ntranslate(test_df)\n# test_df = language_detection(test_df)\n# test_df = translate_markdown(test_df)\ndel lang_detection,translator_ru","metadata":{"execution":{"iopub.status.busy":"2022-08-05T08:42:15.919705Z","iopub.execute_input":"2022-08-05T08:42:15.920358Z","iopub.status.idle":"2022-08-05T08:42:17.119169Z","shell.execute_reply.started":"2022-08-05T08:42:15.920320Z","shell.execute_reply":"2022-08-05T08:42:17.118159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df[\"rank\"] = test_df.groupby([\"id\", \"cell_type\"]).cumcount()\ntest_df[\"pred\"] = test_df.groupby([\"id\", \"cell_type\"])[\"rank\"].rank(pct=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T08:42:17.120820Z","iopub.execute_input":"2022-08-05T08:42:17.121768Z","iopub.status.idle":"2022-08-05T08:42:17.135842Z","shell.execute_reply.started":"2022-08-05T08:42:17.121729Z","shell.execute_reply":"2022-08-05T08:42:17.134809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df","metadata":{"execution":{"iopub.status.busy":"2022-08-05T08:42:17.137216Z","iopub.execute_input":"2022-08-05T08:42:17.137944Z","iopub.status.idle":"2022-08-05T08:42:17.161841Z","shell.execute_reply.started":"2022-08-05T08:42:17.137907Z","shell.execute_reply":"2022-08-05T08:42:17.160961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Additional code cells\ndef clean_code(cell):\n    return str(cell).replace(\"\\\\n\", \"\\n\")\n\n\ndef sample_cells(cells, n):\n    cells = [clean_code(cell) for cell in cells]\n    if n >= len(cells):\n        return [cell[:200] for cell in cells]\n    else:\n        results = []\n        step = len(cells) / n\n        idx = 0\n        while int(np.round(idx)) < len(cells):\n            results.append(cells[int(np.round(idx))])\n            idx += step\n        assert cells[0] in results\n        if cells[-1] not in results:\n            results[-1] = cells[-1]\n        return results\n\n\ndef get_features(df):\n    features = dict()\n    df = df.sort_values(\"rank\").reset_index(drop=True)\n    for idx, sub_df in tqdm(df.groupby(\"id\")):\n        features[idx] = dict()\n        total_md = sub_df[sub_df.cell_type == \"markdown\"].shape[0]\n        code_sub_df = sub_df[sub_df.cell_type == \"code\"]\n        total_code = code_sub_df.shape[0]\n        codes = sample_cells(code_sub_df.source.values, 20)\n        features[idx][\"total_code\"] = total_code\n        features[idx][\"total_md\"] = total_md\n        features[idx][\"codes\"] = codes\n    return features","metadata":{"papermill":{"duration":0.044369,"end_time":"2022-07-18T14:28:22.222327","exception":false,"start_time":"2022-07-18T14:28:22.177958","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-05T08:42:17.162998Z","iopub.execute_input":"2022-08-05T08:42:17.163777Z","iopub.status.idle":"2022-08-05T08:42:17.174737Z","shell.execute_reply.started":"2022-08-05T08:42:17.163749Z","shell.execute_reply":"2022-08-05T08:42:17.173748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_fts = get_features(test_df)","metadata":{"papermill":{"duration":0.034506,"end_time":"2022-07-18T14:28:22.279762","exception":false,"start_time":"2022-07-18T14:28:22.245256","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-05T08:42:17.178774Z","iopub.execute_input":"2022-08-05T08:42:17.179065Z","iopub.status.idle":"2022-08-05T08:42:17.198418Z","shell.execute_reply.started":"2022-08-05T08:42:17.179011Z","shell.execute_reply":"2022-08-05T08:42:17.197455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MarkdownModel(nn.Module):\n    def __init__(self, model_path):\n        super(MarkdownModel, self).__init__()\n        self.model = AutoModel.from_pretrained(model_path)\n        self.top = nn.Linear(769, 1)\n        \n    def forward(self, ids, mask, fts):\n        x = self.model(ids, mask)[0]\n        x = self.top(torch.cat((x[:, 0, :], fts),1))\n        return x\n\nclass MarkdownDataset(Dataset):\n\n    def __init__(self, df, tokenizer_path, total_max_len, md_max_len, fts):\n        super().__init__()\n        self.df = df.reset_index(drop=True)\n        self.md_max_len = md_max_len\n        self.total_max_len = total_max_len  # maxlen allowed by model config\n        self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_path)\n        self.fts = fts\n\n    def __getitem__(self, index):\n        row = self.df.iloc[index]\n\n        inputs = self.tokenizer.encode_plus(\n            row.source,\n            None,\n            add_special_tokens=True,\n            max_length=self.md_max_len,\n            padding=\"max_length\",\n            return_token_type_ids=True,\n            truncation=True\n        )\n        code_inputs = self.tokenizer.batch_encode_plus(\n            [str(x) for x in self.fts[row.id][\"codes\"]],\n            add_special_tokens=True,\n            max_length=23,\n            padding=\"max_length\",\n            truncation=True\n        )\n        n_md = self.fts[row.id][\"total_md\"]\n        n_code = self.fts[row.id][\"total_md\"]\n        if n_md + n_code == 0:\n            fts = torch.FloatTensor([0])\n        else:\n            fts = torch.FloatTensor([n_md / (n_md + n_code)])\n\n        ids = inputs['input_ids']\n        for x in code_inputs['input_ids']:\n            ids.extend(x[:-1])\n        ids = ids[:self.total_max_len]\n        if len(ids) != self.total_max_len:\n            ids = ids + [self.tokenizer.pad_token_id, ] * (self.total_max_len - len(ids))\n        ids = torch.LongTensor(ids)\n\n        mask = inputs['attention_mask']\n        for x in code_inputs['attention_mask']:\n            mask.extend(x[:-1])\n        mask = mask[:self.total_max_len]\n        if len(mask) != self.total_max_len:\n            mask = mask + [self.tokenizer.pad_token_id, ] * (self.total_max_len - len(mask))\n        mask = torch.LongTensor(mask)\n\n        assert len(ids) == self.total_max_len\n\n        return ids, mask, fts, torch.FloatTensor([row.pct_rank])\n\n    def __len__(self):\n        return self.df.shape[0]","metadata":{"papermill":{"duration":7.645045,"end_time":"2022-07-18T14:28:29.938891","exception":false,"start_time":"2022-07-18T14:28:22.293846","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-05T08:42:17.200049Z","iopub.execute_input":"2022-08-05T08:42:17.200712Z","iopub.status.idle":"2022-08-05T08:42:17.216423Z","shell.execute_reply.started":"2022-08-05T08:42:17.200673Z","shell.execute_reply":"2022-08-05T08:42:17.215145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_data(data):\n    return tuple(d.cuda() for d in data[:-1]), data[-1].cuda()\n\n\ndef validate(model, val_loader):\n    model.eval()\n    \n    tbar = tqdm(val_loader, file=sys.stdout)\n    \n    preds = []\n    labels = []\n\n    with torch.no_grad():\n        for idx, data in enumerate(tbar):\n            inputs, target = read_data(data)\n\n            pred = model(*inputs)\n\n            preds.append(pred.detach().cpu().numpy().ravel())\n            labels.append(target.detach().cpu().numpy().ravel())\n    \n    return np.concatenate(labels), np.concatenate(preds)\n\ndef predict(model_path, ckpt_path, tokenizer_path):\n    model = MarkdownModel(model_path)\n    model = model.to('cuda:0')\n    model.eval()\n    model.load_state_dict(torch.load(ckpt_path, map_location='cuda:0'))\n    BS = 8\n    NW = 2\n    MAX_LEN = 64\n    test_df[\"pct_rank\"] = 0\n    test_ds = MarkdownDataset(test_df[test_df[\"cell_type\"] == \"markdown\"].reset_index(drop=True), \n                              md_max_len=64,\n                              tokenizer_path = tokenizer_path,\n                              total_max_len = total_max_len,\n                              fts=test_fts)\n    test_loader = DataLoader(test_ds, batch_size=BS, \n                             shuffle=False,\n                             num_workers=NW,\n                             pin_memory=False,\n                             drop_last=False)\n    _, y_test = validate(model, test_loader)\n    return y_test","metadata":{"papermill":{"duration":0.029813,"end_time":"2022-07-18T14:28:29.98533","exception":false,"start_time":"2022-07-18T14:28:29.955517","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-05T08:42:17.218041Z","iopub.execute_input":"2022-08-05T08:42:17.218691Z","iopub.status.idle":"2022-08-05T08:42:17.230852Z","shell.execute_reply.started":"2022-08-05T08:42:17.218654Z","shell.execute_reply":"2022-08-05T08:42:17.229953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_test = predict(model_path, ckpt_path, tokenizer_path)","metadata":{"papermill":{"duration":17.057781,"end_time":"2022-07-18T14:28:47.098909","exception":false,"start_time":"2022-07-18T14:28:30.041128","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-05T08:42:17.232376Z","iopub.execute_input":"2022-08-05T08:42:17.233002Z","iopub.status.idle":"2022-08-05T08:42:28.710601Z","shell.execute_reply.started":"2022-08-05T08:42:17.232966Z","shell.execute_reply":"2022-08-05T08:42:28.709422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.loc[test_df[\"cell_type\"] == \"markdown\", \"pred\"] = y_test","metadata":{"papermill":{"duration":0.025652,"end_time":"2022-07-18T14:28:47.180736","exception":false,"start_time":"2022-07-18T14:28:47.155084","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-05T08:42:28.712408Z","iopub.execute_input":"2022-08-05T08:42:28.714278Z","iopub.status.idle":"2022-08-05T08:42:28.721719Z","shell.execute_reply.started":"2022-08-05T08:42:28.714236Z","shell.execute_reply":"2022-08-05T08:42:28.720740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df = test_df.sort_values(\"pred\").groupby(\"id\")[\"cell_id\"].apply(lambda x: \" \".join(x)).reset_index()\nsub_df.rename(columns={\"cell_id\": \"cell_order\"}, inplace=True)\nsub_df.head()","metadata":{"papermill":{"duration":0.038273,"end_time":"2022-07-18T14:28:47.234777","exception":false,"start_time":"2022-07-18T14:28:47.196504","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-05T08:42:28.723213Z","iopub.execute_input":"2022-08-05T08:42:28.723805Z","iopub.status.idle":"2022-08-05T08:42:28.743430Z","shell.execute_reply.started":"2022-08-05T08:42:28.723768Z","shell.execute_reply":"2022-08-05T08:42:28.742149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T08:42:28.745278Z","iopub.execute_input":"2022-08-05T08:42:28.746695Z","iopub.status.idle":"2022-08-05T08:42:28.755961Z","shell.execute_reply.started":"2022-08-05T08:42:28.746657Z","shell.execute_reply":"2022-08-05T08:42:28.755020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import json\n# from pathlib import Path\n\n# import numpy as np\n# import pandas as pd\n# from scipy import sparse\n# from tqdm import tqdm\n\n# pd.options.display.width = 180\n# pd.options.display.max_colwidth = 120\n\n# BERT_PATH = \"../input/huggingface-bert-variants/distilbert-base-uncased/distilbert-base-uncased\"\n\n# data_dir = Path('../input/AI4Code')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# NUM_TRAIN = 200\n\n\n# def read_notebook(path):\n#     return (\n#         pd.read_json(\n#             path,\n#             dtype={'cell_type': 'category', 'source': 'str'})\n#         .assign(id=path.stem)\n#         .rename_axis('cell_id')\n#     )\n\n\n# paths_train = list((data_dir / 'train').glob('*.json'))[:NUM_TRAIN]\n# notebooks_train = [\n#     read_notebook(path) for path in tqdm(paths_train, desc='Train NBs')\n# ]\n# df = (\n#     pd.concat(notebooks_train)\n#     .set_index('id', append=True)\n#     .swaplevel()\n#     .sort_index(level='id', sort_remaining=False)\n# )\n\n# df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Get an example notebook\n# nb_id = df.index.unique('id')[6]\n# print('Notebook:', nb_id)\n\n# print(\"The disordered notebook:\")\n# nb = df.loc[nb_id, :]\n# display(nb)\n# print()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_orders = pd.read_csv(\n#     data_dir / 'train_orders.csv',\n#     index_col='id',\n#     squeeze=True,\n# ).str.split()  # Split the string representation of cell_ids into a list\n\n# df_orders","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# cell_order = df_orders.loc[nb_id]\n\n# print(\"The ordered notebook:\")\n# nb.loc[cell_order, :]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def get_ranks(base, derived):\n#     return [base.index(d) for d in derived]\n\n# cell_ranks = get_ranks(cell_order, list(nb.index))\n# nb.insert(0, 'rank', cell_ranks)\n\n# nb","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_orders_ = df_orders.to_frame().join(\n#     df.reset_index('cell_id').groupby('id')['cell_id'].apply(list),\n#     how='right',\n# )\n\n# ranks = {}\n# for id_, cell_order, cell_id in df_orders_.itertuples():\n#     ranks[id_] = {'cell_id': cell_id, 'rank': get_ranks(cell_order, cell_id)}\n\n# df_ranks = (\n#     pd.DataFrame\n#     .from_dict(ranks, orient='index')\n#     .rename_axis('id')\n#     .apply(pd.Series.explode)\n#     .set_index('cell_id', append=True)\n# )\n\n# df_ranks","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_ancestors = pd.read_csv(data_dir / 'train_ancestors.csv', index_col='id')\n# df_ancestors","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df = df.reset_index().merge(df_ranks, on=[\"id\", \"cell_id\"]).merge(df_ancestors, on=[\"id\"])\n# df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df[\"pct_rank\"] = df[\"rank\"] / df.groupby(\"id\")[\"cell_id\"].transform(\"count\")\n\n# df[\"pct_rank\"].hist(bins=10)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dict_cellid_source = dict(zip(df['cell_id'].values, df['source'].values))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import numpy as np\n# import pandas as pd\n# import os\n# import re\n# # import fasttext\n# from tqdm import tqdm\n# import matplotlib.pyplot as plt\n# from sklearn.metrics.pairwise import cosine_similarity\n# from nltk.stem import WordNetLemmatizer\n# from pathlib import Path\n# # import nltk\n# # nltk.download('wordnet')\n\n# stemmer = WordNetLemmatizer()\n\n# def preprocess_text(document):\n#         # Remove all the special characters\n#         document = re.sub(r'\\W', ' ', str(document))\n\n#         # remove all single characters\n#         document = re.sub(r'\\s+[a-zA-Z]\\s+', ' ', document)\n\n#         # Remove single characters from the start\n#         document = re.sub(r'\\^[a-zA-Z]\\s+', ' ', document)\n\n#         # Substituting multiple spaces with single space\n#         document = re.sub(r'\\s+', ' ', document, flags=re.I)\n\n#         # Removing prefixed 'b'\n#         document = re.sub(r'^b\\s+', '', document)\n\n#         # Converting to Lowercase\n#         document = document.lower()\n#         #return document\n\n#         # Lemmatization\n#         tokens = document.split()\n#         tokens = [stemmer.lemmatize(word) for word in tokens]\n#         tokens = [word for word in tokens if len(word) > 3]\n\n#         preprocessed_text = ' '.join(tokens)\n#         return preprocessed_text\n\n    \n# def preprocess_df(df):\n#     \"\"\"\n#     This function is for processing sorce of notebook\n#     returns preprocessed dataframe\n#     \"\"\"\n#     return [preprocess_text(message) for message in df.source]\n\n# df.source = df.source.apply(preprocess_text)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.model_selection import GroupShuffleSplit\n\n# NVALID = 0.1  # size of validation set\n\n# splitter = GroupShuffleSplit(n_splits=1, test_size=NVALID, random_state=0)\n\n# train_ind, val_ind = next(splitter.split(df, groups=df[\"ancestor_id\"]))\n\n# train_df = df.loc[train_ind].reset_index(drop=True)\n# val_df = df.loc[val_ind].reset_index(drop=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from tqdm.notebook import tqdm\n\n# def generate_triplet(df, mode='train'):\n#   triplets = []\n#   ids = df.id.unique()\n#   random_drop = np.random.random(size=10000)>0.9\n#   count = 0\n\n#   for id, df_tmp in tqdm(df.groupby('id')):\n#     df_tmp_markdown = df_tmp[df_tmp['cell_type']=='markdown']\n\n#     df_tmp_code = df_tmp[df_tmp['cell_type']=='code']\n#     df_tmp_code_rank = df_tmp_code['rank'].values\n#     df_tmp_code_cell_id = df_tmp_code['cell_id'].values\n\n#     for cell_id, rank in df_tmp_markdown[['cell_id', 'rank']].values:\n#       labels = np.array([(r==(rank+1)) for r in df_tmp_code_rank]).astype('int')\n\n#       for cid, label in zip(df_tmp_code_cell_id, labels):\n#         count += 1\n#         if label==1:\n#           triplets.append( [cell_id, cid, label] )\n#           # triplets.append( [cid, cell_id, label] )\n#         elif mode == 'test':\n#           triplets.append( [cell_id, cid, label] )\n#           # triplets.append( [cid, cell_id, label] )\n#         elif random_drop[count%10000]:\n#           triplets.append( [cell_id, cid, label] )\n#           # triplets.append( [cid, cell_id, label] )\n    \n#   return triplets\n\n# triplets = generate_triplet(train_df)\n# val_triplets = generate_triplet(val_df, mode = 'test')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from bisect import bisect\n\n\n# def count_inversions(a):\n#     inversions = 0\n#     sorted_so_far = []\n#     for i, u in enumerate(a):\n#         j = bisect(sorted_so_far, u)\n#         inversions += i - j\n#         sorted_so_far.insert(j, u)\n#     return inversions\n\n\n# def kendall_tau(ground_truth, predictions):\n#     total_inversions = 0\n#     total_2max = 0  # twice the maximum possible inversions across all instances\n#     for gt, pred in zip(ground_truth, predictions):\n#         ranks = [gt.index(x) for x in pred]  # rank predicted order in terms of ground truth\n#         total_inversions += count_inversions(ranks)\n#         n = len(gt)\n#         total_2max += n * (n - 1)\n#     return 1 - 4 * total_inversions / total_2max","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from transformers import RobertaTokenizer, RobertaConfig, RobertaModel\n# import torch.nn.functional as F\n# import torch.nn as nn\n# import torch\n# from transformers import AutoModelWithLMHead, AutoTokenizer, AutoModel\n\n# MAX_LEN = 128\n\n    \n# class MarkdownModel(nn.Module):\n#     def __init__(self):\n#         super(MarkdownModel, self).__init__()\n#         self.distill_bert = AutoModel.from_pretrained(\"../input/mymodelpairbertsmallpretrained/models/checkpoint-18000\")\n#         self.top = nn.Linear(512, 1)\n\n#         self.dropout = nn.Dropout(0.2)\n        \n#     def forward(self, ids, mask):\n#         x = self.distill_bert(ids, mask)[0]\n#         x = self.dropout(x)\n#         x = self.top(x[:, 0, :])\n#         x = torch.sigmoid(x) \n#         return x","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from torch.utils.data import DataLoader, Dataset\n\n\n\n# class MarkdownDataset(Dataset):\n    \n#     def __init__(self, df, max_len, mode='train'):\n#         super().__init__()\n#         self.df = df\n#         self.max_len = max_len\n#         self.tokenizer = AutoTokenizer.from_pretrained(\"../input/mymodelpairbertsmallpretrained/my_own_tokenizer\", do_lower_case=True)\n#         self.mode=mode\n\n#     def __getitem__(self, index):\n#         row = self.df[index]\n\n#         label = row[-1]\n\n#         txt = dict_cellid_source[row[0]] + '[SEP]' + dict_cellid_source[row[1]]\n\n#         inputs = self.tokenizer.encode_plus(\n#             txt,\n#             None,\n#             add_special_tokens=True,\n#             max_length=self.max_len,\n#             padding=\"max_length\",\n#             return_token_type_ids=True,\n#             truncation=True\n#         )\n#         ids = torch.LongTensor(inputs['input_ids'])\n#         mask = torch.LongTensor(inputs['attention_mask'])\n\n#         return ids, mask, torch.FloatTensor([label])\n\n\n\n\n#     def __len__(self):\n#         return len(self.df)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def adjust_lr(optimizer, epoch):\n#     if epoch < 1:\n#         lr = 5e-5\n#     elif epoch < 2:\n#         lr = 1e-3\n#     elif epoch < 5:\n#         lr = 1e-4\n#     else:\n#         lr = 1e-5\n\n#     for p in optimizer.param_groups:\n#         p['lr'] = lr\n#     return lr\n    \n# def get_optimizer(net):\n#     optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, net.parameters()), lr=3e-4, betas=(0.9, 0.999),\n#                                  eps=1e-08)\n#     return optimizer\n\n# BS = 128\n# NW = 2","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def read_data(data):\n#     return tuple(d.cuda() for d in data[:-1]), data[-1].cuda()\n\n\n# def validate(model, val_loader, mode='train'):\n#     model.eval()\n    \n#     tbar = tqdm(val_loader, file=sys.stdout)\n    \n#     preds = np.zeros(len(val_loader.dataset), dtype='float32')\n#     labels = []\n#     count = 0\n\n#     with torch.no_grad():\n#         for idx, data in enumerate(tbar):\n#             inputs, target = read_data(data)\n\n#             pred = model(inputs[0], inputs[1]).detach().cpu().numpy().ravel()\n\n#             preds[count:count+len(pred)] = pred\n#             count += len(pred)\n            \n#             if mode=='test':\n#               labels.append(target.detach().cpu().numpy().ravel())\n#     if mode=='test':\n#       return preds\n#     else:\n#       return np.concatenate(labels), np.concatenate(preds)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# paths_test = list((data_dir / 'test').glob('*.json'))\n# notebooks_test = [\n#     read_notebook(path) for path in tqdm(paths_test, desc='Test NBs')\n# ]\n# test_df = (\n#     pd.concat(notebooks_test)\n#     .set_index('id', append=True)\n#     .swaplevel()\n#     .sort_index(level='id', sort_remaining=False)\n# ).reset_index()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_df.source = test_df.source.apply(preprocess_text)\n# dict_cellid_source = dict(zip(test_df['cell_id'].values, test_df['source'].values))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_df[\"rank\"] = test_df.groupby([\"id\", \"cell_type\"]).cumcount()\n# test_df[\"pred\"] = test_df.groupby([\"id\", \"cell_type\"])[\"rank\"].rank(pct=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_triplets = generate_triplet(test_df, mode = 'test')\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_df[\"pct_rank\"] = 0\n# test_ds = MarkdownDataset(test_triplets, max_len=MAX_LEN)\n# test_loader = DataLoader(test_ds, batch_size=BS * 4, shuffle=False, num_workers=NW,\n#                           pin_memory=False, drop_last=False)\n\n\n# import gc \n# gc.collect()\n# len(test_ds), test_ds[0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import sys \n\n# model = MarkdownModel()\n# model = model.cuda()\n# model.load_state_dict(torch.load('../input/bertsmall-pairwise-ckpt/my_own_model_4.bin'))\n# y_test = validate(model, test_loader, mode='test')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# preds_copy = y_test\n# pred_vals = []\n# count = 0\n# for id, df_tmp in tqdm(test_df.groupby('id')):\n#   df_tmp_mark = df_tmp[df_tmp['cell_type']=='markdown']\n#   df_tmp_code = df_tmp[df_tmp['cell_type']!='markdown']\n#   df_tmp_code_rank = df_tmp_code['rank'].rank().values\n#   N_code = len(df_tmp_code_rank)\n#   N_mark = len(df_tmp_mark)\n\n#   preds_tmp = preds_copy[count:count+N_mark * N_code]\n\n#   count += N_mark * N_code\n\n#   for i in range(N_mark):\n#     pred = preds_tmp[i*N_code:i*N_code+N_code] \n\n#     softmax = np.exp((pred-np.mean(pred)) *20)/np.sum(np.exp((pred-np.mean(pred)) *20)) \n\n#     rank = np.sum(softmax * df_tmp_code_rank)\n#     pred_vals.append(rank)\n\n# del model\n# del test_triplets[:]\n# del dict_cellid_source\n# gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_df.loc[test_df[\"cell_type\"] == \"markdown\", \"pred\"] = pred_vals\n# sub_df = test_df.sort_values(\"pred\").groupby(\"id\")[\"cell_id\"].apply(lambda x: \" \".join(x)).reset_index()\n# sub_df.rename(columns={\"cell_id\": \"cell_order\"}, inplace=True)\n# sub_df.to_csv(\"submission_2.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_1 = pd.read_csv('submission_2.csv')\n# df_2 = pd.read_csv('submission_1.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# new_samples = []\n# for sample_idx in range(len(df_1)):\n#     sample_1 = {k: v for v, k in enumerate(df_1.iloc[sample_idx]['cell_order'].split(' '))}\n#     sample_2 = {k: v for v, k in enumerate(df_2.iloc[sample_idx]['cell_order'].split(' '))}\n#     for key in sample_1: sample_1[key] = ( (sample_1[key] * 0.246) + (sample_2[key] * 0.754) )\n#     new_samples.append(' '.join([i[0] for i in list(sorted(sample_1.items(), key=lambda x:x[1]))]))\n# df_1['cell_order'] = new_samples","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_1.to_csv('submission.csv', index = False)\n# df_1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}