{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install sentence-transformers","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-21T18:04:22.532722Z","iopub.execute_input":"2022-07-21T18:04:22.533900Z","iopub.status.idle":"2022-07-21T18:04:36.417785Z","shell.execute_reply.started":"2022-07-21T18:04:22.533802Z","shell.execute_reply":"2022-07-21T18:04:36.416662Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<img src=\"https://miro.medium.com/max/1400/1*CvIsEOdi3XAcr8PE_t2LTQ.png\">\n\n<center><h1> - Training & Tuning using CrossEncoder - </h1></center>\n\n**Goal**: Predict the correct ordering of the **cells** within a Cross Encode.\n\n### ⬇ Libraries","metadata":{}},{"cell_type":"code","source":"import warnings\nfrom sklearn.model_selection import GroupShuffleSplit\nfrom bisect import bisect\nfrom pandas.core.common import SettingWithCopyWarning\nwarnings.simplefilter(action='ignore', category=FutureWarning)\nwarnings.simplefilter(action=\"ignore\", category=SettingWithCopyWarning)\nimport numpy as np\nfrom pathlib import Path\nimport pandas as pd\nimport numpy as np\nimport torch\nimport re\nimport nltk\nfrom sentence_transformers import SentenceTransformer, util\nimport torch.nn.functional as F\nimport torch.nn as nn\nfrom torch.utils.data import DataLoader, Dataset\nfrom time import time\nfrom tqdm.auto import tqdm\ntqdm.pandas()\npd.options.display.width = 180\npd.options.display.max_colwidth = 120\nimport sys\nimport json\nfrom torch.utils.data import DataLoader\nfrom sentence_transformers import SentenceTransformer, LoggingHandler, util, models, evaluation, losses, InputExample\nimport logging\nfrom datetime import datetime\nimport gzip\nimport os\nimport tarfile\nfrom collections import defaultdict\nfrom torch.utils.data import IterableDataset\nfrom torch.utils.data import Dataset\nimport random\nimport pickle\nfrom sentence_transformers.cross_encoder.evaluation import CEBinaryClassificationEvaluator\nfrom sentence_transformers.cross_encoder import CrossEncoder\nimport logging\nlogger = logging.getLogger()\nlogger.setLevel(logging.DEBUG)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T18:04:36.420104Z","iopub.execute_input":"2022-07-21T18:04:36.420703Z","iopub.status.idle":"2022-07-21T18:04:41.353813Z","shell.execute_reply.started":"2022-07-21T18:04:36.420661Z","shell.execute_reply":"2022-07-21T18:04:41.352507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 🧹 Preparation","metadata":{}},{"cell_type":"code","source":"# reading\ndef get_df_orders_and_ranks(df, data_dir):\n    # train orders\n    df_orders = pd.read_csv(\n      data_dir / 'train_orders.csv',\n      index_col='id',\n      squeeze=True,\n    ).str.split()  # cell_ids str -> list\n\n\n    df_orders_ = df_orders.to_frame().join(\n      # reset only one index out of many -> \"cell_id\"; make a list out of cells in train data\n      df.reset_index('cell_id').groupby('id')['cell_id'].apply(list),\n      how='right',\n    )\n\n    ranks = {}\n    for id_, cell_order, cell_id in df_orders_.itertuples():\n        ranks[id_] = {'cell_id': cell_id, 'rank': get_ranks(cell_order, cell_id)}\n\n    df_ranks = (\n      pd.DataFrame\n      .from_dict(ranks, orient='index')\n      .rename_axis('id')\n      .apply(pd.Series.explode)\n      .set_index('cell_id', append=True)\n    )\n    # now we have\n    # id cell_id rank\n    return df_orders, df_ranks\n\ndef get_ranks(base, derived):\n    return [base.index(d) for d in derived]\n\ndef read_train_data(data_dir, NUM_TRAIN = 10000):\n    def read_notebook(path):\n        return (\n            pd.read_json(\n                path,\n                dtype={'cell_type': 'category', 'source': 'str'})\n            .assign(id=path.stem)  # final path component\n            .rename_axis('cell_id')\n        )\n\n    paths_train = list((data_dir / 'train').glob('*.json'))[:NUM_TRAIN]\n    notebooks_train = [\n      read_notebook(path) for path in tqdm(paths_train)\n    ]\n    df = (\n      pd.concat(notebooks_train)\n      .set_index('id', append=True)\n      .swaplevel()\n      .sort_index(level='id', sort_remaining=False)\n    )\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-07-21T18:04:41.358835Z","iopub.execute_input":"2022-07-21T18:04:41.363852Z","iopub.status.idle":"2022-07-21T18:04:41.382324Z","shell.execute_reply.started":"2022-07-21T18:04:41.363810Z","shell.execute_reply":"2022-07-21T18:04:41.381214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# metrics\ndef count_inversions(a):\n    inversions = 0\n    sorted_so_far = []\n    for i, u in enumerate(a):\n        j = bisect(sorted_so_far, u)\n        inversions += i - j\n        sorted_so_far.insert(j, u)\n    return inversions\n\n\ndef kendall_tau(ground_truth, predictions):\n    total_inversions = 0\n    total_2max = 0  # twice the maximum possible inversions across all instances\n    for gt, pred in zip(ground_truth, predictions):\n        ranks = [gt.index(x) for x in pred]  # rank predicted order in terms of ground truth\n        total_inversions += count_inversions(ranks)\n        n = len(gt)\n        total_2max += n * (n - 1)\n    return 1 - 4 * total_inversions / total_2max","metadata":{"execution":{"iopub.status.busy":"2022-07-21T18:04:54.486182Z","iopub.execute_input":"2022-07-21T18:04:54.486872Z","iopub.status.idle":"2022-07-21T18:04:54.494820Z","shell.execute_reply.started":"2022-07-21T18:04:54.486831Z","shell.execute_reply":"2022-07-21T18:04:54.493887Z"},"_kg_hide-output":true,"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dir = Path('../input/AI4Code')\ndf = read_train_data(data_dir, NUM_TRAIN=1000)\ndf_orders, df_ranks = get_df_orders_and_ranks(df, data_dir)\nprint(f\"Df shape is {df.shape}\")","metadata":{"execution":{"iopub.status.busy":"2022-07-21T18:04:41.385177Z","iopub.execute_input":"2022-07-21T18:04:41.385722Z","iopub.status.idle":"2022-07-21T18:04:54.483788Z","shell.execute_reply.started":"2022-07-21T18:04:41.385684Z","shell.execute_reply":"2022-07-21T18:04:54.481990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 📑 Clean markdown, get percentage level and split","metadata":{}},{"cell_type":"code","source":"df_orders, df_ranks = get_df_orders_and_ranks(df, data_dir)\n\ndf['rank'] = df.reset_index().merge(df_ranks, on=[\"id\", \"cell_id\"])['rank'].values\ndf = df.reset_index()\ndf[\"pct_rank\"] = df[\"rank\"] / df.groupby(\"id\")[\"cell_id\"].transform(\"count\")\ndf_ancestors = pd.read_csv(data_dir / 'train_ancestors.csv', index_col='id')\ndf = df.merge(df_ancestors, on=[\"id\"])","metadata":{"execution":{"iopub.status.busy":"2022-07-21T18:04:54.496410Z","iopub.execute_input":"2022-07-21T18:04:54.496763Z","iopub.status.idle":"2022-07-21T18:04:57.262178Z","shell.execute_reply.started":"2022-07-21T18:04:54.496728Z","shell.execute_reply":"2022-07-21T18:04:57.261150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"splitter = GroupShuffleSplit(n_splits=1, test_size=0.1, random_state=47)\ntrain_ind, val_ind = next(splitter.split(df, groups=df[\"ancestor_id\"]))\n\ntrain_df = df.loc[train_ind].reset_index(drop=True)\nval_df = df.loc[val_ind].reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T18:04:57.263541Z","iopub.execute_input":"2022-07-21T18:04:57.264321Z","iopub.status.idle":"2022-07-21T18:04:57.372932Z","shell.execute_reply.started":"2022-07-21T18:04:57.264282Z","shell.execute_reply":"2022-07-21T18:04:57.371799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df_mark = train_df[train_df[\"cell_type\"] == \"markdown\"].reset_index(drop=True)\nval_df_mark= val_df[val_df[\"cell_type\"] == \"markdown\"].reset_index(drop=True)\ntrain_df_cells = train_df[train_df[\"cell_type\"] == \"code\"].reset_index(drop=True)\nval_df_cells = val_df[val_df[\"cell_type\"] == \"code\"].reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T18:04:57.377530Z","iopub.execute_input":"2022-07-21T18:04:57.377897Z","iopub.status.idle":"2022-07-21T18:04:57.420339Z","shell.execute_reply.started":"2022-07-21T18:04:57.377862Z","shell.execute_reply":"2022-07-21T18:04:57.419397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 🦾 Training","metadata":{}},{"cell_type":"code","source":"train_batch_size = 16           #Increasing the train batch size improves the model performance, but requires more GPU memory\nmax_seq_length = 128            #Max length for passages. Increasing it, requires more GPU memory\nce_score_margin = 3             #Margin for the CrossEncoder score between negative and positive passages\nnum_negs_per_system = 5         # We used different systems to mine hard negatives. Number of hard negatives to add from each system\nnum_epochs = 5                 # Number of epochs we want to train","metadata":{"execution":{"iopub.status.busy":"2022-07-21T18:04:57.421824Z","iopub.execute_input":"2022-07-21T18:04:57.427364Z","iopub.status.idle":"2022-07-21T18:04:57.433503Z","shell.execute_reply.started":"2022-07-21T18:04:57.427323Z","shell.execute_reply":"2022-07-21T18:04:57.432481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 📙 Creating corpus and generating triplets 🎶","metadata":{}},{"cell_type":"code","source":"def create_corpus(marks, cells):\n    corpus = {}         #dict in the format: passage_id -> passage. Stores all existent passages\n    for i, row in tqdm(marks.iterrows(), total=len(marks)):\n        source = row.source\n        index = (row.id, row.cell_id)\n        corpus[index] = source\n\n    for i, row in tqdm(cells.iterrows(), total=len(cells)):\n        source = row.source\n        index = (row.id, row.cell_id)\n        corpus[index] = source\n    return corpus","metadata":{"execution":{"iopub.status.busy":"2022-07-21T18:04:57.452219Z","iopub.execute_input":"2022-07-21T18:04:57.453769Z","iopub.status.idle":"2022-07-21T18:04:57.468319Z","shell.execute_reply.started":"2022-07-21T18:04:57.453726Z","shell.execute_reply":"2022-07-21T18:04:57.467404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def find_neg_code_cell(df, pct_rank):\n    if pct_rank <= 0.3:\n        pct_position = 0.65 + pct_rank\n        df = df[df['pct_rank'] > pct_position].sort_values('pct_rank', ascending=True)\n    elif pct_rank >= 0.7:\n        pct_position = pct_rank - 0.65\n        df = df[df['pct_rank'] < pct_position].sort_values('pct_rank', ascending=False)\n    else: \n        return\n    if not df.empty:\n        return df.iloc[0][['cell_id']].values[0]\n    else:\n        return","metadata":{"execution":{"iopub.status.busy":"2022-07-21T18:04:57.473231Z","iopub.execute_input":"2022-07-21T18:04:57.474407Z","iopub.status.idle":"2022-07-21T18:04:57.484624Z","shell.execute_reply.started":"2022-07-21T18:04:57.474362Z","shell.execute_reply":"2022-07-21T18:04:57.483704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generate_triplets(df):\n    triplets = {}\n    for nb_id, df_tmp in tqdm(df.groupby('id')):\n        df_tmp_markdown = df_tmp[df_tmp['cell_type']=='markdown']\n        df_tmp_code = df_tmp[df_tmp['cell_type']=='code']\n        for md_cell_id, md_rank, md_pct_rank in df_tmp_markdown[['cell_id', 'rank', 'pct_rank']].values:\n            if (nb_id, md_cell_id) not in triplets:\n                triplets[nb_id, md_cell_id] = {'cell_id': None, 'label': None}\n            for cd_id, cd_rank in df_tmp_code[['cell_id', 'rank']].values:\n                if cd_rank == md_rank+1:\n                    triplets[nb_id, md_cell_id]['cell_id'] = cd_id\n                    triplets[nb_id, md_cell_id]['label'] = 1\n                    break\n            neg_cd_cell = find_neg_code_cell(df_tmp_code, md_pct_rank)\n            if neg_cd_cell:\n                triplets[nb_id, md_cell_id]['cell_id'] = neg_cd_cell\n                triplets[nb_id, md_cell_id]['label'] = 0\n    triplets_df = pd.DataFrame.from_dict(triplets, orient='index').reset_index()\n    triplets_df.columns = ['nb_id', 'md_id', 'cell_id', 'label']\n    triplets_df = triplets_df.dropna(subset=['cell_id'])\n    return triplets_df","metadata":{"execution":{"iopub.status.busy":"2022-07-21T18:04:57.489497Z","iopub.execute_input":"2022-07-21T18:04:57.490084Z","iopub.status.idle":"2022-07-21T18:04:57.506065Z","shell.execute_reply.started":"2022-07-21T18:04:57.490048Z","shell.execute_reply":"2022-07-21T18:04:57.505165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TripletsDataset(Dataset):\n    def __init__(self, triplets, corpus): \n        # corpus = cells.\n        # triplet notebook id - markdown id - {pos: code id; neg: code id}\n        self.triplets = triplets\n        self.corpus = corpus\n        self.pos_label = 1\n        self.neg_label = 0\n\n    def __getitem__(self, idx):\n        row = self.triplets.iloc[idx]\n        nb_id, md_id, cell_id, label = row.values\n        return InputExample(texts=[\n            self.corpus[(nb_id, md_id)],\n            self.corpus[(nb_id, cell_id)]\n        ], label=label)\n\n    def __len__(self):\n        return len(self.triplets)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T18:04:57.510618Z","iopub.execute_input":"2022-07-21T18:04:57.513416Z","iopub.status.idle":"2022-07-21T18:04:57.523961Z","shell.execute_reply.started":"2022-07-21T18:04:57.513369Z","shell.execute_reply":"2022-07-21T18:04:57.523168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corpus = create_corpus(train_df_cells, train_df_mark)\ntriplets_df = generate_triplets(train_df)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T18:04:57.528739Z","iopub.execute_input":"2022-07-21T18:04:57.531491Z","iopub.status.idle":"2022-07-21T18:05:19.468186Z","shell.execute_reply.started":"2022-07-21T18:04:57.531458Z","shell.execute_reply":"2022-07-21T18:05:19.467160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# use pretrained SBERT model\nmodel = CrossEncoder(\n    \"../input/huggingface-bert-variants/distilbert-base-uncased/distilbert-base-uncased\",\n    max_length=max_seq_length,\n    num_labels=1\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T18:05:19.469539Z","iopub.execute_input":"2022-07-21T18:05:19.470103Z","iopub.status.idle":"2022-07-21T18:05:23.234991Z","shell.execute_reply.started":"2022-07-21T18:05:19.470064Z","shell.execute_reply":"2022-07-21T18:05:23.233988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = TripletsDataset(triplets_df, corpus)\ntrain_dataloader = DataLoader(\n    train_dataset,\n    shuffle=True,\n    batch_size=train_batch_size,\n    collate_fn=model.smart_batching_collate\n)\nmodel_save_path = \"cross_encoder_model\"","metadata":{"execution":{"iopub.status.busy":"2022-07-21T18:05:23.236429Z","iopub.execute_input":"2022-07-21T18:05:23.236793Z","iopub.status.idle":"2022-07-21T18:05:23.245265Z","shell.execute_reply.started":"2022-07-21T18:05:23.236755Z","shell.execute_reply":"2022-07-21T18:05:23.244365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Validation","metadata":{}},{"cell_type":"code","source":"val_df_mark = val_df_mark[['id', 'cell_id', 'cell_type', 'pct_rank', 'source']]\nvalid_mg_df = pd.merge(\n    val_df_cells[['id', 'cell_id', 'source', 'pct_rank']],\n    val_df_mark, how=\"inner\", on='id',\n    suffixes=('_left', '_right')\n)\n# validation\nval_corpus = create_corpus(val_df_cells, val_df_mark)\nval_triplets_df = generate_triplets(val_df)\nval_dataset = TripletsDataset(val_triplets_df, val_corpus)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T18:05:23.247452Z","iopub.execute_input":"2022-07-21T18:05:23.247723Z","iopub.status.idle":"2022-07-21T18:05:25.503431Z","shell.execute_reply.started":"2022-07-21T18:05:23.247700Z","shell.execute_reply":"2022-07-21T18:05:25.502579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"evalutation_df = []\nfor i, pair in tqdm(val_triplets_df.iterrows(), total=len(val_triplets_df)):\n    left_sentence = val_corpus[(pair['nb_id'], pair['md_id'])]\n    right_sentence = val_corpus[(pair['nb_id'], pair['cell_id'])]\n    evalutation_df.append(InputExample(texts=[left_sentence, right_sentence], label=pair['label']))","metadata":{"execution":{"iopub.status.busy":"2022-07-21T18:05:25.504753Z","iopub.execute_input":"2022-07-21T18:05:25.505097Z","iopub.status.idle":"2022-07-21T18:05:25.644799Z","shell.execute_reply.started":"2022-07-21T18:05:25.505062Z","shell.execute_reply":"2022-07-21T18:05:25.643763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"evalt = CEBinaryClassificationEvaluator.from_input_examples(\n    evalutation_df,\n    name='evals',\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T18:05:25.646290Z","iopub.execute_input":"2022-07-21T18:05:25.646875Z","iopub.status.idle":"2022-07-21T18:05:25.653579Z","shell.execute_reply.started":"2022-07-21T18:05:25.646835Z","shell.execute_reply":"2022-07-21T18:05:25.652135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Fitting","metadata":{}},{"cell_type":"code","source":"model.fit(\n    train_dataloader=train_dataloader,\n    epochs=5,\n    warmup_steps=1000,\n    use_amp=True,\n    optimizer_params = {'lr': 2e-5},\n    save_best_model=True,\n    evaluator=evalt,\n    output_path=model_save_path,\n    evaluation_steps=100\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T18:05:25.655492Z","iopub.execute_input":"2022-07-21T18:05:25.655851Z","iopub.status.idle":"2022-07-21T18:16:54.028101Z","shell.execute_reply.started":"2022-07-21T18:05:25.655816Z","shell.execute_reply":"2022-07-21T18:16:54.027161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sentence_pairs = []\nfor i, pair in tqdm(valid_mg_df.iterrows(), total=len(valid_mg_df)):\n    left_sentence = pair['source_left']\n    right_sentence = pair['source_right']\n    sentence_pairs.append([left_sentence, right_sentence])","metadata":{"execution":{"iopub.status.busy":"2022-07-21T18:23:07.156075Z","iopub.execute_input":"2022-07-21T18:23:07.156765Z","iopub.status.idle":"2022-07-21T18:23:12.714595Z","shell.execute_reply.started":"2022-07-21T18:23:07.156729Z","shell.execute_reply":"2022-07-21T18:23:12.713497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"similarities = model.predict(sentence_pairs, show_progress_bar=True)\nvalid_mg_df['similarity_score'] = similarities","metadata":{"execution":{"iopub.status.busy":"2022-07-21T18:23:14.928784Z","iopub.execute_input":"2022-07-21T18:23:14.929358Z","iopub.status.idle":"2022-07-21T18:24:25.752899Z","shell.execute_reply.started":"2022-07-21T18:23:14.929315Z","shell.execute_reply":"2022-07-21T18:24:25.751238Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.concat([val_df_cells, val_df_mark])\npred_vals = []\npreds_copy = similarities.copy()\ncount = 0\nfor id, df_tmp in tqdm(test_df.groupby('id')):\n    df_tmp_mark = df_tmp[df_tmp['cell_type']=='markdown']\n    df_tmp_code = df_tmp[df_tmp['cell_type']!='markdown']\n    df_tmp_code_rank = df_tmp_code['pct_rank'].values\n    N_code = len(df_tmp_code_rank)\n    N_mark = len(df_tmp_mark)\n\n    preds_tmp = preds_copy[count:count+N_mark * N_code]\n\n    count += N_mark * N_code\n    for i in range(N_mark):\n        pred = preds_tmp[i*N_code:i*N_code+N_code]\n        if len(pred) != N_code:\n            print(len(pred))\n            print(N_code)\n            print(i*N_code+N_code)\n            print(len(preds_tmp))\n            print(count+N_mark * N_code)\n        softmax = np.exp((pred-np.mean(pred)) *20)/np.sum(np.exp((pred-np.mean(pred)) *20))\n        rank = np.sum(softmax * df_tmp_code_rank)\n        pred_vals.append(rank)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T18:19:14.662218Z","iopub.execute_input":"2022-07-21T18:19:14.662573Z","iopub.status.idle":"2022-07-21T18:19:14.899693Z","shell.execute_reply.started":"2022-07-21T18:19:14.662531Z","shell.execute_reply":"2022-07-21T18:19:14.898679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.loc[test_df[\"cell_type\"] == \"markdown\", \"pct_rank\"] = pred_vals\ny_dummy = test_df.sort_values(\"pct_rank\").groupby('id')['cell_id'].apply(list)\nscore = kendall_tau(df_orders.loc[y_dummy.index], y_dummy)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T18:19:14.911601Z","iopub.execute_input":"2022-07-21T18:19:14.912534Z","iopub.status.idle":"2022-07-21T18:19:14.943043Z","shell.execute_reply.started":"2022-07-21T18:19:14.912494Z","shell.execute_reply":"2022-07-21T18:19:14.942152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(score)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T18:19:14.944624Z","iopub.execute_input":"2022-07-21T18:19:14.944972Z","iopub.status.idle":"2022-07-21T18:19:14.953003Z","shell.execute_reply.started":"2022-07-21T18:19:14.944937Z","shell.execute_reply":"2022-07-21T18:19:14.951768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**If like it you know how to support it ❤️**","metadata":{}}]}