{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import json\nimport os\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import GroupShuffleSplit\nfrom scipy import sparse\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom xgboost import XGBRanker\nfrom bisect import bisect","metadata":{"execution":{"iopub.status.busy":"2022-07-06T18:37:01.234984Z","iopub.execute_input":"2022-07-06T18:37:01.235402Z","iopub.status.idle":"2022-07-06T18:37:01.779460Z","shell.execute_reply.started":"2022-07-06T18:37:01.235320Z","shell.execute_reply":"2022-07-06T18:37:01.778418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_train = 10000\n\ndef read_notebook(path,id_name):\n    return (\n        pd.read_json(path, dtype={'cell_type': 'category', 'source': 'str'})\n        .assign(id=id_name)\n        .rename_axis('cell_id')\n    )\n\npaths = []\ndirectory = '../input/AI4Code/train'\nfor file in os.scandir(directory):\n    if file.is_file():\n        paths.append(file.path)\n    if len(paths) == num_train:\n        break\n        \nid_names = []\nfor name in paths:\n    name = name.split('/')\n    id_n = name[-1].split('.')\n    id_names.append(id_n[0])\n    \n# print(id_names)\n# print(paths)\n\ntrain_notebooks = []\nfor i in range(len(paths)):\n    train_notebooks.append(read_notebook(paths[i],id_names[i]))\n    \nprint(train_notebooks[0])","metadata":{"execution":{"iopub.status.busy":"2022-07-06T18:37:01.780693Z","iopub.execute_input":"2022-07-06T18:37:01.781187Z","iopub.status.idle":"2022-07-06T18:38:06.146063Z","shell.execute_reply.started":"2022-07-06T18:37:01.781151Z","shell.execute_reply":"2022-07-06T18:38:06.144824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(type(train_notebooks[0]))\ndf = (\n    pd.concat(train_notebooks)\n    .set_index('id', append=True)\n    .swaplevel()\n    .sort_index(level='id', sort_remaining=False)\n)\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-06T18:38:06.147891Z","iopub.execute_input":"2022-07-06T18:38:06.148879Z","iopub.status.idle":"2022-07-06T18:38:09.469691Z","shell.execute_reply.started":"2022-07-06T18:38:06.148845Z","shell.execute_reply":"2022-07-06T18:38:09.468345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_orders = pd.read_csv('../input/AI4Code/train_orders.csv',index_col='id',squeeze=True).str.split() ","metadata":{"execution":{"iopub.status.busy":"2022-07-06T18:38:09.473337Z","iopub.execute_input":"2022-07-06T18:38:09.473709Z","iopub.status.idle":"2022-07-06T18:38:11.192903Z","shell.execute_reply.started":"2022-07-06T18:38:09.473677Z","shell.execute_reply":"2022-07-06T18:38:11.191555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_orders","metadata":{"execution":{"iopub.status.busy":"2022-07-06T18:38:11.194539Z","iopub.execute_input":"2022-07-06T18:38:11.194868Z","iopub.status.idle":"2022-07-06T18:38:11.227803Z","shell.execute_reply.started":"2022-07-06T18:38:11.194827Z","shell.execute_reply":"2022-07-06T18:38:11.226367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_orders_ = df_orders.to_frame().join(\n    df.reset_index('cell_id').groupby('id')['cell_id'].apply(list),\n    how='right',\n)\n\ndef get_ranks(base, derived):\n    return [base.index(d) for d in derived]\n\nranks = {}\nfor id_, cell_order, cell_id in df_orders_.itertuples():\n    ranks[id_] = {'cell_id': cell_id, 'rank': get_ranks(cell_order, cell_id)}\n\ndf_ranks = (\n    pd.DataFrame\n    .from_dict(ranks, orient='index')\n    .rename_axis('id')\n    .apply(pd.Series.explode)\n    .set_index('cell_id', append=True)\n)\n\ndf_ranks","metadata":{"execution":{"iopub.status.busy":"2022-07-06T18:38:11.229548Z","iopub.execute_input":"2022-07-06T18:38:11.229948Z","iopub.status.idle":"2022-07-06T18:38:14.558266Z","shell.execute_reply.started":"2022-07-06T18:38:11.229915Z","shell.execute_reply":"2022-07-06T18:38:14.556942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ancestors = pd.read_csv('../input/AI4Code/train_ancestors.csv', index_col='id')\ndf_ancestors","metadata":{"execution":{"iopub.status.busy":"2022-07-06T18:38:14.559460Z","iopub.execute_input":"2022-07-06T18:38:14.559943Z","iopub.status.idle":"2022-07-06T18:38:14.791901Z","shell.execute_reply.started":"2022-07-06T18:38:14.559892Z","shell.execute_reply":"2022-07-06T18:38:14.790608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"siz = 0.1  # size of validation set\n\nsplitter = GroupShuffleSplit(n_splits=1, test_size=siz, random_state=0)\n\n# Split, keeping notebooks with a common origin (ancestor_id) together\nids = df.index.unique('id')\nancestors = df_ancestors.loc[ids, 'ancestor_id']\nids_train, ids_valid = next(splitter.split(ids, groups=ancestors))\nids_train, ids_valid = ids[ids_train], ids[ids_valid]\n\ndf_train = df.loc[ids_train, :]\ndf_valid = df.loc[ids_valid, :]","metadata":{"execution":{"iopub.status.busy":"2022-07-06T18:38:14.793477Z","iopub.execute_input":"2022-07-06T18:38:14.793910Z","iopub.status.idle":"2022-07-06T18:38:14.914855Z","shell.execute_reply.started":"2022-07-06T18:38:14.793875Z","shell.execute_reply":"2022-07-06T18:38:14.913524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training set\ntfidf = TfidfVectorizer(min_df=0.01)\nX_train = tfidf.fit_transform(df_train['source'].astype(str))\n# Rank of each cell within the notebook\ny_train = df_ranks.loc[ids_train].to_numpy()\n# Number of cells in each notebook\ngroups = df_ranks.loc[ids_train].groupby('id').size().to_numpy()","metadata":{"execution":{"iopub.status.busy":"2022-07-06T18:38:14.916738Z","iopub.execute_input":"2022-07-06T18:38:14.917452Z","iopub.status.idle":"2022-07-06T18:38:33.503057Z","shell.execute_reply.started":"2022-07-06T18:38:14.917420Z","shell.execute_reply":"2022-07-06T18:38:33.501683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Add code cell ordering\nX_train = sparse.hstack((\n    X_train,\n    np.where(\n        df_train['cell_type'] == 'code',\n        df_train.groupby(['id', 'cell_type']).cumcount().to_numpy() + 1,\n        0,\n    ).reshape(-1, 1)\n))\nprint(X_train.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T18:38:33.505259Z","iopub.execute_input":"2022-07-06T18:38:33.505726Z","iopub.status.idle":"2022-07-06T18:38:33.773507Z","shell.execute_reply.started":"2022-07-06T18:38:33.505678Z","shell.execute_reply":"2022-07-06T18:38:33.772135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = XGBRanker(\n    min_child_weight=10,\n    subsample=0.5,\n    tree_method='hist',\n)\nmodel.fit(X_train, y_train, group=groups)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T18:38:33.775339Z","iopub.execute_input":"2022-07-06T18:38:33.778130Z","iopub.status.idle":"2022-07-06T18:38:46.138060Z","shell.execute_reply.started":"2022-07-06T18:38:33.778057Z","shell.execute_reply":"2022-07-06T18:38:46.136802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Validation set\nX_valid = tfidf.transform(df_valid['source'].astype(str))\n# The metric uses cell ids\ny_valid = df_orders.loc[ids_valid]\n\nX_valid = sparse.hstack((\n    X_valid,\n    np.where(\n        df_valid['cell_type'] == 'code',\n        df_valid.groupby(['id', 'cell_type']).cumcount().to_numpy() + 1,\n        0,\n    ).reshape(-1, 1)\n))","metadata":{"execution":{"iopub.status.busy":"2022-07-06T18:38:46.140179Z","iopub.execute_input":"2022-07-06T18:38:46.140663Z","iopub.status.idle":"2022-07-06T18:38:47.936712Z","shell.execute_reply.started":"2022-07-06T18:38:46.140615Z","shell.execute_reply":"2022-07-06T18:38:47.935455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = pd.DataFrame({'rank': model.predict(X_valid)}, index=df_valid.index)\ny_pred = (\n    y_pred\n    .sort_values(['id', 'rank'])  # Sort the cells in each notebook by their rank.\n                                  # The cell_ids are now in the order the model predicted.\n    .reset_index('cell_id')  # Convert the cell_id index into a column.\n    .groupby('id')['cell_id'].apply(list)  # Group the cell_ids for each notebook into a list.\n)\ny_pred.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T18:38:47.942890Z","iopub.execute_input":"2022-07-06T18:38:47.943803Z","iopub.status.idle":"2022-07-06T18:38:48.120831Z","shell.execute_reply.started":"2022-07-06T18:38:47.943753Z","shell.execute_reply":"2022-07-06T18:38:48.119643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nb_id = df_valid.index.get_level_values('id').unique()[8]\n\ndisplay(df.loc[nb_id])\ndisplay(df.loc[nb_id].loc[y_pred.loc[nb_id]])\n","metadata":{"execution":{"iopub.status.busy":"2022-07-06T18:38:48.122428Z","iopub.execute_input":"2022-07-06T18:38:48.122967Z","iopub.status.idle":"2022-07-06T18:38:48.157968Z","shell.execute_reply.started":"2022-07-06T18:38:48.122935Z","shell.execute_reply":"2022-07-06T18:38:48.156919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def count_inversions_slowly(ranks):\n    inversions = 0\n    size = len(ranks)\n    for i in range(size):\n        for j in range(i+1, size):\n            if ranks[i] > ranks[j]:\n                total += 1\n    return total\n\ndef count_inversions(a):\n    inversions = 0\n    sorted_so_far = []\n    for i, u in enumerate(a):  \n        j = bisect(sorted_so_far, u)  \n        inversions += i - j\n        sorted_so_far.insert(j, u)  \n    return inversions\n\ndef kendall_tau(ground_truth, predictions):\n    total_inversions = 0  \n    total_2max = 0  \n    for gt, pred in zip(ground_truth, predictions):\n        ranks = [gt.index(x) for x in pred]  # rank predicted order in terms of ground truth\n        total_inversions += count_inversions(ranks)\n        n = len(gt)\n        total_2max += n * (n - 1)\n    return 1 - 4 * total_inversions / total_2max","metadata":{"execution":{"iopub.status.busy":"2022-07-06T18:38:48.159574Z","iopub.execute_input":"2022-07-06T18:38:48.160238Z","iopub.status.idle":"2022-07-06T18:38:48.171185Z","shell.execute_reply.started":"2022-07-06T18:38:48.160206Z","shell.execute_reply":"2022-07-06T18:38:48.169574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_dummy = df_valid.reset_index('cell_id').groupby('id')['cell_id'].apply(list)\nkendall_tau(y_valid, y_dummy)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T18:38:48.172912Z","iopub.execute_input":"2022-07-06T18:38:48.173806Z","iopub.status.idle":"2022-07-06T18:38:48.316180Z","shell.execute_reply.started":"2022-07-06T18:38:48.173754Z","shell.execute_reply":"2022-07-06T18:38:48.314877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kendall_tau(y_valid, y_pred)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T18:38:48.317643Z","iopub.execute_input":"2022-07-06T18:38:48.318041Z","iopub.status.idle":"2022-07-06T18:38:48.412114Z","shell.execute_reply.started":"2022-07-06T18:38:48.318009Z","shell.execute_reply":"2022-07-06T18:38:48.410843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"paths = []\ndirectory = '../input/AI4Code/test'\nfor file in os.scandir(directory):\n    if file.is_file():\n        paths.append(file.path)\n    if len(paths) == num_train:\n        break\n        \nid_names = []\nfor name in paths:\n    name = name.split('/')\n    id_n = name[-1].split('.')\n    id_names.append(id_n[0])\n    \n# print(id_names)\n# print(paths)\n\ntest_notebooks = []\nfor i in range(len(paths)):\n    test_notebooks.append(read_notebook(paths[i],id_names[i]))\n    \nprint(test_notebooks[0])","metadata":{"execution":{"iopub.status.busy":"2022-07-06T18:38:48.413985Z","iopub.execute_input":"2022-07-06T18:38:48.414888Z","iopub.status.idle":"2022-07-06T18:38:48.471989Z","shell.execute_reply.started":"2022-07-06T18:38:48.414839Z","shell.execute_reply":"2022-07-06T18:38:48.471141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(type(train_notebooks[0]))\ndf_test = (\n    pd.concat(test_notebooks)\n    .set_index('id', append=True)\n    .swaplevel()\n    .sort_index(level='id', sort_remaining=False)\n)\ndf_test.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-06T18:38:48.473006Z","iopub.execute_input":"2022-07-06T18:38:48.473582Z","iopub.status.idle":"2022-07-06T18:38:48.490359Z","shell.execute_reply.started":"2022-07-06T18:38:48.473549Z","shell.execute_reply":"2022-07-06T18:38:48.489154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = tfidf.transform(df_test['source'].astype(str))\nX_test = sparse.hstack((\n    X_test,\n    np.where(\n        df_test['cell_type'] == 'code',\n        df_test.groupby(['id', 'cell_type']).cumcount().to_numpy() + 1,\n        0,\n    ).reshape(-1, 1)\n))","metadata":{"execution":{"iopub.status.busy":"2022-07-06T18:38:48.491896Z","iopub.execute_input":"2022-07-06T18:38:48.492861Z","iopub.status.idle":"2022-07-06T18:38:48.512365Z","shell.execute_reply.started":"2022-07-06T18:38:48.492827Z","shell.execute_reply":"2022-07-06T18:38:48.510967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_infer = pd.DataFrame({'rank': model.predict(X_test)}, index=df_test.index)\ny_infer = y_infer.sort_values(['id', 'rank']).reset_index('cell_id').groupby('id')['cell_id'].apply(list)\ny_infer","metadata":{"execution":{"iopub.status.busy":"2022-07-06T18:38:48.514426Z","iopub.execute_input":"2022-07-06T18:38:48.515212Z","iopub.status.idle":"2022-07-06T18:38:48.539805Z","shell.execute_reply.started":"2022-07-06T18:38:48.515180Z","shell.execute_reply":"2022-07-06T18:38:48.538804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_submit = (\n    y_infer\n    .apply(' '.join)  # list of ids -> string of ids\n    .rename_axis('id')\n    .rename('cell_order')\n)\ny_submit","metadata":{"execution":{"iopub.status.busy":"2022-07-06T18:38:48.541621Z","iopub.execute_input":"2022-07-06T18:38:48.542130Z","iopub.status.idle":"2022-07-06T18:38:48.551450Z","shell.execute_reply.started":"2022-07-06T18:38:48.542060Z","shell.execute_reply":"2022-07-06T18:38:48.550350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_submit.to_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-06T18:38:48.552644Z","iopub.execute_input":"2022-07-06T18:38:48.553452Z","iopub.status.idle":"2022-07-06T18:38:48.562711Z","shell.execute_reply.started":"2022-07-06T18:38:48.553422Z","shell.execute_reply":"2022-07-06T18:38:48.561381Z"},"trusted":true},"execution_count":null,"outputs":[]}]}