{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook will help you to extract the top neighbors of each aid taking in account the number of co-occurrence","metadata":{}},{"cell_type":"code","source":"!pip3 install polars","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-16T11:11:05.743093Z","iopub.execute_input":"2022-12-16T11:11:05.743954Z","iopub.status.idle":"2022-12-16T11:11:17.930781Z","shell.execute_reply.started":"2022-12-16T11:11:05.743581Z","shell.execute_reply":"2022-12-16T11:11:17.929880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\nfrom collections import defaultdict\nimport numpy as np\nimport pandas as pd\nimport gc\nimport torch\nimport polars as pl","metadata":{"execution":{"iopub.status.busy":"2022-12-16T11:11:17.932361Z","iopub.execute_input":"2022-12-16T11:11:17.932616Z","iopub.status.idle":"2022-12-16T11:11:18.504524Z","shell.execute_reply.started":"2022-12-16T11:11:17.932590Z","shell.execute_reply":"2022-12-16T11:11:18.503681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pl.read_parquet('../input/otto-full-optimized-memory-footprint/train.parquet')\ntest_df = pl.read_parquet('../input/otto-full-optimized-memory-footprint/test.parquet')","metadata":{"execution":{"iopub.status.busy":"2022-12-16T11:11:18.507031Z","iopub.execute_input":"2022-12-16T11:11:18.507940Z","iopub.status.idle":"2022-12-16T11:11:47.427210Z","shell.execute_reply.started":"2022-12-16T11:11:18.507910Z","shell.execute_reply":"2022-12-16T11:11:47.426476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In order to consider if an aid is a neighbor to another, we just shift the `aid` column by 1, relatively to each session","metadata":{}},{"cell_type":"code","source":"def lagged_df(df):\n    df =df.with_columns(\n        pl.col(['aid']).shift(1).over('session').prefix(\"prev_\")\n    )\n    df = df.drop_nulls()\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-12-16T11:11:47.429866Z","iopub.execute_input":"2022-12-16T11:11:47.430198Z","iopub.status.idle":"2022-12-16T11:11:47.434979Z","shell.execute_reply.started":"2022-12-16T11:11:47.430155Z","shell.execute_reply":"2022-12-16T11:11:47.434367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = lagged_df(train_df)\ntest_df = lagged_df(test_df)\n\n\ndf = pl.concat([\n    train_df,\n    test_df\n], how=\"vertical\")","metadata":{"execution":{"iopub.status.busy":"2022-12-16T11:11:47.436029Z","iopub.execute_input":"2022-12-16T11:11:47.436483Z","iopub.status.idle":"2022-12-16T11:12:27.950710Z","shell.execute_reply.started":"2022-12-16T11:11:47.436450Z","shell.execute_reply":"2022-12-16T11:12:27.949011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Extract general neighbors ( not type specific ) for each aid","metadata":{}},{"cell_type":"code","source":"def extract_top_k_neighbors(df,k):\n    FFFF = df.groupby([\"aid\",\"prev_aid\"]).agg([\n    pl.count()\n])\n    f_prev_aid = FFFF.sort(\"count\",reverse=True).groupby(['prev_aid']).head(5)\n    f_prev_aid.columns = ['aid','prev_aid','count']\n    ffff =  pl.concat([FFFF.sort(\"count\",reverse=True).groupby(['aid']).head(5),f_prev_aid],how=\"vertical\")\n    fff_2 = ffff[['prev_aid','aid','count']]\n    fff_2.columns = ['aid','prev_aid','count']\n    d  =  pl.concat([ffff,fff_2],how=\"vertical\")\n    dd = d.unique(subset=[\"aid\",'prev_aid'])\n    tmp = dd.sort(\"count\",reverse=True).groupby(['aid']).head(k)\n    df_pd = tmp.to_pandas()\n    return dd","metadata":{"execution":{"iopub.status.busy":"2022-12-16T11:12:27.953683Z","iopub.execute_input":"2022-12-16T11:12:27.954125Z","iopub.status.idle":"2022-12-16T11:12:27.964943Z","shell.execute_reply.started":"2022-12-16T11:12:27.954073Z","shell.execute_reply":"2022-12-16T11:12:27.963801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here for exemple, we extract the top 10 neighbors for each aid","metadata":{}},{"cell_type":"code","source":"top_10_neighbors = extract_top_k_neighbors(df, 10)","metadata":{"execution":{"iopub.status.busy":"2022-12-16T11:12:41.143996Z","iopub.execute_input":"2022-12-16T11:12:41.144396Z","iopub.status.idle":"2022-12-16T11:14:01.140512Z","shell.execute_reply.started":"2022-12-16T11:12:41.144361Z","shell.execute_reply":"2022-12-16T11:14:01.139504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Extract neighbors for each specific type","metadata":{}},{"cell_type":"code","source":"def extract_k_neighbors_by_type(type_sort,k):\n    # 0 for click , 1 for carts, 2 for orders\n    FFFF = df.groupby([\"aid\",\"prev_aid\"]).agg([\n    pl.count(),\n    pl.col(\"aid\").filter(pl.col(\"type\") == type_sort).count().alias(\"count_type\"),\n])\n    f_prev_aid = FFFF.sort([\"count_type\",'count'],reverse=True).groupby(['prev_aid']).head(5)\n    f_prev_aid.columns = ['aid','prev_aid','count','count_type']\n    ffff =  pl.concat([FFFF.sort([\"count_type\",'count'],reverse=True).groupby(['aid']).head(5),f_prev_aid],how=\"vertical\")\n    fff_2 = ffff[['prev_aid','aid','count','count_type']]\n    fff_2.columns = ['aid','prev_aid','count','count_type']\n    d  =  pl.concat([ffff,fff_2],how=\"vertical\")\n    dd = d.unique(subset=[\"aid\",'prev_aid'])\n    tmp = dd.sort([\"count_type\",'count'],reverse=True).groupby(['aid']).head(k)\n    return dd\n","metadata":{"execution":{"iopub.status.busy":"2022-12-16T11:14:01.142064Z","iopub.execute_input":"2022-12-16T11:14:01.142354Z","iopub.status.idle":"2022-12-16T11:14:01.150877Z","shell.execute_reply.started":"2022-12-16T11:14:01.142328Z","shell.execute_reply":"2022-12-16T11:14:01.149843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Extract top 10 neighbors","metadata":{}},{"cell_type":"code","source":"k = 10","metadata":{"execution":{"iopub.status.busy":"2022-12-16T11:14:01.152115Z","iopub.execute_input":"2022-12-16T11:14:01.152936Z","iopub.status.idle":"2022-12-16T11:14:01.167100Z","shell.execute_reply.started":"2022-12-16T11:14:01.152906Z","shell.execute_reply":"2022-12-16T11:14:01.166020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"top_10_neighbors_clicks = extract_k_neighbors_by_type(0,k)\ntop_10_neighbors_carts = extract_k_neighbors_by_type(1,k)\ntop_10_neighbors_orders = extract_k_neighbors_by_type(2,k)\n","metadata":{"execution":{"iopub.status.busy":"2022-12-16T11:14:01.170492Z","iopub.execute_input":"2022-12-16T11:14:01.171289Z","iopub.status.idle":"2022-12-16T11:19:10.246402Z","shell.execute_reply.started":"2022-12-16T11:14:01.171217Z","shell.execute_reply":"2022-12-16T11:19:10.244947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"top_10_neighbors_clicks.filter(pl.col(\"aid\") == 121435)","metadata":{"execution":{"iopub.status.busy":"2022-12-16T11:20:58.247120Z","iopub.execute_input":"2022-12-16T11:20:58.247550Z","iopub.status.idle":"2022-12-16T11:20:58.272167Z","shell.execute_reply.started":"2022-12-16T11:20:58.247516Z","shell.execute_reply":"2022-12-16T11:20:58.270528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"top_10_neighbors_carts.filter(pl.col(\"aid\") == 121435)","metadata":{"execution":{"iopub.status.busy":"2022-12-16T11:20:51.878682Z","iopub.execute_input":"2022-12-16T11:20:51.879832Z","iopub.status.idle":"2022-12-16T11:20:51.905994Z","shell.execute_reply.started":"2022-12-16T11:20:51.879783Z","shell.execute_reply":"2022-12-16T11:20:51.905134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"top_10_neighbors_orders.filter(pl.col(\"aid\") == 121435)","metadata":{"execution":{"iopub.status.busy":"2022-12-16T11:21:04.394082Z","iopub.execute_input":"2022-12-16T11:21:04.394470Z","iopub.status.idle":"2022-12-16T11:21:04.419700Z","shell.execute_reply.started":"2022-12-16T11:21:04.394442Z","shell.execute_reply":"2022-12-16T11:21:04.419014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}