{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# OTTO: Co-visitation Matrix\n\nThere exist products that are frequently viewed and bought together. Here we leverage this idea by computing a co-visitation matrix of products. It's done in the following way:\n\n1. First we look at all pairs of events within the same session that are close to each other in time (< 1 day). We compute co-visitation matrix $M_{aid1,aid2}$ by counting global number of event pairs for each pair across all sessions.\n2. For each $aid1$ we find top 20 most frequent aid2:  `aid2=argsort(M[aid])[-20:]`\n3. We produce test results by concatenating `tail(20)` of test session events (see https://www.kaggle.com/code/simamumu/old-test-data-last-20-aid-get-lb0-947) with the most likely recommendations from co-visitation matrix. These recommendations are generated from session AIDs and `aid2` from the step 2\n\n\n**Please, smash that thumbs up button and subscribe if you like this notebook!**","metadata":{}},{"cell_type":"markdown","source":"## Utils, imports","metadata":{}},{"cell_type":"code","source":"### import numpy as np\nfrom collections import defaultdict\nimport pandas as pd\nfrom tqdm.notebook import tqdm\nimport glob\nimport numpy as np\nimport multiprocessing\nimport os\nimport pickle\n\nimport glob\nfrom collections import Counter\n\nDEBUG=False   \nSAMPLING = 1  # Reduce it to improve performance\n\n","metadata":{"tags":[],"execution":{"iopub.status.busy":"2022-11-04T07:55:50.536587Z","iopub.execute_input":"2022-11-04T07:55:50.538104Z","iopub.status.idle":"2022-11-04T07:55:50.682463Z","shell.execute_reply.started":"2022-11-04T07:55:50.537979Z","shell.execute_reply":"2022-11-04T07:55:50.681184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TOP_40_CACHE = 'top_40_pairs.pkl'\ntry:\n    from kaggle_secrets import UserSecretsClient\n    user_secrets = UserSecretsClient()\n    secret_value_0 = user_secrets.get_secret(\"gcloud\")\n\n    with open('/tmp/json', 'w+') as f:\n        f.write(secret_value_0)\n        \n    !gcloud auth login --cred-file /tmp/json    \n    !gsutil cp gs://nesp/top_40_pairs.pkl .        \n        \nexcept Exception  as ex:\n    pass\n","metadata":{"tags":[],"execution":{"iopub.status.busy":"2022-11-04T07:55:52.231803Z","iopub.execute_input":"2022-11-04T07:55:52.233013Z","iopub.status.idle":"2022-11-04T07:56:19.442949Z","shell.execute_reply.started":"2022-11-04T07:55:52.232957Z","shell.execute_reply":"2022-11-04T07:56:19.441363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Generate AID pairs","metadata":{}},{"cell_type":"code","source":"import sys\nimport gc\ndef gen_pairs(df):\n    df = df.query('session % @SAMPLING == 0').groupby('session', as_index=False, sort=False).apply(lambda g: g.tail(30)).reset_index(drop=True)\n    df = pd.merge(df, df, on='session')\n    pairs = df.query('abs(ts_x - ts_y) < 24 * 60 * 60 * 1000 and aid_x != aid_y')[['session', 'aid_x', 'aid_y']].drop_duplicates()\n    return pairs[['aid_x', 'aid_y']].values\n    \n\n    \n    \ndef gen_aid_pairs():\n    all_pairs = defaultdict(lambda: Counter())\n    all_pair_chunks = []\n    with tqdm(glob.glob('../input/otto-chunk-data-inparquet-format/*_parquet/*'), desc='Chunks') as prog:\n        for idx, chunk_file in enumerate(prog):\n            with multiprocessing.Pool() as p:            \n                chunk = pd.read_parquet(chunk_file).drop(columns=['type'])\n                pair_chunks = p.map(gen_pairs, np.array_split(chunk, 120))            \n                pair_chunks = np.concatenate(pair_chunks, axis=0)\n                all_pair_chunks.append(pair_chunks)\n\n                if DEBUG and idx >= 3:\n                    break\n                del chunk, pair_chunks\n                gc.collect()\n                \n                \n    df = pd.DataFrame(data=np.concatenate(all_pair_chunks), columns=['aid1', 'aid2'])\n    top_aids = df.groupby('aid1').apply(lambda df: Counter(df.aid2).most_common(40)).to_dict()\n    return top_aids","metadata":{"tags":[],"execution":{"iopub.status.busy":"2022-11-04T07:56:19.445326Z","iopub.execute_input":"2022-11-04T07:56:19.445715Z","iopub.status.idle":"2022-11-04T07:56:19.459457Z","shell.execute_reply.started":"2022-11-04T07:56:19.445678Z","shell.execute_reply":"2022-11-04T07:56:19.458113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if os.path.exists(TOP_40_CACHE):\n    print('Reading top40 AIDs from cache')\n    top_40 = pickle.load(open(TOP_40_CACHE, 'rb'))\nelse:\n    top_40 = gen_aid_pairs()    \n    with open('top_40_pairs.pkl', 'wb') as f:\n        pickle.dump(top_40, f)\n    !gsutil cp top_40_pairs.pkl gs://nesp/\n        \nlen(top_40)","metadata":{"execution":{"iopub.status.busy":"2022-11-04T07:56:19.461376Z","iopub.execute_input":"2022-11-04T07:56:19.461734Z","iopub.status.idle":"2022-11-04T07:56:47.920224Z","shell.execute_reply.started":"2022-11-04T07:56:19.461701Z","shell.execute_reply":"2022-11-04T07:56:47.919301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, (k, v) in enumerate(top_40.items()):\n    print(k, v)\n    if i > 10:\n        break","metadata":{"tags":[],"execution":{"iopub.status.busy":"2022-11-04T07:56:47.922233Z","iopub.execute_input":"2022-11-04T07:56:47.923287Z","iopub.status.idle":"2022-11-04T07:56:47.930109Z","shell.execute_reply.started":"2022-11-04T07:56:47.923244Z","shell.execute_reply":"2022-11-04T07:56:47.928889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Test set inference","metadata":{}},{"cell_type":"code","source":"top_40_cnt = {aid: Counter(dict(top)) for aid, top in top_40.items()}","metadata":{"execution":{"iopub.status.busy":"2022-11-04T07:59:33.229472Z","iopub.execute_input":"2022-11-04T07:59:33.229893Z","iopub.status.idle":"2022-11-04T08:00:01.486005Z","shell.execute_reply.started":"2022-11-04T07:59:33.229861Z","shell.execute_reply":"2022-11-04T08:00:01.484714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, (k, v) in enumerate(top_40_cnt.items()):\n    print(k, v)\n    if i > 3:\n        break","metadata":{"execution":{"iopub.status.busy":"2022-11-04T08:00:26.329840Z","iopub.execute_input":"2022-11-04T08:00:26.330294Z","iopub.status.idle":"2022-11-04T08:00:26.337094Z","shell.execute_reply.started":"2022-11-04T08:00:26.330257Z","shell.execute_reply":"2022-11-04T08:00:26.335788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_test():    \n    dfs = []\n    for e, chunk_file in enumerate(tqdm(glob.glob('../input/otto-chunk-data-inparquet-format/test_parquet/*'))):\n        chunk = pd.read_parquet(chunk_file)\n        dfs.append(chunk)\n\n    return pd.concat(dfs).reset_index(drop=True).astype({\"ts\": \"datetime64[ms]\"})","metadata":{"execution":{"iopub.status.busy":"2022-11-04T07:57:00.289628Z","iopub.execute_input":"2022-11-04T07:57:00.290021Z","iopub.status.idle":"2022-11-04T07:57:00.297026Z","shell.execute_reply.started":"2022-11-04T07:57:00.289989Z","shell.execute_reply":"2022-11-04T07:57:00.295523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = load_test()","metadata":{"execution":{"iopub.status.busy":"2022-11-04T07:57:00.596813Z","iopub.execute_input":"2022-11-04T07:57:00.597802Z","iopub.status.idle":"2022-11-04T07:57:04.560292Z","shell.execute_reply.started":"2022-11-04T07:57:00.597749Z","shell.execute_reply":"2022-11-04T07:57:04.558792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import itertools\n\ndef suggest_aids(df):\n    aids = df.tail(20).aid.tolist()\n    \n    if len(aids) >= 20:\n        # We have enough events in the test session\n        return aids\n    \n    # Append it with AIDs from the co-visitation matrix. \n    aids = set(aids)\n    new_aids = Counter()\n    for aid in aids:\n        new_aids.update(top_40_cnt.get(aid, Counter()))\n    \n    top_aids2 = [aid2 for aid2, cnt in new_aids.most_common(20) if aid2 not in aids]        \n    return list(aids) + top_aids2[:20 - len(aids)]\n\n        ","metadata":{"execution":{"iopub.status.busy":"2022-11-04T08:04:57.229764Z","iopub.execute_input":"2022-11-04T08:04:57.230219Z","iopub.status.idle":"2022-11-04T08:04:57.238493Z","shell.execute_reply.started":"2022-11-04T08:04:57.230173Z","shell.execute_reply":"2022-11-04T08:04:57.237265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_df = test_df.sort_values([\"session\", \"type\", \"ts\"]).groupby([\"session\"]).apply(\n    lambda x: suggest_aids(x)\n)","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2022-11-04T08:04:58.373449Z","iopub.execute_input":"2022-11-04T08:04:58.373837Z","iopub.status.idle":"2022-11-04T08:12:39.747989Z","shell.execute_reply.started":"2022-11-04T08:04:58.373806Z","shell.execute_reply":"2022-11-04T08:12:39.746621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clicks_pred_df = pd.DataFrame(pred_df.add_suffix(\"_clicks\"), columns=[\"labels\"]).reset_index()\norders_pred_df = pd.DataFrame(pred_df.add_suffix(\"_orders\"), columns=[\"labels\"]).reset_index()\ncarts_pred_df = pd.DataFrame(pred_df.add_suffix(\"_carts\"), columns=[\"labels\"]).reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-11-04T08:12:39.750277Z","iopub.execute_input":"2022-11-04T08:12:39.750667Z","iopub.status.idle":"2022-11-04T08:12:43.204734Z","shell.execute_reply.started":"2022-11-04T08:12:39.750633Z","shell.execute_reply":"2022-11-04T08:12:43.203533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_df","metadata":{"execution":{"iopub.status.busy":"2022-11-04T08:12:43.206203Z","iopub.execute_input":"2022-11-04T08:12:43.206710Z","iopub.status.idle":"2022-11-04T08:12:43.220088Z","shell.execute_reply.started":"2022-11-04T08:12:43.206665Z","shell.execute_reply":"2022-11-04T08:12:43.218492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_df = pd.concat(\n    [clicks_pred_df, orders_pred_df, carts_pred_df]\n)\npred_df.columns = [\"session_type\", \"labels\"]\npred_df[\"labels\"] = pred_df.labels.apply(lambda x: \" \".join(map(str,x)))\npred_df.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-11-04T08:12:43.222264Z","iopub.execute_input":"2022-11-04T08:12:43.222628Z","iopub.status.idle":"2022-11-04T08:13:33.856360Z","shell.execute_reply.started":"2022-11-04T08:12:43.222598Z","shell.execute_reply":"2022-11-04T08:13:33.855008Z"},"trusted":true},"execution_count":null,"outputs":[]}]}