{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Here I extract the pagerank of each aid using the co-visitation matrix proposed by the community on this competition. I'm using the top 100 candidates for each aid, however because of memory issues I took only the `.head(70)`.","metadata":{}},{"cell_type":"code","source":"!pip3 install polars\n!pip3 install fast-pagerank\nfrom tqdm import tqdm\nfrom collections import defaultdict\nimport numpy as np\nimport pandas as pd\nimport gc\nimport torch\nfrom scipy import sparse\nfrom fast_pagerank import pagerank\nfrom fast_pagerank import pagerank_power\nimport polars as pl\nfrom sklearn.preprocessing import minmax_scale\n\nfrom tqdm import tqdm\nimport pandas as pd, numpy as np\nimport os, sys, pickle, glob, gc\nfrom collections import Counter\nimport itertools\nVER = 1\nimport pandas as pd, numpy as np\nimport pickle, glob, gc\n\n# multiprocessing \nimport psutil\nN_CORES = psutil.cpu_count()     # Available CPU cores\nprint(f\"N Cores : {N_CORES}\")\nfrom multiprocessing import Pool","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-01-19T22:32:49.283864Z","iopub.execute_input":"2023-01-19T22:32:49.284256Z","iopub.status.idle":"2023-01-19T22:33:11.488880Z","shell.execute_reply.started":"2023-01-19T22:32:49.284226Z","shell.execute_reply":"2023-01-19T22:33:11.487454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# LOAD THREE CO-VISITATION MATRICES\ndef pqt_to_dict(df):\n    return df.groupby('aid_x').aid_y.apply(list).to_dict()\n\ndef pr(edges,max_iter = 100,tol=1e-7):\n    weights = np.ones((len(edges),))\n    G = sparse.csr_matrix((weights,\n                       (edges[:,0], edges[:,1])\n                      )\n                     )\n    pr_values =pagerank_power(G, p=0.85,max_iter=max_iter, tol=tol)\n    return pr_values","metadata":{"execution":{"iopub.status.busy":"2023-01-19T22:33:11.491247Z","iopub.execute_input":"2023-01-19T22:33:11.491763Z","iopub.status.idle":"2023-01-19T22:33:11.500262Z","shell.execute_reply.started":"2023-01-19T22:33:11.491727Z","shell.execute_reply":"2023-01-19T22:33:11.498739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pagerank_aid(option,option_2):\n    DISK_PIECES = 5\n    candidates_number = 100\n    if option_2 == \"train\":\n        \n        top_aids = pqt_to_dict( pd.read_parquet(f'/kaggle/input/clicks-orders-carts-candidates/train/top_{candidates_number}_{option}_v6_0.pqt') )\n\n        for k in range(1, DISK_PIECES): \n            top_aids.update( pqt_to_dict( pd.read_parquet(f'/kaggle/input/clicks-orders-carts-candidates/train/top_{candidates_number}_{option}_v6_{k}.pqt') ) )\n    else:\n        top_aids = pqt_to_dict( pd.read_parquet(f'/kaggle/input/clicks-orders-carts-candidates/test/top_{candidates_number}_{option}_v6_0.pqt') )\n\n        for k in range(1, DISK_PIECES): \n            top_aids.update( pqt_to_dict( pd.read_parquet(f'/kaggle/input/clicks-orders-carts-candidates/test/top_{candidates_number}_{option}_v6_{k}.pqt') ) )        \n\n    top_aids_pd = pd.DataFrame()\n    top_aids_pd['aid'] = list(top_aids.keys())\n    top_aids_pd['co_visit_candidates'] = list(top_aids.values())\n\n\n    top_aids_pd = top_aids_pd.explode(\"co_visit_candidates\")\n\n\n    top_aids_pd = top_aids_pd.groupby(\"aid\").head(50)\n\n\n    edges = top_aids_pd[['aid','co_visit_candidates']].values\n\n\n    pagerank_vector = pr(edges,max_iter=100)\n    \n    return pagerank_vector","metadata":{"execution":{"iopub.status.busy":"2023-01-19T22:34:20.821836Z","iopub.execute_input":"2023-01-19T22:34:20.822519Z","iopub.status.idle":"2023-01-19T22:34:20.832940Z","shell.execute_reply.started":"2023-01-19T22:34:20.822474Z","shell.execute_reply":"2023-01-19T22:34:20.831480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Compute the page rank for each aid from each co-visitation matrix","metadata":{}},{"cell_type":"code","source":"%time pagerank_buy2buy = pagerank_aid(\"buy2buy\",\"train\")\n\n%time pagerank_carts_orders = pagerank_aid(\"carts_orders\",\"train\")\n\n%time pagerank_clicks = pagerank_aid(\"clicks\",\"train\")","metadata":{"execution":{"iopub.status.busy":"2023-01-19T22:34:21.824062Z","iopub.execute_input":"2023-01-19T22:34:21.824487Z","iopub.status.idle":"2023-01-19T22:43:44.028520Z","shell.execute_reply.started":"2023-01-19T22:34:21.824455Z","shell.execute_reply":"2023-01-19T22:43:44.027163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I noticed that `pagerank_buy2buy`  doesn't have the last item ( I verified it manually) then we'll add a zero value to this aid\n","metadata":{}},{"cell_type":"code","source":"pagerank_buy2buy.shape\npagerank_buy2buy = np.concatenate((pagerank_buy2buy,np.array([0.0])),axis=0)","metadata":{"execution":{"iopub.status.busy":"2023-01-19T22:43:56.812966Z","iopub.execute_input":"2023-01-19T22:43:56.813750Z","iopub.status.idle":"2023-01-19T22:43:56.827336Z","shell.execute_reply.started":"2023-01-19T22:43:56.813699Z","shell.execute_reply":"2023-01-19T22:43:56.826164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Create the dataframe","metadata":{}},{"cell_type":"code","source":"pr_df = pd.DataFrame()\npr_df['aid'] = np.arange(0,1855603,1)\n\npr_df['pagerank_buy2buy'] = pagerank_buy2buy\npr_df['pagerank_carts_orders'] = pagerank_carts_orders\npr_df['pagerank_clicks'] = pagerank_clicks\n\nfor col in pr_df.columns[-3:]:\n    pr_df[col] = pr_df[col].astype(\"float32\")\n    \npr_df[\"aid\"] = pr_df[\"aid\"].astype(\"int32\")\n\npr_df.to_parquet(\"train_pageRank_per_aid.parquet\",index=False)","metadata":{"execution":{"iopub.status.busy":"2023-01-19T22:45:07.829876Z","iopub.execute_input":"2023-01-19T22:45:07.830334Z","iopub.status.idle":"2023-01-19T22:45:08.340642Z","shell.execute_reply.started":"2023-01-19T22:45:07.830298Z","shell.execute_reply":"2023-01-19T22:45:08.339522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Same for test","metadata":{}},{"cell_type":"code","source":"%time pagerank_buy2buy_test = pagerank_aid(\"buy2buy\",\"test\")\n\n%time pagerank_carts_orders_test = pagerank_aid(\"carts_orders\",\"test\")\n\n%time pagerank_clicks_test = pagerank_aid(\"clicks\",\"test\")","metadata":{"execution":{"iopub.status.busy":"2023-01-19T22:45:10.007972Z","iopub.execute_input":"2023-01-19T22:45:10.008413Z","iopub.status.idle":"2023-01-19T22:54:53.353628Z","shell.execute_reply.started":"2023-01-19T22:45:10.008376Z","shell.execute_reply":"2023-01-19T22:54:53.352284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pagerank_buy2buy_test.shape\npagerank_buy2buy_test = np.concatenate((pagerank_buy2buy_test,np.array([0.0])),axis=0)","metadata":{"execution":{"iopub.status.busy":"2023-01-19T23:00:29.208783Z","iopub.execute_input":"2023-01-19T23:00:29.209861Z","iopub.status.idle":"2023-01-19T23:00:29.219737Z","shell.execute_reply.started":"2023-01-19T23:00:29.209819Z","shell.execute_reply":"2023-01-19T23:00:29.218793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pr_df = pd.DataFrame()\npr_df['aid'] = np.arange(0,1855603,1)\n\npr_df['pagerank_buy2buy'] = pagerank_buy2buy_test\npr_df['pagerank_carts_orders'] = pagerank_carts_orders_test\npr_df['pagerank_clicks'] = pagerank_clicks_test\n\nfor col in pr_df.columns[-3:]:\n    pr_df[col] = pr_df[col].astype(\"float32\")\n    \npr_df[\"aid\"] = pr_df[\"aid\"].astype(\"int32\")\n\npr_df.to_parquet(\"test_pageRank_per_aid.parquet\",index=False)","metadata":{"execution":{"iopub.status.busy":"2023-01-19T23:00:29.388062Z","iopub.execute_input":"2023-01-19T23:00:29.388568Z","iopub.status.idle":"2023-01-19T23:00:29.948646Z","shell.execute_reply.started":"2023-01-19T23:00:29.388529Z","shell.execute_reply":"2023-01-19T23:00:29.947403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}