{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import gc\nimport glob\nimport json\nimport os\nimport pickle\nfrom typing import List\n\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2023-01-29T02:32:15.001354Z","iopub.execute_input":"2023-01-29T02:32:15.001837Z","iopub.status.idle":"2023-01-29T02:32:15.029335Z","shell.execute_reply.started":"2023-01-29T02:32:15.001750Z","shell.execute_reply":"2023-01-29T02:32:15.028458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEBUG = False","metadata":{"execution":{"iopub.status.busy":"2023-01-29T02:32:15.030718Z","iopub.execute_input":"2023-01-29T02:32:15.031650Z","iopub.status.idle":"2023-01-29T02:32:15.036676Z","shell.execute_reply.started":"2023-01-29T02:32:15.031594Z","shell.execute_reply":"2023-01-29T02:32:15.035550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_working_dir():\n    paths = [\n        \"../working/train\",\n        \"../working/valid\",\n        \"../working/test\"\n    ]\n    \n    for p in paths:\n        if os.path.exists(p):\n            shutil.rmtree(p)\n\n    print(os.listdir(\"../working\"))\n    ","metadata":{"execution":{"iopub.status.busy":"2023-01-29T02:32:15.037793Z","iopub.execute_input":"2023-01-29T02:32:15.038072Z","iopub.status.idle":"2023-01-29T02:32:15.048409Z","shell.execute_reply.started":"2023-01-29T02:32:15.038045Z","shell.execute_reply":"2023-01-29T02:32:15.047591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clean_working_dir()","metadata":{"execution":{"iopub.status.busy":"2023-01-29T02:32:15.050194Z","iopub.execute_input":"2023-01-29T02:32:15.050947Z","iopub.status.idle":"2023-01-29T02:32:15.059569Z","shell.execute_reply.started":"2023-01-29T02:32:15.050919Z","shell.execute_reply":"2023-01-29T02:32:15.058762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Validation set(jsonl)\n\nhttps://www.kaggle.com/code/theoviel/pretraining-with-merlin-s-transformers4rec#1.-Create-a-validation-set\n\nhttps://www.kaggle.com/datasets/theoviel/split-otto","metadata":{}},{"cell_type":"code","source":"# trainset = \"../input/split-otto/train_sessions.jsonl\"\n# validset = \"../input/split-otto/val_sessions.jsonl\"\n\ntrainset = \"../input/otto-train-valid-split/out/train_sessions.jsonl\"\nvalidset = \"../input/otto-train-valid-split/out/valid_sessions.jsonl\"\n\ntestset = \"../input/otto-recommender-system/test.jsonl\"","metadata":{"execution":{"iopub.status.busy":"2023-01-29T02:32:15.060839Z","iopub.execute_input":"2023-01-29T02:32:15.061368Z","iopub.status.idle":"2023-01-29T02:32:15.070022Z","shell.execute_reply.started":"2023-01-29T02:32:15.061337Z","shell.execute_reply":"2023-01-29T02:32:15.068922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Candidates\n\n## \"Carts Orders\" Co-visitation Matrix - Type Weighted\n\nhttps://www.kaggle.com/code/cdeotte/candidate-rerank-model-lb-0-575?scriptVersionId=111214204&cellId=7\n\n## \"Buy2Buy\" Co-visitation Matrix\n\nhttps://www.kaggle.com/code/cdeotte/candidate-rerank-model-lb-0-575?scriptVersionId=111214204&cellId=9\n\n## \"Clicks\" Co-visitation Matrix - Time Weighted\n\nhttps://www.kaggle.com/code/cdeotte/candidate-rerank-model-lb-0-575?scriptVersionId=111214204&cellId=11","metadata":{}},{"cell_type":"code","source":"top_15_buy2buy_path = \"../input/candidate-rerank-model-lb-0-575/top_15_buy2buy_v5_0.pqt\"\ntop_15_carts_orders_path = glob.glob(\"../input/candidate-rerank-model-lb-0-575/top_15_carts_orders_v5_*.pqt\")\ntop_20_clicks_path = glob.glob(\"/kaggle/input/candidate-rerank-model-lb-0-575/top_20_clicks_v5_*.pqt\")","metadata":{"execution":{"iopub.status.busy":"2023-01-29T02:32:15.071595Z","iopub.execute_input":"2023-01-29T02:32:15.071957Z","iopub.status.idle":"2023-01-29T02:32:15.090756Z","shell.execute_reply.started":"2023-01-29T02:32:15.071922Z","shell.execute_reply":"2023-01-29T02:32:15.089662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pqt_to_dict(df):\n    return df.groupby('aid_x').aid_y.apply(list).to_dict()","metadata":{"execution":{"iopub.status.busy":"2023-01-29T02:32:15.091945Z","iopub.execute_input":"2023-01-29T02:32:15.093109Z","iopub.status.idle":"2023-01-29T02:32:15.098192Z","shell.execute_reply.started":"2023-01-29T02:32:15.093058Z","shell.execute_reply":"2023-01-29T02:32:15.096951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntop_20_clicks = pqt_to_dict(pd.read_parquet(top_20_clicks_path))\ntop_15_buy2buy = pqt_to_dict(pd.read_parquet(top_15_buy2buy_path))\ntop_15_carts_orders = pqt_to_dict(pd.read_parquet(top_15_carts_orders_path))","metadata":{"execution":{"iopub.status.busy":"2023-01-29T02:32:15.099693Z","iopub.execute_input":"2023-01-29T02:32:15.100319Z","iopub.status.idle":"2023-01-29T02:33:54.726621Z","shell.execute_reply.started":"2023-01-29T02:32:15.100280Z","shell.execute_reply":"2023-01-29T02:33:54.725270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('top_20_clicks.pkl', 'wb') as f:\n    pickle.dump(top_20_clicks, f)\n\nwith open('top_15_buy2buy.pkl', 'wb') as f:\n    pickle.dump(top_15_buy2buy, f)\n    \nwith open('top_15_carts_orders.pkl', 'wb') as f:\n    pickle.dump(top_15_carts_orders, f)","metadata":{"execution":{"iopub.status.busy":"2023-01-29T02:33:54.728033Z","iopub.execute_input":"2023-01-29T02:33:54.728320Z","iopub.status.idle":"2023-01-29T02:33:58.609170Z","shell.execute_reply.started":"2023-01-29T02:33:54.728297Z","shell.execute_reply":"2023-01-29T02:33:58.607696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ncandidates = np.concatenate(list(top_20_clicks.values()))\ncandidates_unique, candidates_counts = np.unique(candidates, return_counts=True)\n\ncounts_sort_index = np.argsort(-candidates_counts)","metadata":{"execution":{"iopub.status.busy":"2023-01-29T02:33:58.613029Z","iopub.execute_input":"2023-01-29T02:33:58.613557Z","iopub.status.idle":"2023-01-29T02:34:04.639467Z","shell.execute_reply.started":"2023-01-29T02:33:58.613525Z","shell.execute_reply":"2023-01-29T02:34:04.638778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 19998 candidates, idx=0 for PAD, idx=1 for OOV, idx=2 ~ 19999 for candidates\nMAX_CANDIDATES = 20000 - 2","metadata":{"execution":{"iopub.status.busy":"2023-01-29T02:34:04.640358Z","iopub.execute_input":"2023-01-29T02:34:04.640891Z","iopub.status.idle":"2023-01-29T02:34:04.644455Z","shell.execute_reply.started":"2023-01-29T02:34:04.640867Z","shell.execute_reply":"2023-01-29T02:34:04.643668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"candidates_filtered = candidates_unique[counts_sort_index][:MAX_CANDIDATES]\ncounts_filtered = candidates_counts[counts_sort_index][:MAX_CANDIDATES]\n\ncandidates_filtered.shape, counts_filtered.shape, candidates_filtered, counts_filtered","metadata":{"execution":{"iopub.status.busy":"2023-01-29T02:34:04.645459Z","iopub.execute_input":"2023-01-29T02:34:04.645712Z","iopub.status.idle":"2023-01-29T02:34:04.700007Z","shell.execute_reply.started":"2023-01-29T02:34:04.645689Z","shell.execute_reply":"2023-01-29T02:34:04.698596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.save(\"candidates.npy\", candidates_filtered)","metadata":{"execution":{"iopub.status.busy":"2023-01-29T02:34:04.701455Z","iopub.execute_input":"2023-01-29T02:34:04.701813Z","iopub.status.idle":"2023-01-29T02:34:04.706973Z","shell.execute_reply.started":"2023-01-29T02:34:04.701781Z","shell.execute_reply":"2023-01-29T02:34:04.705723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create parquet dataset with co-visitation","metadata":{}},{"cell_type":"code","source":"# session\n# evnet_type\n# top_20_clicks-list\n# top_15_buy2buy-list\n# top_15_carts_orders-list\n# aid-list\n# ts-list","metadata":{"execution":{"iopub.status.busy":"2023-01-29T02:34:04.708367Z","iopub.execute_input":"2023-01-29T02:34:04.708633Z","iopub.status.idle":"2023-01-29T02:34:04.718481Z","shell.execute_reply.started":"2023-01-29T02:34:04.708608Z","shell.execute_reply":"2023-01-29T02:34:04.717365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def event_type_to_int(event_types):\n    \"\"\"\n    event_type\n    1 : session with only clicks\n    2 : session with carts, without orders\n    3 : session with orders\n    \"\"\"\n    \n    event_type = 1\n    \n    if \"carts\" in event_types:\n        event_type = 2\n    \n    if \"orders\" in event_types:\n        event_type = 3\n        \n    return event_type","metadata":{"execution":{"iopub.status.busy":"2023-01-29T02:34:04.721308Z","iopub.execute_input":"2023-01-29T02:34:04.722419Z","iopub.status.idle":"2023-01-29T02:34:04.729784Z","shell.execute_reply.started":"2023-01-29T02:34:04.722384Z","shell.execute_reply":"2023-01-29T02:34:04.728330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_labels(events: List[dict]):\n    prev_labels = {\"clicks\": None, \"carts\": set(), \"orders\": set()}\n\n    labels_clicks = events[-1][\"aid\"] if events[-1][\"type\"] == \"clicks\" else 0\n    \n    carts = []\n    orders = []\n    for event in events:\n        if event[\"type\"] == \"carts\":\n            carts.append(event[\"aid\"])\n        elif event[\"type\"] == \"clicks\":\n            orders.append(event[\"aid\"])\n\n    labels_carts = set(carts) if carts else [0]\n    labels_orders = set(orders) if orders else [0]\n\n    return labels_clicks, labels_carts, labels_orders","metadata":{"execution":{"iopub.status.busy":"2023-01-29T02:34:04.731587Z","iopub.execute_input":"2023-01-29T02:34:04.732021Z","iopub.status.idle":"2023-01-29T02:34:04.743390Z","shell.execute_reply.started":"2023-01-29T02:34:04.731980Z","shell.execute_reply":"2023-01-29T02:34:04.741590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def jsonl_to_parquet(filename, output=\"train\", debug=False):\n    chunks = pd.read_json(filename, lines=True, chunksize=100_000)\n    \n    for idx, chunk in tqdm(enumerate(chunks)):\n        session_list, event_type_list = [], []\n        top_20_clicks_list, top_15_buy2buy_list, top_15_carts_orders_list = [], [], []\n        aids_list, ts_list = [], []\n        labels_clicks, labels_carts, labels_orders =[], [], []\n        \n        for row, sample in chunk.iterrows():\n            aids_, ts_ = [], []\n            top_20_clicks_, top_15_buy2buy_, top_15_carts_orders_ = [], [], []\n            labels_clicks_, labels_carts_, labels_orders_ = [], [], []\n            \n            session = sample[\"session\"]\n            aids_ = [event[\"aid\"] for event in sample[\"events\"]]\n            ts_ = [event[\"ts\"] for event in sample[\"events\"]]\n            event_types = [event[\"type\"] for event in sample[\"events\"]]\n            event_type = event_type_to_int(event_types)\n\n            # co-visitation matrix items retrieved from last aids_ click\n            top_20_clicks_ = top_20_clicks.get(aids_[-1], 20 * [0])\n            top_15_buy2buy_ = top_15_buy2buy.get(aids_[-1], 15 * [0])\n            top_15_carts_orders_ = top_15_carts_orders.get(aids_[-1], 15 * [0])\n\n            labels_clicks_, labels_carts_, labels_orders_ = get_labels(sample[\"events\"])\n            \n            session_list.append(session)\n            event_type_list.append(event_type)\n            \n            top_20_clicks_list.append(top_20_clicks_)\n            top_15_buy2buy_list.append(top_15_buy2buy_)\n            top_15_carts_orders_list.append(top_15_carts_orders_)\n            \n            aids_list.append(aids_)\n            ts_list.append(ts_)\n            \n            labels_clicks.append(labels_clicks_)\n            labels_carts.append(labels_carts_)\n            labels_orders.append(labels_orders_)            \n                \n            \n        df = pd.DataFrame(data={\n            \"session\": session_list,\n            \"event_type\": event_type_list,\n            \"top_20_clicks-list\": top_20_clicks_list,\n            \"top_15_buy2buy-list\": top_15_buy2buy_list,\n            \"top_15_carts_orders-list\": top_15_carts_orders_list,\n            \"aid-list\": aids_list,\n            \"ts-list\": ts_list,\n            \"labels_clicks\": labels_clicks,\n            \"labels_carts\": labels_carts,\n            \"labels_orders\": labels_orders,\n        })\n        \n        df.to_parquet(f\"{output}_{idx}.parquet\", index=False)\n        gc.collect()\n        \n        if debug:\n            break","metadata":{"execution":{"iopub.status.busy":"2023-01-29T02:34:04.746022Z","iopub.execute_input":"2023-01-29T02:34:04.746555Z","iopub.status.idle":"2023-01-29T02:34:04.761712Z","shell.execute_reply.started":"2023-01-29T02:34:04.746515Z","shell.execute_reply":"2023-01-29T02:34:04.760243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"jsonl_to_parquet(trainset, \"train\", debug=DEBUG)","metadata":{"execution":{"iopub.status.busy":"2023-01-29T02:34:04.763652Z","iopub.execute_input":"2023-01-29T02:34:04.764883Z","iopub.status.idle":"2023-01-29T02:34:31.009765Z","shell.execute_reply.started":"2023-01-29T02:34:04.764851Z","shell.execute_reply":"2023-01-29T02:34:31.008729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"jsonl_to_parquet(validset, \"valid\", debug=DEBUG)","metadata":{"execution":{"iopub.status.busy":"2023-01-29T02:34:31.012100Z","iopub.execute_input":"2023-01-29T02:34:31.012443Z","iopub.status.idle":"2023-01-29T02:34:43.590716Z","shell.execute_reply.started":"2023-01-29T02:34:31.012416Z","shell.execute_reply":"2023-01-29T02:34:43.588790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"jsonl_to_parquet(testset, \"test\", debug=DEBUG)","metadata":{"execution":{"iopub.status.busy":"2023-01-29T02:34:43.592047Z","iopub.execute_input":"2023-01-29T02:34:43.592362Z","iopub.status.idle":"2023-01-29T02:34:56.200157Z","shell.execute_reply.started":"2023-01-29T02:34:43.592335Z","shell.execute_reply":"2023-01-29T02:34:56.199155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_parquet(\"../working/train_0.parquet\")\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-29T02:34:56.201672Z","iopub.execute_input":"2023-01-29T02:34:56.201954Z","iopub.status.idle":"2023-01-29T02:34:56.605185Z","shell.execute_reply.started":"2023-01-29T02:34:56.201928Z","shell.execute_reply":"2023-01-29T02:34:56.603737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_df = pd.read_parquet(\"../working/valid_0.parquet\")\nvalid_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-29T02:34:56.607043Z","iopub.execute_input":"2023-01-29T02:34:56.607424Z","iopub.status.idle":"2023-01-29T02:34:56.864613Z","shell.execute_reply.started":"2023-01-29T02:34:56.607391Z","shell.execute_reply":"2023-01-29T02:34:56.863349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_parquet(\"../working/test_0.parquet\")\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-29T02:34:56.866004Z","iopub.execute_input":"2023-01-29T02:34:56.866576Z","iopub.status.idle":"2023-01-29T02:34:57.179276Z","shell.execute_reply.started":"2023-01-29T02:34:56.866546Z","shell.execute_reply":"2023-01-29T02:34:57.178090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls ../working/*.parquet","metadata":{"execution":{"iopub.status.busy":"2023-01-29T02:34:57.182732Z","iopub.execute_input":"2023-01-29T02:34:57.183606Z","iopub.status.idle":"2023-01-29T02:34:57.542655Z","shell.execute_reply.started":"2023-01-29T02:34:57.183563Z","shell.execute_reply":"2023-01-29T02:34:57.541487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}