{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nfrom IPython.display import display\nfrom tqdm import tqdm\nimport json\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-01-20T13:08:06.027792Z","iopub.execute_input":"2023-01-20T13:08:06.028287Z","iopub.status.idle":"2023-01-20T13:08:06.070697Z","shell.execute_reply.started":"2023-01-20T13:08:06.028193Z","shell.execute_reply":"2023-01-20T13:08:06.069402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_base_path = \"/kaggle/input/otto-recommender-system/\"\ntest_path = os.path.join(input_base_path, \"test.jsonl\")\ntrain_path = os.path.join(input_base_path, \"train.jsonl\")","metadata":{"execution":{"iopub.status.busy":"2023-01-20T13:08:10.063640Z","iopub.execute_input":"2023-01-20T13:08:10.064068Z","iopub.status.idle":"2023-01-20T13:08:10.070888Z","shell.execute_reply.started":"2023-01-20T13:08:10.064028Z","shell.execute_reply":"2023-01-20T13:08:10.069078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type_mapping = {\n    \"clicks\": 0,\n    \"carts\": 1,\n    \"orders\": 2\n}\n\ndef create_df(filepath):\n#     data = {\n#         \"session\": [], #np.array([], dtype=\"int32\"),\n#         \"aid\": [], #np.array([], dtype=\"int32\"),\n#         \"ts\": [], #np.array([], dtype=\"int32\"),\n#         \"type\": [], #np.array([], dtype=\"int8\")\n#     }\n    \n    \n    n = 6928123\n    data = {\n        \"session\": np.zeros((n,), dtype=\"int32\"),\n        \"aid\": np.zeros((n,), dtype=\"int32\"),\n        \"ts\": np.zeros((n,), dtype=\"int32\"),\n        \"type\": np.zeros((n,), dtype=\"int8\")\n    }\n    \n    #print(data[\"session\"].shape)\n    \n    with open(filepath, \"r\") as f:\n        cnt = 0\n        for line in tqdm(f):\n            obj = json.loads(line)\n            \n            for event in obj[\"events\"]:\n                aid = event[\"aid\"]\n                ts = event[\"ts\"]\n                typ = event[\"type\"]\n                \n                #np.append(data[\"session\"], [obj[\"session\"]])\n                #np.append(data[\"aid\"], [aid])\n                #np.append(data[\"ts\"], [ts])\n                #np.append(data[\"type\"], [type_mapping[typ]])\n                \n                data[\"session\"][cnt] = obj[\"session\"]\n                data[\"aid\"][cnt] = aid\n                data[\"ts\"][cnt] = ts\n                data[\"type\"][cnt] = type_mapping[typ]\n                \n#                 data[\"session\"].append(obj[\"session\"])\n#                 data[\"aid\"].append(aid)\n#                 data[\"ts\"].append(ts)\n#                 data[\"type\"].append(type_mapping[typ])\n                \n                cnt += 1\n                #row = [obj[\"session\"], aid, ts, type_mapping[typ]]\n                #df = df.append({\"session\": obj[\"session\"], \"aid\": aid, \"ts\": ts, \"type\": type_mapping[type]})\n                #df.loc[len(df)] = row\n            #cnt += 1\n            #if cnt == 10:\n            #    return data\n    return data","metadata":{"execution":{"iopub.status.busy":"2023-01-20T13:12:28.163307Z","iopub.execute_input":"2023-01-20T13:12:28.163688Z","iopub.status.idle":"2023-01-20T13:12:28.174428Z","shell.execute_reply.started":"2023-01-20T13:12:28.163658Z","shell.execute_reply":"2023-01-20T13:12:28.172757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = create_df(test_path)","metadata":{"execution":{"iopub.status.busy":"2023-01-20T13:12:29.019056Z","iopub.execute_input":"2023-01-20T13:12:29.019998Z","iopub.status.idle":"2023-01-20T13:12:45.111303Z","shell.execute_reply.started":"2023-01-20T13:12:29.019961Z","shell.execute_reply":"2023-01-20T13:12:45.109985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame(data)","metadata":{"execution":{"iopub.status.busy":"2023-01-20T13:12:49.956415Z","iopub.execute_input":"2023-01-20T13:12:49.956839Z","iopub.status.idle":"2023-01-20T13:12:49.980195Z","shell.execute_reply.started":"2023-01-20T13:12:49.956802Z","shell.execute_reply":"2023-01-20T13:12:49.978539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.info()","metadata":{"execution":{"iopub.status.busy":"2023-01-20T13:12:52.975432Z","iopub.execute_input":"2023-01-20T13:12:52.975871Z","iopub.status.idle":"2023-01-20T13:12:52.987585Z","shell.execute_reply.started":"2023-01-20T13:12:52.975837Z","shell.execute_reply":"2023-01-20T13:12:52.986342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.to_parquet(\"/kaggle/working/test_data.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-01-20T13:13:05.403864Z","iopub.execute_input":"2023-01-20T13:13:05.404292Z","iopub.status.idle":"2023-01-20T13:13:05.901545Z","shell.execute_reply.started":"2023-01-20T13:13:05.404265Z","shell.execute_reply":"2023-01-20T13:13:05.900637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = {\n    \"session\": [],\n    \"aid\": [],\n    \"ts\": [],\n    \"type\": []\n}\ndf = pd.DataFrame(data)\ndf['type'] = df['type'].astype(\"int8\")\ndf.info()","metadata":{"execution":{"iopub.status.busy":"2023-01-20T10:18:37.147963Z","iopub.execute_input":"2023-01-20T10:18:37.148544Z","iopub.status.idle":"2023-01-20T10:18:37.167202Z","shell.execute_reply.started":"2023-01-20T10:18:37.148488Z","shell.execute_reply":"2023-01-20T10:18:37.165595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(train_path, \"r\") as f:\n    counts = {}\n    \n    for line in tqdm(f):\n        obj = json.loads(line)\n        \n        for event in obj[\"events\"]:\n            aid = event[\"aid\"]\n            \n            if aid not in counts:\n                counts[aid] = 0\n                \n            counts[aid] += 1","metadata":{"execution":{"iopub.status.busy":"2023-01-15T06:25:30.677234Z","iopub.execute_input":"2023-01-15T06:25:30.678180Z","iopub.status.idle":"2023-01-15T06:33:08.100464Z","shell.execute_reply.started":"2023-01-15T06:25:30.678104Z","shell.execute_reply":"2023-01-15T06:33:08.098526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame.from_dict(counts, orient=\"index\", columns=[\"count\"])","metadata":{"execution":{"iopub.status.busy":"2023-01-15T06:35:19.960627Z","iopub.execute_input":"2023-01-15T06:35:19.961101Z","iopub.status.idle":"2023-01-15T06:35:21.239757Z","shell.execute_reply.started":"2023-01-15T06:35:19.961056Z","shell.execute_reply":"2023-01-15T06:35:21.238361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.index.name = \"aid\"\ndf = df.reset_index()\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-15T06:35:21.242417Z","iopub.execute_input":"2023-01-15T06:35:21.242967Z","iopub.status.idle":"2023-01-15T06:35:21.266531Z","shell.execute_reply.started":"2023-01-15T06:35:21.242910Z","shell.execute_reply":"2023-01-15T06:35:21.265219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Number of products: {df.shape[0]: ,}\")","metadata":{"execution":{"iopub.status.busy":"2023-01-15T06:36:04.151110Z","iopub.execute_input":"2023-01-15T06:36:04.152419Z","iopub.status.idle":"2023-01-15T06:36:04.159364Z","shell.execute_reply.started":"2023-01-15T06:36:04.152360Z","shell.execute_reply":"2023-01-15T06:36:04.157889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df.sort_values('count', ascending=False)\n\n# top 20 most popular products\ntop_20 = df.head(20)","metadata":{"execution":{"iopub.status.busy":"2023-01-15T06:38:05.465010Z","iopub.execute_input":"2023-01-15T06:38:05.465602Z","iopub.status.idle":"2023-01-15T06:38:05.766867Z","shell.execute_reply.started":"2023-01-15T06:38:05.465558Z","shell.execute_reply":"2023-01-15T06:38:05.765217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"top_20","metadata":{"execution":{"iopub.status.busy":"2023-01-15T06:38:14.382506Z","iopub.execute_input":"2023-01-15T06:38:14.383078Z","iopub.status.idle":"2023-01-15T06:38:14.398992Z","shell.execute_reply.started":"2023-01-15T06:38:14.383023Z","shell.execute_reply":"2023-01-15T06:38:14.397475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"aids = top_20[\"aid\"].values\naids","metadata":{"execution":{"iopub.status.busy":"2023-01-15T06:40:13.236669Z","iopub.execute_input":"2023-01-15T06:40:13.237186Z","iopub.status.idle":"2023-01-15T06:40:13.245621Z","shell.execute_reply.started":"2023-01-15T06:40:13.237143Z","shell.execute_reply":"2023-01-15T06:40:13.244583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"string_values = [str(x) for x in aids]\naids_str = ' '.join(string_values)","metadata":{"execution":{"iopub.status.busy":"2023-01-15T06:41:16.139241Z","iopub.execute_input":"2023-01-15T06:41:16.139857Z","iopub.status.idle":"2023-01-15T06:41:16.147352Z","shell.execute_reply.started":"2023-01-15T06:41:16.139813Z","shell.execute_reply":"2023-01-15T06:41:16.145837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df.to_csv(os.path.join(input_base_path, 'aid_counts_train.csv'), index=False, header=True)\ninput_base_path","metadata":{"execution":{"iopub.status.busy":"2023-01-15T06:42:52.312639Z","iopub.execute_input":"2023-01-15T06:42:52.313205Z","iopub.status.idle":"2023-01-15T06:42:52.322038Z","shell.execute_reply.started":"2023-01-15T06:42:52.313155Z","shell.execute_reply":"2023-01-15T06:42:52.320841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.to_csv(os.path.join(\"/kaggle/working/\", 'aid_counts_train.csv'), index=False, header=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-15T06:43:07.816525Z","iopub.execute_input":"2023-01-15T06:43:07.817417Z","iopub.status.idle":"2023-01-15T06:43:09.814698Z","shell.execute_reply.started":"2023-01-15T06:43:07.817366Z","shell.execute_reply":"2023-01-15T06:43:09.813379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read the CSV file into a DataFrame\nsubmit = pd.read_csv(os.path.join(input_base_path, 'sample_submission.csv'))\n\nsubmit.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-15T06:44:31.442505Z","iopub.execute_input":"2023-01-15T06:44:31.443042Z","iopub.status.idle":"2023-01-15T06:44:38.738944Z","shell.execute_reply.started":"2023-01-15T06:44:31.442993Z","shell.execute_reply":"2023-01-15T06:44:38.737622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit['labels'] = aids_str\nsubmit.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-15T06:45:20.506424Z","iopub.execute_input":"2023-01-15T06:45:20.506972Z","iopub.status.idle":"2023-01-15T06:45:20.599620Z","shell.execute_reply.started":"2023-01-15T06:45:20.506932Z","shell.execute_reply":"2023-01-15T06:45:20.595107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-15T06:45:33.624222Z","iopub.execute_input":"2023-01-15T06:45:33.624714Z","iopub.status.idle":"2023-01-15T06:45:33.631693Z","shell.execute_reply.started":"2023-01-15T06:45:33.624672Z","shell.execute_reply":"2023-01-15T06:45:33.630752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit.to_csv(os.path.join(\"/kaggle/working/\", 'submission.csv'), index=False, header=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-15T06:46:13.869554Z","iopub.execute_input":"2023-01-15T06:46:13.870042Z","iopub.status.idle":"2023-01-15T06:46:34.458863Z","shell.execute_reply.started":"2023-01-15T06:46:13.870004Z","shell.execute_reply":"2023-01-15T06:46:34.456591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prediction_on_test():\n    return","metadata":{"execution":{"iopub.status.busy":"2023-01-15T06:57:12.843034Z","iopub.execute_input":"2023-01-15T06:57:12.843900Z","iopub.status.idle":"2023-01-15T06:57:12.849370Z","shell.execute_reply.started":"2023-01-15T06:57:12.843850Z","shell.execute_reply":"2023-01-15T06:57:12.848100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data_dict = {\n    \"session_type\": []\n}\nwith open(os.path.join(input_base_path, \"test.jsonl\")) as f:\n    cnt = 0\n    for line in tqdm(f):\n        obj = json.loads(line)\n        sess = str(obj[\"session\"])\n        test_data_dict[\"session_type\"].append(sess+\"_clicks\")\n        test_data_dict[\"session_type\"].append(sess+\"_carts\")\n        test_data_dict[\"session_type\"].append(sess+\"_orders\")","metadata":{"execution":{"iopub.status.busy":"2023-01-15T07:23:08.966962Z","iopub.execute_input":"2023-01-15T07:23:08.967504Z","iopub.status.idle":"2023-01-15T07:23:24.775473Z","shell.execute_reply.started":"2023-01-15T07:23:08.967461Z","shell.execute_reply":"2023-01-15T07:23:24.773603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data_dict[\"labels\"] = [aids_str]*len(test_data_dict[\"session_type\"])","metadata":{"execution":{"iopub.status.busy":"2023-01-15T07:25:22.995816Z","iopub.execute_input":"2023-01-15T07:25:22.996321Z","iopub.status.idle":"2023-01-15T07:25:23.032036Z","shell.execute_reply.started":"2023-01-15T07:25:22.996280Z","shell.execute_reply":"2023-01-15T07:25:23.030574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df = pd.DataFrame(test_data_dict)","metadata":{"execution":{"iopub.status.busy":"2023-01-15T07:26:06.659670Z","iopub.execute_input":"2023-01-15T07:26:06.660180Z","iopub.status.idle":"2023-01-15T07:26:07.500975Z","shell.execute_reply.started":"2023-01-15T07:26:06.660137Z","shell.execute_reply":"2023-01-15T07:26:07.499618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.to_csv(\"/kaggle/working/submission_2.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-01-15T07:26:49.232042Z","iopub.execute_input":"2023-01-15T07:26:49.232622Z","iopub.status.idle":"2023-01-15T07:27:09.827279Z","shell.execute_reply.started":"2023-01-15T07:26:49.232573Z","shell.execute_reply":"2023-01-15T07:27:09.825637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(submit)","metadata":{"execution":{"iopub.status.busy":"2023-01-15T07:30:46.542318Z","iopub.execute_input":"2023-01-15T07:30:46.542834Z","iopub.status.idle":"2023-01-15T07:30:46.551157Z","shell.execute_reply.started":"2023-01-15T07:30:46.542788Z","shell.execute_reply":"2023-01-15T07:30:46.549889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(submission_df)","metadata":{"execution":{"iopub.status.busy":"2023-01-15T07:31:06.068882Z","iopub.execute_input":"2023-01-15T07:31:06.069741Z","iopub.status.idle":"2023-01-15T07:31:06.076483Z","shell.execute_reply.started":"2023-01-15T07:31:06.069693Z","shell.execute_reply":"2023-01-15T07:31:06.075541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}