{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Install/Import Libraries","metadata":{}},{"cell_type":"code","source":"%%capture --no-stderr pip_install_stdout_log\n! pip install git+https://github.com/yomura-yomura/plotly_utility","metadata":{"execution":{"iopub.status.busy":"2022-12-04T07:17:59.533670Z","iopub.execute_input":"2022-12-04T07:17:59.534117Z","iopub.status.idle":"2022-12-04T07:18:17.166267Z","shell.execute_reply.started":"2022-12-04T07:17:59.534071Z","shell.execute_reply":"2022-12-04T07:18:17.164503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport pathlib\nimport tqdm\nimport numpy as np\nimport plotly_utility.express as pux","metadata":{"execution":{"iopub.status.busy":"2022-12-04T07:19:07.020488Z","iopub.execute_input":"2022-12-04T07:19:07.020982Z","iopub.status.idle":"2022-12-04T07:19:08.482188Z","shell.execute_reply.started":"2022-12-04T07:19:07.020861Z","shell.execute_reply":"2022-12-04T07:19:08.480729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utility Functions","metadata":{}},{"cell_type":"code","source":"chunksize=10_000\n\nkaggle_data_dir_path = pathlib.Path(\"/kaggle/input/otto-recommender-system\")\n\n\ndef get_n_records(filename):\n    with open(kaggle_data_dir_path / filename, \"r\") as f:\n        return sum(1 for _ in f)\n\n    \ndef get_json_reader(filename, with_progress_bar=True):\n    json_reader = pd.read_json(\n        kaggle_data_dir_path / filename, lines=True, \n        chunksize=chunksize\n    )\n    if with_progress_bar:\n        n_records = get_n_records(filename)\n        json_reader = tqdm.tqdm(\n            json_reader, \n            total=int(np.ceil(n_records / chunksize))\n        )\n    return json_reader\n\n    \ndef get_n_events_list(filename):\n    json_reader = get_json_reader(filename)\n    return [\n        n_events\n        for chunk_df in json_reader\n        for n_events in chunk_df[\"events\"].map(len)\n    ]\n\n\ndef get_unique_session_array(filename):\n    json_reader = get_json_reader(filename)\n    return np.unique([\n        session\n        for chunk_df in json_reader\n        for session in chunk_df[\"session\"].unique()\n    ])\n\n\ndef get_unique_aid_array(filename):\n    json_reader = get_json_reader(filename)\n    return np.unique([\n        aid\n        for chunk_df in json_reader\n        for aid in np.unique([\n            event[\"aid\"] \n            for events in chunk_df[\"events\"] \n            for event in events\n        ])\n    ])","metadata":{"execution":{"iopub.status.busy":"2022-12-04T08:00:53.318013Z","iopub.execute_input":"2022-12-04T08:00:53.318483Z","iopub.status.idle":"2022-12-04T08:00:53.333430Z","shell.execute_reply.started":"2022-12-04T08:00:53.318449Z","shell.execute_reply":"2022-12-04T08:00:53.332009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA","metadata":{}},{"cell_type":"markdown","source":"## train","metadata":{}},{"cell_type":"code","source":"target_fn = \"train.jsonl\"","metadata":{"execution":{"iopub.status.busy":"2022-12-04T08:40:54.133821Z","iopub.execute_input":"2022-12-04T08:40:54.135286Z","iopub.status.idle":"2022-12-04T08:40:54.141583Z","shell.execute_reply.started":"2022-12-04T08:40:54.135225Z","shell.execute_reply":"2022-12-04T08:40:54.140246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_records = get_n_records(target_fn)  # jsonlファイルのレコード数（=行数）\nprint(f\"The number of records = {n_records:,}\")","metadata":{"execution":{"iopub.status.busy":"2022-12-04T07:29:39.952884Z","iopub.execute_input":"2022-12-04T07:29:39.953357Z","iopub.status.idle":"2022-12-04T07:31:43.418812Z","shell.execute_reply.started":"2022-12-04T07:29:39.953326Z","shell.execute_reply":"2022-12-04T07:31:43.416438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Distribution of The Number of Events","metadata":{}},{"cell_type":"code","source":"n_events_list = get_n_events_list(target_fn)","metadata":{"execution":{"iopub.status.busy":"2022-12-03T09:25:37.369991Z","iopub.execute_input":"2022-12-03T09:25:37.370559Z","iopub.status.idle":"2022-12-03T09:34:59.978038Z","shell.execute_reply.started":"2022-12-03T09:25:37.370522Z","shell.execute_reply":"2022-12-03T09:34:59.974979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q = np.linspace(0, 1, 11)\nprint(\"n_events-dist quantile values:\")\nfor iq, x in zip(q, np.quantile(n_events_list, q)):\n    print(f\"{iq:>4.0%}: {x:>3.0f}\")","metadata":{"execution":{"iopub.status.busy":"2022-12-03T09:54:08.588094Z","iopub.execute_input":"2022-12-03T09:54:08.588585Z","iopub.status.idle":"2022-12-03T09:54:09.816918Z","shell.execute_reply.started":"2022-12-03T09:54:08.588546Z","shell.execute_reply":"2022-12-03T09:54:09.815378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pux.histogram(n_events_list, log_y=True).show()","metadata":{"execution":{"iopub.status.busy":"2022-12-03T09:37:01.876324Z","iopub.execute_input":"2022-12-03T09:37:01.876822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Unique Session","metadata":{}},{"cell_type":"code","source":"unique_session_array = get_unique_session_array(target_fn)","metadata":{"execution":{"iopub.status.busy":"2022-12-04T07:34:20.840162Z","iopub.execute_input":"2022-12-04T07:34:20.840602Z","iopub.status.idle":"2022-12-04T07:43:16.865448Z","shell.execute_reply.started":"2022-12-04T07:34:20.840568Z","shell.execute_reply":"2022-12-04T07:43:16.864056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_session_array","metadata":{"execution":{"iopub.status.busy":"2022-12-04T08:30:13.995964Z","iopub.execute_input":"2022-12-04T08:30:13.996398Z","iopub.status.idle":"2022-12-04T08:30:14.005031Z","shell.execute_reply.started":"2022-12-04T08:30:13.996363Z","shell.execute_reply":"2022-12-04T08:30:14.003580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"The number of unique sessions = {unique_session_array.size:,}\")","metadata":{"execution":{"iopub.status.busy":"2022-12-04T07:46:41.647944Z","iopub.execute_input":"2022-12-04T07:46:41.648448Z","iopub.status.idle":"2022-12-04T07:46:41.655904Z","shell.execute_reply.started":"2022-12-04T07:46:41.648412Z","shell.execute_reply":"2022-12-04T07:46:41.654564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# jsonlファイルのレコード数と全く同じ\nassert unique_session_array.size == n_records  \n\n# しかも連番になってる\nassert np.all(unique_session_array == np.arange(unique_session_array.size))","metadata":{"execution":{"iopub.status.busy":"2022-12-04T07:51:25.779233Z","iopub.execute_input":"2022-12-04T07:51:25.779671Z","iopub.status.idle":"2022-12-04T07:51:25.839128Z","shell.execute_reply.started":"2022-12-04T07:51:25.779634Z","shell.execute_reply":"2022-12-04T07:51:25.837791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 32bitsのint/uint型で表現可能\nassert np.all(unique_session_array == unique_session_array.astype(np.int32))\nassert np.all(unique_session_array == unique_session_array.astype(np.uint32))\n\n# 16bitsのuint型では表現できない\nassert not np.all(unique_session_array == unique_session_array.astype(np.uint16))","metadata":{"execution":{"iopub.status.busy":"2022-12-04T08:32:17.804471Z","iopub.execute_input":"2022-12-04T08:32:17.804908Z","iopub.status.idle":"2022-12-04T08:32:17.933367Z","shell.execute_reply.started":"2022-12-04T08:32:17.804874Z","shell.execute_reply":"2022-12-04T08:32:17.931732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Unique aid","metadata":{}},{"cell_type":"code","source":"unique_aid_array = get_unique_aid_array(target_fn)","metadata":{"execution":{"iopub.status.busy":"2022-12-04T08:40:57.667972Z","iopub.execute_input":"2022-12-04T08:40:57.668369Z","iopub.status.idle":"2022-12-04T08:59:27.694641Z","shell.execute_reply.started":"2022-12-04T08:40:57.668338Z","shell.execute_reply":"2022-12-04T08:59:27.693615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_aid_array","metadata":{"execution":{"iopub.status.busy":"2022-12-04T08:26:09.635344Z","iopub.execute_input":"2022-12-04T08:26:09.636689Z","iopub.status.idle":"2022-12-04T08:26:09.644468Z","shell.execute_reply.started":"2022-12-04T08:26:09.636639Z","shell.execute_reply":"2022-12-04T08:26:09.643131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"The number of unique aid = {unique_aid_array.size:,}\")","metadata":{"execution":{"iopub.status.busy":"2022-12-04T08:26:55.954268Z","iopub.execute_input":"2022-12-04T08:26:55.954749Z","iopub.status.idle":"2022-12-04T08:26:55.961547Z","shell.execute_reply.started":"2022-12-04T08:26:55.954714Z","shell.execute_reply":"2022-12-04T08:26:55.960135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 連番になってる\nassert np.all(unique_aid_array == np.arange(unique_aid_array.min(), unique_aid_array.max() + 1))","metadata":{"execution":{"iopub.status.busy":"2022-12-04T09:10:04.244540Z","iopub.execute_input":"2022-12-04T09:10:04.247616Z","iopub.status.idle":"2022-12-04T09:10:04.270109Z","shell.execute_reply.started":"2022-12-04T09:10:04.247551Z","shell.execute_reply":"2022-12-04T09:10:04.268066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 32bitsのint/uint型で表現可能\nassert np.all(unique_aid_array == unique_aid_array.astype(np.int32))\nassert np.all(unique_aid_array == unique_aid_array.astype(np.uint32))\n\n# 16bitsのuint型では表現できない\nassert not np.all(unique_aid_array == unique_aid_array.astype(np.uint16))","metadata":{"execution":{"iopub.status.busy":"2022-12-04T09:10:06.004142Z","iopub.execute_input":"2022-12-04T09:10:06.004536Z","iopub.status.idle":"2022-12-04T09:10:06.027211Z","shell.execute_reply.started":"2022-12-04T09:10:06.004506Z","shell.execute_reply":"2022-12-04T09:10:06.025979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_unique_aid_array = unique_aid_array","metadata":{"execution":{"iopub.status.busy":"2022-12-04T09:10:15.556212Z","iopub.execute_input":"2022-12-04T09:10:15.556663Z","iopub.status.idle":"2022-12-04T09:10:15.562413Z","shell.execute_reply.started":"2022-12-04T09:10:15.556626Z","shell.execute_reply":"2022-12-04T09:10:15.561128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## test","metadata":{}},{"cell_type":"code","source":"target_fn = \"test.jsonl\"","metadata":{"execution":{"iopub.status.busy":"2022-12-04T09:11:11.884364Z","iopub.execute_input":"2022-12-04T09:11:11.884860Z","iopub.status.idle":"2022-12-04T09:11:11.891951Z","shell.execute_reply.started":"2022-12-04T09:11:11.884813Z","shell.execute_reply":"2022-12-04T09:11:11.890409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_records = get_n_records(target_fn)  # jsonlファイルのレコード数（=行数）\nprint(f\"The number of records = {n_records:,}\")","metadata":{"execution":{"iopub.status.busy":"2022-12-04T08:37:05.069143Z","iopub.execute_input":"2022-12-04T08:37:05.069556Z","iopub.status.idle":"2022-12-04T08:37:09.671741Z","shell.execute_reply.started":"2022-12-04T08:37:05.069523Z","shell.execute_reply":"2022-12-04T08:37:09.670633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Distribution of The Number of Events","metadata":{}},{"cell_type":"code","source":"n_events_list = get_n_events_list(target_fn)","metadata":{"execution":{"iopub.status.busy":"2022-12-03T09:25:37.369991Z","iopub.execute_input":"2022-12-03T09:25:37.370559Z","iopub.status.idle":"2022-12-03T09:34:59.978038Z","shell.execute_reply.started":"2022-12-03T09:25:37.370522Z","shell.execute_reply":"2022-12-03T09:34:59.974979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q = np.linspace(0, 1, 11)\nprint(\"n_events-dist quantile values:\")\nfor iq, x in zip(q, np.quantile(n_events_list, q)):\n    print(f\"{iq:>4.0%}: {x:>3.0f}\")","metadata":{"execution":{"iopub.status.busy":"2022-12-03T09:54:08.588094Z","iopub.execute_input":"2022-12-03T09:54:08.588585Z","iopub.status.idle":"2022-12-03T09:54:09.816918Z","shell.execute_reply.started":"2022-12-03T09:54:08.588546Z","shell.execute_reply":"2022-12-03T09:54:09.815378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pux.histogram(n_events_list, log_y=True).show()","metadata":{"execution":{"iopub.status.busy":"2022-12-03T09:37:01.876324Z","iopub.execute_input":"2022-12-03T09:37:01.876822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Unique Session","metadata":{}},{"cell_type":"code","source":"unique_session_array = get_unique_session_array(target_fn)","metadata":{"execution":{"iopub.status.busy":"2022-12-04T08:37:15.728785Z","iopub.execute_input":"2022-12-04T08:37:15.729207Z","iopub.status.idle":"2022-12-04T08:37:42.253659Z","shell.execute_reply.started":"2022-12-04T08:37:15.729174Z","shell.execute_reply":"2022-12-04T08:37:42.252392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_session_array","metadata":{"execution":{"iopub.status.busy":"2022-12-04T08:37:45.068874Z","iopub.execute_input":"2022-12-04T08:37:45.069296Z","iopub.status.idle":"2022-12-04T08:37:45.077351Z","shell.execute_reply.started":"2022-12-04T08:37:45.069263Z","shell.execute_reply":"2022-12-04T08:37:45.076263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"The number of unique sessions = {unique_session_array.size:,}\")","metadata":{"execution":{"iopub.status.busy":"2022-12-04T08:37:52.560068Z","iopub.execute_input":"2022-12-04T08:37:52.561073Z","iopub.status.idle":"2022-12-04T08:37:52.567674Z","shell.execute_reply.started":"2022-12-04T08:37:52.561027Z","shell.execute_reply":"2022-12-04T08:37:52.566583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# jsonlファイルのレコード数と全く同じ\nassert unique_session_array.size == n_records  \n\n# しかも連番になってる\nassert np.all(unique_session_array == np.arange(unique_session_array.min(), unique_session_array.max() + 1))","metadata":{"execution":{"iopub.status.busy":"2022-12-04T08:38:22.357244Z","iopub.execute_input":"2022-12-04T08:38:22.357649Z","iopub.status.idle":"2022-12-04T08:38:22.371062Z","shell.execute_reply.started":"2022-12-04T08:38:22.357616Z","shell.execute_reply":"2022-12-04T08:38:22.369844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 32bitsのint/uint型で表現可能\nassert np.all(unique_session_array == unique_session_array.astype(np.int32))\nassert np.all(unique_session_array == unique_session_array.astype(np.uint32))\n\n# 16bitsのuint型では表現できない\nassert not np.all(unique_session_array == unique_session_array.astype(np.uint16))","metadata":{"execution":{"iopub.status.busy":"2022-12-04T08:38:43.022507Z","iopub.execute_input":"2022-12-04T08:38:43.022973Z","iopub.status.idle":"2022-12-04T08:38:43.041481Z","shell.execute_reply.started":"2022-12-04T08:38:43.022904Z","shell.execute_reply":"2022-12-04T08:38:43.040575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Unique aid","metadata":{}},{"cell_type":"code","source":"unique_aid_array = get_unique_aid_array(target_fn)","metadata":{"execution":{"iopub.status.busy":"2022-12-04T09:11:15.877856Z","iopub.execute_input":"2022-12-04T09:11:15.878871Z","iopub.status.idle":"2022-12-04T09:11:55.075801Z","shell.execute_reply.started":"2022-12-04T09:11:15.878822Z","shell.execute_reply":"2022-12-04T09:11:55.074578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_aid_array","metadata":{"execution":{"iopub.status.busy":"2022-12-04T09:15:08.652345Z","iopub.execute_input":"2022-12-04T09:15:08.652836Z","iopub.status.idle":"2022-12-04T09:15:08.661519Z","shell.execute_reply.started":"2022-12-04T09:15:08.652802Z","shell.execute_reply":"2022-12-04T09:15:08.660206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"The number of unique aid = {unique_aid_array.size:,}\")","metadata":{"execution":{"iopub.status.busy":"2022-12-04T09:15:10.842641Z","iopub.execute_input":"2022-12-04T09:15:10.843076Z","iopub.status.idle":"2022-12-04T09:15:10.849544Z","shell.execute_reply.started":"2022-12-04T09:15:10.843038Z","shell.execute_reply":"2022-12-04T09:15:10.848240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 番号が飛び飛びになってる\nassert unique_aid_array.max() != unique_aid_array.size\n\n# trainで出現する番号に必ず含まれる\nassert np.all(np.isin(unique_aid_array, train_unique_aid_array))","metadata":{"execution":{"iopub.status.busy":"2022-12-04T09:18:04.755963Z","iopub.execute_input":"2022-12-04T09:18:04.756465Z","iopub.status.idle":"2022-12-04T09:18:04.898235Z","shell.execute_reply.started":"2022-12-04T09:18:04.756428Z","shell.execute_reply":"2022-12-04T09:18:04.896993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 32bitsのint/uint型で表現可能\nassert np.all(unique_aid_array == unique_aid_array.astype(np.int32))\nassert np.all(unique_aid_array == unique_aid_array.astype(np.uint32))\n\n# 16bitsのuint型では表現できない\nassert not np.all(unique_aid_array == unique_aid_array.astype(np.uint16))","metadata":{"execution":{"iopub.status.busy":"2022-12-04T09:18:14.601971Z","iopub.execute_input":"2022-12-04T09:18:14.602366Z","iopub.status.idle":"2022-12-04T09:18:14.614304Z","shell.execute_reply.started":"2022-12-04T09:18:14.602335Z","shell.execute_reply":"2022-12-04T09:18:14.612819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}