{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Adapted from: https://www.kaggle.com/code/columbia2131/otto-read-a-chunk-of-jsonl-to-manageable-df/","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n","metadata":{"_uuid":"051d70d956493feee0c6d64651c6a088724dca2a","_execution_state":"idle","execution":{"iopub.status.busy":"2022-11-02T13:06:16.942025Z","iopub.execute_input":"2022-11-02T13:06:16.942407Z","iopub.status.idle":"2022-11-02T13:06:16.965818Z","shell.execute_reply.started":"2022-11-02T13:06:16.942329Z","shell.execute_reply":"2022-11-02T13:06:16.964815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    data_path = '/kaggle/input/otto-recommender-system/'","metadata":{"execution":{"iopub.status.busy":"2022-11-02T13:06:16.967382Z","iopub.execute_input":"2022-11-02T13:06:16.967679Z","iopub.status.idle":"2022-11-02T13:06:16.971519Z","shell.execute_reply.started":"2022-11-02T13:06:16.967655Z","shell.execute_reply":"2022-11-02T13:06:16.970730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sessions = pd.DataFrame()\nchunks = pd.read_json(CFG.data_path + 'train.jsonl', lines=True, chunksize=100_000)\n\n\nfor e, chunk in enumerate(chunks):\n    event_dict = {\n        'session': [],\n        'aid': [],\n        'ts': [],\n        'type': [],\n    }\n    if e < 2:\n        # train_sessions = pd.concat([train_sessions, chunk])\n        for session, events in zip(chunk['session'].tolist(), chunk['events'].tolist()):\n            for event in events:\n                event_dict['session'].append(session)\n                event_dict['aid'].append(event['aid'])\n                event_dict['ts'].append(event['ts'])\n                event_dict['type'].append(event['type'])\n        chunk_session = pd.DataFrame(event_dict)\n        train_sessions = pd.concat([train_sessions, chunk_session])\n    else:\n        break\n        \ntrain_sessions = train_sessions.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-11-02T13:06:16.972473Z","iopub.execute_input":"2022-11-02T13:06:16.972744Z","iopub.status.idle":"2022-11-02T13:06:59.371141Z","shell.execute_reply.started":"2022-11-02T13:06:16.972720Z","shell.execute_reply":"2022-11-02T13:06:59.370166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train_sessions)","metadata":{"execution":{"iopub.status.busy":"2022-11-02T13:06:59.372954Z","iopub.execute_input":"2022-11-02T13:06:59.373344Z","iopub.status.idle":"2022-11-02T13:06:59.392017Z","shell.execute_reply.started":"2022-11-02T13:06:59.373317Z","shell.execute_reply":"2022-11-02T13:06:59.391174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sessions.to_csv('train_data_v1.csv', index = False)\ntrain_sessions.to_parquet('train_data_v1.parquet')","metadata":{"execution":{"iopub.status.busy":"2022-11-02T11:52:34.595580Z","iopub.execute_input":"2022-11-02T11:52:34.596302Z","iopub.status.idle":"2022-11-02T11:52:36.581280Z","shell.execute_reply.started":"2022-11-02T11:52:34.596267Z","shell.execute_reply":"2022-11-02T11:52:36.579944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Test sessions","metadata":{}},{"cell_type":"code","source":"test_sessions = pd.DataFrame()\nchunks = pd.read_json(CFG.data_path + 'test.jsonl', lines=True, chunksize=100_000)\n\n\nfor e, chunk in enumerate(chunks):\n    event_dict = {\n        'session': [],\n        'aid': [],\n        'ts': [],\n        'type': [],\n    }\n    if e < 2:\n        # train_sessions = pd.concat([train_sessions, chunk])\n        for session, events in zip(chunk['session'].tolist(), chunk['events'].tolist()):\n            for event in events:\n                event_dict['session'].append(session)\n                event_dict['aid'].append(event['aid'])\n                event_dict['ts'].append(event['ts'])\n                event_dict['type'].append(event['type'])\n        chunk_session = pd.DataFrame(event_dict)\n        test_sessions = pd.concat([test_sessions, chunk_session])\n    else:\n        break\n        \ntest_sessions = test_sessions.reset_index(drop=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_sessions.to_csv('test_data_v1.csv', index = False)\ntest_sessions.to_parquet('test_data_v1.parquet')","metadata":{},"execution_count":null,"outputs":[]}]}