{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nimport json\nfrom tqdm import tqdm\nimport multiprocessing","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-09T08:01:19.151642Z","iopub.execute_input":"2022-12-09T08:01:19.152353Z","iopub.status.idle":"2022-12-09T08:01:19.185686Z","shell.execute_reply.started":"2022-12-09T08:01:19.152253Z","shell.execute_reply":"2022-12-09T08:01:19.184648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"multi_process = 24","metadata":{"execution":{"iopub.status.busy":"2022-12-09T08:01:19.188086Z","iopub.execute_input":"2022-12-09T08:01:19.188804Z","iopub.status.idle":"2022-12-09T08:01:19.194201Z","shell.execute_reply.started":"2022-12-09T08:01:19.188756Z","shell.execute_reply":"2022-12-09T08:01:19.193102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def convert_json_as_dataframe(lines, res):\n    for line in lines:\n        sesion_record = json.loads(line)\n        session_id =sesion_record['session']\n        for event in sesion_record['events']:\n            res.append(pd.DataFrame({'session': [session_id],\n                      'aid':[event['aid']],\n                      'ts':[event['ts']],\n                      'type':[event['type']]\n                  }))\n        \n    return ","metadata":{"execution":{"iopub.status.busy":"2022-12-09T08:01:19.19572Z","iopub.execute_input":"2022-12-09T08:01:19.196076Z","iopub.status.idle":"2022-12-09T08:01:19.20688Z","shell.execute_reply.started":"2022-12-09T08:01:19.196045Z","shell.execute_reply":"2022-12-09T08:01:19.20565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"manager = multiprocessing.Manager()\nres = manager.list()\nprocess_list = []\nwith open('/kaggle/input/otto-recommender-system/train.jsonl', 'r') as f:\n    total_records = f.readlines()\n    ","metadata":{"execution":{"iopub.status.busy":"2022-12-09T08:09:25.253495Z","iopub.execute_input":"2022-12-09T08:09:25.25671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"offset = int(len(total_records) / multi_process)\nfor i in range(multi_process):\n    if i + 1 == multi_process:\n        data_part = total_records[i * offset:]\n    else:\n        data_part = total_records[i * offset:(i + 1) * offset]\n    p = multiprocessing.Process(\n        target=convert_json_as_dataframe,\n        kwargs={\n            'lines': data_part,\n            'res': res\n        }\n    )\n\n    p.daemon = True\n    process_list.append(p)\n    p.start()\nfor process in process_list:\n    print('join')\n    process.join()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.concat(list(res))","metadata":{"execution":{"iopub.status.busy":"2022-12-09T08:05:29.274648Z","iopub.execute_input":"2022-12-09T08:05:29.275414Z","iopub.status.idle":"2022-12-09T08:05:30.866743Z","shell.execute_reply.started":"2022-12-09T08:05:29.275376Z","shell.execute_reply":"2022-12-09T08:05:30.865647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2022-12-09T08:06:44.971264Z","iopub.execute_input":"2022-12-09T08:06:44.971708Z","iopub.status.idle":"2022-12-09T08:06:44.987095Z","shell.execute_reply.started":"2022-12-09T08:06:44.971673Z","shell.execute_reply":"2022-12-09T08:06:44.985899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}