{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nimport jsonlines","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-11-02T03:07:32.164219Z","iopub.execute_input":"2022-11-02T03:07:32.164671Z","iopub.status.idle":"2022-11-02T03:07:32.182388Z","shell.execute_reply.started":"2022-11-02T03:07:32.164620Z","shell.execute_reply":"2022-11-02T03:07:32.181025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_path = Path('/kaggle/input/otto-recommender-system/')","metadata":{"execution":{"iopub.status.busy":"2022-11-02T03:07:32.184809Z","iopub.execute_input":"2022-11-02T03:07:32.185271Z","iopub.status.idle":"2022-11-02T03:07:32.190294Z","shell.execute_reply.started":"2022-11-02T03:07:32.185230Z","shell.execute_reply":"2022-11-02T03:07:32.189123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### First look of the data","metadata":{}},{"cell_type":"markdown","source":"We can read the data using context manager","metadata":{"execution":{"iopub.status.busy":"2022-11-02T02:57:38.129271Z","iopub.execute_input":"2022-11-02T02:57:38.129677Z","iopub.status.idle":"2022-11-02T02:57:38.137072Z","shell.execute_reply.started":"2022-11-02T02:57:38.129632Z","shell.execute_reply":"2022-11-02T02:57:38.135444Z"}}},{"cell_type":"code","source":"with jsonlines.open(data_path / 'train.jsonl') as reader:\n    first_obs = reader.read()\n    \nfirst_obs","metadata":{"execution":{"iopub.status.busy":"2022-11-02T03:07:32.191899Z","iopub.execute_input":"2022-11-02T03:07:32.192573Z","iopub.status.idle":"2022-11-02T03:07:32.248216Z","shell.execute_reply.started":"2022-11-02T03:07:32.192522Z","shell.execute_reply":"2022-11-02T03:07:32.246928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can also read the data using the Reader object","metadata":{"execution":{"iopub.status.busy":"2022-11-02T02:58:38.826828Z","iopub.execute_input":"2022-11-02T02:58:38.827750Z","iopub.status.idle":"2022-11-02T02:58:38.834252Z","shell.execute_reply.started":"2022-11-02T02:58:38.827691Z","shell.execute_reply":"2022-11-02T02:58:38.832627Z"}}},{"cell_type":"code","source":"fp = open(data_path / 'train.jsonl')\nreader = jsonlines.Reader(fp)\nfirst = reader.read()\nsecond = reader.read()\nreader.close()\nfp.close()\n\nsecond","metadata":{"execution":{"iopub.status.busy":"2022-11-02T03:07:32.250853Z","iopub.execute_input":"2022-11-02T03:07:32.251202Z","iopub.status.idle":"2022-11-02T03:07:32.265341Z","shell.execute_reply.started":"2022-11-02T03:07:32.251171Z","shell.execute_reply":"2022-11-02T03:07:32.263889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Get the iterator","metadata":{}},{"cell_type":"markdown","source":"We can read each datum using the iterator.","metadata":{}},{"cell_type":"code","source":"i = 0\nfirst_three_obs = []\nwith jsonlines.open(data_path / 'train.jsonl') as reader:\n    iterator = reader.iter()\n    for session in iterator:\n        if i != 3:\n            first_three_obs.append(session)\n            i += 1\n        else:\n            break\n","metadata":{"execution":{"iopub.status.busy":"2022-11-02T03:07:32.266828Z","iopub.execute_input":"2022-11-02T03:07:32.267215Z","iopub.status.idle":"2022-11-02T03:07:32.280627Z","shell.execute_reply.started":"2022-11-02T03:07:32.267175Z","shell.execute_reply":"2022-11-02T03:07:32.279715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(first_three_obs)","metadata":{"execution":{"iopub.status.busy":"2022-11-02T03:07:32.282390Z","iopub.execute_input":"2022-11-02T03:07:32.283139Z","iopub.status.idle":"2022-11-02T03:07:32.293788Z","shell.execute_reply.started":"2022-11-02T03:07:32.283092Z","shell.execute_reply":"2022-11-02T03:07:32.292079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can also read all the data into memory using the iterator and the context manager. <br>\nYet this will consume all the memory of the kaggle machine due to the large file size. So we comment out the code below.","metadata":{}},{"cell_type":"code","source":"# with jsonlines.open(data_path / 'train.jsonl') as reader:\n#     iterator = reader.iter()\n#     all_data = list(iterator)","metadata":{"execution":{"iopub.status.busy":"2022-11-02T03:07:32.297412Z","iopub.execute_input":"2022-11-02T03:07:32.297849Z","iopub.status.idle":"2022-11-02T03:07:32.304761Z","shell.execute_reply.started":"2022-11-02T03:07:32.297812Z","shell.execute_reply":"2022-11-02T03:07:32.303845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Check if all the dictionary (JSON) object has the same strcuture\n\nSince the JSON structure can be different from each other theoretically, we check to make sure they all have the same structure.<br>\nThis will take some time to iterate all the data. We put it in a function for timing it using %time.\n\nIt takes around 4 mins to run on kaggle machine. And it is confirmed that they have identical structure as there is no assertion error.","metadata":{}},{"cell_type":"code","source":"def check_same_structure():\n    with jsonlines.open(data_path / 'train.jsonl') as reader:\n        iterator = reader.iter()\n        for session in iterator:\n            assert len(session.keys())==2 and 'session' in session.keys() and 'events' in session.keys()\n    return ","metadata":{"execution":{"iopub.status.busy":"2022-11-02T03:07:32.306130Z","iopub.execute_input":"2022-11-02T03:07:32.306535Z","iopub.status.idle":"2022-11-02T03:07:32.317471Z","shell.execute_reply.started":"2022-11-02T03:07:32.306502Z","shell.execute_reply":"2022-11-02T03:07:32.316197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%time check_same_structure()","metadata":{"execution":{"iopub.status.busy":"2022-11-02T03:07:32.320530Z","iopub.execute_input":"2022-11-02T03:07:32.321261Z","iopub.status.idle":"2022-11-02T03:12:04.799779Z","shell.execute_reply.started":"2022-11-02T03:07:32.321209Z","shell.execute_reply":"2022-11-02T03:12:04.798340Z"},"trusted":true},"execution_count":null,"outputs":[]}]}