{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom pathlib import Path\nfrom tqdm import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Writing","metadata":{}},{"cell_type":"code","source":"train_path = \"../input/otto-recommender-system/train.jsonl\"\ntest_path = \"../input/otto-recommender-system/test.jsonl\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type_dict = {\n    'clicks': 0,\n    'carts': 1,\n    'orders': 2\n}\n\ndef create_pandas_dataset(path: Path) -> pd.DataFrame:\n    \"\"\"\n    Creates pd.DataFrame object from the data stored in a json file.\n    \n    Args:\n        path: path to the json file\n        \n    Returns:\n        dataset read from the json file in pd.Dataset format (columns = [\"session\", \"aid\", \"ts\", \"type\"])\n    \"\"\"\n    data = pd.DataFrame()\n    \n    num_lines = sum(1 for line in open(path))\n    chunksize = 100_000\n    num_chunks = int(np.ceil(num_lines / chunksize))\n\n    chunks = pd.read_json(path, lines=True, chunksize=chunksize)\n    \n    for chunk in tqdm(chunks, total=num_chunks):\n        event_dict = {\n            'session': [],\n            'aid': [],\n            'ts': [],\n            'type': [],\n        }\n        for session, events in zip(chunk['session'].tolist(), chunk['events'].tolist()):\n            for event in events:\n                event_dict['session'].append(session)\n                event_dict['aid'].append(event['aid'])\n                event_dict['ts'].append(event['ts'])\n                event_dict['type'].append(type_dict[event['type']])\n        chunk_data = pd.DataFrame(event_dict)\n        chunk_data[\"session\"] = chunk_data[\"session\"].astype(np.uint32)\n        chunk_data[\"aid\"] = chunk_data[\"aid\"].astype(np.uint32)\n        chunk_data[\"ts\"] = pd.to_datetime(chunk_data[\"ts\"], unit='ms')\n        chunk_data[\"type\"] = chunk_data[\"type\"].astype(np.uint8)\n        data = pd.concat([data, chunk_data])\n    return data","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd_train = create_pandas_dataset(train_path)\ntrain_pd_path = \"./train.csv\" \npd_train.to_csv(train_pd_path, index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd_train.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd_test = create_pandas_dataset(test_path)\ntest_pd_path = \"./test.csv\" \npd_test.to_csv(test_pd_path, index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd_test.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del pd_train\ndel pd_test","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reading","metadata":{}},{"cell_type":"markdown","source":"## Recommended way","metadata":{}},{"cell_type":"code","source":"train_read = pd.read_csv(train_pd_path, dtype={\"session\": np.uint32, \"aid\": np.uint32, \"type\": np.uint8}, parse_dates=[\"ts\"])\ntrain_read.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_read = pd.read_csv(test_pd_path, dtype={\"session\": np.uint32, \"aid\": np.uint32, \"type\": np.uint8}, parse_dates=[\"ts\"])\ntest_read.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Not recommended way","metadata":{}},{"cell_type":"code","source":"# train_read_standard = pd.read_csv(train_pd_path)\n# train_read_standard.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_read_standard = pd.read_csv(test_pd_path)\n# test_read_standard.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Quick inside","metadata":{}},{"cell_type":"markdown","source":"It's worth specifying the data types.","metadata":{}}]}