{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":38760,"databundleVersionId":4493939,"sourceType":"competition"}],"dockerImageVersionId":30558,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nfrom pathlib import Path\n\n\nnum_lines = sum(1 for line in open(\"/kaggle/input/otto-recommender-system/train.jsonl\"))\nprint(f'number of lines in train: {num_lines:_}')\n\nchunksize = 100_000\nnum_chunks = int(np.ceil(num_lines / 100_000))\nprint(f'number of chunks: {num_chunks:_}')\n\nn = 2\ntrain_sessions = pd.DataFrame()\nchunks = pd.read_json(\"/kaggle/input/otto-recommender-system/train.jsonl\", lines=True, chunksize=chunksize)\n\nfor e, chunk in enumerate(chunks):\n    if e < 2:\n        train_sessions = pd.concat([train_sessions, chunk])\n    else:\n        break\ntrain_sessions = train_sessions.set_index('session', drop=True).sort_index()\n\ntrain_sessions","metadata":{"execution":{"iopub.status.busy":"2023-12-22T14:25:10.019047Z","iopub.execute_input":"2023-12-22T14:25:10.019483Z","iopub.status.idle":"2023-12-22T14:28:28.220164Z","shell.execute_reply.started":"2023-12-22T14:25:10.019447Z","shell.execute_reply":"2023-12-22T14:28:28.219073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Task1\n# transform the dataframe train_sessions to a dataframe train_sessions_detailed : columns ['aid', 'ts', 'type' , 'session']\ndf_session = []\nfor session_id in range(1000):\n    df_detailed = pd.DataFrame(train_sessions.iloc[session_id][0])\n    df_detailed['session'] = session_id\n    df_session.append(df_detailed)\n\ndf_result = pd.concat(df_session, ignore_index=True)\ndf_result\n\n#Task2 Explore the data","metadata":{"execution":{"iopub.status.busy":"2023-12-22T14:28:28.222263Z","iopub.execute_input":"2023-12-22T14:28:28.222688Z","iopub.status.idle":"2023-12-22T14:28:29.245011Z","shell.execute_reply.started":"2023-12-22T14:28:28.222655Z","shell.execute_reply":"2023-12-22T14:28:29.243729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_result.head()\n","metadata":{"execution":{"iopub.status.busy":"2023-12-22T14:28:29.247157Z","iopub.execute_input":"2023-12-22T14:28:29.248695Z","iopub.status.idle":"2023-12-22T14:28:29.260172Z","shell.execute_reply.started":"2023-12-22T14:28:29.248648Z","shell.execute_reply":"2023-12-22T14:28:29.259079Z"},"trusted":true},"execution_count":null,"outputs":[]}]}