{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# The training dataset into small chunks\nWhen using statistics such as the mean obtained by groupby `session_id` and `level_group` as features, it is not necessary to store the entire time series dataset in memory. This can be used to avoid OOM. Therefore, the policy is to split `train.csv` into smaller chunks and calculate statistics for each of these chunks. However, it must be handled in such a way that `session_id` is not split into multiple chunks.\n\n<img src=\"https://www.googleapis.com/download/storage/v1/b/kaggle-forum-message-attachments/o/inbox%2F2473952%2F1688baae242a52befb659da01ffbbb40%2Fchunk_image.png?generation=1679855369596237&alt=media\" width=\"700\">\n\ncf. Discussion: https://www.kaggle.com/competitions/predict-student-performance-from-game-play/discussion/397688","metadata":{}},{"cell_type":"code","source":"import pandas as pd","metadata":{"execution":{"iopub.status.busy":"2023-03-26T19:31:24.645769Z","iopub.execute_input":"2023-03-26T19:31:24.646209Z","iopub.status.idle":"2023-03-26T19:31:24.675032Z","shell.execute_reply.started":"2023-03-26T19:31:24.646174Z","shell.execute_reply":"2023-03-26T19:31:24.673894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nread_chunk_size = 5_000_000\ntrain_csv_path = \"/kaggle/input/predict-student-performance-from-game-play/train.csv\"\ntrain_chunks = pd.read_csv(train_csv_path, chunksize=read_chunk_size)\n\nchunks = []\nfor idx, chunk_df in enumerate(train_chunks):\n    chunks.append(chunk_df)\n    if len(chunks) == 1:\n        continue\n\n    chunk_1_df = chunks[0]  # 1st chunk\n    chunk_2_df = chunks[1]  # 2nd chunk\n    sid_1_set = set(chunk_1_df[\"session_id\"])\n    sid_2_set = set(chunk_2_df[\"session_id\"])\n    sid_inter = list(sid_1_set & sid_2_set)  # Interception of session_ids\n    inter_df = chunk_2_df[chunk_2_df[\"session_id\"].isin(sid_inter)]\n\n    csv_1_path = f\"chunk_{idx}.csv\"\n    # Concat interception\n    pd.concat([chunk_1_df, inter_df]).to_csv(csv_1_path)\n    \n    # Delete interception and convert 2nd chunk to 1st chunk\n    chunks = []\n    chunk_1_df = chunk_2_df[~chunk_2_df[\"session_id\"].isin(sid_inter)]\n    chunks.append(chunk_1_df)\n\n# Make csv last chunk\nchunks[0].to_csv(f\"chunk_{idx+1}.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-03-26T19:31:24.676981Z","iopub.execute_input":"2023-03-26T19:31:24.677575Z","iopub.status.idle":"2023-03-26T19:38:40.853864Z","shell.execute_reply.started":"2023-03-26T19:31:24.677537Z","shell.execute_reply":"2023-03-26T19:38:40.852482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Calculate the average from the chunks as an example","metadata":{}},{"cell_type":"code","source":"def feature_engineer(train):\n    dfs = []\n    for c in [\"elapsed_time\", \"level\", \"page\"]:\n        tmp = train.groupby([\"session_id\", \"level_group\"])[c].agg(\"mean\")\n        tmp.name = tmp.name + '_mean'\n        dfs.append(tmp)\n    df = pd.concat(dfs, axis=1)\n    df = df.reset_index()\n    df = df.set_index('session_id')\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-03-26T19:38:40.855560Z","iopub.execute_input":"2023-03-26T19:38:40.856308Z","iopub.status.idle":"2023-03-26T19:38:40.864060Z","shell.execute_reply.started":"2023-03-26T19:38:40.856270Z","shell.execute_reply":"2023-03-26T19:38:40.862856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = []\nfor idx in range(6):\n    csv_path = f\"chunk_{idx+1}.csv\"\n    chunk_df = pd.read_csv(csv_path) \n    feature_df = feature_engineer(chunk_df)\n    features.append(feature_df)\n\nfeature_from_chunks_df = pd.concat(features)\nfeature_from_chunks_df","metadata":{"execution":{"iopub.status.busy":"2023-03-26T19:38:40.865392Z","iopub.execute_input":"2023-03-26T19:38:40.865996Z","iopub.status.idle":"2023-03-26T19:40:15.761871Z","shell.execute_reply.started":"2023-03-26T19:38:40.865959Z","shell.execute_reply":"2023-03-26T19:40:15.760579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Validation","metadata":{}},{"cell_type":"code","source":"# Validation that the statistics of data loaded directly from train.csv and data loaded from csv divided into chunks match\ntrain_csv_path = \"/kaggle/input/predict-student-performance-from-game-play/train.csv\"\ntrain_df = pd.read_csv(train_csv_path, usecols=[\"session_id\", \"level_group\", \"elapsed_time\", \"level\", \"page\"])\nfeature_from_bulk_df = feature_engineer(train_df)\n\nfeature_from_bulk_df.equals(feature_from_chunks_df)","metadata":{"execution":{"iopub.status.busy":"2023-03-26T19:40:15.764571Z","iopub.execute_input":"2023-03-26T19:40:15.765094Z","iopub.status.idle":"2023-03-26T19:42:01.030720Z","shell.execute_reply.started":"2023-03-26T19:40:15.765057Z","shell.execute_reply":"2023-03-26T19:42:01.029765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}