{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"dockerImageVersionId":30664,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from pathlib import Path\n\nimport polars as pl\nimport pandas as pd","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-03-21T15:33:47.392241Z","iopub.execute_input":"2024-03-21T15:33:47.392742Z","iopub.status.idle":"2024-03-21T15:33:48.236965Z","shell.execute_reply.started":"2024-03-21T15:33:47.392702Z","shell.execute_reply":"2024-03-21T15:33:48.235923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"image from [@sergiosaharovskiy](https://www.kaggle.com/sergiosaharovskiy)'s amazing [[Home Credit CRMS, 2024] EDA and Submission](https://www.kaggle.com/code/sergiosaharovskiy/home-credit-crms-2024-eda-and-submission) notebook\n\n<p align=\"right\">\n  <img src=\"https://api.monosnap.com/file/download?id=QBYoGNfr4osgQfmrNPKGU4k05hUDkZ\"/>\n</p>","metadata":{}},{"cell_type":"markdown","source":"## 1. feature_definitions","metadata":{}},{"cell_type":"code","source":"df_feature_definitions = pl.read_csv('/kaggle/input/home-credit-credit-risk-model-stability/feature_definitions.csv')\ndf_feature_definitions","metadata":{"execution":{"iopub.status.busy":"2024-03-21T15:17:58.214421Z","iopub.execute_input":"2024-03-21T15:17:58.215579Z","iopub.status.idle":"2024-03-21T15:17:58.440858Z","shell.execute_reply.started":"2024-03-21T15:17:58.215526Z","shell.execute_reply":"2024-03-21T15:17:58.439818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Show the default polars config\npl.Config.state()","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-03-21T15:17:58.803395Z","iopub.execute_input":"2024-03-21T15:17:58.804584Z","iopub.status.idle":"2024-03-21T15:17:58.815711Z","shell.execute_reply.started":"2024-03-21T15:17:58.804542Z","shell.execute_reply":"2024-03-21T15:17:58.814388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Show all rows and long string for `feature_definitions` table\n# https://docs.pola.rs/py-polars/html/reference/config.html\n# https://docs.pola.rs/py-polars/html/reference/api/polars.Config.set_fmt_str_lengths.html\npl.Config.set_fmt_str_lengths(100)\npl.Config.set_tbl_rows(-1)","metadata":{"execution":{"iopub.status.busy":"2024-03-21T15:17:59.703213Z","iopub.execute_input":"2024-03-21T15:17:59.703688Z","iopub.status.idle":"2024-03-21T15:17:59.712583Z","shell.execute_reply.started":"2024-03-21T15:17:59.703654Z","shell.execute_reply":"2024-03-21T15:17:59.710985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## . . 1.1. feature_definitions (a-c)","metadata":{}},{"cell_type":"code","source":"df_feature_definitions.filter(\n    pl.col('Variable').str.contains('^[a-c]')  # rows that start with 'a' ~ 'c'\n)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-03-21T15:18:00.560095Z","iopub.execute_input":"2024-03-21T15:18:00.560550Z","iopub.status.idle":"2024-03-21T15:18:00.642888Z","shell.execute_reply.started":"2024-03-21T15:18:00.560517Z","shell.execute_reply":"2024-03-21T15:18:00.641511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## . . 1.2. feature_definitions (d-i)","metadata":{}},{"cell_type":"code","source":"df_feature_definitions.filter(\n    pl.col('Variable').str.contains('^[d-i]')\n)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-03-21T15:18:01.251263Z","iopub.execute_input":"2024-03-21T15:18:01.252038Z","iopub.status.idle":"2024-03-21T15:18:01.269250Z","shell.execute_reply.started":"2024-03-21T15:18:01.251994Z","shell.execute_reply":"2024-03-21T15:18:01.267921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## . . 1.3. feature_definitions (j-n)","metadata":{}},{"cell_type":"code","source":"df_feature_definitions.filter(\n    pl.col('Variable').str.contains('^[j-n]')\n)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-03-21T15:18:02.129857Z","iopub.execute_input":"2024-03-21T15:18:02.130299Z","iopub.status.idle":"2024-03-21T15:18:02.148861Z","shell.execute_reply.started":"2024-03-21T15:18:02.130250Z","shell.execute_reply":"2024-03-21T15:18:02.147842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## . . 1.4. feature_definitions (o-z)","metadata":{}},{"cell_type":"code","source":"df_feature_definitions.filter(\n    pl.col('Variable').str.contains('^[o-z]')\n)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-03-21T15:18:03.433301Z","iopub.execute_input":"2024-03-21T15:18:03.433783Z","iopub.status.idle":"2024-03-21T15:18:03.455513Z","shell.execute_reply.started":"2024-03-21T15:18:03.433741Z","shell.execute_reply":"2024-03-21T15:18:03.454103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Show head(3) and tail(3) only\npl.Config.set_fmt_str_lengths(None)\npl.Config.set_tbl_rows(6)","metadata":{"execution":{"iopub.status.busy":"2024-03-21T15:34:03.813901Z","iopub.execute_input":"2024-03-21T15:34:03.814482Z","iopub.status.idle":"2024-03-21T15:34:03.824221Z","shell.execute_reply.started":"2024-03-21T15:34:03.814447Z","shell.execute_reply":"2024-03-21T15:34:03.822746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Utility Function\n- based on [@greysky](https://www.kaggle.com/greysky)'s amazing [Home Credit Baseline](https://www.kaggle.com/code/greysky/home-credit-baseline) notebook.\n- {python docs} [pathlib](https://docs.python.org/3/library/pathlib.html#module-pathlib)\n- {python docs} [Path.glob()](https://docs.python.org/3/library/pathlib.html#pathlib.Path.glob)","metadata":{}},{"cell_type":"code","source":"ROOT            = Path(\"/kaggle/input/home-credit-credit-risk-model-stability\")\nTRAIN_DIR       = ROOT / \"parquet_files\" / \"train\"","metadata":{"execution":{"iopub.status.busy":"2024-03-21T15:34:04.829950Z","iopub.execute_input":"2024-03-21T15:34:04.830372Z","iopub.status.idle":"2024-03-21T15:34:04.836499Z","shell.execute_reply.started":"2024-03-21T15:34:04.830328Z","shell.execute_reply":"2024-03-21T15:34:04.835193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_files(matching_files):\n    chunks = []\n    for file in matching_files:\n        df = pl.read_parquet(file)\n        chunks.append(df)\n        \n    df = pl.concat(chunks, how=\"vertical\")\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2024-03-21T15:34:05.411962Z","iopub.execute_input":"2024-03-21T15:34:05.412964Z","iopub.status.idle":"2024-03-21T15:34:05.419692Z","shell.execute_reply.started":"2024-03-21T15:34:05.412912Z","shell.execute_reply":"2024-03-21T15:34:05.418367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. train_base","metadata":{}},{"cell_type":"code","source":"%%time\n\npl.read_parquet(TRAIN_DIR / 'train_base.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-03-21T15:18:07.863301Z","iopub.execute_input":"2024-03-21T15:18:07.863717Z","iopub.status.idle":"2024-03-21T15:18:08.100422Z","shell.execute_reply.started":"2024-03-21T15:18:07.863679Z","shell.execute_reply":"2024-03-21T15:18:08.099381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. Internal Source - Depth 0","metadata":{}},{"cell_type":"markdown","source":"## . . 3.1. static_0","metadata":{}},{"cell_type":"code","source":"matching_files = sorted(TRAIN_DIR.glob(\"train_static_0_*.parquet\"))\nmatching_files","metadata":{"execution":{"iopub.status.busy":"2024-03-21T15:18:10.005538Z","iopub.execute_input":"2024-03-21T15:18:10.007628Z","iopub.status.idle":"2024-03-21T15:18:10.020083Z","shell.execute_reply.started":"2024-03-21T15:18:10.007580Z","shell.execute_reply":"2024-03-21T15:18:10.018478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nread_files(matching_files)","metadata":{"execution":{"iopub.status.busy":"2024-03-21T15:18:11.436447Z","iopub.execute_input":"2024-03-21T15:18:11.436911Z","iopub.status.idle":"2024-03-21T15:18:16.331156Z","shell.execute_reply.started":"2024-03-21T15:18:11.436873Z","shell.execute_reply":"2024-03-21T15:18:16.329913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4. Internal Source - Depth 1","metadata":{}},{"cell_type":"markdown","source":"## . . 4.1. applprev_1","metadata":{}},{"cell_type":"code","source":"matching_files = sorted(TRAIN_DIR.glob(\"train_applprev_1_*.parquet\"))\nmatching_files","metadata":{"execution":{"iopub.status.busy":"2024-03-21T15:18:16.333054Z","iopub.execute_input":"2024-03-21T15:18:16.333402Z","iopub.status.idle":"2024-03-21T15:18:16.342407Z","shell.execute_reply.started":"2024-03-21T15:18:16.333373Z","shell.execute_reply":"2024-03-21T15:18:16.341396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nread_files(matching_files)","metadata":{"execution":{"iopub.status.busy":"2024-03-21T15:18:16.343983Z","iopub.execute_input":"2024-03-21T15:18:16.345004Z","iopub.status.idle":"2024-03-21T15:18:24.069616Z","shell.execute_reply.started":"2024-03-21T15:18:16.344966Z","shell.execute_reply":"2024-03-21T15:18:24.068223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## . . 4.2. other_1","metadata":{}},{"cell_type":"code","source":"%%time\n\npl.read_parquet(TRAIN_DIR / 'train_other_1.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-03-21T15:18:24.072206Z","iopub.execute_input":"2024-03-21T15:18:24.072613Z","iopub.status.idle":"2024-03-21T15:18:24.105583Z","shell.execute_reply.started":"2024-03-21T15:18:24.072580Z","shell.execute_reply":"2024-03-21T15:18:24.104332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## . . 4.3. deposit_1","metadata":{}},{"cell_type":"code","source":"%%time\n\npl.read_parquet(TRAIN_DIR / 'train_deposit_1.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-03-21T15:18:24.107128Z","iopub.execute_input":"2024-03-21T15:18:24.107614Z","iopub.status.idle":"2024-03-21T15:18:24.145866Z","shell.execute_reply.started":"2024-03-21T15:18:24.107580Z","shell.execute_reply":"2024-03-21T15:18:24.144596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## . . 4.4. person_1","metadata":{}},{"cell_type":"code","source":"%%time\n\npl.read_parquet(TRAIN_DIR / 'train_person_1.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-03-21T15:18:24.149331Z","iopub.execute_input":"2024-03-21T15:18:24.150412Z","iopub.status.idle":"2024-03-21T15:18:25.952860Z","shell.execute_reply.started":"2024-03-21T15:18:24.150356Z","shell.execute_reply":"2024-03-21T15:18:25.951460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## . . 4.5. debitcard_1","metadata":{}},{"cell_type":"code","source":"%%time\n\npl.read_parquet(TRAIN_DIR / 'train_debitcard_1.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-03-21T15:18:25.956895Z","iopub.execute_input":"2024-03-21T15:18:25.957323Z","iopub.status.idle":"2024-03-21T15:18:25.999200Z","shell.execute_reply.started":"2024-03-21T15:18:25.957290Z","shell.execute_reply":"2024-03-21T15:18:25.997991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 5. Internal Source - Depth 2","metadata":{}},{"cell_type":"markdown","source":"## . . 5.2. applprev_2","metadata":{}},{"cell_type":"code","source":"%%time\n\npl.read_parquet(TRAIN_DIR / 'train_applprev_2.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-03-21T15:18:26.171090Z","iopub.execute_input":"2024-03-21T15:18:26.171895Z","iopub.status.idle":"2024-03-21T15:18:27.779735Z","shell.execute_reply.started":"2024-03-21T15:18:26.171844Z","shell.execute_reply":"2024-03-21T15:18:27.778500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 6. External Source - Depth 0","metadata":{}},{"cell_type":"markdown","source":"## . . 6.1. static_cb","metadata":{}},{"cell_type":"code","source":"%%time\n\npl.read_parquet(TRAIN_DIR / 'train_static_cb_0.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-03-21T15:18:30.381354Z","iopub.execute_input":"2024-03-21T15:18:30.381823Z","iopub.status.idle":"2024-03-21T15:18:31.456073Z","shell.execute_reply.started":"2024-03-21T15:18:30.381774Z","shell.execute_reply":"2024-03-21T15:18:31.454894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 7. External Source - Depth 1","metadata":{}},{"cell_type":"markdown","source":"## . . 7.1. tax_registry_a_1","metadata":{}},{"cell_type":"code","source":"%%time\n\npl.read_parquet(TRAIN_DIR / 'train_tax_registry_a_1.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-03-21T15:18:32.776958Z","iopub.execute_input":"2024-03-21T15:18:32.778021Z","iopub.status.idle":"2024-03-21T15:18:33.181430Z","shell.execute_reply.started":"2024-03-21T15:18:32.777974Z","shell.execute_reply":"2024-03-21T15:18:33.180188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## . . 7.2. tax_registry_b_1","metadata":{}},{"cell_type":"code","source":"%%time\n\npl.read_parquet(TRAIN_DIR / 'train_tax_registry_a_1.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-03-21T15:18:34.810861Z","iopub.execute_input":"2024-03-21T15:18:34.811313Z","iopub.status.idle":"2024-03-21T15:18:35.139146Z","shell.execute_reply.started":"2024-03-21T15:18:34.811262Z","shell.execute_reply":"2024-03-21T15:18:35.138212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## . . 7.3. tax_registry_c_1","metadata":{}},{"cell_type":"code","source":"%%time\n\npl.read_parquet(TRAIN_DIR / 'train_tax_registry_a_1.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-03-21T15:18:36.094268Z","iopub.execute_input":"2024-03-21T15:18:36.095436Z","iopub.status.idle":"2024-03-21T15:18:36.404900Z","shell.execute_reply.started":"2024-03-21T15:18:36.095385Z","shell.execute_reply":"2024-03-21T15:18:36.403465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## . . 7.4. credit_bureau_a_1\n- `read_files()` fails with out of memory\n- row counts\n\n```\n- 0: 4_108_212\n- 1: 6_009_192\n- 2: 3_743_810\n- 3: 2_079_323\n```","metadata":{}},{"cell_type":"code","source":"matching_files = sorted(TRAIN_DIR.glob(\"train_credit_bureau_a_1_*.parquet\"))\nmatching_files","metadata":{"execution":{"iopub.status.busy":"2024-03-21T15:18:42.235754Z","iopub.execute_input":"2024-03-21T15:18:42.236210Z","iopub.status.idle":"2024-03-21T15:18:42.244342Z","shell.execute_reply.started":"2024-03-21T15:18:42.236177Z","shell.execute_reply":"2024-03-21T15:18:42.242728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\npl.read_parquet(TRAIN_DIR / 'train_credit_bureau_a_1_0.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-03-21T15:18:58.611138Z","iopub.execute_input":"2024-03-21T15:18:58.611625Z","iopub.status.idle":"2024-03-21T15:19:02.642342Z","shell.execute_reply.started":"2024-03-21T15:18:58.611578Z","shell.execute_reply":"2024-03-21T15:19:02.640952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## . . 7.5. credit_bureau_b_1","metadata":{}},{"cell_type":"code","source":"%%time\n\npl.read_parquet(TRAIN_DIR / 'train_credit_bureau_b_1.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-03-21T15:19:36.441251Z","iopub.execute_input":"2024-03-21T15:19:36.441733Z","iopub.status.idle":"2024-03-21T15:19:36.530456Z","shell.execute_reply.started":"2024-03-21T15:19:36.441696Z","shell.execute_reply":"2024-03-21T15:19:36.529154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 8. External Source - Depth 2","metadata":{}},{"cell_type":"markdown","source":"## . . 8.1. credit_bureau_a_2\n- `read_files()` fails with out of memory\n- row counts\n\n```\n-  0:  5_296_031\n-  1:  7_861_809\n-  2: 17_893_536\n-  3: 26_563_901\n-  4: 27_025_737\n-  5: 33_053_760\n-  6: 25_511_332\n-  7:  8_055_986\n-  8: 13_927_071\n-  9: 18_723_227\n- 10:  4_386_062\n```","metadata":{}},{"cell_type":"code","source":"matching_files = sorted(TRAIN_DIR.glob(\"train_credit_bureau_a_2_*.parquet\"), key=lambda x: int(x.stem.split('_')[-1]))\nmatching_files","metadata":{"execution":{"iopub.status.busy":"2024-03-21T15:29:28.297238Z","iopub.execute_input":"2024-03-21T15:29:28.298731Z","iopub.status.idle":"2024-03-21T15:29:28.316292Z","shell.execute_reply.started":"2024-03-21T15:29:28.298676Z","shell.execute_reply":"2024-03-21T15:29:28.315148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\npl.read_parquet(TRAIN_DIR / 'train_credit_bureau_a_2_0.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-03-21T15:29:29.730071Z","iopub.execute_input":"2024-03-21T15:29:29.731293Z","iopub.status.idle":"2024-03-21T15:29:31.498078Z","shell.execute_reply.started":"2024-03-21T15:29:29.731231Z","shell.execute_reply":"2024-03-21T15:29:31.496826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## . . 8.2. credit_bureau_b_2","metadata":{}},{"cell_type":"code","source":"%%time\n\npl.read_parquet(TRAIN_DIR / 'train_credit_bureau_b_2.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-03-21T15:37:22.409147Z","iopub.execute_input":"2024-03-21T15:37:22.409731Z","iopub.status.idle":"2024-03-21T15:37:22.565998Z","shell.execute_reply.started":"2024-03-21T15:37:22.409690Z","shell.execute_reply":"2024-03-21T15:37:22.564579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 9. Depth example\n- {discussion} [What is the meaning of num_group1 and num_group2 in depth1 and 2 dataset?](https://www.kaggle.com/competitions/home-credit-credit-risk-model-stability/discussion/476763)\n- {discussion} [Understanding the dataset (depths, num_groupN)](https://www.kaggle.com/competitions/home-credit-credit-risk-model-stability/discussion/482392)","metadata":{}},{"cell_type":"markdown","source":"## . . 9.1. Depth 1","metadata":{}},{"cell_type":"code","source":"# Reset max row config\npl.Config.set_tbl_rows(None)","metadata":{"execution":{"iopub.status.busy":"2024-03-21T00:39:36.242971Z","iopub.execute_input":"2024-03-21T00:39:36.243393Z","iopub.status.idle":"2024-03-21T00:39:36.250304Z","shell.execute_reply.started":"2024-03-21T00:39:36.243362Z","shell.execute_reply":"2024-03-21T00:39:36.249445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_applprev_1_0 = pl.read_parquet(TRAIN_DIR / 'train_applprev_1_0.parquet')\ndf_train_applprev_1_0","metadata":{"execution":{"iopub.status.busy":"2024-03-21T15:39:02.146989Z","iopub.execute_input":"2024-03-21T15:39:02.147498Z","iopub.status.idle":"2024-03-21T15:39:04.906809Z","shell.execute_reply.started":"2024-03-21T15:39:02.147459Z","shell.execute_reply":"2024-03-21T15:39:04.905928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_applprev_1_0.filter(\n    pl.col('case_id') <= 10\n).select(['case_id', 'num_group1'])","metadata":{"execution":{"iopub.status.busy":"2024-03-21T15:39:13.180349Z","iopub.execute_input":"2024-03-21T15:39:13.180756Z","iopub.status.idle":"2024-03-21T15:39:13.228839Z","shell.execute_reply.started":"2024-03-21T15:39:13.180725Z","shell.execute_reply":"2024-03-21T15:39:13.227515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Each `case_id` could have multiple records. `num_group1` is the index.","metadata":{}},{"cell_type":"markdown","source":"## . . 9.2. Depth 2","metadata":{}},{"cell_type":"code","source":"df_train_applprev_2 = pl.read_parquet(TRAIN_DIR / 'train_applprev_2.parquet')\ndf_train_applprev_2","metadata":{"execution":{"iopub.status.busy":"2024-03-21T15:39:53.092913Z","iopub.execute_input":"2024-03-21T15:39:53.093412Z","iopub.status.idle":"2024-03-21T15:39:54.622953Z","shell.execute_reply.started":"2024-03-21T15:39:53.093374Z","shell.execute_reply":"2024-03-21T15:39:54.621658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_applprev_2.filter(\n    pl.col('case_id') <= 6\n).select(['case_id', 'num_group1', 'num_group2'])","metadata":{"execution":{"iopub.status.busy":"2024-03-21T15:40:02.171216Z","iopub.execute_input":"2024-03-21T15:40:02.171665Z","iopub.status.idle":"2024-03-21T15:40:02.199404Z","shell.execute_reply.started":"2024-03-21T15:40:02.171630Z","shell.execute_reply":"2024-03-21T15:40:02.198153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Each (`case_id`, `num_group1`) pair could have multiple records. `num_group2` is the index.","metadata":{}}]}