{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"dockerImageVersionId":30664,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### 🚧 Work in progress","metadata":{}},{"cell_type":"code","source":"import polars as pl","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-03-29T16:13:25.390616Z","iopub.execute_input":"2024-03-29T16:13:25.391817Z","iopub.status.idle":"2024-03-29T16:13:25.679839Z","shell.execute_reply.started":"2024-03-29T16:13:25.391755Z","shell.execute_reply":"2024-03-29T16:13:25.678513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<img src=\"https://raw.githubusercontent.com/pola-rs/polars-static/master/logos/polars_github_logo_rect_dark_name.svg\" width=\"600\">\n\n**References**\n- https://docs.pola.rs/user-guide/getting-started/","metadata":{}},{"cell_type":"code","source":"# Show head(2) and tail(2) only\n# https://docs.pola.rs/py-polars/html/reference/config.html\n# https://docs.pola.rs/py-polars/html/reference/api/polars.Config.set_tbl_rows.html\npl.Config.set_tbl_rows(4)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:13:25.682234Z","iopub.execute_input":"2024-03-29T16:13:25.682591Z","iopub.status.idle":"2024-03-29T16:13:25.690213Z","shell.execute_reply.started":"2024-03-29T16:13:25.682559Z","shell.execute_reply":"2024-03-29T16:13:25.689295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1. Read datasets\n- {User guide} [Getting started > **Reading & writing**](https://docs.pola.rs/user-guide/getting-started/#reading-writing)\n- {API reference} [pl.**read_csv()**](https://docs.pola.rs/py-polars/html/reference/api/polars.read_csv.html#polars.read_csv)\n- {API reference} [pl.**read_parquet()**](https://docs.pola.rs/py-polars/html/reference/api/polars.read_parquet.html#polars.read_parquet)","metadata":{}},{"cell_type":"code","source":"# Read csv file\ndf_base = pl.read_csv(\n    \"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_base.csv\",\n)\ndf_base","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:13:25.691488Z","iopub.execute_input":"2024-03-29T16:13:25.692592Z","iopub.status.idle":"2024-03-29T16:13:26.137774Z","shell.execute_reply.started":"2024-03-29T16:13:25.692550Z","shell.execute_reply":"2024-03-29T16:13:26.136496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read parquet file\ndf_static_0_0 = pl.read_parquet(\n    \"/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_static_0_0.parquet\",\n)\ndf_static_0_0","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:13:26.141157Z","iopub.execute_input":"2024-03-29T16:13:26.141904Z","iopub.status.idle":"2024-03-29T16:13:27.907567Z","shell.execute_reply.started":"2024-03-29T16:13:26.141857Z","shell.execute_reply":"2024-03-29T16:13:27.906559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_static_0_1 = pl.read_parquet(\n    \"/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_static_0_1.parquet\",\n)\ndf_static_0_1","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:13:27.911518Z","iopub.execute_input":"2024-03-29T16:13:27.913575Z","iopub.status.idle":"2024-03-29T16:13:28.866057Z","shell.execute_reply.started":"2024-03-29T16:13:27.913534Z","shell.execute_reply":"2024-03-29T16:13:28.865030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_person_1 = pl.read_parquet(\n    \"/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_person_1.parquet\"\n)\ndf_person_1","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:13:28.868176Z","iopub.execute_input":"2024-03-29T16:13:28.868528Z","iopub.status.idle":"2024-03-29T16:13:31.245141Z","shell.execute_reply.started":"2024-03-29T16:13:28.868498Z","shell.execute_reply":"2024-03-29T16:13:31.243845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. Concatenation\n- {User guide} [Transformation > **Concatenation**](https://docs.pola.rs/user-guide/transformations/concatenation/)","metadata":{}},{"cell_type":"code","source":"df_static_0 = pl.concat(\n    [df_static_0_0, df_static_0_1],\n    how = 'vertical'\n)\ndf_static_0","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:13:31.246593Z","iopub.execute_input":"2024-03-29T16:13:31.246926Z","iopub.status.idle":"2024-03-29T16:13:33.611869Z","shell.execute_reply.started":"2024-03-29T16:13:31.246899Z","shell.execute_reply":"2024-03-29T16:13:33.610469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. Join\n- {User guide} [Getting started > **Join**](https://docs.pola.rs/user-guide/getting-started/#join)\n- {User guide} [Transformations > **Joins**](https://docs.pola.rs/user-guide/transformations/joins/)\n- {API reference} [DataFrame.**join()**](https://docs.pola.rs/py-polars/html/reference/dataframe/api/polars.DataFrame.join.html)","metadata":{}},{"cell_type":"code","source":"df_train = df_base.join(\n    df_static_0,\n    on = 'case_id',\n    how = 'left',    \n)\ndf_train","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:13:33.613305Z","iopub.execute_input":"2024-03-29T16:13:33.613640Z","iopub.status.idle":"2024-03-29T16:13:35.189021Z","shell.execute_reply.started":"2024-03-29T16:13:33.613609Z","shell.execute_reply":"2024-03-29T16:13:35.188052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4. Pipe\n- {API reference} [DataFrame.**pipe()**](https://docs.pola.rs/py-polars/html/reference/dataframe/api/polars.DataFrame.pipe.html#polars.DataFrame.pipe)\n- {User quide) [Migrating > Coming from Pandas > **Pipe littering**](https://docs.pola.rs/user-guide/migration/pandas/#pipe-littering)\n- {Notebook} [Home Credit Baseline > Pipeline](https://www.kaggle.com/code/greysky/home-credit-baseline?scriptVersionId=166530112&cellId=5)","metadata":{}},{"cell_type":"code","source":"# Prepare function\n# https://www.kaggle.com/code/greysky/home-credit-baseline\ndef set_table_dtypes(df):\n    for col in df.columns:\n        if col in [\"case_id\", \"WEEK_NUM\", \"num_group1\", \"num_group2\"]:\n            df = df.with_columns(pl.col(col).cast(pl.Int32))\n        elif col in [\"date_decision\"]:\n            df = df.with_columns(pl.col(col).cast(pl.Date))\n        elif col[-1] in (\"P\", \"A\"):\n            df = df.with_columns(pl.col(col).cast(pl.Float64))\n        elif col[-1] in (\"M\",):\n            df = df.with_columns(pl.col(col).cast(pl.String))\n        elif col[-1] in (\"D\",):\n            df = df.with_columns(pl.col(col).cast(pl.Date))            \n\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:13:35.190467Z","iopub.execute_input":"2024-03-29T16:13:35.191579Z","iopub.status.idle":"2024-03-29T16:13:35.200151Z","shell.execute_reply.started":"2024-03-29T16:13:35.191536Z","shell.execute_reply":"2024-03-29T16:13:35.199140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_base['date_decision'].dtype","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:13:35.205516Z","iopub.execute_input":"2024-03-29T16:13:35.205966Z","iopub.status.idle":"2024-03-29T16:13:35.214647Z","shell.execute_reply.started":"2024-03-29T16:13:35.205936Z","shell.execute_reply":"2024-03-29T16:13:35.213777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_base = df_base.pipe(set_table_dtypes)\ndf_base","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:13:35.216140Z","iopub.execute_input":"2024-03-29T16:13:35.216764Z","iopub.status.idle":"2024-03-29T16:13:35.439013Z","shell.execute_reply.started":"2024-03-29T16:13:35.216719Z","shell.execute_reply":"2024-03-29T16:13:35.437860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_base['date_decision'].dtype","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:13:35.440527Z","iopub.execute_input":"2024-03-29T16:13:35.441295Z","iopub.status.idle":"2024-03-29T16:13:35.447633Z","shell.execute_reply.started":"2024-03-29T16:13:35.441262Z","shell.execute_reply":"2024-03-29T16:13:35.446541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_static_0 = df_static_0.pipe(set_table_dtypes)\ndf_person_1 = df_person_1.pipe(set_table_dtypes)\ndf_train = df_train.pipe(set_table_dtypes)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:13:35.449035Z","iopub.execute_input":"2024-03-29T16:13:35.449805Z","iopub.status.idle":"2024-03-29T16:13:38.978649Z","shell.execute_reply.started":"2024-03-29T16:13:35.449775Z","shell.execute_reply":"2024-03-29T16:13:38.977425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 5. Select Columns\n- {User guide} [Expressions > **Column selections**](https://docs.pola.rs/user-guide/expressions/column-selections/)\n- {User guide} [Expressions > **Basic operators**](https://docs.pola.rs/user-guide/expressions/operators/)\n- {API reference} [DataFrame.**select()** -> DataFrame](https://docs.pola.rs/py-polars/html/reference/dataframe/api/polars.DataFrame.select.html)","metadata":{}},{"cell_type":"code","source":"# Use expressions\ndf_base.select(\n    pl.col('date_decision'),\n    pl.col('MONTH'),\n)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:13:38.980302Z","iopub.execute_input":"2024-03-29T16:13:38.980692Z","iopub.status.idle":"2024-03-29T16:13:38.990044Z","shell.execute_reply.started":"2024-03-29T16:13:38.980657Z","shell.execute_reply":"2024-03-29T16:13:38.988867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# By multiple strings\ndf_base.select(\n    pl.col('date_decision', 'MONTH'),\n)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:13:38.992020Z","iopub.execute_input":"2024-03-29T16:13:38.992510Z","iopub.status.idle":"2024-03-29T16:13:39.010428Z","shell.execute_reply.started":"2024-03-29T16:13:38.992465Z","shell.execute_reply":"2024-03-29T16:13:39.009171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Select all columns\ndf_base.select(\n    pl.col(\"*\"),  # equivalent to `pl.all()`\n)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:13:39.012558Z","iopub.execute_input":"2024-03-29T16:13:39.013523Z","iopub.status.idle":"2024-03-29T16:13:39.022880Z","shell.execute_reply.started":"2024-03-29T16:13:39.013487Z","shell.execute_reply":"2024-03-29T16:13:39.021637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Exclude some columns\ndf_base.select(\n    pl.col(\"*\").exclude('WEEK_NUM', 'MONTH')\n)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:13:39.024739Z","iopub.execute_input":"2024-03-29T16:13:39.025619Z","iopub.status.idle":"2024-03-29T16:13:39.039149Z","shell.execute_reply.started":"2024-03-29T16:13:39.025567Z","shell.execute_reply":"2024-03-29T16:13:39.037628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# By data type\ndf_base.select(\n    pl.col(pl.Int64)\n)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:13:39.040505Z","iopub.execute_input":"2024-03-29T16:13:39.040839Z","iopub.status.idle":"2024-03-29T16:13:39.052915Z","shell.execute_reply.started":"2024-03-29T16:13:39.040811Z","shell.execute_reply":"2024-03-29T16:13:39.051540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Select colmuns with some operations and `alias`\ndf_base.select(\n    (pl.col('WEEK_NUM') + 1).alias('WEEK_NUM+1'),\n    (pl.col('WEEK_NUM') + 2).alias('WEEK_NUM+2'),\n)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:13:39.054660Z","iopub.execute_input":"2024-03-29T16:13:39.055027Z","iopub.status.idle":"2024-03-29T16:13:39.078453Z","shell.execute_reply.started":"2024-03-29T16:13:39.054994Z","shell.execute_reply":"2024-03-29T16:13:39.077599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 6. Filter rows\n- {User guide} [Getting started > **Filter**](https://docs.pola.rs/user-guide/getting-started/#filter)\n- {API reference} [DataFrame.**filter()**](https://docs.pola.rs/py-polars/html/reference/dataframe/api/polars.DataFrame.filter.html)","metadata":{}},{"cell_type":"code","source":"df_base.filter(\n    pl.col(\"WEEK_NUM\") == 0,\n)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:13:39.079615Z","iopub.execute_input":"2024-03-29T16:13:39.080622Z","iopub.status.idle":"2024-03-29T16:13:39.128985Z","shell.execute_reply.started":"2024-03-29T16:13:39.080590Z","shell.execute_reply":"2024-03-29T16:13:39.128130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_base.filter(\n    (pl.col(\"WEEK_NUM\") == 0) & (pl.col(\"target\")==1)\n)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:13:39.130503Z","iopub.execute_input":"2024-03-29T16:13:39.131078Z","iopub.status.idle":"2024-03-29T16:13:39.160705Z","shell.execute_reply.started":"2024-03-29T16:13:39.131047Z","shell.execute_reply":"2024-03-29T16:13:39.159576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_base.filter(\n    pl.col(\"WEEK_NUM\").is_between(0, 2),\n)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:13:39.162447Z","iopub.execute_input":"2024-03-29T16:13:39.162876Z","iopub.status.idle":"2024-03-29T16:13:39.196903Z","shell.execute_reply.started":"2024-03-29T16:13:39.162837Z","shell.execute_reply":"2024-03-29T16:13:39.196030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 7. Add Columns\n\n- {User guide} [Getting started > **Add columns**](https://docs.pola.rs/user-guide/getting-started/#add-columns)\n- {User guide} [Expressions > **Column selections**](https://docs.pola.rs/user-guide/expressions/column-selections/)\n- {API reference} [DataFrame.**with_colums()**](https://docs.pola.rs/py-polars/html/reference/dataframe/api/polars.DataFrame.with_columns.html)\n- {API reference} [Expressions > **Temporal**](https://docs.pola.rs/py-polars/html/reference/expressions/temporal.html) : `dt` related methods","metadata":{}},{"cell_type":"code","source":"# Use `alias`\ndf_base.with_columns(\n    pl.col(\"date_decision\").dt.year().alias(\"year\"),\n)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:13:39.198048Z","iopub.execute_input":"2024-03-29T16:13:39.198427Z","iopub.status.idle":"2024-03-29T16:13:39.229560Z","shell.execute_reply.started":"2024-03-29T16:13:39.198398Z","shell.execute_reply":"2024-03-29T16:13:39.228614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Use keyword arguments to easily name your expression inputs\ndf_base.with_columns(\n    month = pl.col(\"date_decision\").dt.month(),\n    day = pl.col(\"date_decision\").dt.day(),\n)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:13:39.230882Z","iopub.execute_input":"2024-03-29T16:13:39.231426Z","iopub.status.idle":"2024-03-29T16:13:39.256619Z","shell.execute_reply.started":"2024-03-29T16:13:39.231393Z","shell.execute_reply":"2024-03-29T16:13:39.255697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 8. Group by\n- {User guide} [Getting started > **Group by**](https://docs.pola.rs/user-guide/getting-started/#group-by)\n- {User guide} [Expressions > **Aggregation**](https://docs.pola.rs/user-guide/expressions/aggregation/)\n- {User guide} [Concepts > Contexts > **Group by / aggregation**](https://docs.pola.rs/user-guide/concepts/contexts/#group-by-aggregation)\n- {User guide} [Expressions > Functions > **Column naming**](https://docs.pola.rs/user-guide/expressions/functions/#column-naming)\n- {API Reference} [DataFrame.**group_by()**](https://docs.pola.rs/py-polars/html/reference/dataframe/api/polars.DataFrame.group_by.html)","metadata":{}},{"cell_type":"code","source":"# Show `float64` columns\ndf_person_1.select(\n    pl.col(pl.Float64)\n)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:13:39.257921Z","iopub.execute_input":"2024-03-29T16:13:39.258416Z","iopub.status.idle":"2024-03-29T16:13:39.265803Z","shell.execute_reply.started":"2024-03-29T16:13:39.258388Z","shell.execute_reply":"2024-03-29T16:13:39.264845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Use aggregation functions and add a suffix to the column name \ndf_person_1.group_by('case_id').agg(\n    pl.col('mainoccupationinc_384A').mean().name.suffix('_mean'),\n    pl.col('personindex_1023L').max().name.suffix('_max'),\n)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:13:39.267342Z","iopub.execute_input":"2024-03-29T16:13:39.267906Z","iopub.status.idle":"2024-03-29T16:13:39.657218Z","shell.execute_reply.started":"2024-03-29T16:13:39.267874Z","shell.execute_reply":"2024-03-29T16:13:39.656256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Iterate\ni = 0\nfor name, data in df_person_1.group_by(['case_id']):\n    if i > 3:\n        break\n    print(f'\\n\\n=== case_id: {name}')\n    display(data)\n    i += 1","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:14:33.458164Z","iopub.execute_input":"2024-03-29T16:14:33.458603Z","iopub.status.idle":"2024-03-29T16:14:34.271166Z","shell.execute_reply.started":"2024-03-29T16:14:33.458569Z","shell.execute_reply":"2024-03-29T16:14:34.270197Z"},"trusted":true},"execution_count":null,"outputs":[]}]}