{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"dockerImageVersionId":30664,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"I just wanted to find tables by feature names.","metadata":{}},{"cell_type":"code","source":"from pathlib import Path\n\nimport polars as pl\nimport polars.selectors as cs","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-04-11T15:49:26.881512Z","iopub.execute_input":"2024-04-11T15:49:26.882965Z","iopub.status.idle":"2024-04-11T15:49:26.986805Z","shell.execute_reply.started":"2024-04-11T15:49:26.882905Z","shell.execute_reply":"2024-04-11T15:49:26.985541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1. Prepare LazyFrames\n- {Polars User guide} [Lazy API > Usage](https://docs.pola.rs/user-guide/lazy/using/)\n- {Polars API reference} [LazyFrame](https://docs.pola.rs/py-polars/html/reference/lazyframe/index.html)","metadata":{}},{"cell_type":"code","source":"def set_table_dtypes(df):\n    for col in df.columns:\n        if col in [\"case_id\", \"WEEK_NUM\", \"num_group1\", \"num_group2\"]:\n            df = df.with_columns(pl.col(col).cast(pl.Int64))\n        elif col in [\"date_decision\"]:\n            df = df.with_columns(pl.col(col).cast(pl.Date))\n        elif col[-1] in (\"P\", \"A\"):\n            df = df.with_columns(pl.col(col).cast(pl.Float64))\n        elif col[-1] in (\"M\",):\n            df = df.with_columns(pl.col(col).cast(pl.String))\n        elif col[-1] in (\"D\",):\n            df = df.with_columns(pl.col(col).cast(pl.Date))\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-04-11T15:49:26.988800Z","iopub.execute_input":"2024-04-11T15:49:26.989147Z","iopub.status.idle":"2024-04-11T15:49:26.997777Z","shell.execute_reply.started":"2024-04-11T15:49:26.989118Z","shell.execute_reply":"2024-04-11T15:49:26.996931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def scan_file(path):\n    lazy_df = pl.scan_parquet(path)\n    lazy_df.pipe(set_table_dtypes)\n    return lazy_df\n\ndef scan_files(matching_files):\n    chunks = []    \n    for path in matching_files:\n        lazy_df = pl.scan_parquet(path)\n        lazy_df = lazy_df.pipe(set_table_dtypes)        \n        chunks.append(lazy_df)\n    \n    lazy_df = pl.concat(chunks, how=\"vertical_relaxed\")\n    lazy_df = lazy_df.unique(subset=[\"case_id\"])\n    return lazy_df","metadata":{"execution":{"iopub.status.busy":"2024-04-11T15:49:26.998940Z","iopub.execute_input":"2024-04-11T15:49:27.000027Z","iopub.status.idle":"2024-04-11T15:49:27.011424Z","shell.execute_reply.started":"2024-04-11T15:49:26.999992Z","shell.execute_reply":"2024-04-11T15:49:27.010117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROOT            = Path(\"/kaggle/input/home-credit-credit-risk-model-stability\")\nTRAIN_DIR       = ROOT / \"parquet_files\" / \"train\"","metadata":{"execution":{"iopub.status.busy":"2024-04-11T15:49:27.014785Z","iopub.execute_input":"2024-04-11T15:49:27.015211Z","iopub.status.idle":"2024-04-11T15:49:27.023282Z","shell.execute_reply.started":"2024-04-11T15:49:27.015179Z","shell.execute_reply":"2024-04-11T15:49:27.022172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_store = {\n    # depth_0\n    \"train_static_cb_0\": scan_file(TRAIN_DIR / \"train_static_cb_0.parquet\"),\n    \"train_static_0\": scan_files(TRAIN_DIR.glob(\"train_static_0_*.parquet\")),\n\n    # depth_1\n    \"train_applprev_1\": scan_files(TRAIN_DIR.glob(\"train_applprev_1_*.parquet\")),\n    \"train_tax_registry_a_1\": scan_file(TRAIN_DIR / \"train_tax_registry_a_1.parquet\"),\n    \"train_tax_registry_b_1\": scan_file(TRAIN_DIR / \"train_tax_registry_b_1.parquet\"),\n    \"train_tax_registry_c_1\": scan_file(TRAIN_DIR / \"train_tax_registry_c_1.parquet\"),\n    \"train_credit_bureau_a_1\": scan_files(TRAIN_DIR.glob(\"train_credit_bureau_a_1_*.parquet\")),\n    \"train_credit_bureau_b_1\": scan_file(TRAIN_DIR / \"train_credit_bureau_b_1.parquet\"),\n    \"train_other_1\": scan_file(TRAIN_DIR / \"train_other_1.parquet\"),\n    \"train_person_1\": scan_file(TRAIN_DIR / \"train_person_1.parquet\"),\n    \"train_deposit_1\": scan_file(TRAIN_DIR / \"train_deposit_1.parquet\"),\n    \"train_debitcard_1\": scan_file(TRAIN_DIR / \"train_debitcard_1.parquet\"),\n    \n    # depth_2\n    \"train_credit_bureau_b_2\": scan_file(TRAIN_DIR / \"train_credit_bureau_b_2.parquet\"),\n    \"train_credit_bureau_a_2\": scan_files(TRAIN_DIR.glob(\"train_credit_bureau_a_2_*.parquet\")),\n    \"train_applprev_2\": scan_file(TRAIN_DIR / \"train_applprev_2.parquet\"),\n    \"train_person_2\": scan_file(TRAIN_DIR / \"train_person_2.parquet\")\n}","metadata":{"execution":{"iopub.status.busy":"2024-04-11T15:49:27.025394Z","iopub.execute_input":"2024-04-11T15:49:27.025805Z","iopub.status.idle":"2024-04-11T15:49:27.141506Z","shell.execute_reply.started":"2024-04-11T15:49:27.025772Z","shell.execute_reply":"2024-04-11T15:49:27.140440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. Prepare Utility Functions","metadata":{}},{"cell_type":"code","source":"df_feature_definitions = pl.read_csv(ROOT / 'feature_definitions.csv')\n\n# https://docs.pola.rs/py-polars/html/reference/config.html#use-as-a-decorator\n@pl.Config(set_fmt_str_lengths=100, set_tbl_rows=-1)  # # Show whole rows and description\ndef get_features_by_keyword(keyword, show_description = True):\n    df_filtered = df_feature_definitions.filter(\n        pl.col('Variable').str.contains(keyword) |\n        pl.col('Description').str.contains(keyword)\n    )\n    if show_description:\n        display(df_filtered)\n    return df_filtered['Variable'].to_list()\n\ndef show_origin_tables(feature_names):\n    for table_name, lazy_df in data_store.items():\n        match_cols = set(lazy_df.columns) & set(feature_names)\n        if len(match_cols) > 0:\n            print(f'==== {table_name} ====')\n            display(lazy_df.select(cs.by_name(match_cols)).describe())","metadata":{"execution":{"iopub.status.busy":"2024-04-11T15:49:27.143011Z","iopub.execute_input":"2024-04-11T15:49:27.143633Z","iopub.status.idle":"2024-04-11T15:49:27.157506Z","shell.execute_reply.started":"2024-04-11T15:49:27.143597Z","shell.execute_reply":"2024-04-11T15:49:27.155956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. Usage (Birthday)\n- {discussion} [Analysis of birthday](https://www.kaggle.com/competitions/home-credit-credit-risk-model-stability/discussion/476463)","metadata":{}},{"cell_type":"code","source":"birth_related_features = get_features_by_keyword('birth')\nbirth_related_features","metadata":{"execution":{"iopub.status.busy":"2024-04-11T15:49:27.159296Z","iopub.execute_input":"2024-04-11T15:49:27.159970Z","iopub.status.idle":"2024-04-11T15:49:27.177452Z","shell.execute_reply.started":"2024-04-11T15:49:27.159934Z","shell.execute_reply":"2024-04-11T15:49:27.176190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_origin_tables(birth_related_features)","metadata":{"execution":{"iopub.status.busy":"2024-04-11T15:49:27.179058Z","iopub.execute_input":"2024-04-11T15:49:27.179810Z","iopub.status.idle":"2024-04-11T15:49:27.427031Z","shell.execute_reply.started":"2024-04-11T15:49:27.179762Z","shell.execute_reply":"2024-04-11T15:49:27.425491Z"},"trusted":true},"execution_count":null,"outputs":[]}]}