{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"dockerImageVersionId":30673,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport polars as pl\nimport glob\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.model_selection import train_test_split\n\nimport wandb\nfrom kaggle_secrets import UserSecretsClient","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-04-04T20:11:18.147866Z","iopub.execute_input":"2024-04-04T20:11:18.148394Z","iopub.status.idle":"2024-04-04T20:11:19.840291Z","shell.execute_reply.started":"2024-04-04T20:11:18.148357Z","shell.execute_reply":"2024-04-04T20:11:19.839059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PATH = '/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train'\n\nTRAIN_DEPTH_0 = [\n    'train_static_0_*.parquet',    \n    'train_static_cb_0.parquet',\n]\nTEST_DEPTH_0 = [\n    'test_static_0_*.parquet',\n    'test_static_cb_0.parquet'\n]\n\nTRAIN_DEPTH_1 = [\n    'train_applprev_1_*.parquet',\n    'train_other_1.parquet',\n    'train_tax_registry_a_1.parquet',\n    'train_tax_registry_b_1.parquet',\n    'train_tax_registry_c_1.parquet',\n    'train_credit_bureau_a_1_*.parquet',\n    'train_credit_bureau_b_1.parquet',\n    'train_deposit_1.parquet',\n    'train_person_1.parquet',\n    'train_debitcard_1.parquet',\n    \n]\nTEST_DEPTH_1 = [\n    'test_applprev_1_*.parquet',\n    'test_other_1.parquet',\n    'test_tax_registry_a_1.parquet',\n    'test_tax_registry_b_1.parquet',\n    'test_tax_registry_c_1.parquet',\n    'test_credit_bureau_a_1_*.parquet',\n    'test_credit_bureau_b_1.parquet',\n    'test_deposit_1.parquet',\n    'test_person_1.parquet',\n    'test_debitcard_1.parquet'\n]\n\n\nTRAIN_DEPTH_2 = [\n    #'train_applprev_2.parquet',\n    #'train_person_2.parquet',\n    'train_credit_bureau_a_2_*.parquet',\n    'train_credit_bureau_b_2.parquet',\n]\n\nTEST_DEPTH_2 = [\n    #'test_applprev_2.parquet',\n    #'test_person_2.parquet',\n    'test_credit_bureau_a_2_*.parquet',\n    'test_credit_bureau_b_2.parquet'\n]","metadata":{"execution":{"iopub.status.busy":"2024-04-04T19:55:40.453405Z","iopub.execute_input":"2024-04-04T19:55:40.454062Z","iopub.status.idle":"2024-04-04T19:55:40.461887Z","shell.execute_reply.started":"2024-04-04T19:55:40.454022Z","shell.execute_reply":"2024-04-04T19:55:40.460433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pipeline Funcitons","metadata":{}},{"cell_type":"code","source":"def filter_cols(lazy_df, threshold=0.95):   \n    #len_df = lazy_df.select(pl.len()).collect().item()\n    #null_percent = lazy_df.null_count().melt(id_vars=\"case_id\").with_columns( pl.col('value') /  len_df)\n    null_percent = lazy_df.select( pl.all().is_null().sum() / pl.all().len() ).melt(id_vars=\"case_id\")\n    columns_to_select = null_percent.filter(pl.col('value') < threshold).select('variable')\n    \n    columns_list = columns_to_select.collect().get_column('variable').to_list()\n    \n    return lazy_df.select(['case_id'] + columns_list )\n    ","metadata":{"execution":{"iopub.status.busy":"2024-04-04T19:55:40.483965Z","iopub.execute_input":"2024-04-04T19:55:40.484731Z","iopub.status.idle":"2024-04-04T19:55:40.494612Z","shell.execute_reply.started":"2024-04-04T19:55:40.484691Z","shell.execute_reply":"2024-04-04T19:55:40.493302Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_dtypes(lazy_df):\n    \n    expressions = []\n    for c in lazy_df.columns:\n        if c.endswith('P') or c.endswith('A'):\n            expressions.extend([\n                pl.col(c).cast(pl.Float32)\n            ])\n        elif c.endswith('M'):\n            expressions.extend([\n                pl.col(c).cast(pl.String)\n            ])\n        elif c.endswith('D'):\n            expressions.extend([\n                pl.col(c).cast(pl.Date)\n            ])\n    \n    return lazy_df.with_columns(expressions)\n            ","metadata":{"execution":{"iopub.status.busy":"2024-04-04T19:55:40.495914Z","iopub.execute_input":"2024-04-04T19:55:40.496237Z","iopub.status.idle":"2024-04-04T19:55:40.506459Z","shell.execute_reply.started":"2024-04-04T19:55:40.496209Z","shell.execute_reply":"2024-04-04T19:55:40.505206Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fill_empty_columns(lazy_df):\n    return lazy_df","metadata":{"execution":{"iopub.status.busy":"2024-04-04T19:55:40.507653Z","iopub.execute_input":"2024-04-04T19:55:40.507962Z","iopub.status.idle":"2024-04-04T19:55:40.516930Z","shell.execute_reply.started":"2024-04-04T19:55:40.507935Z","shell.execute_reply":"2024-04-04T19:55:40.516077Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def aggregate_p(lazy_df):\n    \n    agg_expressions = []\n    for c in lazy_df.columns:\n        if c.endswith('P'):\n            agg_expressions.extend([\n                pl.col(c).mean().alias(f\"{c}_mean\"),\n                pl.col(c).max().alias(f\"{c}_max\"),\n                pl.col(c).min().alias(f\"{c}_min\"),\n                pl.col(c).sum().alias(f\"{c}_sum\")\n            ])\n    \n    return lazy_df.groupby('case_id').agg(agg_expressions)","metadata":{"execution":{"iopub.status.busy":"2024-04-04T19:55:40.518037Z","iopub.execute_input":"2024-04-04T19:55:40.518381Z","iopub.status.idle":"2024-04-04T19:55:40.528604Z","shell.execute_reply.started":"2024-04-04T19:55:40.518354Z","shell.execute_reply":"2024-04-04T19:55:40.527509Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def aggregate_m(lazy_df):\n    \n    agg_expressions = []\n    for c in lazy_df.columns:\n        if c.endswith('M'):\n            agg_expressions.extend([\n                pl.col(c).max().alias(f\"{c}_max\"),\n            ])\n    \n    return lazy_df.groupby('case_id').agg(agg_expressions)","metadata":{"execution":{"iopub.status.busy":"2024-04-04T19:55:40.530123Z","iopub.execute_input":"2024-04-04T19:55:40.530579Z","iopub.status.idle":"2024-04-04T19:55:40.539768Z","shell.execute_reply.started":"2024-04-04T19:55:40.530549Z","shell.execute_reply":"2024-04-04T19:55:40.538891Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def aggregate_a(lazy_df):\n    \n    agg_expressions = []\n    for c in lazy_df.columns:\n        if c.endswith('A'):\n            agg_expressions.extend([\n                pl.col(c).mean().alias(f\"{c}_mean\"),\n                pl.col(c).max().alias(f\"{c}_max\"),\n                pl.col(c).min().alias(f\"{c}_min\"),\n                pl.col(c).sum().alias(f\"{c}_sum\")\n            ])\n    \n    return lazy_df.groupby('case_id').agg(agg_expressions)","metadata":{"execution":{"iopub.status.busy":"2024-04-04T19:55:40.541028Z","iopub.execute_input":"2024-04-04T19:55:40.541390Z","iopub.status.idle":"2024-04-04T19:55:40.551805Z","shell.execute_reply.started":"2024-04-04T19:55:40.541360Z","shell.execute_reply":"2024-04-04T19:55:40.550310Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def aggregate_d(lazy_df):\n    \n    agg_expressions = []\n    for c in lazy_df.columns:\n        if c.endswith('D'):\n            agg_expressions.extend([\n                pl.col(c).first().alias(f\"{c}_first\"),\n                pl.col(c).max().alias(f\"{c}_last\")\n            ])\n    \n    return lazy_df.groupby('case_id').agg(agg_expressions)","metadata":{"execution":{"iopub.status.busy":"2024-04-04T19:55:40.554899Z","iopub.execute_input":"2024-04-04T19:55:40.555271Z","iopub.status.idle":"2024-04-04T19:55:40.566318Z","shell.execute_reply.started":"2024-04-04T19:55:40.555217Z","shell.execute_reply":"2024-04-04T19:55:40.564985Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def aggregate_t(lazy_df):\n    \n    agg_expressions = []\n    for c in lazy_df.columns:\n        if c.endswith('T'):\n            agg_expressions.extend([\n                pl.col(c).mean().alias(f\"{c}_mean\"),\n                pl.col(c).max().alias(f\"{c}_max\"),\n                pl.col(c).min().alias(f\"{c}_min\"),\n                pl.col(c).sum().alias(f\"{c}_sum\")\n            ])\n    \n    return lazy_df.groupby('case_id').agg(agg_expressions)","metadata":{"execution":{"iopub.status.busy":"2024-04-04T19:55:40.567945Z","iopub.execute_input":"2024-04-04T19:55:40.568335Z","iopub.status.idle":"2024-04-04T19:55:40.577980Z","shell.execute_reply.started":"2024-04-04T19:55:40.568294Z","shell.execute_reply":"2024-04-04T19:55:40.576767Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def aggregate_l(lazy_df):\n    \n    agg_expressions = []\n    for c in lazy_df.columns:\n        if c.endswith('L'):\n            agg_expressions.extend([\n                pl.col(c).max().alias(f\"{c}_max\"),\n            ])\n    \n    return lazy_df.groupby('case_id').agg(agg_expressions)","metadata":{"execution":{"iopub.status.busy":"2024-04-04T19:55:40.579625Z","iopub.execute_input":"2024-04-04T19:55:40.580058Z","iopub.status.idle":"2024-04-04T19:55:40.588674Z","shell.execute_reply.started":"2024-04-04T19:55:40.580017Z","shell.execute_reply":"2024-04-04T19:55:40.587794Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Depth 0","metadata":{}},{"cell_type":"code","source":"def get_depth_0(path, files):\n    for n in files:\n        temp = []\n        for file in glob.glob(f'{path}/{n}'):\n             temp.append(pl.scan_parquet(file))\n\n        d0_df = pl.concat(\n            temp,\n            how='vertical_relaxed'\n        )\n\n    d0_df = (\n        d0_df\n        .pipe(filter_cols)\n        .pipe(set_dtypes)\n    )\n\n    d0_df = d0_df.collect()\n    \n    return d0_df","metadata":{"execution":{"iopub.status.busy":"2024-04-04T19:55:40.589768Z","iopub.execute_input":"2024-04-04T19:55:40.590382Z","iopub.status.idle":"2024-04-04T19:55:40.601962Z","shell.execute_reply.started":"2024-04-04T19:55:40.590353Z","shell.execute_reply":"2024-04-04T19:55:40.600711Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Depth 1","metadata":{}},{"cell_type":"code","source":"def get_depth_1(path, files):\n    for n in files:\n        temp = []\n        for file in glob.glob(f'{path}/{n}'):\n             temp.append(pl.scan_parquet(file))\n\n        d1_df = pl.concat(\n            temp,\n            how='vertical_relaxed'\n        )\n\n    d1_df = (\n            d1_df\n            .pipe(filter_cols)\n            .pipe(set_dtypes)\n            .pipe(fill_empty_columns)\n        )\n    d1_df = pl.concat(\n        [\n            d1_df.pipe(aggregate_p), \n            d1_df.pipe(aggregate_m), \n            d1_df.pipe(aggregate_a),\n            d1_df.pipe(aggregate_d),\n            d1_df.pipe(aggregate_t),\n            d1_df.pipe(aggregate_l)\n        ],\n        how='align'\n    )\n\n    d1_df = d1_df.collect()\n    \n    return d1_df\n","metadata":{"execution":{"iopub.status.busy":"2024-04-04T19:55:40.603284Z","iopub.execute_input":"2024-04-04T19:55:40.604266Z","iopub.status.idle":"2024-04-04T19:55:40.614515Z","shell.execute_reply.started":"2024-04-04T19:55:40.604218Z","shell.execute_reply":"2024-04-04T19:55:40.613639Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Depth 2","metadata":{}},{"cell_type":"code","source":"def get_depth_2(path, files):\n    for n in files:\n        temp = []\n        for file in glob.glob(f'{path}/{n}'):\n            temp.append(pl.scan_parquet(file))\n\n        d2_df = pl.concat(\n            temp,\n            how='vertical_relaxed'\n        )\n\n    d2_df = d2_df.pipe(filter_cols)\\\n            .pipe(set_dtypes)\\\n            .pipe(fill_empty_columns)\n\n    d2_df = pl.concat(\n        [\n            d2_df.pipe(aggregate_p), \n            d2_df.pipe(aggregate_m), \n            d2_df.pipe(aggregate_a),\n            d2_df.pipe(aggregate_d),\n            d2_df.pipe(aggregate_t),\n            d2_df.pipe(aggregate_l)\n        ],\n        how='align'\n    )\n\n    d2_df = d2_df.collect()\n    return d2_df","metadata":{"execution":{"iopub.status.busy":"2024-04-04T19:55:40.615762Z","iopub.execute_input":"2024-04-04T19:55:40.616451Z","iopub.status.idle":"2024-04-04T19:55:40.629129Z","shell.execute_reply.started":"2024-04-04T19:55:40.616410Z","shell.execute_reply":"2024-04-04T19:55:40.628341Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Main","metadata":{}},{"cell_type":"markdown","source":"## Train","metadata":{}},{"cell_type":"code","source":"base = pl.scan_parquet(f'{PATH}/train_base.parquet')\nbase = base.pipe(filter_cols).pipe(set_dtypes).collect()\n\nd0_df = get_depth_0(PATH, TRAIN_DEPTH_0)\nd1_df = get_depth_1(PATH, TRAIN_DEPTH_1)\nd2_df = get_depth_2(PATH, TRAIN_DEPTH_2)\n\nbase = base.join(\n    other=d0_df,\n    on='case_id',\n    how='left'\n)\nbase = base.join(\n    other=d1_df,\n    on='case_id',\n    how='left'\n)\nbase = base.join(\n    other=d2_df,\n    on='case_id',\n    how='left'\n)\nbase = base.to_pandas()\n\ntrain, test = train_test_split(base, test_size=0.2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(nrows=1,ncols=2, sharey='row', figsize=(10,2))\nax[0].set_title('Train Target Count Plot')\nax[1].set_title('Test Target Count Plot')\nsns.countplot(data=train, x=\"target\", ax=ax[0])\nsns.countplot(data=test, x=\"target\", ax=ax[1])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-04T20:04:49.955582Z","iopub.execute_input":"2024-04-04T20:04:49.955972Z","iopub.status.idle":"2024-04-04T20:04:50.348591Z","shell.execute_reply.started":"2024-04-04T20:04:49.955942Z","shell.execute_reply":"2024-04-04T20:04:50.347361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.to_csv('train.csv', index=False)\ntest.to_csv('test.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Wandb Artifact","metadata":{}},{"cell_type":"code","source":"user_secrets = UserSecretsClient()\nwandb_key = user_secrets.get_secret(\"wandb_key\")\n\nwandb.login(key=wandb_key)\nrun = wandb.init(project=\"HC-CRMS\", job_type=\"add-dataset\")\n\nartifact = wandb.Artifact(name=\"data\", type=\"dataset\")\n\nartifact.add_file(local_path=\"/kaggle/working/train.csv\", name=\"train.csv\")\nartifact.add_file(local_path=\"/kaggle/working/test.csv\", name=\"test.csv\")\n\nrun.log_artifact(artifact)","metadata":{"execution":{"iopub.status.busy":"2024-04-04T20:22:54.768697Z","iopub.execute_input":"2024-04-04T20:22:54.769179Z","iopub.status.idle":"2024-04-04T20:23:32.222959Z","shell.execute_reply.started":"2024-04-04T20:22:54.769144Z","shell.execute_reply":"2024-04-04T20:23:32.221866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wandb.finish()","metadata":{"execution":{"iopub.status.busy":"2024-04-04T20:24:37.941039Z","iopub.execute_input":"2024-04-04T20:24:37.941512Z","iopub.status.idle":"2024-04-04T20:24:40.388616Z","shell.execute_reply.started":"2024-04-04T20:24:37.941476Z","shell.execute_reply":"2024-04-04T20:24:40.387401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}