{"metadata":{"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"},{"sourceId":7600559,"sourceType":"datasetVersion","datasetId":4424545}],"dockerImageVersionId":30646,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.13"},"papermill":{"default_parameters":{},"duration":558.016249,"end_time":"2024-02-21T12:22:15.771938","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-02-21T12:12:57.755689","version":"2.5.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Introduction","metadata":{"papermill":{"duration":0.015517,"end_time":"2024-02-21T12:13:02.020293","exception":false,"start_time":"2024-02-21T12:13:02.004776","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"**Based on:**\n\nhttps://www.kaggle.com/code/greysky/home-credit-baseline\n\n\n**Related notebooks**\n\nTraining model-1 notebook:\n\nhttps://www.kaggle.com/andreynesterov/home-credit-baseline-training\n\nInference notebook:\n\nhttps://www.kaggle.com/andreynesterov/home-credit-baseline-inference","metadata":{"papermill":{"duration":0.011828,"end_time":"2024-02-21T12:13:02.046026","exception":false,"start_time":"2024-02-21T12:13:02.034198","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# Dependencies","metadata":{"papermill":{"duration":0.012795,"end_time":"2024-02-21T12:13:02.071155","exception":false,"start_time":"2024-02-21T12:13:02.05836","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import os\nimport gc\nfrom glob import glob\nfrom pathlib import Path\nfrom datetime import datetime\nimport re\n\nimport numpy as np\nimport pandas as pd\nimport polars as pl\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)\npd.set_option('display.max_columns', None)\npd.set_option('display.max_rows', 500)","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":3.770999,"end_time":"2024-02-21T12:13:05.854883","exception":false,"start_time":"2024-02-21T12:13:02.083884","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-20T10:00:32.857051Z","iopub.execute_input":"2024-05-20T10:00:32.857415Z","iopub.status.idle":"2024-05-20T10:00:35.497912Z","shell.execute_reply.started":"2024-05-20T10:00:32.857384Z","shell.execute_reply":"2024-05-20T10:00:35.496939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Configuration","metadata":{"papermill":{"duration":0.012114,"end_time":"2024-02-21T12:13:05.878951","exception":false,"start_time":"2024-02-21T12:13:05.866837","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class CFG:\n    root_dir = Path(\"/kaggle/input/home-credit-credit-risk-model-stability/\")\n    train_dir = Path(\"/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train\")\n    test_dir = Path(\"/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/test\")","metadata":{"papermill":{"duration":0.033103,"end_time":"2024-02-21T12:13:05.924976","exception":false,"start_time":"2024-02-21T12:13:05.891873","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-20T10:00:40.293190Z","iopub.execute_input":"2024-05-20T10:00:40.293743Z","iopub.status.idle":"2024-05-20T10:00:40.299074Z","shell.execute_reply.started":"2024-05-20T10:00:40.293708Z","shell.execute_reply":"2024-05-20T10:00:40.297830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature definitions","metadata":{"papermill":{"duration":0.014312,"end_time":"2024-02-21T12:13:05.952994","exception":false,"start_time":"2024-02-21T12:13:05.938682","status":"completed"},"tags":[]}},{"cell_type":"code","source":"if __name__ == '__main__':\n    with pd.option_context('display.max_rows', None, 'display.max_columns', None, 'max_colwidth', 400): \n        enhanced_feat_def_df = pd.read_parquet(\"/kaggle/input/home-credit-enhanced-feature-definitions/feature_definitions_dtypes_tables.parquet\")\n        display(enhanced_feat_def_df)","metadata":{"_kg_hide-output":true,"scrolled":true,"execution":{"iopub.status.busy":"2024-05-20T10:00:43.843914Z","iopub.execute_input":"2024-05-20T10:00:43.844295Z","iopub.status.idle":"2024-05-20T10:00:44.177437Z","shell.execute_reply.started":"2024-05-20T10:00:43.844265Z","shell.execute_reply":"2024-05-20T10:00:44.176413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Collection and Preprocessing","metadata":{"papermill":{"duration":0.018087,"end_time":"2024-02-21T12:13:06.119827","exception":false,"start_time":"2024-02-21T12:13:06.10174","status":"completed"},"tags":[]}},{"cell_type":"code","source":"### from https://www.kaggle.com/code/batprem/home-credit-risk-mode-utility-scripts\n\ndef reduce_mem_usage(df, float16_as32=True):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        if str(col_type)==\"category\":\n            continue\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    if float16_as32:\n                        df[col] = df[col].astype(np.float32)\n                    else:\n                        df[col] = df[col].astype(np.float16)                    \n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype('category')\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df","metadata":{"papermill":{"duration":0.048912,"end_time":"2024-02-21T12:13:06.186394","exception":false,"start_time":"2024-02-21T12:13:06.137482","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-20T10:01:05.557866Z","iopub.execute_input":"2024-05-20T10:01:05.558400Z","iopub.status.idle":"2024-05-20T10:01:05.571038Z","shell.execute_reply.started":"2024-05-20T10:01:05.558368Z","shell.execute_reply":"2024-05-20T10:01:05.570011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Pipeline","metadata":{"papermill":{"duration":0.021127,"end_time":"2024-02-21T12:13:06.229322","exception":false,"start_time":"2024-02-21T12:13:06.208195","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class Pipeline:\n    @staticmethod\n    def set_table_dtypes(df):\n        for col in df.columns:\n            if col in [\"case_id\", \"WEEK_NUM\", \"num_group1\", \"num_group2\"]:\n                df = df.with_columns(pl.col(col).cast(pl.Int64))\n            elif col in [\"date_decision\"]:\n                df = df.with_columns(pl.col(col).cast(pl.Date))\n            elif col[-1] in (\"P\", \"A\"):\n                df = df.with_columns(pl.col(col).cast(pl.Float64))\n            elif col[-1] in (\"M\",):\n                df = df.with_columns(pl.col(col).cast(pl.String))\n            elif col[-1] in (\"D\",):\n                df = df.with_columns(pl.col(col).cast(pl.Date))            \n\n        return df\n    \n    @staticmethod\n    def handle_dates(df):\n        for col in df.columns:\n            if col[-1] in (\"D\",):\n                df = df.with_columns(pl.col(col) - pl.col(\"date_decision\"))\n                df = df.with_columns(pl.col(col).dt.total_days())\n                \n        df = df.drop(\"date_decision\", \"MONTH\")\n\n        return df\n    \n    @staticmethod\n    def filter_cols(df):\n        for col in df.columns:\n            if col not in [\"target\", \"case_id\", \"WEEK_NUM\"]:\n                isnull = df[col].is_null().mean()\n\n                if isnull > 0.95:\n                    df = df.drop(col)\n\n        for col in df.columns:\n            if (col not in [\"target\", \"case_id\", \"WEEK_NUM\"]) & (df[col].dtype == pl.String):\n                freq = df[col].n_unique()\n\n                if (freq == 1) | (freq > 200):\n                    df = df.drop(col)\n\n        return df","metadata":{"papermill":{"duration":0.04415,"end_time":"2024-02-21T12:13:06.293737","exception":false,"start_time":"2024-02-21T12:13:06.249587","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-20T10:01:09.353004Z","iopub.execute_input":"2024-05-20T10:01:09.353403Z","iopub.status.idle":"2024-05-20T10:01:09.365772Z","shell.execute_reply.started":"2024-05-20T10:01:09.353370Z","shell.execute_reply":"2024-05-20T10:01:09.364390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Automatic Aggregation","metadata":{"papermill":{"duration":0.019447,"end_time":"2024-02-21T12:13:06.331153","exception":false,"start_time":"2024-02-21T12:13:06.311706","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class Aggregator:\n    def __init__(\n        self, \n        num_aggregators=[pl.max, pl.min, pl.first, pl.last, pl.mean], \n        str_aggregators=[pl.max, pl.min, pl.first, pl.last],  # n_unique\n        group_aggregators=[pl.max, pl.min, pl.first, pl.last],\n        str_mode=True\n    ):\n        self.num_aggregators = num_aggregators\n        self.str_aggregators = str_aggregators\n        self.group_aggregators = group_aggregators\n        self.str_mode=str_mode\n    \n    def num_expr(self, df_cols):\n        cols = [col for col in df_cols if col[-1] in (\"P\", \"A\")]\n        expr_all = []\n        for method in self.num_aggregators:\n            expr = [method(col).alias(f\"{method.__name__}_{col}\") for col in cols]\n            expr_all += expr\n\n        return expr_all\n\n    def date_expr(self, df_cols):\n        cols = [col for col in df_cols if col[-1] in (\"D\",)]\n        expr_all = []\n        for method in self.num_aggregators:\n            expr = [method(col).alias(f\"{method.__name__}_{col}\") for col in cols]  \n            expr_all += expr\n\n        return expr_all\n\n    def str_expr(self, df_cols):\n        cols = [col for col in df_cols if col[-1] in (\"M\",)]\n        \n        expr_all = []\n        for method in self.str_aggregators:\n            expr = [method(col).alias(f\"{method.__name__}_{col}\") for col in cols]  \n            expr_all += expr\n            \n        if self.str_mode:\n            expr_mode = [\n                pl.col(col)\n                .drop_nulls()\n                .mode()\n                .first()\n                .alias(f\"mode_{col}\")\n                for col in cols\n            ]\n        else:\n            expr_mode = []\n\n        return expr_all + expr_mode\n\n    def other_expr(self, df_cols):\n        cols = [col for col in df_cols if col[-1] in (\"T\", \"L\")]\n        \n        expr_all = []\n        for method in self.str_aggregators:\n            expr = [method(col).alias(f\"{method.__name__}_{col}\") for col in cols]  \n            expr_all += expr\n\n        return expr_all\n    \n    def count_expr(self, df_cols):\n        cols = [col for col in df_cols if \"num_group\" in col]\n\n        expr_all = []\n        for method in self.group_aggregators:\n            expr = [method(col).alias(f\"{method.__name__}_{col}\") for col in cols]  \n            expr_all += expr\n\n        return expr_all\n\n    def get_exprs(self, df_cols):\n        exprs = (\n            self.num_expr(df_cols) + \n            self.date_expr(df_cols) + \n            self.str_expr(df_cols) + \n            self.other_expr(df_cols) + \n            self.count_expr(df_cols)\n        )\n\n        return exprs","metadata":{"papermill":{"duration":0.047549,"end_time":"2024-02-21T12:13:06.396332","exception":false,"start_time":"2024-02-21T12:13:06.348783","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-20T10:01:12.732002Z","iopub.execute_input":"2024-05-20T10:01:12.732369Z","iopub.status.idle":"2024-05-20T10:01:12.749160Z","shell.execute_reply.started":"2024-05-20T10:01:12.732342Z","shell.execute_reply":"2024-05-20T10:01:12.747817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_files_by_path(pattern_path, aggregator, depth=None, agg_chunks=False, num_group1_filter=None):\n    chunks = []\n    for i, path in enumerate(glob(str(pattern_path))):\n        print(\"  chunk: \", i)\n        chunk = pl.read_parquet(path).pipe(Pipeline.set_table_dtypes)\n        if agg_chunks:\n            if num_group1_filter != None:\n                chunk = chunk.filter((pl.col(\"num_group1\") == num_group1_filter)).drop(columns=[\"num_group1\"])\n            chunk = chunk.group_by(\"case_id\").agg(aggregator.get_exprs(chunk.columns))\n        chunks.append(chunk)\n    df = pl.concat(chunks, how=\"vertical_relaxed\")\n    \n    if depth in [1, 2]:\n        print(f\"  agg, depth {depth}\")\n        if num_group1_filter != None:\n            df = df.filter((pl.col(\"num_group1\") == num_group1_filter)).drop(columns=[\"num_group1\"])\n        df = df.group_by(\"case_id\").agg(aggregator.get_exprs(df.columns))\n    return df","metadata":{"papermill":{"duration":0.038084,"end_time":"2024-02-21T12:13:06.452628","exception":false,"start_time":"2024-02-21T12:13:06.414544","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-20T10:01:16.985071Z","iopub.execute_input":"2024-05-20T10:01:16.986219Z","iopub.status.idle":"2024-05-20T10:01:16.994069Z","shell.execute_reply.started":"2024-05-20T10:01:16.986180Z","shell.execute_reply":"2024-05-20T10:01:16.992879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_files(files_arr, data_dir, aggregator, mode=\"train\", agg_chunks=False, num_group1_filter=None):\n    base_file_name = f\"{mode}_base.parquet\"\n    print(\"  files: \", base_file_name)\n    feats_df = read_files_by_path(\n        data_dir / base_file_name, data_dir, mode\n    )\n    \n    for i, file_name in enumerate(files_arr):\n        depth = re.findall(\"\\w_(\\d)\", file_name)\n        if len(depth) > 0:\n            depth = depth[0]\n        else:\n            continue\n        print(\"  files: \", file_name, f\"(depth: {depth})\")\n        files_df = read_files_by_path(\n            data_dir / f\"{mode}{file_name}\", \n            aggregator, int(depth), agg_chunks, \n            num_group1_filter=num_group1_filter\n        )\n        feats_df = feats_df.join(\n            files_df, \n            how=\"left\", on=\"case_id\", suffix=f\"_{depth}_{i}\"\n        )\n        del files_df\n        gc.collect()\n\n    return feats_df","metadata":{"papermill":{"duration":0.041201,"end_time":"2024-02-21T12:13:06.514663","exception":false,"start_time":"2024-02-21T12:13:06.473462","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-20T10:01:21.760075Z","iopub.execute_input":"2024-05-20T10:01:21.760472Z","iopub.status.idle":"2024-05-20T10:01:21.768557Z","shell.execute_reply.started":"2024-05-20T10:01:21.760427Z","shell.execute_reply":"2024-05-20T10:01:21.767431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def to_pandas(df_data, cat_cols=None):\n    df_data = df_data.to_pandas()\n    \n    if cat_cols is None:\n        cat_cols = list(df_data.select_dtypes(\"object\").columns)\n    \n    df_data[cat_cols] = df_data[cat_cols].astype(\"category\")\n    \n    return df_data","metadata":{"papermill":{"duration":0.033236,"end_time":"2024-02-21T12:13:06.570484","exception":false,"start_time":"2024-02-21T12:13:06.537248","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-20T10:01:25.681218Z","iopub.execute_input":"2024-05-20T10:01:25.682082Z","iopub.status.idle":"2024-05-20T10:01:25.688380Z","shell.execute_reply.started":"2024-05-20T10:01:25.682039Z","shell.execute_reply":"2024-05-20T10:01:25.687305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare df","metadata":{"papermill":{"duration":0.021875,"end_time":"2024-02-21T12:13:06.659545","exception":false,"start_time":"2024-02-21T12:13:06.63767","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def prepare_df(\n    files_arr, data_dir, aggregator, \n    mode=\"train\", cat_cols=None, train_cols=[], \n    agg_chunks=False, feat_eng=True, num_group1_filter=None\n):\n    print()\n    print(\"Collecting data...\")\n    feats_df = read_files(files_arr, data_dir, aggregator, mode=mode, agg_chunks=agg_chunks)\n    print(\"  feats_df shape:\\t\", feats_df.shape)\n    \n    if feat_eng:\n        print(\"Feature Engineering...\")\n        feats_df = feats_df.with_columns(\n            month_decision = pl.col(\"date_decision\").dt.month(),\n            weekday_decision = pl.col(\"date_decision\").dt.weekday(),\n        )\n        neworder = feats_df.columns\n        neworder.remove(\"month_decision\")\n        neworder.remove(\"weekday_decision\")\n        neworder.insert(5, \"month_decision\")\n        neworder.insert(6, \"weekday_decision\")\n        feats_df = feats_df.select(neworder)\n    \n    feats_df = feats_df.pipe(Pipeline.handle_dates)\n#     print(\"  feats_df shape:\\t\", feats_df.shape)\n    \n    print(\"Filter cols...\")\n    if mode == \"train\":\n        feats_df = feats_df.pipe(Pipeline.filter_cols)\n    else:\n        train_cols = feats_df.columns if len(train_cols) == 0 else train_cols\n        feats_df = feats_df.select([col for col in train_cols if col != \"target\"])\n    print(\"  feats_df shape:\\t\", feats_df.shape)\n    \n    print(\"Convert to pandas...\")\n    feats_df = to_pandas(feats_df, cat_cols)\n    return feats_df","metadata":{"papermill":{"duration":0.041754,"end_time":"2024-02-21T12:13:06.722422","exception":false,"start_time":"2024-02-21T12:13:06.680668","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-20T10:01:32.229422Z","iopub.execute_input":"2024-05-20T10:01:32.229820Z","iopub.status.idle":"2024-05-20T10:01:32.239704Z","shell.execute_reply.started":"2024-05-20T10:01:32.229791Z","shell.execute_reply":"2024-05-20T10:01:32.238303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"credit_bureau_a_1_files = [\n    \"_credit_bureau_a_1_*.parquet\",\n]\ncredit_b_a_1_agg = Aggregator(\n    num_aggregators = [pl.max, pl.min, pl.first, pl.last, pl.mean],\n    str_aggregators = [pl.max, pl.min, pl.first, pl.last], # n_unique\n    group_aggregators = [pl.max, pl.min, pl.first, pl.last]\n)","metadata":{"papermill":{"duration":0.031657,"end_time":"2024-02-21T12:13:06.771666","exception":false,"start_time":"2024-02-21T12:13:06.740009","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-20T10:01:35.938790Z","iopub.execute_input":"2024-05-20T10:01:35.939619Z","iopub.status.idle":"2024-05-20T10:01:35.945124Z","shell.execute_reply.started":"2024-05-20T10:01:35.939581Z","shell.execute_reply":"2024-05-20T10:01:35.943966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == '__main__':\n    credit_bureau_a_1_train_df = prepare_df(\n        credit_bureau_a_1_files, CFG.train_dir, credit_b_a_1_agg, feat_eng=False\n    )\n    gc.collect()\n    cat_cols_credit_bureau_a_1 = list(credit_bureau_a_1_train_df.select_dtypes(\"category\").columns)\n    display(credit_bureau_a_1_train_df)","metadata":{"papermill":{"duration":178.322003,"end_time":"2024-02-21T12:16:05.116002","exception":false,"start_time":"2024-02-21T12:13:06.793999","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-20T10:01:38.795720Z","iopub.execute_input":"2024-05-20T10:01:38.796115Z","iopub.status.idle":"2024-05-20T10:03:45.730011Z","shell.execute_reply.started":"2024-05-20T10:01:38.796083Z","shell.execute_reply":"2024-05-20T10:03:45.728970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"credit_bureau_a_2_files = [\n    \"_credit_bureau_a_2_*.parquet\",\n]\ncredit_b_a_2_agg = Aggregator(\n    num_aggregators = [pl.first],\n    str_aggregators = [pl.first],\n    group_aggregators = [pl.first],\n    str_mode = False\n)\ncredit_b_a_2_agg_2 = Aggregator(\n    num_aggregators = [pl.max],\n    str_aggregators = [pl.max],\n    group_aggregators = [pl.max],\n    str_mode = False\n)","metadata":{"papermill":{"duration":0.034419,"end_time":"2024-02-21T12:16:05.174834","exception":false,"start_time":"2024-02-21T12:16:05.140415","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-20T10:03:58.344961Z","iopub.execute_input":"2024-05-20T10:03:58.345620Z","iopub.status.idle":"2024-05-20T10:03:58.351384Z","shell.execute_reply.started":"2024-05-20T10:03:58.345586Z","shell.execute_reply":"2024-05-20T10:03:58.350192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == '__main__':\n    credit_bureau_a_2_train_df = prepare_df(\n        credit_bureau_a_2_files, CFG.train_dir, credit_b_a_2_agg, agg_chunks=True, feat_eng=False\n    )\n    \n    credit_bureau_a_2_train_df_2 = prepare_df(\n        credit_bureau_a_2_files, CFG.train_dir, credit_b_a_2_agg_2, agg_chunks=True, feat_eng=False\n    )\n    credit_bureau_a_2_train_df_2.set_index(\"case_id\", inplace=True)\n    \n    credit_bureau_a_2_train_df = credit_bureau_a_2_train_df.join(\n        credit_bureau_a_2_train_df_2.drop(columns=[\"WEEK_NUM\", \"target\"]), \n        how=\"left\", on=\"case_id\"\n    )\n    \n    cat_cols_credit_bureau_a_2 = list(credit_bureau_a_2_train_df.select_dtypes(\"category\").columns)\n    del credit_bureau_a_2_train_df_2\n    gc.collect()\n    display(credit_bureau_a_2_train_df)","metadata":{"papermill":{"duration":93.666036,"end_time":"2024-02-21T12:17:38.865264","exception":false,"start_time":"2024-02-21T12:16:05.199228","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-20T10:04:01.201057Z","iopub.execute_input":"2024-05-20T10:04:01.201435Z","iopub.status.idle":"2024-05-20T10:05:50.483943Z","shell.execute_reply.started":"2024-05-20T10:04:01.201405Z","shell.execute_reply":"2024-05-20T10:05:50.482932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == '__main__':\n    display(credit_bureau_a_2_train_df.describe())","metadata":{"papermill":{"duration":2.646133,"end_time":"2024-02-21T12:17:41.541216","exception":false,"start_time":"2024-02-21T12:17:38.895083","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-20T10:06:06.659064Z","iopub.execute_input":"2024-05-20T10:06:06.659440Z","iopub.status.idle":"2024-05-20T10:06:08.798665Z","shell.execute_reply.started":"2024-05-20T10:06:06.659413Z","shell.execute_reply":"2024-05-20T10:06:08.797537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == '__main__':\n    display(credit_bureau_a_2_train_df[credit_bureau_a_2_train_df.case_id == 405])","metadata":{"papermill":{"duration":0.071666,"end_time":"2024-02-21T12:17:41.643755","exception":false,"start_time":"2024-02-21T12:17:41.572089","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-20T10:06:11.790587Z","iopub.execute_input":"2024-05-20T10:06:11.791376Z","iopub.status.idle":"2024-05-20T10:06:11.827052Z","shell.execute_reply.started":"2024-05-20T10:06:11.791341Z","shell.execute_reply":"2024-05-20T10:06:11.825902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_files = [\n    \"_static_cb_0.parquet\",\n    \"_static_0_*.parquet\",\n    \"_applprev_1_*.parquet\",\n    \"_tax_registry_a_1.parquet\",\n    \"_tax_registry_b_1.parquet\",\n    \"_tax_registry_c_1.parquet\",\n    \"_other_1.parquet\",\n    \"_person_1.parquet\",\n    \"_deposit_1.parquet\",\n    \"_debitcard_1.parquet\",\n    \"_credit_bureau_b_1.parquet\",\n    \"_credit_bureau_b_2.parquet\",\n]\nbase_agg = Aggregator(\n    num_aggregators = [pl.max, pl.min, pl.first, pl.last, pl.mean],\n    str_aggregators = [pl.max, pl.min, pl.first, pl.last],\n    group_aggregators = [pl.max, pl.min, pl.first, pl.last],\n    str_mode = True\n)","metadata":{"papermill":{"duration":0.041564,"end_time":"2024-02-21T12:17:41.715201","exception":false,"start_time":"2024-02-21T12:17:41.673637","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-20T10:06:18.274423Z","iopub.execute_input":"2024-05-20T10:06:18.275352Z","iopub.status.idle":"2024-05-20T10:06:18.281478Z","shell.execute_reply.started":"2024-05-20T10:06:18.275316Z","shell.execute_reply":"2024-05-20T10:06:18.280040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == '__main__':\n    train_base_df = prepare_df(base_files, CFG.train_dir, base_agg)\n    cat_cols_base = list(train_base_df.select_dtypes(\"category\").columns)\n    display(train_base_df)","metadata":{"_kg_hide-output":true,"papermill":{"duration":187.099649,"end_time":"2024-02-21T12:20:48.845675","exception":false,"start_time":"2024-02-21T12:17:41.746026","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-20T10:06:23.511483Z","iopub.execute_input":"2024-05-20T10:06:23.511861Z","iopub.status.idle":"2024-05-20T10:08:45.241108Z","shell.execute_reply.started":"2024-05-20T10:06:23.511820Z","shell.execute_reply":"2024-05-20T10:08:45.240072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == '__main__':\n    display(cat_cols_base)","metadata":{"_kg_hide-output":true,"papermill":{"duration":0.063353,"end_time":"2024-02-21T12:20:48.955498","exception":false,"start_time":"2024-02-21T12:20:48.892145","status":"completed"},"scrolled":true,"tags":[],"execution":{"iopub.status.busy":"2024-05-20T10:08:59.641703Z","iopub.execute_input":"2024-05-20T10:08:59.642072Z","iopub.status.idle":"2024-05-20T10:08:59.650747Z","shell.execute_reply.started":"2024-05-20T10:08:59.642045Z","shell.execute_reply":"2024-05-20T10:08:59.649550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == '__main__':\n    test_base_df = prepare_df(\n        base_files, CFG.test_dir, base_agg, mode=\"test\", cat_cols=cat_cols_base, train_cols=train_base_df.columns\n    )\n    display(test_base_df)","metadata":{"_kg_hide-output":true,"papermill":{"duration":2.797915,"end_time":"2024-02-21T12:20:51.800227","exception":false,"start_time":"2024-02-21T12:20:49.002312","status":"completed"},"scrolled":true,"tags":[],"execution":{"iopub.status.busy":"2024-05-20T10:09:04.363438Z","iopub.execute_input":"2024-05-20T10:09:04.364345Z","iopub.status.idle":"2024-05-20T10:09:06.306399Z","shell.execute_reply.started":"2024-05-20T10:09:04.364304Z","shell.execute_reply":"2024-05-20T10:09:06.305382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Save files","metadata":{"papermill":{"duration":0.058164,"end_time":"2024-02-21T12:20:51.917764","exception":false,"start_time":"2024-02-21T12:20:51.8596","status":"completed"},"tags":[]}},{"cell_type":"code","source":"if __name__ == '__main__':\n    train_base_df.to_parquet(\"train_base.parquet\")\n    credit_bureau_a_1_train_df.to_parquet(\"credit_bureau_a_1_train_df.parquet\")\n    credit_bureau_a_2_train_df.to_parquet(\"credit_bureau_a_2_train_df.parquet\")","metadata":{"papermill":{"duration":61.377014,"end_time":"2024-02-21T12:21:53.353688","exception":false,"start_time":"2024-02-21T12:20:51.976674","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### EDA","metadata":{"papermill":{"duration":0.059571,"end_time":"2024-02-21T12:21:53.472392","exception":false,"start_time":"2024-02-21T12:21:53.412821","status":"completed"},"tags":[]}},{"cell_type":"code","source":"if __name__ == '__main__':\n    print(\"Train is duplicated:\\t\", train_base_df[\"case_id\"].duplicated().any())\n    print(\"Train Week Range:\\t\", (train_base_df[\"WEEK_NUM\"].min(), train_base_df[\"WEEK_NUM\"].max()))\n\n    print()\n\n    print(\"Test is duplicated:\\t\", test_base_df[\"case_id\"].duplicated().any())\n    print(\"Test Week Range:\\t\", (test_base_df[\"WEEK_NUM\"].min(), test_base_df[\"WEEK_NUM\"].max()))","metadata":{"papermill":{"duration":0.113167,"end_time":"2024-02-21T12:21:53.646678","exception":false,"start_time":"2024-02-21T12:21:53.533511","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == '__main__':\n    sns.lineplot(\n        data=train_base_df,\n        x=\"WEEK_NUM\",\n        y=\"target\",\n    )\n    plt.show()","metadata":{"papermill":{"duration":19.971297,"end_time":"2024-02-21T12:22:13.680728","exception":false,"start_time":"2024-02-21T12:21:53.709431","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]}]}