{"metadata":{"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"},{"sourceId":7600559,"sourceType":"datasetVersion","datasetId":4424545},{"sourceId":8023315,"sourceType":"datasetVersion","datasetId":4728096}],"dockerImageVersionId":30698,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.13"},"papermill":{"default_parameters":{},"duration":461.002542,"end_time":"2024-03-24T14:35:00.083838","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-03-24T14:27:19.081296","version":"2.5.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Introduction","metadata":{"papermill":{"duration":0.013764,"end_time":"2024-03-24T14:27:22.090612","exception":false,"start_time":"2024-03-24T14:27:22.076848","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# Dependencies","metadata":{"papermill":{"duration":0.012547,"end_time":"2024-03-24T14:27:22.141901","exception":false,"start_time":"2024-03-24T14:27:22.129354","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import os\nimport gc\nfrom glob import glob\nfrom pathlib import Path\nfrom datetime import datetime\nimport re\nimport copy\n\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nimport polars.selectors as cs\n\n# import matplotlib.pyplot as plt\n# import seaborn as sns\nimport joblib\nimport itertools\nimport time\n\nfrom statistics import mode as mode_statistics\nfrom category_encoders import CountEncoder, CatBoostEncoder, TargetEncoder, BaseNEncoder, OneHotEncoder\n\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)\npd.set_option('display.max_columns', None)\npd.set_option('display.max_rows', 500)","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":2.848759,"end_time":"2024-03-24T14:27:25.004944","exception":false,"start_time":"2024-03-24T14:27:22.156185","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-17T17:53:25.219753Z","iopub.execute_input":"2024-05-17T17:53:25.220232Z","iopub.status.idle":"2024-05-17T17:53:27.038328Z","shell.execute_reply.started":"2024-05-17T17:53:25.220174Z","shell.execute_reply":"2024-05-17T17:53:27.037152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n root_dir = Path(\"/kaggle/input/home-credit-credit-risk-model-stability/\")\n train_dir = Path(\"/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train\")\n test_dir = Path(\"/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/test\")\n isnull_threshold = 0.95\n# isnull_threshold = 0.99\n freq_threshold = 200\n freq_threshold_upper = 400\n cat_values_threshold = 0.999 #9\n correlation_threshold = 1\n correlation_threshold_lin = 0\n date_decision_custom = False","metadata":{"papermill":{"duration":0.023594,"end_time":"2024-03-24T14:27:25.067858","exception":false,"start_time":"2024-03-24T14:27:25.044264","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-17T17:53:27.040354Z","iopub.execute_input":"2024-05-17T17:53:27.040949Z","iopub.status.idle":"2024-05-17T17:53:27.047611Z","shell.execute_reply.started":"2024-05-17T17:53:27.040911Z","shell.execute_reply":"2024-05-17T17:53:27.046492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_0 = pl.read_parquet(\"/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_credit_bureau_b_2.parquet\")\n# display(df_0)","metadata":{"execution":{"iopub.status.busy":"2024-05-17T17:53:27.048974Z","iopub.execute_input":"2024-05-17T17:53:27.052539Z","iopub.status.idle":"2024-05-17T17:53:27.060348Z","shell.execute_reply.started":"2024-05-17T17:53:27.052507Z","shell.execute_reply":"2024-05-17T17:53:27.059227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Configuration","metadata":{"papermill":{"duration":0.012687,"end_time":"2024-03-24T14:27:25.030973","exception":false,"start_time":"2024-03-24T14:27:25.018286","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## Feature definitions","metadata":{"papermill":{"duration":0.012742,"end_time":"2024-03-24T14:27:25.094296","exception":false,"start_time":"2024-03-24T14:27:25.081554","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# if __name__ == '__main__':\n# feature_definitions_df = pd.read_csv(CFG.root_dir / \"feature_definitions.csv\")\n# with pd.option_context('display.max_rows', 500, 'display.max_columns', None, 'max_colwidth', 400): \n# display(feature_definitions_df)","metadata":{"_kg_hide-output":true,"papermill":{"duration":0.022597,"end_time":"2024-03-24T14:27:25.129917","exception":false,"start_time":"2024-03-24T14:27:25.10732","status":"completed"},"scrolled":true,"tags":[],"execution":{"iopub.status.busy":"2024-05-17T17:53:27.063189Z","iopub.execute_input":"2024-05-17T17:53:27.063888Z","iopub.status.idle":"2024-05-17T17:53:27.069270Z","shell.execute_reply.started":"2024-05-17T17:53:27.063856Z","shell.execute_reply":"2024-05-17T17:53:27.068099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == '__main__':\n with pd.option_context('display.max_rows', None, 'display.max_columns', None, 'max_colwidth', 400): \n enhanced_feat_def_df = pd.read_parquet(\"/kaggle/input/home-credit-enhanced-feature-definitions/feature_definitions_dtypes_tables.parquet\")\n display(enhanced_feat_def_df)","metadata":{"_kg_hide-output":true,"papermill":{"duration":0.388607,"end_time":"2024-03-24T14:27:25.531627","exception":false,"start_time":"2024-03-24T14:27:25.14302","status":"completed"},"scrolled":true,"tags":[],"execution":{"iopub.status.busy":"2024-05-17T17:53:27.070699Z","iopub.execute_input":"2024-05-17T17:53:27.071037Z","iopub.status.idle":"2024-05-17T17:53:27.424120Z","shell.execute_reply.started":"2024-05-17T17:53:27.070990Z","shell.execute_reply":"2024-05-17T17:53:27.423024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Collection and Preprocessing","metadata":{"papermill":{"duration":0.028884,"end_time":"2024-03-24T14:27:25.590172","exception":false,"start_time":"2024-03-24T14:27:25.561288","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def reduce_mem_usage(df: pd.DataFrame, exclude_cols=[]) -> pd.DataFrame:\n \"\"\" iterate through all the columns of a dataframe and modify the data type\n to reduce memory usage. \n \"\"\"\n start_mem = df.memory_usage().sum() / 1024**2\n print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n\n int_types = [\n np.int8,\n np.int16,\n np.int32,\n np.int64,\n np.uint8,\n np.uint16,\n np.uint32,\n np.uint64,\n ]\n float_types = [np.float32, np.float64]\n\n for col in df.columns:\n col_type = df[col].dtype\n if col_type == \"category\" or col in exclude_cols:\n continue\n \n if col_type in int_types + float_types:\n c_min = df[col].min()\n c_max = df[col].max()\n\n if pd.notna(c_min) and pd.notna(c_max):\n if col_type in int_types:\n if c_min >= 0:\n if (\n c_min >= np.iinfo(np.uint8).min\n and c_max <= np.iinfo(np.uint8).max\n ):\n df[col] = df[col].astype(np.uint8)\n elif (\n c_min >= np.iinfo(np.uint16).min\n and c_max <= np.iinfo(np.uint16).max\n ):\n df[col] = df[col].astype(np.uint16)\n elif (\n c_min >= np.iinfo(np.uint32).min\n and c_max <= np.iinfo(np.uint32).max\n ):\n df[col] = df[col].astype(np.uint32)\n elif (\n c_min >= np.iinfo(np.uint64).min\n and c_max <= np.iinfo(np.uint64).max\n ):\n df[col] = df[col].astype(np.uint64)\n else:\n if (\n c_min >= np.iinfo(np.int8).min\n and c_max <= np.iinfo(np.int8).max\n ):\n df[col] = df[col].astype(np.int8)\n elif (\n c_min >= np.iinfo(np.int16).min\n and c_max <= np.iinfo(np.int16).max\n ):\n df[col] = df[col].astype(np.int16)\n elif (\n c_min >= np.iinfo(np.int32).min\n and c_max <= np.iinfo(np.int32).max\n ):\n df[col] = df[col].astype(np.int32)\n elif (\n c_min >= np.iinfo(np.int64).min\n and c_max <= np.iinfo(np.int64).max\n ):\n df[col] = df[col].astype(np.int64)\n elif col_type in float_types:\n if (\n c_min > np.finfo(np.float32).min\n and c_max < np.finfo(np.float32).max\n ):\n df[col] = df[col].astype(np.float32)\n\n end_mem = df.memory_usage().sum() / 1024**2\n print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n\n return df\n","metadata":{"execution":{"iopub.status.busy":"2024-05-17T17:53:27.425680Z","iopub.execute_input":"2024-05-17T17:53:27.426114Z","iopub.status.idle":"2024-05-17T17:53:27.447561Z","shell.execute_reply.started":"2024-05-17T17:53:27.426064Z","shell.execute_reply":"2024-05-17T17:53:27.446525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_memory_usage_pl(df: pl.DataFrame, name='', exclude_cols=[]) -> pl.DataFrame:\n \"\"\"\n Reduces memory usage of a DataFrame by converting column types.\n\n Args:\n - df (pl.DataFrame): DataFrame to optimize.\n - name (str): Name of the DataFrame.\n\n Returns:\n - pl.DataFrame: Optimized DataFrame.\n \"\"\"\n print(\n f\"Memory usage of dataframe \\\"{name}\\\" is {round(df.estimated_size('mb'), 4)} MB.\"\n )\n\n int_types = [\n pl.Int8,\n pl.Int16,\n pl.Int32,\n pl.Int64,\n pl.UInt8,\n pl.UInt16,\n pl.UInt32,\n pl.UInt64,\n ]\n float_types = [pl.Float32, pl.Float64]\n\n for col in df.columns:\n col_type = df[col].dtype\n if str(col_type)==\"category\" or col in exclude_cols:\n continue\n \n if col_type in int_types + float_types:\n c_min = df[col].min()\n c_max = df[col].max()\n\n if c_min is not None and c_max is not None:\n if col_type in int_types:\n if c_min >= 0:\n if (\n c_min >= np.iinfo(np.uint8).min\n and c_max <= np.iinfo(np.uint8).max\n ):\n df = df.with_columns(df[col].cast(pl.UInt8))\n elif (\n c_min >= np.iinfo(np.uint16).min\n and c_max <= np.iinfo(np.uint16).max\n ):\n df = df.with_columns(df[col].cast(pl.UInt16))\n elif (\n c_min >= np.iinfo(np.uint32).min\n and c_max <= np.iinfo(np.uint32).max\n ):\n df = df.with_columns(df[col].cast(pl.UInt32))\n elif (\n c_min >= np.iinfo(np.uint64).min\n and c_max <= np.iinfo(np.uint64).max\n ):\n df = df.with_columns(df[col].cast(pl.UInt64))\n else:\n if (\n c_min >= np.iinfo(np.int8).min\n and c_max <= np.iinfo(np.int8).max\n ):\n df = df.with_columns(df[col].cast(pl.Int8))\n elif (\n c_min >= np.iinfo(np.int16).min\n and c_max <= np.iinfo(np.int16).max\n ):\n df = df.with_columns(df[col].cast(pl.Int16))\n elif (\n c_min >= np.iinfo(np.int32).min\n and c_max <= np.iinfo(np.int32).max\n ):\n df = df.with_columns(df[col].cast(pl.Int32))\n elif (\n c_min >= np.iinfo(np.int64).min\n and c_max <= np.iinfo(np.int64).max\n ):\n df = df.with_columns(df[col].cast(pl.Int64))\n elif col_type in float_types:\n if (\n c_min > np.finfo(np.float32).min\n and c_max < np.finfo(np.float32).max\n ):\n df = df.with_columns(df[col].cast(pl.Float32))\n\n print(\n f\"Memory usage of dataframe \\\"{name}\\\" became {round(df.estimated_size('mb'), 4)} MB.\"\n )\n\n return df","metadata":{"execution":{"iopub.status.busy":"2024-05-17T17:53:27.448753Z","iopub.execute_input":"2024-05-17T17:53:27.449045Z","iopub.status.idle":"2024-05-17T17:53:27.478409Z","shell.execute_reply.started":"2024-05-17T17:53:27.449022Z","shell.execute_reply":"2024-05-17T17:53:27.477293Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## CorrelationFilter","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n\nclass CorrelationFilter:\n def __init__(self, threshold=0.8):\n self.threshold = threshold\n\n def get_nan_groups(self, df):\n num_cols = df.select_dtypes(exclude='category').columns\n nans_df = df[num_cols].isna()\n nans_groups = {}\n for col in num_cols:\n cur_group = nans_df[col].sum()\n try:\n nans_groups[cur_group].append(col)\n except:\n nans_groups[cur_group] = [col]\n return nans_groups\n\n def reduce_group(self, groups, df):\n best_cols_all = []\n for sub_group in groups:\n max_unique_n = 0\n best_col = sub_group[0]\n for col in sub_group:\n col_unique_n = df[col].nunique()\n if col_unique_n > max_unique_n:\n max_unique_n = col_unique_n\n best_col = col\n best_cols_all.append(best_col)\n return best_cols_all\n\n def group_columns_by_correlation(self, df):\n correlation_matrix = df.corr()\n groups = []\n remaining_cols = list(df.columns)\n while remaining_cols:\n col = remaining_cols.pop(0)\n group = [col]\n correlated_cols = [col]\n for c in remaining_cols:\n if correlation_matrix.loc[col, c] >= self.threshold:\n group.append(c)\n correlated_cols.append(c)\n groups.append(group)\n remaining_cols = [c for c in remaining_cols if c not in correlated_cols]\n return groups\n\n def get_uncorrelated_cols(self, df):\n uncorrelated_cols = []\n nans_groups = self.get_nan_groups(df)\n for k, v in nans_groups.items():\n if len(v) > 1:\n Vs = nans_groups[k]\n group_columns = self.group_columns_by_correlation(df[Vs])\n best_cols = self.reduce_group(group_columns, df)\n uncorrelated_cols += best_cols\n else:\n uncorrelated_cols += v\n return uncorrelated_cols\n\n def filter_correlated_cols(self, df):\n cols_before = df.columns\n \n obj_cols = list(df.select_dtypes(\"object\").columns)\n df[obj_cols] = df[obj_cols].astype(\"category\")\n \n uncorrelated_cols = self.get_uncorrelated_cols(df)\n uncorrelated_cols = list(set(uncorrelated_cols))\n if \"WEEK_NUM\" in cols_before and \"WEEK_NUM\" not in uncorrelated_cols:\n uncorrelated_cols.append(\"WEEK_NUM\")\n uncorrelated_cols = uncorrelated_cols + list(df.select_dtypes(include='category').columns)\n df = df[uncorrelated_cols]\n cols_after = df.columns\n dropped_cols = set(cols_before).difference(cols_after) \n print(\" dropped correlated cols len: \", len(dropped_cols))\n print(\" dropped correlated cols: \", list(dropped_cols))\n return df\n \n def filter_correlated_cols_pl(self, df):\n df_pd = to_pandas(df)\n df_pd = self.filter_correlated_cols(df_pd)\n df = df.select(df_pd.columns)\n return df\n","metadata":{"execution":{"iopub.status.busy":"2024-05-17T17:53:27.480294Z","iopub.execute_input":"2024-05-17T17:53:27.480689Z","iopub.status.idle":"2024-05-17T17:53:27.496813Z","shell.execute_reply.started":"2024-05-17T17:53:27.480659Z","shell.execute_reply":"2024-05-17T17:53:27.495694Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n\nclass LinearCorrelationFilter:\n def __init__(self, threshold=0.0025):\n self.threshold = threshold\n\n def pearson_corr(self, x1, x2):\n mean_x1 = np.mean(x1)\n mean_x2 = np.mean(x2)\n std_x1 = np.std(x1)\n std_x2 = np.std(x2)\n pearson = np.mean((x1 - mean_x1) * (x2 - mean_x2))/(std_x1 * std_x2)\n return pearson\n\n def filter_correlated_cols(self, df):\n cols_before = df.columns\n base_cols = ['target']\n choose_cols = []\n drop_cols = []\n \n obj_cols = list(df.select_dtypes(\"object\").columns)\n df[obj_cols] = df[obj_cols].astype(\"category\")\n num_cols = df.select_dtypes(exclude='category').columns\n cat_cols = list(df.select_dtypes(include='category').columns)\n \n for col in num_cols:\n if col != 'target':\n pearson_score = self.pearson_corr(\n df[col].values,\n df['target'].values\n ) \n if abs(pearson_score) > self.threshold:\n choose_cols.append(col)\n else:\n drop_cols.append(col)\n print(\" dropped cols len: \", len(drop_cols))\n print(\" dropped cols: \", list(drop_cols))\n df = df[base_cols + choose_cols + cat_cols]\n return df\n \n def filter_correlated_cols_pl(self, df):\n df_pd = to_pandas(df)\n df_pd = self.filter_correlated_cols(df_pd)\n df = df.select(df_pd.columns)\n return df\n","metadata":{"execution":{"iopub.status.busy":"2024-05-17T17:53:27.498177Z","iopub.execute_input":"2024-05-17T17:53:27.498983Z","iopub.status.idle":"2024-05-17T17:53:27.511564Z","shell.execute_reply.started":"2024-05-17T17:53:27.498953Z","shell.execute_reply":"2024-05-17T17:53:27.510686Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Pipeline","metadata":{"papermill":{"duration":0.021591,"end_time":"2024-03-24T14:27:25.696444","exception":false,"start_time":"2024-03-24T14:27:25.674853","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def replace_rare_categories(df, col, threshold=200):\n df_counts = df[col].value_counts()\n rare_categories = df_counts.filter(pl.col('count') < threshold)[:, 0].to_list()\n df = df.with_columns(\n pl.when(\n pl.col(col).is_in(rare_categories)\n ).then(pl.lit('rare_categories')).otherwise(\n pl.col(col)\n ).alias(col)\n )\n return df","metadata":{"execution":{"iopub.status.busy":"2024-05-17T17:53:27.515162Z","iopub.execute_input":"2024-05-17T17:53:27.515764Z","iopub.status.idle":"2024-05-17T17:53:27.528860Z","shell.execute_reply.started":"2024-05-17T17:53:27.515724Z","shell.execute_reply":"2024-05-17T17:53:27.527770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Pipeline:\n @staticmethod\n def set_table_dtypes(df):\n cat_cols = [\"persontype_1072L\", \"persontype_792L\"]\n for col in df.columns:\n if col in [\"case_id\", \"WEEK_NUM\", \"num_group1\", \"num_group2\"]:\n df = df.with_columns(pl.col(col).cast(pl.Int64))\n elif col in [\"date_decision\"]:\n df = df.with_columns(pl.col(col).cast(pl.Date))\n elif col[-1] in (\"P\", \"A\"):\n df = df.with_columns(pl.col(col).cast(pl.Float64))\n elif col[-1] in (\"M\",) or (col in cat_cols):\n df = df.with_columns(pl.col(col).cast(pl.String))\n elif col[-1] in (\"D\",):\n df = df.with_columns(pl.col(col).cast(pl.Date))\n\n return df\n \n @staticmethod\n def set_oh_enc_dtypes(df):\n for col in df.columns:\n if \"__\" in col:\n df = df.with_columns(pl.col(col).cast(pl.Float64))\n\n return df\n \n @staticmethod\n def handle_dates(df):\n for col in df.columns:\n if col[-1] in (\"D\",) and (\"__\" not in col):\n df = df.with_columns(pl.col(col) - pl.col(\"date_decision\"))\n df = df.with_columns(pl.col(col).dt.total_days().cast(pl.Int32))\n df = df.drop(\"date_decision\", \"MONTH\")\n return df\n \n @staticmethod\n def filter_cols(\n df, \n isnull_threshold=0.99, \n freq_threshold=200, \n freq_threshold_upper=400,\n cat_values_threshold=0.999, \n verbose=True,\n exclude_cols=[],\n ):\n drop_cols = []\n isnull_cols = []\n freq_threshold_cols = []\n none_over_category_cols = []\n same_val_cols = []\n print(\"isnull_threshold:\", isnull_threshold)\n for col in df.columns:\n # filter null columns\n if col not in [\"target\", \"case_id\", \"WEEK_NUM\"]:\n isnull = df[col].is_null().mean()\n if isnull > isnull_threshold:\n isnull_cols.append(col)\n drop_cols.append(col)\n \n # filter cat cols\n if (col not in [\"target\", \"case_id\", \"WEEK_NUM\"]) & (df[col].dtype == pl.String):\n freq = df[col].n_unique()\n if (freq == 1) | (freq > freq_threshold):\n freq_threshold_cols.append(col)\n drop_cols.append(col)\n else:\n none_over_category = df[col].value_counts().select(\n pl.col(col),\n pl.col('count'),\n (pl.col('count')/pl.col('count').sum()).alias('perc')\n ).filter(pl.col('perc') > cat_values_threshold).is_empty()\n if not none_over_category:\n none_over_category_cols.append(col)\n drop_cols.append(col)\n \n # filter num cols\n if (col not in [\"target\", \"case_id\", \"WEEK_NUM\"]) & (df[col].dtype != pl.String):\n# if df[col].max() == df[col].min() and df[col].is_null().mean() == 0:\n if df[col].max() == df[col].min():\n same_val_cols.append(col)\n drop_cols.append(col)\n \n if verbose:\n print(\"cols to drop:\")\n print(\" len isnull_cols: \", len(isnull_cols))\n print(\" len freq_threshold_cols: \", len(freq_threshold_cols))\n print(\" len none_over_category_cols: \", len(isnull_cols))\n print(\" len same_val_cols: \", len(same_val_cols))\n print(\" len drop_cols: \", len(drop_cols))\n \n for col in exclude_cols:\n if col in drop_cols:\n drop_cols.remove(col)\n \n# print(\" isnull_cols: \", isnull_cols)\n# print(\" freq_threshold_cols: \", freq_threshold_cols)\n# print(\" none_over_category_cols: \", isnull_cols)\n# print(\" same_val_cols: \", same_val_cols)\n\n \n df = df.drop(drop_cols)\n \n if verbose:\n print(\" drop_cols: \", drop_cols)\n return df","metadata":{"papermill":{"duration":0.040422,"end_time":"2024-03-24T14:27:25.75835","exception":false,"start_time":"2024-03-24T14:27:25.717928","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-17T17:53:27.530629Z","iopub.execute_input":"2024-05-17T17:53:27.530957Z","iopub.status.idle":"2024-05-17T17:53:27.550727Z","shell.execute_reply.started":"2024-05-17T17:53:27.530927Z","shell.execute_reply":"2024-05-17T17:53:27.549716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Automatic Aggregation","metadata":{"papermill":{"duration":0.022498,"end_time":"2024-03-24T14:27:25.802625","exception":false,"start_time":"2024-03-24T14:27:25.780127","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def coef_var(col) -> pl.Expr:\n return pl.std(col) / pl.mean(col)","metadata":{"execution":{"iopub.status.busy":"2024-05-17T17:53:27.551731Z","iopub.execute_input":"2024-05-17T17:53:27.552677Z","iopub.status.idle":"2024-05-17T17:53:27.567669Z","shell.execute_reply.started":"2024-05-17T17:53:27.552646Z","shell.execute_reply":"2024-05-17T17:53:27.566270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def mean_to_max(col) -> pl.Expr:\n return pl.mean(col) / pl.max(col)","metadata":{"execution":{"iopub.status.busy":"2024-05-17T17:53:27.569160Z","iopub.execute_input":"2024-05-17T17:53:27.569767Z","iopub.status.idle":"2024-05-17T17:53:27.576290Z","shell.execute_reply.started":"2024-05-17T17:53:27.569736Z","shell.execute_reply":"2024-05-17T17:53:27.575222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def entropy_pl(col) -> pl.Expr:\n return pl.col(col).filter(pl.col(col) != 0).entropy()","metadata":{"execution":{"iopub.status.busy":"2024-05-17T17:53:27.577635Z","iopub.execute_input":"2024-05-17T17:53:27.578150Z","iopub.status.idle":"2024-05-17T17:53:27.584499Z","shell.execute_reply.started":"2024-05-17T17:53:27.578122Z","shell.execute_reply":"2024-05-17T17:53:27.583534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def norm_mean(col) -> pl.Expr:\n return (pl.col(col) / pl.col(col).pow(2).sum().sqrt()).mean()","metadata":{"execution":{"iopub.status.busy":"2024-05-17T17:53:27.585807Z","iopub.execute_input":"2024-05-17T17:53:27.586379Z","iopub.status.idle":"2024-05-17T17:53:27.594465Z","shell.execute_reply.started":"2024-05-17T17:53:27.586349Z","shell.execute_reply":"2024-05-17T17:53:27.593489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def zero_freq(col) -> pl.Expr:\n return pl.col(col).filter(pl.col(col) == 0).count() / pl.col(col).count()","metadata":{"execution":{"iopub.status.busy":"2024-05-17T17:53:27.595680Z","iopub.execute_input":"2024-05-17T17:53:27.596024Z","iopub.status.idle":"2024-05-17T17:53:27.605806Z","shell.execute_reply.started":"2024-05-17T17:53:27.595995Z","shell.execute_reply":"2024-05-17T17:53:27.604654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Aggregator:\n def __init__(\n self, \n num_aggregators=[pl.max, pl.min, pl.first, pl.last, pl.mean], \n str_aggregators=[pl.max, pl.min, pl.first, pl.last], # n_unique\n group_aggregators=[pl.max, pl.min, pl.first, pl.last],\n date_aggregators=[pl.max, pl.min, pl.first, pl.last, pl.mean],\n one_hot_aggregators=[pl.sum, pl.mean],\n feat_agg_additional={},\n str_mode=False\n ):\n self.num_aggregators = num_aggregators\n self.str_aggregators = str_aggregators\n self.group_aggregators = group_aggregators\n self.date_aggregators = date_aggregators\n self.one_hot_aggregators = one_hot_aggregators\n self.feat_agg_additional = feat_agg_additional\n self.str_mode=str_mode\n \n def feat_agg(self, columns):\n expr_all = []\n for col, aggregators in self.feat_agg_additional.items():\n if col not in columns:\n continue\n for method in aggregators:\n expr = [method(col).alias(f\"{method.__name__}_{col}\")]\n expr_all += expr\n return expr_all\n \n def num_expr(self, df_cols):\n cols = [col for col in df_cols if (col[-1] in (\"P\", \"A\"))]\n expr_all = []\n for method in self.num_aggregators:\n expr = [method(col).alias(f\"{method.__name__}_{col}\") for col in cols]\n expr_all += expr\n\n return expr_all\n\n def date_expr(self, df_cols):\n cols = [col for col in df_cols if (col[-1] in (\"D\",))]\n expr_all = []\n for method in self.date_aggregators:\n expr = [method(col).alias(f\"{method.__name__}_{col}\") for col in cols] \n expr_all += expr\n\n return expr_all\n\n def str_expr(self, df_cols):\n cols = [col for col in df_cols if (col[-1] in (\"M\",))]\n \n expr_all = []\n for method in self.str_aggregators:\n expr = [method(col).alias(f\"{method.__name__}_{col}\") for col in cols] \n expr_all += expr\n \n if self.str_mode:\n# print(\"mode calculation...\")\n expr_mode = [\n pl.col(col)\n .drop_nulls()\n# .map_batches(lambda x: mode_statistics(x))\n .mode()\n .sort()\n .last()\n .alias(f\"mode_{col}\")\n for col in cols\n ]\n else:\n expr_mode = []\n\n return expr_all + expr_mode\n\n def other_expr(self, df_cols):\n cols = [col for col in df_cols if (col[-1] in (\"T\", \"L\"))]\n \n expr_all = []\n for method in self.str_aggregators:\n expr = [method(col).alias(f\"{method.__name__}_{col}\") for col in cols]\n expr_all += expr\n\n return expr_all\n \n def count_expr(self, df_cols, suffix=\"\"):\n cols = [col for col in df_cols if \"num_group\" in col]\n\n expr_all = []\n for method in self.group_aggregators:\n expr = [method(col).alias(f\"{method.__name__}_{re.sub('_', '', col)}{suffix}\") for col in cols] \n expr_all += expr\n return expr_all\n\n\n def get_exprs(self, df_cols, suffix=\"\"):\n# print(\"suffix: \", suffix)\n exprs = (\n self.num_expr(df_cols) + \n self.date_expr(df_cols) + \n self.str_expr(df_cols) + \n self.other_expr(df_cols) + \n self.count_expr(df_cols, suffix) + \n# self.one_hot_expr(df_cols) +\n self.feat_agg(df_cols)\n )\n \n return exprs","metadata":{"papermill":{"duration":0.04786,"end_time":"2024-03-24T14:27:25.944758","exception":false,"start_time":"2024-03-24T14:27:25.896898","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-17T17:53:27.607569Z","iopub.execute_input":"2024-05-17T17:53:27.607920Z","iopub.status.idle":"2024-05-17T17:53:27.625854Z","shell.execute_reply.started":"2024-05-17T17:53:27.607884Z","shell.execute_reply":"2024-05-17T17:53:27.624818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_by_chunks(file_pattern, aggregator=None, agg_chunks=False, col_additional_suffix=\"\", \n agg_by_group1=False, \n lazy_mode=False,\n ):\n chunks = []\n for i, path in enumerate(glob(str(file_pattern))):\n print(\" chunk: \", i)\n if not lazy_mode:\n chunk = pl.read_parquet(path).pipe(Pipeline.set_table_dtypes)\n else:\n chunk = pl.scan_parquet(path).pipe(Pipeline.set_table_dtypes)\n \n sort_cols = [\"case_id\"]\n descending_order = [False]\n if \"num_group1\" in chunk.columns:\n sort_cols += [\"num_group1\"]\n if \"num_group2\" in chunk.columns:\n sort_cols += [\"num_group2\"]\n if len(sort_cols) > 1:\n print(\" sorting by num_group...\")\n chunk = chunk.sort(sort_cols)\n if agg_chunks and aggregator != None:\n if not agg_by_group1:\n exprs = aggregator.get_exprs(chunk.columns, col_additional_suffix)\n chunk = chunk.group_by(\"case_id\").agg(exprs)\n else:\n exprs = aggregator.get_exprs(chunk.drop(\"num_group1\").columns, col_additional_suffix + \"_A\")\n chunk = chunk.group_by([\"case_id\", \"num_group1\"]).agg(exprs)\n chunks.append(chunk)\n df = pl.concat(chunks, how=\"vertical_relaxed\")\n return df","metadata":{"papermill":{"duration":0.035442,"end_time":"2024-03-24T14:27:26.053054","exception":false,"start_time":"2024-03-24T14:27:26.017612","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-17T17:53:27.627130Z","iopub.execute_input":"2024-05-17T17:53:27.627631Z","iopub.status.idle":"2024-05-17T17:53:27.641786Z","shell.execute_reply.started":"2024-05-17T17:53:27.627603Z","shell.execute_reply":"2024-05-17T17:53:27.640828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def join_gr1_gr2(b1_path, b2_path, aggregator, col_additional_suffix=\"\", filter_isnull_col_1=None, agg_chunks_1=True):\n print(\" join_gr1_gr2...\")\n print(\"col_additional_suffix:\", col_additional_suffix)\n b1_df = read_by_chunks(b1_path, lazy_mode=False).pipe(Pipeline.set_table_dtypes)\n if filter_isnull_col_1 != None and filter_isnull_col_1 in b1_df.columns:\n b1_df = b1_df.filter(pl.col(filter_isnull_col_1).is_not_null())\n b2_df = read_by_chunks(\n b2_path, \n aggregator, \n lazy_mode=False, \n agg_chunks=agg_chunks_1, \n agg_by_group1=agg_chunks_1,\n col_additional_suffix=col_additional_suffix,\n ).pipe(Pipeline.set_table_dtypes)\n if not agg_chunks_1:\n exprs_b2 = aggregator.get_exprs(b2_df.drop(\"num_group1\").columns, col_additional_suffix + \"_A\")\n b2_df = b2_df.group_by(['case_id', 'num_group1']).agg(exprs_b2)\n b1_df = b1_df.join(b2_df, on=['case_id', 'num_group1'], how=\"left\")\n return b1_df","metadata":{"execution":{"iopub.status.busy":"2024-05-17T17:53:27.643189Z","iopub.execute_input":"2024-05-17T17:53:27.644260Z","iopub.status.idle":"2024-05-17T17:53:27.654246Z","shell.execute_reply.started":"2024-05-17T17:53:27.644223Z","shell.execute_reply":"2024-05-17T17:53:27.653388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def join_similar_files(join_similar_dict, aggregator):\n print(\" join_similar_files...\")\n files_df_list = []\n col_additional_suffix = \"_join_similar\"\n for file in join_similar_dict[\"files\"]:\n file_df = read_by_chunks(file, lazy_mode=True).pipe(Pipeline.set_table_dtypes)\n rename_dict = {}\n drop_cols = []\n for name in join_similar_dict[\"names\"]:\n for col in file_df.columns:\n if name in col:\n rename_dict[col] = f\"{name}_{col[-1]}\"\n if name in join_similar_dict[\"drop_cols\"]:\n drop_cols.append(rename_dict[col])\n file_df = file_df.rename(rename_dict)\n file_df = file_df.drop(drop_cols)\n files_df_list.append(file_df)\n df = pl.concat(files_df_list, how=\"diagonal_relaxed\")\n return df ","metadata":{"execution":{"iopub.status.busy":"2024-05-17T17:53:27.655468Z","iopub.execute_input":"2024-05-17T17:53:27.656297Z","iopub.status.idle":"2024-05-17T17:53:27.669948Z","shell.execute_reply.started":"2024-05-17T17:53:27.656261Z","shell.execute_reply":"2024-05-17T17:53:27.668885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def count_encoding(df):\n cnt_encoding_cols = df.select(pl.selectors.by_dtype([pl.String, pl.Boolean, pl.Categorical])).columns\n mappings = {}\n for col in cnt_encoding_cols:\n mappings[col] = df.group_by(col).len()\n\n df_lazy = df.select(mappings.keys()).lazy()\n\n for col, mapping in mappings.items():\n remapping = {category: count for category, count in mapping.rows()}\n remapping[None] = -2\n expr = pl.col(col).replace(\n remapping,\n default=-1,\n )\n df_lazy = df_lazy.with_columns(expr.alias('cnt_' + col))\n\n df_transformed = df_lazy.collect()\n df = pl.concat([df, df_transformed.select(\"^cnt_.*$\")], how='horizontal')\n print(\" df shape:\\t\", df.shape)\n return df","metadata":{"execution":{"iopub.status.busy":"2024-05-17T17:53:27.671240Z","iopub.execute_input":"2024-05-17T17:53:27.671603Z","iopub.status.idle":"2024-05-17T17:53:27.680108Z","shell.execute_reply.started":"2024-05-17T17:53:27.671574Z","shell.execute_reply":"2024-05-17T17:53:27.679265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_files_by_path(\n pattern_path, \n aggregator, \n depth=0, \n agg_chunks=False, \n num_group1_filter=None, #oh_enc_cols=[], oh_enc=False,\n sort=True, \n join_by_groups_dict={}, \n join_similar_dict={},\n col_additional_suffix=\"\",\n agg_by_group1=False,\n aggregator_L2=None,\n mode=\"train\",\n):\n print('depth:', depth) \n if len(join_by_groups_dict) > 0: # and pattern_path == '':\n df = join_gr1_gr2(\n join_by_groups_dict[\"files\"][0], \n join_by_groups_dict[\"files\"][1], \n aggregator, \n col_additional_suffix=col_additional_suffix,\n filter_isnull_col_1=join_by_groups_dict[\"filter_isnull_col_1\"], \n agg_chunks_1=True\n ).pipe(Pipeline.set_table_dtypes)\n \n if pattern_path != '':\n# agg_by_group1 = True if depth == 2 else False\n agg_chunks = True if (int(depth) > 0 and agg_chunks) else False\n df = read_by_chunks(\n pattern_path, aggregator, agg_chunks, col_additional_suffix, agg_by_group1\n ).pipe(Pipeline.set_table_dtypes)\n \n if len(join_similar_dict) > 0:\n df = join_similar_files(\n join_similar_dict, aggregator\n ).pipe(Pipeline.set_table_dtypes).collect()\n# df = new_features(df)\n\n if depth in [1, 2]:\n print(f\" agg, depth {depth}\")\n if not agg_chunks:\n print(\" aggregation...\")\n # if aggregator_L2 == None:\n # aggregator_L2 = aggregator\n exprs = aggregator.get_exprs(df.columns, col_additional_suffix)\n df = df.group_by(\"case_id\").agg(exprs)\n# if len(join_similar_dict) > 0:\n# df = df.collect()\n return df","metadata":{"papermill":{"duration":0.035842,"end_time":"2024-03-24T14:27:26.218254","exception":false,"start_time":"2024-03-24T14:27:26.182412","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-17T17:53:27.681607Z","iopub.execute_input":"2024-05-17T17:53:27.681901Z","iopub.status.idle":"2024-05-17T17:53:27.694785Z","shell.execute_reply.started":"2024-05-17T17:53:27.681878Z","shell.execute_reply":"2024-05-17T17:53:27.693777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_files(files_arr, data_dir, aggregator, \n mode=\"train\", \n agg_chunks=False, \n num_group1_filter=None, \n join_by_groups=[],\n join_similar=[],\n agg_by_group1=False,\n# aggregator_L2=None\n# oh_enc_cols=[], oh_enc=False\n ):\n base_file_name = f\"{mode}_base.parquet\"\n print(\" files: \", base_file_name)\n feats_df = read_files_by_path(\n data_dir / base_file_name, \n data_dir, \n mode=mode,\n )\n \n for i, file_name in enumerate(files_arr):\n depth = re.findall(\"\\w_(\\d)\", file_name)\n if len(depth) > 0:\n depth = depth[0]\n else:\n continue\n print(\" files: \", file_name, f\"(depth: {depth})\")\n file_name_short = re.sub('\\*?\\..*', '', str(file_name))\n files_df = read_files_by_path(\n data_dir / f\"{mode}{file_name}\", \n aggregator, \n depth=int(depth), \n agg_chunks=agg_chunks, \n num_group1_filter=num_group1_filter,\n col_additional_suffix=file_name_short,\n agg_by_group1=agg_by_group1,\n mode=mode,\n# aggregator_L2=aggregator_L2,\n )\n feats_df = feats_df.join(\n files_df, \n how=\"left\", on=\"case_id\", suffix=f\"_{depth}_{i}\"\n )\n del files_df\n gc.collect()\n \n# if len(join_by_groups) == 2:\n join_by_groups = copy.deepcopy(join_by_groups)\n for i, join_by_groups_dict in enumerate(join_by_groups):\n files_pair = join_by_groups_dict[\"files\"]\n file_name_short = re.sub('\\*?\\..*', '', str(files_pair[1]))\n print(\" files: \", files_pair)\n join_by_groups_dict[\"files\"] = [data_dir / f\"{mode}{file_name}\" for file_name in files_pair]\n files_df = read_files_by_path(\n pattern_path=\"\", \n aggregator=aggregator, \n depth=1,\n join_by_groups_dict=join_by_groups_dict,\n col_additional_suffix=file_name_short,\n mode=mode,\n )\n feats_df = feats_df.join(\n files_df, \n how=\"left\", on=\"case_id\", suffix=f\"_bygroups_{i}\"\n )\n del files_df\n gc.collect()\n \n join_similar = copy.deepcopy(join_similar)\n for i, join_similar_dict in enumerate(join_similar):\n print(\" files: \", join_similar_dict)\n depth = re.findall(\"\\w_(\\d)\", join_similar_dict[\"files\"][0])[0]\n join_similar_dict[\"files\"] = [data_dir / f\"{mode}{file_name}\" for file_name in join_similar_dict[\"files\"]]\n print(\"depth:\", depth)\n# file_name_short = re.sub('\\*?\\..*', '', str(files_pair[1]))\n files_df = read_files_by_path(\n pattern_path=\"\", \n aggregator=aggregator, \n depth=int(depth),\n join_similar_dict=join_similar_dict,\n mode=mode,\n )\n feats_df = feats_df.join(\n files_df, \n how=\"left\", on=\"case_id\", suffix=f\"_bysimilar_{i}\"\n )\n del files_df\n gc.collect()\n\n return feats_df","metadata":{"papermill":{"duration":0.127923,"end_time":"2024-03-24T14:27:26.368251","exception":false,"start_time":"2024-03-24T14:27:26.240328","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-17T17:53:27.696256Z","iopub.execute_input":"2024-05-17T17:53:27.696984Z","iopub.status.idle":"2024-05-17T17:53:27.711944Z","shell.execute_reply.started":"2024-05-17T17:53:27.696955Z","shell.execute_reply":"2024-05-17T17:53:27.710903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def to_pandas(df_data, cat_cols=None):\n df_data = df_data.to_pandas()\n if cat_cols is None:\n cat_cols = list(df_data.select_dtypes(\"object\").columns)\n if len(cat_cols) > 0:\n df_data[cat_cols] = df_data[cat_cols].astype(\"category\")\n return df_data","metadata":{"papermill":{"duration":0.037908,"end_time":"2024-03-24T14:27:26.42888","exception":false,"start_time":"2024-03-24T14:27:26.390972","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-17T17:53:27.713183Z","iopub.execute_input":"2024-05-17T17:53:27.713850Z","iopub.status.idle":"2024-05-17T17:53:27.725784Z","shell.execute_reply.started":"2024-05-17T17:53:27.713823Z","shell.execute_reply":"2024-05-17T17:53:27.724788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def collapse_categories(df, col, verbose=True):\n freq_prev = len(df[col].unique())\n df = df.with_columns(pl.col(col).str.replace(r\"_.*\", \"\").alias(col))\n freq = len(df[col].unique())\n if freq_prev != freq and verbose:\n print(f' col {col} collapsed')\n print(' col len prev:', freq_prev)\n print(' col len:', freq)\n# col = new_col_name\n \n return df","metadata":{"execution":{"iopub.status.busy":"2024-05-17T17:53:27.727018Z","iopub.execute_input":"2024-05-17T17:53:27.727569Z","iopub.status.idle":"2024-05-17T17:53:27.734828Z","shell.execute_reply.started":"2024-05-17T17:53:27.727538Z","shell.execute_reply":"2024-05-17T17:53:27.733892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def collapse_cat_cols(\n df, \n freq_threshold=200, \n freq_threshold_upper=400\n):\n cat_cols = df.select(pl.selectors.by_dtype([pl.String, pl.Categorical])).columns\n for col in cat_cols:\n freq = len(df[col].unique())\n if (freq > freq_threshold) and (freq <= freq_threshold_upper):\n df = replace_rare_categories(df, col)\n# freq_after = len(df[col].unique()) \n return df","metadata":{"execution":{"iopub.status.busy":"2024-05-17T17:53:27.736005Z","iopub.execute_input":"2024-05-17T17:53:27.736543Z","iopub.status.idle":"2024-05-17T17:53:27.744765Z","shell.execute_reply.started":"2024-05-17T17:53:27.736514Z","shell.execute_reply":"2024-05-17T17:53:27.743601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def cat_encoding(\n df, target_col='', cat_cols=[], \n path_to_encoder='encoder.pkl', \n polars_mode=True, force_fit=False, \n enc_type='targ'\n):\n if polars_mode:\n df = to_pandas(df, cat_cols)\n \n if len(cat_cols) == 0:\n cat_cols = list(df.select_dtypes(include=[\"category\", \"object\"]).columns)\n print(\" cat_cols len:\", len(cat_cols))\n bool_cols = list(df.select_dtypes(include=[\"boolean\"]).columns)\n print(\" bool_cols len:\", len(bool_cols))\n df[bool_cols] = df[bool_cols].astype(\"string\").astype(\"category\")\n cat_cols += bool_cols\n \n if os.path.isfile(path_to_encoder) and not force_fit:\n print(' load encoder...')\n encoder = joblib.load(path_to_encoder)\n else:\n print(' fit encoder...')\n enc_dict = {\n 'catb': CatBoostEncoder(cols=cat_cols),\n 'targ': TargetEncoder(cols=cat_cols),\n 'basen': BaseNEncoder(cols=cat_cols, base=4),\n 'ohe': OneHotEncoder(cols=cat_cols)\n }\n encoder = enc_dict[enc_type]\n encoder.fit(df[cat_cols], df[target_col])\n joblib.dump(encoder, path_to_encoder)\n \n print(\" feature_names_in_ len:\", len(encoder.feature_names_in_))\n print(' encoder transform...')\n df_enc = encoder.transform(df[encoder.feature_names_in_]) #[encoder.feature_names_out_]\n df = pd.concat([df, df_enc], axis=1)\n df.rename(columns={col: col+'_catenc' for col in cat_cols}, inplace=True)\n \n if polars_mode:\n# df = pl.from_pandas(df)\n df = pd_to_polars(df)\n return df","metadata":{"execution":{"iopub.status.busy":"2024-05-17T17:53:27.746230Z","iopub.execute_input":"2024-05-17T17:53:27.746746Z","iopub.status.idle":"2024-05-17T17:53:27.758867Z","shell.execute_reply.started":"2024-05-17T17:53:27.746719Z","shell.execute_reply":"2024-05-17T17:53:27.758080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pd_to_polars(df):\n for column in df.select_dtypes(include=['category']).columns:\n df[column] = df[column].astype(str)\n df = pl.from_pandas(df)\n return df","metadata":{"execution":{"iopub.status.busy":"2024-05-17T17:53:27.763714Z","iopub.execute_input":"2024-05-17T17:53:27.764096Z","iopub.status.idle":"2024-05-17T17:53:27.769013Z","shell.execute_reply.started":"2024-05-17T17:53:27.764052Z","shell.execute_reply":"2024-05-17T17:53:27.768276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare df","metadata":{"papermill":{"duration":0.021927,"end_time":"2024-03-24T14:27:26.525241","exception":false,"start_time":"2024-03-24T14:27:26.503314","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def reorder_cols(df):\n if 'WEEK_NUM' in df.columns:\n base_cols = ['case_id', 'WEEK_NUM']\n else:\n base_cols = ['case_id']\n if 'target' in df.columns:\n base_cols.append('target')\n if 'month_decision' in df.columns:\n base_cols.append('month_decision')\n if 'weekday_decision' in df.columns:\n base_cols.append('weekday_decision')\n df = df[ base_cols + [ col for col in df.columns if col not in base_cols ] ]\n return df","metadata":{"execution":{"iopub.status.busy":"2024-05-17T17:53:27.770931Z","iopub.execute_input":"2024-05-17T17:53:27.771379Z","iopub.status.idle":"2024-05-17T17:53:27.785827Z","shell.execute_reply.started":"2024-05-17T17:53:27.771350Z","shell.execute_reply":"2024-05-17T17:53:27.784648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def new_features_1(df):\n rates = ['totaldebt_9A', 'maininc_215A', 'avgpmtlast12m_4525200A', 'credamount_770A', 'annuity_780A']\n rates_comb = list(itertools.combinations(rates, 2))\n \n for pair in rates_comb:\n if 'credamount_770A' in pair and 'annuity_780A' in pair:\n rates_comb.remove(pair)\n for pair in rates_comb:\n if 'max_mainoccupationinc_384A' in pair and 'annuity_780A' in pair:\n rates_comb.remove(pair)\n for pair in rates_comb:\n if 'maininc_215A' in pair and 'max_mainoccupationinc_384A' in pair:\n rates_comb.remove(pair)\n \n for comb_pair in rates_comb:\n df = df.with_columns(\n (pl.col(comb_pair[0]) / pl.col(comb_pair[1])).alias(f\"{comb_pair[0]}_to_{comb_pair[1]}_rate\")\n )\n \n df = df.with_columns((\n (pl.col('maxdpdlast3m_392P') > 0) | (pl.col('maxdpdlast6m_474P') > 0) | (pl.col('maxdpdlast9m_1059P') > 0)\n ).alias('has_dpd'))\n df = df.with_columns((pl.col('applicationscnt_1086L') > 3).alias('has_misinfo'))\n df = df.with_columns(\n ( pl.col('maininc_215A') / (1 + pl.col('max_childnum_21L')) ).alias('inc_per_child')\n )\n return df","metadata":{"execution":{"iopub.status.busy":"2024-05-17T17:53:27.786925Z","iopub.execute_input":"2024-05-17T17:53:27.787780Z","iopub.status.idle":"2024-05-17T17:53:27.798309Z","shell.execute_reply.started":"2024-05-17T17:53:27.787749Z","shell.execute_reply":"2024-05-17T17:53:27.797480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def new_features_2(df):\n if \"riskassesment_302T\" in df.columns:\n if df[\"riskassesment_302T\"].dtype == pl.Null:\n print(\"riskassesment_302T is null\")\n df = df.with_columns(\n [\n pl.Series(\n \"riskassesment_302T_rng\", df[\"riskassesment_302T\"], pl.UInt8\n ),\n pl.Series(\n \"riskassesment_302T_mean\", df[\"riskassesment_302T\"], pl.UInt8\n ),\n ]\n )\n else:\n print(\"riskassesment_302T is not null\")\n pct_low: pl.Series = (\n df[\"riskassesment_302T\"]\n .map_elements(lambda x: int(x.split(\" - \")[0].replace(\"%\", \"\")), return_dtype=pl.UInt8)\n )\n pct_high: pl.Series = (\n df[\"riskassesment_302T\"]\n .map_elements(lambda x: int(x.split(\" - \")[1].replace(\"%\", \"\")), return_dtype=pl.UInt8)\n )\n\n diff: pl.Series = pct_high - pct_low\n avg: pl.Series = ((pct_low + pct_high) / 2).cast(pl.Float32)\n \n del pct_high, pct_low\n gc.collect()\n\n df = df.with_columns(\n [\n diff.alias(\"riskassesment_302T_rng\"),\n avg.alias(\"riskassesment_302T_mean\"),\n ]\n )\n\n return df.drop(\"riskassesment_302T\")","metadata":{"execution":{"iopub.status.busy":"2024-05-17T17:53:27.799924Z","iopub.execute_input":"2024-05-17T17:53:27.800287Z","iopub.status.idle":"2024-05-17T17:53:27.811379Z","shell.execute_reply.started":"2024-05-17T17:53:27.800258Z","shell.execute_reply":"2024-05-17T17:53:27.810072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prepare_df(\n files_arr, \n data_dir, \n aggregator, \n mode=\"train\", \n cat_cols=None, \n train_cols=[], \n agg_chunks=False, \n feat_eng=True, \n num_group1_filter=None,\n join_by_groups=[],\n join_similar=[],\n agg_by_group1=False,\n isnull_threshold=0.99, \n freq_threshold=200, \n freq_threshold_upper=400,\n cat_values_threshold=0.999,\n correlation_threshold=0.8,\n count_encode=False,\n cat_encode=False,\n path_to_encoder=None,\n enc_type='targ',\n new_features=[],\n correlation_threshold_lin=0,\n convert_to_cat_cols=[],\n filter_exclude_cols=[],\n):\n print()\n print(\"Collecting data...\")\n feats_df = read_files(\n files_arr, \n data_dir, \n aggregator, \n mode=mode, \n agg_chunks=agg_chunks, \n join_by_groups=join_by_groups,\n join_similar=join_similar,\n agg_by_group1=agg_by_group1,\n )\n print(\" feats_df shape:\\t\", feats_df.shape)\n \n if feat_eng:\n print(\"Feature Engineering...\")\n feats_df = feats_df.with_columns(\n month_decision = pl.col(\"date_decision\").dt.month().cast(pl.Int16),\n weekday_decision = pl.col(\"date_decision\").dt.weekday().cast(pl.Int16),\n\n day_decision = pl.col(\"date_decision\").dt.day().cast(pl.UInt8),\n \n# decision_year = pl.col(\"date_decision\").dt.year().cast(pl.UInt16),\n# decision_quarter = pl.col(\"date_decision\").dt.quarter().cast(pl.String), #.astype('category')\n# decision_month_of_year = pl.col(\"date_decision\").dt.month().cast(pl.String), #.astype('category')\n# decision_day_of_month = pl.col(\"date_decision\").dt.day().cast(pl.UInt8),\n# decision_day_of_year = pl.col(\"date_decision\").dt.ordinal_day().cast(pl.UInt16),\n# decision_week_of_year = pl.col(\"date_decision\").dt.week().cast(pl.UInt8),\n# decision_day_of_week = (pl.col(\"date_decision\").dt.weekday() + 1).cast(pl.String), #.astype('category')\n )\n# feats_df = feats_df.with_columns(\n# decision_year = pl.col(\"date_decision\").dt.year().cast(pl.UInt16),\n# )\n# year_cols = [col for col in feats_df.columns if ('year' in col) and (col[-1] in ('T',))]\n# print(\"year_cols:\", year_cols)\n# ref_cols = ['decision_year']\n# for y_col in year_cols:\n# for r_col in ref_cols:\n# col_name = f'delta_{y_col}_{r_col}'\n# feats_df = feats_df.with_columns(\n# (pl.col(y_col) - pl.col(r_col)).abs().cast(pl.UInt8).alias(col_name)\n# )\n# feats_df = feats_df.drop(ref_cols + year_cols)\n \n# if 'max_birth_259D' in df.columns:\n# feats_df = feats_df.with_columns(\n# 'birth_year = df_pd.max_birth_259D.dt.year\n# )\n neworder = feats_df.columns\n neworder.remove(\"month_decision\")\n neworder.remove(\"weekday_decision\")\n neworder.remove(\"day_decision\")\n neworder.insert(5, \"month_decision\")\n neworder.insert(6, \"weekday_decision\")\n neworder.insert(7, \"day_decision\")\n feats_df = feats_df.select(neworder)\n \n if CFG.date_decision_custom:\n feats_df = feats_df.pipe(Pipeline.handle_dates_custom)\n else:\n feats_df = feats_df.pipe(Pipeline.handle_dates)\n \n# if len(new_features) > 0:\n# print(\"Create new feats...\")\n# for new_feat_fx in new_features:\n# feats_df = new_feat_fx(feats_df)\n \n if mode == \"train\":\n print('collapse cat cols...')\n feats_df = collapse_cat_cols(\n feats_df, \n freq_threshold=freq_threshold, \n freq_threshold_upper=freq_threshold_upper,\n )\n \n print(\"Filter cols...\")\n feats_df = feats_df.pipe(\n Pipeline.filter_cols, \n isnull_threshold=isnull_threshold, \n freq_threshold=freq_threshold, \n freq_threshold_upper=freq_threshold_upper, \n cat_values_threshold=cat_values_threshold,\n exclude_cols=filter_exclude_cols,\n )\n if len(new_features) > 0:\n print(\"Create new feats...\")\n for new_feat_fx in new_features:\n feats_df = new_feat_fx(feats_df)\n ##\n if count_encode:\n print(f\" count_encoding...\")\n feats_df = count_encoding(feats_df)\n if cat_encode:\n print(f\" cat_encoding...\")\n feats_df = cat_encoding(\n feats_df, \n target_col='target',\n path_to_encoder=path_to_encoder,\n enc_type=enc_type,\n ) \n else:\n if len(new_features) > 0:\n print(\"Create new feats...\")\n for new_feat_fx in new_features:\n feats_df = new_feat_fx(feats_df)\n \n if count_encode:\n print(f\" count_encoding...\")\n feats_df = count_encoding(feats_df)\n if cat_encode: \n print(f\" cat_encoding...\")\n feats_df = cat_encoding(\n feats_df, \n target_col='', \n cat_cols=cat_cols, \n path_to_encoder=path_to_encoder,\n enc_type=enc_type,\n )\n \n if len(train_cols) == 0:\n train_cols = feats_df.columns\n train_cols = [col for col in train_cols if col != \"target\"]\n feats_df = feats_df.select(train_cols)\n else:\n train_cols = [col for col in train_cols if col != \"target\"]\n new_cols = np.array(train_cols)[~np.in1d(train_cols, feats_df.columns)]\n same_cols = np.array(train_cols)[np.in1d(train_cols, feats_df.columns)]\n feats_df = feats_df.select(same_cols)\n if len(new_cols) > 0:\n print(f\" adding {len(new_cols)} new cols:\", new_cols)\n feats_df_new_cols = pl.DataFrame([], schema=list(new_cols)).pipe(Pipeline.set_oh_enc_dtypes) #.fill_nan(0)\n feats_df = pl.concat([feats_df, feats_df_new_cols], how=\"horizontal\")\n\n print(\" feats_df shape:\\t\", feats_df.shape)\n print(\"Convert to pandas...\")\n if cat_encode:\n feats_df = to_pandas(feats_df, [])\n else:\n feats_df = to_pandas(feats_df, cat_cols)\n \n if mode == \"train\" and correlation_threshold < 1:\n print(\"Filter correlated cols...\")\n print(\" cols before: \", feats_df.shape[1])\n feats_df = CorrelationFilter(correlation_threshold).filter_correlated_cols(feats_df)\n print(\" cols after: \", feats_df.shape[1])\n if mode == \"train\" and correlation_threshold_lin > 0:\n print(\"Filter lin correlated cols...\")\n print(\" cols before: \", feats_df.shape[1])\n feats_df = LinearCorrelationFilter(\n correlation_threshold_lin\n ).filter_correlated_cols(feats_df)\n print(\" cols after: \", feats_df.shape[1])\n \n cat_cnt_cols = feats_df[\n feats_df.columns[feats_df.columns.str.contains(\"cnt_\")]\n ].select_dtypes(include=['object', 'category']).columns\n if len(cat_cnt_cols) > 0:\n feats_df[cat_cnt_cols] = feats_df[cat_cnt_cols].astype(np.float64)\n\n feats_df = reorder_cols(feats_df)\n \n df_mem = feats_df.memory_usage().sum() / 1024**2\n print('Memory usage of dataframe is {:.2f} MB'.format(df_mem))\n return feats_df","metadata":{"papermill":{"duration":0.042055,"end_time":"2024-03-24T14:27:26.590174","exception":false,"start_time":"2024-03-24T14:27:26.548119","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-17T17:57:56.914390Z","iopub.execute_input":"2024-05-17T17:57:56.914960Z","iopub.status.idle":"2024-05-17T17:57:56.945812Z","shell.execute_reply.started":"2024-05-17T17:57:56.914923Z","shell.execute_reply":"2024-05-17T17:57:56.944359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Collection","metadata":{"papermill":{"duration":0.029519,"end_time":"2024-03-24T14:30:15.323083","exception":false,"start_time":"2024-03-24T14:30:15.293564","status":"completed"},"tags":[]}},{"cell_type":"code","source":"feat_agg_additional = {\n \"actualdpd_943P\": [pl.sum, pl.mean, coef_var],\n \"annuity_853A\": [pl.sum, pl.mean, coef_var],\n \"credamount_590A\": [pl.sum, pl.mean, coef_var],\n \"downpmt_134A\": [pl.sum, pl.mean, coef_var],\n \"mainoccupationinc_437A\": [pl.sum, pl.mean, coef_var],\n \"pmtnum_8L\": [pl.sum, pl.mean, coef_var],\n \"tenor_203L\": [pl.sum, pl.mean, coef_var],\n \"dpdmax_139P\": [pl.sum, pl.mean, coef_var],\n \"dpdmaxdatemonth_89T\": [pl.sum, pl.mean, coef_var],\n \"monthlyinstlamount_332A\": [pl.sum, pl.mean, coef_var],\n \"numberofoverdueinstlmax_1039L\": [pl.sum, pl.mean, coef_var],\n \"numberofoverdueinstls_725L\": [pl.sum, pl.mean, coef_var],\n \"overdueamount_659A\": [pl.sum, pl.mean, coef_var],\n \"overdueamountmax2_14A\": [pl.sum, pl.mean, coef_var],\n \"overdueamountmax_155A\": [pl.sum, pl.mean, coef_var],\n \"collater_valueofguarantee_1124L\": [pl.sum, pl.mean, coef_var],\n \"amount_1115A\": [pl.sum, pl.mean, coef_var],\n \"credlmt_3940954A\": [pl.sum, pl.mean, coef_var],\n \"debtpastduevalue_732A\": [pl.sum, pl.mean, coef_var],\n \"debtvalue_227A\": [pl.sum, pl.mean, coef_var],\n \"dpd_550P\": [pl.sum, pl.mean, coef_var],\n \"dpd_733P\": [pl.sum, pl.mean, coef_var],\n \"dpdmax_851P\": [pl.sum, pl.mean, coef_var],\n \"installmentamount_644A\": [pl.sum, pl.mean, coef_var],\n \"installmentamount_833A\": [pl.sum, pl.mean, coef_var],\n \"instlamount_892A\": [pl.sum, pl.mean, coef_var],\n \"interesteffectiverate_369L\": [pl.sum, pl.mean, coef_var],\n \"interestrateyearly_538L\": [pl.sum, pl.mean, coef_var],\n \"overdueamountmax_950A\": [pl.sum, pl.mean, coef_var],\n \"pmtnumpending_403L\": [pl.sum, pl.mean, coef_var],\n \"pmts_dpdvalue_108P\": [pl.sum, pl.mean, coef_var],\n \"amount_416A\": [pl.sum, pl.mean, coef_var],\n \"amtdebitincoming_4809443A\": [pl.sum, pl.mean, coef_var],\n \"amtdebitoutgoing_4809440A\": [pl.sum, pl.mean, coef_var],\n \"amtdepositbalance_4809441A\": [pl.sum, pl.mean, coef_var],\n \"amtdepositincoming_4809444A\": [pl.sum, pl.mean, coef_var],\n \"amtdepositoutgoing_4809442A\": [pl.sum, pl.mean, coef_var],\n \"amount_4527230A\": [pl.sum, pl.mean, coef_var],\n \"amount_4917619A\": [pl.sum, pl.mean, coef_var],\n \"pmts_dpd_1073P\": [pl.sum, pl.mean, coef_var],\n \"pmts_overdue_1140A\": [pl.sum, pl.mean, coef_var],\n \"totalamount_6A\": [pl.sum, pl.mean, coef_var],\n \"pmts_dpd_303P\": [pl.sum, pl.mean, coef_var],\n \"dpdmax_757P\": [pl.sum, pl.mean, coef_var],\n \"pmtamount_36A\": [pl.sum, pl.mean, coef_var],\n \"byoccupationinc_3656910L\": [pl.sum, pl.mean, coef_var],\n \"pmtdaysoverdue_1135P\": [pl.sum, pl.mean, coef_var],\n \"totalamount_503A\": [pl.sum, pl.mean, coef_var],\n \"credquantity_1099L\": [pl.sum],\n \"credquantity_984L\": [pl.sum],\n}","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# feat_agg_additional = {\n# \"actualdpd_943P\": [pl.sum, coef_var],\n# \"annuity_853A\": [pl.sum, coef_var],\n# \"credamount_590A\": [pl.sum, coef_var],\n# \"downpmt_134A\": [pl.sum, coef_var],\n# \"mainoccupationinc_437A\": [pl.sum, coef_var],\n# \"pmtnum_8L\": [pl.sum, coef_var],\n# \"tenor_203L\": [pl.sum, coef_var],\n# \"dpdmax_139P\": [pl.sum, coef_var],\n# \"dpdmaxdatemonth_89T\": [pl.sum, coef_var],\n# \"monthlyinstlamount_332A\": [pl.sum, coef_var],\n# \"numberofoverdueinstlmax_1039L\": [pl.sum, coef_var],\n# \"numberofoverdueinstls_725L\": [pl.sum, coef_var],\n# \"overdueamount_659A\": [pl.sum, coef_var],\n# \"overdueamountmax2_14A\": [pl.sum, coef_var],\n# \"overdueamountmax_155A\": [pl.sum, coef_var],\n# \"collater_valueofguarantee_1124L\": [pl.sum, coef_var],\n# \"amount_1115A\": [pl.sum, coef_var],\n# \"credlmt_3940954A\": [pl.sum, coef_var],\n# \"debtpastduevalue_732A\": [pl.sum, coef_var],\n# \"debtvalue_227A\": [pl.sum, coef_var],\n# \"dpd_550P\": [pl.sum, coef_var],\n# \"dpd_733P\": [pl.sum, coef_var],\n# \"dpdmax_851P\": [pl.sum, coef_var],\n# \"installmentamount_644A\": [pl.sum, coef_var],\n# \"installmentamount_833A\": [pl.sum, coef_var],\n# \"instlamount_892A\": [pl.sum, coef_var],\n# \"interesteffectiverate_369L\": [pl.sum, coef_var],\n# \"interestrateyearly_538L\": [pl.sum, coef_var],\n# \"overdueamountmax_950A\": [pl.sum, coef_var],\n# \"pmtnumpending_403L\": [pl.sum, coef_var],\n# \"pmts_dpdvalue_108P\": [pl.sum, coef_var],\n# \"amount_416A\": [pl.sum, coef_var],\n# \"amtdebitincoming_4809443A\": [pl.sum, coef_var],\n# \"amtdebitoutgoing_4809440A\": [pl.sum, coef_var],\n# \"amtdepositbalance_4809441A\": [pl.sum, coef_var],\n# \"amtdepositincoming_4809444A\": [pl.sum, coef_var],\n# \"amtdepositoutgoing_4809442A\": [pl.sum, coef_var],\n# \"amount_4527230A\": [pl.sum, coef_var],\n# \"amount_4917619A\": [pl.sum, coef_var],\n# \"pmts_dpd_1073P\": [pl.sum, coef_var],\n# \"pmts_overdue_1140A\": [pl.sum, coef_var],\n# \"totalamount_6A\": [pl.sum, coef_var],\n# \"pmts_dpd_303P\": [pl.sum, coef_var],\n# \"dpdmax_757P\": [pl.sum, coef_var],\n# \"pmtamount_36A\": [pl.sum, coef_var],\n# \"byoccupationinc_3656910L\": [pl.sum, coef_var],\n# \"pmtdaysoverdue_1135P\": [pl.sum, coef_var],\n# \"totalamount_503A\": [pl.sum, coef_var],\n# \"credquantity_1099L\": [pl.sum],\n# \"credquantity_984L\": [pl.sum],\n# }","metadata":{"execution":{"iopub.status.busy":"2024-05-17T17:53:27.841178Z","iopub.execute_input":"2024-05-17T17:53:27.841823Z","iopub.status.idle":"2024-05-17T17:53:27.858380Z","shell.execute_reply.started":"2024-05-17T17:53:27.841793Z","shell.execute_reply":"2024-05-17T17:53:27.857407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_files = [\n \"_static_cb_0.parquet\",\n \"_static_0_*.parquet\",\n \"_applprev_1_*.parquet\", \n \"_tax_registry_a_1.parquet\",\n \"_tax_registry_b_1.parquet\",\n \"_tax_registry_c_1.parquet\",\n# \"_credit_bureau_a_1_*.parquet\",\n \"_credit_bureau_b_1.parquet\",\n \"_other_1.parquet\",\n \"_person_1.parquet\",\n \"_deposit_1.parquet\",\n \"_debitcard_1.parquet\",\n \"_credit_bureau_b_2.parquet\",\n# \"_credit_bureau_a_2_*.parquet\",\n \"_applprev_2.parquet\",\n \"_person_2.parquet\",\n]","metadata":{"execution":{"iopub.status.busy":"2024-05-17T17:53:27.859684Z","iopub.execute_input":"2024-05-17T17:53:27.860360Z","iopub.status.idle":"2024-05-17T17:53:27.873499Z","shell.execute_reply.started":"2024-05-17T17:53:27.860318Z","shell.execute_reply":"2024-05-17T17:53:27.872273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# base_files_all = [\n# \"_static_cb_0.parquet\",\n# \"_static_0_*.parquet\",\n# \"_applprev_1_*.parquet\", \n# \"_tax_registry_a_1.parquet\",\n# \"_tax_registry_b_1.parquet\",\n# \"_tax_registry_c_1.parquet\",\n# \"_credit_bureau_a_1_*.parquet\",\n# \"_credit_bureau_b_1.parquet\",\n# \"_other_1.parquet\",\n# \"_person_1.parquet\",\n# \"_deposit_1.parquet\",\n# \"_debitcard_1.parquet\",\n# \"_credit_bureau_b_2.parquet\",\n# \"_credit_bureau_a_2_*.parquet\",\n# \"_applprev_2.parquet\",\n# \"_person_2.parquet\",\n# ]","metadata":{"execution":{"iopub.status.busy":"2024-05-17T17:53:27.874916Z","iopub.execute_input":"2024-05-17T17:53:27.875585Z","iopub.status.idle":"2024-05-17T17:53:27.887461Z","shell.execute_reply.started":"2024-05-17T17:53:27.875552Z","shell.execute_reply":"2024-05-17T17:53:27.886391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cred_b_a_files = [\n \"_credit_bureau_a_1_*.parquet\",\n \"_credit_bureau_a_2_*.parquet\",\n]","metadata":{"execution":{"iopub.status.busy":"2024-05-17T17:53:27.889419Z","iopub.execute_input":"2024-05-17T17:53:27.890536Z","iopub.status.idle":"2024-05-17T17:53:27.897537Z","shell.execute_reply.started":"2024-05-17T17:53:27.890494Z","shell.execute_reply":"2024-05-17T17:53:27.896580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"join_by_groups_applprev = [\n {\n \"files\": [\"_applprev_1_*.parquet\", \"_applprev_2.parquet\"],\n \"filter_isnull_col_1\": None\n }\n]\njoin_by_groups_cred_b_b = [\n {\n \"files\": [\"_credit_bureau_b_1.parquet\", \"_credit_bureau_b_2.parquet\"],\n \"filter_isnull_col_1\": None\n }\n]\n\njoin_by_groups_applprev_cred_b_b = [\n {\n \"files\": [\"_applprev_1_*.parquet\", \"_applprev_2.parquet\"],\n \"filter_isnull_col_1\": None\n },\n {\n \"files\": [\"_credit_bureau_b_1.parquet\", \"_credit_bureau_b_2.parquet\"],\n \"filter_isnull_col_1\": None\n }\n]\n\njoin_by_groups_applprev_cred_b_a = [\n {\n \"files\": [\"_applprev_1_*.parquet\", \"_applprev_2.parquet\"],\n \"filter_isnull_col_1\": None\n },\n {\n \"files\": [\"_credit_bureau_a_1_*.parquet\", \"_credit_bureau_a_2_*.parquet\"],\n \"filter_isnull_col_1\": \"lastupdate_1112D\",\n }\n]\n\njoin_by_groups_cred_b_a = [\n {\n \"files\": [\"_credit_bureau_a_1_*.parquet\", \"_credit_bureau_a_2_*.parquet\"],\n \"filter_isnull_col_1\": \"lastupdate_1112D\",\n }\n]\n\nbase_agg = Aggregator(\n num_aggregators = [pl.max, pl.last],\n date_aggregators = [pl.max, pl.last, pl.mean],\n str_aggregators = [pl.max, pl.last],\n group_aggregators = [pl.max, pl.last],\n feat_agg_additional = feat_agg_additional,\n str_mode = False\n)\n\n# filter_exclude_cols = ['riskassesment_302T', 'riskassesment_940T']\nfilter_exclude_cols = ['riskassesment_940T'] \n\n# base_agg = Aggregator(\n# num_aggregators = [pl.min, pl.max, pl.first, pl.last],\n# date_aggregators = [pl.min, pl.max, pl.first, pl.last],\n# str_aggregators = [pl.min, pl.max, pl.first, pl.last],\n# group_aggregators = [pl.min, pl.max, pl.first, pl.last],\n# feat_agg_additional = feat_agg_additional,\n# str_mode = False\n# )","metadata":{"papermill":{"duration":0.043065,"end_time":"2024-03-24T14:30:15.397258","exception":false,"start_time":"2024-03-24T14:30:15.354193","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-17T17:53:27.899234Z","iopub.execute_input":"2024-05-17T17:53:27.900154Z","iopub.status.idle":"2024-05-17T17:53:27.909760Z","shell.execute_reply.started":"2024-05-17T17:53:27.900103Z","shell.execute_reply":"2024-05-17T17:53:27.908856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def norm_mean(col) -> pl.Expr:\n# pl.mean(pl.col(col) / pl.sqrt(pl.sum(pl.pow(col, 2))))","metadata":{"execution":{"iopub.status.busy":"2024-05-17T17:53:27.911348Z","iopub.execute_input":"2024-05-17T17:53:27.912241Z","iopub.status.idle":"2024-05-17T17:53:27.924922Z","shell.execute_reply.started":"2024-05-17T17:53:27.912185Z","shell.execute_reply":"2024-05-17T17:53:27.923559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def norm_mean(col) -> pl.Expr:\n pl.mean(pl.col(col) / pl.Expr.sqrt(pl.sum(pl.Expr.pow(col, 2))))","metadata":{"execution":{"iopub.status.busy":"2024-05-17T17:53:27.926244Z","iopub.execute_input":"2024-05-17T17:53:27.926734Z","iopub.status.idle":"2024-05-17T17:53:27.935952Z","shell.execute_reply.started":"2024-05-17T17:53:27.926701Z","shell.execute_reply":"2024-05-17T17:53:27.934861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# train_base_full_df","metadata":{}},{"cell_type":"code","source":"# if __name__ == '__main__':\n# train_base_full_df = prepare_df(\n# base_files_all,\n# CFG.train_dir,\n# base_agg,\n# agg_chunks=True,\n# cat_encode=True,\n# new_features=[new_features_1],\n \n# isnull_threshold=CFG.isnull_threshold, \n# freq_threshold=CFG.freq_threshold,\n# freq_threshold_upper=CFG.freq_threshold_upper,\n# cat_values_threshold=CFG.cat_values_threshold,\n# correlation_threshold=CFG.correlation_threshold,\n# correlation_threshold_lin=CFG.correlation_threshold_lin,\n# )\n# cat_base_cols = list(train_base_full_df.select_dtypes(\"category\").columns)\n# train_base_cols = list(train_base_full_df.columns)\n# display(train_base_full_df)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-05-17T17:53:27.937067Z","iopub.execute_input":"2024-05-17T17:53:27.937406Z","iopub.status.idle":"2024-05-17T17:53:27.947889Z","shell.execute_reply.started":"2024-05-17T17:53:27.937379Z","shell.execute_reply":"2024-05-17T17:53:27.946283Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if __name__ == '__main__':\n# train_base_full_df.to_parquet(\"train_base_full_df.parquet\")\n# del train_base_full_df\n# gc.collect()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-05-17T17:53:27.949129Z","iopub.execute_input":"2024-05-17T17:53:27.949521Z","iopub.status.idle":"2024-05-17T17:53:27.961097Z","shell.execute_reply.started":"2024-05-17T17:53:27.949492Z","shell.execute_reply":"2024-05-17T17:53:27.960271Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# train_base_df","metadata":{}},{"cell_type":"code","source":"if __name__ == '__main__':\n train_base_df = prepare_df(\n base_files,\n CFG.train_dir,\n base_agg,\n agg_chunks=True,\n new_features=[new_features_1],\n \n isnull_threshold=CFG.isnull_threshold, \n freq_threshold=CFG.freq_threshold,\n freq_threshold_upper=CFG.freq_threshold_upper,\n cat_values_threshold=CFG.cat_values_threshold,\n correlation_threshold=CFG.correlation_threshold,\n filter_exclude_cols=filter_exclude_cols,\n )\n cat_base_cols = list(train_base_df.select_dtypes(\"category\").columns)\n train_base_cols = list(train_base_df.columns)\n display(train_base_df)","metadata":{"execution":{"iopub.status.busy":"2024-05-17T17:58:10.966424Z","iopub.execute_input":"2024-05-17T17:58:10.966855Z","iopub.status.idle":"2024-05-17T17:59:43.040885Z","shell.execute_reply.started":"2024-05-17T17:58:10.966827Z","shell.execute_reply":"2024-05-17T17:59:43.039685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == '__main__':\n train_base_df.to_parquet(\"train_base_df.parquet\")\n del train_base_df\n gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-17T18:00:13.537986Z","iopub.execute_input":"2024-05-17T18:00:13.540570Z","iopub.status.idle":"2024-05-17T18:00:39.163553Z","shell.execute_reply.started":"2024-05-17T18:00:13.540529Z","shell.execute_reply":"2024-05-17T18:00:39.162164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# train_cred_bureau_a_df","metadata":{}},{"cell_type":"code","source":"if __name__ == '__main__':\n train_cred_bureau_a_df = prepare_df(\n cred_b_a_files,\n CFG.train_dir,\n base_agg,\n agg_chunks=True,\n feat_eng=False,\n \n isnull_threshold=CFG.isnull_threshold, \n freq_threshold=CFG.freq_threshold,\n freq_threshold_upper=CFG.freq_threshold_upper,\n cat_values_threshold=CFG.cat_values_threshold,\n correlation_threshold=CFG.correlation_threshold,\n filter_exclude_cols=filter_exclude_cols,\n )\n display(train_cred_bureau_a_df)","metadata":{"execution":{"iopub.status.busy":"2024-05-17T18:00:39.165775Z","iopub.execute_input":"2024-05-17T18:00:39.166136Z","iopub.status.idle":"2024-05-17T18:03:16.911663Z","shell.execute_reply.started":"2024-05-17T18:00:39.166106Z","shell.execute_reply":"2024-05-17T18:03:16.910549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == '__main__':\n train_cred_bureau_a_df.to_parquet(\"train_cred_bureau_a_df.parquet\")\n del train_cred_bureau_a_df\n gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-17T18:03:16.913323Z","iopub.execute_input":"2024-05-17T18:03:16.913666Z","iopub.status.idle":"2024-05-17T18:03:30.689429Z","shell.execute_reply.started":"2024-05-17T18:03:16.913626Z","shell.execute_reply":"2024-05-17T18:03:30.688146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Test df","metadata":{"papermill":{"duration":0.056929,"end_time":"2024-03-24T14:33:54.910169","exception":false,"start_time":"2024-03-24T14:33:54.85324","status":"completed"},"tags":[]}},{"cell_type":"code","source":"if __name__ == '__main__':\n test_base_df = prepare_df(\n base_files, \n# base_files_all,\n CFG.test_dir, \n base_agg, \n mode=\"test\", \n cat_cols=cat_base_cols, \n train_cols=train_base_cols,\n agg_chunks=True,\n new_features=[new_features_1, new_features_2],\n )\n display(test_base_df)\n display(test_base_df.shape)","metadata":{"execution":{"iopub.status.busy":"2024-05-17T18:03:30.691688Z","iopub.execute_input":"2024-05-17T18:03:30.692038Z","iopub.status.idle":"2024-05-17T18:03:32.772671Z","shell.execute_reply.started":"2024-05-17T18:03:30.692009Z","shell.execute_reply":"2024-05-17T18:03:32.771580Z"},"trusted":true},"execution_count":null,"outputs":[]}]}