{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"dockerImageVersionId":30673,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport gc\nfrom glob import glob\nfrom pathlib import Path\nfrom datetime import datetime\n\nimport tqdm\nimport numpy as np\nimport pandas as pd\nimport polars as pl\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# from sklearn.model_selection import KFold, StratifiedGroupKFold, GridSearchCV\nfrom sklearn.metrics import roc_auc_score\n\nimport lightgbm as lgb\n# import xgboost as xgb","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-05-06T20:19:06.796934Z","iopub.execute_input":"2024-05-06T20:19:06.797285Z","iopub.status.idle":"2024-05-06T20:19:11.294531Z","shell.execute_reply.started":"2024-05-06T20:19:06.797258Z","shell.execute_reply":"2024-05-06T20:19:11.293501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CAT_NAN_VALUE = '_NAN_VALUE_'\nCAT_RARE_VALUE = '_RARE_VALUE_'","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:19:11.296949Z","iopub.execute_input":"2024-05-06T20:19:11.297493Z","iopub.status.idle":"2024-05-06T20:19:11.302425Z","shell.execute_reply.started":"2024-05-06T20:19:11.297461Z","shell.execute_reply":"2024-05-06T20:19:11.301197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Pipeline","metadata":{}},{"cell_type":"code","source":"class Pipeline:\n\n @staticmethod\n def set_table_dtypes(df):\n for col in df.columns:\n if col in [\"case_id\", \"WEEK_NUM\", \"num_group1\", \"num_group2\"]:\n df = df.with_columns(pl.col(col).cast(pl.Int32))\n elif col in [\"date_decision\"]:\n df = df.with_columns(pl.col(col).cast(pl.Date))\n elif col[-1] in (\"P\", \"A\"):\n df = df.with_columns(pl.col(col).cast(pl.Float64))\n elif col[-1] in (\"M\",):\n df = df.with_columns(pl.col(col).cast(pl.String))\n elif col[-1] in (\"D\",):\n df = df.with_columns(pl.col(col).cast(pl.Date)) \n\n return df\n \n @staticmethod\n def handle_dates(df):\n for col in df.columns:\n if col[-1] in (\"D\",):\n df = df.with_columns(pl.col(col) - pl.col(\"date_decision\"))\n df = df.with_columns(pl.col(col).dt.total_days())\n df = df.with_columns(pl.col(col).cast(pl.Float32))\n\n df = df.drop(\"date_decision\", \"MONTH\")\n\n return df\n \n @staticmethod\n def filter_cols(df):\n for col in df.columns:\n if col not in [\"target\", \"case_id\", \"WEEK_NUM\"]:\n isnull = df[col].is_null().mean()\n\n if isnull > 0.95: # 0.7\n df = df.drop(col)\n\n for col in df.columns:\n if (col not in [\"target\", \"case_id\", \"WEEK_NUM\"]) and (df[col].dtype == pl.String):\n freq = df[col].n_unique()\n\n if (freq <= 1) or (freq > 255): # (freq <= 1) | (freq > 200)\n df = df.drop(col)\n\n return df","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:19:11.303982Z","iopub.execute_input":"2024-05-06T20:19:11.304459Z","iopub.status.idle":"2024-05-06T20:19:11.321198Z","shell.execute_reply.started":"2024-05-06T20:19:11.304421Z","shell.execute_reply":"2024-05-06T20:19:11.319931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Automatic Aggregation","metadata":{}},{"cell_type":"code","source":"class Aggregator:\n\n @staticmethod\n def _applicant_max_expr(df, depth, cols):\n assert depth == 1\n assert \"num_group1\" in df.columns\n assert \"num_group2\" not in df.columns\n assert \"num_group1\" not in cols\n\n redundant_cols = [\n 'creationdate_885D',\n 'recorddate_4527225D',\n 'deductiondate_4917603D',\n 'debtoutstand_525A',\n 'debtoverdue_47A',\n 'empl_employedfrom_271D',\n 'empl_employedtotal_800L',\n 'empl_industry_691L',\n 'familystate_447L',\n 'housetype_905L',\n 'incometype_1044T',\n 'safeguarantyflag_411L',\n 'sex_738L',\n 'revolvingaccount_394A',\n 'residualamount_488A',\n 'instlamount_768A',\n 'instlamount_852A',\n 'outstandingamount_362A',\n 'totaldebtoverduevalue_178A',\n 'totaloutstanddebtvalue_39A',\n 'dateofcredstart_181D',\n 'numberofoverdueinstlmaxdat_641D',\n 'overdueamountmax2date_1142D',\n 'numberofcontrsvalue_258L',\n 'maxdpdtolerance_577P',\n 'outstandingdebt_522A',\n 'actualdpd_943P',\n 'financialinstitution_591M',\n 'contractst_545M',\n 'contractst_964M',\n ]\n\n return [\n pl.col(\"num_group1\", col)\n .filter(pl.col(\"num_group1\") == 0)\n .exclude(\"num_group1\")\n .max()\n .alias(f\"applicant_max_{col}\")\n for col in cols if col not in redundant_cols\n ]\n\n @staticmethod\n def _above_expr(df, depth, cols, threshold):\n assert depth in [1, 2]\n assert \"num_group1\" in df.columns\n return [\n pl.col(col)\n .filter(pl.col(col) > threshold)\n .len()\n .cast(pl.Float64)\n .truediv(1 + pl.len())\n .alias(f\"above_{threshold}_share_{col}\")\n for col in cols\n ]\n\n @staticmethod\n def _mode_share_expr(df, depth, cols):\n assert depth in [1, 2]\n assert \"num_group1\" in df.columns\n \n additional_param_cols = [\n 'numberofoverdueinstlmax_1151L',\n 'postype_4733339M',\n 'rejectreasonclient_4145042M',\n 'purposeofcred_874M',\n 'purposeofcred_426M',\n 'purposeofcred_722M',\n 'classificationofcontr_13M',\n 'contractst_545M',\n 'financialinstitution_591M',\n 'numberofoutstandinstls_520L',\n 'employername_M',\n 'name_4527232M',\n 'name_4917606M',\n 'employername_160M',\n 'numberofcontrsvalue_358L',\n 'contaddr_district_15M',\n 'registaddr_district_1083M',\n 'empladdr_district_926M',\n 'empladdr_zipcode_114M',\n ]\n\n expr = []\n\n for col in cols:\n if df[col].dtype == pl.String:\n mode1 = pl.col(col).drop_nulls().mode().max()\n # mode2 = pl.col(col).filter(pl.col(col) != mode1).mode().first()\n expr.append(mode1.alias(f\"mode1_{col}\"))\n\n if col in additional_param_cols:\n expr.append(\n (pl.col(col) == mode1).cast(pl.Float64).sum()\n .truediv(1 + pl.len())\n .alias(f\"mode1_additional_param_{col}\")\n )\n else:\n expr.append(\n pl.mean(col).alias(f\"mode1_{col}\")\n )\n\n if col in additional_param_cols:\n expr.append(\n pl.max(col).alias(f\"mode1_additional_param_{col}\")\n )\n\n return expr\n\n @staticmethod\n def num_dpd_expr(df, depth):\n assert depth in [1, 2]\n \n redundant_cols = [\n 'avgdbdtollast24m_4525197P',\n ]\n\n cols = [\n col for col in df.columns\n if (col[-1] == \"P\") and (col not in redundant_cols)\n ]\n\n expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n expr_mean = [pl.mean(col).alias(f\"mean_{col}\") for col in cols]\n\n expr_above = Aggregator._above_expr(\n df,\n depth,\n [col for col in cols if 'pmts_dpd' in col],\n threshold=0.0\n )\n \n if depth == 2:\n return expr_mean + expr_max + expr_above\n\n expr_applicant_max = Aggregator._applicant_max_expr(df, depth, cols)\n return expr_mean + expr_max + expr_above + expr_applicant_max\n\n @staticmethod\n def num_amnt_expr(df, depth):\n assert depth in [1, 2]\n \n redundant_cols = [\n 'avgpmtlast12m_4525200A',\n 'disbursedcredamount_1113A',\n 'maxpmtlast3m_4525190A',\n 'pmtamount_36A',\n 'totaloutstanddebtvalue_668A',\n 'mainoccupationinc_384A',\n 'credacc_credlmt_575A',\n ]\n\n cols = [\n col for col in df.columns\n if (col[-1] == \"A\") and (col not in redundant_cols)\n ]\n\n expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n expr_min = [\n pl.min(col).alias(f\"min_{col}\") for col in cols\n if col not in [\n 'debtoutstand_525A',\n 'debtoverdue_47A',\n 'outstandingamount_362A',\n 'residualamount_488A',\n 'totalamount_996A',\n 'credlmt_935A',\n 'residualamount_856A',\n 'instlamount_768A',\n \n ]\n ]\n\n expr_mean = [pl.mean(col).alias(f\"mean_{col}\") for col in cols]\n expr_std = [\n pl.std(col).alias(f\"std_{col}\")\n for col in cols if 'amount' in col\n ]\n\n if depth == 2:\n return expr_max + expr_min + expr_mean + expr_std\n\n expr_applicant_max = Aggregator._applicant_max_expr(df, depth, cols)\n return expr_max + expr_min + expr_mean + expr_std + expr_applicant_max\n\n @staticmethod\n def date_expr(df, depth):\n assert depth in [1, 2]\n\n redundant_cols = [\n 'responsedate_1012D',\n 'processingdate_168D',\n 'numberofoverdueinstlmaxdat_148D',\n ]\n\n cols = [\n col for col in df.columns\n if (col[-1] == \"D\") and (col not in redundant_cols)\n ]\n\n expr_max = [\n pl.max(col).alias(f\"max_{col}\") for col in cols\n if col not in [\n 'recorddate_4527225D',\n 'approvaldate_319D',\n 'creationdate_885D',\n ]\n ]\n expr_min = [\n pl.min(col).alias(f\"min_{col}\") for col in cols\n if col not in [\n 'empl_employedfrom_271D',\n 'recorddate_4527225D',\n ]\n ]\n\n if depth == 1:\n expr_applicant_max = Aggregator._applicant_max_expr(df, depth, cols)\n return expr_max + expr_min + expr_applicant_max\n\n return expr_max + expr_min\n\n @staticmethod\n def str_expr(df, depth):\n assert depth in [1, 2]\n\n redundant_cols = [\n 'lastrejectcommodtypec_5251769M',\n 'language1_981M',\n 'subjectrole_182M',\n 'subjectrole_93M',\n 'subjectroles_name_838M',\n ]\n\n cols = [\n col for col in df.columns\n if (col[-1] == \"M\") and (col not in redundant_cols)\n ]\n\n expr_mode_share = Aggregator._mode_share_expr(df, depth, cols)\n\n if depth == 2:\n return expr_mode_share\n\n expr_applicant_max = Aggregator._applicant_max_expr(df, depth, cols)\n return expr_mode_share + expr_applicant_max\n\n @staticmethod\n def other_expr(df, depth):\n assert depth in [1, 2]\n \n redundant_cols = [\n 'pmts_month_158T',\n 'pmts_year_1139T',\n 'pmts_month_706T',\n 'dpdmaxdateyear_596T',\n 'overdueamountmaxdatemonth_284T',\n 'overdueamountmaxdatemonth_365T',\n 'dpdmaxdatemonth_89T',\n 'dpdmaxdatemonth_442T',\n\n 'contractssum_5085716L',\n 'applicationscnt_629L',\n 'clientscnt_1130L',\n 'clientscnt_360L',\n 'clientscnt_533L',\n 'clientscnt_887L',\n 'clientscnt_946L',\n 'numinstpaidlastcontr_4325080L',\n 'numnotactivated_1143L',\n 'contaddr_smempladdr_334L',\n 'type_25L',\n\n # only 1 unique value\n 'personindex_1023L',\n 'persontype_1072L',\n 'persontype_792L',\n\n # only 2 unique values\n 'contaddr_matchlist_1032L', # [null, false]\n 'remitter_829L', # [ 0. null]\n\n # duplicates\n 'tenor_203L', # ~ pmtnum_8L\n \n # score decrease\n 'periodicityofpmts_837L',\n 'overdueamountmaxdateyear_2T',\n 'numberofoutstandinstls_59L',\n 'annualeffectiverate_63L',\n 'pmtnum_8L',\n 'status_219L',\n ]\n\n cols = [\n col for col in df.columns\n if (col[-1] in (\"T\", \"L\")) and (col not in redundant_cols)\n ]\n\n expr_mode_share = Aggregator._mode_share_expr(df, depth, cols)\n\n if depth == 2:\n return expr_mode_share\n\n expr_applicant_max = Aggregator._applicant_max_expr(df, depth, cols)\n return expr_mode_share + expr_applicant_max\n \n @staticmethod\n def count_expr(df, depth):\n assert depth in [1, 2]\n\n cols = [\n col for col in df.columns\n if \"num_group\" in col\n ]\n\n if depth == 1:\n assert \"num_group1\" in df.columns\n assert \"num_group2\" not in df.columns\n expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n return expr_max\n\n assert \"num_group1\" in df.columns\n assert \"num_group2\" in df.columns\n\n return [] # expr_n_unique + expr_applicant_share\n\n @staticmethod\n def get_exprs(df, depth):\n assert depth in [1, 2]\n\n exprs = Aggregator.num_dpd_expr(df, depth) + \\\n Aggregator.num_amnt_expr(df, depth) + \\\n Aggregator.date_expr(df, depth) + \\\n Aggregator.str_expr(df, depth) + \\\n Aggregator.other_expr(df, depth) + \\\n Aggregator.count_expr(df, depth)\n\n return exprs","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:19:11.323163Z","iopub.execute_input":"2024-05-06T20:19:11.323621Z","iopub.status.idle":"2024-05-06T20:19:11.371652Z","shell.execute_reply.started":"2024-05-06T20:19:11.323581Z","shell.execute_reply":"2024-05-06T20:19:11.370390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### File I/O","metadata":{}},{"cell_type":"code","source":"def read_file(path, depth=None):\n df = pl.read_parquet(path)\n df = df.pipe(Pipeline.set_table_dtypes)\n\n if depth in [1, 2]:\n df = df.group_by(\"case_id\").agg(Aggregator.get_exprs(df, depth))\n\n return df\n\n\ndef read_files(regex_path, depth=None):\n chunks = []\n for path in glob(str(regex_path)):\n df = pl.read_parquet(path)\n df = df.pipe(Pipeline.set_table_dtypes)\n\n if depth in [1, 2]:\n df = df.group_by(\"case_id\").agg(Aggregator.get_exprs(df, depth))\n \n chunks.append(df)\n\n df = pl.concat(chunks, how=\"vertical_relaxed\")\n df = df.unique(subset=[\"case_id\"])\n \n return df","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:19:11.373451Z","iopub.execute_input":"2024-05-06T20:19:11.373924Z","iopub.status.idle":"2024-05-06T20:19:11.389672Z","shell.execute_reply.started":"2024-05-06T20:19:11.373881Z","shell.execute_reply":"2024-05-06T20:19:11.388371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Feature Engineering","metadata":{}},{"cell_type":"code","source":"def feature_eng(df_base, depth_0, depth_1, depth_2):\n df_base = (\n df_base.with_columns(\n month_decision = pl.col(\"date_decision\").dt.month(),\n weekday_decision = pl.col(\"date_decision\").dt.weekday(),\n )\n )\n\n for i, df in enumerate(depth_0 + depth_1 + depth_2):\n df_base = df_base.join(df, how=\"left\", on=\"case_id\", suffix=f\"_{i}\")\n\n df_base = df_base.pipe(Pipeline.handle_dates)\n\n return df_base","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:19:11.391389Z","iopub.execute_input":"2024-05-06T20:19:11.392670Z","iopub.status.idle":"2024-05-06T20:19:11.405703Z","shell.execute_reply.started":"2024-05-06T20:19:11.392627Z","shell.execute_reply":"2024-05-06T20:19:11.404364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def to_pandas(df_data, cat_cols=None):\n df_data = df_data.to_pandas()\n\n if cat_cols is None:\n cat_cols = list(df_data.select_dtypes(\"object\").columns)\n\n df_data[cat_cols] = df_data[cat_cols].astype(\"category\")\n\n return df_data, cat_cols","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:19:11.409961Z","iopub.execute_input":"2024-05-06T20:19:11.410402Z","iopub.status.idle":"2024-05-06T20:19:11.416813Z","shell.execute_reply.started":"2024-05-06T20:19:11.410368Z","shell.execute_reply":"2024-05-06T20:19:11.415855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Configuration","metadata":{}},{"cell_type":"code","source":"ROOT = Path(\"/kaggle/input/home-credit-credit-risk-model-stability\")\nTRAIN_DIR = ROOT / \"parquet_files\" / \"train\"\nTEST_DIR = ROOT / \"parquet_files\" / \"test\"","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:19:11.418282Z","iopub.execute_input":"2024-05-06T20:19:11.418672Z","iopub.status.idle":"2024-05-06T20:19:11.429489Z","shell.execute_reply.started":"2024-05-06T20:19:11.418642Z","shell.execute_reply":"2024-05-06T20:19:11.428422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train Files Read & Feature Engineering","metadata":{}},{"cell_type":"code","source":"data_store = {\n \"df_base\": read_file(TRAIN_DIR / \"train_base.parquet\"),\n \"depth_0\": [\n read_file(TRAIN_DIR / \"train_static_cb_0.parquet\"),\n read_files(TRAIN_DIR / \"train_static_0_*.parquet\"),\n ],\n \"depth_1\": [\n read_files(TRAIN_DIR / \"train_applprev_1_*.parquet\", 1),\n read_file(TRAIN_DIR / \"train_tax_registry_a_1.parquet\", 1),\n read_file(TRAIN_DIR / \"train_tax_registry_b_1.parquet\", 1),\n read_file(TRAIN_DIR / \"train_tax_registry_c_1.parquet\", 1),\n read_files(TRAIN_DIR / \"train_credit_bureau_a_1_*.parquet\", 1),\n read_file(TRAIN_DIR / \"train_credit_bureau_b_1.parquet\", 1),\n read_file(TRAIN_DIR / \"train_other_1.parquet\", 1),\n read_file(TRAIN_DIR / \"train_person_1.parquet\", 1),\n read_file(TRAIN_DIR / \"train_deposit_1.parquet\", 1),\n read_file(TRAIN_DIR / \"train_debitcard_1.parquet\", 1),\n ],\n \"depth_2\": [\n read_file(TRAIN_DIR / \"train_credit_bureau_b_2.parquet\", 2),\n read_files(TRAIN_DIR / \"train_credit_bureau_a_2_*.parquet\", 2),\n ]\n}","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:19:11.431013Z","iopub.execute_input":"2024-05-06T20:19:11.431470Z","iopub.status.idle":"2024-05-06T20:25:56.335343Z","shell.execute_reply.started":"2024-05-06T20:19:11.431438Z","shell.execute_reply":"2024-05-06T20:25:56.332974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = feature_eng(**data_store)\nprint(\"train data shape:\\t\", df_train.shape)","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:25:56.339195Z","iopub.execute_input":"2024-05-06T20:25:56.339721Z","iopub.status.idle":"2024-05-06T20:26:19.124960Z","shell.execute_reply.started":"2024-05-06T20:25:56.339673Z","shell.execute_reply":"2024-05-06T20:26:19.123809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del data_store\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:26:19.127011Z","iopub.execute_input":"2024-05-06T20:26:19.127884Z","iopub.status.idle":"2024-05-06T20:26:19.879100Z","shell.execute_reply.started":"2024-05-06T20:26:19.127842Z","shell.execute_reply":"2024-05-06T20:26:19.877340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Take a look at one file","metadata":{}},{"cell_type":"code","source":"# df = pl.read_parquet(TRAIN_DIR / \"train_person_1.parquet\")\n# df = df.pipe(Pipeline.set_table_dtypes)","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:26:19.881132Z","iopub.execute_input":"2024-05-06T20:26:19.881469Z","iopub.status.idle":"2024-05-06T20:26:19.888651Z","shell.execute_reply.started":"2024-05-06T20:26:19.881434Z","shell.execute_reply":"2024-05-06T20:26:19.887667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:26:19.890237Z","iopub.execute_input":"2024-05-06T20:26:19.891352Z","iopub.status.idle":"2024-05-06T20:26:19.898798Z","shell.execute_reply.started":"2024-05-06T20:26:19.891318Z","shell.execute_reply":"2024-05-06T20:26:19.897594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df.select(pl.col('remitter_829L'))","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:26:19.902602Z","iopub.execute_input":"2024-05-06T20:26:19.903045Z","iopub.status.idle":"2024-05-06T20:26:19.908431Z","shell.execute_reply.started":"2024-05-06T20:26:19.903002Z","shell.execute_reply":"2024-05-06T20:26:19.907245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df.filter(pl.col('case_id') == 40791)","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:26:19.909656Z","iopub.execute_input":"2024-05-06T20:26:19.909970Z","iopub.status.idle":"2024-05-06T20:26:19.918817Z","shell.execute_reply.started":"2024-05-06T20:26:19.909944Z","shell.execute_reply":"2024-05-06T20:26:19.917892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Feature Elimination","metadata":{}},{"cell_type":"code","source":"df_train = df_train.pipe(Pipeline.filter_cols)\n\nassert len(df_train.shape) == 2\nprint(\"train data shape:\\t\", df_train.shape)","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:26:19.919949Z","iopub.execute_input":"2024-05-06T20:26:19.922409Z","iopub.status.idle":"2024-05-06T20:26:24.991503Z","shell.execute_reply.started":"2024-05-06T20:26:19.922379Z","shell.execute_reply":"2024-05-06T20:26:24.990308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Pandas Conversion","metadata":{}},{"cell_type":"code","source":"df_train, cat_cols = to_pandas(df_train)\ntrain_cols = list(df_train.columns)","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:26:24.993987Z","iopub.execute_input":"2024-05-06T20:26:24.995075Z","iopub.status.idle":"2024-05-06T20:26:53.443880Z","shell.execute_reply.started":"2024-05-06T20:26:24.995033Z","shell.execute_reply":"2024-05-06T20:26:53.442906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Constructing features","metadata":{}},{"cell_type":"code","source":"df_train","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:42:18.846314Z","iopub.execute_input":"2024-05-06T20:42:18.846701Z","iopub.status.idle":"2024-05-06T20:42:19.142746Z","shell.execute_reply.started":"2024-05-06T20:42:18.846674Z","shell.execute_reply":"2024-05-06T20:42:19.141558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Preprocessor:\n\n def __init__(self):\n self._cat_col2valid_values = dict()\n\n def fit(self, X):\n assert len(X.shape) == 2\n assert X.shape[0] >= 2\n\n self._cat_col2valid_values.clear()\n for col in X.columns:\n if X[col].dtype.name == 'category':\n value_counts = X[col].value_counts(normalize=True, sort=True)\n max_unique_values_n = 20\n assert max_unique_values_n >= 2\n valid_values = value_counts.index[:max_unique_values_n]\n valid_values = set(valid_values.tolist())\n valid_values = valid_values.union({CAT_NAN_VALUE, CAT_RARE_VALUE})\n self._cat_col2valid_values[col] = list(valid_values)\n\n return self\n\n def transform(self, X):\n assert len(X.shape) == 2\n assert X.shape[0] >= 1\n\n out_df = X.copy()\n out_df = out_df.set_index('case_id')\n\n target = None\n if 'target' in out_df.columns:\n out_df = out_df.sort_values(by=['WEEK_NUM', 'case_id'])\n target = out_df['target']\n out_df = out_df.drop(columns=['target'])\n\n # ============================================================\n # Preprocess categorical features\n for cat_col, valid_values in self._cat_col2valid_values.items():\n assert out_df[cat_col].dtype.name == 'category'\n out_df[cat_col] = out_df[cat_col].cat.add_categories([\n CAT_NAN_VALUE, CAT_RARE_VALUE\n ])\n out_df[cat_col] = out_df[cat_col].fillna(CAT_NAN_VALUE)\n mask = ~out_df[cat_col].isin(valid_values)\n out_df.loc[mask, cat_col] = CAT_RARE_VALUE\n out_df[cat_col] = out_df[cat_col].cat.set_categories(valid_values)\n assert out_df[cat_col].dtype.name == 'category'\n # ============================================================\n\n # ============================================================\n # drop redundant columns\n out_df.drop(columns=Preprocessor._get_redundant_columns(), inplace=True)\n # ============================================================\n\n # ============================================================\n # aggregate some columns\n agg2group = Preprocessor._get_agg2group()\n agg_df = Preprocessor._get_agg_df(out_df, agg2group)\n\n mode2group = Preprocessor._get_mode2group()\n mode_df = Preprocessor._get_mode_df(out_df, mode2group)\n\n out_df.drop(\n columns=Preprocessor._get_groups_to_drop(agg2group, mode2group),\n inplace=True\n )\n out_df = pd.concat([agg_df, mode_df, out_df], axis=1)\n # ============================================================\n\n # ============================================================\n # construct new features (sometimes, e.g.\n # feature1 - feature2 > 0 => higher chance of default)\n mix_df = pd.concat([\n Preprocessor._get_add_df(out_df, Preprocessor._get_add_pairs()),\n Preprocessor._get_sub_df(out_df, Preprocessor._get_sub_pairs()),\n Preprocessor._get_mul_df(out_df, Preprocessor._get_mul_pairs()),\n Preprocessor._get_div_df(out_df, Preprocessor._get_div_pairs()),\n # Preprocessor._get_compound_df(out_df, agg_df),\n ], axis=1)\n\n out_df = pd.concat([mix_df, out_df], axis=1)\n\n # ============================================================\n # select columns\n out_df = out_df[\n ['WEEK_NUM']\n + Preprocessor._get_primary_features()\n + Preprocessor._get_secondary_features()\n ]\n # ============================================================\n\n # ============================================================\n # handle NaNs - fillna?\n # out_df['missing_values_n'] = out_df.isna().sum(axis=1)\n # ============================================================\n\n return out_df, target\n \n @staticmethod\n def _get_redundant_columns():\n return [\n 'deferredmnthsnum_166L', # only 1 unique value\n 'commnoinclast6m_3546845L', # only 0 and nan\n 'mastercontrelectronic_519L', # only 0 and nan\n 'mastercontrexist_109L', # [ 0. nan]\n 'bankacctype_710L', # [NaN, 'CA']\n 'isdebitcard_729L', # [NaN, False]\n 'paytype1st_925L', # ['OTHER', NaN]\n 'paytype_783L', # ['OTHER', NaN]\n 'typesuite_864L', # [NaN, 'AL'] \n # 'mode1_subjectrole_182M', # [NaN, 'a55475b1']\n # 'mode1_subjectrole_93M', # [NaN, 'a55475b1']\n # 'mode1_subjectroles_name_838M', # [NaN, 'a55475b1']\n\n # duplicated columns\n 'interestrate_311L',\n 'paytype_783L',\n 'mean_debtoutstand_525A',\n \n # other columns contain the same values, but have less nans\n 'assignmentdate_4527235D',\n\n # score decrease\n 'month_decision',\n 'weekday_decision',\n 'firstclxcampaign_1125D',\n\n 'firstquarter_103L',\n 'secondquarter_766L',\n # 'thirdquarter_1082L',\n 'fourthquarter_440L',\n 'max_num_group1_5',\n 'max_num_group1_6',\n 'max_num_group1',\n 'max_num_group1_9',\n 'max_num_group1_10',\n 'max_num_group1_11',\n\n 'pmtcount_4527229L',\n 'pmtcount_693L',\n 'pmtscount_423L',\n 'pmtssum_45A',\n # 'assignmentdate_238D',\n # 'responsedate_1012D',\n 'applicationscnt_464L',\n 'applicationscnt_629L',\n ]\n\n @staticmethod\n def _get_mode2group():\n return {\n 'agg_applicant_education_M': [\n 'education_1103M',\n 'applicant_max_education_1138M',\n 'education_88M',\n 'applicant_max_education_927M',\n ],\n }\n \n @staticmethod\n def _get_mode_df(df, mode2group):\n assert len(df.shape) == 2\n assert type(mode2group) is dict\n assert len(mode2group) == len(Preprocessor._get_mode2group())\n \n mode_data = dict()\n for mode, group in mode2group.items():\n mode_data[mode] = df[group].mode(axis=1).iloc[:, 0].astype('category')\n\n return pd.DataFrame(index=df.index, data=mode_data).astype('category')\n\n @staticmethod\n def _get_agg2group():\n # Return aggregation feature name -> group of features\n agg2group = {\n 'agg_birth_D': [\n 'birthdate_574D', 'dateofbirth_337D',\n 'applicant_max_birth_259D',\n 'max_birth_259D', 'min_birth_259D',\n ],\n 'agg_credit_bureau_queries_L': [\n 'days120_123L', 'days180_256L', 'days30_165L',\n 'days360_512L', 'days90_310L',\n ],\n 'agg_avgdbddpdlast24m_P': [\n 'avgdpdtolclosure24_3658938P',\n 'avgdbddpdlast24m_3658932P',\n # 'avgdbddpdlast3m_4187120P',\n 'avgdbdtollast24m_4525197P',\n ],\n 'agg_phone_number_shares_L': [\n 'clientscnt_100L', 'clientscnt_1022L',\n 'clientscnt_1071L', 'clientscnt_1130L',\n 'clientscnt_157L', 'clientscnt_257L',\n 'clientscnt_304L', 'clientscnt_360L',\n 'clientscnt_493L', 'clientscnt_533L',\n 'clientscnt_887L', 'clientscnt_946L',\n 'clientscnt12m_3712952L',\n 'clientscnt3m_3712950L',\n 'clientscnt6m_3712949L',\n ],\n 'agg_applicant_dtlastpmt_D': [\n 'applicant_max_dtlastpmt_581D',\n 'applicant_max_dtlastpmtallstes_3545839D',\n ],\n 'agg_min_dtlastpmt_D': [\n 'min_dtlastpmt_581D',\n 'min_dtlastpmtallstes_3545839D',\n ],\n 'agg_maxdbddpd_P': [ # diffs are interesting\n 'maxdbddpdlast1m_3658939P',\n 'maxdbddpdtollast12m_3658940P',\n 'maxdbddpdtollast6m_4187119P',\n ],\n 'agg_maxdpdlast_P': [ # diffs are interesting\n 'maxdpdlast12m_727P',\n 'maxdpdlast24m_143P',\n 'maxdpdlast3m_392P',\n 'maxdpdlast6m_474P',\n 'maxdpdlast9m_1059P',\n ],\n 'agg_mindpdlast24m_P': [\n 'mindbddpdlast24m_3658935P',\n 'mindbdtollast24m_4525191P',\n ],\n 'agg_numactivecreds_L': [\n 'numactivecreds_622L',\n 'numactivecredschannel_414L',\n ],\n # 'agg_pmtcount_L': [\n # 'pmtcount_4527229L',\n # 'pmtcount_693L',\n # 'pmtscount_423L',\n # ],\n 'agg_pmtaverage_A': [\n 'pmtaverage_3A',\n 'pmtaverage_4527227A',\n ],\n 'agg_numinstpaid_L': [\n 'numinstlsallpaid_934L',\n 'numinstpaid_4499208L',\n ],\n 'agg_numinstnotpaid_L': [\n 'numinsttopaygr_769L',\n 'numinsttopaygrest_4493213L',\n ],\n 'agg_numinstpaidearlymix_L': [\n 'numinstpaidearly3d_3546850L',\n 'numinstpaidearly3dest_4493216L',\n 'numinstmatpaidtearly2d_4499204L',\n 'numinstlallpaidearly3d_817L',\n 'numinstpaidearly5dobd_4499205L',\n 'numinstpaidearly_338L',\n 'numinstpaidearlyest_4493214L',\n ],\n 'agg_numinstpaidearly5d_L': [\n 'numinstpaidearly5d_1087L',\n 'numinstpaidearly5dest_4493211L',\n ],\n 'agg_numinstregularpaid_L': [\n 'numinstregularpaid_973L',\n 'numinstregularpaidest_4493210L',\n ],\n 'agg_numinstunpaidmax_L': [\n 'numinstunpaidmax_3546851L',\n 'numinstunpaidmaxest_4493212L',\n ],\n 'agg_pctinstlsallpaidlat_L': [\n 'pctinstlsallpaidlat10d_839L',\n 'pctinstlsallpaidlate1d_3546856L',\n 'pctinstlsallpaidlate4d_3546849L',\n 'pctinstlsallpaidlate6d_3546844L',\n ],\n\n # correlation is very close to 1 (0.9999...)\n 'agg_dpd_P': [\n 'actualdpdtolerance_344P',\n 'max_actualdpd_943P',\n ],\n 'agg_currtotaldebt_A': [\n 'currdebt_22A',\n 'totaldebt_9A',\n ],\n 'agg_activateddate_dateactivated_D': [\n 'lastactivateddate_801D',\n 'max_dateactivated_425D',\n 'applicant_max_dateactivated_425D',\n ],\n 'agg_apprdate_D': [\n 'lastapprdate_640D',\n 'applicant_max_approvaldate_319D',\n ],\n 'agg_sumoutstandtotal_A': [\n 'sumoutstandtotal_3546847A',\n 'sumoutstandtotalest_4493215A',\n ],\n 'agg_firstnonzeroinstldate_D': [\n 'max_firstnonzeroinstldate_307D',\n 'applicant_max_firstnonzeroinstldate_307D',\n ],\n 'agg_dpdmax_139_pmts_dpd_1073_P': [\n 'max_dpdmax_139P',\n 'max_pmts_dpd_1073P',\n ],\n 'agg_dpdmax_757_pmts_dpd_303_P': [\n 'max_dpdmax_757P',\n 'max_pmts_dpd_303P',\n ],\n 'agg_totaldebtoverduevalue_A': [\n 'max_overdueamount_31A',\n 'max_totaldebtoverduevalue_718A',\n 'mean_totaldebtoverduevalue_718A',\n 'min_totaldebtoverduevalue_718A',\n 'applicant_max_totaldebtoverduevalue_718A',\n ],\n 'agg_numberofoutstandinstls_L': [\n 'mode1_numberofoutstandinstls_520L',\n 'mode1_additional_param_numberofoutstandinstls_520L',\n ],\n\n # correlation analysis (correlations 0.95+)\n 'agg_dtlastpmtallstes_D': [\n 'dtlastpmtallstes_4499206D',\n 'max_dtlastpmtallstes_3545839D',\n ],\n 'agg_approvaldate_firstnonzeroinstldate_D': [\n 'lastapplicationdate_877D',\n 'applicant_max_approvaldate_319D',\n 'applicant_max_dateactivated_425D',\n 'lastapprdate_640D',\n 'applicant_max_firstnonzeroinstldate_307D',\n ],\n 'agg_min_approvaldate_dateactivated_D': [\n 'min_approvaldate_319D',\n 'min_dateactivated_425D',\n ],\n 'agg_creationdate_firstnonzeroinstldate_D': [\n 'min_creationdate_885D',\n 'min_firstnonzeroinstldate_307D',\n ],\n 'agg_max_amount_A': [\n 'max_amount_4527230A',\n 'max_amount_4917619A',\n 'applicant_max_amount_4527230A',\n 'applicant_max_amount_4917619A',\n ],\n 'agg_min_amount_A': [\n 'min_amount_4527230A',\n 'min_amount_4917619A',\n ],\n 'agg_mean_amount_A': [\n 'mean_amount_4527230A',\n 'mean_amount_4917619A',\n ],\n 'agg_std_amount_A': [\n 'std_amount_4527230A',\n 'std_amount_4917619A',\n ],\n 'agg_mode1_additional_param_employername_M': [\n 'mode1_additional_param_name_4527232M',\n 'mode1_additional_param_name_4917606M',\n 'mode1_additional_param_employername_160M',\n ],\n 'agg_max_num_group1_3_4': [\n 'max_num_group1_3',\n 'max_num_group1_4',\n ],\n 'agg_totaldebtoverdue_A': [\n 'max_debtoverdue_47A',\n 'max_overdueamount_659A',\n 'max_totaldebtoverduevalue_178A',\n 'mean_totaldebtoverduevalue_178A',\n ],\n 'agg_applicant_credlmt_totalamount_A': [\n 'applicant_max_credlmt_230A',\n 'applicant_max_totalamount_6A',\n ],\n 'agg_overdueamountmaxdateyear_T': [\n 'mode1_dpdmaxdateyear_896T',\n 'mode1_overdueamountmaxdateyear_994T',\n ],\n 'agg_numberofcontrsvalue_L': [\n 'mode1_additional_param_numberofcontrsvalue_358L',\n 'applicant_max_numberofcontrsvalue_358L',\n 'mode1_numberofcontrsvalue_358L',\n ],\n 'agg_applicant_overdueamountmaxdateyear_T': [\n 'applicant_max_dpdmaxdateyear_896T',\n 'applicant_max_overdueamountmaxdateyear_994T',\n ],\n 'agg_mode1_additional_param_contaddr_registaddr_district_M': [\n 'mode1_additional_param_contaddr_district_15M',\n 'mode1_additional_param_registaddr_district_1083M',\n ],\n 'agg_mode1_additional_param_empladdr_district_zipcode_M': [\n 'mode1_additional_param_empladdr_district_926M',\n 'mode1_additional_param_empladdr_zipcode_114M',\n ],\n 'agg_max_openingdate_D': [\n 'max_openingdate_313D',\n 'max_openingdate_857D',\n ],\n 'agg_min_openingdate_D': [\n 'min_openingdate_313D',\n 'min_openingdate_857D',\n ],\n 'agg_applicant_openingdate_D': [\n 'applicant_max_openingdate_313D',\n 'applicant_max_openingdate_857D',\n ],\n 'agg_mode1_additional_param_13_545_591_426_M': [\n 'mode1_additional_param_classificationofcontr_13M',\n 'mode1_additional_param_contractst_545M',\n 'mode1_additional_param_financialinstitution_591M',\n 'mode1_additional_param_purposeofcred_426M',\n ],\n }\n\n for prefix in ['max', 'min', 'mean', 'std', 'applicant_max']:\n agg2group['agg_' + prefix + '_overdueamountmax_14155A'] = [\n prefix + '_overdueamountmax2_14A',\n prefix + '_overdueamountmax_155A',\n ]\n agg2group['agg_' + prefix + '_overdueamountmax_35398A'] = [\n prefix + '_overdueamountmax2_398A',\n prefix + '_overdueamountmax_35A',\n ]\n \n return agg2group\n \n @staticmethod\n def _get_agg_df(df, agg2group):\n assert len(df.shape) == 2\n assert type(agg2group) is dict\n assert len(agg2group) == len(Preprocessor._get_agg2group())\n\n agg_data = dict()\n for agg, group in agg2group.items():\n agg_data[agg] = df[group].mean(axis=1)\n\n return pd.DataFrame(index=df.index, data=agg_data)\n\n @staticmethod\n def _get_add_pairs():\n return [\n ['agg_currtotaldebt_A', 'agg_sumoutstandtotal_A', 1.4, 1.0],\n [\n 'mode1_numberofoverdueinstlmax_1039L',\n 'mode1_numberofoverdueinstlmax_1151L',\n 4.0, 1.0\n ],\n # ['mean_dpdmax_757P', 'maxdpdlast24m_143P', 0.35, 1.0],\n # ['mean_pmts_dpd_303P', 'maxdpdlast24m_143P', 0.38, 1.0],\n ['mean_pmts_overdue_1140A', 'mean_pmts_overdue_1152A', 3.0, 1.0],\n ]\n \n @staticmethod\n def _get_add_df(df, add_pairs):\n assert len(df.shape) == 2\n assert type(add_pairs) is list\n assert len(add_pairs) == len(Preprocessor._get_add_pairs())\n \n for add_pair in add_pairs:\n assert len(add_pair) == 4\n\n add_data = dict()\n for (f1, f2, w1, w2) in add_pairs:\n feature_name = '_'.join(['add', f1, f2])\n add_data[feature_name] = w1 * df[f1] + w2 * df[f2]\n \n return pd.DataFrame(index=df.index, data=add_data)\n\n @staticmethod\n def _get_sub_pairs():\n return [\n ['agg_currtotaldebt_A', 'max_outstandingdebt_522A', 1.27, 1.0],\n # ['agg_totaldebtoverdue_A', 'mean_totaldebtoverduevalue_178A', 1.0, 1.0],\n # ['agg_totaldebtoverdue_A', 'max_pmts_overdue_1140A', 2.0, 1.0],\n ['agg_totaldebtoverdue_A', 'agg_max_overdueamountmax_14155A', 2.0, 1.0],\n ['mean_totaldebtoverduevalue_178A', 'mean_pmts_overdue_1140A', 0.65, 1.0],\n ['numincomingpmts_3546848L', 'numinstlswithdpd10_728L', 0.11, 1.0],\n \n # ['min_annuity_853A', 'min_totaldebtoverduevalue_178A', 0.75, 1.0],\n ['numinstlswithdpd5_4187116L', 'numinstlswithoutdpd_562L', 14.1, 1.0],\n # ['datelastinstal40dpd_247D', 'max_dateofcredstart_181D', 0.42, 1.0],\n ['agg_min_overdueamountmax_35398A', 'min_annuity_853A', 1.3, 1.0],\n # ['agg_totaldebtoverdue_A', 'mean_annuity_853A', 1.65, 1.0],\n ['annuity_780A', 'mean_annuity_853A', 0.83, 1.0],\n ['annuity_780A', 'applicant_max_annuity_853A', 0.9, 1.0],\n ['agg_avgdbddpdlast24m_P', 'avgdbddpdlast3m_4187120P', 0.65, 1.0],\n ['maxdbddpdlast1m_3658939P', 'maxdbddpdtollast12m_3658940P', 1.17, 1.0],\n ['mean_overdueamount_659A', 'mean_overdueamount_31A', 0.02, 1.0],\n # ['maxdpdlast12m_727P', 'maxdpdlast24m_143P', 1.6, 1.0],\n # ['maxdpdlast9m_1059P', 'maxdpdlast12m_727P', 1.24, 1.0],\n # ['maxdpdlast6m_474P', 'maxdpdlast9m_1059P', 1.33, 1.0],\n ['maxdpdlast3m_392P', 'maxdpdlast6m_474P', 1.5, 1.0],\n # ['maxdpdlast3m_392P', 'maxdpdlast24m_143P', 4.0, 1.0],\n # ['currdebt_22A', 'currdebt_94A', 1.0, 1.0],\n ['maininc_215A', 'mean_mainoccupationinc_437A', 0.83, 1.0],\n # ['mean_dpdmax_139P', 'mean_dpdmax_757P', 4.5, 1.0],\n ['mean_monthlyinstlamount_332A', 'mean_monthlyinstlamount_674A', 1.2, 1.0],\n ['mean_totalamount_996A', 'mean_totalamount_6A', 0.35, 1.0],\n ]\n\n @staticmethod\n def _get_sub_df(df, sub_pairs):\n assert len(df.shape) == 2\n assert type(sub_pairs) is list\n assert len(sub_pairs) == len(Preprocessor._get_sub_pairs())\n \n for sub_pair in sub_pairs:\n assert len(sub_pair) == 4\n\n sub_data = dict()\n for (f1, f2, w1, w2) in sub_pairs:\n feature_name = '_'.join(['sub', f1, f2])\n sub_data[feature_name] = w1 * df[f1] - w2 * df[f2]\n\n return pd.DataFrame(index=df.index, data=sub_data)\n \n @staticmethod\n def _get_mul_pairs():\n return [\n # ['maxdpdlast12m_727P', 'agg_sumoutstandtotal_A'],\n # ['agg_sumoutstandtotal_A', 'applicant_max_dateofcredend_289D'],\n # ['mean_residualamount_488A', 'mean_pmts_overdue_1152A'],\n ]\n \n @staticmethod\n def _get_mul_df(df, mul_pairs):\n assert len(df.shape) == 2\n assert type(mul_pairs) is list\n assert len(mul_pairs) == len(Preprocessor._get_mul_pairs())\n \n for mul_pair in mul_pairs:\n assert len(mul_pair) == 2\n\n mul_data = dict()\n for (f1, f2) in mul_pairs:\n feature_name = '_'.join(['mul', f1, f2])\n mul_data[feature_name] = df[f1] * df[f2]\n\n return pd.DataFrame(index=df.index, data=mul_data)\n\n @staticmethod\n def _get_div_pairs():\n return [\n ['annuity_780A', 'maininc_215A'],\n ['agg_currtotaldebt_A', 'maininc_215A'],\n ['mean_debtoverdue_47A', 'maininc_215A'],\n ['mean_totaloutstanddebtvalue_39A', 'maininc_215A'],\n ['avglnamtstart24m_4525187A', 'maininc_215A'],\n ['avgpmtlast12m_4525200A', 'maininc_215A'],\n ['agg_sumoutstandtotal_A', 'maininc_215A'],\n ['maxdpdlast12m_727P', 'maininc_215A'],\n ['maxpmtlast3m_4525190A', 'maininc_215A'],\n ['inittransactionamount_650A', 'price_1097A'],\n ['agg_sumoutstandtotal_A', 'price_1097A'],\n ['agg_currtotaldebt_A', 'price_1097A'],\n # ['maxpmtlast3m_4525190A', 'price_1097A'],\n ['mean_credacc_actualbalance_314A', 'agg_numactivecreds_L'],\n ['mean_credacc_actualbalance_314A', 'avgpmtlast12m_4525200A'],\n ['agg_currtotaldebt_A', 'applicant_max_dateofcredend_289D'],\n # ['mean_debtoverdue_47A', 'applicant_max_dateofcredend_289D'],\n # ['agg_sumoutstandtotal_A', 'applicant_max_dateofcredend_289D'],\n ['maininc_215A', 'applicant_max_childnum_21L'],\n ['mean_monthlyinstlamount_332A', 'inittransactionamount_650A'],\n ['mean_monthlyinstlamount_332A', 'maininc_215A'],\n ['mean_totaldebtoverduevalue_178A', 'maininc_215A'],\n ]\n \n @staticmethod\n def _get_div_df(df, div_pairs):\n assert len(df.shape) == 2\n assert type(div_pairs) is list\n assert len(div_pairs) == len(Preprocessor._get_div_pairs())\n \n for div_pair in div_pairs:\n assert len(div_pair) == 2\n\n div_data = dict()\n for (f1, f2) in div_pairs:\n feature_name = '_'.join(['div', f1, f2])\n div_data[feature_name] = df[f1] / (1.0 + df[f2])\n\n return pd.DataFrame(index=df.index, data=div_data)\n \n# @staticmethod\n# def _get_compound_df(df):\n# assert len(df.shape) == 2\n\n# compound_data = {\n# 'compound_coef_1_DA': (\n# df['mean_monthlyinstlamount_332A']\n# / (1.0 + df['maininc_215A'])\n# * df['applicant_max_dateofcredend_289D']\n# ),\n# 'compound_coef_2_DA': (\n# df['maininc_215A']\n# * df['applicant_max_dateofcredend_289D']\n# / (1.0 + df['price_1097A'])\n# ),\n# 'compound_coef_3_DA': (\n# df['maininc_215A']\n# * df['applicant_max_dateofcredend_289D']\n# - df['agg_sumoutstandtotal_A']\n# ),\n# 'compound_coef_4_DA': (\n# df['maininc_215A'] * df['applicant_max_dateofcredend_289D']\n# + df['mean_credacc_actualbalance_314A']\n# - df['agg_sumoutstandtotal_A']\n# ) / (1.0 + df['agg_numactivecreds_L']),\n# }\n\n# return pd.DataFrame(index=df.index, data=compound_data)\n\n @staticmethod\n def _get_groups_to_drop(agg2group, mode2group):\n assert type(agg2group) is dict\n assert type(mode2group) is dict\n assert len(agg2group) == len(Preprocessor._get_agg2group())\n assert len(mode2group) == len(Preprocessor._get_mode2group())\n\n features_to_drop = []\n features_to_keep = [\n 'mean_totaldebtoverduevalue_178A',\n 'maxdbddpdlast1m_3658939P',\n 'maxdbddpdtollast12m_3658940P',\n 'maxdpdlast12m_727P',\n 'maxdpdlast24m_143P',\n 'maxdpdlast9m_1059P',\n 'maxdpdlast6m_474P',\n 'maxdpdlast3m_392P',\n ]\n\n for _, group in agg2group.items():\n for feature in group:\n if feature not in features_to_keep:\n features_to_drop.append(feature)\n\n for _, group in mode2group.items():\n for feature in group:\n if feature not in features_to_keep:\n features_to_drop.append(feature)\n\n return features_to_drop\n\n @staticmethod\n def _get_primary_features():\n return [\n 'annuity_780A',\n 'pmtnum_254L',\n 'eir_270L',\n 'mobilephncnt_593L',\n 'price_1097A',\n\n 'agg_birth_D',\n 'requesttype_4525192L',\n 'amtinstpaidbefduel24m_4187115A',\n 'annuitynextmonth_57A',\n 'agg_avgdbddpdlast24m_P',\n\n 'mean_pmts_dpd_1073P',\n 'mean_pmts_overdue_1140A',\n 'applicant_max_dpdmax_139P',\n 'above_0.0_share_pmts_dpd_1073P',\n 'lastrejectdate_50D',\n\n 'agg_applicant_education_M',\n 'mean_dpdmax_139P',\n 'lastcancelreason_561M',\n\n 'mode1_relationshiptoclient_415T',\n 'min_dateofcredstart_739D',\n 'agg_max_amount_A',\n 'datefirstoffer_1144D',\n 'agg_currtotaldebt_A',\n\n 'mode1_incometype_1044T',\n 'mode1_numberofoverdueinstlmax_1151L',\n 'mode1_additional_param_numberofoverdueinstlmax_1151L',\n 'agg_mean_overdueamountmax_35398A',\n 'above_0.0_share_pmts_dpd_303P',\n \n 'monthsannuity_845L',\n 'lastdelinqdate_224D',\n 'isbidproduct_1095L',\n 'maxdpdtolerance_374P',\n 'mean_residualamount_856A',\n 'std_overdueamount_659A',\n 'agg_maxdbddpd_P',\n 'mode1_numberofoverdueinstlmax_1039L',\n 'agg_numberofcontrsvalue_L',\n 'mean_maxdpdtolerance_577P',\n 'mean_totalamount_6A',\n 'agg_numinstunpaidmax_L',\n 'applicationscnt_867L',\n\n 'sub_numincomingpmts_3546848L_numinstlswithdpd10_728L',\n 'sub_numinstlswithdpd5_4187116L_numinstlswithoutdpd_562L',\n 'div_inittransactionamount_650A_price_1097A',\n 'min_credacc_actualbalance_314A',\n 'add_mode1_numberofoverdueinstlmax_1039L_mode1_numberofoverdueinstlmax_1151L',\n 'sub_agg_currtotaldebt_A_max_outstandingdebt_522A',\n 'sub_maxdbddpdlast1m_3658939P_maxdbddpdtollast12m_3658940P',\n 'agg_totaldebtoverduevalue_A',\n 'min_mainoccupationinc_437A',\n ]\n \n @staticmethod\n def _get_secondary_features():\n return [\n 'agg_credit_bureau_queries_L',\n 'maritalst_385M',\n 'responsedate_4527233D',\n 'max_employedfrom_700D',\n 'mode1_education_1138M',\n 'mode1_additional_param_postype_4733339M',\n 'mode1_additional_param_rejectreasonclient_4145042M',\n 'mode1_isdebitcard_527L',\n 'agg_max_num_group1_3_4',\n 'max_instlamount_852A',\n 'max_monthlyinstlamount_332A',\n 'min_totaloutstanddebtvalue_39A',\n 'mean_totalamount_996A',\n 'agg_std_overdueamountmax_35398A',\n 'applicant_max_purposeofcred_426M',\n 'mode1_collater_valueofguarantee_1124L',\n\n 'agg_mean_amount_A',\n 'agg_overdueamountmaxdateyear_T',\n 'agg_applicant_overdueamountmaxdateyear_T',\n 'agg_mode1_additional_param_13_545_591_426_M',\n 'cntincpaycont9m_3716944L',\n 'lastrejectreason_759M',\n 'posfpd30lastmonth_3976960P',\n 'max_credamount_590A',\n 'mean_credamount_590A',\n 'max_overdueamountmax2date_1142D',\n 'applicant_max_dateofcredend_289D',\n 'applicant_max_overdueamountmax2date_1002D',\n 'mode1_additional_param_purposeofcred_874M',\n 'mode1_sex_738L',\n 'max_amount_416A',\n 'mean_amount_416A',\n 'max_pmts_overdue_1152A',\n \n 'equalitydataagreement_891L',\n 'lastapprcredamount_781A',\n 'lastrejectcredamount_222A',\n 'maxdpdfrom6mto36m_3546853P',\n 'mode1_financialinstitution_382M', # check\n\n # 'sub_agg_totaldebtoverdue_A_max_overdueamount_659A',\n 'sub_agg_totaldebtoverdue_A_agg_max_overdueamountmax_14155A',\n 'sub_mean_totalamount_996A_mean_totalamount_6A',\n 'div_mean_totaloutstanddebtvalue_39A_maininc_215A',\n 'div_maxdpdlast12m_727P_maininc_215A',\n 'div_agg_currtotaldebt_A_price_1097A',\n 'div_mean_monthlyinstlamount_332A_maininc_215A',\n 'agg_numinstpaid_L',\n 'agg_dtlastpmtallstes_D',\n 'agg_creationdate_firstnonzeroinstldate_D',\n 'agg_max_overdueamountmax_35398A',\n 'thirdquarter_1082L',\n 'avgdbddpdlast3m_4187120P',\n 'credamount_770A',\n 'credtype_322L',\n 'daysoverduetolerancedd_3976961L',\n 'inittransactioncode_186L',\n 'lastrejectcommoditycat_161M',\n 'maxannuity_159A',\n 'numinstls_657L',\n 'numinstpaidlate1d_3546852L',\n 'sellerplacescnt_216L',\n 'max_credacc_maxhisbal_375A',\n 'max_credacc_minhisbal_90A',\n 'min_credacc_maxhisbal_375A',\n 'min_credacc_minhisbal_90A',\n 'mean_credacc_minhisbal_90A',\n 'mean_currdebt_94A',\n 'mean_mainoccupationinc_437A',\n 'mean_revolvingaccount_394A',\n 'applicant_max_credamount_590A',\n 'applicant_max_currdebt_94A',\n 'applicant_max_downpmt_134A',\n 'mode1_credacc_status_367L',\n 'mode1_credtype_587L',\n 'mode1_familystate_726L',\n 'max_credlmt_935A',\n 'max_outstandingamount_362A',\n 'min_totalamount_6A',\n 'mean_overdueamount_659A',\n 'std_instlamount_768A',\n 'std_monthlyinstlamount_674A',\n 'applicant_max_totalamount_996A',\n 'max_dateofcredend_289D',\n 'max_dateofcredstart_739D',\n 'min_numberofoverdueinstlmaxdat_641D',\n 'mode1_nominalrate_281L',\n 'mode1_nominalrate_498L',\n 'mode1_numberofinstls_229L',\n 'mode1_periodicityofpmts_1102L',\n 'applicant_max_numberofoverdueinstlmax_1039L',\n 'mode1_collater_valueofguarantee_876L',\n \n 'mean_outstandingdebt_522A',\n 'agg_mean_overdueamountmax_14155A',\n 'agg_std_amount_A',\n 'agg_firstnonzeroinstldate_D',\n 'applicant_max_familystate_726L',\n 'applicant_max_dateofcredstart_739D',\n 'min_overdueamountmax2date_1002D',\n 'maxdbddpdtollast12m_3658940P',\n 'maxdbddpdlast1m_3658939P',\n 'agg_maxdpdlast_P',\n 'div_annuity_780A_maininc_215A',\n 'div_agg_sumoutstandtotal_A_price_1097A',\n 'sub_mean_totaldebtoverduevalue_178A_mean_pmts_overdue_1140A',\n\n 'add_agg_currtotaldebt_A_agg_sumoutstandtotal_A',\n 'add_mean_pmts_overdue_1140A_mean_pmts_overdue_1152A',\n 'sub_annuity_780A_applicant_max_annuity_853A',\n 'sub_agg_avgdbddpdlast24m_P_avgdbddpdlast3m_4187120P',\n 'sub_mean_monthlyinstlamount_332A_mean_monthlyinstlamount_674A',\n 'div_avglnamtstart24m_4525187A_maininc_215A',\n 'div_maxpmtlast3m_4525190A_maininc_215A',\n 'agg_min_dtlastpmt_D',\n 'agg_mindpdlast24m_P',\n 'agg_numinstregularpaid_L',\n 'agg_pctinstlsallpaidlat_L',\n 'agg_activateddate_dateactivated_D',\n 'agg_sumoutstandtotal_A',\n 'agg_dpdmax_139_pmts_dpd_1073_P',\n 'agg_dpdmax_757_pmts_dpd_303_P',\n 'agg_approvaldate_firstnonzeroinstldate_D',\n 'agg_min_amount_A',\n 'agg_applicant_credlmt_totalamount_A',\n 'agg_mode1_additional_param_empladdr_district_zipcode_M',\n 'agg_min_openingdate_D',\n 'agg_max_overdueamountmax_14155A',\n 'agg_min_overdueamountmax_14155A',\n 'agg_min_overdueamountmax_35398A',\n 'agg_applicant_max_overdueamountmax_14155A',\n 'avglnamtstart24m_4525187A',\n 'avgmaxdpdlast9m_3716943P',\n 'avgpmtlast12m_4525200A',\n 'cntpmts24_3658933L',\n 'datelastinstal40dpd_247D',\n 'disbursedcredamount_1113A',\n 'homephncnt_628L',\n 'inittransactionamount_650A',\n 'lastapprcommoditycat_1041M',\n 'lastrejectreasonclient_4145040M',\n 'lastst_736L',\n 'maxdebt4_972A',\n 'maxdpdlast24m_143P',\n 'maxdpdlast9m_1059P',\n 'maxinstallast24m_3658928A',\n 'maxlnamtstart6m_4525199A',\n 'numcontrs3months_479L',\n 'numincomingpmts_3546848L',\n 'numnotactivated_1143L',\n 'numrejects9m_859L',\n 'pctinstlsallpaidearl3d_427L',\n 'totalsettled_863A',\n 'totinstallast1m_4525188A',\n 'max_maxdpdtolerance_577P',\n 'max_annuity_853A',\n 'max_mainoccupationinc_437A',\n 'max_outstandingdebt_522A',\n 'mean_annuity_853A',\n 'std_credamount_590A',\n 'applicant_max_annuity_853A',\n 'min_employedfrom_700D',\n 'applicant_max_employedfrom_700D',\n 'mode1_childnum_21L',\n 'applicant_max_credtype_587L',\n 'applicant_max_inittransactioncode_279L',\n 'mean_dpdmax_757P',\n 'applicant_max_dpdmax_757P',\n 'max_credlmt_230A',\n 'max_instlamount_768A',\n 'max_monthlyinstlamount_674A',\n 'max_residualamount_488A',\n 'max_residualamount_856A',\n 'max_totalamount_6A',\n 'max_totalamount_996A',\n 'min_credlmt_230A',\n 'min_instlamount_852A',\n 'min_monthlyinstlamount_332A',\n 'min_monthlyinstlamount_674A',\n 'min_totaldebtoverduevalue_178A',\n 'mean_credlmt_230A',\n 'mean_outstandingamount_362A',\n 'mean_residualamount_488A',\n 'mean_totaldebtoverduevalue_178A',\n 'std_instlamount_852A',\n 'std_monthlyinstlamount_332A',\n 'std_residualamount_488A',\n 'std_residualamount_856A',\n 'std_totalamount_6A',\n 'applicant_max_monthlyinstlamount_332A',\n 'max_dateofcredstart_181D',\n 'max_dateofrealrepmt_138D',\n 'max_lastupdate_388D',\n 'max_numberofoverdueinstlmaxdat_641D',\n 'max_overdueamountmax2date_1002D',\n 'min_dateofcredend_289D',\n 'min_dateofcredstart_181D',\n 'min_dateofrealrepmt_138D',\n 'applicant_max_dateofcredend_353D',\n 'applicant_max_dateofrealrepmt_138D',\n 'applicant_max_lastupdate_388D',\n 'applicant_max_refreshdate_3813885D',\n 'mode1_numberofinstls_320L',\n 'mode1_prolongationcount_1120L',\n 'applicant_max_numberofinstls_320L',\n 'applicant_max_numberofoverdueinstlmax_1151L',\n 'mean_pmts_overdue_1152A',\n 'maxdpdinstldate_3546855D',\n 'maxdpdlast3m_392P',\n 'mode1_numberofoverdueinstls_725L',\n 'agg_numinstnotpaid_L',\n\n 'mode1_rejectreason_755M',\n 'mode1_financialinstitution_591M',\n 'mode1_familystate_447L',\n 'mode1_education_927M',\n 'datelastunpaid_3546854D',\n 'mode1_cancelreason_3545846M',\n ]","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:26:53.684956Z","iopub.execute_input":"2024-05-06T20:26:53.685474Z","iopub.status.idle":"2024-05-06T20:26:53.780010Z","shell.execute_reply.started":"2024-05-06T20:26:53.685444Z","shell.execute_reply":"2024-05-06T20:26:53.779192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# group = [\n# 'mode1_additional_param_contaddr_district_15M',\n# 'mode1_additional_param_registaddr_district_1083M',\n# ]\n\n# display(X[group])\n# display(X[group].corr())\n\n# for f in group:\n# print(f, ' : ', X[f].isna().sum() / X.shape[0], ' (nans)')\n\n# print()\n\n# for i in range(len(group)):\n# for j in range(i, len(group)):\n# fi = group[i]\n# fj = group[j]\n# if X[fi].dtype.name != 'category' and X[fj].dtype.name != 'category':\n# if i != j and fi[-1] == fj[-1]:\n# corr_coef = X[[fi, fj]].corr().iloc[0, 1]\n# print(fi, ' - ', fj, ' : ', corr_coef, ' (corr)')\n\n# tmp_df = pd.DataFrame(\n# index=X.index,\n# data={\n# fi: X[fi], fj: X[fj], 'target': y\n# }\n# )\n# neg_df = tmp_df[tmp_df[fi] - tmp_df[fj] < 0]\n# pos_df = tmp_df[tmp_df[fi] - tmp_df[fj] > 0]\n \n# neg_mean = neg_df['target'].mean()\n# pos_mean = pos_df['target'].mean()\n# threshold = 0.001\n\n# if neg_mean > 0.09 and neg_df.shape[0] / X.shape[0] > threshold:\n# print('HIGH, NEG:', fi, fj, neg_mean)\n# elif neg_mean < 0.01 and neg_df.shape[0] / X.shape[0] > threshold:\n# print(' LOW, NEG:', fi, fj, neg_mean)\n# elif pos_mean > 0.09 and pos_df.shape[0] / X.shape[0] > threshold:\n# print('HIGH, POS:', fi, fj, pos_mean)\n# elif pos_mean < 0.01 and pos_df.shape[0] / X.shape[0] > threshold:\n# print(' LOW, POS:', fi, fj, pos_mean)\n# else:\n# print('############### --- ###############')\n# print()","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:26:53.781592Z","iopub.execute_input":"2024-05-06T20:26:53.781986Z","iopub.status.idle":"2024-05-06T20:26:53.790302Z","shell.execute_reply.started":"2024-05-06T20:26:53.781953Z","shell.execute_reply":"2024-05-06T20:26:53.789193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\npreprocessor = Preprocessor()\npreprocessor.fit(df_train)\n\n# print(preprocessor._cat_col2valid_values)\n\nX, y = preprocessor.transform(df_train)\nweek_num = X['WEEK_NUM']\nX = X.drop(columns=['WEEK_NUM'])\n\nassert y.shape == week_num.shape\nassert y.shape == (X.shape[0],)\nassert (week_num.index == X.index).all()\nassert (X.index == y.index).all()\n\nfor i in range(1, week_num.shape[0]):\n assert week_num.iloc[i - 1] <= week_num.iloc[i]","metadata":{"execution":{"iopub.status.busy":"2024-05-06T19:07:35.422454Z","iopub.execute_input":"2024-05-06T19:07:35.422753Z","iopub.status.idle":"2024-05-06T19:16:09.400129Z","shell.execute_reply.started":"2024-05-06T19:07:35.422726Z","shell.execute_reply":"2024-05-06T19:16:09.399079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.isna().sum().sum() / X.shape[0] / X.shape[1]","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:42:02.515263Z","iopub.execute_input":"2024-05-06T20:42:02.515931Z","iopub.status.idle":"2024-05-06T20:42:03.069075Z","shell.execute_reply.started":"2024-05-06T20:42:02.515897Z","shell.execute_reply":"2024-05-06T20:42:03.067558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"assert df_train.shape[0] == X.shape[0]\n\nfor col in X.columns:\n assert X[col].isna().sum() <= 0.95 * X.shape[0]\n assert X[col].unique().shape[0] >= 2\n assert X[col].dtype.name != 'object'\n\n if X[col].dtype.name == 'category':\n assert X[col].isna().sum() == 0","metadata":{"execution":{"iopub.status.busy":"2024-05-06T19:16:10.383339Z","iopub.execute_input":"2024-05-06T19:16:10.383785Z","iopub.status.idle":"2024-05-06T19:16:22.810785Z","shell.execute_reply.started":"2024-05-06T19:16:10.383751Z","shell.execute_reply":"2024-05-06T19:16:22.809893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.shape, X.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-06T19:16:22.812012Z","iopub.execute_input":"2024-05-06T19:16:22.812331Z","iopub.status.idle":"2024-05-06T19:16:22.818803Z","shell.execute_reply.started":"2024-05-06T19:16:22.812305Z","shell.execute_reply":"2024-05-06T19:16:22.817641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df_train\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-06T19:16:22.827598Z","iopub.execute_input":"2024-05-06T19:16:22.828078Z","iopub.status.idle":"2024-05-06T19:16:22.942172Z","shell.execute_reply.started":"2024-05-06T19:16:22.828048Z","shell.execute_reply":"2024-05-06T19:16:22.941016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Feature selection","metadata":{}},{"cell_type":"code","source":"assert (\n len(Preprocessor._get_primary_features()) ==\n len(set(Preprocessor._get_primary_features()))\n)\nassert (\n len(Preprocessor._get_secondary_features()) ==\n len(set(Preprocessor._get_secondary_features()))\n)\nassert 0 == len(\n set(Preprocessor._get_primary_features()).intersection(\n set(Preprocessor._get_secondary_features())\n))","metadata":{"execution":{"iopub.status.busy":"2024-05-06T19:16:22.943565Z","iopub.execute_input":"2024-05-06T19:16:22.943927Z","iopub.status.idle":"2024-05-06T19:16:22.950565Z","shell.execute_reply.started":"2024-05-06T19:16:22.943898Z","shell.execute_reply":"2024-05-06T19:16:22.949570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_features = (\n Preprocessor._get_primary_features() +\n Preprocessor._get_secondary_features()\n)\nassert len(base_features) == len(set(base_features))","metadata":{"execution":{"iopub.status.busy":"2024-05-06T19:16:22.951802Z","iopub.execute_input":"2024-05-06T19:16:22.952452Z","iopub.status.idle":"2024-05-06T19:16:22.959779Z","shell.execute_reply.started":"2024-05-06T19:16:22.952411Z","shell.execute_reply":"2024-05-06T19:16:22.958885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(base_features)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_features = X.shape[1]\n# corr_df = X[[\n# col for col in X.columns\n# if X[col].dtype.name != 'category'\n# ]].corr()\n\nduplicated_columns = []\nexact_correlations = []\nhigh_correlations = []\n\nfor i in tqdm.tqdm(range(n_features)):\n for j in range(i + 1, n_features):\n fi = X.columns[i]\n fj = X.columns[j]\n pair = [fi, fj]\n \n if X[fi].equals(X[fj]):\n duplicated_columns.append(pair)\n\n# if fi != fj and X[fi].dtype.name != 'category' and X[fj].dtype.name != 'category':\n# if X[pair].dropna().shape[0] >= 10:\n# corr_df = X[pair].corr()\n# assert corr_df.shape == (2, 2)\n# corr_coef = corr_df.iloc[0, 1]\n\n# if not np.isnan(corr_coef):\n# assert -1.0 <= corr_coef <= 1.0\n \n# if np.isclose(corr_coef, 1.0):\n# exact_correlations.append(pair)\n# elif corr_coef > 0.95:\n# high_correlations.append(pair)","metadata":{"execution":{"iopub.status.busy":"2024-05-06T19:16:22.962630Z","iopub.execute_input":"2024-05-06T19:16:22.963000Z","iopub.status.idle":"2024-05-06T20:02:19.105729Z","shell.execute_reply.started":"2024-05-06T19:16:22.962970Z","shell.execute_reply":"2024-05-06T20:02:19.104142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for (f1, f2) in duplicated_columns:\n assert X[f1].equals(X[f2])\n\nduplicated_columns","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:02:24.166459Z","iopub.execute_input":"2024-05-06T20:02:24.167223Z","iopub.status.idle":"2024-05-06T20:02:24.174706Z","shell.execute_reply.started":"2024-05-06T20:02:24.167188Z","shell.execute_reply":"2024-05-06T20:02:24.173709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for (f1, f2) in exact_correlations:\n# corr_coef = X[[f1, f2]].corr().iloc[0, 1]\n# assert np.isclose(corr_coef, 1.0)\n\n# pair_df = X[[f1, f2]].dropna()\n# if pair_df[f1].equals(pair_df[f2]):\n# print(f1, f2, pair_df.shape[0])\n\n# exact_correlations","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:02:25.900098Z","iopub.execute_input":"2024-05-06T20:02:25.901011Z","iopub.status.idle":"2024-05-06T20:02:25.974353Z","shell.execute_reply.started":"2024-05-06T20:02:25.900979Z","shell.execute_reply":"2024-05-06T20:02:25.973381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for (f1, f2) in high_correlations:\n# corr_coef = X[[f1, f2]].corr().iloc[0, 1]\n# assert 0.95 < corr_coef < 1.0\n# # if 0.95 <= corr_coef < 0.97:\n# print(\"'\", f1, \"', '\", f2, \"'\", \" corr_coef: \", corr_coef, sep='')\n\n# # high_correlations","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:02:19.111662Z","iopub.status.idle":"2024-05-06T20:02:19.112042Z","shell.execute_reply.started":"2024-05-06T20:02:19.111867Z","shell.execute_reply":"2024-05-06T20:02:19.111883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def analyze_features_pair(x1, x2, target):\n assert len(x1.shape) == len(x2.shape) == len(target.shape) == 1\n assert x1.shape == x2.shape == target.shape\n \n scores = []\n \n scaled_x1 = (x1 - x1.mean()) / x1.std()\n scaled_x2 = (x2 - x2.mean()) / x2.std()\n\n df = pd.DataFrame(\n index=target.index,\n data={\n '_'.join(['add', x1.name, x2.name]): scaled_x1 + scaled_x2,\n '_'.join(['sub', x1.name, x2.name]): scaled_x1 - scaled_x2,\n '_'.join(['mul', x1.name, x2.name]): scaled_x1 * scaled_x2,\n '_'.join(['div', x1.name, x2.name]): scaled_x1 / scaled_x2,\n 'target': target\n }\n ).replace([np.inf, -np.inf], np.nan)\n\n for pair_feature in df.columns:\n if pair_feature != 'target':\n pair_df = df[[pair_feature, 'target']].dropna()\n if pair_df.shape[0] > 0.3 * df.shape[0]:\n score = roc_auc_score(\n pair_df['target'],\n pair_df[pair_feature]\n )\n if score < 0.3 or score > 0.7:\n scores.append([pair_feature, score])\n \n return scores\n \n\ndef is_pair_analysis(x1, x2):\n assert len(x1.shape) == len(x2.shape) == 1\n assert x1.shape == x2.shape\n \n if x1.name == x2.name:\n return False\n\n if x1.name[-1] != x2.name[-1]:\n return False\n\n if x1.dtype.name == 'category' or x2.dtype.name == 'category':\n return False\n\n prefixes = ['add_', 'sub_', 'mul_', 'div_']\n for prefix in prefixes:\n if prefix in x1.name or prefix in x2.name:\n return False\n\n prefixes = ['max_', 'min_', 'mean_', 'std_']\n for prefix1 in prefixes:\n for prefix2 in prefixes:\n if prefix1 != prefix2:\n if prefix1 in x1.name and prefix2 in x2.name:\n return False\n\n if x1.isna().sum() > 0.8 * x1.shape[0]:\n return False\n if x2.isna().sum() > 0.8 * x2.shape[0]:\n return False\n\n return True\n\n\ndef find_pair_features(df, target):\n assert len(df.shape) == 2\n assert 'target' not in df.columns\n assert 'y' not in df.columns\n assert target.shape == (df.shape[0],)\n \n pairs = []\n\n for i in tqdm.tqdm(range(df.shape[1])):\n for j in range(df.shape[1]):\n f1 = df.columns[i]\n f2 = df.columns[j]\n\n if is_pair_analysis(df[f1], df[f2]):\n pairs.extend(\n analyze_features_pair(df[f1], df[f2], target)\n )\n\n return pairs","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:02:19.113543Z","iopub.status.idle":"2024-05-06T20:02:19.113921Z","shell.execute_reply.started":"2024-05-06T20:02:19.113715Z","shell.execute_reply":"2024-05-06T20:02:19.113729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"assert X.shape[0] == y.shape[0]\n\nfor col in X.columns:\n if X[col].dtype.name != 'category':\n score = roc_auc_score(\n y,\n X[col].fillna(X[col].mean())\n )\n if score < 0.4 or score > 0.6:\n print(col, score)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pairs = find_pair_features(X, y)\npairs.sort(key=lambda pair: pair[1], reverse=True)\npairs","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:02:19.116081Z","iopub.status.idle":"2024-05-06T20:02:19.116450Z","shell.execute_reply.started":"2024-05-06T20:02:19.116270Z","shell.execute_reply":"2024-05-06T20:02:19.116285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# high_corr_features = set()\n# for (f1, f2) in high_correlations:\n# high_corr_features.add(f1)\n# high_corr_features.add(f2)","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:02:19.118013Z","iopub.status.idle":"2024-05-06T20:02:19.118363Z","shell.execute_reply.started":"2024-05-06T20:02:19.118187Z","shell.execute_reply":"2024-05-06T20:02:19.118202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# find_diff_features(X[list(high_corr_features)], y)","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:02:19.119359Z","iopub.status.idle":"2024-05-06T20:02:19.119699Z","shell.execute_reply.started":"2024-05-06T20:02:19.119528Z","shell.execute_reply":"2024-05-06T20:02:19.119543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# feature_definitions_df = pd.read_csv(\n# '/kaggle/input/home-credit-credit-risk-model-stability/feature_definitions.csv'\n# )","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:02:19.121536Z","iopub.status.idle":"2024-05-06T20:02:19.121894Z","shell.execute_reply.started":"2024-05-06T20:02:19.121707Z","shell.execute_reply":"2024-05-06T20:02:19.121721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def check_feature(feature):\n# for base_feature in base_features:\n# if feature in base_feature:\n# return False\n# return True","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:02:19.124313Z","iopub.status.idle":"2024-05-06T20:02:19.124648Z","shell.execute_reply.started":"2024-05-06T20:02:19.124485Z","shell.execute_reply":"2024-05-06T20:02:19.124499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# unused_features = []\n\n# for i in range(feature_definitions_df.shape[0]):\n# feature = feature_definitions_df.iloc[i]['Variable']\n# if check_feature(feature):\n# unused_features.append(feature)","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:02:19.125685Z","iopub.status.idle":"2024-05-06T20:02:19.126048Z","shell.execute_reply.started":"2024-05-06T20:02:19.125873Z","shell.execute_reply":"2024-05-06T20:02:19.125889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for unused_feature in unused_features:\n# for col in X.columns:\n# if unused_feature in col:\n# print(col, ' ', unused_feature)\n# print()","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:02:19.127142Z","iopub.status.idle":"2024-05-06T20:02:19.127473Z","shell.execute_reply.started":"2024-05-06T20:02:19.127307Z","shell.execute_reply":"2024-05-06T20:02:19.127320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgb_params = {\n 'objective': 'binary',\n 'boosting_type': 'gbdt',\n # 'bagging_freq': 2,\n # 'pos_bagging_fraction': 0.9,\n # 'neg_bagging_fraction': 0.9,\n # 'colsample_bynode': 0.9,\n # 'colsample_bytree': 0.9,\n 'device': 'cpu',\n # 'is_unbalance': True,\n 'learning_rate': 0.1,\n 'max_depth': 4,\n 'metric': 'auc',\n 'n_estimators': 100,\n # 'lambda_l1': 0.001,\n # 'lambda_l2': 0.001,\n # \"reg_alpha\": 0.1,\n # \"reg_lambda\": 10,\n \"verbose\": -1,\n}","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:02:19.128444Z","iopub.status.idle":"2024-05-06T20:02:19.128779Z","shell.execute_reply.started":"2024-05-06T20:02:19.128612Z","shell.execute_reply":"2024-05-06T20:02:19.128626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"border = 1000000\n\nmodel = lgb.LGBMClassifier(**lgb_params)\nmodel.fit(X[base_features].iloc[:border], y.iloc[:border])\n\nbase_score = roc_auc_score(\n y.iloc[border:].to_numpy(),\n model.predict_proba(X[base_features].iloc[border:])[:, 1]\n)\n\nfeatures = [feature for feature in base_features]\nscores = [base_score for _ in base_features]\n\nprint('base score:', base_score)\n\nfor feature in tqdm.tqdm(X.columns):\n if feature not in features:\n curr_features = base_features + [feature]\n # curr_features = features + [feature]\n\n X_train = X[curr_features].iloc[:border]\n y_train = y.iloc[:border]\n X_valid = X[curr_features].iloc[border:]\n y_valid = y.iloc[border:]\n\n model = lgb.LGBMClassifier(**lgb_params)\n model.fit(X_train, y_train)\n\n curr_score = roc_auc_score(\n y_valid.to_numpy(),\n model.predict_proba(X_valid)[:, 1]\n )\n\n # if curr_score > base_score + 0.0001:\n # base_score = curr_score\n features.append(feature)\n scores.append(curr_score)\n\nassert len(features) == len(scores)\nprint('final score:', base_score)","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:02:19.130762Z","iopub.status.idle":"2024-05-06T20:02:19.131120Z","shell.execute_reply.started":"2024-05-06T20:02:19.130953Z","shell.execute_reply":"2024-05-06T20:02:19.130968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:02:19.132360Z","iopub.status.idle":"2024-05-06T20:02:19.132871Z","shell.execute_reply.started":"2024-05-06T20:02:19.132591Z","shell.execute_reply":"2024-05-06T20:02:19.132612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:02:19.134104Z","iopub.status.idle":"2024-05-06T20:02:19.134435Z","shell.execute_reply.started":"2024-05-06T20:02:19.134268Z","shell.execute_reply":"2024-05-06T20:02:19.134281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(16, 8))\nplt.plot(range(len(features)), scores)","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:02:19.135645Z","iopub.status.idle":"2024-05-06T20:02:19.136019Z","shell.execute_reply.started":"2024-05-06T20:02:19.135821Z","shell.execute_reply":"2024-05-06T20:02:19.135853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(features)","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:02:19.137406Z","iopub.status.idle":"2024-05-06T20:02:19.137760Z","shell.execute_reply.started":"2024-05-06T20:02:19.137589Z","shell.execute_reply":"2024-05-06T20:02:19.137604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(1, len(scores)):\n if scores[i] - scores[0] < -0.0001:\n print(i, features[i], 10000.0 * (scores[i] - scores[0]))","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:02:19.138850Z","iopub.status.idle":"2024-05-06T20:02:19.139181Z","shell.execute_reply.started":"2024-05-06T20:02:19.139022Z","shell.execute_reply":"2024-05-06T20:02:19.139036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(1, len(scores)):\n if scores[i] - scores[0] > 0.0001:\n print(i, features[i], 10000.0 * (scores[i] - scores[0]))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# nans_shares = X.isna().sum() / X.shape[0]\n# diff_pairs = []\n\n# for i in range(nans_shares.shape[0]):\n# for j in range(i + 1, nans_shares.shape[0]):\n# if nans_shares.iloc[i] > 0.5 and nans_shares.iloc[j] > 0.5:\n# fi = nans_shares.index[i]\n# fj = nans_shares.index[j]\n# if X[fi].dtype.name != 'category' and X[fj].dtype.name != 'category':\n# if fi[-1] == fj[-1] and 'std' not in fi and 'std' not in fj:\n# if 'agg_' not in fi and 'agg_' not in fj:\n# if X[[fi, fj]].corr().iloc[0, 1] > 0.8:\n# diff_pairs.append([nans_shares.index[i], nans_shares.index[j]])","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:02:19.140778Z","iopub.status.idle":"2024-05-06T20:02:19.141149Z","shell.execute_reply.started":"2024-05-06T20:02:19.140981Z","shell.execute_reply":"2024-05-06T20:02:19.140996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# diff_pairs","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:02:19.142202Z","iopub.status.idle":"2024-05-06T20:02:19.142604Z","shell.execute_reply.started":"2024-05-06T20:02:19.142431Z","shell.execute_reply":"2024-05-06T20:02:19.142447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for (fi, fj) in diff_pairs:\n# analyze_features_pair(X[fi], X[fj], y)","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:02:19.144021Z","iopub.status.idle":"2024-05-06T20:02:19.144352Z","shell.execute_reply.started":"2024-05-06T20:02:19.144185Z","shell.execute_reply":"2024-05-06T20:02:19.144199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# cols = []\n\n# for col in X.columns:\n# if 'credacc_actualbalance_314A' in col:\n# cols.append(col)\n \n# X[cols].isna().sum()","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:02:19.145664Z","iopub.status.idle":"2024-05-06T20:02:19.146031Z","shell.execute_reply.started":"2024-05-06T20:02:19.145850Z","shell.execute_reply":"2024-05-06T20:02:19.145870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# del df_train\ndel X\ndel y\n# del week_num\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:02:19.147218Z","iopub.status.idle":"2024-05-06T20:02:19.147549Z","shell.execute_reply.started":"2024-05-06T20:02:19.147389Z","shell.execute_reply":"2024-05-06T20:02:19.147402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_store = {\n \"df_base\": read_file(TEST_DIR / \"test_base.parquet\"),\n \"depth_0\": [\n read_file(TEST_DIR / \"test_static_cb_0.parquet\"),\n read_files(TEST_DIR / \"test_static_0_*.parquet\"),\n ],\n \"depth_1\": [\n read_files(TEST_DIR / \"test_applprev_1_*.parquet\", 1),\n read_file(TEST_DIR / \"test_tax_registry_a_1.parquet\", 1),\n read_file(TEST_DIR / \"test_tax_registry_b_1.parquet\", 1),\n read_file(TEST_DIR / \"test_tax_registry_c_1.parquet\", 1),\n read_files(TEST_DIR / \"test_credit_bureau_a_1_*.parquet\", 1),\n read_file(TEST_DIR / \"test_credit_bureau_b_1.parquet\", 1),\n read_file(TEST_DIR / \"test_other_1.parquet\", 1),\n read_file(TEST_DIR / \"test_person_1.parquet\", 1),\n read_file(TEST_DIR / \"test_deposit_1.parquet\", 1),\n read_file(TEST_DIR / \"test_debitcard_1.parquet\", 1),\n ],\n \"depth_2\": [\n read_file(TEST_DIR / \"test_credit_bureau_b_2.parquet\", 2),\n read_files(TEST_DIR / \"test_credit_bureau_a_2_*.parquet\", 2),\n ]\n}","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:02:19.149297Z","iopub.status.idle":"2024-05-06T20:02:19.149660Z","shell.execute_reply.started":"2024-05-06T20:02:19.149487Z","shell.execute_reply":"2024-05-06T20:02:19.149502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = feature_eng(**data_store)\nprint(\"test data shape:\\t\", df_test.shape)","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:02:19.151342Z","iopub.status.idle":"2024-05-06T20:02:19.151671Z","shell.execute_reply.started":"2024-05-06T20:02:19.151512Z","shell.execute_reply":"2024-05-06T20:02:19.151525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del data_store\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:02:19.152995Z","iopub.status.idle":"2024-05-06T20:02:19.153338Z","shell.execute_reply.started":"2024-05-06T20:02:19.153170Z","shell.execute_reply":"2024-05-06T20:02:19.153185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = df_test.select([col for col in train_cols if col != \"target\"])\nassert len(df_test.shape) == 2\nprint(\"test data shape:\\t\", df_test.shape)","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:02:19.154730Z","iopub.status.idle":"2024-05-06T20:02:19.155090Z","shell.execute_reply.started":"2024-05-06T20:02:19.154921Z","shell.execute_reply":"2024-05-06T20:02:19.154936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test, _ = to_pandas(df_test, cat_cols)","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:02:19.156543Z","iopub.status.idle":"2024-05-06T20:02:19.157051Z","shell.execute_reply.started":"2024-05-06T20:02:19.156783Z","shell.execute_reply":"2024-05-06T20:02:19.156803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test, _ = preprocessor.transform(df_test)","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:02:19.158183Z","iopub.status.idle":"2024-05-06T20:02:19.158528Z","shell.execute_reply.started":"2024-05-06T20:02:19.158358Z","shell.execute_reply":"2024-05-06T20:02:19.158373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df_test\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:02:19.160044Z","iopub.status.idle":"2024-05-06T20:02:19.160527Z","shell.execute_reply.started":"2024-05-06T20:02:19.160275Z","shell.execute_reply":"2024-05-06T20:02:19.160295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Prediction","metadata":{}},{"cell_type":"code","source":"y_pred = X_test['agg_currtotaldebt_A']\ny_pred = y_pred.to_numpy()","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:02:19.162221Z","iopub.status.idle":"2024-05-06T20:02:19.162713Z","shell.execute_reply.started":"2024-05-06T20:02:19.162452Z","shell.execute_reply":"2024-05-06T20:02:19.162472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = pd.Series(y_pred, index=X_test.index)\ny_pred = y_pred.fillna(0.0)","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:02:19.164142Z","iopub.status.idle":"2024-05-06T20:02:19.164634Z","shell.execute_reply.started":"2024-05-06T20:02:19.164373Z","shell.execute_reply":"2024-05-06T20:02:19.164394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Submission","metadata":{}},{"cell_type":"code","source":"df_subm = pd.read_csv(ROOT / \"sample_submission.csv\")\ndf_subm = df_subm.set_index(\"case_id\")\n\ndf_subm[\"score\"] = y_pred","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:02:19.166212Z","iopub.status.idle":"2024-05-06T20:02:19.166560Z","shell.execute_reply.started":"2024-05-06T20:02:19.166389Z","shell.execute_reply":"2024-05-06T20:02:19.166404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_subm","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:02:19.167818Z","iopub.status.idle":"2024-05-06T20:02:19.168201Z","shell.execute_reply.started":"2024-05-06T20:02:19.168028Z","shell.execute_reply":"2024-05-06T20:02:19.168044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_subm.to_csv(\"submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-05-06T20:02:19.169783Z","iopub.status.idle":"2024-05-06T20:02:19.170176Z","shell.execute_reply.started":"2024-05-06T20:02:19.169988Z","shell.execute_reply":"2024-05-06T20:02:19.170003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}