{"metadata":{"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"dockerImageVersionId":30684,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false},"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Example Notebook\n\nWelcome to the example notebook for the Home Credit Kaggle competition. The goal of this competition is to determine how likely a customer is going to default on an issued loan. The main difference between the [first](https://www.kaggle.com/c/home-credit-default-risk) and this competition is that now your submission will be scored with a custom metric that will take into account how well the model performs in future. A decline in performance will be penalized. The goal is to create a model that is stable and performs well in the future.\n\nIn this notebook you will see how to:\n* Load the data\n* Join tables with Polars - a DataFrame library implemented in Rust language, designed to be blazingy fast and memory efficient.  \n* Create simple aggregation features\n* Train a LightGBM model\n* Create a submission table","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5"}},{"cell_type":"markdown","source":"## Load the data","metadata":{}},{"cell_type":"code","source":"import polars as pl\nimport numpy as np\nimport pandas as pd\nimport lightgbm as lgb\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score \nimport gc\n\ndataPath = \"/kaggle/input/home-credit-credit-risk-model-stability/\"\n#dataPath = './'","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_table_dtypes(df: pl.DataFrame) -> pl.DataFrame:\n    # implement here all desired dtypes for tables\n    # the following is just an example\n    for col in df.columns:\n        # last letter of column name will help you determine the type\n        if col[-1] in (\"P\", \"A\"):\n            df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n        elif col[-1] == 'D' or 'date' in col:\n            df = df.with_columns(pl.col(col).cast(pl.Date).cast(pl.Float64).alias(col))\n\n    return df\n\ndef convert_strings(df: pd.DataFrame) -> pd.DataFrame:\n    for col in df.columns:  \n        if df[col].dtype.name in ['object', 'string']:\n            df[col] = df[col].astype(\"string\").fillna('').astype('category')\n            current_categories = df[col].cat.categories\n            new_categories = current_categories.to_list() + [\"Unknown\"]\n            new_dtype = pd.CategoricalDtype(categories=new_categories, ordered=True)\n            df[col] = df[col].astype(new_dtype)\n    return df","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_basetable = pl.read_parquet(dataPath + \"parquet_files/train/train_base.parquet\").pipe(set_table_dtypes)\ntest_basetable = pl.read_parquet(dataPath + \"parquet_files/test/test_base.parquet\").pipe(set_table_dtypes)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"static_0 = pl.concat(\n    [\n        pl.read_parquet(dataPath + \"parquet_files/train/train_static_0_0.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/train/train_static_0_1.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/test/test_static_0_0.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/test/test_static_0_1.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/test/test_static_0_2.parquet\").pipe(set_table_dtypes),\n    ],\n    how=\"vertical_relaxed\",\n)\n\ndata = train_basetable.join(\n    static_0, how=\"left\", on=\"case_id\"\n)\ndata_submission = test_basetable.join(\n    static_0, how=\"left\", on=\"case_id\"\n)\n\ndel static_0\ngc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"static_cb_0 = pl.concat(\n    [\n        pl.read_parquet(dataPath + \"parquet_files/train/train_static_cb_0.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/test/test_static_cb_0.parquet\").pipe(set_table_dtypes),\n    ],\n    how=\"vertical_relaxed\",\n)\ndata = data.join(\n    static_cb_0, how=\"left\", on=\"case_id\"\n)\ndata_submission = data_submission.join(\n    static_cb_0, how=\"left\", on=\"case_id\"\n)\n\ndel static_cb_0\ngc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"applprev_1 = pl.concat(\n    [\n        pl.read_parquet(dataPath + \"parquet_files/train/train_applprev_1_0.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/train/train_applprev_1_1.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/test/test_applprev_1_0.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/test/test_applprev_1_1.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/test/test_applprev_1_2.parquet\").pipe(set_table_dtypes),\n    ],\n    how=\"vertical_relaxed\",\n)\n\ndata = data.join(\n    applprev_1, how='left', on=\"case_id\"\n).with_columns(\n    ( 2*((pl.col('creationdate_885D') < pl.col('date_decision')).cast(pl.Int64)-0.5)*pl.col('creationdate_885D') ).alias('sort')\n).group_by('case_id').agg([\n    pl.len().alias(\"applprev_1_len\"),\n    pl.all().sort_by('date_decision', -pl.col('num_group1'), nulls_last=False).last(),\n]).drop(\n    ['sort','num_group1']\n)\ndata_submission = data_submission.join(\n    applprev_1, how='left', on=\"case_id\"\n).with_columns(\n    ( 2*((pl.col('creationdate_885D') < pl.col('date_decision')).cast(pl.Int64)-0.5)*pl.col('creationdate_885D') ).alias('sort')\n).group_by('case_id').agg([\n    pl.len().alias(\"applprev_1_len\"),\n    pl.all().sort_by('date_decision', -pl.col('num_group1'), nulls_last=False).last(),\n]).drop(\n    ['sort','num_group1']\n)\n\ndel applprev_1\ngc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"other_1 = pl.concat(\n    [\n        pl.read_parquet(dataPath + \"parquet_files/train/train_other_1.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/test/test_other_1.parquet\").pipe(set_table_dtypes),\n    ],\n    how=\"vertical_relaxed\",\n)\n\ndata = data.join(\n    other_1, how='left', on=\"case_id\"\n).drop('num_group1')\ndata_submission = data_submission.join(\n    other_1, how='left', on=\"case_id\"\n).drop('num_group1')\n\ndel other_1\ngc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tax_registry_a_1 = pl.concat(\n    [\n        pl.read_parquet(dataPath + \"parquet_files/train/train_tax_registry_a_1.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/test/test_tax_registry_a_1.parquet\").pipe(set_table_dtypes),\n    ],\n    how=\"vertical_relaxed\",\n)\ndata = data.join(\n    tax_registry_a_1, how='left', on=\"case_id\"\n).with_columns(\n    ( 2*((pl.col('recorddate_4527225D') < pl.col('date_decision')).cast(pl.Int64)-0.5)*pl.col('recorddate_4527225D') ).alias('sort')\n).group_by('case_id').agg([\n    pl.len().alias(\"tax_registry_a_1_len\"),\n    pl.all().sort_by('date_decision', -pl.col('num_group1'), nulls_last=False).last(),\n]).drop(\n    ['sort','num_group1']\n)\ndata_submission = data_submission.join(\n    tax_registry_a_1, how='left', on=\"case_id\"\n).with_columns(\n    ( 2*((pl.col('recorddate_4527225D') < pl.col('date_decision')).cast(pl.Int64)-0.5)*pl.col('recorddate_4527225D') ).alias('sort')\n).group_by('case_id').agg([\n    pl.len().alias(\"tax_registry_a_1_len\"),\n    pl.all().sort_by('date_decision', -pl.col('num_group1'), nulls_last=False).last(),\n]).drop(\n    ['sort','num_group1']\n)\n\ndel tax_registry_a_1\ngc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tax_registry_b_1 = pl.concat(\n    [\n        pl.read_parquet(dataPath + \"parquet_files/train/train_tax_registry_b_1.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/test/test_tax_registry_b_1.parquet\").pipe(set_table_dtypes),\n    ],\n    how=\"vertical_relaxed\",\n)\ndata = data.join(\n    tax_registry_b_1, how='left', on=\"case_id\"\n).with_columns(\n    ( 2*((pl.col('deductiondate_4917603D') < pl.col('date_decision')).cast(pl.Int64)-0.5)*pl.col('deductiondate_4917603D') ).alias('sort')\n).group_by('case_id').agg([\n    pl.len().alias(\"tax_registry_b_1_len\"),\n    pl.all().sort_by('date_decision', -pl.col('num_group1'), nulls_last=False).last(),\n]).drop(\n    ['sort','num_group1']\n)\ndata_submission = data_submission.join(\n    tax_registry_b_1, how='left', on=\"case_id\"\n).with_columns(\n    ( 2*((pl.col('deductiondate_4917603D') < pl.col('date_decision')).cast(pl.Int64)-0.5)*pl.col('deductiondate_4917603D') ).alias('sort')\n).group_by('case_id').agg([\n    pl.len().alias(\"tax_registry_b_1_len\"),\n    pl.all().sort_by('date_decision', -pl.col('num_group1'), nulls_last=False).last(),\n]).drop(\n    ['sort','num_group1']\n)\n\ndel tax_registry_b_1\ngc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tax_registry_c_1 = pl.concat(\n    [\n        pl.read_parquet(dataPath + \"parquet_files/train/train_tax_registry_c_1.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/test/test_tax_registry_c_1.parquet\").pipe(set_table_dtypes),\n    ],\n    how=\"vertical_relaxed\",\n)\n\ndata = data.join(\n    tax_registry_c_1, how='left', on=\"case_id\"\n).with_columns(\n    ( 2*((pl.col('processingdate_168D') < pl.col('date_decision')).cast(pl.Int64)-0.5)*pl.col('processingdate_168D') ).alias('sort')\n).group_by('case_id').agg([\n    pl.len().alias(\"tax_registry_c_1_len\"),\n    pl.all().sort_by('date_decision', -pl.col('num_group1'), nulls_last=False).last(),\n]).drop(\n    ['sort','num_group1']\n)\ndata_submission = data_submission.join(\n    tax_registry_c_1, how='left', on=\"case_id\"\n).with_columns(\n    ( 2*((pl.col('processingdate_168D') < pl.col('date_decision')).cast(pl.Int64)-0.5)*pl.col('processingdate_168D') ).alias('sort')\n).group_by('case_id').agg([\n    pl.len().alias(\"tax_registry_c_1_len\"),\n    pl.all().sort_by('date_decision', -pl.col('num_group1'), nulls_last=False).last(),\n]).drop(\n    ['sort','num_group1']\n)\n\ndel tax_registry_c_1\ngc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"credit_bureau_a_1 = pl.concat(\n    [\n        pl.read_parquet(dataPath + \"parquet_files/train/train_credit_bureau_a_1_0.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/train/train_credit_bureau_a_1_1.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/train/train_credit_bureau_a_1_2.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/train/train_credit_bureau_a_1_3.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/test/test_credit_bureau_a_1_0.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/test/test_credit_bureau_a_1_1.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/test/test_credit_bureau_a_1_2.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/test/test_credit_bureau_a_1_3.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/test/test_credit_bureau_a_1_4.parquet\").pipe(set_table_dtypes),\n    ],\n    how=\"vertical_relaxed\",\n)\ndata = data.join(\n    credit_bureau_a_1, how='left', on=\"case_id\"\n).with_columns(\n    ( 2*((pl.col('dateofcredstart_739D') < pl.col('date_decision')).cast(pl.Int64)-0.5)*pl.col('dateofcredstart_739D') ).alias('sort')\n).group_by('case_id').agg([\n    pl.len().alias(\"credit_bureau_a_1_len\"),\n    pl.all().sort_by('date_decision', -pl.col('num_group1'), nulls_last=False).last(),\n]).drop(\n    ['sort','num_group1']\n)\ndata_submission = data_submission.join(\n    credit_bureau_a_1, how='left', on=\"case_id\"\n).with_columns(\n    ( 2*((pl.col('dateofcredstart_739D') < pl.col('date_decision')).cast(pl.Int64)-0.5)*pl.col('dateofcredstart_739D') ).alias('sort')\n).group_by('case_id').agg([\n    pl.len().alias(\"credit_bureau_a_1_len\"),\n    pl.all().sort_by('date_decision', -pl.col('num_group1'), nulls_last=False).last(),\n]).drop(\n    ['sort','num_group1']\n)\n\ndel credit_bureau_a_1\ngc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"credit_bureau_b_1 = pl.concat(\n    [\n        pl.read_parquet(dataPath + \"parquet_files/train/train_credit_bureau_b_1.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/test/test_credit_bureau_b_1.parquet\").pipe(set_table_dtypes),\n    ],\n    how=\"vertical_relaxed\",\n)\n\ndata = data.join(\n    credit_bureau_b_1, how='left', on=\"case_id\"\n).with_columns(\n    ( 2*((pl.col('contractdate_551D') < pl.col('date_decision')).cast(pl.Int64)-0.5)*pl.col('contractdate_551D') ).alias('sort')\n).group_by('case_id').agg([\n    pl.len().alias(\"credit_bureau_b_1_len\"),\n    pl.all().sort_by('date_decision', -pl.col('num_group1'), nulls_last=False).last(),\n]).drop(\n    ['sort','num_group1']\n)\ndata_submission = data_submission.join(\n    credit_bureau_b_1, how='left', on=\"case_id\"\n).with_columns(\n    ( 2*((pl.col('contractdate_551D') < pl.col('date_decision')).cast(pl.Int64)-0.5)*pl.col('contractdate_551D') ).alias('sort')\n).group_by('case_id').agg([\n    pl.len().alias(\"credit_bureau_b_1_len\"),\n    pl.all().sort_by('date_decision', -pl.col('num_group1'), nulls_last=False).last(),\n]).drop(\n    ['sort','num_group1']\n)\n\ndel credit_bureau_b_1\ngc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"deposit_1 =pl.concat(\n    [\n        pl.read_parquet(dataPath + \"parquet_files/train/train_deposit_1.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/test/test_deposit_1.parquet\").pipe(set_table_dtypes),\n    ],\n    how=\"vertical_relaxed\",\n)\n\ndata = data.join(\n    deposit_1, how='left', on=\"case_id\"\n).with_columns(\n    ( 2*((pl.col('openingdate_313D') < pl.col('date_decision')).cast(pl.Int64)-0.5)*pl.col('openingdate_313D') ).alias('sort')\n).group_by('case_id').agg([\n    pl.len().alias(\"deposit_1_len\"),\n    pl.all().sort_by('date_decision', -pl.col('num_group1'), nulls_last=False).last(),\n]).drop(\n    ['sort','num_group1']\n)\ndata_submission = data_submission.join(\n    deposit_1, how='left', on=\"case_id\"\n).with_columns(\n    ( 2*((pl.col('openingdate_313D') < pl.col('date_decision')).cast(pl.Int64)-0.5)*pl.col('openingdate_313D') ).alias('sort')\n).group_by('case_id').agg([\n    pl.len().alias(\"deposit_1_len\"),\n    pl.all().sort_by('date_decision', -pl.col('num_group1'), nulls_last=False).last(),\n]).drop(\n    ['sort','num_group1']\n)\n\ndel deposit_1\ngc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"person_1 = pl.concat(\n    [\n        pl.read_parquet(dataPath + \"parquet_files/train/train_person_1.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/test/test_person_1.parquet\").pipe(set_table_dtypes),\n    ],\n    how=\"vertical_relaxed\",\n)\n\ndata = data.join(\n    person_1.filter(pl.col(\"num_group1\") == 0\n).drop(\"num_group1\"), how=\"left\", on=\"case_id\"\n)\ndata_submission = data_submission.join(\n    person_1.filter(pl.col(\"num_group1\") == 0\n).drop(\"num_group1\"), how=\"left\", on=\"case_id\"\n)\n\ndel person_1\ngc.collect()\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"debitcard_1 = pl.concat(\n    [\n        pl.read_parquet(dataPath + \"parquet_files/train/train_debitcard_1.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/test/test_debitcard_1.parquet\").pipe(set_table_dtypes),\n    ],\n    how=\"vertical_relaxed\",\n)\n\ndata = data.join(\n    debitcard_1, how='left', on=\"case_id\"\n).with_columns(\n    ( 2*((pl.col('openingdate_857D') < pl.col('date_decision')).cast(pl.Int64)-0.5)*pl.col('openingdate_857D') ).alias('sort')\n).group_by('case_id').agg([\n    pl.len().alias(\"debitcard_1_len\"),\n    pl.all().sort_by('date_decision', -pl.col('num_group1'), nulls_last=False).last(),\n]).drop(\n    ['sort','num_group1']\n)\ndata_submission = data_submission.join(\n    debitcard_1, how='left', on=\"case_id\"\n).with_columns(\n    ( 2*((pl.col('openingdate_857D') < pl.col('date_decision')).cast(pl.Int64)-0.5)*pl.col('openingdate_857D') ).alias('sort')\n).group_by('case_id').agg([\n    pl.len().alias(\"debitcard_1_len\"),\n    pl.all().sort_by('date_decision', -pl.col('num_group1'), nulls_last=False).last(),\n]).drop(\n    ['sort','num_group1']\n)\n\ndel debitcard_1\ngc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"applprev_2 = pl.concat(\n    [\n        pl.read_parquet(dataPath + \"parquet_files/train/train_applprev_2.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/test/test_applprev_2.parquet\").pipe(set_table_dtypes),\n    ],\n    how=\"vertical_relaxed\",\n)\n\ndata = data.join(\n    applprev_2.filter(pl.col(\"num_group1\") == 0\n).drop(\"num_group1\").filter(pl.col(\"num_group2\") == 0\n).drop(\"num_group2\"), how=\"left\", on=\"case_id\"\n)\ndata_submission = data_submission.join(\n    applprev_2.filter(pl.col(\"num_group1\") == 0\n).drop(\"num_group1\").filter(pl.col(\"num_group2\") == 0\n).drop(\"num_group2\"), how=\"left\", on=\"case_id\"\n)\n\ndel applprev_2\ngc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"person_2 = pl.concat(\n    [\n        pl.read_parquet(dataPath + \"parquet_files/train/train_person_2.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/test/test_person_2.parquet\").pipe(set_table_dtypes),\n    ],\n    how=\"vertical_relaxed\",\n)\n\ndata = data.join(\n    person_2.filter(pl.col(\"num_group1\") == 0\n).drop(\"num_group1\").filter(pl.col(\"num_group2\") == 0\n).drop(\"num_group2\"), how=\"left\", on=\"case_id\"\n)\ndata_submission = data_submission.join(\n    person_2.filter(pl.col(\"num_group1\") == 0\n).drop(\"num_group1\").filter(pl.col(\"num_group2\") == 0\n).drop(\"num_group2\"), how=\"left\", on=\"case_id\"\n)\n\ndel person_2\ngc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\ncredit_bureau_a_2 = pl.concat(\n    [\n        pl.read_parquet(dataPath + \"parquet_files/train/train_credit_bureau_a_2_0.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/train/train_credit_bureau_a_2_1.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/train/train_credit_bureau_a_2_2.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/train/train_credit_bureau_a_2_3.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/train/train_credit_bureau_a_2_4.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/train/train_credit_bureau_a_2_5.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/train/train_credit_bureau_a_2_6.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/train/train_credit_bureau_a_2_7.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/train/train_credit_bureau_a_2_8.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/train/train_credit_bureau_a_2_9.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/train/train_credit_bureau_a_2_10.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/test/test_credit_bureau_a_2_0.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/test/test_credit_bureau_a_2_1.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/test/test_credit_bureau_a_2_2.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/test/test_credit_bureau_a_2_3.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/test/test_credit_bureau_a_2_4.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/test/test_credit_bureau_a_2_5.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/test/test_credit_bureau_a_2_6.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/test/test_credit_bureau_a_2_7.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/test/test_credit_bureau_a_2_8.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/test/test_credit_bureau_a_2_9.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/test/test_credit_bureau_a_2_10.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/test/test_credit_bureau_a_2_11.parquet\").pipe(set_table_dtypes),\n    ],\n    how=\"vertical_relaxed\",\n)\n\ndata = data.join(\n    credit_bureau_a_2.filter(pl.col(\"num_group1\") == 0\n).drop(\"num_group1\").filter(pl.col(\"num_group2\") == 0\n).drop(\"num_group2\"), how=\"left\", on=\"case_id\"\n)\ndata_submission = data_submission.join(\n    credit_bureau_a_2.filter(pl.col(\"num_group1\") == 0\n).drop(\"num_group1\").filter(pl.col(\"num_group2\") == 0\n).drop(\"num_group2\"), how=\"left\", on=\"case_id\"\n)\n\ndel credit_bureau_a_2\ngc.collect()\n'''","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"credit_bureau_b_2 = pl.concat(\n    [\n        pl.read_parquet(dataPath + \"parquet_files/train/train_credit_bureau_b_2.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/test/test_credit_bureau_b_2.parquet\").pipe(set_table_dtypes),\n    ],\n    how=\"vertical_relaxed\",\n)\n\ndata = data.join(\n    credit_bureau_b_2, how='left', on=\"case_id\"\n).with_columns(\n    ( 2*((pl.col('pmts_date_1107D') < pl.col('date_decision')).cast(pl.Int64)-0.5)*pl.col('pmts_date_1107D') ).alias('sort')\n).group_by('case_id').agg([\n    pl.len().alias(\"credit_bureau_b_2_len\"),\n    pl.all().sort_by('date_decision', -pl.col('num_group1'),-pl.col('num_group2'), nulls_last=False).last(),\n]).drop(\n    ['sort','num_group1']\n)\ndata_submission = data_submission.join(\n    credit_bureau_b_2, how='left', on=\"case_id\"\n).with_columns(\n    ( 2*((pl.col('pmts_date_1107D') < pl.col('date_decision')).cast(pl.Int64)-0.5)*pl.col('pmts_date_1107D') ).alias('sort')\n).group_by('case_id').agg([\n    pl.len().alias(\"credit_bureau_b_2_len\"),\n    pl.all().sort_by('date_decision', -pl.col('num_group1'),-pl.col('num_group2'), nulls_last=False).last(),\n]).drop(\n    ['sort','num_group1']\n)\n\ndel credit_bureau_b_2\ngc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Join the data","metadata":{}},{"cell_type":"markdown","source":"!mlflow server --host 127.0.0.1 --port 8899","metadata":{}},{"cell_type":"code","source":"import mlflow\n\nmlflow.set_tracking_uri(\"http://127.0.0.1:8899\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"exp = mlflow.get_experiment_by_name('My experiment')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = train_basetable.join(\n    static_0, how=\"left\", on=\"case_id\"\n).join(\n    static_cb_0, how=\"left\", on=\"case_id\"\n).join(\n    applprev_1.filter(pl.col(\"num_group1\") == 0\n).drop(\"num_group1\"), how=\"left\", on=\"case_id\"\n).join(\n    other_1.filter(pl.col(\"num_group1\") == 0\n).drop(\"num_group1\"), how=\"left\", on=\"case_id\"\n).join(\n    tax_registry_a_1.filter(pl.col(\"num_group1\") == 0\n).drop(\"num_group1\"), how=\"left\", on=\"case_id\"\n).join(\n    tax_registry_b_1.filter(pl.col(\"num_group1\") == 0\n).drop(\"num_group1\"), how=\"left\", on=\"case_id\"\n).join(\n    deposit_1.filter(pl.col(\"num_group1\") == 0\n).drop(\"num_group1\"), how=\"left\", on=\"case_id\"\n).join(\n    person_1.filter(pl.col(\"num_group1\") == 0\n).drop(\"num_group1\"), how=\"left\", on=\"case_id\"\n).join(\n    debitcard_1.filter(pl.col(\"num_group1\") == 0\n).drop(\"num_group1\"), how=\"left\", on=\"case_id\"\n).join(\n    applprev_2.filter(pl.col(\"num_group1\") == 0\n).drop(\"num_group1\").filter(pl.col(\"num_group2\") == 0\n).drop(\"num_group2\"), how=\"left\", on=\"case_id\"\n).join(\n    person_2.filter(pl.col(\"num_group1\") == 0\n).drop(\"num_group1\").filter(pl.col(\"num_group2\") == 0\n).drop(\"num_group2\"), how=\"left\", on=\"case_id\"\n).join(\n    credit_bureau_a_2.filter(pl.col(\"num_group1\") == 0\n).drop(\"num_group1\").filter(pl.col(\"num_group2\") == 0\n).drop(\"num_group2\"), how=\"left\", on=\"case_id\"\n).join(\n    credit_bureau_b_2.filter(pl.col(\"num_group1\") == 0\n).drop(\"num_group1\").filter(pl.col(\"num_group2\") == 0\n).drop(\"num_group2\"), how=\"left\", on=\"case_id\"\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_submission = test_basetable.join(\n    static_0, how=\"left\", on=\"case_id\"\n).join(\n    static_cb_0, how=\"left\", on=\"case_id\"\n).join(\n    applprev_1.filter(pl.col(\"num_group1\") == 0\n).drop(\"num_group1\"), how=\"left\", on=\"case_id\"\n).join(\n    other_1.filter(pl.col(\"num_group1\") == 0\n).drop(\"num_group1\"), how=\"left\", on=\"case_id\"\n).join(\n    tax_registry_a_1.filter(pl.col(\"num_group1\") == 0\n).drop(\"num_group1\"), how=\"left\", on=\"case_id\"\n).join(\n    tax_registry_b_1.filter(pl.col(\"num_group1\") == 0\n).drop(\"num_group1\"), how=\"left\", on=\"case_id\"\n).join(\n    deposit_1.filter(pl.col(\"num_group1\") == 0\n).drop(\"num_group1\"), how=\"left\", on=\"case_id\"\n).join(\n    person_1.filter(pl.col(\"num_group1\") == 0\n).drop(\"num_group1\"), how=\"left\", on=\"case_id\"\n).join(\n    debitcard_1.filter(pl.col(\"num_group1\") == 0\n).drop(\"num_group1\"), how=\"left\", on=\"case_id\"\n).join(\n    applprev_2.filter(pl.col(\"num_group1\") == 0\n).drop(\"num_group1\").filter(pl.col(\"num_group2\") == 0\n).drop(\"num_group2\"), how=\"left\", on=\"case_id\"\n).join(\n    person_2.filter(pl.col(\"num_group1\") == 0\n).drop(\"num_group1\").filter(pl.col(\"num_group2\") == 0\n).drop(\"num_group2\"), how=\"left\", on=\"case_id\"\n).join(\n    credit_bureau_a_2.filter(pl.col(\"num_group1\") == 0\n).drop(\"num_group1\").filter(pl.col(\"num_group2\") == 0\n).drop(\"num_group2\"), how=\"left\", on=\"case_id\"\n).join(\n    credit_bureau_b_2.filter(pl.col(\"num_group1\") == 0\n).drop(\"num_group1\").filter(pl.col(\"num_group2\") == 0\n).drop(\"num_group2\"), how=\"left\", on=\"case_id\"\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"case_ids = data[\"case_id\"].unique().shuffle(seed=1)\ncase_ids_train, case_ids_test = train_test_split(case_ids, train_size=0.6, random_state=1)\ncase_ids_valid, case_ids_test = train_test_split(case_ids_test, train_size=0.5, random_state=1)\n\ncols_pred = []\nfor col in data.columns:\n    if col[-1].isupper() and col[:-1].islower():\n        cols_pred.append(col)\n\n#print(cols_pred)\n\ndef from_polars_to_pandas(case_ids: pl.DataFrame) -> pl.DataFrame:\n    return (\n        data.filter(pl.col(\"case_id\").is_in(case_ids))[[\"case_id\", \"WEEK_NUM\", \"target\"]].to_pandas(),\n        data.filter(pl.col(\"case_id\").is_in(case_ids))[cols_pred].to_pandas(),\n        data.filter(pl.col(\"case_id\").is_in(case_ids))[\"target\"].to_pandas()\n    )\n\nbase_train, X_train, y_train = from_polars_to_pandas(case_ids_train)\nbase_valid, X_valid, y_valid = from_polars_to_pandas(case_ids_valid)\nbase_test, X_test, y_test = from_polars_to_pandas(case_ids_test)\n\nfor df in [X_train, X_valid, X_test]:\n    df = convert_strings(df)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import catboost as cb","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_features = X_train.columns[X_train.dtypes == 'category'].tolist()\n\npool_tr = cb.Pool(X_train, y_train, cat_features=cat_features)\npool_val = cb.Pool(X_valid, y_valid, cat_features=cat_features)\n\nparams = {\n    'objective': 'Logloss',\n    'learning_rate': 0.2,\n    'iterations': 300,\n    'verbose': True,\n    'random_seed': 911,\n    'eval_metric':'AUC',\n    'early_stopping_rounds': 5,\n    'metric_period':10\n}\nmodel = cb.CatBoost(params)\nmodel.fit(pool_tr, eval_set=pool_val)\n\nmodel.predict(pool_tr)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_submission = data_submission[cols_pred].to_pandas()\nX_submission = convert_strings(X_submission)\ncategorical_cols = X_train.select_dtypes(include=['category']).columns\n\nfor col in categorical_cols:\n    train_categories = set(X_train[col].cat.categories)\n    submission_categories = set(X_submission[col].cat.categories)\n    new_categories = submission_categories - train_categories\n    X_submission.loc[X_submission[col].isin(new_categories), col] = \"Unknown\"\n    new_dtype = pd.CategoricalDtype(categories=train_categories, ordered=True)\n    X_train[col] = X_train[col].astype(new_dtype)\n    X_submission[col] = X_submission[col].astype(new_dtype)\n\ny_submission_pred = model.predict(X_submission)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import roc_auc_score\ny_pred = model.predict(X_test)\nroc_auc_score(y_test, y_pred)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({\n    \"case_id\": data_submission[\"case_id\"].to_numpy(),\n    \"score\": y_submission_pred\n}).set_index('case_id')\nsubmission.to_csv(\"./submission.csv\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Интересно, такая модель дает score 0.87259 в контесте","metadata":{}},{"cell_type":"code","source":"import shap\nimport matplotlib.pyplot as plt\nexplainer = shap.TreeExplainer(model)\nshap_values = explainer(pool_val)\nshap.summary_plot(shap_values, X_test, show=False)\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fi = model.get_feature_importance(pool_val, prettified=True)\nfi","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"indices, scores = model.get_object_importance(\n    pool_val, pool_tr,\n    top_size=-1, # не влияет на скорость\n    type='PerObject', # не влияет на скорость\n    update_method='SinglePoint', # очень влияет на скорость! лучше SinglePoint :)\n    importance_values_sign='All', # не влияет на скорость\n    thread_count=16\n)\nindices, scores","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from local_manipulations import *\n\nt = train_model_get_t(X_tr, y_tr)\nplot_tree_info(t)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We need to use aggregation functions in tables with depth > 1, so tables that contain num_group1 column or \n# also num_group2 column.\ntrain_person_1_feats_1 = train_person_1.group_by(\"case_id\").agg(\n    pl.col(\"mainoccupationinc_384A\").max().alias(\"mainoccupationinc_384A_max\"),\n    (pl.col(\"incometype_1044T\") == \"SELFEMPLOYED\").max().alias(\"mainoccupationinc_384A_any_selfemployed\")\n)\n\n# Here num_group1=0 has special meaning, it is the person who applied for the loan.\ntrain_person_1_feats_2 = train_person_1.select([\"case_id\", \"num_group1\", \"housetype_905L\"]).filter(\n    pl.col(\"num_group1\") == 0\n).drop(\"num_group1\").rename({\"housetype_905L\": \"person_housetype\"})\n\n# Here we have num_goup1 and num_group2, so we need to aggregate again.\ntrain_credit_bureau_b_2_feats = train_credit_bureau_b_2.group_by(\"case_id\").agg(\n    pl.col(\"pmts_pmtsoverdue_635A\").max().alias(\"pmts_pmtsoverdue_635A_max\"),\n    (pl.col(\"pmts_dpdvalue_108P\") > 31).max().alias(\"pmts_dpdvalue_108P_over31\")\n)\n\n# We will process in this examples only A-type and M-type columns, so we need to select them.\nselected_static_cols = []\nfor col in train_static.columns:\n    if col[-1] in (\"A\", \"M\"):\n        selected_static_cols.append(col)\nprint(selected_static_cols)\n\nselected_static_cb_cols = []\nfor col in train_static_cb.columns:\n    if col[-1] in (\"A\", \"M\"):\n        selected_static_cb_cols.append(col)\nprint(selected_static_cb_cols)\n\n# Join all tables together.\ndata = train_basetable.join(\n    train_static.select([\"case_id\"]+selected_static_cols), how=\"left\", on=\"case_id\"\n).join(\n    train_static_cb.select([\"case_id\"]+selected_static_cb_cols), how=\"left\", on=\"case_id\"\n).join(\n    train_person_1_feats_1, how=\"left\", on=\"case_id\"\n).join(\n    train_person_1_feats_2, how=\"left\", on=\"case_id\"\n).join(\n    train_credit_bureau_b_2_feats, how=\"left\", on=\"case_id\"\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"desc = feature_definitions.filter(feature_definitions['Variable'].is_in(static_0.columns))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(len(desc)):\n    print(desc[i]['Variable'][0],':',desc[i]['Description'][0])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from utils import *","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_rows', 500)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_df_info(static_0.to_pandas())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Много фичей с высоким trash_score, но так как их слишком много, то не будем ничего менять и дадим возможность бустингу с этим всем разобраться","metadata":{}},{"cell_type":"code","source":"desc = feature_definitions.filter(feature_definitions['Variable'].is_in(static_cb_0.columns))\nfor i in range(len(desc)):\n    print(desc[i]['Variable'][0],':',desc[i]['Description'][0])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_df_info(static_cb_0.to_pandas())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Аналогичная ситуация, идем дальше (много фичей с высоким trash_score, но так как их слишком много, то не будем ничего менять и дадим возможность бустингу с этим всем разобраться)","metadata":{}},{"cell_type":"code","source":"desc = feature_definitions.filter(feature_definitions['Variable'].is_in(applprev_1.columns))\nfor i in range(len(desc)):\n    print(desc[i]['Variable'][0],':',desc[i]['Description'][0])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_df_info(applprev_1.to_pandas())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Посмотрим для примера какой-нибудь case_id","metadata":{}},{"cell_type":"code","source":"applprev_1.filter(pl.col('case_id')==2703453)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Похоже, в данном случае num_group1 отвечает за предыдущие заявки\n\nТеперь поймем, какой признак хотим оставить: так как предсказываем на будущее, то большая склонность к выбору более поздних значений, однако имеет смысл в зависимости от признака будем брать максимум / среднее / ненулевое значение","metadata":{}},{"cell_type":"code","source":"applprev_1_agg = applprev_1.group_by(\"case_id\").agg(\n    pl.len().alias(\"num_applprev1\"),\n    pl.col(\"actualdpd_943P\").max().alias(\"actualdpd_943P_max\"),\n    pl.col(\"annuity_853A\").max().alias(\"annuity_853A_max\"),\n    pl.col(\"annuity_853A\").mean().alias(\"annuity_853A_mean\"),\n    pl.col(\"approvaldate_319D\").min().alias(\"approvaldate_319D_min\"),\n    pl.col(\"approvaldate_319D\").max().alias(\"approvaldate_319D_max\"),\n    pl.col('byoccupationinc_3656910L').drop_nulls().first().alias(\"byoccupationinc_3656910L_first\"),\n    pl.col('cancelreason_3545846M').mode().first().alias(\"cancelreason_3545846M_mode\"),\n    pl.col('childnum_21L').drop_nulls().first().alias(\"childnum_21L_first\"),\n    (pl.col(\"approvaldate_319D\") - pl.col('creationdate_885D')).max().alias(\"process_time_max\"),\n    (pl.col(\"approvaldate_319D\") - pl.col('creationdate_885D')).min().alias(\"process_time_min\"),\n    pl.col('credacc_actualbalance_314A').drop_nulls().first().alias(\"credacc_actualbalance_314A_first\"),\n    pl.col('credacc_credlmt_575A').drop_nulls().first().alias(\"credacc_credlmt_575A_first\"),\n    pl.col('credacc_maxhisbal_375A').drop_nulls().first().alias(\"credacc_maxhisbal_375A_first\"),\n    pl.col('credacc_minhisbal_90A').drop_nulls().first().alias(\"credacc_minhisbal_90A_first\"),\n    pl.col('credacc_status_367L').drop_nulls().first().alias(\"credacc_status_367L_first\"),\n    pl.col('credacc_transactions_402L').sum().alias(\"credacc_transactions_402L_sum\"),\n    pl.col('credamount_590A').sum().alias(\"credamount_590A_sum\"),\n    pl.col('credamount_590A').mean().alias(\"credamount_590A_mean\"),    \n    pl.col('credtype_587L').mode().first().alias(\"credtype_587L_mode\"),   \n    pl.col('currdebt_94A').max().alias(\"currdebt_94A_max\"),\n    pl.col('district_544M').drop_nulls().first().alias(\"district_544M_first\"),\n    pl.col('downpmt_134A').max().alias(\"downpmt_134A_max\"),\n    pl.col('education_1138M').drop_nulls().first().alias(\"education_1138M_first\"),\n    pl.col('employedfrom_700D').drop_nulls().first().alias(\"employedfrom_700D_first\"),\n    pl.col('familystate_726L').drop_nulls().first().alias(\"familystate_726L_first\"),\n    (-pl.col(\"approvaldate_319D\") + pl.col('firstnonzeroinstldate_307D')).max().alias(\"time_to_first_instal\"),\n    pl.col('inittransactioncode_279L').mode().first().alias(\"inittransactioncode_279L_mode\"),\n    pl.col('isbidproduct_390L').mode().first().alias(\"isbidproduct_390L_mode\"),\n    pl.col('isdebitcard_527L').mode().first().alias(\"isdebitcard_527L_mode\"),\n    pl.col('mainoccupationinc_437A').drop_nulls().first().alias(\"mainoccupationinc_437A_first\"),\n    pl.col('maxdpdtolerance_577P').max().alias('maxdpdtolerance_577P_max'),\n    pl.col('outstandingdebt_522A').max().alias('outstandingdebt_522A_max'),\n    pl.col('pmtnum_8L').sum().alias('pmtnum_8L_sum'),\n    pl.col('pmtnum_8L').mean().alias('pmtnum_8L_mean'),\n    pl.col('postype_4733339M').mode().first().alias(\"postype_4733339M_mode\"),\n    pl.col('postype_4733339M').drop_nulls().first().alias(\"postype_4733339M_first\"),\n    pl.col('profession_152M').drop_nulls().first().alias(\"profession_152M_first\"),\n    pl.col('rejectreason_755M').mode().first().alias(\"rejectreason_755M_mode\"),\n    pl.col('rejectreason_755M').first().alias(\"rejectreason_755M_first\"),\n    pl.col('revolvingaccount_394A').max().alias('revolvingaccount_394A_max'),\n    pl.col('status_219L').mode().first().alias(\"status_219L_755M_mode\"),\n    pl.col('status_219L').first().alias(\"status_219L_755M_first\"),\n    # (pl.col('pmtnum_8L') - pl.col('tenor_203L')).max().alias('difference_in_payments_max') оказывается, всегда ноль\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"applprev_1_agg","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_df_info(applprev_1_agg.to_pandas())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"В целом, фичи стали заметно лучше","metadata":{}},{"cell_type":"code","source":"desc = feature_definitions.filter(feature_definitions['Variable'].is_in(other_1.columns))\nfor i in range(len(desc)):\n    print(desc[i]['Variable'][0],':',desc[i]['Description'][0])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_df_info(other_1.to_pandas())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"В этом случае num_group1 всегда равна нулю, поэтому можно только добавить новые фичи","metadata":{}},{"cell_type":"code","source":"other_1_agg = other_1.drop('num_group1').with_columns(\n    [\n        (pl.col('amtdepositincoming_4809444A') - pl.col('amtdepositoutgoing_4809442A')).alias(\"amtdeposit_diff\"),\n        (pl.col('amtdebitincoming_4809443A') - pl.col('amtdebitoutgoing_4809440A')).alias(\"amtdebit_diff\"),\n    ]\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"other_1_agg.corr()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Корреляция есть, но не слишком высокая, поэтому такие фичи оправданы","metadata":{}},{"cell_type":"code","source":"desc = feature_definitions.filter(feature_definitions['Variable'].is_in(tax_registry_a_1.columns))\nfor i in range(len(desc)):\n    print(desc[i]['Variable'][0],':',desc[i]['Description'][0])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_df_info(tax_registry_a_1.to_pandas())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Теперь пытаемся понять, что означает num_group1 в этом случае","metadata":{}},{"cell_type":"code","source":"tax_registry_a_1.filter(pl.col('case_id')==1819728)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Похоже, что это все налоговые вычеты, отслеживаемые государством для человека, подающего заявку: везде имя работодателя совпадает. Логично их просуммировать, а также посмотреть на период, за который они известны. Также можем взять моду по работодателю. ","metadata":{}},{"cell_type":"code","source":"tax_registry_a_1_agg = tax_registry_a_1.group_by(\"case_id\").agg(\n    pl.len().alias(\"tax_registry_a_1_len\"),\n    pl.col('amount_4527230A').sum().alias('amount_4527230A_sum'),\n    pl.col('amount_4527230A').mean().alias('amount_4527230A_mean'),\n    pl.col('recorddate_4527225D').min().alias('recorddate_4527225D_min'),\n    pl.col('recorddate_4527225D').max().alias('recorddate_4527225D_max'),\n    (pl.col('recorddate_4527225D').max() - pl.col('recorddate_4527225D').min()).alias('recorddate_4527225D_diff'),\n    pl.col('name_4527232M').mode().first().alias('name_4527232M_mode'),\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"desc = feature_definitions.filter(feature_definitions['Variable'].is_in(tax_registry_b_1.columns))\nfor i in range(len(desc)):\n    print(desc[i]['Variable'][0],':',desc[i]['Description'][0])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_df_info(tax_registry_b_1.to_pandas())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tax_registry_b_1.filter(pl.col('case_id')==2703452)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Агрегируем аналогичным образом","metadata":{}},{"cell_type":"code","source":"tax_registry_b_1_agg = tax_registry_b_1.group_by(\"case_id\").agg(\n    pl.len().alias(\"tax_registry_b_1_len\"),\n    pl.col('amount_4917619A').sum().alias('amount_4917619A_sum'),\n    pl.col('amount_4917619A').mean().alias('amount_4917619A_mean'),\n    pl.col('deductiondate_4917603D').min().alias('deductiondate_4917603D_min'),\n    pl.col('deductiondate_4917603D').max().alias('deductiondate_4917603D_max'),\n    (pl.col('deductiondate_4917603D').max() - pl.col('deductiondate_4917603D').min()).alias('deductiondate_4917603D_diff'),\n    pl.col('name_4917606M').mode().first().alias('name_4917606M_mode'),\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"desc = feature_definitions.filter(feature_definitions['Variable'].is_in(tax_registry_c_1.columns))\nfor i in range(len(desc)):\n    print(desc[i]['Variable'][0],':',desc[i]['Description'][0])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_df_info(tax_registry_c_1.to_pandas())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tax_registry_c_1","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tax_registry_c_1.filter(pl.col('case_id')==357)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"И еще раз делаем так же, чтобы получить хорошие признаки","metadata":{}},{"cell_type":"code","source":"tax_registry_с_1_agg = tax_registry_c_1.group_by(\"case_id\").agg(\n    pl.len().alias(\"tax_registry_с_1_len\"),\n    pl.col('pmtamount_36A').sum().alias('pmtamount_36A_sum'),\n    pl.col('pmtamount_36A').mean().alias('pmtamount_36A_mean'),\n    pl.col('processingdate_168D').min().alias('processingdate_168D_min'),\n    pl.col('processingdate_168D').max().alias('processingdate_168D_max'),\n    (pl.col('processingdate_168D').max() - pl.col('processingdate_168D').min()).alias('processingdate_168D_diff'),\n    pl.col('employername_160M').mode().first().alias('employername_160M_mode'),\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tax_registry_с_1_agg","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"desc = feature_definitions.filter(feature_definitions['Variable'].is_in(credit_bureau_a_1.columns))\nfor i in range(len(desc)):\n    print(desc[i]['Variable'][0],':',desc[i]['Description'][0])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_df_info(credit_bureau_a_1.to_pandas())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"credit_bureau_a_1","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"credit_bureau_a_1.filter(pl.col('case_id')==388)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Также для простоты оставим только первую (самую свежую) непустую запись по каждому клиенту","metadata":{}},{"cell_type":"code","source":"credit_bureau_a_1_agg = credit_bureau_a_1.group_by(\"case_id\").agg(\n   pl.len().alias(\"credit_bureau_a_1_len\"),    \n   pl.exclude(\"num_group1\").drop_nulls().first().name.suffix(\"_first\")\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"desc = feature_definitions.filter(feature_definitions['Variable'].is_in(credit_bureau_b_1.columns))\nfor i in range(len(desc)):\n    print(desc[i]['Variable'][0],':',desc[i]['Description'][0])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_df_info(credit_bureau_b_1.to_pandas())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"credit_bureau_b_1_agg = credit_bureau_b_1.group_by(\"case_id\").agg(\n   pl.len().alias(\"credit_bureau_b_1_len\"),    \n   pl.exclude(\"num_group1\").drop_nulls().first().name.suffix(\"_first\")\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"desc = feature_definitions.filter(feature_definitions['Variable'].is_in(deposit_1.columns))\nfor i in range(len(desc)):\n    print(desc[i]['Variable'][0],':',desc[i]['Description'][0])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_df_info(deposit_1.to_pandas())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"deposit_1","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"deposit_1.filter(pl.col('case_id')==2703430)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Похоже, что это разные счета одного и того же человека, поэтому берем похожие характеристики, как с налогами","metadata":{}},{"cell_type":"code","source":"deposit_1_agg = deposit_1.group_by(\"case_id\").agg(\n    pl.len().alias(\"tax_registry_с_1_len\"),\n    pl.col('amount_416A').sum().alias('amount_416A_sum'),\n    pl.col('amount_416A').mean().alias('amount_416A_mean'),\n    pl.col('openingdate_313D').min().alias('openingdate_313D_min'),\n    pl.col('openingdate_313D').max().alias('openingdate_313D_max'),\n    (pl.col('contractenddate_991D') - pl.col('openingdate_313D')).max().alias('contractenddate_991D_max_diff'),\n    (pl.col('contractenddate_991D') - pl.col('openingdate_313D')).mean().alias('contractenddate_991D_mean_diff'),\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"deposit_1_agg","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"desc = feature_definitions.filter(feature_definitions['Variable'].is_in(person_1.columns))\nfor i in range(len(desc)):\n    print(desc[i]['Variable'][0],':',desc[i]['Description'][0])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_df_info(person_1.to_pandas())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Здесь для простоты берем только numgroup1 = 0, то есть человека, который берет кредит","metadata":{}},{"cell_type":"code","source":"person_1_agg = person_1.filter(pl.col(\"num_group1\") == 0).drop(\"num_group1\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"desc = feature_definitions.filter(feature_definitions['Variable'].is_in(debitcard_1.columns))\nfor i in range(len(desc)):\n    print(desc[i]['Variable'][0],':',desc[i]['Description'][0])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_df_info(debitcard_1.to_pandas())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"debitcard_1","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"debitcard_1.filter(pl.col('case_id')==2703430)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"debitcard_1_agg = debitcard_1.group_by(\"case_id\").agg(\n    pl.len().alias(\"debitcard_1_agg_len\"),\n    pl.col('last180dayaveragebalance_704A').mean().alias('last180dayaveragebalance_704A_mean'),\n    pl.col('last180dayturnover_1134A').mean().alias('last180dayturnover_1134A_mean'),\n    pl.col('last30dayturnover_651A').mean().alias('last30dayturnover_651A_mean'),\n    pl.col('openingdate_857D').min().alias('openingdate_857D_min'),\n    pl.col('openingdate_857D').max().alias('openingdate_857D_max'),\n    (pl.col('openingdate_857D').max() - pl.col('openingdate_857D').min()).alias('openingdate_857D_991D_diff')\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_df_info(debitcard_1_agg.to_pandas())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = train_basetable.join(\n    static_0, how=\"left\", on=\"case_id\"\n).join(\n    static_cb_0, how=\"left\", on=\"case_id\"\n).join(\n    applprev_1, how='left', on=\"case_id\"\n).with_columns(\n    ( 2*((pl.col('creationdate_885D') < pl.col('date_decision')).cast(pl.Int64)-0.5)*pl.col('creationdate_885D') ).alias('sort')\n).group_by('case_id').agg([\n    pl.len().alias(\"applprev_1_len\"),\n    pl.all().sort_by('date_decision', -pl.col('num_group1'), nulls_last=False).last(),\n]).drop(\n    ['sort','num_group1']\n).join(\n    other_1, how='left', on=\"case_id\"\n).drop('num_group1').join(\n    tax_registry_a_1, how='left', on=\"case_id\"\n).with_columns(\n    ( 2*((pl.col('recorddate_4527225D') < pl.col('date_decision')).cast(pl.Int64)-0.5)*pl.col('recorddate_4527225D') ).alias('sort')\n).group_by('case_id').agg([\n    pl.len().alias(\"tax_registry_a_1_len\"),\n    pl.all().sort_by('date_decision', -pl.col('num_group1'), nulls_last=False).last(),\n]).drop(\n    ['sort','num_group1']\n).join(\n    tax_registry_b_1, how='left', on=\"case_id\"\n).with_columns(\n    ( 2*((pl.col('deductiondate_4917603D') < pl.col('date_decision')).cast(pl.Int64)-0.5)*pl.col('deductiondate_4917603D') ).alias('sort')\n).group_by('case_id').agg([\n    pl.len().alias(\"tax_registry_b_1_len\"),\n    pl.all().sort_by('date_decision', -pl.col('num_group1'), nulls_last=False).last(),\n]).drop(\n    ['sort','num_group1']\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = data.join(\n    tax_registry_c_1, how='left', on=\"case_id\"\n).with_columns(\n    ( 2*((pl.col('processingdate_168D') < pl.col('date_decision')).cast(pl.Int64)-0.5)*pl.col('processingdate_168D') ).alias('sort')\n).group_by('case_id').agg([\n    pl.len().alias(\"tax_registry_c_1_len\"),\n    pl.all().sort_by('date_decision', -pl.col('num_group1'), nulls_last=False).last(),\n]).drop(\n    ['sort','num_group1']\n).join(\n    credit_bureau_a_1, how='left', on=\"case_id\"\n).with_columns(\n    ( 2*((pl.col('dateofcredstart_739D') < pl.col('date_decision')).cast(pl.Int64)-0.5)*pl.col('dateofcredstart_739D') ).alias('sort')\n).group_by('case_id').agg([\n    pl.len().alias(\"credit_bureau_a_1_len\"),\n    pl.all().sort_by('date_decision', -pl.col('num_group1'), nulls_last=False).last(),\n]).drop(\n    ['sort','num_group1']\n).join(\n    credit_bureau_b_1, how='left', on=\"case_id\"\n).with_columns(\n    ( 2*((pl.col('contractdate_551D') < pl.col('date_decision')).cast(pl.Int64)-0.5)*pl.col('contractdate_551D') ).alias('sort')\n).group_by('case_id').agg([\n    pl.len().alias(\"credit_bureau_b_1_len\"),\n    pl.all().sort_by('date_decision', -pl.col('num_group1'), nulls_last=False).last(),\n]).drop(\n    ['sort','num_group1']\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = data.join(\n    deposit_1, how='left', on=\"case_id\"\n).with_columns(\n    ( 2*((pl.col('openingdate_313D') < pl.col('date_decision')).cast(pl.Int64)-0.5)*pl.col('openingdate_313D') ).alias('sort')\n).group_by('case_id').agg([\n    pl.len().alias(\"deposit_1_len\"),\n    pl.all().sort_by('date_decision', -pl.col('num_group1'), nulls_last=False).last(),\n]).drop(\n    ['sort','num_group1']\n).join(\n    person_1.filter(pl.col(\"num_group1\") == 0\n).drop(\"num_group1\"), how=\"left\", on=\"case_id\"\n).join(\n    debitcard_1, how='left', on=\"case_id\"\n).with_columns(\n    ( 2*((pl.col('openingdate_857D') < pl.col('date_decision')).cast(pl.Int64)-0.5)*pl.col('openingdate_857D') ).alias('sort')\n).group_by('case_id').agg([\n    pl.len().alias(\"debitcard_1_len\"),\n    pl.all().sort_by('date_decision', -pl.col('num_group1'), nulls_last=False).last(),\n]).drop(\n    ['sort','num_group1']\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = data.join(\n    applprev_2.filter(pl.col(\"num_group1\") == 0\n).drop(\"num_group1\").filter(pl.col(\"num_group2\") == 0\n).drop(\"num_group2\"), how=\"left\", on=\"case_id\"\n).join(\n    person_2.filter(pl.col(\"num_group1\") == 0\n).drop(\"num_group1\").filter(pl.col(\"num_group2\") == 0\n).drop(\"num_group2\"), how=\"left\", on=\"case_id\"\n).join(\n    credit_bureau_a_2.filter(pl.col(\"num_group1\") == 0\n).drop(\"num_group1\").filter(pl.col(\"num_group2\") == 0\n).drop(\"num_group2\"), how=\"left\", on=\"case_id\"\n).join(\n    credit_bureau_b_2, how='left', on=\"case_id\"\n).with_columns(\n    ( 2*((pl.col('pmts_date_1107D') < pl.col('date_decision')).cast(pl.Int64)-0.5)*pl.col('pmts_date_1107D') ).alias('sort')\n).group_by('case_id').agg([\n    pl.len().alias(\"credit_bureau_b_2_len\"),\n    pl.all().sort_by('date_decision', -pl.col('num_group1'),-pl.col('num_group2'), nulls_last=False).last(),\n]).drop(\n    ['sort','num_group1']\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_submission = test_basetable.join(\n    static_0, how=\"left\", on=\"case_id\"\n).join(\n    static_cb_0, how=\"left\", on=\"case_id\"\n).join(\n    applprev_1, how='left', on=\"case_id\"\n).with_columns(\n    ( 2*((pl.col('creationdate_885D') < pl.col('date_decision')).cast(pl.Int64)-0.5)*pl.col('creationdate_885D') ).alias('sort')\n).group_by('case_id').agg([\n    pl.len().alias(\"applprev_1_len\"),\n    pl.all().sort_by('date_decision', -pl.col('num_group1'), nulls_last=False).last(),\n]).drop(\n    ['sort','num_group1']\n).join(\n    other_1, how='left', on=\"case_id\"\n).drop('num_group1').join(\n    tax_registry_a_1, how='left', on=\"case_id\"\n).with_columns(\n    ( 2*((pl.col('recorddate_4527225D') < pl.col('date_decision')).cast(pl.Int64)-0.5)*pl.col('recorddate_4527225D') ).alias('sort')\n).group_by('case_id').agg([\n    pl.len().alias(\"tax_registry_a_1_len\"),\n    pl.all().sort_by('date_decision', -pl.col('num_group1'), nulls_last=False).last(),\n]).drop(\n    ['sort','num_group1']\n).join(\n    tax_registry_b_1, how='left', on=\"case_id\"\n).with_columns(\n    ( 2*((pl.col('deductiondate_4917603D') < pl.col('date_decision')).cast(pl.Int64)-0.5)*pl.col('deductiondate_4917603D') ).alias('sort')\n).group_by('case_id').agg([\n    pl.len().alias(\"tax_registry_b_1_len\"),\n    pl.all().sort_by('date_decision', -pl.col('num_group1'), nulls_last=False).last(),\n]).drop(\n    ['sort','num_group1']\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_submission = data_submission.join(\n    tax_registry_c_1, how='left', on=\"case_id\"\n).with_columns(\n    ( 2*((pl.col('processingdate_168D') < pl.col('date_decision')).cast(pl.Int64)-0.5)*pl.col('processingdate_168D') ).alias('sort')\n).group_by('case_id').agg([\n    pl.len().alias(\"tax_registry_c_1_len\"),\n    pl.all().sort_by('date_decision', -pl.col('num_group1'), nulls_last=False).last(),\n]).drop(\n    ['sort','num_group1']\n).join(\n    credit_bureau_a_1, how='left', on=\"case_id\"\n).with_columns(\n    ( 2*((pl.col('dateofcredstart_739D') < pl.col('date_decision')).cast(pl.Int64)-0.5)*pl.col('dateofcredstart_739D') ).alias('sort')\n).group_by('case_id').agg([\n    pl.len().alias(\"credit_bureau_a_1_len\"),\n    pl.all().sort_by('date_decision', -pl.col('num_group1'), nulls_last=False).last(),\n]).drop(\n    ['sort','num_group1']\n).join(\n    credit_bureau_b_1, how='left', on=\"case_id\"\n).with_columns(\n    ( 2*((pl.col('contractdate_551D') < pl.col('date_decision')).cast(pl.Int64)-0.5)*pl.col('contractdate_551D') ).alias('sort')\n).group_by('case_id').agg([\n    pl.len().alias(\"credit_bureau_b_1_len\"),\n    pl.all().sort_by('date_decision', -pl.col('num_group1'), nulls_last=False).last(),\n]).drop(\n    ['sort','num_group1']\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_submission = data_submission.join(\n    deposit_1, how='left', on=\"case_id\"\n).with_columns(\n    ( 2*((pl.col('openingdate_313D') < pl.col('date_decision')).cast(pl.Int64)-0.5)*pl.col('openingdate_313D') ).alias('sort')\n).group_by('case_id').agg([\n    pl.len().alias(\"deposit_1_len\"),\n    pl.all().sort_by('date_decision', -pl.col('num_group1'), nulls_last=False).last(),\n]).drop(\n    ['sort','num_group1']\n).join(\n    person_1.filter(pl.col(\"num_group1\") == 0\n).drop(\"num_group1\"), how=\"left\", on=\"case_id\"\n).join(\n    debitcard_1, how='left', on=\"case_id\"\n).with_columns(\n    ( 2*((pl.col('openingdate_857D') < pl.col('date_decision')).cast(pl.Int64)-0.5)*pl.col('openingdate_857D') ).alias('sort')\n).group_by('case_id').agg([\n    pl.len().alias(\"debitcard_1_len\"),\n    pl.all().sort_by('date_decision', -pl.col('num_group1'), nulls_last=False).last(),\n]).drop(\n    ['sort','num_group1']\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_submission = data_submission.join(\n    applprev_2.filter(pl.col(\"num_group1\") == 0\n).drop(\"num_group1\").filter(pl.col(\"num_group2\") == 0\n).drop(\"num_group2\"), how=\"left\", on=\"case_id\"\n).join(\n    person_2.filter(pl.col(\"num_group1\") == 0\n).drop(\"num_group1\").filter(pl.col(\"num_group2\") == 0\n).drop(\"num_group2\"), how=\"left\", on=\"case_id\"\n).join(\n    credit_bureau_a_2.filter(pl.col(\"num_group1\") == 0\n).drop(\"num_group1\").filter(pl.col(\"num_group2\") == 0\n).drop(\"num_group2\"), how=\"left\", on=\"case_id\"\n).join(\n    credit_bureau_b_2, how='left', on=\"case_id\"\n).with_columns(\n    ( 2*((pl.col('pmts_date_1107D') < pl.col('date_decision')).cast(pl.Int64)-0.5)*pl.col('pmts_date_1107D') ).alias('sort')\n).group_by('case_id').agg([\n    pl.len().alias(\"credit_bureau_b_2_len\"),\n    pl.all().sort_by('date_decision', -pl.col('num_group1'),-pl.col('num_group2'), nulls_last=False).last(),\n]).drop(\n    ['sort','num_group1']\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train the model","metadata":{}},{"cell_type":"code","source":"case_ids = data[\"case_id\"].unique().shuffle(seed=1)\ncase_ids_train, case_ids_test = train_test_split(case_ids, train_size=0.9, random_state=1)\ncase_ids_valid, case_ids_test = train_test_split(case_ids_test, train_size=0.9, random_state=1)\n\ncols_pred = []\nfor col in data.columns:\n    if col[-1].isupper() and col[:-1].islower():\n        cols_pred.append(col)\n\n#print(cols_pred)\n\ndef from_polars_to_pandas(case_ids: pl.DataFrame) -> pl.DataFrame:\n    return (\n        data.filter(pl.col(\"case_id\").is_in(case_ids))[[\"case_id\", \"WEEK_NUM\", \"target\"]].to_pandas(),\n        data.filter(pl.col(\"case_id\").is_in(case_ids))[cols_pred].to_pandas(),\n        data.filter(pl.col(\"case_id\").is_in(case_ids))[\"target\"].to_pandas()\n    )\n\nbase_train, X_train, y_train = from_polars_to_pandas(case_ids_train)\nbase_valid, X_valid, y_valid = from_polars_to_pandas(case_ids_valid)\nbase_test, X_test, y_test = from_polars_to_pandas(case_ids_test)\n\nfor df in [X_train, X_valid, X_test]:\n    df = convert_strings(df)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import catboost as cb","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_features = X_train.columns[X_train.dtypes == 'category'].tolist()\n\npool_tr = cb.Pool(X_train, y_train, cat_features=cat_features)\npool_val = cb.Pool(X_valid, y_valid, cat_features=cat_features)\n\nparams = {\n    'objective': 'Logloss',\n    'learning_rate': 0.5,\n    'iterations': 500,\n    'verbose': True,\n    'random_seed': 911,\n    'eval_metric':'AUC',\n    'early_stopping_rounds': 5,\n    'metric_period':10\n}\nmodel = cb.CatBoost(params)\nmodel.fit(pool_tr, eval_set=pool_val)\n\nmodel.predict(pool_tr)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for base, X in [(base_train, X_train), (base_valid, X_valid)]:\n    y_pred = model.predict(X)\n    base[\"score\"] = y_pred\n\ndef gini_stability(base, w_fallingrate=88.0, w_resstd=-0.5):\n    gini_in_time = base.loc[:, [\"WEEK_NUM\", \"target\", \"score\"]]\\\n        .sort_values(\"WEEK_NUM\")\\\n        .groupby(\"WEEK_NUM\")[[\"target\", \"score\"]]\\\n        .apply(lambda x: 2*roc_auc_score(x[\"target\"], x[\"score\"])-1).tolist()\n    \n    x = np.arange(len(gini_in_time))\n    y = gini_in_time\n    a, b = np.polyfit(x, y, 1)\n    y_hat = a*x + b\n    residuals = y - y_hat\n    res_std = np.std(residuals)\n    avg_gini = np.mean(gini_in_time)\n    return avg_gini + w_fallingrate * min(0, a) + w_resstd * res_std\n\nstability_score_train = gini_stability(base_train)\nstability_score_valid = gini_stability(base_valid)\n\n\nprint(f'The stability score on the train set is: {stability_score_train}') \nprint(f'The stability score on the valid set is: {stability_score_valid}') ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_submission","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_submission.select_dtypes(include=['category']).columns","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.select_dtypes(include=['category']).columns","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_submission.iloc[:,142].fillna('')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_submission = data_submission[cols_pred].to_pandas()\nX_submission.iloc[:,142] = X_submission.iloc[:,142].astype(str)\nX_submission = convert_strings(X_submission)\ncategorical_cols = X_train.select_dtypes(include=['category']).columns\n#X_submission = X_submission.astype(X_train.dtypes)\nfor col in categorical_cols:\n    train_categories = set(X_train[col].cat.categories)\n    submission_categories = set(X_submission[col].cat.categories)\n    new_categories = submission_categories - train_categories\n    X_submission.loc[X_submission[col].isin(new_categories), col] = \"Unknown\"\n    new_dtype = pd.CategoricalDtype(categories=train_categories, ordered=True)\n    X_train[col] = X_train[col].astype(new_dtype)\n    X_submission[col] = X_submission[col].astype(new_dtype)\n\ny_submission_pred = model.predict(X_submission)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({\n    \"case_id\": data_submission[\"case_id\"].to_numpy(),\n    \"score\": y_submission_pred\n}).set_index('case_id')\nsubmission.to_csv(\"./submission.csv\")","metadata":{},"execution_count":null,"outputs":[]}]}