{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"},{"sourceId":7869941,"sourceType":"datasetVersion","datasetId":4617786},{"sourceId":7870374,"sourceType":"datasetVersion","datasetId":4618074},{"sourceId":7885740,"sourceType":"datasetVersion","datasetId":4629105}],"dockerImageVersionId":30664,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install /kaggle/input/lgbm4-3-0/lightgbm-4.3.0-py3-none-manylinux_2_28_x86_64.whl\n!pip install /kaggle/input/sk1-4-1/scikit_learn-1.4.1.post1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl","metadata":{"execution":{"iopub.status.busy":"2024-03-21T12:39:26.909641Z","iopub.execute_input":"2024-03-21T12:39:26.909916Z","iopub.status.idle":"2024-03-21T12:40:37.317797Z","shell.execute_reply.started":"2024-03-21T12:39:26.909890Z","shell.execute_reply":"2024-03-21T12:40:37.316662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import polars as pl\ndef set_table_dtypes(df: pl.DataFrame) -> pl.DataFrame:\n    for col in df.columns:\n        if col[-1] == \"D\":\n            df = df.with_columns(pl.col(col).str.strptime(pl.Date, \"%Y-%m-%d\").alias(col))\n        elif col[-1] in [\"M\", \"T\"]:\n            df = df.with_columns(pl.col(col).cast(pl.Utf8).alias(col))\n        elif col[-1] in [\"P\", \"A\"]:\n            df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n        elif col[-1] == \"L\":\n            non_null_values = df.filter(pl.col(col).is_not_null())[col].limit(1).to_list()\n            if non_null_values:\n                first_non_null = non_null_values[0]\n                if isinstance(first_non_null, bool):\n                    df = df.with_columns(pl.col(col).cast(pl.Boolean).alias(col))\n                elif isinstance(first_non_null, (float, int)):\n                    df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n                else:\n                    df = df.with_columns(pl.col(col).cast(pl.Utf8).alias(col))\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-03-21T12:40:37.319883Z","iopub.execute_input":"2024-03-21T12:40:37.320184Z","iopub.status.idle":"2024-03-21T12:40:37.538521Z","shell.execute_reply.started":"2024-03-21T12:40:37.320159Z","shell.execute_reply":"2024-03-21T12:40:37.537779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\n\n# join train data\n\ntrain_basetable = pl.read_csv(\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_base.csv\")\ndef process_week_num(week_num):\n    return week_num % 52 if week_num >= 52 else week_num\ntrain_basetable = train_basetable.with_columns(\n    pl.col(\"WEEK_NUM\").map_elements(process_week_num).alias(\"WEEK_NUM\")\n)\ntrain_basetable = train_basetable.drop(['date_decision','MONTH'])\n\n\ntrain_static = pl.concat(\n    [\n        pl.read_csv(\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_static_0_0.csv\").pipe(set_table_dtypes),\n        pl.read_csv(\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_static_0_1.csv\").pipe(set_table_dtypes)\n    ],\n    how=\"vertical_relaxed\",\n)\ncolumns_of_interest = [\n    'case_id',\n    'price_1097A',\n    'numrejects9m_859L',\n    'pctinstlsallpaidlate1d_3546856L',\n    'maininc_215A',\n    'maxdpdfrom6mto36m_3546853P',\n    'numinstlsallpaid_934L',\n    'lastapprcredamount_781A',\n    'totalsettled_863A',\n    'maxannuity_159A',\n    'disbursedcredamount_1113A',\n    'inittransactionamount_650A',\n    'maxdpdlast24m_143P',\n    'maxdpdlast12m_727P',\n    'avgdpdtolclosure24_3658938P',\n    'numinstlswithdpd10_728L',\n    'annuity_780A',\n    'maxdpdtolerance_374P',\n    'credamount_770A',\n    'maxdebt4_972A',\n    'eir_270L'\n]\ntrain_static = train_static[columns_of_interest]\ntrain_basetable = train_basetable.join(train_static, on='case_id', how='left')\ndel columns_of_interest,train_static\ngc.collect()\n\n\ntrain_person_1 = (\n    pl.read_csv(\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_person_1.csv\").pipe(set_table_dtypes)\n    .select(['case_id', 'mainoccupationinc_384A', 'num_group1'])\n    .filter(pl.col('num_group1') == 0)\n    .drop('num_group1')\n)\ntrain_basetable = train_basetable.join(train_person_1, on='case_id', how='left')\ndel train_person_1\ngc.collect()\n\ntrain_other_1 = pl.read_csv(\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_other_1.csv\").pipe(set_table_dtypes)\ntrain_other_1 = (\n    train_other_1\n    .select(['case_id', 'amtdepositbalance_4809441A','amtdebitincoming_4809443A','amtdebitoutgoing_4809440A','num_group1'])\n    .filter(pl.col('num_group1') == 0)\n    .drop('num_group1')\n)\ntrain_basetable = train_basetable.join(train_other_1, on='case_id', how='left')\ndel train_other_1\ngc.collect()\n\ntrain_deposit_1 = pl.read_csv(\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_deposit_1.csv\").pipe(set_table_dtypes)\ndeposit_sums = (\n    train_deposit_1\n    .group_by('case_id')\n    .agg(pl.sum('amount_416A').alias('deposit'))\n)\ntrain_deposit_1 = train_deposit_1.join(deposit_sums, on='case_id', how='left')\ntrain_deposit_1 = (\n    train_deposit_1\n    .filter(pl.col('num_group1') == 0)\n    .drop('num_group1')\n    .drop('contractenddate_991D')\n    .drop('openingdate_313D')\n    .drop('amount_416A')\n)\ntrain_basetable = train_basetable.join(train_deposit_1, on='case_id', how='left')\ndel train_deposit_1\ngc.collect()\n\n\ntrain_credit_bureau_a_1 = pl.concat(\n    [\n        pl.read_csv(\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_credit_bureau_a_1_0.csv\").pipe(set_table_dtypes),\n        pl.read_csv(\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_credit_bureau_a_1_1.csv\").pipe(set_table_dtypes),\n        pl.read_csv(\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_credit_bureau_a_1_2.csv\").pipe(set_table_dtypes),\n        pl.read_csv(\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_credit_bureau_a_1_3.csv\").pipe(set_table_dtypes)\n    ],\n    how=\"vertical_relaxed\",\n)\ntrain_credit_bureau_a_1 = (\n    train_credit_bureau_a_1\n    .select(['case_id', 'debtoutstand_525A', 'totaloutstanddebtvalue_39A','overdueamountmax2_14A',\n             'overdueamountmax_155A','monthlyinstlamount_332A','dpdmax_139P','numberofoverdueinstlmax_1039L','num_group1'])\n    .filter(pl.col('num_group1') == 0)\n    .drop('num_group1')\n)\ntrain_basetable = train_basetable.join(train_credit_bureau_a_1, on='case_id', how='left')\ndel train_credit_bureau_a_1\ngc.collect()\n\n\ntrain_credit_bureau_b_1 = pl.read_csv('/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_credit_bureau_b_1.csv').pipe(set_table_dtypes)\ntrain_credit_bureau_b_1 = (\n    train_credit_bureau_b_1\n    .select(['case_id', 'totalamount_881A','dpdmax_851P','num_group1'])\n    .filter(pl.col('num_group1') == 0)\n    .drop('num_group1')\n)\ntrain_basetable = train_basetable.join(train_credit_bureau_b_1, on='case_id', how='left')\ndel train_credit_bureau_b_1\ngc.collect()\n\n\ntrain_credit_bureau_b_2 = pl.read_csv(\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_credit_bureau_b_2.csv\").pipe(set_table_dtypes)\nsums = (\n    train_credit_bureau_b_2\n    .group_by('case_id')\n    .agg([\n        pl.sum('pmts_dpdvalue_108P').alias('all_pmts_dpdvalue_108P'),\n        pl.sum('pmts_pmtsoverdue_635A').alias('all_pmts_pmtsoverdue_635A')\n    ])\n)\ntrain_credit_bureau_b_2 = train_credit_bureau_b_2.join(sums, on='case_id', how='left')\ntrain_credit_bureau_b_2 = train_credit_bureau_b_2.sort([\"case_id\"])\ntrain_credit_bureau_b_2 = train_credit_bureau_b_2.unique(subset=['case_id'], keep='first')\ntrain_credit_bureau_b_2 = train_credit_bureau_b_2.select(['case_id', 'all_pmts_dpdvalue_108P','all_pmts_pmtsoverdue_635A'])\ntrain_basetable = train_basetable.join(train_credit_bureau_b_2, on='case_id', how='left')\ndel train_credit_bureau_b_2\ngc.collect()\n\n\ntrain_tax_registry_a_1 = pl.read_csv('/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_tax_registry_a_1.csv').pipe(set_table_dtypes)\ntrain_tax_registry_a_1 = (\n    train_tax_registry_a_1\n    .select(['case_id', 'amount_4527230A','num_group1'])\n    .filter(pl.col('num_group1') == 0)\n    .drop('num_group1')\n)\ntrain_basetable = train_basetable.join(train_tax_registry_a_1, on='case_id', how='left')\ndel train_tax_registry_a_1\ngc.collect()\n\n\ntrain_tax_registry_b_1 = pl.read_csv('/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_tax_registry_b_1.csv').pipe(set_table_dtypes)\ntrain_tax_registry_b_1 = (\n    train_tax_registry_b_1\n    .select(['case_id', 'amount_4917619A','num_group1'])\n    .filter(pl.col('num_group1') == 0)\n    .drop('num_group1')\n)\ntrain_basetable = train_basetable.join(train_tax_registry_b_1, on='case_id', how='left')\ndel train_tax_registry_b_1\ngc.collect()\n\n\ntrain_tax_registry_c_1 = pl.read_csv('/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_tax_registry_c_1.csv').pipe(set_table_dtypes)\ntrain_tax_registry_c_1 = (\n    train_tax_registry_c_1\n    .select(['case_id', 'pmtamount_36A','num_group1'])\n    .filter(pl.col('num_group1') == 0)\n    .drop('num_group1')\n)\ntrain_basetable = train_basetable.join(train_tax_registry_c_1, on='case_id', how='left')\ndel train_tax_registry_c_1\ngc.collect()\n\n\ntrain_debitcard_1 = pl.read_csv('/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_debitcard_1.csv').pipe(set_table_dtypes)\ntrain_debitcard_1 = train_debitcard_1.with_columns([\n    pl.col(\"last180dayaveragebalance_704A\").cast(pl.Float64),\n    pl.col(\"last180dayturnover_1134A\").cast(pl.Float64)\n])\ncredit_balance = (\n    train_debitcard_1\n    .group_by('case_id')\n    .agg([\n        pl.sum('last180dayaveragebalance_704A').alias('all_last180dayaveragebalance_704A'),\n        pl.sum(\"last180dayturnover_1134A\").alias('all_last180dayturnover_1134A')\n    ])\n)\ntrain_debitcard_1 = train_debitcard_1.join(credit_balance, on='case_id', how='left')\ntrain_debitcard_1 = (\n    train_debitcard_1\n    .select(['case_id', 'all_last180dayaveragebalance_704A','all_last180dayturnover_1134A','num_group1'])\n    .filter(pl.col('num_group1') == 0)\n    .drop('num_group1')\n)\ntrain_basetable = train_basetable.join(train_debitcard_1, on='case_id', how='left')\ndel train_debitcard_1\ngc.collect()\n\n\ntrain_applprev_1 = pl.concat(\n    [\n        pl.read_csv(\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_applprev_1_0.csv\").pipe(set_table_dtypes),\n        pl.read_csv(\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_applprev_1_1.csv\").pipe(set_table_dtypes)\n    ],\n    how=\"vertical_relaxed\",\n)\ntrain_applprev_1 = (\n    train_applprev_1\n    .select(['case_id', 'annuity_853A','mainoccupationinc_437A','num_group1'])\n    .filter(pl.col('num_group1') == 0)\n    .drop('num_group1')\n)\ntrain_basetable = train_basetable.join(train_applprev_1, on='case_id', how='left')\ndel train_applprev_1\ngc.collect()\n\n\ntrain_basetable_special_case = train_basetable.filter(pl.col('eir_270L').is_null())\ntrain_basetable_base_case = train_basetable.filter(pl.col('eir_270L').is_not_null())\ndel train_basetable\ngc.collect()\n\ntrain_basetable_base_case = train_basetable_base_case.drop(['inittransactionamount_650A', 'maxdpdtolerance_374P', 'numinstlsallpaid_934L','eir_270L','credamount_770A'])\ntrain_basetable_special_case = train_basetable_special_case.drop(['avgdpdtolclosure24_3658938P', 'pctinstlsallpaidlate1d_3546856L', 'maxdpdlast12m_727P', \n                                                                    'numinstlswithdpd10_728L', 'numrejects9m_859L', 'maininc_215A','eir_270L'])\n\ntrain_basetable_base_case = train_basetable_base_case.sort(\"case_id\")\ntrain_basetable_base_case = train_basetable_base_case.to_pandas()\ntrain_basetable_base_case.to_csv(\"/kaggle/working/train_basetable_merged_base_case.csv\",index=False)\ndel train_basetable_base_case\ngc.collect()\n\ntrain_basetable_special_case = train_basetable_special_case.sort(\"case_id\")\ntrain_basetable_special_case = train_basetable_special_case.to_pandas()\ntrain_basetable_special_case.to_csv(\"/kaggle/working/train_basetable_merged_special_case.csv\",index=False)\ndel train_basetable_special_case\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-03-21T12:40:37.539905Z","iopub.execute_input":"2024-03-21T12:40:37.540243Z","iopub.status.idle":"2024-03-21T12:45:32.008719Z","shell.execute_reply.started":"2024-03-21T12:40:37.540213Z","shell.execute_reply":"2024-03-21T12:45:32.007832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# join test data\n\ntest_basetable = pl.read_csv(\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/test/test_base.csv\")\ntest_basetable = test_basetable.with_columns(\n    pl.col(\"WEEK_NUM\").map_elements(process_week_num).alias(\"WEEK_NUM\")\n)\ntest_basetable = test_basetable.drop(['date_decision','MONTH'])\n\ncolumns_of_interest = [\n    'case_id',\n    'price_1097A',\n    'numrejects9m_859L',\n    'pctinstlsallpaidlate1d_3546856L',\n    'maininc_215A',\n    'maxdpdfrom6mto36m_3546853P',\n    'numinstlsallpaid_934L',\n    'lastapprcredamount_781A',\n    'totalsettled_863A',\n    'maxannuity_159A',\n    'disbursedcredamount_1113A',\n    'inittransactionamount_650A',\n    'maxdpdlast24m_143P',\n    'maxdpdlast12m_727P',\n    'avgdpdtolclosure24_3658938P',\n    'numinstlswithdpd10_728L',\n    'annuity_780A',\n    'maxdpdtolerance_374P',\n    'credamount_770A',\n    'maxdebt4_972A',\n    'eir_270L'\n]\ntest_static = pl.concat(\n    [\n        pl.read_csv(\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/test/test_static_0_0.csv\").pipe(set_table_dtypes),\n        pl.read_csv(\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/test/test_static_0_1.csv\").pipe(set_table_dtypes),\n        pl.read_csv(\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/test/test_static_0_2.csv\").pipe(set_table_dtypes)\n    ],\n    how=\"vertical_relaxed\",\n)\ntest_static = test_static[columns_of_interest]\ntest_basetable = test_basetable.join(test_static, on='case_id', how='left')\ndel test_static,columns_of_interest\ngc.collect()\n\n\ntest_person_1 = (\n    pl.read_csv(\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/test/test_person_1.csv\").pipe(set_table_dtypes)\n    .select(['case_id', 'mainoccupationinc_384A', 'num_group1'])\n    .filter(pl.col('num_group1') == 0)\n    .drop('num_group1')\n)\ntest_basetable = test_basetable.join(test_person_1, on='case_id', how='left')\ndel test_person_1\ngc.collect()\n\n\ntest_other_1 = pl.read_csv(\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/test/test_other_1.csv\").pipe(set_table_dtypes)\ntest_other_1 = (\n    test_other_1\n    .select(['case_id', 'amtdepositbalance_4809441A','amtdebitincoming_4809443A','amtdebitoutgoing_4809440A','num_group1'])\n    .filter(pl.col('num_group1') == 0)\n    .drop('num_group1')\n)\ntest_basetable = test_basetable.join(test_other_1, on='case_id', how='left')\ndel test_other_1\ngc.collect()\n\n\ntest_deposit_1 = pl.read_csv(\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/test/test_deposit_1.csv\").pipe(set_table_dtypes)\ndeposit_sums = (\n    test_deposit_1\n    .group_by('case_id')\n    .agg(pl.sum('amount_416A').alias('deposit'))\n)\ntest_deposit_1 = test_deposit_1.join(deposit_sums, on='case_id', how='left')\ntest_deposit_1 = (\n    test_deposit_1\n    .filter(pl.col('num_group1') == 0)\n    .drop('num_group1')\n    .drop('contractenddate_991D')\n    .drop('openingdate_313D')\n    .drop('amount_416A')\n)\ntest_basetable = test_basetable.join(test_deposit_1, on='case_id', how='left')\ndel test_deposit_1\ngc.collect()\n\n\ntest_credit_bureau_a_1 = pl.concat(\n    [\n        pl.read_csv(\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/test/test_credit_bureau_a_1_0.csv\").pipe(set_table_dtypes),\n        pl.read_csv(\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/test/test_credit_bureau_a_1_1.csv\").pipe(set_table_dtypes),\n        pl.read_csv(\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/test/test_credit_bureau_a_1_2.csv\").pipe(set_table_dtypes),\n        pl.read_csv(\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/test/test_credit_bureau_a_1_3.csv\").pipe(set_table_dtypes),\n        pl.read_csv(\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/test/test_credit_bureau_a_1_4.csv\").pipe(set_table_dtypes)\n    ],\n    how=\"vertical_relaxed\",\n)\ntest_credit_bureau_a_1 = (\n    test_credit_bureau_a_1\n    .select(['case_id', 'debtoutstand_525A', 'totaloutstanddebtvalue_39A','overdueamountmax2_14A',\n             'overdueamountmax_155A','monthlyinstlamount_332A','dpdmax_139P','numberofoverdueinstlmax_1039L','num_group1'])\n    .filter(pl.col('num_group1') == 0)\n    .drop('num_group1')\n)\ntest_basetable = test_basetable.join(test_credit_bureau_a_1, on='case_id', how='left')\ndel test_credit_bureau_a_1\ngc.collect()\n\n\ntest_credit_bureau_b_1 =pl.read_csv(\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/test/test_credit_bureau_b_1.csv\").pipe(set_table_dtypes)\ntest_credit_bureau_b_1 = (\n    test_credit_bureau_b_1\n    .select(['case_id', 'totalamount_881A','dpdmax_851P','num_group1'])\n    .filter(pl.col('num_group1') == 0)\n    .drop('num_group1')\n)\ntest_basetable = test_basetable.join(test_credit_bureau_b_1, on='case_id', how='left')\ndel test_credit_bureau_b_1\ngc.collect()\n\n\ntest_credit_bureau_b_2 = pl.read_csv(\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/test/test_credit_bureau_b_2.csv\").pipe(set_table_dtypes)\nsums = (\n    test_credit_bureau_b_2\n    .group_by('case_id')\n    .agg([\n        pl.sum('pmts_dpdvalue_108P').alias('all_pmts_dpdvalue_108P'),\n        pl.sum('pmts_pmtsoverdue_635A').alias('all_pmts_pmtsoverdue_635A')\n    ])\n)\ntest_credit_bureau_b_2 = test_credit_bureau_b_2.join(sums, on='case_id', how='left')\ntest_credit_bureau_b_2 = test_credit_bureau_b_2.sort([\"case_id\"])\ntest_credit_bureau_b_2 = test_credit_bureau_b_2.unique(subset=['case_id'], keep='first')\ntest_credit_bureau_b_2 = test_credit_bureau_b_2.select(['case_id', 'all_pmts_dpdvalue_108P','all_pmts_pmtsoverdue_635A'])\ntest_basetable = test_basetable.join(test_credit_bureau_b_2, on='case_id', how='left')\ndel test_credit_bureau_b_2\ngc.collect()\n\n\ntest_tax_registry_a_1 = pl.read_csv('/kaggle/input/home-credit-credit-risk-model-stability/csv_files/test/test_tax_registry_a_1.csv').pipe(set_table_dtypes)\ntest_tax_registry_a_1 = (\n    test_tax_registry_a_1\n    .select(['case_id', 'amount_4527230A','num_group1'])\n    .filter(pl.col('num_group1') == 0)\n    .drop('num_group1')\n)\ntest_basetable = test_basetable.join(test_tax_registry_a_1, on='case_id', how='left')\ndel test_tax_registry_a_1\ngc.collect()\n\n\ntest_tax_registry_b_1 = pl.read_csv('/kaggle/input/home-credit-credit-risk-model-stability/csv_files/test/test_tax_registry_b_1.csv').pipe(set_table_dtypes)\ntest_tax_registry_b_1 = (\n    test_tax_registry_b_1\n    .select(['case_id', 'amount_4917619A','num_group1'])\n    .filter(pl.col('num_group1') == 0)\n    .drop('num_group1')\n)\ntest_basetable = test_basetable.join(test_tax_registry_b_1, on='case_id', how='left')\ndel test_tax_registry_b_1\ngc.collect()\n\n\ntest_tax_registry_c_1 = pl.read_csv('/kaggle/input/home-credit-credit-risk-model-stability/csv_files/test/test_tax_registry_c_1.csv').pipe(set_table_dtypes)\ntest_tax_registry_c_1 = (\n    test_tax_registry_c_1\n    .select(['case_id', 'pmtamount_36A','num_group1'])\n    .with_columns(pl.col('num_group1').cast(int))\n    .with_columns(pl.col(\"case_id\").cast(int))\n    .filter(pl.col('num_group1') == 0)\n    .drop('num_group1')\n)\ntest_basetable = test_basetable.join(test_tax_registry_c_1, on='case_id', how='left')\ndel test_tax_registry_c_1\ngc.collect()\n\n\ntest_debitcard_1 = pl.read_csv('/kaggle/input/home-credit-credit-risk-model-stability/csv_files/test/test_debitcard_1.csv').pipe(set_table_dtypes)\ntest_debitcard_1 = test_debitcard_1.with_columns([\n    pl.col(\"last180dayaveragebalance_704A\").cast(pl.Float64),\n    pl.col(\"last180dayturnover_1134A\").cast(pl.Float64)\n])\ncredit_balance = (\n    test_debitcard_1\n    .group_by('case_id')\n    .agg([\n        pl.sum('last180dayaveragebalance_704A').alias('all_last180dayaveragebalance_704A'),\n        pl.sum(\"last180dayturnover_1134A\").alias('all_last180dayturnover_1134A')\n    ])\n)\ntest_debitcard_1 = test_debitcard_1.join(credit_balance, on='case_id', how='left')\ntest_debitcard_1 = (\n    test_debitcard_1\n    .select(['case_id', 'all_last180dayaveragebalance_704A','all_last180dayturnover_1134A','num_group1'])\n    .filter(pl.col('num_group1') == 0)\n    .drop('num_group1')\n)\ntest_basetable = test_basetable.join(test_debitcard_1, on='case_id', how='left')\ndel test_debitcard_1\ngc.collect()\n\n\ntest_applprev_1 = pl.concat(\n    [\n        pl.read_csv(\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/test/test_applprev_1_0.csv\").pipe(set_table_dtypes),\n        pl.read_csv(\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/test/test_applprev_1_1.csv\").pipe(set_table_dtypes),\n        pl.read_csv(\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/test/test_applprev_1_2.csv\").pipe(set_table_dtypes)\n    ],\n    how=\"vertical_relaxed\",\n)\ntest_applprev_1 = (\n    test_applprev_1\n    .select(['case_id', 'annuity_853A','mainoccupationinc_437A','num_group1'])\n    .filter(pl.col('num_group1') == 0)\n    .drop('num_group1')\n)\ntest_basetable = test_basetable.join(test_applprev_1, on='case_id', how='left')\ndel test_applprev_1\ngc.collect()\n\n\ntest_basetable_special_case = test_basetable.filter(pl.col('eir_270L').is_null())\ntest_basetable_base_case = test_basetable.filter(pl.col('eir_270L').is_not_null())\ndel test_basetable\ngc.collect()\n\ntest_basetable_base_case = test_basetable_base_case.drop(['inittransactionamount_650A', 'maxdpdtolerance_374P', 'numinstlsallpaid_934L','eir_270L','credamount_770A'])\ntest_basetable_special_case = test_basetable_special_case.drop(['avgdpdtolclosure24_3658938P', 'pctinstlsallpaidlate1d_3546856L', 'maxdpdlast12m_727P', \n                                                                    'numinstlswithdpd10_728L', 'numrejects9m_859L', 'maininc_215A','eir_270L'])\n\ntest_basetable_base_case = test_basetable_base_case.sort(\"case_id\")\ntest_basetable_base_case = test_basetable_base_case.to_pandas()\ntest_basetable_base_case.to_csv(\"/kaggle/working/test_basetable_merged_base_case.csv\",index=False)\ndel test_basetable_base_case\ngc.collect()\n\n\ntest_basetable_special_case = test_basetable_special_case.sort(\"case_id\")\ntest_basetable_special_case = test_basetable_special_case.to_pandas()\ntest_basetable_special_case.to_csv(\"/kaggle/working/test_basetable_merged_special_case.csv\",index=False)\ndel test_basetable_special_case\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-03-21T12:45:32.011007Z","iopub.execute_input":"2024-03-21T12:45:32.011275Z","iopub.status.idle":"2024-03-21T12:45:33.083480Z","shell.execute_reply.started":"2024-03-21T12:45:32.011252Z","shell.execute_reply":"2024-03-21T12:45:33.082547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom sklearn.ensemble import StackingClassifier\nfrom lightgbm import LGBMClassifier\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.utils import resample\n\n\ntrain_data_base_case = pd.read_csv(\"/kaggle/working/train_basetable_merged_base_case.csv\",index_col='case_id')\nmodels = {}\n\ndef assessment(csv_name):\n    \n    case_ids = []\n    all_predictions = []\n    df = pd.read_csv(csv_name, index_col='case_id')\n    \n    for i in range(len(df)):\n        \n        sample = df.iloc[i]\n        case_ids.append(sample.name)\n        print(sample.name)\n        sample = sample.dropna()\n        non_null_columns = sample.index.tolist()\n        sample_dict = {col: [val] for col, val in zip(non_null_columns, sample.values)}\n        sample = pd.DataFrame(sample_dict)\n        sample.columns = sample.columns.map(str)\n        sample = sample.astype('float32')\n        columns_key = '_'.join(non_null_columns)\n        \n        if columns_key not in models:\n\n            non_null_columns_target = non_null_columns.copy()\n            non_null_columns_target.append('target')\n            chosen = train_data_base_case[non_null_columns_target]\n            chosen = chosen.dropna()\n\n            majority_class = chosen[chosen.target == 0]\n            minority_class = chosen[chosen.target == 1]\n            \n            if len(minority_class) <=3 or len(majority_class) <=3:\n                all_predictions.append(0)\n            \n            else:\n                majority_downsampled = resample(majority_class,\n                                            replace=False,\n                                            n_samples=len(minority_class) * 7,\n                                            random_state=10)\n                chosen = pd.concat([majority_downsampled, minority_class])\n                \n                X = chosen.drop('target', axis=1)\n                X.columns = X.columns.map(str)\n                X = X.astype('float32')\n                y = chosen['target']\n                \n                base_learner = [('lgb', LGBMClassifier(verbose=-1, device='gpu',gpu_platform_id=0, gpu_device_id=0, n_estimators=300, learning_rate=0.05, max_depth=15, num_leaves=25, random_state=10, force_col_wise=True))]\n                meta_learner = LogisticRegression(random_state=10,n_jobs= -1)\n                cv_method = StratifiedKFold(n_splits=4, shuffle=True, random_state=10)\n                stacked_model = StackingClassifier(\n                                estimators=base_learner,\n                                final_estimator=meta_learner,\n                                stack_method='predict_proba',\n                                cv=cv_method\n                 )\n\n                stacked_model.fit(X, y)\n                result = stacked_model.predict_proba(sample)\n                all_predictions.append(result[:, 1][0])\n                models[columns_key] = stacked_model\n                print(result[:, 1][0])\n                \n        else:\n            result = models[columns_key].predict_proba(sample)\n            all_predictions.append(result[:, 1][0])\n            print(result[:, 1][0])\n\n    output = {\n        'case_id': case_ids,\n        'score': all_predictions\n    }\n    df = pd.DataFrame(output)\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-03-21T12:45:33.084753Z","iopub.execute_input":"2024-03-21T12:45:33.085050Z","iopub.status.idle":"2024-03-21T12:45:43.193943Z","shell.execute_reply.started":"2024-03-21T12:45:33.085006Z","shell.execute_reply":"2024-03-21T12:45:43.192686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data_special_case = pd.read_csv(\"/kaggle/working/train_basetable_merged_special_case.csv\",index_col='case_id')\nmodels_special_case = {}\n\ndef assessment_special_case(csv_name):\n    \n    case_ids = []\n    all_predictions = []\n    df = pd.read_csv(csv_name, index_col='case_id')\n    \n    if df.empty:\n        output = {\n        'case_id': [],\n        'score': []\n        }\n        df = pd.DataFrame(output)\n        return df\n    \n    for i in range(len(df)):\n        \n        sample = df.iloc[i]\n        case_ids.append(sample.name)\n        print(sample.name)\n        sample = sample.dropna()\n        non_null_columns = sample.index.tolist()\n        sample_dict = {col: [val] for col, val in zip(non_null_columns, sample.values)}\n        sample = pd.DataFrame(sample_dict)\n        sample.columns = sample.columns.map(str)\n        sample = sample.astype('float32')\n        columns_key = '_'.join(non_null_columns)\n        \n        if columns_key not in models_special_case:\n\n            non_null_columns_target = non_null_columns.copy()\n            non_null_columns_target.append('target')\n            chosen = train_data_special_case[non_null_columns_target]\n            chosen = chosen.dropna()\n            \n            majority_class = chosen[chosen.target == 0]\n            minority_class = chosen[chosen.target == 1]\n            \n            if len(minority_class) <=3 or len(majority_class) <=3:\n                all_predictions.append(0)\n            \n            else:\n                \n                majority_downsampled = resample(majority_class,\n                                            replace=False,\n                                            n_samples=len(minority_class) * 7,\n                                            random_state=10)\n                chosen = pd.concat([majority_downsampled, minority_class])\n                \n                X = chosen.drop('target', axis=1)\n                X.columns = X.columns.map(str)\n                X = X.astype('float32')\n                y = chosen['target']\n                \n                base_learner = [('lgb', LGBMClassifier(verbose=-1, device='gpu',gpu_platform_id=0, gpu_device_id=0, n_estimators=250, learning_rate=0.05, max_depth=15, num_leaves=25, random_state=10, force_col_wise=True))]\n                meta_learner = LogisticRegression(random_state=10,n_jobs= -1)\n                cv_method = StratifiedKFold(n_splits=4, shuffle=True, random_state=10)\n                stacked_model = StackingClassifier(\n                                estimators=base_learner,\n                                final_estimator=meta_learner,\n                                stack_method='predict_proba',\n                                cv=cv_method\n                 )\n\n                stacked_model.fit(X, y)\n                result = stacked_model.predict_proba(sample)\n                all_predictions.append(result[:, 1][0])\n                models_special_case[columns_key] = stacked_model\n                print(result[:, 1][0])\n                \n        else:\n            result = models_special_case[columns_key].predict_proba(sample)\n            all_predictions.append(result[:, 1][0])\n            print(result[:, 1][0])\n            \n    output = {\n        'case_id': case_ids,\n        'score': all_predictions\n    }\n    df = pd.DataFrame(output)\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-03-21T12:45:43.195748Z","iopub.execute_input":"2024-03-21T12:45:43.196127Z","iopub.status.idle":"2024-03-21T12:45:43.976920Z","shell.execute_reply.started":"2024-03-21T12:45:43.196092Z","shell.execute_reply":"2024-03-21T12:45:43.976126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#bug_row = train_data_base_case[train_data_base_case.index==51132]\n#bug_row = bug_row.drop('target', axis=1)\n#bug_row.to_csv('/kaggle/working/bug_row.csv')\n#assessment('/kaggle/working/bug_row.csv')","metadata":{"execution":{"iopub.status.busy":"2024-03-21T12:46:30.384146Z","iopub.execute_input":"2024-03-21T12:46:30.384792Z","iopub.status.idle":"2024-03-21T12:46:30.566487Z","shell.execute_reply.started":"2024-03-21T12:46:30.384757Z","shell.execute_reply":"2024-03-21T12:46:30.565488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearnex import patch_sklearn\npatch_sklearn()\n\nimport logging\nlogging.getLogger().setLevel(logging.ERROR)\n\nresult1 = assessment(\"/kaggle/working/test_basetable_merged_base_case.csv\")\nresult2 = assessment_special_case(\"/kaggle/working/test_basetable_merged_special_case.csv\")\n\ncombined_df = pd.concat([result1, result2], ignore_index=True)\ncombined_df.set_index('case_id', inplace=True)\nsorted_df = combined_df.sort_index()\nsorted_df.to_csv('/kaggle/working/submission.csv')","metadata":{"execution":{"iopub.status.busy":"2024-03-21T12:45:43.983640Z","iopub.execute_input":"2024-03-21T12:45:43.983945Z","iopub.status.idle":"2024-03-21T12:46:25.427974Z","shell.execute_reply.started":"2024-03-21T12:45:43.983907Z","shell.execute_reply":"2024-03-21T12:46:25.426350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#import logging\n#logging.getLogger().setLevel(logging.ERROR)\n\n#from sklearnex import patch_sklearn\n#patch_sklearn()\n\n#df1 = pd.read_csv(\"/kaggle/working/train_basetable_merged_base_case.csv\",index_col='case_id')\n#df1_1 = df1.drop('target', axis=1)\n#df1_1.to_csv('/kaggle/working/a.csv')\n\n#df2 = pd.read_csv(\"/kaggle/working/train_basetable_merged_special_case.csv\",index_col='case_id')\n#df2_1 = df2.drop('target', axis=1)\n#df2_1.to_csv('/kaggle/working/b.csv')\n\n#result1 = assessment('/kaggle/working/a.csv')\n#result2 = assessment('/kaggle/working/b.csv')\n#combined_df = pd.concat([result1, result2], ignore_index=True)\n#combined_df.set_index('case_id', inplace=True)\n#sorted_df = combined_df.sort_index()\n#sorted_df.to_csv('/kaggle/working/submission.csv')\n\n#df1_0 = df1[['case_id','target']]\n#df2_0 = df2[['case_id','target']]\n#combined_target = pd.concat([df1_0, df2_0], ignore_index=True)\n#combined_target.set_index('case_id', inplace=True)\n#sorted_target = combined_target.sort_index()","metadata":{"execution":{"iopub.status.busy":"2024-03-21T12:46:25.429101Z","iopub.status.idle":"2024-03-21T12:46:25.429553Z","shell.execute_reply.started":"2024-03-21T12:46:25.429330Z","shell.execute_reply":"2024-03-21T12:46:25.429348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#from sklearn.metrics import roc_auc_score\n\n#predict = sorted_df['score'].values\n#true = sorted_target['target'].values\n#print(true)\n#print(predict)\n#auc = roc_auc_score(true, predict)\n#print(auc)","metadata":{"execution":{"iopub.status.busy":"2024-03-21T12:46:25.430803Z","iopub.status.idle":"2024-03-21T12:46:25.431107Z","shell.execute_reply.started":"2024-03-21T12:46:25.430956Z","shell.execute_reply":"2024-03-21T12:46:25.430969Z"},"trusted":true},"execution_count":null,"outputs":[]}]}