{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"dockerImageVersionId":30635,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Team 4 DSCI 552 Initial Submission\n\n## Load the data","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"import polars as pl\nimport numpy as np\nimport pandas as pd\nimport lightgbm as lgb\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nfrom tensorflow import keras as ks\nfrom keras.models import Sequential as Seq\nfrom keras.layers import Dense, SimpleRNN, Dropout\nfrom keras.callbacks import ReduceLROnPlateau, EarlyStopping\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score \nfrom sklearn.utils.class_weight import compute_class_weight\nimport sys\nfrom keras.regularizers import l2\nfrom sklearn.preprocessing import StandardScaler\nfrom keras.metrics import AUC\n\ndataPath = \"/kaggle/input/home-credit-credit-risk-model-stability/\"","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2024-05-08T03:54:21.371298Z","iopub.execute_input":"2024-05-08T03:54:21.371704Z","iopub.status.idle":"2024-05-08T03:54:34.684557Z","shell.execute_reply.started":"2024-05-08T03:54:21.371663Z","shell.execute_reply":"2024-05-08T03:54:34.683636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_table_dtypes(df: pl.DataFrame) -> pl.DataFrame:\n    for col in df.columns:\n        if col[-1] in (\"P\", \"A\"):\n            df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n        elif col[-1] in (\"D\"):\n            df = df.with_columns(pl.col(col).cast(pl.Date).alias(col))\n    return df\n\ndef convert_strings(df: pd.DataFrame) -> pd.DataFrame:\n    for col in df.columns:  \n        if df[col].dtype.name in ['object', 'string']:\n            df[col] = df[col].astype(\"string\").astype('category')\n            current_categories = df[col].cat.categories\n            new_categories = current_categories.to_list() + [\"Unknown\"]\n            new_dtype = pd.CategoricalDtype(categories=new_categories, ordered=True)\n            df[col] = df[col].astype(new_dtype)\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-05-08T03:54:34.686567Z","iopub.execute_input":"2024-05-08T03:54:34.687562Z","iopub.status.idle":"2024-05-08T03:54:34.696405Z","shell.execute_reply.started":"2024-05-08T03:54:34.687522Z","shell.execute_reply":"2024-05-08T03:54:34.695483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_basetable = pl.read_csv(dataPath + \"csv_files/train/train_base.csv\", low_memory=False)\ntrain_static = pl.concat(\n    [\n        pl.read_csv(dataPath + \"csv_files/train/train_static_0_0.csv\", low_memory=False).pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/train/train_static_0_1.csv\", low_memory=False).pipe(set_table_dtypes),\n    ],\n    how=\"vertical_relaxed\",\n)\ntrain_static_cb = pl.read_csv(dataPath + \"csv_files/train/train_static_cb_0.csv\", low_memory=False).pipe(set_table_dtypes)\n'''train_applprev_1 = pl.concat(\n    [\n        pl.read_csv(dataPath + \"csv_files/train/train_applprev_1_0.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/train/train_applprev_1_1.csv\").pipe(set_table_dtypes),\n    ],\n    how=\"vertical_relaxed\",\n)'''\n#train_other_1 = pl.read_csv(dataPath + \"csv_files/train/train_other_1.csv\").pipe(set_table_dtypes)\n#train_tax_registry_a_1 = pl.read_csv(dataPath + \"csv_files/train/train_tax_registry_a_1.csv\").pipe(set_table_dtypes)\n#train_tax_registry_b_1 = pl.read_csv(dataPath + \"csv_files/train/train_tax_registry_b_1.csv\").pipe(set_table_dtypes)\n#train_tax_registry_c_1 = pl.read_csv(dataPath + \"csv_files/train/train_tax_registry_c_1.csv\").pipe(set_table_dtypes)\n\n#train_credit_bureau_b_1 = pl.read_csv(dataPath + \"csv_files/train/train_credit_bureau_b_1.csv\").pipe(set_table_dtypes) \n'''train_deposit_1 = pl.read_csv(dataPath + \"csv_files/train/train_deposit_1.csv\").pipe(set_table_dtypes)\ntrain_person_1 = pl.read_csv(dataPath + \"csv_files/train/train_person_1.csv\").pipe(set_table_dtypes) \ntrain_debitcard_1 = pl.read_csv(dataPath + \"csv_files/train/train_debitcard_1.csv\").pipe(set_table_dtypes)'''\n#train_applprev_2 = pl.read_csv(dataPath + \"csv_files/train/train_applprev_2.csv\").pipe(set_table_dtypes)\n#train_person_2 = pl.read_csv(dataPath + \"csv_files/train/train_person_2.csv\").pipe(set_table_dtypes) \n\n#train_credit_bureau_b_2 = pl.read_csv(dataPath + \"csv_files/train/train_credit_bureau_b_2.csv\").pipe(set_table_dtypes) ","metadata":{"execution":{"iopub.status.busy":"2024-05-08T03:54:34.698001Z","iopub.execute_input":"2024-05-08T03:54:34.698354Z","iopub.status.idle":"2024-05-08T03:54:50.463373Z","shell.execute_reply.started":"2024-05-08T03:54:34.698313Z","shell.execute_reply":"2024-05-08T03:54:50.462351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''train_credit_bureau_a_1 = pl.concat(\n    [\n        pl.read_csv(dataPath + \"csv_files/train/train_credit_bureau_a_1_0.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/train/train_credit_bureau_a_1_1.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/train/train_credit_bureau_a_1_2.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/train/train_credit_bureau_a_1_3.csv\").pipe(set_table_dtypes),\n    ],\n    how=\"vertical_relaxed\",\n)'''","metadata":{"execution":{"iopub.status.busy":"2024-05-08T03:54:50.465544Z","iopub.execute_input":"2024-05-08T03:54:50.465891Z","iopub.status.idle":"2024-05-08T03:54:50.472855Z","shell.execute_reply.started":"2024-05-08T03:54:50.465846Z","shell.execute_reply":"2024-05-08T03:54:50.471621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''train_credit_bureau_a_2 = pl.concat(\n    [\n        pl.read_csv(dataPath + \"csv_files/train/train_credit_bureau_a_2_0.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/train/train_credit_bureau_a_2_1.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/train/train_credit_bureau_a_2_2.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/train/train_credit_bureau_a_2_3.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/train/train_credit_bureau_a_2_4.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/train/train_credit_bureau_a_2_5.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/train/train_credit_bureau_a_2_6.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/train/train_credit_bureau_a_2_7.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/train/train_credit_bureau_a_2_8.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/train/train_credit_bureau_a_2_9.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/train/train_credit_bureau_a_2_10.csv\").pipe(set_table_dtypes),\n    ],\n    how=\"vertical_relaxed\",\n)'''","metadata":{"execution":{"iopub.status.busy":"2024-05-08T03:54:50.474136Z","iopub.execute_input":"2024-05-08T03:54:50.474481Z","iopub.status.idle":"2024-05-08T03:54:50.488990Z","shell.execute_reply.started":"2024-05-08T03:54:50.474454Z","shell.execute_reply":"2024-05-08T03:54:50.488027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_basetable = pl.read_csv(dataPath + \"csv_files/test/test_base.csv\", low_memory=False)\ntest_static = pl.concat(\n    [\n        pl.read_csv(dataPath + \"csv_files/test/test_static_0_0.csv\", low_memory=False).pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/test/test_static_0_1.csv\", low_memory=False).pipe(set_table_dtypes),\n    ],\n    how=\"vertical_relaxed\",\n)\ntest_static_cb = pl.read_csv(dataPath + \"csv_files/test/test_static_cb_0.csv\", low_memory=False).pipe(set_table_dtypes)\n'''test_applprev_1 = pl.concat(\n    [\n        pl.read_csv(dataPath + \"csv_files/test/test_applprev_1_0.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/test/test_applprev_1_1.csv\").pipe(set_table_dtypes),\n    ],\n    how=\"vertical_relaxed\",\n)'''\n#test_other_1 = pl.read_csv(dataPath + \"csv_files/test/test_other_1.csv\").pipe(set_table_dtypes)\n#test_tax_registry_a_1 = pl.read_csv(dataPath + \"csv_files/test/test_tax_registry_a_1.csv\").pipe(set_table_dtypes)\n#test_tax_registry_b_1 = pl.read_csv(dataPath + \"csv_files/test/test_tax_registry_b_1.csv\").pipe(set_table_dtypes)\n#test_tax_registry_c_1 = pl.read_csv(dataPath + \"csv_files/test/test_tax_registry_c_1.csv\").pipe(set_table_dtypes)\n'''test_credit_bureau_a_1 = pl.concat(\n    [\n        pl.read_csv(dataPath + \"csv_files/test/test_credit_bureau_a_1_0.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/test/test_credit_bureau_a_1_1.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/test/test_credit_bureau_a_1_2.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/test/test_credit_bureau_a_1_3.csv\").pipe(set_table_dtypes),\n    ],\n    how=\"vertical_relaxed\",\n)'''\n#test_credit_bureau_b_1 = pl.read_csv(dataPath + \"csv_files/test/test_credit_bureau_b_1.csv\").pipe(set_table_dtypes) \n'''test_deposit_1 = pl.read_csv(dataPath + \"csv_files/test/test_deposit_1.csv\").pipe(set_table_dtypes)\ntest_person_1 = pl.read_csv(dataPath + \"csv_files/test/test_person_1.csv\").pipe(set_table_dtypes) \ntest_debitcard_1 = pl.read_csv(dataPath + \"csv_files/test/test_debitcard_1.csv\").pipe(set_table_dtypes)'''\n#test_applprev_2 = pl.read_csv(dataPath + \"csv_files/test/test_applprev_2.csv\").pipe(set_table_dtypes)\n#test_person_2 = pl.read_csv(dataPath + \"csv_files/test/test_person_2.csv\").pipe(set_table_dtypes) \n'''test_credit_bureau_a_2 = pl.concat(\n    [\n        pl.read_csv(dataPath + \"csv_files/test/test_credit_bureau_a_2_0.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/test/test_credit_bureau_a_2_1.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/test/test_credit_bureau_a_2_2.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/test/test_credit_bureau_a_2_3.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/test/test_credit_bureau_a_2_4.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/test/test_credit_bureau_a_2_5.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/test/test_credit_bureau_a_2_6.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/test/test_credit_bureau_a_2_7.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/test/test_credit_bureau_a_2_8.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/test/test_credit_bureau_a_2_9.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/test/test_credit_bureau_a_2_10.csv\").pipe(set_table_dtypes),\n    ],\n    how=\"vertical_relaxed\",\n)'''\n#test_credit_bureau_b_2 = pl.read_csv(dataPath + \"csv_files/test/test_credit_bureau_b_2.csv\").pipe(set_table_dtypes) ","metadata":{"execution":{"iopub.status.busy":"2024-05-08T03:54:50.490685Z","iopub.execute_input":"2024-05-08T03:54:50.491150Z","iopub.status.idle":"2024-05-08T03:54:50.537358Z","shell.execute_reply.started":"2024-05-08T03:54:50.491111Z","shell.execute_reply":"2024-05-08T03:54:50.536315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_basetable\n#train_static\n#train_static_cb\n#train_applprev_1\n#train_other_1\n#train_tax_registry_a_1\n#train_tax_registry_b_1\n#train_tax_registry_c_1\n##train_credit_bureau_a_1\n#train_credit_bureau_b_1\n#train_deposit_1\n#train_person_1\n#train_debitcard_1\n#train_applprev_2\n#train_person_2\n##train_credit_bureau_a_2\n#train_credit_bureau_b_2","metadata":{"execution":{"iopub.status.busy":"2024-05-08T03:54:50.538645Z","iopub.execute_input":"2024-05-08T03:54:50.538953Z","iopub.status.idle":"2024-05-08T03:54:50.544088Z","shell.execute_reply.started":"2024-05-08T03:54:50.538927Z","shell.execute_reply":"2024-05-08T03:54:50.543078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''train_person_1_feat = train_person_1.group_by(\"case_id\").agg(\n    pl.col(\"mainoccupationinc_384A\").max().alias(\"mainoccupationinc_384A_max\"),\n    (pl.col(\"incometype_1044T\") == \"SELFEMPLOYED\").max().alias(\"mainoccupationinc_384A_any_selfemployed\")\n)\n\n# num_group1 = 0 is the person who applied for the loan\ntrain_person_1_feats_2 = train_person_1.select([\"case_id\", \"num_group1\",\"housetype_905L\"]).filter(\n    pl.col(\"num_group1\") == 0\n).drop(\"num_group1\").rename({\"housetype_905L\": \"person_housetype\"})\n\n# Here we have num_group1 and num_group2, so we need to aggregate again.\ntrain_credit_bureau_b_2_feats = train_credit_bureau_b_2.group_by(\"case_id\").agg(\n    pl.col(\"pmts_pmtsoverdue_635A\").max().alias(\"pmts_pmtsoverdue_635A_max\"),\n    (pl.col(\"pmts_dpdvalue_108P\") > 31).max().alias(\"pmts_dpdvalue_108P_over31\")\n)\n\ntrain_debitcard_feat = train_debitcard_1.group_by(\"case_id\").agg(\n    pl.col(\"last180dayaveragebalance_704A\").max().alias(\"max_last180dayaveragebalance_704A\")\n)\n\ntrain_deposit_feat = train_deposit_1.group_by('case_id').agg(\n    pl.sum('amount_416A').alias('sum_deposit_amount_A')\n).sort(by='case_id')'''\n\n        \nselected_static_cols = []\nfor col in train_static.columns:\n    if col[-1] in (\"A\", \"M\", \"P\"):\n        selected_static_cols.append(col)\n    if col[-1] in (\"L\", \"T\"):\n        if train_static[col].dtype.is_float():\n            selected_static_cols.append(col)\n        elif \"type\" in col or \"equality\" in col:\n            selected_static_cols.append(col)\n        elif \"num\" in col:\n            train_static = train_static.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n            selected_static_cols.append(col)\n        \nselected_static_cb_cols = []\nfor col in train_static_cb.columns:\n    if col[-1] in (\"A\", \"M\", \"P\"):\n        selected_static_cb_cols.append(col)\n    if col[-1] in (\"L\", \"T\"):\n        if train_static_cb[col].dtype.is_float():\n            train_static_cb = train_static_cb.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n            selected_static_cb_cols.append(col)\n        elif \"type\" in col:\n            selected_static_cb_cols.append(col)\n    \nlist_cb =[\"contractssum_5085716L\", \"pmtcount_4527229L\", \"pmtcount_4955617L\"]\nfor col in list_cb:\n    train_static_cb = train_static_cb.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n    selected_static_cb_cols.append(col)\n                \ndata = train_basetable.join(\n    train_static.select([\"case_id\"] + selected_static_cols), how=\"left\", on=\"case_id\"\n).join(\n    train_static_cb.select([\"case_id\"] + selected_static_cb_cols), how=\"left\", on=\"case_id\"\n)\n'''.join(\n    train_person_1_feat, how=\"left\", on=\"case_id\"\n).join(\n    train_person_1_feats_2, how=\"left\", on=\"case_id\"\n).join(\n    train_credit_bureau_b_2_feats, how=\"left\", on=\"case_id\"\n).join(\n    train_debitcard_feat, how=\"left\", on=\"case_id\"\n).join(\n    train_deposit_feat, how=\"left\", on=\"case_id\")'''","metadata":{"execution":{"iopub.status.busy":"2024-05-08T03:54:50.545460Z","iopub.execute_input":"2024-05-08T03:54:50.545772Z","iopub.status.idle":"2024-05-08T03:54:53.022338Z","shell.execute_reply.started":"2024-05-08T03:54:50.545746Z","shell.execute_reply":"2024-05-08T03:54:53.021215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''test_person_1_feats_1 = train_person_1.group_by(\"case_id\").agg(\n    pl.col(\"mainoccupationinc_384A\").max().alias(\"mainoccupationinc_384A_max\"),\n    (pl.col(\"incometype_1044T\") == \"SELFEMPLOYED\").max().alias(\"mainoccupationinc_384A_any_selfemployed\")\n)\n\ntest_person_1_feats_2 = train_person_1.select([\"case_id\", \"num_group1\", \"housetype_905L\"]).filter(\n    pl.col(\"num_group1\") == 0\n).drop(\"num_group1\").rename({\"housetype_905L\": \"person_housetype\"})\n\ntest_credit_bureau_b_2_feats = train_credit_bureau_b_2.group_by(\"case_id\").agg(\n    pl.col(\"pmts_pmtsoverdue_635A\").max().alias(\"pmts_pmtsoverdue_635A_max\"),\n    (pl.col(\"pmts_dpdvalue_108P\") > 31).max().alias(\"pmts_dpdvalue_108P_over31\")\n)\n\ntest_debitcard_feat = test_debitcard_1.group_by(\"case_id\").agg(\n    pl.col(\"last180dayaveragebalance_704A\").max().alias(\"max_last180dayaveragebalance_704A\")\n)\n\n### deposit ###\ntest_deposit_feat = test_deposit_1.group_by('case_id').agg(\n    pl.sum('amount_416A').alias('sum_deposit_amount_A')\n).sort(by='case_id')'''\n\n\n\ndata_submission = test_basetable.join(\n    test_static.select([\"case_id\"] + selected_static_cols), how=\"left\", on=\"case_id\"\n).join(\n    test_static_cb.select([\"case_id\"] + selected_static_cb_cols), how=\"left\", on=\"case_id\"\n)\n'''.join(\n    test_person_1_feats_1, how=\"left\", on=\"case_id\"\n).join(\n    test_person_1_feats_2, how=\"left\", on=\"case_id\"\n).join(\n    test_credit_bureau_b_2_feats, how=\"left\", on=\"case_id\"\n).join(\n    test_debitcard_feat, how=\"left\", on=\"case_id\"\n).join(\n    test_deposit_feat, how=\"left\", on=\"case_id\")'''\n\nfor col in data_submission.columns:\n    if col[-1] in (\"L\", \"T\") and data[col].dtype.is_float():\n        data_submission = data_submission.with_columns(pl.col(col).cast(pl.Float64).alias(col))","metadata":{"execution":{"iopub.status.busy":"2024-05-08T03:54:53.023863Z","iopub.execute_input":"2024-05-08T03:54:53.024947Z","iopub.status.idle":"2024-05-08T03:54:53.051895Z","shell.execute_reply.started":"2024-05-08T03:54:53.024905Z","shell.execute_reply":"2024-05-08T03:54:53.050520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"case_ids = data[\"case_id\"].unique().shuffle(seed = 1)\ncase_ids_train, case_ids_test = train_test_split(case_ids, train_size=0.60, random_state=1)\ncase_ids_valid, case_ids_test = train_test_split(case_ids_test, train_size=0.5, random_state=1)\n\ncols_pred = []\nfor col in data.columns:\n    if (col[-1].isupper() and col[:-1].islower()) or (\"_max\" in col):\n        cols_pred.append(col)\n\ndef from_polars_to_pandas(case_ids: pl.DataFrame) -> pl.DataFrame:\n    return (\n        data.filter(pl.col(\"case_id\").is_in(case_ids))[[\"case_id\", \"WEEK_NUM\", \"target\"]].to_pandas(),\n        data.filter(pl.col(\"case_id\").is_in(case_ids))[cols_pred].to_pandas(),\n        data.filter(pl.col(\"case_id\").is_in(case_ids))[\"target\"].to_pandas()\n    )\n\nbase_train, X_train, y_train = from_polars_to_pandas(case_ids_train)\nbase_valid, X_valid, y_valid = from_polars_to_pandas(case_ids_valid)\nbase_test, X_test, y_test = from_polars_to_pandas(case_ids_test)\n\nfor df in [X_train, X_valid, X_test]:\n    df = convert_strings(df)","metadata":{"execution":{"iopub.status.busy":"2024-05-08T03:54:53.056030Z","iopub.execute_input":"2024-05-08T03:54:53.056490Z","iopub.status.idle":"2024-05-08T03:55:07.952678Z","shell.execute_reply.started":"2024-05-08T03:54:53.056448Z","shell.execute_reply":"2024-05-08T03:55:07.951691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_train","metadata":{"execution":{"iopub.status.busy":"2024-05-08T03:55:07.954065Z","iopub.execute_input":"2024-05-08T03:55:07.954411Z","iopub.status.idle":"2024-05-08T03:55:07.971452Z","shell.execute_reply.started":"2024-05-08T03:55:07.954379Z","shell.execute_reply":"2024-05-08T03:55:07.970471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_valid","metadata":{"execution":{"iopub.status.busy":"2024-05-08T03:55:07.972632Z","iopub.execute_input":"2024-05-08T03:55:07.972937Z","iopub.status.idle":"2024-05-08T03:55:07.985299Z","shell.execute_reply.started":"2024-05-08T03:55:07.972912Z","shell.execute_reply":"2024-05-08T03:55:07.984285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_test","metadata":{"execution":{"iopub.status.busy":"2024-05-08T03:55:07.986507Z","iopub.execute_input":"2024-05-08T03:55:07.986826Z","iopub.status.idle":"2024-05-08T03:55:08.002927Z","shell.execute_reply.started":"2024-05-08T03:55:07.986797Z","shell.execute_reply":"2024-05-08T03:55:08.001782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"null_count_train = X_train.isnull().sum() / len(X_train)\ndrop_cols_train = null_count_train[null_count_train >= 0.4].index.tolist()","metadata":{"execution":{"iopub.status.busy":"2024-05-08T03:55:08.004315Z","iopub.execute_input":"2024-05-08T03:55:08.004720Z","iopub.status.idle":"2024-05-08T03:55:08.269800Z","shell.execute_reply.started":"2024-05-08T03:55:08.004684Z","shell.execute_reply":"2024-05-08T03:55:08.268559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.columns","metadata":{"execution":{"iopub.status.busy":"2024-05-08T03:55:08.271631Z","iopub.execute_input":"2024-05-08T03:55:08.272603Z","iopub.status.idle":"2024-05-08T03:55:08.280305Z","shell.execute_reply.started":"2024-05-08T03:55:08.272558Z","shell.execute_reply":"2024-05-08T03:55:08.279188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"null_count_valid = X_valid.isnull().sum() / len(X_valid)\ndrop_cols_valid = null_count_valid[null_count_valid >= 0.4].index.tolist()","metadata":{"execution":{"iopub.status.busy":"2024-05-08T03:55:08.281673Z","iopub.execute_input":"2024-05-08T03:55:08.282036Z","iopub.status.idle":"2024-05-08T03:55:08.381764Z","shell.execute_reply.started":"2024-05-08T03:55:08.282007Z","shell.execute_reply":"2024-05-08T03:55:08.380426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"null_count_test = X_test.isnull().sum() / len(X_test)\ndrop_cols_test = null_count_test[null_count_test >= 0.4].index.tolist()","metadata":{"execution":{"iopub.status.busy":"2024-05-08T03:55:08.383095Z","iopub.execute_input":"2024-05-08T03:55:08.383490Z","iopub.status.idle":"2024-05-08T03:55:08.476612Z","shell.execute_reply.started":"2024-05-08T03:55:08.383454Z","shell.execute_reply":"2024-05-08T03:55:08.475613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"drop_cols_train == drop_cols_valid","metadata":{"execution":{"iopub.status.busy":"2024-05-08T03:55:08.477930Z","iopub.execute_input":"2024-05-08T03:55:08.478323Z","iopub.status.idle":"2024-05-08T03:55:08.485309Z","shell.execute_reply.started":"2024-05-08T03:55:08.478290Z","shell.execute_reply":"2024-05-08T03:55:08.484214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"drop_cols_valid == drop_cols_test","metadata":{"execution":{"iopub.status.busy":"2024-05-08T03:55:08.486420Z","iopub.execute_input":"2024-05-08T03:55:08.486745Z","iopub.status.idle":"2024-05-08T03:55:08.497378Z","shell.execute_reply.started":"2024-05-08T03:55:08.486716Z","shell.execute_reply":"2024-05-08T03:55:08.496258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"drop_cols_train == drop_cols_test","metadata":{"execution":{"iopub.status.busy":"2024-05-08T03:55:08.498948Z","iopub.execute_input":"2024-05-08T03:55:08.499350Z","iopub.status.idle":"2024-05-08T03:55:08.508904Z","shell.execute_reply.started":"2024-05-08T03:55:08.499318Z","shell.execute_reply":"2024-05-08T03:55:08.507918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.drop(drop_cols_train, inplace=True, axis=1)\nX_valid.drop(drop_cols_valid, inplace=True, axis=1)\nX_test.drop(drop_cols_test, inplace=True, axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-05-08T03:55:08.510144Z","iopub.execute_input":"2024-05-08T03:55:08.510539Z","iopub.status.idle":"2024-05-08T03:55:08.903060Z","shell.execute_reply.started":"2024-05-08T03:55:08.510500Z","shell.execute_reply":"2024-05-08T03:55:08.901772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nulls_less = null_count_train[null_count_train < 0.4]\nnulls = nulls_less[nulls_less > 0]\nprint(nulls.index.tolist())","metadata":{"execution":{"iopub.status.busy":"2024-05-08T03:55:08.904601Z","iopub.execute_input":"2024-05-08T03:55:08.904937Z","iopub.status.idle":"2024-05-08T03:55:08.912210Z","shell.execute_reply.started":"2024-05-08T03:55:08.904907Z","shell.execute_reply":"2024-05-08T03:55:08.910914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"categoricals = X_train.dtypes[X_train.dtypes != 'float64'].index.tolist()\nfloats = X_train.dtypes[X_train.dtypes == 'float64'].index.tolist()","metadata":{"execution":{"iopub.status.busy":"2024-05-08T03:55:08.913733Z","iopub.execute_input":"2024-05-08T03:55:08.914060Z","iopub.status.idle":"2024-05-08T03:55:08.927126Z","shell.execute_reply.started":"2024-05-08T03:55:08.914031Z","shell.execute_reply":"2024-05-08T03:55:08.925880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"categoricals_set = set(categoricals)\nfloats_set = set(floats)\nnulls_set = set(nulls.index.tolist())\nprint(nulls_set.intersection(categoricals_set))\nprint(floats_set.intersection(nulls_set))","metadata":{"execution":{"iopub.status.busy":"2024-05-08T03:55:08.928864Z","iopub.execute_input":"2024-05-08T03:55:08.929734Z","iopub.status.idle":"2024-05-08T03:55:08.939291Z","shell.execute_reply.started":"2024-05-08T03:55:08.929694Z","shell.execute_reply":"2024-05-08T03:55:08.938292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train['description_5085714M'] = X_train['description_5085714M'].cat.add_categories('Credit Bureau 5085714M Not Provided')\nX_train['education_1103M'] = X_train['education_1103M'].cat.add_categories('(External Source) Education 1103M Not Provided')\nX_train['paytype1st_925L'] = X_train['paytype1st_925L'].cat.add_categories('Type of 1st Payment 925L Not Provided')\nX_train['paytype_783L'] = X_train['paytype_783L'].cat.add_categories('Payment Type 783L Not Provided')\nX_train['disbursementtype_67L'] = X_train['disbursementtype_67L'].cat.add_categories('Disbursement Type 67L Not Provided')\nX_train['education_88M'] = X_train['education_88M'].cat.add_categories('Education Level 88M Not Provided')\nX_train['maritalst_893M'] = X_train['maritalst_893M'].cat.add_categories('Marital Status 893M Not Provided')\nX_train['maritalst_385M'] = X_train['maritalst_385M'].cat.add_categories('Marital Status 385M Not Provided')\nX_train.fillna(value={'description_5085714M': 'Credit Bureau 5085714M Not Provided', \n                      'education_1103M': '(External Source) Education 1103M Not Provided', \n                      'paytype1st_925L': 'Type of 1st Payment 925L Not Provided', \n                      'paytype_783L': 'Payment Type 783L Not Provided', \n                      'disbursementtype_67L': 'Disbursement Type 67L Not Provided', \n                      'education_88M': 'Education Level 88M Not Provided', \n                      'maritalst_893M': 'Marital Status 893M Not Provided', \n                      'maritalst_385M': 'Marital Status 385M Not Provided', \n                      'maxdpdfrom6mto36m_3546853P': X_train['maxdpdfrom6mto36m_3546853P'].median(), \n                      'firstquarter_103L': X_train['firstquarter_103L'].median(), \n                      'numinsttopaygr_769L': X_train['numinsttopaygr_769L'].median(), \n                      'daysoverduetolerancedd_3976961L': X_train['daysoverduetolerancedd_3976961L'].median(), \n                      'amtinstpaidbefduel24m_4187115A': X_train['amtinstpaidbefduel24m_4187115A'].median(), \n                      'interestrate_311L': X_train['interestrate_311L'].median(), \n                      'currdebtcredtyperange_828A': X_train['currdebtcredtyperange_828A'].median(), \n                      'days30_165L': X_train['days30_165L'].median(), \n                      'numinstlsallpaid_934L': X_train['numinstlsallpaid_934L'].median(), \n                      'maxdpdlast9m_1059P': X_train['maxdpdlast9m_1059P'].median(), \n                      'numinstunpaidmax_3546851L': X_train['numinstunpaidmax_3546851L'].median(), \n                      'numinstlswithdpd10_728L': X_train['numinstlswithdpd10_728L'].median(), \n                      'cntincpaycont9m_3716944L': X_train['cntincpaycont9m_3716944L'].median(), \n                      'maxdebt4_972A': X_train['maxdebt4_972A'].median(), \n                      'thirdquarter_1082L': X_train['thirdquarter_1082L'].median(), \n                      'pctinstlsallpaidlate4d_3546849L': X_train['pctinstlsallpaidlate4d_3546849L'].median(), \n                      'days90_310L': X_train['days90_310L'].median(), \n                      'numincomingpmts_3546848L': X_train['numincomingpmts_3546848L'].median(), \n                      'eir_270L': X_train['eir_270L'].median(), \n                      'pctinstlsallpaidearl3d_427L': X_train['pctinstlsallpaidearl3d_427L'].median(), \n                      'posfpd10lastmonth_333P': X_train['posfpd10lastmonth_333P'].median(), \n                      'cntpmts24_3658933L': X_train['cntpmts24_3658933L'].median(), \n                      'commnoinclast6m_3546845L': X_train['commnoinclast6m_3546845L'].median(), \n                      'pctinstlsallpaidlat10d_839L': X_train['pctinstlsallpaidlat10d_839L'].median(), \n                      'sumoutstandtotal_3546847A': X_train['sumoutstandtotal_3546847A'].median(), \n                      'maxdpdtolerance_374P': X_train['maxdpdtolerance_374P'].median(), \n                      'numinstlswithdpd5_4187116L': X_train['numinstlswithdpd5_4187116L'].median(), \n                      'maxannuity_159A': X_train['maxannuity_159A'].median(), \n                      'mastercontrelectronic_519L': X_train['mastercontrelectronic_519L'].median(), \n                      'numinstpaidearly3d_3546850L': X_train['numinstpaidearly3d_3546850L'].median(), \n                      'numinstregularpaid_973L': X_train['numinstregularpaid_973L'].median(), \n                      'maxdpdlast12m_727P': X_train['maxdpdlast12m_727P'].median(), \n                      'numberofqueries_373L': X_train['numberofqueries_373L'].median(), \n                      'totaldebt_9A': X_train['totaldebt_9A'].median(), \n                      'days180_256L': X_train['days180_256L'].median(), \n                      'maxdpdlast24m_143P': X_train['maxdpdlast24m_143P'].median(), \n                      'numinstpaidearly5d_1087L': X_train['numinstpaidearly5d_1087L'].median(), \n                      'actualdpdtolerance_344P': X_train['actualdpdtolerance_344P'].median(), \n                      'pctinstlsallpaidlate6d_3546844L': X_train['pctinstlsallpaidlate6d_3546844L'].median(), \n                      'numinstls_657L': X_train['numinstls_657L'].median(), \n                      'numinstpaidearly_338L': X_train['numinstpaidearly_338L'].median(), \n                      'mastercontrexist_109L': X_train['mastercontrexist_109L'].median(), \n                      'maxdpdlast6m_474P': X_train['maxdpdlast6m_474P'].median(), \n                      'pctinstlsallpaidlate1d_3546856L': X_train['pctinstlsallpaidlate1d_3546856L'].median(), \n                      'price_1097A': X_train['price_1097A'].median(), \n                      'annuitynextmonth_57A': X_train['annuitynextmonth_57A'].median(), \n                      'posfpd30lastmonth_3976960P': X_train['posfpd30lastmonth_3976960P'].median(), \n                      'posfstqpd30lastmonth_3976962P': X_train['posfstqpd30lastmonth_3976962P'].median(), \n                      'secondquarter_766L': X_train['secondquarter_766L'].median(), \n                      'monthsannuity_845L': X_train['monthsannuity_845L'].median(), \n                      'fourthquarter_440L': X_train['fourthquarter_440L'].median(), \n                      'lastapprcredamount_781A': X_train['lastapprcredamount_781A'].median(), \n                      'numinstlallpaidearly3d_817L': X_train['numinstlallpaidearly3d_817L'].median(), \n                      'maininc_215A': X_train['maininc_215A'].median(), \n                      'numinstpaidlate1d_3546852L': X_train['numinstpaidlate1d_3546852L'].median(), \n                      'currdebt_22A': X_train['currdebt_22A'].median(), \n                      'maxdpdlast3m_392P': X_train['maxdpdlast3m_392P'].median(), \n                      'totalsettled_863A': X_train['totalsettled_863A'].median(), \n                      'days360_512L': X_train['days360_512L'].median(), \n                      'avgdpdtolclosure24_3658938P': X_train['avgdpdtolclosure24_3658938P'].median(), \n                      'numinstlswithoutdpd_562L': X_train['numinstlswithoutdpd_562L'].median(), \n                      'days120_123L': X_train['days120_123L'].median(), \n                      'pmtnum_254L': X_train['pmtnum_254L'].median()}, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-05-08T03:55:08.940949Z","iopub.execute_input":"2024-05-08T03:55:08.941353Z","iopub.status.idle":"2024-05-08T03:55:10.521359Z","shell.execute_reply.started":"2024-05-08T03:55:08.941318Z","shell.execute_reply":"2024-05-08T03:55:10.520189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_valid['description_5085714M'] = X_valid['description_5085714M'].cat.add_categories('Credit Bureau 5085714M Not Provided')\nX_valid['education_1103M'] = X_valid['education_1103M'].cat.add_categories('(External Source) Education 1103M Not Provided')\nX_valid['paytype1st_925L'] = X_valid['paytype1st_925L'].cat.add_categories('Type of 1st Payment 925L Not Provided')\nX_valid['paytype_783L'] = X_valid['paytype_783L'].cat.add_categories('Payment Type 783L Not Provided')\nX_valid['disbursementtype_67L'] = X_valid['disbursementtype_67L'].cat.add_categories('Disbursement Type 67L Not Provided')\nX_valid['education_88M'] = X_valid['education_88M'].cat.add_categories('Education Level 88M Not Provided')\nX_valid['maritalst_893M'] = X_valid['maritalst_893M'].cat.add_categories('Marital Status 893M Not Provided')\nX_valid['maritalst_385M'] = X_valid['maritalst_385M'].cat.add_categories('Marital Status 385M Not Provided')\nX_valid.fillna(value={'description_5085714M': 'Credit Bureau 5085714M Not Provided', \n                      'education_1103M': '(External Source) Education 1103M Not Provided', \n                      'paytype1st_925L': 'Type of 1st Payment 925L Not Provided', \n                      'paytype_783L': 'Payment Type 783L Not Provided', \n                      'disbursementtype_67L': 'Disbursement Type 67L Not Provided', \n                      'education_88M': 'Education Level 88M Not Provided', \n                      'maritalst_893M': 'Marital Status 893M Not Provided', \n                      'maritalst_385M': 'Marital Status 385M Not Provided', \n                      'maxdpdfrom6mto36m_3546853P': X_valid['maxdpdfrom6mto36m_3546853P'].median(), \n                      'firstquarter_103L': X_valid['firstquarter_103L'].median(), \n                      'numinsttopaygr_769L': X_valid['numinsttopaygr_769L'].median(), \n                      'daysoverduetolerancedd_3976961L': X_valid['daysoverduetolerancedd_3976961L'].median(), \n                      'amtinstpaidbefduel24m_4187115A': X_valid['amtinstpaidbefduel24m_4187115A'].median(), \n                      'interestrate_311L': X_valid['interestrate_311L'].median(), \n                      'currdebtcredtyperange_828A': X_valid['currdebtcredtyperange_828A'].median(), \n                      'days30_165L': X_valid['days30_165L'].median(), \n                      'numinstlsallpaid_934L': X_valid['numinstlsallpaid_934L'].median(), \n                      'maxdpdlast9m_1059P': X_valid['maxdpdlast9m_1059P'].median(), \n                      'numinstunpaidmax_3546851L': X_valid['numinstunpaidmax_3546851L'].median(), \n                      'numinstlswithdpd10_728L': X_valid['numinstlswithdpd10_728L'].median(), \n                      'cntincpaycont9m_3716944L': X_valid['cntincpaycont9m_3716944L'].median(), \n                      'maxdebt4_972A': X_valid['maxdebt4_972A'].median(), \n                      'thirdquarter_1082L': X_valid['thirdquarter_1082L'].median(), \n                      'pctinstlsallpaidlate4d_3546849L': X_valid['pctinstlsallpaidlate4d_3546849L'].median(), \n                      'days90_310L': X_valid['days90_310L'].median(), \n                      'numincomingpmts_3546848L': X_valid['numincomingpmts_3546848L'].median(), \n                      'eir_270L': X_valid['eir_270L'].median(), \n                      'pctinstlsallpaidearl3d_427L': X_valid['pctinstlsallpaidearl3d_427L'].median(), \n                      'posfpd10lastmonth_333P': X_valid['posfpd10lastmonth_333P'].median(), \n                      'cntpmts24_3658933L': X_valid['cntpmts24_3658933L'].median(), \n                      'commnoinclast6m_3546845L': X_valid['commnoinclast6m_3546845L'].median(), \n                      'pctinstlsallpaidlat10d_839L': X_valid['pctinstlsallpaidlat10d_839L'].median(), \n                      'sumoutstandtotal_3546847A': X_valid['sumoutstandtotal_3546847A'].median(), \n                      'maxdpdtolerance_374P': X_valid['maxdpdtolerance_374P'].median(), \n                      'numinstlswithdpd5_4187116L': X_valid['numinstlswithdpd5_4187116L'].median(), \n                      'maxannuity_159A': X_valid['maxannuity_159A'].median(), \n                      'mastercontrelectronic_519L': X_valid['mastercontrelectronic_519L'].median(), \n                      'numinstpaidearly3d_3546850L': X_valid['numinstpaidearly3d_3546850L'].median(), \n                      'numinstregularpaid_973L': X_valid['numinstregularpaid_973L'].median(), \n                      'maxdpdlast12m_727P': X_valid['maxdpdlast12m_727P'].median(), \n                      'numberofqueries_373L': X_valid['numberofqueries_373L'].median(), \n                      'totaldebt_9A': X_valid['totaldebt_9A'].median(), \n                      'days180_256L': X_valid['days180_256L'].median(), \n                      'maxdpdlast24m_143P': X_valid['maxdpdlast24m_143P'].median(), \n                      'numinstpaidearly5d_1087L': X_valid['numinstpaidearly5d_1087L'].median(), \n                      'actualdpdtolerance_344P': X_valid['actualdpdtolerance_344P'].median(), \n                      'pctinstlsallpaidlate6d_3546844L': X_valid['pctinstlsallpaidlate6d_3546844L'].median(), \n                      'numinstls_657L': X_valid['numinstls_657L'].median(), \n                      'numinstpaidearly_338L': X_valid['numinstpaidearly_338L'].median(), \n                      'mastercontrexist_109L': X_valid['mastercontrexist_109L'].median(), \n                      'maxdpdlast6m_474P': X_valid['maxdpdlast6m_474P'].median(), \n                      'pctinstlsallpaidlate1d_3546856L': X_valid['pctinstlsallpaidlate1d_3546856L'].median(), \n                      'price_1097A': X_valid['price_1097A'].median(), \n                      'annuitynextmonth_57A': X_valid['annuitynextmonth_57A'].median(), \n                      'posfpd30lastmonth_3976960P': X_valid['posfpd30lastmonth_3976960P'].median(), \n                      'posfstqpd30lastmonth_3976962P': X_valid['posfstqpd30lastmonth_3976962P'].median(), \n                      'secondquarter_766L': X_valid['secondquarter_766L'].median(), \n                      'monthsannuity_845L': X_valid['monthsannuity_845L'].median(), \n                      'fourthquarter_440L': X_valid['fourthquarter_440L'].median(), \n                      'lastapprcredamount_781A': X_valid['lastapprcredamount_781A'].median(), \n                      'numinstlallpaidearly3d_817L': X_valid['numinstlallpaidearly3d_817L'].median(), \n                      'maininc_215A': X_valid['maininc_215A'].median(), \n                      'numinstpaidlate1d_3546852L': X_valid['numinstpaidlate1d_3546852L'].median(), \n                      'currdebt_22A': X_valid['currdebt_22A'].median(), \n                      'maxdpdlast3m_392P': X_valid['maxdpdlast3m_392P'].median(), \n                      'totalsettled_863A': X_valid['totalsettled_863A'].median(), \n                      'days360_512L': X_valid['days360_512L'].median(), \n                      'avgdpdtolclosure24_3658938P': X_valid['avgdpdtolclosure24_3658938P'].median(), \n                      'numinstlswithoutdpd_562L': X_valid['numinstlswithoutdpd_562L'].median(), \n                      'days120_123L': X_valid['days120_123L'].median(), \n                      'pmtnum_254L': X_valid['pmtnum_254L'].median()}, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-05-08T03:55:10.523516Z","iopub.execute_input":"2024-05-08T03:55:10.523837Z","iopub.status.idle":"2024-05-08T03:55:11.016023Z","shell.execute_reply.started":"2024-05-08T03:55:10.523809Z","shell.execute_reply":"2024-05-08T03:55:11.014851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test['description_5085714M'] = X_test['description_5085714M'].cat.add_categories('Credit Bureau 5085714M Not Provided')\nX_test['education_1103M'] = X_test['education_1103M'].cat.add_categories('(External Source) Education 1103M Not Provided')\nX_test['paytype1st_925L'] = X_test['paytype1st_925L'].cat.add_categories('Type of 1st Payment 925L Not Provided')\nX_test['paytype_783L'] = X_test['paytype_783L'].cat.add_categories('Payment Type 783L Not Provided')\nX_test['disbursementtype_67L'] = X_test['disbursementtype_67L'].cat.add_categories('Disbursement Type 67L Not Provided')\nX_test['education_88M'] = X_test['education_88M'].cat.add_categories('Education Level 88M Not Provided')\nX_test['maritalst_893M'] = X_test['maritalst_893M'].cat.add_categories('Marital Status 893M Not Provided')\nX_test['maritalst_385M'] = X_test['maritalst_385M'].cat.add_categories('Marital Status 385M Not Provided')\nX_test.fillna(value={'description_5085714M': 'Credit Bureau 5085714M Not Provided', \n                      'education_1103M': '(External Source) Education 1103M Not Provided', \n                      'paytype1st_925L': 'Type of 1st Payment 925L Not Provided', \n                      'paytype_783L': 'Payment Type 783L Not Provided', \n                      'disbursementtype_67L': 'Disbursement Type 67L Not Provided', \n                      'education_88M': 'Education Level 88M Not Provided', \n                      'maritalst_893M': 'Marital Status 893M Not Provided', \n                      'maritalst_385M': 'Marital Status 385M Not Provided', \n                      'maxdpdfrom6mto36m_3546853P': X_test['maxdpdfrom6mto36m_3546853P'].median(), \n                      'firstquarter_103L': X_test['firstquarter_103L'].median(), \n                      'numinsttopaygr_769L': X_test['numinsttopaygr_769L'].median(), \n                      'daysoverduetolerancedd_3976961L': X_test['daysoverduetolerancedd_3976961L'].median(), \n                      'amtinstpaidbefduel24m_4187115A': X_test['amtinstpaidbefduel24m_4187115A'].median(), \n                      'interestrate_311L': X_test['interestrate_311L'].median(), \n                      'currdebtcredtyperange_828A': X_test['currdebtcredtyperange_828A'].median(), \n                      'days30_165L': X_test['days30_165L'].median(), \n                      'numinstlsallpaid_934L': X_test['numinstlsallpaid_934L'].median(), \n                      'maxdpdlast9m_1059P': X_test['maxdpdlast9m_1059P'].median(), \n                      'numinstunpaidmax_3546851L': X_test['numinstunpaidmax_3546851L'].median(), \n                      'numinstlswithdpd10_728L': X_test['numinstlswithdpd10_728L'].median(), \n                      'cntincpaycont9m_3716944L': X_test['cntincpaycont9m_3716944L'].median(), \n                      'maxdebt4_972A': X_test['maxdebt4_972A'].median(), \n                      'thirdquarter_1082L': X_test['thirdquarter_1082L'].median(), \n                      'pctinstlsallpaidlate4d_3546849L': X_test['pctinstlsallpaidlate4d_3546849L'].median(), \n                      'days90_310L': X_test['days90_310L'].median(), \n                      'numincomingpmts_3546848L': X_test['numincomingpmts_3546848L'].median(), \n                      'eir_270L': X_test['eir_270L'].median(), \n                      'pctinstlsallpaidearl3d_427L': X_test['pctinstlsallpaidearl3d_427L'].median(), \n                      'posfpd10lastmonth_333P': X_test['posfpd10lastmonth_333P'].median(), \n                      'cntpmts24_3658933L': X_test['cntpmts24_3658933L'].median(), \n                      'commnoinclast6m_3546845L': X_test['commnoinclast6m_3546845L'].median(), \n                      'pctinstlsallpaidlat10d_839L': X_test['pctinstlsallpaidlat10d_839L'].median(), \n                      'sumoutstandtotal_3546847A': X_test['sumoutstandtotal_3546847A'].median(), \n                      'maxdpdtolerance_374P': X_test['maxdpdtolerance_374P'].median(), \n                      'numinstlswithdpd5_4187116L': X_test['numinstlswithdpd5_4187116L'].median(), \n                      'maxannuity_159A': X_test['maxannuity_159A'].median(), \n                      'mastercontrelectronic_519L': X_test['mastercontrelectronic_519L'].median(), \n                      'numinstpaidearly3d_3546850L': X_test['numinstpaidearly3d_3546850L'].median(), \n                      'numinstregularpaid_973L': X_test['numinstregularpaid_973L'].median(), \n                      'maxdpdlast12m_727P': X_test['maxdpdlast12m_727P'].median(), \n                      'numberofqueries_373L': X_test['numberofqueries_373L'].median(), \n                      'totaldebt_9A': X_test['totaldebt_9A'].median(), \n                      'days180_256L': X_test['days180_256L'].median(), \n                      'maxdpdlast24m_143P': X_test['maxdpdlast24m_143P'].median(), \n                      'numinstpaidearly5d_1087L': X_test['numinstpaidearly5d_1087L'].median(), \n                      'actualdpdtolerance_344P': X_test['actualdpdtolerance_344P'].median(), \n                      'pctinstlsallpaidlate6d_3546844L': X_test['pctinstlsallpaidlate6d_3546844L'].median(), \n                      'numinstls_657L': X_test['numinstls_657L'].median(), \n                      'numinstpaidearly_338L': X_test['numinstpaidearly_338L'].median(), \n                      'mastercontrexist_109L': X_test['mastercontrexist_109L'].median(), \n                      'maxdpdlast6m_474P': X_test['maxdpdlast6m_474P'].median(), \n                      'pctinstlsallpaidlate1d_3546856L': X_test['pctinstlsallpaidlate1d_3546856L'].median(), \n                      'price_1097A': X_test['price_1097A'].median(), \n                      'annuitynextmonth_57A': X_test['annuitynextmonth_57A'].median(), \n                      'posfpd30lastmonth_3976960P': X_test['posfpd30lastmonth_3976960P'].median(), \n                      'posfstqpd30lastmonth_3976962P': X_test['posfstqpd30lastmonth_3976962P'].median(), \n                      'secondquarter_766L': X_test['secondquarter_766L'].median(), \n                      'monthsannuity_845L': X_test['monthsannuity_845L'].median(), \n                      'fourthquarter_440L': X_test['fourthquarter_440L'].median(), \n                      'lastapprcredamount_781A': X_test['lastapprcredamount_781A'].median(), \n                      'numinstlallpaidearly3d_817L': X_test['numinstlallpaidearly3d_817L'].median(), \n                      'maininc_215A': X_test['maininc_215A'].median(), \n                      'numinstpaidlate1d_3546852L': X_test['numinstpaidlate1d_3546852L'].median(), \n                      'currdebt_22A': X_test['currdebt_22A'].median(), \n                      'maxdpdlast3m_392P': X_test['maxdpdlast3m_392P'].median(), \n                      'totalsettled_863A': X_test['totalsettled_863A'].median(), \n                      'days360_512L': X_test['days360_512L'].median(), \n                      'avgdpdtolclosure24_3658938P': X_test['avgdpdtolclosure24_3658938P'].median(), \n                      'numinstlswithoutdpd_562L': X_test['numinstlswithoutdpd_562L'].median(), \n                      'days120_123L': X_test['days120_123L'].median(), \n                      'pmtnum_254L': X_test['pmtnum_254L'].median()}, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-05-08T03:55:11.017986Z","iopub.execute_input":"2024-05-08T03:55:11.018354Z","iopub.status.idle":"2024-05-08T03:55:11.510706Z","shell.execute_reply.started":"2024-05-08T03:55:11.018323Z","shell.execute_reply":"2024-05-08T03:55:11.509732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_dummy = pd.get_dummies(X_train)\nX_valid_dummy = pd.get_dummies(X_valid)\nX_test_dummy = pd.get_dummies(X_test)","metadata":{"execution":{"iopub.status.busy":"2024-05-08T03:55:11.517778Z","iopub.execute_input":"2024-05-08T03:55:11.518130Z","iopub.status.idle":"2024-05-08T03:55:18.809037Z","shell.execute_reply.started":"2024-05-08T03:55:11.518099Z","shell.execute_reply":"2024-05-08T03:55:18.807677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_dummy.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-08T03:55:18.810344Z","iopub.execute_input":"2024-05-08T03:55:18.810684Z","iopub.status.idle":"2024-05-08T03:55:18.817675Z","shell.execute_reply.started":"2024-05-08T03:55:18.810654Z","shell.execute_reply":"2024-05-08T03:55:18.816643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_valid_dummy.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-08T03:55:18.819063Z","iopub.execute_input":"2024-05-08T03:55:18.819510Z","iopub.status.idle":"2024-05-08T03:55:18.830132Z","shell.execute_reply.started":"2024-05-08T03:55:18.819455Z","shell.execute_reply":"2024-05-08T03:55:18.829193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test_dummy.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-08T03:55:18.831424Z","iopub.execute_input":"2024-05-08T03:55:18.831798Z","iopub.status.idle":"2024-05-08T03:55:18.843127Z","shell.execute_reply.started":"2024-05-08T03:55:18.831766Z","shell.execute_reply":"2024-05-08T03:55:18.841963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def standardize_columns(df1, df2): \n    colslist_df1 = np.setdiff1d(df1.columns, df2.columns).tolist()\n    colslist_df2 = np.setdiff1d(df2.columns, df1.columns).tolist()\n    valueslist_df1 = [0.0] * df1.shape[0]\n    valueslist_df2 = [0.0] * df2.shape[0]\n    datadict_df1 = {col: valueslist_df1 for col in colslist_df2}\n    datadict_df2 = {col: valueslist_df2 for col in colslist_df1}\n    append_df1 = pd.DataFrame(datadict_df1, columns=colslist_df2)\n    append_df2 = pd.DataFrame(datadict_df2, columns=colslist_df1)\n    return pd.concat([df1, append_df1], axis=1), pd.concat([df2, append_df2], axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-05-08T03:55:18.844663Z","iopub.execute_input":"2024-05-08T03:55:18.845014Z","iopub.status.idle":"2024-05-08T03:55:18.854142Z","shell.execute_reply.started":"2024-05-08T03:55:18.844984Z","shell.execute_reply":"2024-05-08T03:55:18.853076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"while (len(X_train_dummy.columns) != len(X_test_dummy.columns)) or (len(X_train_dummy.columns) != len(X_valid_dummy.columns)) or (len(X_valid_dummy.columns) != len(X_test_dummy.columns)):\n    X_train_dummy, X_test_dummy = standardize_columns(X_train_dummy, X_test_dummy)\n    X_train_dummy, X_valid_dummy = standardize_columns(X_train_dummy, X_valid_dummy)\n    X_valid_dummy, X_test_dummy = standardize_columns(X_valid_dummy, X_test_dummy)","metadata":{"execution":{"iopub.status.busy":"2024-05-08T03:55:59.954915Z","iopub.execute_input":"2024-05-08T03:55:59.955971Z","iopub.status.idle":"2024-05-08T03:56:14.770814Z","shell.execute_reply.started":"2024-05-08T03:55:59.955928Z","shell.execute_reply":"2024-05-08T03:56:14.769848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_dummy.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-08T03:56:14.772677Z","iopub.execute_input":"2024-05-08T03:56:14.773023Z","iopub.status.idle":"2024-05-08T03:56:14.779969Z","shell.execute_reply.started":"2024-05-08T03:56:14.772994Z","shell.execute_reply":"2024-05-08T03:56:14.778887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_valid_dummy.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-08T03:56:14.781338Z","iopub.execute_input":"2024-05-08T03:56:14.781692Z","iopub.status.idle":"2024-05-08T03:56:14.792961Z","shell.execute_reply.started":"2024-05-08T03:56:14.781654Z","shell.execute_reply":"2024-05-08T03:56:14.791899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test_dummy.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-08T03:56:14.795071Z","iopub.execute_input":"2024-05-08T03:56:14.795426Z","iopub.status.idle":"2024-05-08T03:56:14.804459Z","shell.execute_reply.started":"2024-05-08T03:56:14.795398Z","shell.execute_reply":"2024-05-08T03:56:14.803598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_dummy = np.asarray(X_train_dummy).astype('float32')\nX_valid_dummy = np.asarray(X_valid_dummy).astype('float32')\nX_test_dummy = np.asarray(X_test_dummy).astype('float32')","metadata":{"execution":{"iopub.status.busy":"2024-05-08T03:56:14.805452Z","iopub.execute_input":"2024-05-08T03:56:14.805727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Exploration","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(10, 10))\nflierprops = dict(marker='+', markerfacecolor='orange', markersize=3,\n                  linestyle='none')\n#plt.boxplot(data['avginstallast24m_3658937A'], flierprops=flierprops)\nplt.hist(data['avginstallast24m_3658937A'], bins=75, color='red', edgecolor='red')\nplt.yscale('log')\nplt.xlabel('Average installment paid by the client over the past 24 months ($)')\nplt.ylabel('Number of observations')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 10))\nplt.boxplot(data['annuity_780A'], flierprops=flierprops)\nplt.yscale('log')\nplt.ylabel('Monthly Annuity ($)')\nplt.title('Distribution of Monthly Annuity Amount')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(5, 5))\ntarget_counts = data['target'].to_pandas().value_counts()\ncounts = target_counts.values\nlabels = target_counts.index\nplt.bar(labels, counts, color='lightblue', edgecolor='black')\nplt.xlabel('Defaulted?')\nplt.ylabel('Count')\nplt.xticks([0, 1], labels=['False', 'True'])\nplt.title('Count of each value in the target column')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 10))\nplt.hist(data['currdebt_22A'], color='green', edgecolor='black')\nplt.yscale('log')\nplt.xlabel('Amount of Debt ($)')\nplt.ylabel('Count')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training the Model","metadata":{}},{"cell_type":"code","source":"%whos","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Code for viewing how much memory each session variable is taking up\n\nipython_vars = [\"In\", \"Out\", \"exit\", \"quit\", \"get_ipython\", \"ipython_vars\"]\n\nmem = {\n    key: value\n    for key, value in sorted(\n        [\n            (x, sys.getsizeof(globals().get(x)))\n            for x in dir()\n            if not x.startswith(\"_\") and x not in sys.modules and x not in ipython_vars\n        ],\n        key=lambda x: x[1],\n        reverse=True,\n    )\n}\n\nmem","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_basetable, train_static, train_static_cb, test_basetable, test_static, test_static_cb\ndel selected_static_cols, selected_static_cb_cols, list_cb, case_ids, case_ids_train, case_ids_test\ndel case_ids_valid, cols_pred, df, floats, drop_cols_test, drop_cols_train, drop_cols_valid, flierprops\ndel floats_set, nulls_set, null_count_test, null_count_train, null_count_valid, nulls_less, nulls\ndel categoricals, categoricals_set, convert_strings, from_polars_to_pandas, set_table_dtypes\ndel standardize_columns, train_test_split, counts, dataPath, target_counts","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mem = {\n    key: value\n    for key, value in sorted(\n        [\n            (x, sys.getsizeof(globals().get(x)))\n            for x in dir()\n            if not x.startswith(\"_\") and x not in sys.modules and x not in ipython_vars\n        ],\n        key=lambda x: x[1],\n        reverse=True,\n    )\n}\nmem","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del mem","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nn = Seq()\nnn.add(Dense(2500, input_shape=(977,), activation='relu', kernel_regularizer=l2(0.0001)))\nnn.add(Dense(1250, activation='relu', kernel_regularizer=l2(0.0001)))\nnn.add(Dense(625, activation='relu', kernel_regularizer=l2(0.0001)))\nnn.add(Dropout(0.4))\nnn.add(Dense(400, activation='relu', kernel_regularizer=l2(0.0001)))\nnn.add(Dropout(0.2))\nnn.add(Dense(250, activation='relu', kernel_regularizer=l2(0.0001)))\nnn.add(Dense(125, activation='relu', kernel_regularizer=l2(0.0001)))\nnn.add(Dropout(0.5))\nnn.add(Dense(1, activation='softmax'))\nadam = ks.optimizers.Adam(learning_rate=0.000001)","metadata":{"execution":{"iopub.status.busy":"2024-05-08T03:55:38.581874Z","iopub.status.idle":"2024-05-08T03:55:38.582285Z","shell.execute_reply.started":"2024-05-08T03:55:38.582066Z","shell.execute_reply":"2024-05-08T03:55:38.582084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scaler = StandardScaler()\nX_train_scaled = scaler.fit_transform(X_train_dummy)\nX_valid_scaled = scaler.transform(X_valid_dummy)\naucmetric = AUC(num_thresholds=500)","metadata":{"execution":{"iopub.status.busy":"2024-05-08T03:55:38.583558Z","iopub.status.idle":"2024-05-08T03:55:38.583993Z","shell.execute_reply.started":"2024-05-08T03:55:38.583797Z","shell.execute_reply":"2024-05-08T03:55:38.583816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nn.compile(loss='binary_crossentropy', optimizer=adam, metrics=[aucmetric])\nclass_weights = compute_class_weight(class_weight='balanced', classes=np.unique(y_train), y=y_train)\nweights_dict = {i:w for i,w in enumerate(class_weights)}","metadata":{"execution":{"iopub.status.busy":"2024-05-08T03:55:38.585766Z","iopub.status.idle":"2024-05-08T03:55:38.586334Z","shell.execute_reply.started":"2024-05-08T03:55:38.586028Z","shell.execute_reply":"2024-05-08T03:55:38.586054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.debugging.disable_traceback_filtering()","metadata":{"execution":{"iopub.status.busy":"2024-05-08T03:55:38.588248Z","iopub.status.idle":"2024-05-08T03:55:38.588796Z","shell.execute_reply.started":"2024-05-08T03:55:38.588508Z","shell.execute_reply":"2024-05-08T03:55:38.588535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = nn.fit(X_train_scaled, y_train, batch_size=64, verbose=1, epochs=10, validation_data=(X_valid_scaled, y_valid), class_weight=weights_dict)","metadata":{"execution":{"iopub.status.busy":"2024-05-08T03:55:38.590218Z","iopub.status.idle":"2024-05-08T03:55:38.590680Z","shell.execute_reply.started":"2024-05-08T03:55:38.590473Z","shell.execute_reply":"2024-05-08T03:55:38.590492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_test = np.asarray(y_test).astype('float32')\nauc1 = nn.evaluate(X_test_dummy, y_test)\nprint(auc1)\nprint('AUC: %.2f' % (auc1[1]*100))","metadata":{"execution":{"iopub.status.busy":"2024-05-07T23:08:40.160788Z","iopub.execute_input":"2024-05-07T23:08:40.161340Z","iopub.status.idle":"2024-05-07T23:08:58.557028Z","shell.execute_reply.started":"2024-05-07T23:08:40.161297Z","shell.execute_reply":"2024-05-07T23:08:58.555868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplot(1, 1, figsize=(8, 6), dpi=300)\nax.plot(history.history['auc'])\nax.plot(history.history['val_auc'])\nax.set_title('AUC')\nax.set_ylabel('AUC')\nax.set_xlabel('Epoch')\nax.legend(['training', 'validation'], loc='upper right')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Train: {X_train_dummy.shape}\")\nprint(f\"Valid: {X_valid_dummy.shape}\")\nprint(f\"Test: {X_test_dummy.shape}\")","metadata":{"execution":{"iopub.status.busy":"2024-04-23T10:47:03.204844Z","iopub.status.idle":"2024-04-23T10:47:03.205244Z","shell.execute_reply.started":"2024-04-23T10:47:03.205056Z","shell.execute_reply":"2024-04-23T10:47:03.205074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del X_train_dummy, X_valid_dummy, X_test_dummy","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# after\nlgb_train = lgb.Dataset(X_train, label=y_train)\nlgb_valid = lgb.Dataset(X_valid, label=y_valid, reference=lgb_train)\n\nparams = {\n    \"boosting_type\": \"gbdt\",\n    \"objective\": \"binary\",\n    \"metric\": \"auc\",\n    \"max_depth\": 10,  \n    \"learning_rate\": 0.05,\n    \"n_estimators\": 1000,  \n    \"colsample_bytree\": 0.8,\n    \"colsample_bynode\": 0.8,\n    \"verbose\": -1,\n    \"random_state\": 42,\n    \"reg_alpha\": 0.1,\n    \"reg_lambda\": 10,\n    \"extra_trees\":True,\n    'num_leaves':64,\n    \"verbose\": -1,\n}\n\ngbm = lgb.train(\n    params,\n    lgb_train,\n    valid_sets=lgb_valid,\n    callbacks=[lgb.log_evaluation(50), lgb.early_stopping(10)]\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-07T22:40:16.025033Z","iopub.execute_input":"2024-05-07T22:40:16.025450Z","iopub.status.idle":"2024-05-07T22:43:26.215809Z","shell.execute_reply.started":"2024-05-07T22:40:16.025416Z","shell.execute_reply":"2024-05-07T22:43:26.214571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for base, X in [(base_train, X_train), (base_valid, X_valid), (base_test, X_test)]:\n    y_pred = gbm.predict(X, num_iteration=gbm.best_iteration)\n    base[\"score\"] = y_pred\n\nprint(f'The AUC score on the train set is: {roc_auc_score(base_train[\"target\"], base_train[\"score\"])}') \nprint(f'The AUC score on the valid set is: {roc_auc_score(base_valid[\"target\"], base_valid[\"score\"])}') \nprint(f'The AUC score on the test set is: {roc_auc_score(base_test[\"target\"], base_test[\"score\"])}')  ","metadata":{"execution":{"iopub.status.busy":"2024-05-07T22:44:58.814933Z","iopub.execute_input":"2024-05-07T22:44:58.816001Z","iopub.status.idle":"2024-05-07T22:45:55.739652Z","shell.execute_reply.started":"2024-05-07T22:44:58.815959Z","shell.execute_reply":"2024-05-07T22:45:55.738305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def gini_stability(base, w_fallingrate=88.0, w_resstd=-0.5):\n    gini_in_time = base.loc[:, [\"WEEK_NUM\", \"target\", \"score\"]]\\\n        .sort_values(\"WEEK_NUM\")\\\n        .groupby(\"WEEK_NUM\")[[\"target\", \"score\"]]\\\n        .apply(lambda x: 2*roc_auc_score(x[\"target\"], x[\"score\"])-1).tolist()\n    \n    x = np.arange(len(gini_in_time))\n    y = gini_in_time\n    a, b = np.polyfit(x, y, 1)\n    y_hat = a*x + b\n    residuals = y - y_hat\n    res_std = np.std(residuals)\n    avg_gini = np.mean(gini_in_time)\n    return avg_gini + w_fallingrate * min(0, a) + w_resstd * res_std\n\nstability_score_train = gini_stability(base_train)\nstability_score_valid = gini_stability(base_valid)\nstability_score_test = gini_stability(base_test)\n\nprint(f'The stability score on the train set is: {stability_score_train}') \nprint(f'The stability score on the valid set is: {stability_score_valid}') \nprint(f'The stability score on the test set is: {stability_score_test}')  ","metadata":{"execution":{"iopub.status.busy":"2024-05-07T22:45:55.741428Z","iopub.execute_input":"2024-05-07T22:45:55.742220Z","iopub.status.idle":"2024-05-07T22:45:56.918761Z","shell.execute_reply.started":"2024-05-07T22:45:55.742184Z","shell.execute_reply":"2024-05-07T22:45:56.917578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"########################################","metadata":{"execution":{"iopub.status.busy":"2024-04-23T08:53:53.547025Z","iopub.execute_input":"2024-04-23T08:53:53.547636Z","iopub.status.idle":"2024-04-23T08:53:53.552859Z","shell.execute_reply.started":"2024-04-23T08:53:53.547597Z","shell.execute_reply":"2024-04-23T08:53:53.551726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def filter_cols(df):\n#     for col in df.columns:\n#         if col not in [\"target\", \"case_id\", \"WEEK_NUM\"]:\n#             isnull = df[col].is_null().mean()\n#             if isnull > 0.7:\n#                 df = df.drop(col)\n\n#     for col in df.columns:\n#         if (col not in [\"target\", \"case_id\", \"WEEK_NUM\"]) & (df[col].dtype == pl.String):\n#             freq = df[col].n_unique()\n#             if (freq == 1) | (freq > 200):\n#                 df = df.drop(col)\n\n#     return df","metadata":{"execution":{"iopub.status.busy":"2024-04-23T08:53:53.554348Z","iopub.execute_input":"2024-04-23T08:53:53.554839Z","iopub.status.idle":"2024-04-23T08:53:53.567184Z","shell.execute_reply.started":"2024-04-23T08:53:53.554796Z","shell.execute_reply":"2024-04-23T08:53:53.565866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# t = filter_cols(data)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T08:53:53.569411Z","iopub.execute_input":"2024-04-23T08:53:53.569883Z","iopub.status.idle":"2024-04-23T08:53:54.785527Z","shell.execute_reply.started":"2024-04-23T08:53:53.569848Z","shell.execute_reply":"2024-04-23T08:53:54.784426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# t.shape","metadata":{"execution":{"iopub.status.busy":"2024-04-23T08:53:54.786924Z","iopub.execute_input":"2024-04-23T08:53:54.787957Z","iopub.status.idle":"2024-04-23T08:53:54.794506Z","shell.execute_reply.started":"2024-04-23T08:53:54.787916Z","shell.execute_reply":"2024-04-23T08:53:54.793392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # dropping these bc xgboost did not like these\n# cols = ['credtype_322L', 'disbursementtype_67L', 'lastapprcommoditycat_1041M', \n#         'lastcancelreason_561M', 'lastrejectcommoditycat_161M', \n#         'lastrejectcommodtypec_5251769M', 'lastrejectreason_759M', \n#         'lastrejectreasonclient_4145040M', 'paytype1st_925L', 'paytype_783L',\n#         'description_5085714M', 'education_1103M', 'education_88M', \n#         'maritalst_385M', 'maritalst_893M', 'requesttype_4525192L']","metadata":{"execution":{"iopub.status.busy":"2024-04-23T08:53:54.795998Z","iopub.execute_input":"2024-04-23T08:53:54.797128Z","iopub.status.idle":"2024-04-23T08:53:54.807218Z","shell.execute_reply.started":"2024-04-23T08:53:54.79709Z","shell.execute_reply":"2024-04-23T08:53:54.806309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df = t.drop(cols)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T08:53:54.808607Z","iopub.execute_input":"2024-04-23T08:53:54.809709Z","iopub.status.idle":"2024-04-23T08:53:54.828881Z","shell.execute_reply.started":"2024-04-23T08:53:54.809658Z","shell.execute_reply":"2024-04-23T08:53:54.827649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# case_ids = df[\"case_id\"].unique().shuffle(seed = 1)\n# case_ids_train, case_ids_test = train_test_split(case_ids, train_size=0.60, random_state=1)\n# case_ids_valid, case_ids_test = train_test_split(case_ids_test, train_size=0.5, random_state=1)\n\n# cols_pred = []\n# for col in df.columns:\n#     if (col[-1].isupper() and col[:-1].islower()) or (\"_max\" in col):\n#         cols_pred.append(col)\n\n# def from_polars_to_pandas(case_ids: pl.DataFrame) -> pl.DataFrame:\n#     return (\n#         df.filter(pl.col(\"case_id\").is_in(case_ids))[[\"case_id\", \"WEEK_NUM\", \"target\"]].to_pandas(),\n#         df.filter(pl.col(\"case_id\").is_in(case_ids))[cols_pred].to_pandas(),\n#         df.filter(pl.col(\"case_id\").is_in(case_ids))[\"target\"].to_pandas()\n#     )\n\n# base_train, X_train, y_train = from_polars_to_pandas(case_ids_train)\n# base_valid, X_valid, y_valid = from_polars_to_pandas(case_ids_valid)\n# base_test, X_test, y_test = from_polars_to_pandas(case_ids_test)\n\n# # for df in [X_train, X_valid, X_test]:\n# #     df = convert_strings(df)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T08:53:54.830476Z","iopub.execute_input":"2024-04-23T08:53:54.833989Z","iopub.status.idle":"2024-04-23T08:53:59.390361Z","shell.execute_reply.started":"2024-04-23T08:53:54.833945Z","shell.execute_reply":"2024-04-23T08:53:59.389333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.preprocessing import LabelEncoder\n\n# # Initialize the label encoder\n# label_encoder = LabelEncoder()\n\n# # Apply label encoding to each categorical column\n# for column in cols_pred:\n#     if X_train[column].dtype.name == 'category':\n#         # Fit and transform the data\n#         X_train[column] = label_encoder.fit_transform(X_train[column].astype(str))\n#         X_valid[column] = label_encoder.transform(X_valid[column].astype(str))\n#         X_test[column] = label_encoder.transform(X_test[column].astype(str))\n","metadata":{"execution":{"iopub.status.busy":"2024-04-23T08:53:59.392791Z","iopub.execute_input":"2024-04-23T08:53:59.393733Z","iopub.status.idle":"2024-04-23T08:53:59.409294Z","shell.execute_reply.started":"2024-04-23T08:53:59.393682Z","shell.execute_reply":"2024-04-23T08:53:59.408301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.model_selection import train_test_split\n# from sklearn.metrics import accuracy_score\n# from xgboost import XGBClassifier\n# import xgboost as xgb\n# model = xgb.XGBClassifier(\n#     objective='binary:logistic',\n#     tree_method=\"hist\",\n#     enable_categorical=True,\n#     eval_metric='auc',\n#     subsample=1,\n#     colsample_bytree=1,\n#     min_child_weight=1,\n#     max_depth=20,\n#     n_estimators=800,\n#     random_state=42,\n# )","metadata":{"execution":{"iopub.status.busy":"2024-04-23T09:01:49.957024Z","iopub.execute_input":"2024-04-23T09:01:49.957997Z","iopub.status.idle":"2024-04-23T09:01:49.965807Z","shell.execute_reply.started":"2024-04-23T09:01:49.95795Z","shell.execute_reply":"2024-04-23T09:01:49.964385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model.fit(\n#     X_train, y_train,\n#     eval_set=[(X_valid, y_valid)],\n#     early_stopping_rounds=10,\n#     verbose=True,\n# )","metadata":{"execution":{"iopub.status.busy":"2024-04-23T09:01:52.67274Z","iopub.execute_input":"2024-04-23T09:01:52.674164Z","iopub.status.idle":"2024-04-23T09:05:01.050074Z","shell.execute_reply.started":"2024-04-23T09:01:52.674107Z","shell.execute_reply":"2024-04-23T09:05:01.048833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for base, X in [(base_train, X_train), (base_valid, X_valid), (base_test, X_test)]:\n#     y_pred = model.predict(X)\n#     base[\"score\"] = y_pred\n\n# print(f'The AUC score on the train set is: {roc_auc_score(base_train[\"target\"], base_train[\"score\"])}') \n# print(f'The AUC score on the valid set is: {roc_auc_score(base_valid[\"target\"], base_valid[\"score\"])}') \n# print(f'The AUC score on the test set is: {roc_auc_score(base_test[\"target\"], base_test[\"score\"])}')  ","metadata":{"execution":{"iopub.status.busy":"2024-04-23T09:07:37.981662Z","iopub.execute_input":"2024-04-23T09:07:37.982692Z","iopub.status.idle":"2024-04-23T09:07:55.384182Z","shell.execute_reply.started":"2024-04-23T09:07:37.982646Z","shell.execute_reply":"2024-04-23T09:07:55.382872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def gini_stability(base, w_fallingrate=88.0, w_resstd=-0.5):\n#     gini_in_time = base.loc[:, [\"WEEK_NUM\", \"target\", \"score\"]]\\\n#         .sort_values(\"WEEK_NUM\")\\\n#         .groupby(\"WEEK_NUM\")[[\"target\", \"score\"]]\\\n#         .apply(lambda x: 2*roc_auc_score(x[\"target\"], x[\"score\"])-1).tolist()\n    \n#     x = np.arange(len(gini_in_time))\n#     y = gini_in_time\n#     a, b = np.polyfit(x, y, 1)\n#     y_hat = a*x + b\n#     residuals = y - y_hat\n#     res_std = np.std(residuals)\n#     avg_gini = np.mean(gini_in_time)\n#     return avg_gini + w_fallingrate * min(0, a) + w_resstd * res_std\n\n# stability_score_train = gini_stability(base_train)\n# stability_score_valid = gini_stability(base_valid)\n# stability_score_test = gini_stability(base_test)\n\n# print(f'The stability score on the train set is: {stability_score_train}') \n# print(f'The stability score on the valid set is: {stability_score_valid}') \n# print(f'The stability score on the test set is: {stability_score_test}')  ","metadata":{"execution":{"iopub.status.busy":"2024-04-23T09:07:57.531752Z","iopub.execute_input":"2024-04-23T09:07:57.532215Z","iopub.status.idle":"2024-04-23T09:07:58.445442Z","shell.execute_reply.started":"2024-04-23T09:07:57.53218Z","shell.execute_reply":"2024-04-23T09:07:58.444176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2024-04-23T09:05:02.247356Z","iopub.status.idle":"2024-04-23T09:05:02.247815Z","shell.execute_reply.started":"2024-04-23T09:05:02.247585Z","shell.execute_reply":"2024-04-23T09:05:02.247605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submission\n\nScoring the submission dataset is below, we need to take care of new categories. Then we save the score as a last step. ","metadata":{}},{"cell_type":"code","source":"X_submission = data_submission[cols_pred].to_pandas()\nX_submission = convert_strings(X_submission)\ncategorical_cols = X_train.select_dtypes(include=['category']).columns\n\nfor col in categorical_cols:\n    train_categories = set(X_train[col].cat.categories)\n    submission_categories = set(X_submission[col].cat.categories)\n    new_categories = submission_categories - train_categories\n    X_submission.loc[X_submission[col].isin(new_categories), col] = \"Unknown\"\n    new_dtype = pd.CategoricalDtype(categories=train_categories, ordered=True)\n    X_train[col] = X_train[col].astype(new_dtype)\n    X_submission[col] = X_submission[col].astype(new_dtype)\n\ny_submission_pred = gbm.predict(X_submission, num_iteration=gbm.best_iteration)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T09:30:20.786355Z","iopub.execute_input":"2024-04-23T09:30:20.786924Z","iopub.status.idle":"2024-04-23T09:30:20.970728Z","shell.execute_reply.started":"2024-04-23T09:30:20.786883Z","shell.execute_reply":"2024-04-23T09:30:20.969346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({\n    \"case_id\": data_submission[\"case_id\"].to_numpy(),\n    \"score\": y_submission_pred\n}).set_index('case_id')\nsubmission.to_csv(\"./submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-04-23T09:30:21.078566Z","iopub.execute_input":"2024-04-23T09:30:21.080123Z","iopub.status.idle":"2024-04-23T09:30:21.091374Z","shell.execute_reply.started":"2024-04-23T09:30:21.080057Z","shell.execute_reply":"2024-04-23T09:30:21.089894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2024-04-23T09:30:21.33005Z","iopub.execute_input":"2024-04-23T09:30:21.330533Z","iopub.status.idle":"2024-04-23T09:30:21.344126Z","shell.execute_reply.started":"2024-04-23T09:30:21.330492Z","shell.execute_reply":"2024-04-23T09:30:21.342626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}