{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"dockerImageVersionId":30665,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\nimport matplotlib.pyplot as plt\nfrom IPython.display import Markdown\nimport time\n\nimport glob\nimport numpy as np\nimport polars as pl\nimport pandas as pd\n\nimport seaborn as sns\nimport collections\n\nimport lightgbm as lgb\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.metrics import (\n    accuracy_score,\n    roc_curve,\n    roc_auc_score,\n    classification_report,\n    confusion_matrix,\n    ConfusionMatrixDisplay,\n)\n\nimport shap","metadata":{"execution":{"iopub.status.busy":"2025-05-22T10:50:00.781254Z","iopub.execute_input":"2025-05-22T10:50:00.782102Z","iopub.status.idle":"2025-05-22T10:50:10.607404Z","shell.execute_reply.started":"2025-05-22T10:50:00.782068Z","shell.execute_reply":"2025-05-22T10:50:10.606486Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"PATH_DATA_ROOT = \"/kaggle/input/home-credit-credit-risk-model-stability/\"\n\nbatches = [\"train\", \"test\"]\n\nPATH_DATA = {\n    batch: f\"{PATH_DATA_ROOT}parquet_files/{batch}/\"\n    for batch in batches\n}","metadata":{"execution":{"iopub.status.busy":"2025-05-22T10:50:10.609102Z","iopub.execute_input":"2025-05-22T10:50:10.609626Z","iopub.status.idle":"2025-05-22T10:50:10.613988Z","shell.execute_reply.started":"2025-05-22T10:50:10.609603Z","shell.execute_reply":"2025-05-22T10:50:10.613082Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def set_pl_dtypes(df):\n    for col in df.columns:\n        if col[-1] in (\"P\", \"A\"):\n            df = df.with_columns(pl.col(col).cast(pl.Float64))\n        if col[-1] in (\"M\"):\n            df = df.with_columns(pl.col(col).cast(pl.Categorical(\"lexical\")))\n        if col[-1] in (\"D\"):\n            df = df.with_columns(pl.col(col).cast(pl.Date))\n        if col[-1] in (\"L\", \"T\") and not df[col].is_numeric:\n            df = df.with_columns(pl.col(col).cast(pl.Categorical(\"lexical\")))\n    return df","metadata":{"execution":{"iopub.status.busy":"2025-05-22T10:50:10.615099Z","iopub.execute_input":"2025-05-22T10:50:10.615415Z","iopub.status.idle":"2025-05-22T10:50:10.630500Z","shell.execute_reply.started":"2025-05-22T10:50:10.615389Z","shell.execute_reply":"2025-05-22T10:50:10.629742Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dt_data = {\n    batch: {\n        \"base\": (pl.read_parquet(f\"{PATH_DATA[batch]}{batch}_base.parquet\")\\\n                 .with_columns(pl.col(\"date_decision\").cast(pl.Date))),\n        \"static\": pl.concat([pl.read_parquet(PATH_DATA_STATIC)\\\n                             .pipe(set_pl_dtypes) for PATH_DATA_STATIC in\n                             glob.glob(f\"{PATH_DATA[batch]}{batch}_static_0*.parquet\")],\n                            how=\"vertical_relaxed\"),\n        \"static_cb\": (pl.read_parquet(f\"{PATH_DATA[batch]}{batch}_static_cb_0.parquet\")\\\n                      .pipe(set_pl_dtypes)),\n        \"person_1\": (pl.read_parquet(f\"{PATH_DATA[batch]}{batch}_person_1.parquet\")\\\n                     .pipe(set_pl_dtypes)),\n        \"credit_bureau_b_2\": (pl.read_parquet(f\"{PATH_DATA[batch]}{batch}_credit_bureau_b_2.parquet\")\\\n                              .pipe(set_pl_dtypes))\n\n    }   \n    for batch in batches\n}","metadata":{"execution":{"iopub.status.busy":"2025-05-22T10:50:10.631529Z","iopub.execute_input":"2025-05-22T10:50:10.632274Z","iopub.status.idle":"2025-05-22T10:50:22.651701Z","shell.execute_reply.started":"2025-05-22T10:50:10.632245Z","shell.execute_reply":"2025-05-22T10:50:22.650756Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for batch in batches:\n\n    df_person_1_1 = dt_data[batch][\"person_1\"].group_by(\"case_id\").agg(\n        pl.col(\"mainoccupationinc_384A\").max().alias(\"mainoccupationinc_max_A\").cast(pl.Float64),\n        (pl.col(\"incometype_1044T\") == \"SELFEMPLOYED\").any().alias(\"anyselfemployed_T\").cast(pl.Boolean)\n    )\n    df_person_1_2 = dt_data[batch][\"person_1\"].drop(\n        [\"mainoccupationinc_384A\", \"incometype_1044T\"])\\\n        .filter(pl.col(\"num_group1\") == 0).drop(\"num_group1\")\n    df_person_1_2 = df_person_1_2.rename(\n        {col: col.rsplit(\"_\",1)[0] + \"_applicant_\" + col.rsplit(\"_\",1)[1] for\n         col in df_person_1_2.drop(\"case_id\").columns}\n    )\n    dt_data[batch][\"person_1\"] = (df_person_1_1\\\n        .join(other=df_person_1_2,\n              how=\"left\",\n              on=\"case_id\")\n    )\n    \n    dt_data[batch][\"credit_bureau_b_2\"] = dt_data[batch][\"credit_bureau_b_2\"]\\\n        .group_by(\"case_id\").agg(\n            pl.col(\"pmts_pmtsoverdue_635A\").max().alias(\"pmts_pmtsoverdue_max_A\").cast(pl.Float64),\n            (pl.col(\"pmts_dpdvalue_108P\") > 31).any().alias(\"pmts_dpdvalue_anyover31_P\").cast(pl.Boolean)\n    )","metadata":{"execution":{"iopub.status.busy":"2025-05-22T10:50:22.654841Z","iopub.execute_input":"2025-05-22T10:50:22.655127Z","iopub.status.idle":"2025-05-22T10:50:25.240734Z","shell.execute_reply.started":"2025-05-22T10:50:22.655099Z","shell.execute_reply":"2025-05-22T10:50:25.239750Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dt_data = {\n    batch: dt_data[batch][\"base\"]\\\n    .join(dt_data[batch][\"static\"],\n          how=\"left\",\n          on=\"case_id\")\\\n    .join(dt_data[batch][\"static_cb\"],\n          how=\"left\",\n          on=\"case_id\")\\\n    .join(dt_data[batch][\"person_1\"],\n          how=\"left\",\n          on=\"case_id\")\\\n    .join(dt_data[batch][\"credit_bureau_b_2\"],\n          how=\"left\",\n          on=\"case_id\")\n    for batch in batches\n}","metadata":{"execution":{"iopub.status.busy":"2025-05-22T10:50:25.241881Z","iopub.execute_input":"2025-05-22T10:50:25.242151Z","iopub.status.idle":"2025-05-22T10:50:27.398922Z","shell.execute_reply.started":"2025-05-22T10:50:25.242129Z","shell.execute_reply":"2025-05-22T10:50:27.398158Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dt_data[\"train\"]","metadata":{"execution":{"iopub.status.busy":"2025-05-22T10:50:27.400005Z","iopub.execute_input":"2025-05-22T10:50:27.400316Z","iopub.status.idle":"2025-05-22T10:50:27.434520Z","shell.execute_reply.started":"2025-05-22T10:50:27.400289Z","shell.execute_reply":"2025-05-22T10:50:27.433628Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dt_data[\"test\"].head()","metadata":{"execution":{"iopub.status.busy":"2025-05-22T10:50:27.435687Z","iopub.execute_input":"2025-05-22T10:50:27.436003Z","iopub.status.idle":"2025-05-22T10:50:27.451292Z","shell.execute_reply.started":"2025-05-22T10:50:27.435969Z","shell.execute_reply":"2025-05-22T10:50:27.450439Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dt_N_y_train = dict(collections.Counter(dt_data[\"train\"][\"target\"]))\n\ndt_N_y_train[\"ratio\"] = dt_N_y_train[0] / dt_N_y_train[1]\n\nplt.figure(figsize=(6.4, 4.8))\nax = plt.axes()\nplt.title(\"Number of counts per class\", pad=20)\n\nsns.countplot(\n    ax=ax,\n    x=dt_data[\"train\"][\"target\"].to_numpy(),\n    color=\"blue\",\n    alpha=0.5,\n    edgecolor=\"black\",\n    linewidth=1.0,\n    width=0.075,\n    hatch=\"////\",\n    zorder=2\n)\n\nax.set_xlabel(r\"Class, $y$\", fontdict={\"fontsize\": 10})\nax.set_ylabel(r\"Counts\", fontdict={\"fontsize\": 10})\n\nax.minorticks_on()\n\nax.grid(\n    visible=True,\n    which=\"major\",\n    color=\"lightgray\",\n    linestyle=\"solid\",\n    linewidth=0.5\n)\nax.grid(\n    visible=True,\n    which=\"minor\",\n    color=\"lightgray\",\n    linestyle=\"dotted\",\n    linewidth=0.5\n)\n\nplt.show() \n\nprint()\ndisplay(pd.DataFrame(data={\"$n^-/n^+$\": dt_N_y_train[\"ratio\"]},\n                     index=[0])\\\n        .style\\\n        .format({\"$n^-/n^+$\": \"{:.2f}\"})\\\n        .set_caption(\"Ratio between numbers of settled ($y=0$) and defaulted ($y=1$) credit contract cases\")\\\n        .set_table_styles([\n                {\"selector\": \"th.col_heading,td\",\n                 \"props\": [(\"width\", \"300px\")]\n                 },\n                {\"selector\": \"caption\",\n                 \"props\": [(\"font-size\", \"16px\"),\n                           (\"font-weight\", \"bold\"),\n                           (\"font-style\", \"italic\")]\n                 }\n            ]))","metadata":{"execution":{"iopub.status.busy":"2025-05-22T10:50:27.452586Z","iopub.execute_input":"2025-05-22T10:50:27.453075Z","iopub.status.idle":"2025-05-22T10:50:28.067293Z","shell.execute_reply.started":"2025-05-22T10:50:27.453046Z","shell.execute_reply":"2025-05-22T10:50:28.066488Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ---> Add a sample weight column to the training dataframe\ndt_data[\"train\"] = dt_data[\"train\"].with_columns(\n    pl.when(pl.col(\"target\") == 1).then(dt_N_y_train[\"ratio\"]).otherwise(1)\\\n    .alias(\"sample_weight\").cast(pl.Float64)\n)","metadata":{"execution":{"iopub.status.busy":"2025-05-22T10:50:28.068236Z","iopub.execute_input":"2025-05-22T10:50:28.068465Z","iopub.status.idle":"2025-05-22T10:50:28.088672Z","shell.execute_reply.started":"2025-05-22T10:50:28.068446Z","shell.execute_reply":"2025-05-22T10:50:28.088075Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dt_emp = {\n    \"emptiness\": {col: dt_data[\"train\"][col].null_count() / len(dt_data[\"train\"][col]) for\n                  col in dt_data[\"train\"].columns},\n    \"almost_empty\": {col: dt_data[\"train\"][col].null_count() / len(dt_data[\"train\"][col]) > 0.995 for\n                     col in dt_data[\"train\"].columns}\n}","metadata":{"execution":{"iopub.status.busy":"2025-05-22T10:50:28.089647Z","iopub.execute_input":"2025-05-22T10:50:28.089958Z","iopub.status.idle":"2025-05-22T10:50:28.099309Z","shell.execute_reply.started":"2025-05-22T10:50:28.089937Z","shell.execute_reply":"2025-05-22T10:50:28.098387Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(6.4, 4.8))\nax = plt.axes()\nplt.title(\"Number of counts per emptiness range in the training dataset\", pad=20)\n\nsns.histplot(\n    ax=ax,\n    data=dt_emp[\"emptiness\"].values(),\n    stat=\"count\",\n    bins=20,\n    binrange=(0, 1),\n    palette=[\"blue\"],\n    alpha=0.5,\n    edgecolor=\"black\",\n    linewidth=1.0,\n    hatch=\"////\",\n    zorder=2,\n    legend=False\n)\n\nplt.axvline(\n    x=0.995,\n    color=\"black\",\n    linewidth=2,\n    alpha=1,\n    linestyle=\"dashed\",\n    label=\"$\\mathrm{emptiness}=0.995$\"\n)\n\nax.set_xlabel(r\"Emptiness\", fontdict={\"fontsize\": 10})\nax.set_ylabel(r\"Counts\", fontdict={\"fontsize\": 10})\n\nax.set_xlim(\n    left=0,\n    right=1\n)\n    \nax.minorticks_on()\n\nax.grid(\n    visible=True,\n    which=\"major\",\n    color=\"lightgray\",\n    linestyle=\"solid\",\n    linewidth=0.5\n)\nax.grid(\n    visible=True,\n    which=\"minor\",\n    color=\"lightgray\",\n    linestyle=\"dotted\",\n    linewidth=0.5\n)\n\nax.legend(fontsize=8)\n\nplt.show() \n\nprint()\ndisplay(pd.DataFrame(data={\"Counts ($\\mathrm{emptiness}>0.995$)\":\n                           sum(dt_emp[\"almost_empty\"].values())},\n                     index=[0])\\\n        .style\\\n        .format({\"Counts ($\\mathrm{emptiness}>0.995$)\": \"{:d}\"})\\\n        .set_caption(\"Number of almost empty columns in the training dataset\")\\\n        .set_table_styles([\n                {\"selector\": \"th.col_heading,td\",\n                 \"props\": [(\"width\", \"300px\")]\n                 },\n                {\"selector\": \"caption\",\n                 \"props\": [(\"font-size\", \"16px\"),\n                           (\"font-weight\", \"bold\"),\n                           (\"font-style\", \"italic\")]\n                 }\n            ]))","metadata":{"execution":{"iopub.status.busy":"2025-05-22T10:50:28.100589Z","iopub.execute_input":"2025-05-22T10:50:28.100932Z","iopub.status.idle":"2025-05-22T10:50:28.453890Z","shell.execute_reply.started":"2025-05-22T10:50:28.100904Z","shell.execute_reply":"2025-05-22T10:50:28.452750Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cols_drop = [key for key in dt_emp[\"almost_empty\"].keys() if dt_emp[\"almost_empty\"][key] == True]\ndt_data[\"train\"] = dt_data[\"train\"].drop(cols_drop)\ndt_data[\"test\"] = dt_data[\"test\"].drop(cols_drop)","metadata":{"execution":{"iopub.status.busy":"2025-05-22T10:50:28.455334Z","iopub.execute_input":"2025-05-22T10:50:28.455721Z","iopub.status.idle":"2025-05-22T10:50:28.462526Z","shell.execute_reply.started":"2025-05-22T10:50:28.455671Z","shell.execute_reply":"2025-05-22T10:50:28.461516Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cols_cat = dt_data[\"train\"].select(pl.col(pl.Categorical)).columns\n\ndt_N_cat = {\n    \"N_cat\": {col: len(dt_data[\"train\"][col].cat.get_categories()) for\n              col in cols_cat},\n}\n\ndt_N_cat.update({\n    \"N_cat==1\": {col: dt_N_cat[\"N_cat\"][col] == 1 for col in cols_cat},\n    \"N_cat>1 and N_cat<=1\": {col: dt_N_cat[\"N_cat\"][col] > 1 and\n                             dt_N_cat[\"N_cat\"][col] <= 1000 for col in cols_cat},\n    \"N_cat>1000\": {col: dt_N_cat[\"N_cat\"][col] > 1000 for col in cols_cat}\n})","metadata":{"execution":{"iopub.status.busy":"2025-05-22T10:50:28.468759Z","iopub.execute_input":"2025-05-22T10:50:28.469195Z","iopub.status.idle":"2025-05-22T10:50:28.483366Z","shell.execute_reply.started":"2025-05-22T10:50:28.469155Z","shell.execute_reply":"2025-05-22T10:50:28.482204Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"x = list(range(len([\"N_cat==1\", \"N_cat>1 and N_cat<=1\", \"N_cat>1000\"]) + 1))\n\ni_x = list(range(len(x)))\nx_bar_points = sum(\n    [[x[i]] * \n     (2 if i in (0, i_x[-1]) else 4)\n     for i in i_x],\n    []\n)\n\ny_bar_points = sum(\n    [[0, sum(dt_N_cat[key].values()), sum(dt_N_cat[key].values()), 0]\n     for key in [\"N_cat==1\", \"N_cat>1 and N_cat<=1\", \"N_cat>1000\"]],\n    []\n)\n\nx_ticks = [(x[i] + x[i + 1]) / 2 for i in i_x[:-1]]\n\nplt.figure(figsize=(6.4, 4.8))\nax = plt.axes()\nplt.title(\"Number of counts per number of categories range in the training dataset\", pad=20)\n\nax.plot(\n    x_bar_points,\n    y_bar_points,\n    color=\"black\",\n    linewidth=1.0,\n    zorder=2\n)\n\nax.fill_between(\n    x=x_bar_points,\n    y1=np.zeros(len(x_bar_points)),\n    y2=y_bar_points,\n    facecolor=\"blue\",\n    edgecolor=\"black\",\n    hatch=\"////\",\n    linewidth=1.0,\n    alpha=0.5,\n    zorder=2\n)\n                                      \nax.set_xlabel(r\"Number of categories, $N_{\\mathrm{cat}}$\", fontdict={\"fontsize\": 10})\nax.set_ylabel(r\"Counts\", fontdict={\"fontsize\": 10})\n\nax.set_xlim(\n    left=0,\n    right=3\n)\nax.set_ylim(\n    bottom=0\n)\n\nax.set_xticks(x_ticks, labels=[\"$1$\", \"$[1,\\,1000]$\", f\"$>1000$\"])\n\nax.grid(\n    visible=True,\n    which=\"major\",\n    color=\"lightgray\",\n    linestyle=\"solid\",\n    linewidth=0.5\n)\nax.grid(\n    visible=True,\n    which=\"minor\",\n    color=\"lightgray\",\n    linestyle=\"dotted\",\n    linewidth=0.5\n)\n\nplt.show() \n\nprint()\ndisplay(pd.DataFrame(data={\"Counts ($N_{\\mathrm{cat}}=1$)\": sum(dt_N_cat[\"N_cat==1\"].values()),\n                           \"Counts ($N_{\\mathrm{cat}}>1000$)\": sum(dt_N_cat[\"N_cat>1000\"].values())},\n                     index=[0])\\\n        .style\\\n        .format({\"N\": \"{:d}\"})\\\n        .set_caption(\"Numbers of columns of single and of too many categories in the training dataset\")\\\n        .set_table_styles([\n                {\"selector\": \"th.col_heading,td\",\n                 \"props\": [(\"width\", \"300px\")]\n                 },\n                {\"selector\": \"caption\",\n                 \"props\": [(\"font-size\", \"16px\"),\n                           (\"font-weight\", \"bold\"),\n                           (\"font-style\", \"italic\")]\n                 }\n            ]))","metadata":{"execution":{"iopub.status.busy":"2025-05-22T10:50:28.484730Z","iopub.execute_input":"2025-05-22T10:50:28.485453Z","iopub.status.idle":"2025-05-22T10:50:28.687100Z","shell.execute_reply.started":"2025-05-22T10:50:28.485421Z","shell.execute_reply":"2025-05-22T10:50:28.686193Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cols_drop = [col for col in cols_cat if\n             True in (dt_N_cat[\"N_cat==1\"][col], dt_N_cat[\"N_cat>1000\"])]\n\ndt_data[\"train\"] = dt_data[\"train\"].drop(cols_drop)\ndt_data[\"test\"] = dt_data[\"test\"].drop(cols_drop)","metadata":{"execution":{"iopub.status.busy":"2025-05-22T10:50:28.688069Z","iopub.execute_input":"2025-05-22T10:50:28.688286Z","iopub.status.idle":"2025-05-22T10:50:28.694024Z","shell.execute_reply.started":"2025-05-22T10:50:28.688269Z","shell.execute_reply":"2025-05-22T10:50:28.693080Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def dist_plot(df, col, col_fancy, binning=True, N_bins=10, xticklabels_rotation=0):\n\n    if binning == True:\n        x_bin_limit = np.linspace(df[col].min(), df[col].max(), N_bins + 1)\n\n        i_x_bin_limit = range(len(x_bin_limit))\n\n        df_dist = pl.DataFrame(\n            schema=[\n                \"Bin's $x_{\\mathrm{min}}$\",\n                \"Bin's $x_{\\mathrm{av}}$\",\n                \"Bin's $x_{\\mathrm{max}}$\",\n                \"Counts ($y=0$)\",\n                \"Counts ($y=1$)\",\n                \"$P(y=1)\\,[\\%]$\"\n            ]\n        )\n\n        df_dist_plot = pl.DataFrame(\n            schema=[\n                \"x\",\n                \"y\"\n            ]\n        )\n\n        for i in i_x_bin_limit[:-1]:\n            df_dist_i = (\n                df.select(col, \"target\")\\\n                    .groupby(\n                        ((pl.col(col) >= x_bin_limit[i]) &\n                         ((pl.col(col) < x_bin_limit[i + 1]) if \n                          i != i_x_bin_limit[-2] else \n                          (pl.col(col) <= x_bin_limit[i + 1])))\n                    )\\\n                    .agg(\n                        (pl.col(\"target\") == 0).sum().alias(\"Counts ($y=0$)\"),\n                        (pl.col(\"target\") == 1).sum().alias(\"Counts ($y=1$)\")\n                    )\\\n                    .with_columns(\n                        (pl.col(\"Counts ($y=1$)\") /\n                         (pl.col(\"Counts ($y=0$)\") + pl.col(\"Counts ($y=1$)\")) *\n                         100).alias(\"$P(y=1)\\,[\\%]$\")\n                    )\\\n                    .filter(pl.col(col) == True)\\\n                    .drop(col)\n                    .with_columns(\n                        pl.lit(x_bin_limit[i]).alias(\"Bin's $x_{\\mathrm{min}}$\"),\n                        pl.lit((x_bin_limit[i] + x_bin_limit[i + 1]) / 2).alias(\"Bin's $x_{\\mathrm{av}}$\"),\n                        pl.lit(x_bin_limit[i + 1]).alias(\"Bin's $x_{\\mathrm{max}}$\"),\n                    )\n                    .select(\n                        \"Bin's $x_{\\mathrm{min}}$\",\n                        \"Bin's $x_{\\mathrm{av}}$\",\n                        \"Bin's $x_{\\mathrm{max}}$\",\n                        \"Counts ($y=0$)\",\n                        \"Counts ($y=1$)\",\n                        \"$P(y=1)\\,[\\%]$\"\n                    )\n            )\n\n            df_dist = pl.concat([\n                df_dist,\n                df_dist_i\n            ],\n                how=\"vertical_relaxed\"\n            )\n\n            df_dist_plot = pl.concat([\n                df_dist_plot,\n                df_dist_i.select(pl.col(\"Bin's $x_{\\mathrm{min}}$\").alias(\"x\")).with_columns(pl.lit(0).alias(\"y\")),\n                df_dist_i.select(pl.col(\"Bin's $x_{\\mathrm{min}}$\").alias(\"x\"), pl.col(\"$P(y=1)\\,[\\%]$\").alias(\"y\")),\n                df_dist_i.select(pl.col(\"Bin's $x_{\\mathrm{max}}$\").alias(\"x\"), pl.col(\"$P(y=1)\\,[\\%]$\").alias(\"y\")),\n                df_dist_i.select(pl.col(\"Bin's $x_{\\mathrm{max}}$\").alias(\"x\")).with_columns(pl.lit(0).alias(\"y\"))\n            ],\n                how=\"vertical_relaxed\"\n            )\n    else:\n        df_dist = (df.select(col, \"target\")\\\n            .groupby(col)\\\n            .agg((pl.col(\"target\") == 0).sum().alias(\"Counts ($y=0$)\"),\n                 (pl.col(\"target\") == 1).sum().alias(\"Counts ($y=1$)\"))\\\n            .with_columns(\n            (pl.col(\"Counts ($y=1$)\") /\n             (pl.col(\"Counts ($y=0$)\") + pl.col(\"Counts ($y=1$)\")) *\n             100).alias(\"$P(y=1)\\,[\\%]$\"))\\\n            .rename({col: col_fancy.capitalize()})\n        )\n\n    print()\n    display(Markdown('---'))\n    print()\n\n    plt.figure(figsize=(6.4, 4.8))\n    ax = plt.axes()\n    plt.title(f\"Relation between probability of credit default and {col_fancy}\", pad=20)\n\n    if binning == True:\n        # Plot bar lines\n        ax.plot(\n            df_dist_plot[\"x\"],\n            df_dist_plot[\"y\"],\n            color=\"blue\",\n            linewidth=1.0,\n            zorder=2\n        )\n\n        # Fill bars\n        ax.fill_between(\n            x=df_dist_plot[\"x\"],\n            y1=np.zeros(len(df_dist_plot[\"x\"])),\n            y2=df_dist_plot[\"y\"],\n            facecolor=\"blue\",\n            edgecolor=\"black\",\n            hatch=\"////\",\n            linewidth=1.0,\n            alpha=0.5,\n            zorder=2\n        )\n\n        for row in df_dist.rows(named=True):\n            ax.annotate(\n                text=\"${:.2f}\\,\\%$\".format(row[\"$P(y=1)\\,[\\%]$\"]),\n                xy=(row[\"Bin's $x_{\\mathrm{av}}$\"], \n                    row[\"$P(y=1)\\,[\\%]$\"]),\n                ha=\"left\",\n                va=\"bottom\",\n                size=9,\n                xytext=(-7.5, 2),\n                textcoords=\"offset points\",\n                rotation=70\n            )\n    else:\n        sns.barplot(\n            ax=ax,\n            data=df_dist.to_pandas(),\n            x=col_fancy.capitalize(),\n            y=\"$P(y=1)\\,[\\%]$\",\n            color=\"blue\",\n            edgecolor=\"black\",\n            hatch=\"////\",\n            linewidth=1.0,\n            alpha=0.5,\n            width=0.200,\n            zorder=2\n        )\n\n        for bar in ax.patches:\n            ax.annotate(\n                text=\"${:.2f}\\,\\%$\".format(bar.get_height()),\n                xy=(bar.get_x() + bar.get_width() / 2, \n                    bar.get_height()),\n                ha=\"left\",\n                va=\"bottom\",\n                size=9,\n                xytext=(-5, 2),\n                textcoords=\"offset points\",\n                rotation=70\n            )\n               \n    ax.set_xlabel(col_fancy.capitalize(), fontdict={\"fontsize\": 10})\n    ax.set_ylabel(r\"$P(y=1)\\,[\\%]$\", fontdict={\"fontsize\": 10})\n\n    ax.tick_params(axis=\"x\", rotation=xticklabels_rotation)\n\n    if binning == True:\n        ax.set_xlim(\n            left=min(df_dist_plot[\"x\"]),\n            right=max(df_dist_plot[\"x\"])\n        )\n    ax.set_ylim(bottom=0, top=1.125 * ax.get_ylim()[1])\n\n    ax.minorticks_on()\n\n    ax.grid(\n        visible=True,\n        which=\"major\",\n        color=\"lightgray\",\n        linestyle=\"solid\",\n        linewidth=0.5\n    )\n    ax.grid(\n        visible=True,\n        which=\"minor\",\n        color=\"lightgray\",\n        linestyle=\"dotted\",\n        linewidth=0.5\n    )\n\n    plt.show()\n\n    print()\n    display(df_dist.to_pandas()\\\n            .style\\\n            .format({\n                \"P_1\": \"{:.2f}\",\n                \"Bin's $x_{\\mathrm{min}}$\": \"{:.2f}\",\n                \"Bin's $x_{\\mathrm{av}}$\": \"{:.2f}\",\n                \"Bin's $x_{\\mathrm{max}}$\": \"{:.2f}\"\n            } if binning==True else\n            {\"P_1\": \"{:.2f}\"})\\\n            .set_caption(f\"{col_fancy.capitalize()} - counts and probabilities\")\\\n            .set_table_styles([\n                    {\"selector\": \"th.col_heading,td\",\n                     \"props\": [(\"width\", \"150px\")]\n                     },\n                    {\"selector\": \"caption\",\n                     \"props\": [(\"font-size\", \"16px\"),\n                               (\"font-weight\", \"bold\"),\n                               (\"font-style\", \"italic\")]\n                     }\n                ]))","metadata":{"execution":{"iopub.status.busy":"2025-05-22T10:50:28.695349Z","iopub.execute_input":"2025-05-22T10:50:28.695600Z","iopub.status.idle":"2025-05-22T10:50:28.718964Z","shell.execute_reply.started":"2025-05-22T10:50:28.695580Z","shell.execute_reply":"2025-05-22T10:50:28.718064Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dt_data = {\n    batch: dt_data[batch].rename({\"annuity_780A\": \"monthly_payment_A\"})\n    for batch in batches\n}\n\n# Plot\ndist_plot(\n    df=dt_data[\"train\"],\n    col=\"monthly_payment_A\",\n    col_fancy=\"contract's monthly payment\",\n    binning=True,\n    N_bins=10,\n    xticklabels_rotation=0\n)\n\ndt_data = {\n    batch: dt_data[batch].with_columns(\n        (pl.col(\"date_decision\").sub(pl.col(\"birth_applicant_259D\"))\\\n         .alias(\"age_applicant_A\").dt.days() / 365.25).cast(pl.Float64)\n    ).drop(\"birth_applicant_259D\")\n    for batch in batches\n}\n\ndist_plot(\n    df=dt_data[\"train\"],\n    col=\"age_applicant_A\",\n    col_fancy=\"applicant's age at decision time\",\n    binning=True,\n    N_bins=20,\n    xticklabels_rotation=0\n)\n\ndt_data = {\n    batch: dt_data[batch].with_columns(\n        (pl.col(\"date_decision\").sub(pl.col(\"empl_employedfrom_applicant_271D\"))\\\n         .alias(\"employment_time_applicant_A\").dt.days() / 365.25).cast(pl.Float64)\n    ).drop(\"empl_employedfrom_applicant_271D\")\n    for batch in batches\n}\n\ndist_plot(\n    df=dt_data[\"train\"],\n    col=\"employment_time_applicant_A\",\n    col_fancy=\"applicant's employment time (in years) at decision date\",\n    binning=True,\n    N_bins=12,\n    xticklabels_rotation=0\n)\n\ndt_data = {\n    batch: dt_data[batch].with_columns(\n        (25 - pl.col(\"cntpmts24_3658933L\"))\\\n         .alias(\"N_missing_payments_24_L\").cast(pl.Int64)\n    ).drop(\"cntpmts24_3658933L\")\n    for batch in batches\n}\n\ndist_plot(\n    df=dt_data[\"train\"],\n    col=\"N_missing_payments_24_L\",\n    col_fancy=\"number of missing payments in the last $24$ months (and current one)\",\n    binning=True,\n    N_bins=10,\n    xticklabels_rotation=0\n)\n\ndist_plot(\n    df=dt_data[\"train\"],\n    col=\"sex_applicant_738L\",\n    col_fancy=\"applicant's gender\",\n    binning=False,\n    xticklabels_rotation=0\n)\n\n\ndt_data = {\n    batch: dt_data[batch].with_columns(\n        pl.col(\"mobilephncnt_593L\")\\\n        .alias(\"n_persons_same_phone_A\").cast(pl.Int64)\n    ).drop(\"mobilephncnt_593L\")\n    for batch in batches\n}\n\ndist_plot(\n    df=dt_data[\"train\"],\n    col=\"n_persons_same_phone_A\",\n    col_fancy=\"number of persons using the same phone number\",\n    binning=True,\n    N_bins=10,\n    xticklabels_rotation=0\n)\n\ndt_data = {\n    batch: dt_data[batch].with_columns(\n        pl.col(\"date_decision\").dt.weekday().cast(pl.Int64)\\\n        .alias(\"weekday_date_decision_A\"),\n        pl.col(\"date_decision\").dt.month().cast(pl.Int64)\\\n        .alias(\"month_date_decision_A\"),\n        pl.col(\"date_decision\").dt.month().cast(pl.Int64)\\\n        .alias(\"year_date_decision_A\"),\n    )\n    for batch in batches\n}\n\ndist_plot(\n    df=dt_data[\"train\"],\n    col=\"weekday_date_decision_A\",\n    col_fancy=\"weekday of contract's decision date\",\n    binning=False,\n    xticklabels_rotation=45\n)\n\ndt_data = {\n    batch: dt_data[batch].with_columns(\n        pl.col(\"pmtnum_254L\")\\\n        .alias(\"n_payments_A\").cast(pl.Int64)\n    ).drop(\"pmtnum_254L\")\n    for batch in batches\n}\n\n# Plot\ndist_plot(\n    df=dt_data[\"train\"],\n    col=\"n_payments_A\",\n    col_fancy=\"number of done payments\",\n    binning=True,\n    N_bins=12,\n    xticklabels_rotation=0\n)","metadata":{"execution":{"iopub.status.busy":"2025-05-22T10:50:28.720169Z","iopub.execute_input":"2025-05-22T10:50:28.720456Z","iopub.status.idle":"2025-05-22T10:50:36.168284Z","shell.execute_reply.started":"2025-05-22T10:50:28.720434Z","shell.execute_reply":"2025-05-22T10:50:36.167365Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dt_data[\"train\"] = dt_data[\"train\"].sort(\"case_id\")\ndt_data[\"train\"] = dt_data[\"train\"].sample(fraction=1, shuffle=True, seed=42)","metadata":{"execution":{"iopub.status.busy":"2025-05-22T10:50:36.169427Z","iopub.execute_input":"2025-05-22T10:50:36.169756Z","iopub.status.idle":"2025-05-22T10:50:40.489427Z","shell.execute_reply.started":"2025-05-22T10:50:36.169704Z","shell.execute_reply":"2025-05-22T10:50:40.488748Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cols_x = []\nfor col in dt_data[\"train\"].columns:\n    if col[-1].isupper() and col[:-1].islower():\n        cols_x.append(col)","metadata":{"execution":{"iopub.status.busy":"2025-05-22T10:50:40.490609Z","iopub.execute_input":"2025-05-22T10:50:40.491260Z","iopub.status.idle":"2025-05-22T10:50:40.496403Z","shell.execute_reply.started":"2025-05-22T10:50:40.491217Z","shell.execute_reply":"2025-05-22T10:50:40.495429Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def convert_cols_obj_to_cols_cat(*dfs):\n    cols_object = list(set().union(*(df.select_dtypes(include=[\"object\"]).columns for df in dfs)))\n    for col in cols_object:\n        for df in dfs:\n            df[col] = df[col].astype(\"category\")\n            new_dtype = pd.CategoricalDtype(categories=df[col].cat.categories.to_list() +\n                                            [\"Unknown\"],\n                                            ordered=True)\n            df[col] = df[col].astype(new_dtype)\n    return dfs\n\ndef convert_cols_date_to_cols_ord(df):\n    cols_date = df.select_dtypes(include=[\"datetime64\"]).columns\n    for col in cols_date:\n        df[col] = df[col].apply(lambda x: x.toordinal() if not\n                                pd.isnull else -1000)\n    return df\n\n\ndef convert_cols_bool_to_cols_int(df):\n    cols_bool = df.select_dtypes(include=[\"bool\"]).columns\n    for col in cols_bool:\n        df[col] = df[col].astype(\"int64\")\n    return df\n\ndef make_cat_excl_unknown(df, df_ref):\n    for col in df_ref.select_dtypes(include=[\"category\"]).columns:\n        cat_ref = df_ref[col].cat.categories.to_list()\n        cat = df[col].cat.categories.to_list()\n        cat_common = list(set(cat).intersection(cat_ref))\n        cat_exc = list(set(cat).difference(cat_common))\n        new_dtype = pd.CategoricalDtype(categories=cat_common,\n                                        ordered=True)\n        df[col] = df[col].replace(to_replace=cat_exc, value=\"Unknown\")\n        df[col] = df[col].astype(new_dtype)\n    return df\n\ndef covert_cols_cat_to_cols_code(df):\n    dt_map = {}\n    \n    for col in df.select_dtypes(include=[\"category\"]).columns:\n        \n    \n        dt_map.update({col:\n            dict(enumerate(df[col].cat.categories))\n        })\n        \n        df[col] = df[col].cat.codes\n\n    return (df, dt_map)","metadata":{"execution":{"iopub.status.busy":"2025-05-22T10:50:40.498007Z","iopub.execute_input":"2025-05-22T10:50:40.498289Z","iopub.status.idle":"2025-05-22T10:50:40.523221Z","shell.execute_reply.started":"2025-05-22T10:50:40.498256Z","shell.execute_reply":"2025-05-22T10:50:40.522392Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dt_data[\"train\"] = {\n    \"base\": (dt_data[\"train\"][[\"case_id\", \"date_decision\", \"WEEK_NUM\", \"target\", \"sample_weight\"]]\\\n             .rename({\"target\": \"y\"}).to_pandas()),\n    \"x\": dt_data[\"train\"][cols_x].to_pandas(),\n    \"y\": dt_data[\"train\"][\"target\"].to_pandas(),\n}\ndt_data[\"test\"] = {\n    \"base\": dt_data[\"test\"][[\"case_id\", \"date_decision\", \"WEEK_NUM\"]].to_pandas(),\n    \"x\": dt_data[\"test\"][cols_x].to_pandas()\n}\n(dt_data[\"train\"][\"x\"], dt_data[\"test\"][\"x\"]) = convert_cols_obj_to_cols_cat(\n    dt_data[\"train\"][\"x\"], dt_data[\"test\"][\"x\"]\n)\n\ndt_data[\"train\"][\"x\"] = convert_cols_date_to_cols_ord(dt_data[\"train\"][\"x\"])\ndt_data[\"test\"][\"x\"] = convert_cols_date_to_cols_ord(dt_data[\"test\"][\"x\"])\n\ndt_data[\"test\"][\"x\"] = make_cat_excl_unknown(\n    df=dt_data[\"test\"][\"x\"], df_ref=dt_data[\"train\"][\"x\"]\n)\n\n(dt_data[\"train\"][\"x\"], dt_data[\"train\"][\"cat_map\"]) = covert_cols_cat_to_cols_code(dt_data[\"train\"][\"x\"])\n(dt_data[\"test\"][\"x\"], dt_data[\"test\"][\"cat_map\"]) = covert_cols_cat_to_cols_code(dt_data[\"test\"][\"x\"])\n\ndt_data[\"train\"][\"x\"] = convert_cols_bool_to_cols_int(dt_data[\"train\"][\"x\"])\ndt_data[\"test\"][\"x\"] = convert_cols_bool_to_cols_int(dt_data[\"test\"][\"x\"])","metadata":{"execution":{"iopub.status.busy":"2025-05-22T10:50:40.524383Z","iopub.execute_input":"2025-05-22T10:50:40.524643Z","iopub.status.idle":"2025-05-22T10:51:21.770079Z","shell.execute_reply.started":"2025-05-22T10:50:40.524623Z","shell.execute_reply":"2025-05-22T10:51:21.769374Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"params = {\n    \"boosting_type\": \"gbdt\",\n    \"objective\": \"binary\",\n    \"metric\": \"auc\",\n    \"num_leaves\": 31,\n    \"learning_rate\": 0.05,\n    \"feature_fraction\": 1,\n    \"bagging_fraction\": 1,\n    \"bagging_freq\": 0,\n    \"n_estimators\": 1200,\n    \"max_depth\": 3,\n    \"n_jobs\": -1,\n    \"random_state\": 42,\n    \"device_type\": \"gpu\",\n    \"gpu_use_dp\": True,\n    \"verbose\": -1,\n}\n\nlgb_estimator = lgb.LGBMClassifier(**params)","metadata":{"execution":{"iopub.status.busy":"2025-05-22T10:51:21.771328Z","iopub.execute_input":"2025-05-22T10:51:21.771893Z","iopub.status.idle":"2025-05-22T10:51:21.779260Z","shell.execute_reply.started":"2025-05-22T10:51:21.771862Z","shell.execute_reply":"2025-05-22T10:51:21.778328Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"param_grid = {\n    \"n_estimators\": [600, 800, 1000],\n    \"max_depth\": [8, 9, 10]\n}\n\ngs = GridSearchCV(\n    estimator=lgb_estimator, \n    param_grid=param_grid,\n    cv=5,\n    scoring=\"roc_auc\",\n    return_train_score=True,\n    n_jobs=1,\n    refit=True,\n    verbose=0\n)\n\nt_i = time.perf_counter()\ngs.fit(\n    X=dt_data[\"train\"][\"x\"],\n    y=dt_data[\"train\"][\"y\"],\n    sample_weight=dt_data[\"train\"][\"base\"][\"sample_weight\"]\n)\n\nt_f = time.perf_counter()\n\nprint()\nprint(f\"Running time: {(t_f - t_i)/3600:.2f} h\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:51:21.780456Z","iopub.execute_input":"2025-05-22T10:51:21.781038Z","iopub.status.idle":"2025-05-22T12:20:50.782950Z","shell.execute_reply.started":"2025-05-22T10:51:21.781007Z","shell.execute_reply":"2025-05-22T12:20:50.782222Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cv_results = pd.DataFrame(gs.cv_results_)\n\ncv_results.columns = cv_results.columns.str.replace(\"score\", \"auc\")\n\ncv_results.to_csv(\"cv_results.csv\", index=True)\n\nprint()\ndisplay(cv_results[cv_results.filter(like=\"param_\", axis=\"columns\").columns.values.tolist() + \n                   [\"mean_train_auc\", \"std_train_auc\", \"mean_test_auc\", \"std_test_auc\"]]\\\n        .style.set_caption(\"Mean AUCs and respective standard devations\")\\\n        .set_table_styles(\n            [\n                {\"selector\": \"th.col_heading,td\",\n                 \"props\": [(\"width\", \"100px\")]\n                 },\n                {\"selector\": \"caption\",\n                 \"props\": [(\"font-size\", \"16px\"),\n                           (\"font-weight\", \"bold\"),\n                           (\"font-style\", \"italic\")]\n                 }\n            ]))\n\n(cv_results_best_train_auc, cv_results_best_valid_auc) = (\n    pd.DataFrame(\n        data=(cv_results[cv_results.filter(like=\"param_\", axis=\"columns\").columns.values.tolist() + \n                         [\"mean_train_auc\", \"std_train_auc\", \"mean_test_auc\", \"std_test_auc\"]]\\\n              .loc[cv_results[f\"mean_{batch}_auc\"].idxmax()].to_dict()),\n        index=[cv_results[f\"mean_{batch}_auc\"].idxmax()]\n    ) for batch in [\"train\", \"test\"]\n)\n\nfor (df, batch_fancy) in ((cv_results_best_train_auc, \"training\"),\n                          (cv_results_best_valid_auc, \"validation\")):\n    print()\n    display(df\\\n        .style.set_caption(f\"Mean AUCs and respective standard devations at best {batch_fancy} mean AUC\")\\\n        .set_table_styles(\n            [\n                {\"selector\": \"th.col_heading,td\",\n                 \"props\": [(\"width\", \"100px\")]\n                 },\n                {\"selector\": \"caption\",\n                 \"props\": [(\"font-size\", \"16px\"),\n                           (\"font-weight\", \"bold\"),\n                           (\"font-style\", \"italic\")]\n                 }\n            ])\n    )","metadata":{"execution":{"iopub.status.busy":"2025-05-22T12:20:50.783838Z","iopub.execute_input":"2025-05-22T12:20:50.784098Z","iopub.status.idle":"2025-05-22T12:20:50.818952Z","shell.execute_reply.started":"2025-05-22T12:20:50.784078Z","shell.execute_reply":"2025-05-22T12:20:50.818215Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"n_estimators = cv_results[\"param_n_estimators\"].unique()\nmax_depth = cv_results[\"param_max_depth\"].unique()\nmean_test_auc = (cv_results.groupby(\"param_n_estimators\")\\\n                 .agg(list)[\"mean_test_auc\"].to_list())\n\nplt.figure(figsize=(6.4, 4.8))\nax = plt.axes()\nplt.title(\n    r\"Dependence of $\\mathrm{AUC}_{\\mathrm{valid},\\,\\mathrm{av}}$ on $N_{\\mathrm{trees}}$ and $\\mathrm{d}_{\\mathrm{max}}$\",\n    pad=20\n)\n\nX = n_estimators\nY = max_depth\nZ = np.transpose(mean_test_auc)\n\ncs = ax.contour(X, Y, Z, levels=10, colors=\"white\", linestyles=\"solid\", linewidths=0.5)\n\ncsf = ax.contourf(X, Y, Z, levels=10, cmap=\"cividis\")\n\nax.clabel(CS=cs, fmt=\"%1.3f\", inline=True, fontsize=8)\n                   \nax.set_xlabel(r\"Number of trees, $N_{\\mathrm{trees}}$\", fontdict={\"fontsize\": 10})\nax.set_ylabel(r\"Maximum tree depth, $\\mathrm{d}_{\\mathrm{max}}$\", fontdict={\"fontsize\": 10})\n\ncbar = plt.colorbar(mappable=csf, label=\"$\\mathrm{AUC}_{\\mathrm{valid},\\,\\mathrm{av}}$\", ax=ax)\ncbar.add_lines(cs)\n\nplt.show() ","metadata":{"execution":{"iopub.status.busy":"2025-05-22T12:20:50.819969Z","iopub.execute_input":"2025-05-22T12:20:50.820223Z","iopub.status.idle":"2025-05-22T12:20:51.246029Z","shell.execute_reply.started":"2025-05-22T12:20:50.820203Z","shell.execute_reply":"2025-05-22T12:20:51.245166Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"best_estimator = gs.best_estimator_","metadata":{"execution":{"iopub.status.busy":"2025-05-22T12:20:51.247312Z","iopub.execute_input":"2025-05-22T12:20:51.247651Z","iopub.status.idle":"2025-05-22T12:20:51.252054Z","shell.execute_reply.started":"2025-05-22T12:20:51.247621Z","shell.execute_reply":"2025-05-22T12:20:51.251190Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dt_data[\"train\"][\"base\"][\"P_pred\"] = best_estimator.predict_proba(\n    X=dt_data[\"train\"][\"x\"]\n)[:, 1]\n\ndt_data[\"train\"][\"base\"][\"y_pred\"] = ((dt_data[\"train\"][\"base\"][\"P_pred\"] >= 0.5)\\\n                                      .astype(dtype=\"int32\"))","metadata":{"execution":{"iopub.status.busy":"2025-05-22T12:20:51.253042Z","iopub.execute_input":"2025-05-22T12:20:51.253314Z","iopub.status.idle":"2025-05-22T12:21:29.105461Z","shell.execute_reply.started":"2025-05-22T12:20:51.253294Z","shell.execute_reply":"2025-05-22T12:21:29.104751Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cm = confusion_matrix(\n    y_true=dt_data[\"train\"][\"y\"],\n    y_pred=dt_data[\"train\"][\"base\"][\"y_pred\"]\n)\n\ndisp = ConfusionMatrixDisplay(cm).plot(cmap=\"cividis\")\ndisp.ax_.set_title(\"Confusion matrix\", pad=20);","metadata":{"execution":{"iopub.status.busy":"2025-05-22T12:21:29.106549Z","iopub.execute_input":"2025-05-22T12:21:29.107160Z","iopub.status.idle":"2025-05-22T12:21:29.822460Z","shell.execute_reply.started":"2025-05-22T12:21:29.107125Z","shell.execute_reply":"2025-05-22T12:21:29.821543Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cr = pd.DataFrame(\n    classification_report(y_true=dt_data[\"train\"][\"y\"],\n                          y_pred=dt_data[\"train\"][\"base\"][\"y_pred\"],\n                          sample_weight=dt_data[\"train\"][\"base\"][\"sample_weight\"],\n                          output_dict=True)).transpose()\n\ncr.loc[\"accuracy\"] = [\"---\", \"---\", cr.loc[\"accuracy\", \"precision\"], cr.loc[\"macro avg\", \"support\"]]\n\ncr[\"support\"] = cr[\"support\"].astype(int) \n\ndt_data[\"train\"][\"precision\"] = cr[\"precision\"].loc[\"1\"]\ndt_data[\"train\"][\"recall\"] = cr[\"recall\"].loc[\"1\"]\ndt_data[\"train\"][\"f1-score\"] = cr[\"f1-score\"].loc[\"1\"]\ndt_data[\"train\"][\"accuracy\"] = cr[\"f1-score\"].loc[\"accuracy\"]\n\nprint()\ndisplay(cr.style.set_caption(\"Classification report\")\\\n        .set_table_styles(\n            [\n                {\"selector\": \"th.col_heading,td\",\n                 \"props\": [(\"width\", \"100px\")]\n                 },\n                {\"selector\": \"caption\",\n                 \"props\": [(\"font-size\", \"16px\"),\n                           (\"font-weight\", \"bold\"),\n                           (\"font-style\", \"italic\")]\n                 }\n            ]))","metadata":{"execution":{"iopub.status.busy":"2025-05-22T12:21:29.823612Z","iopub.execute_input":"2025-05-22T12:21:29.823948Z","iopub.status.idle":"2025-05-22T12:21:35.013788Z","shell.execute_reply.started":"2025-05-22T12:21:29.823924Z","shell.execute_reply":"2025-05-22T12:21:35.012859Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ---> Compute ROC curve, AUC and Gini coefficient for the traning dataset\n\n# Add ROC's FPR and TPR to the dataset dictionary\n(dt_data[\"train\"][\"FPR\"], dt_data[\"train\"][\"TPR\"], _) = roc_curve(\n    y_true=dt_data[\"train\"][\"y\"],\n    y_score=dt_data[\"train\"][\"base\"][\"P_pred\"]\n)\n\n# Add AUC to the dataset dictionary\ndt_data[\"train\"][\"auc\"] = roc_auc_score(\n    y_true=dt_data[\"train\"][\"y\"],\n    y_score=dt_data[\"train\"][\"base\"][\"P_pred\"]\n)\n\n# Add Gini coefficient to the dataset dictionary\ndt_data[\"train\"][\"g\"] = 2 * dt_data[\"train\"][\"auc\"] - 1","metadata":{"execution":{"iopub.status.busy":"2025-05-22T12:21:35.014785Z","iopub.execute_input":"2025-05-22T12:21:35.015014Z","iopub.status.idle":"2025-05-22T12:21:36.120798Z","shell.execute_reply.started":"2025-05-22T12:21:35.014995Z","shell.execute_reply":"2025-05-22T12:21:36.120033Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(4.8, 4.8))\nax = plt.axes()\nplt.title(rf\"ROC curves, AUC and Gini coefficient\", pad=20)\n\nax.plot(dt_data[\"train\"][\"FPR\"],\n        dt_data[\"train\"][\"TPR\"],\n        linestyle=\"solid\",\n        color=\"black\",\n        alpha=1,\n        linewidth=1.5,\n        label=\"Estimator's ROC\")\n\nax.fill_between(\n    x=dt_data[\"train\"][\"FPR\"],\n    y1=np.zeros(len(dt_data[\"train\"][\"FPR\"])),\n    y2=dt_data[\"train\"][\"TPR\"],\n    edgecolor=\"midnightblue\",\n    facecolor=\"midnightblue\",\n    linewidth=0,\n    alpha=0.65,\n    label=(r\"Estimator's $\\mathrm{AUC}$ ($\\mathrm{AUC} = \" +\n           rf\"{dt_data['train']['auc']:.3f}$)\"))\n\nax.fill_between(\n    x=dt_data[\"train\"][\"FPR\"],\n    y1=dt_data[\"train\"][\"FPR\"],\n    y2=dt_data[\"train\"][\"TPR\"],\n    edgecolor=\"yellow\",\n    facecolor=\"None\",\n    linewidth=0,\n    hatch = \"xxx\",\n    alpha=1,\n    label=(r\"Estimator's $1/2$ Gini ($G = \" +\n           rf\"{dt_data['train']['g']:.3f}$)\"))\n\nax.plot(\n    [0, 1],\n    [0, 1],\n    linestyle=\"dashed\",\n    color=\"black\",\n    alpha=1,\n    linewidth=1.25,\n    label=\"Random estimator's ROC\")\n\nax.set_xlabel(r\"$\\mathrm{FPR}$\", fontdict={\"fontsize\": 10})\nax.set_ylabel(r\"$\\mathrm{TPR}$\", fontdict={\"fontsize\": 10})\n\nax.set_xlim([0, 1])\nax.set_ylim([0, 1])\n\nax.set_aspect(\"equal\")\n\nax.legend(loc=\"upper left\", fontsize=8)\n\nplt.show() ","metadata":{"execution":{"iopub.status.busy":"2025-05-22T12:21:36.121847Z","iopub.execute_input":"2025-05-22T12:21:36.122098Z","iopub.status.idle":"2025-05-22T12:21:36.532497Z","shell.execute_reply.started":"2025-05-22T12:21:36.122079Z","shell.execute_reply":"2025-05-22T12:21:36.531616Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_stability_score(\n    df_base,\n    w_G_av=1,\n    w_a=88.0,\n    w_RMSD=-0.5\n):\n    G = df_base[[\"WEEK_NUM\", \"y\", \"P_pred\"]]\\\n        .sort_values(by=\"WEEK_NUM\")\\\n        .groupby(by=\"WEEK_NUM\")[[\"y\", \"P_pred\"]]\\\n        .apply(lambda x:\n               2 * roc_auc_score(x[\"y\"], x[\"P_pred\"]) - 1).tolist()\n    \n    G_av = np.mean(G)\n\n    i = np.arange(len(G))\n    \n    [a, b] = np.polyfit(x=i, y=G, deg=1)\n    \n    G_fit = a * i + b\n    \n    RMSD = np.sqrt(np.mean((G_fit - G)**2))\n\n    stability_score = w_G_av * G_av + w_a * min(0, a) + w_RMSD * RMSD\n    \n    dt = {\n        \"g_week\": G,\n        \"a\": a,\n        \"b\": b,\n        \"RMSD\": RMSD,\n        \"stability_score\": stability_score\n    }\n    \n    return dt\n\ndt_data[\"train\"].update(get_stability_score(dt_data[\"train\"][\"base\"]))","metadata":{"execution":{"iopub.status.busy":"2025-05-22T12:21:36.533783Z","iopub.execute_input":"2025-05-22T12:21:36.534162Z","iopub.status.idle":"2025-05-22T12:21:37.307547Z","shell.execute_reply.started":"2025-05-22T12:21:36.534135Z","shell.execute_reply":"2025-05-22T12:21:37.306871Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(6.4, 4.8))\nax = plt.axes()\nplt.title(rf\"Stability score's elements\", pad=20)\n\nax.plot(range(len(dt_data[\"train\"][\"g_week\"])),\n        dt_data[\"train\"][\"g_week\"],\n        linestyle=\"None\",\n        marker=\"o\",\n        markeredgecolor=\"black\",\n        markerfacecolor=\"None\",\n        markersize=3,\n        alpha=1,\n        label=\"Points ($G_i$)\")\n\ni = np.array(range(len(dt_data[\"train\"][\"g_week\"])))[[0, -1]]\nax.plot(i,\n        dt_data[\"train\"][\"a\"] * i + dt_data[\"train\"][\"b\"],\n        linestyle=\"solid\",\n        color=\"blue\",\n        alpha=1,\n        linewidth=1,\n        label=(\"Fitting line ($G_{\\mathrm{fit}}(i) = a \\cdot i +b$, with \" +\n               f\"$a=${dt_data['train']['a']:.2e})\"))\n                \nax.set_xlabel(r\"Week number, $i$\", fontdict={\"fontsize\": 10})\nax.set_ylabel(r\"Gini coefficient, $G$\", fontdict={\"fontsize\": 10})\n\nax.minorticks_on()\n\nax.grid(visible=True, which=\"major\", color=\"lightgray\", linestyle=\"solid\",\n        linewidth=0.5)\nax.grid(visible=True, which=\"minor\", color=\"lightgray\", linestyle=\"dotted\",\n        linewidth=0.5)\n\n# Legend\nax.legend(fontsize=8)\n\n# Show plot\nplt.show() ","metadata":{"execution":{"iopub.status.busy":"2025-05-22T12:21:37.308516Z","iopub.execute_input":"2025-05-22T12:21:37.308773Z","iopub.status.idle":"2025-05-22T12:21:37.754327Z","shell.execute_reply.started":"2025-05-22T12:21:37.308753Z","shell.execute_reply":"2025-05-22T12:21:37.753487Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dt_summary = {\n    \"dataset\": \"train\",\n    \"precision\": dt_data[\"train\"][\"precision\"],\n    \"recall\": dt_data[\"train\"][\"recall\"],\n    \"f1-score\": dt_data[\"train\"][\"f1-score\"],\n    \"accuracy\": dt_data[\"train\"][\"accuracy\"],\n    \"auc\": dt_data[\"train\"][\"auc\"],\n    \"g\": dt_data[\"train\"][\"g\"],\n    \"stability_score\": dt_data[\"train\"][\"stability_score\"]\n}\n\nprint()\ndisplay(pd.DataFrame(data=dt_summary, index=[0])\\\n        .style.set_caption(\"Performance metrics for the training data\")\\\n        .set_table_styles(\n            [\n                {\"selector\": \"th.col_heading,td\",\n                 \"props\": [(\"width\", \"100px\")]\n                 },\n                {\"selector\": \"caption\",\n                 \"props\": [(\"font-size\", \"16px\"),\n                           (\"font-weight\", \"bold\"),\n                           (\"font-style\", \"italic\")]\n                 }\n            ]))","metadata":{"execution":{"iopub.status.busy":"2025-05-22T12:21:37.755713Z","iopub.execute_input":"2025-05-22T12:21:37.756312Z","iopub.status.idle":"2025-05-22T12:21:37.768401Z","shell.execute_reply.started":"2025-05-22T12:21:37.756279Z","shell.execute_reply":"2025-05-22T12:21:37.767535Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dt_data[\"test\"][\"base\"][\"P_pred\"] = best_estimator.predict_proba(\n    X=dt_data[\"test\"][\"x\"]\n)[:, 1]","metadata":{"execution":{"iopub.status.busy":"2025-05-22T12:21:37.769467Z","iopub.execute_input":"2025-05-22T12:21:37.769805Z","iopub.status.idle":"2025-05-22T12:21:37.785270Z","shell.execute_reply.started":"2025-05-22T12:21:37.769783Z","shell.execute_reply":"2025-05-22T12:21:37.784493Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission = pd.DataFrame({\n    \"case_id\": dt_data[\"test\"][\"base\"][\"case_id\"].to_numpy(),\n    \"score\": dt_data[\"test\"][\"base\"][\"P_pred\"]\n})\n\nprint()\ndisplay(submission.style.set_caption(\"Submission dataframe\")\\\n        .set_table_styles(\n            [\n                {\"selector\": \"th.col_heading,td\",\n                 \"props\": [(\"width\", \"100px\")]\n                 },\n                {\"selector\": \"caption\",\n                 \"props\": [(\"font-size\", \"16px\"),\n                           (\"font-weight\", \"bold\"),\n                           (\"font-style\", \"italic\")]\n                 }\n            ]))","metadata":{"execution":{"iopub.status.busy":"2025-05-22T12:21:37.786113Z","iopub.execute_input":"2025-05-22T12:21:37.786378Z","iopub.status.idle":"2025-05-22T12:21:37.804490Z","shell.execute_reply.started":"2025-05-22T12:21:37.786339Z","shell.execute_reply":"2025-05-22T12:21:37.803469Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2025-05-22T12:21:37.805915Z","iopub.execute_input":"2025-05-22T12:21:37.806822Z","iopub.status.idle":"2025-05-22T12:21:37.815238Z","shell.execute_reply.started":"2025-05-22T12:21:37.806798Z","shell.execute_reply":"2025-05-22T12:21:37.814520Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"masker = shap.maskers.Independent(\n    data=dt_data[\"train\"][\"x\"],\n    max_samples=100)\nexplainer = shap.Explainer(model=best_estimator.predict_proba, masker=masker)","metadata":{"execution":{"iopub.status.busy":"2025-05-22T12:21:37.816203Z","iopub.execute_input":"2025-05-22T12:21:37.816449Z","iopub.status.idle":"2025-05-22T12:21:38.876660Z","shell.execute_reply.started":"2025-05-22T12:21:37.816429Z","shell.execute_reply":"2025-05-22T12:21:38.875675Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"t_i = time.perf_counter()\n\nshapley = explainer(dt_data[\"train\"][\"x\"].sample(frac=0.005, random_state=42))\n\nt_f = time.perf_counter()\n\nprint()\nprint(f\"Running time: {(t_f - t_i)/3600:.2f} h\")","metadata":{"execution":{"iopub.status.busy":"2025-05-22T12:21:38.877860Z","iopub.execute_input":"2025-05-22T12:21:38.878124Z","iopub.status.idle":"2025-05-22T12:23:55.416040Z","shell.execute_reply.started":"2025-05-22T12:21:38.878104Z","shell.execute_reply":"2025-05-22T12:23:55.414335Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"shap.plots.bar(\n    shap_values = shapley[:,:,1].abs.mean(0),\n    max_display=11,\n    show=False\n)\n\nplt.title(\n    label=\"Feature importance, $I^{(j)}$\\n (the mean absolute Shapley value, $|\\phi^{(j)}|_{\\mathrm{av}}$)\",\n    loc=\"center\",\n    pad=20,\n    fontdict={\n        \"fontsize\": 15,\n        \"verticalalignment\": \"baseline\",\n        \"horizontalalignment\": \"center\"})\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2025-05-22T12:23:55.416980Z","iopub.status.idle":"2025-05-22T12:23:55.417379Z","shell.execute_reply.started":"2025-05-22T12:23:55.417168Z","shell.execute_reply":"2025-05-22T12:23:55.417187Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"shap.plots.beeswarm(\n    shap_values = shapley[:,:,1], \n    max_display=11,\n    color=shap.plots.colors.red_blue,\n    show=False\n)\n\nplt.title(\n    label=\"Features and respective Shapley values,\\n sorted by their importance ($I^{(j)}$)\",\n    loc=\"center\",\n    pad=20,\n    fontdict={\n        \"fontsize\": 15,\n        \"verticalalignment\": \"baseline\",\n        \"horizontalalignment\": \"center\"})\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2025-05-22T12:23:55.418972Z","iopub.status.idle":"2025-05-22T12:23:55.419371Z","shell.execute_reply.started":"2025-05-22T12:23:55.419167Z","shell.execute_reply":"2025-05-22T12:23:55.419184Z"},"trusted":true},"outputs":[],"execution_count":null}]}