{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7602123,"sourceType":"competition"},{"sourceId":7575001,"sourceType":"datasetVersion","datasetId":3383787}],"dockerImageVersionId":30646,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<div style=\"padding: 20px; background-color: #000080; border-radius: 10px; box-shadow: 0 4px 8px rgba(0, 0, 0, 0.1);\">\n    <div style=\"border: 2px solid #000080; padding: 20px; text-align: center; border-radius: 10px; background-color: #ffffff;\">\n        <h1 style=\"color: #ffff00; font-size: 32px; text-transform: uppercase; letter-spacing: 2px; margin-bottom: 20px;\">Easy EDA & modelling with BlueCast</h1>\n        <div><em>\n       If you like the content please consider an upvote. It is a great motivator to keep sharing code and ideas.\n        Thank you!!!\n    </em></div>\n</div>","metadata":{"_uuid":"dd880da3-22cc-40a3-bd1e-751ca320b86a","_cell_guid":"9bf4c6ad-abec-4da2-9feb-6179790fb30e","papermill":{"duration":0.016843,"end_time":"2023-12-17T05:29:21.962764","exception":false,"start_time":"2023-12-17T05:29:21.945921","status":"completed"},"tags":[],"trusted":true}},{"cell_type":"markdown","source":"<h1 style=\"background-color: #000080; color: #ffff00;\">Introducing BlueCast</h1>\n\nBluecast is an automl framework that offers a lightweight library with EDA, automl and xperiment tracking capabilities.\nIt offers many options for customization. Check out the repo to see many examples:\nhttps://github.com/ThomasMeissnerDS/BlueCast","metadata":{"_uuid":"a2d2a3ff-8448-49a5-86e1-59942b817c30","_cell_guid":"4526e3a0-af37-44f4-8967-a0f7b2efcb6d","papermill":{"duration":0.016187,"end_time":"2023-12-17T05:29:21.995242","exception":false,"start_time":"2023-12-17T05:29:21.979055","status":"completed"},"tags":[],"trusted":true}},{"cell_type":"markdown","source":"<h1 style=\"background-color: #000080; color: #ffff00;\">Table of contents</h1>\n\n* [Load the data](#1)\n* [Feature engineering](#2)\n    * [Add running features](#2.1)\n* [EDA with BlueCast](#3)\n    * [Feature type detection](#3.1)\n    * [Univariate plots](#3.2)    \n    * [Bivariate plots](#3.3) \n    * [Correlation to target](#3.4)  \n    * [Correlation heatmap](#3.5) \n    * [Mutual informtion score](#3.6)\n    * [Dimensionality reduction using PCA](#3.7) \n    * [Dimensionality reduction using t-SNE](#3.8) \n    * [Map of associations between categorical features](#3.9)\n    * [Missing values](#3.10)\n    * [Do we have columns with high cardinality?](#2.11) \n* [Leakage detection](#4)\n    * [Leakage detection for numerical columns](#4.1)\n    * [Leakage detection for categorical columns](#4.2)  \n* [Check for data drift](#5)\n    * [Data drift for numerical columns](#5.1)\n    * [## Data drift for categorical columns](#5.2)\n* [Building the pipeline in a few lines of code](#6)\n* [Plot decision trees](#7)\n* [Predict on new data](#8) \n* [Accessing the inbuilt experiment tracker](#9) \n    * [Understand most impactful parameters across all hyperparameters tests and model trainings](#9.1)\n    * [Don't lose your progress!](#9.2) \n* [Submission time](#10)","metadata":{"_uuid":"1497ad99-273c-4627-be8f-b42560f35d12","_cell_guid":"e1453dd8-52f9-47bc-b806-c2c36164a0b3","papermill":{"duration":0.016069,"end_time":"2023-12-17T05:29:22.027566","exception":false,"start_time":"2023-12-17T05:29:22.011497","status":"completed"},"tags":[],"trusted":true}},{"cell_type":"code","source":"from IPython.core.display import HTML\n\n# Define custom CSS directly in Python variable\ncustom_css = \"\"\"\n<style>\n  :root {\n    --header1_color: #204709;\n    --header2_color: #42841F;\n    --header3_color: #6EAF4B;\n    --keyword_color: #cc241d; /* import */\n    --string_color: #79740e;\n    --number_color: #b16286;\n    --def_color: #689d6a; /* class name */\n    --property_color: #458588; /* python properties */\n    --builtin_color: #689d6a;\n    --comment_color: #9f9f9f;\n    --comment_color_2: #458588; /* equals sign */\n    --operator_color: #a221f2;\n    --font_color: #3c3836; /* general font */\n    --variable2_color: #b16286; /*self keyworda */\n    --box_color: #fffdee66;\n  }\n\n  /* Add the following style for headers with background color */\n  h1,\n  .h1 {\n    font-family: \"Trebuchet MS\", sans-serif;\n    font-size: 2em !important;\n    letter-spacing: 1px;\n    color: var(--header1_color);\n    border-bottom: 3px solid var(--header1_color);\n    background-color: #000080;\n    padding: 0.5em;\n    color: #ffff00 !important;\n  }\n\n  h2,\n  .h2 {\n    font-family: \"Trebuchet MS\";\n    font-size: 1.7em !important;\n    color: var(--header2_color);\n    background-color: #000080;\n    padding: 0.5em;\n    color: #ffff00 !important;\n  }\n\n  h3,\n  .h3 {\n    font-family: \"Trebuchet MS\";\n    font-size: 1.4em !important;\n    color: var(--header3_color);\n    background-color: #000080;\n    padding: 0.5em;\n    color: #ffff00 !important;\n  }\n\n  /* Rest of your existing styles... */\n\n  body[data-jp-theme-light=\"true\"] .jp-Notebook .CodeMirror.cm-s-jupyter {\n    background-color: var(--box_color) !important;\n  }\n\n  div.input_area {\n    background-color: var(--box_color) !important;\n  }\n</style>\n\n\"\"\"\n\n# Apply custom CSS\nHTML(custom_css)","metadata":{"_uuid":"35ed8771-29fc-4d90-89cb-502213df4e41","_cell_guid":"012e49ce-7934-4fa7-b57a-c2e8e7172a1d","collapsed":false,"_kg_hide-input":true,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-02-06T16:03:52.982674Z","iopub.execute_input":"2024-02-06T16:03:52.983069Z","iopub.status.idle":"2024-02-06T16:03:53.029963Z","shell.execute_reply.started":"2024-02-06T16:03:52.983036Z","shell.execute_reply":"2024-02-06T16:03:53.028656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install bluecast --no-index --find-links=file:/kaggle/input/bluecast/bluecast-0.90-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2024-02-06T16:03:53.03209Z","iopub.execute_input":"2024-02-06T16:03:53.03249Z","iopub.status.idle":"2024-02-06T16:04:10.614745Z","shell.execute_reply.started":"2024-02-06T16:03:53.032457Z","shell.execute_reply":"2024-02-06T16:04:10.613261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#%%capture\n#!pip install bluecast #--no-index --find-links=file:/kaggle/input/bluecast/bluecast-0.90-py3-none-any.whl","metadata":{"_uuid":"0304eaa2-0d67-4e7a-a6f9-bd2ceecad7a4","_cell_guid":"9ceea33e-b435-4aaa-a082-e4969166e025","collapsed":false,"papermill":{"duration":11.823096,"end_time":"2023-12-17T05:29:52.721068","exception":false,"start_time":"2023-12-17T05:29:40.897972","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-02-06T16:04:10.620209Z","iopub.execute_input":"2024-02-06T16:04:10.620655Z","iopub.status.idle":"2024-02-06T16:04:10.627032Z","shell.execute_reply.started":"2024-02-06T16:04:10.620621Z","shell.execute_reply":"2024-02-06T16:04:10.62557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"toc\"></a>\n\n<a href=\"#toc\" style=\"background-color: #E1B12D; color: #ffffff; padding: 7px 10px; text-decoration: none; border-radius: 50px;\">Back to top</a><a id=\"toc\"></a>\n\n<a id=\"1.2\"></a>","metadata":{"_uuid":"d05c7bd1-1a2b-40c7-a254-7f780077c345","_cell_guid":"43cde00e-234b-452f-8200-6ceb2ca387c9","trusted":true}},{"cell_type":"markdown","source":"# Load the data","metadata":{"_uuid":"629c0c80-c595-4197-b05a-dd65ada05fc5","_cell_guid":"7ea7cad9-87e9-4073-9dfa-b18cd60a87ae","papermill":{"duration":0.017152,"end_time":"2023-12-17T05:29:52.755869","exception":false,"start_time":"2023-12-17T05:29:52.738717","status":"completed"},"tags":[],"trusted":true}},{"cell_type":"code","source":"from category_encoders import (\n    GLMMEncoder,\n    LeaveOneOutEncoder,\n    OneHotEncoder,\n    OrdinalEncoder,\n    TargetEncoder,\n    WOEEncoder,\n)\n\nfrom sklearn.decomposition import PCA\nfrom sklearn.preprocessing import RobustScaler\n\nimport numpy as np\nimport pandas as pd\nimport re\nfrom typing import Optional, Tuple, Union\n\nimport matplotlib.pyplot as plt\nfrom xgboost import plot_tree\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n\nfrom bluecast.blueprints.cast import BlueCast\nfrom bluecast.blueprints.cast_cv import BlueCastCV\nfrom bluecast.config.training_config import TrainingConfig, XgboostTuneParamsConfig\nfrom bluecast.preprocessing.custom import CustomPreprocessing\nfrom bluecast.general_utils.general_utils import save_to_production, load_for_production\n\nfrom sklearn.model_selection import StratifiedKFold, RepeatedStratifiedKFold\nfrom sklearn.preprocessing import PowerTransformer, LabelEncoder\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score \n\nimport polars as pl","metadata":{"_uuid":"24fda393-bbe0-4ee1-a2e1-135d0a305a85","_cell_guid":"d4b66a39-ecd1-4d28-9b35-89e76071aae0","collapsed":false,"papermill":{"duration":8.211699,"end_time":"2023-12-17T05:30:00.984724","exception":false,"start_time":"2023-12-17T05:29:52.773025","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-02-06T16:04:10.630513Z","iopub.execute_input":"2024-02-06T16:04:10.631334Z","iopub.status.idle":"2024-02-06T16:04:20.904928Z","shell.execute_reply.started":"2024-02-06T16:04:10.631291Z","shell.execute_reply":"2024-02-06T16:04:20.903815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataPath = \"/kaggle/input/home-credit-credit-risk-model-stability/\"","metadata":{"execution":{"iopub.status.busy":"2024-02-06T16:04:20.907114Z","iopub.execute_input":"2024-02-06T16:04:20.908525Z","iopub.status.idle":"2024-02-06T16:04:20.912786Z","shell.execute_reply.started":"2024-02-06T16:04:20.90847Z","shell.execute_reply":"2024-02-06T16:04:20.911971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_table_dtypes(df: pl.DataFrame) -> pl.DataFrame:\n    # implement here all desired dtypes for tables\n    # the following is just an example\n    for col in df.columns:\n        # last letter of column name will help you determine the type\n        if col[-1] in (\"P\", \"A\"):\n            df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n\n    return df\n\ndef convert_strings(df: pd.DataFrame) -> pd.DataFrame:\n    for col in df.columns:  \n        if df[col].dtype.name in ['object', 'string']:\n            df[col] = df[col].astype(\"string\").astype('category')\n            current_categories = df[col].cat.categories\n            new_categories = current_categories.to_list() + [\"Unknown\"]\n            new_dtype = pd.CategoricalDtype(categories=new_categories, ordered=True)\n            df[col] = df[col].astype(new_dtype)\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-02-06T16:04:20.914596Z","iopub.execute_input":"2024-02-06T16:04:20.914963Z","iopub.status.idle":"2024-02-06T16:04:20.930747Z","shell.execute_reply.started":"2024-02-06T16:04:20.914934Z","shell.execute_reply":"2024-02-06T16:04:20.92955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_basetable = pl.read_csv(dataPath + \"csv_files/train/train_base.csv\")\ntrain_static = pl.concat(\n    [\n        pl.read_csv(dataPath + \"csv_files/train/train_static_0_0.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/train/train_static_0_1.csv\").pipe(set_table_dtypes),\n    ],\n    how=\"vertical_relaxed\",\n)\ntrain_static_cb = pl.read_csv(dataPath + \"csv_files/train/train_static_cb_0.csv\").pipe(set_table_dtypes)\ntrain_person_1 = pl.read_csv(dataPath + \"csv_files/train/train_person_1.csv\").pipe(set_table_dtypes) \ntrain_credit_bureau_b_2 = pl.read_csv(dataPath + \"csv_files/train/train_credit_bureau_b_2.csv\").pipe(set_table_dtypes) ","metadata":{"execution":{"iopub.status.busy":"2024-02-06T16:04:20.932159Z","iopub.execute_input":"2024-02-06T16:04:20.932535Z","iopub.status.idle":"2024-02-06T16:04:39.763959Z","shell.execute_reply.started":"2024-02-06T16:04:20.932506Z","shell.execute_reply":"2024-02-06T16:04:39.76308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_basetable = pl.read_csv(dataPath + \"csv_files/test/test_base.csv\")\ntest_static = pl.concat(\n    [\n        pl.read_csv(dataPath + \"csv_files/test/test_static_0_0.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/test/test_static_0_1.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/test/test_static_0_2.csv\").pipe(set_table_dtypes),\n    ],\n    how=\"vertical_relaxed\",\n)\ntest_static_cb = pl.read_csv(dataPath + \"csv_files/test/test_static_cb_0.csv\").pipe(set_table_dtypes)\ntest_person_1 = pl.read_csv(dataPath + \"csv_files/test/test_person_1.csv\").pipe(set_table_dtypes) \ntest_credit_bureau_b_2 = pl.read_csv(dataPath + \"csv_files/test/test_credit_bureau_b_2.csv\").pipe(set_table_dtypes) ","metadata":{"_uuid":"2e7fbe36-5534-4296-9472-5bacb6ae3ea0","_cell_guid":"cd18463f-6bdd-47e0-8400-991d63d760a3","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-02-06T16:04:39.765323Z","iopub.execute_input":"2024-02-06T16:04:39.766033Z","iopub.status.idle":"2024-02-06T16:04:39.841336Z","shell.execute_reply.started":"2024-02-06T16:04:39.766002Z","shell.execute_reply":"2024-02-06T16:04:39.840472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"toc\"></a>\n\n<a href=\"#toc\" style=\"background-color: #E1B12D; color: #ffffff; padding: 7px 10px; text-decoration: none; border-radius: 50px;\">Back to top</a><a id=\"toc\"></a>\n\n<a id=\"1.2\"></a>","metadata":{"_uuid":"f1098f00-e870-4cf9-be2d-90ba3fb94188","_cell_guid":"fb5e5bd7-4cc8-44ab-a6bb-c05aff64decf","trusted":true}},{"cell_type":"markdown","source":"# Feature engineering\n\nIn this section we will build features using various techniques.","metadata":{"_uuid":"f4c395be-28a1-41cc-9266-42393758a34e","_cell_guid":"f1818465-c2b3-4601-be0e-e4fde97bb43e","trusted":true}},{"cell_type":"markdown","source":"Kudos to: https://www.kaggle.com/code/jetakow/home-credit-2024-starter-notebook","metadata":{}},{"cell_type":"code","source":"# We need to use aggregation functions in tables with depth > 1, so tables that contain num_group1 column or \n# also num_group2 column.\ntrain_person_1_feats_1 = train_person_1.group_by(\"case_id\").agg(\n    pl.col(\"mainoccupationinc_384A\").max().alias(\"mainoccupationinc_384A_max\"),\n    (pl.col(\"incometype_1044T\") == \"SELFEMPLOYED\").max().alias(\"mainoccupationinc_384A_any_selfemployed\")\n)\n\n# Here num_group1=0 has special meaning, it is the person who applied for the loan.\ntrain_person_1_feats_2 = train_person_1.select([\"case_id\", \"num_group1\", \"housetype_905L\"]).filter(\n    pl.col(\"num_group1\") == 0\n).drop(\"num_group1\").rename({\"housetype_905L\": \"person_housetype\"})\n\n# Here we have num_goup1 and num_group2, so we need to aggregate again.\ntrain_credit_bureau_b_2_feats = train_credit_bureau_b_2.group_by(\"case_id\").agg(\n    pl.col(\"pmts_pmtsoverdue_635A\").max().alias(\"pmts_pmtsoverdue_635A_max\"),\n    (pl.col(\"pmts_dpdvalue_108P\") > 31).max().alias(\"pmts_dpdvalue_108P_over31\")\n)\n\n# We will process in this examples only A-type and M-type columns, so we need to select them.\nselected_static_cols = []\nfor col in train_static.columns:\n    if col[-1] in (\"A\", \"M\"):\n        selected_static_cols.append(col)\nprint(selected_static_cols)\n\nselected_static_cb_cols = []\nfor col in train_static_cb.columns:\n    if col[-1] in (\"A\", \"M\"):\n        selected_static_cb_cols.append(col)\nprint(selected_static_cb_cols)\n\n# Join all tables together.\ndata = train_basetable.join(\n    train_static.select([\"case_id\"]+selected_static_cols), how=\"left\", on=\"case_id\"\n).join(\n    train_static_cb.select([\"case_id\"]+selected_static_cb_cols), how=\"left\", on=\"case_id\"\n).join(\n    train_person_1_feats_1, how=\"left\", on=\"case_id\"\n).join(\n    train_person_1_feats_2, how=\"left\", on=\"case_id\"\n).join(\n    train_credit_bureau_b_2_feats, how=\"left\", on=\"case_id\"\n)","metadata":{"execution":{"iopub.status.busy":"2024-02-06T16:04:39.845518Z","iopub.execute_input":"2024-02-06T16:04:39.846177Z","iopub.status.idle":"2024-02-06T16:04:42.915825Z","shell.execute_reply.started":"2024-02-06T16:04:39.846147Z","shell.execute_reply":"2024-02-06T16:04:42.914686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_person_1_feats_1 = train_person_1.group_by(\"case_id\").agg(\n    pl.col(\"mainoccupationinc_384A\").max().alias(\"mainoccupationinc_384A_max\"),\n    (pl.col(\"incometype_1044T\") == \"SELFEMPLOYED\").max().alias(\"mainoccupationinc_384A_any_selfemployed\")\n)\n\ntest_person_1_feats_2 = train_person_1.select([\"case_id\", \"num_group1\", \"housetype_905L\"]).filter(\n    pl.col(\"num_group1\") == 0\n).drop(\"num_group1\").rename({\"housetype_905L\": \"person_housetype\"})\n\ntest_credit_bureau_b_2_feats = train_credit_bureau_b_2.group_by(\"case_id\").agg(\n    pl.col(\"pmts_pmtsoverdue_635A\").max().alias(\"pmts_pmtsoverdue_635A_max\"),\n    (pl.col(\"pmts_dpdvalue_108P\") > 31).max().alias(\"pmts_dpdvalue_108P_over31\")\n)\n\ndata_submission = test_basetable.join(\n    test_static.select([\"case_id\"]+selected_static_cols), how=\"left\", on=\"case_id\"\n).join(\n    test_static_cb.select([\"case_id\"]+selected_static_cb_cols), how=\"left\", on=\"case_id\"\n).join(\n    test_person_1_feats_1, how=\"left\", on=\"case_id\"\n).join(\n    test_person_1_feats_2, how=\"left\", on=\"case_id\"\n).join(\n    test_credit_bureau_b_2_feats, how=\"left\", on=\"case_id\"\n)","metadata":{"execution":{"iopub.status.busy":"2024-02-06T16:04:42.918882Z","iopub.execute_input":"2024-02-06T16:04:42.919419Z","iopub.status.idle":"2024-02-06T16:04:44.419662Z","shell.execute_reply.started":"2024-02-06T16:04:42.919357Z","shell.execute_reply":"2024-02-06T16:04:44.418733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"case_ids = data[\"case_id\"].unique().shuffle(seed=1)\ncase_ids_train, case_ids_test = train_test_split(case_ids, train_size=0.6, random_state=1)\ncase_ids_valid, case_ids_test = train_test_split(case_ids_test, train_size=0.5, random_state=1)\n\ncols_pred = []\nfor col in data.columns:\n    if col[-1].isupper() and col[:-1].islower():\n        cols_pred.append(col)\n\nprint(cols_pred)\n\ndef from_polars_to_pandas(case_ids: pl.DataFrame) -> pl.DataFrame:\n    return (\n        data.filter(pl.col(\"case_id\").is_in(case_ids))[[\"case_id\", \"WEEK_NUM\", \"target\"]].to_pandas(),\n        data.filter(pl.col(\"case_id\").is_in(case_ids))[cols_pred].to_pandas(),\n        data.filter(pl.col(\"case_id\").is_in(case_ids))[\"target\"].to_pandas()\n    )\n\nbase_train, train_1, y_train = from_polars_to_pandas(case_ids_train)\nbase_train_2, train_2, y_train_2 = from_polars_to_pandas(case_ids_valid)\nbase_test, test, y_test = from_polars_to_pandas(case_ids_test)\n\nfor df in [train_1, train_2, test]:\n    df = convert_strings(df)","metadata":{"execution":{"iopub.status.busy":"2024-02-06T16:04:44.420669Z","iopub.execute_input":"2024-02-06T16:04:44.42098Z","iopub.status.idle":"2024-02-06T16:04:53.625236Z","shell.execute_reply.started":"2024-02-06T16:04:44.420953Z","shell.execute_reply":"2024-02-06T16:04:53.624024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Train: {train_1.shape}\")\nprint(f\"Valid: {train_2.shape}\")\nprint(f\"Test: {test.shape}\")","metadata":{"execution":{"iopub.status.busy":"2024-02-06T16:04:53.626783Z","iopub.execute_input":"2024-02-06T16:04:53.627151Z","iopub.status.idle":"2024-02-06T16:04:53.633518Z","shell.execute_reply.started":"2024-02-06T16:04:53.627119Z","shell.execute_reply":"2024-02-06T16:04:53.632189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_1[\"score\"] = y_train\ntrain_2[\"score\"] = y_train_2\ntest[\"score\"] = y_test","metadata":{"execution":{"iopub.status.busy":"2024-02-06T16:04:53.635004Z","iopub.execute_input":"2024-02-06T16:04:53.635593Z","iopub.status.idle":"2024-02-06T16:04:53.657961Z","shell.execute_reply.started":"2024-02-06T16:04:53.635561Z","shell.execute_reply":"2024-02-06T16:04:53.656407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.concat([train_1, train_2, test]).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-02-06T16:04:53.659626Z","iopub.execute_input":"2024-02-06T16:04:53.660577Z","iopub.status.idle":"2024-02-06T16:04:55.411591Z","shell.execute_reply.started":"2024-02-06T16:04:53.660527Z","shell.execute_reply":"2024-02-06T16:04:55.41044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_submission = data_submission[cols_pred].to_pandas()\nX_submission = convert_strings(X_submission)\ncategorical_cols = train.select_dtypes(include=['category']).columns\n\nfor col in categorical_cols:\n    train_categories = set(train[col].cat.categories)\n    submission_categories = set(X_submission[col].cat.categories)\n    new_categories = submission_categories - train_categories\n    X_submission.loc[X_submission[col].isin(new_categories), col] = \"Unknown\"\n    new_dtype = pd.CategoricalDtype(categories=train_categories, ordered=True)\n    train[col] = train[col].astype(new_dtype)\n    X_submission[col] = X_submission[col].astype(new_dtype)","metadata":{"execution":{"iopub.status.busy":"2024-02-06T16:04:55.413464Z","iopub.execute_input":"2024-02-06T16:04:55.413826Z","iopub.status.idle":"2024-02-06T16:04:55.491981Z","shell.execute_reply.started":"2024-02-06T16:04:55.413796Z","shell.execute_reply":"2024-02-06T16:04:55.490525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target = \"score\"","metadata":{"_uuid":"92447bd6-308a-4976-8f9d-3821fecdd9ad","_cell_guid":"add086ae-66aa-4504-b296-728af7021df5","collapsed":false,"papermill":{"duration":0.026318,"end_time":"2023-12-17T05:30:01.699639","exception":false,"start_time":"2023-12-17T05:30:01.673321","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-02-06T16:04:55.493511Z","iopub.execute_input":"2024-02-06T16:04:55.493892Z","iopub.status.idle":"2024-02-06T16:04:55.498774Z","shell.execute_reply.started":"2024-02-06T16:04:55.49386Z","shell.execute_reply":"2024-02-06T16:04:55.497643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[target].value_counts()","metadata":{"_uuid":"9b452d62-9d44-4baf-879d-1d5ead024c16","_cell_guid":"808d99fb-2271-4c91-8e04-07112224ffee","collapsed":false,"papermill":{"duration":0.028996,"end_time":"2023-12-17T05:30:01.746888","exception":false,"start_time":"2023-12-17T05:30:01.717892","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-02-06T16:04:55.500208Z","iopub.execute_input":"2024-02-06T16:04:55.500607Z","iopub.status.idle":"2024-02-06T16:04:55.531166Z","shell.execute_reply.started":"2024-02-06T16:04:55.500577Z","shell.execute_reply":"2024-02-06T16:04:55.530243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"toc\"></a>\n\n<a href=\"#toc\" style=\"background-color: #E1B12D; color: #ffffff; padding: 7px 10px; text-decoration: none; border-radius: 50px;\">Back to top</a><a id=\"toc\"></a>\n\n<a id=\"1.2\"></a>","metadata":{"_uuid":"e3ccf3a4-f685-43f6-b5ca-8b60623d45b7","_cell_guid":"c3a2b46b-fc39-407b-88b0-c711c64aa66b","trusted":true}},{"cell_type":"markdown","source":"# EDA with BlueCast\n\nHere you can get an overview of the data and its distribution.","metadata":{"_uuid":"0d420ad6-8d10-49eb-950c-759bed745db1","_cell_guid":"648a942f-5ff5-4c5e-96c1-d9ee42e41439","papermill":{"duration":0.018033,"end_time":"2023-12-17T05:30:01.78321","exception":false,"start_time":"2023-12-17T05:30:01.765177","status":"completed"},"tags":[],"trusted":true}},{"cell_type":"code","source":"from bluecast.eda.analyse import (\n    bi_variate_plots,\n    correlation_heatmap,\n    correlation_to_target,\n    plot_pca,\n    plot_theil_u_heatmap,\n    plot_tsne,\n    univariate_plots,\n    check_unique_values,\n    plot_null_percentage,\n    mutual_info_to_target\n)\n\nfrom bluecast.preprocessing.feature_types import FeatureTypeDetector","metadata":{"_uuid":"cdf189ad-a86d-4342-ae73-4d2da07c2f12","_cell_guid":"125abdc1-5404-4d56-bdf8-819182351cf8","collapsed":false,"papermill":{"duration":0.080426,"end_time":"2023-12-17T05:30:01.88187","exception":false,"start_time":"2023-12-17T05:30:01.801444","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-02-06T16:04:55.532961Z","iopub.execute_input":"2024-02-06T16:04:55.53342Z","iopub.status.idle":"2024-02-06T16:04:55.613745Z","shell.execute_reply.started":"2024-02-06T16:04:55.533357Z","shell.execute_reply":"2024-02-06T16:04:55.612419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature type detection","metadata":{"_uuid":"0a2e78cd-4284-43d3-a490-a2a9b549e307","_cell_guid":"dc010086-a23c-4291-960e-a291e3a64f74","papermill":{"duration":0.018069,"end_time":"2023-12-17T05:30:01.918744","exception":false,"start_time":"2023-12-17T05:30:01.900675","status":"completed"},"tags":[],"trusted":true}},{"cell_type":"code","source":"ignore_cols = []\n\nfeat_type_detector = FeatureTypeDetector()\ntrain_data = feat_type_detector.fit_transform_feature_types(train.drop(ignore_cols, axis=1))\n\nlen(feat_type_detector.num_columns)","metadata":{"_uuid":"1d2ae9aa-32ff-4df4-bd7b-a427e0d20b6d","_cell_guid":"9dc8f0c4-0eea-422d-9f5d-2d5e80834d90","collapsed":false,"papermill":{"duration":0.050219,"end_time":"2023-12-17T05:30:01.987293","exception":false,"start_time":"2023-12-17T05:30:01.937074","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-02-06T16:04:55.616286Z","iopub.execute_input":"2024-02-06T16:04:55.616988Z","iopub.status.idle":"2024-02-06T16:05:01.061616Z","shell.execute_reply.started":"2024-02-06T16:04:55.616956Z","shell.execute_reply":"2024-02-06T16:05:01.060758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eda_sample = train_data.sample(50000, random_state=25)","metadata":{"_uuid":"0a19bca1-c748-40a5-91e5-59df536f45ba","_cell_guid":"02337ccd-26da-4da2-8be9-98a64527431c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-02-06T16:05:01.062726Z","iopub.execute_input":"2024-02-06T16:05:01.063466Z","iopub.status.idle":"2024-02-06T16:05:01.068404Z","shell.execute_reply.started":"2024-02-06T16:05:01.063435Z","shell.execute_reply":"2024-02-06T16:05:01.067092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Univariate plots","metadata":{"_uuid":"8bed970d-9482-4d73-8180-ce325525d0fc","_cell_guid":"99c8759a-6895-40dd-a2eb-66778e4e696b","papermill":{"duration":0.018469,"end_time":"2023-12-17T05:30:02.026089","exception":false,"start_time":"2023-12-17T05:30:02.00762","status":"completed"},"tags":[],"trusted":true}},{"cell_type":"code","source":"univariate_plots(\n        eda_sample.loc[\n            :, feat_type_detector.num_columns\n        ],\n    )","metadata":{"_uuid":"84d7bda4-ef89-42dd-ae71-999565d1c16e","_cell_guid":"c43c7a36-80e0-4524-b5d2-26b0221c8a9a","collapsed":false,"papermill":{"duration":8.794109,"end_time":"2023-12-17T05:30:10.838647","exception":false,"start_time":"2023-12-17T05:30:02.044538","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-02-06T16:05:01.069911Z","iopub.execute_input":"2024-02-06T16:05:01.070695Z","iopub.status.idle":"2024-02-06T16:12:13.378835Z","shell.execute_reply.started":"2024-02-06T16:05:01.070663Z","shell.execute_reply":"2024-02-06T16:12:13.377483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Bivariate plots","metadata":{"_uuid":"360727ae-fd20-4b34-a7e3-a7b5f3e8f1b6","_cell_guid":"dc297ba6-b5c6-40b0-95e7-820ae5b3a81d","papermill":{"duration":0.025798,"end_time":"2023-12-17T05:30:10.890694","exception":false,"start_time":"2023-12-17T05:30:10.864896","status":"completed"},"tags":[],"trusted":true}},{"cell_type":"code","source":"bi_variate_plots(\n        eda_sample.loc[\n            :, feat_type_detector.num_columns\n        ],\n        target,\n    )","metadata":{"_uuid":"5e9be562-7e8a-4358-8201-831bf84d73e1","_cell_guid":"b626d4c0-f033-4675-8e97-e4d634efc247","collapsed":false,"papermill":{"duration":3.300788,"end_time":"2023-12-17T05:30:14.217101","exception":false,"start_time":"2023-12-17T05:30:10.916313","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-02-06T16:12:13.380226Z","iopub.execute_input":"2024-02-06T16:12:13.380587Z","iopub.status.idle":"2024-02-06T16:13:27.974903Z","shell.execute_reply.started":"2024-02-06T16:12:13.380558Z","shell.execute_reply":"2024-02-06T16:13:27.97355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Correlation to target","metadata":{"_uuid":"2af23814-dd7a-4f9d-b8d1-6bd54301b842","_cell_guid":"b4d02ac6-5d28-4be8-b5ad-7fac8341316a","papermill":{"duration":0.029717,"end_time":"2023-12-17T05:30:14.277554","exception":false,"start_time":"2023-12-17T05:30:14.247837","status":"completed"},"tags":[],"trusted":true}},{"cell_type":"code","source":"# show correlation to target\ncorrelation_to_target(\n    eda_sample.loc[:, feat_type_detector.num_columns],\n      target,\n      )","metadata":{"_uuid":"3975835a-9670-4d5a-b64e-65c042b10fb5","_cell_guid":"9f126412-22db-405b-8b4b-b58b3c918d82","collapsed":false,"papermill":{"duration":0.453592,"end_time":"2023-12-17T05:30:14.761039","exception":false,"start_time":"2023-12-17T05:30:14.307447","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-02-06T16:13:27.976886Z","iopub.execute_input":"2024-02-06T16:13:27.977338Z","iopub.status.idle":"2024-02-06T16:13:32.57776Z","shell.execute_reply.started":"2024-02-06T16:13:27.9773Z","shell.execute_reply":"2024-02-06T16:13:32.576282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Correlation heatmap","metadata":{"_uuid":"b5ef5f2e-b607-4721-92e2-d1f956a1a74f","_cell_guid":"88c56320-4cca-4ed6-b857-922b1bcf5ea3","papermill":{"duration":0.029897,"end_time":"2023-12-17T05:30:14.821487","exception":false,"start_time":"2023-12-17T05:30:14.79159","status":"completed"},"tags":[],"trusted":true}},{"cell_type":"code","source":"correlation_heatmap(eda_sample.loc[\n            :, feat_type_detector.num_columns])","metadata":{"_uuid":"462aa3f1-853c-46ed-90e9-6f86c0a68c26","_cell_guid":"d08cd338-ecea-4839-a3ed-d6f13d562bbb","collapsed":false,"papermill":{"duration":0.526088,"end_time":"2023-12-17T05:30:15.377318","exception":false,"start_time":"2023-12-17T05:30:14.85123","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-02-06T16:13:32.585083Z","iopub.execute_input":"2024-02-06T16:13:32.585487Z","iopub.status.idle":"2024-02-06T16:13:37.234903Z","shell.execute_reply.started":"2024-02-06T16:13:32.585455Z","shell.execute_reply":"2024-02-06T16:13:37.233474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Mutual information score","metadata":{"_uuid":"d097c3ee-60ee-4630-9b59-76f6bf884fd9","_cell_guid":"a8e08928-3fdb-4d63-af68-ab90896a8a07","papermill":{"duration":0.033624,"end_time":"2023-12-17T05:30:15.443677","exception":false,"start_time":"2023-12-17T05:30:15.410053","status":"completed"},"tags":[],"trusted":true}},{"cell_type":"code","source":"# show mutual information of categorical features to target\n# features are expected to be numerical format\nextra_params = {\"random_state\": 30}\nmutual_info_to_target(eda_sample.loc[:, feat_type_detector.num_columns].fillna(0), target, class_problem=\"binary\", **extra_params)","metadata":{"_uuid":"25fb17e4-1e49-4449-920f-63fdb636e94c","_cell_guid":"0b2fb354-fc96-4484-9192-3b215d2c24c0","collapsed":false,"papermill":{"duration":1.163219,"end_time":"2023-12-17T05:30:16.639502","exception":false,"start_time":"2023-12-17T05:30:15.476283","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-02-06T16:13:37.23667Z","iopub.execute_input":"2024-02-06T16:13:37.23704Z","iopub.status.idle":"2024-02-06T16:22:50.500317Z","shell.execute_reply.started":"2024-02-06T16:13:37.23701Z","shell.execute_reply":"2024-02-06T16:22:50.498688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dimensionality reduction using PCA","metadata":{"_uuid":"a1ad7b01-bc79-4a99-a228-f6686c7b5374","_cell_guid":"3744ea3e-cd17-4672-afb1-4762c2def18e","papermill":{"duration":0.033741,"end_time":"2023-12-17T05:30:16.712174","exception":false,"start_time":"2023-12-17T05:30:16.678433","status":"completed"},"tags":[],"trusted":true}},{"cell_type":"code","source":"# show feature space after principal component analysis\nplot_pca(eda_sample.loc[\n            :, feat_type_detector.num_columns\n        ].fillna(0), target)","metadata":{"_uuid":"4ec1b774-860f-44eb-9040-241c49a3fc62","_cell_guid":"be6aa479-04b9-438f-a57a-3dfac8f1b908","collapsed":false,"papermill":{"duration":0.943879,"end_time":"2023-12-17T05:30:17.688576","exception":false,"start_time":"2023-12-17T05:30:16.744697","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-02-06T16:22:50.503454Z","iopub.execute_input":"2024-02-06T16:22:50.504194Z","iopub.status.idle":"2024-02-06T16:23:57.118159Z","shell.execute_reply.started":"2024-02-06T16:22:50.504143Z","shell.execute_reply":"2024-02-06T16:23:57.116751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It looks like linear models can be promising here. Most classes can be linearly separated very well.","metadata":{"_uuid":"a83d186b-609d-4c9d-ae05-58b4d1e384df","_cell_guid":"fc73ef64-a68f-48f6-939f-3ba0881a57c5","trusted":true}},{"cell_type":"markdown","source":"## Dimensionality reduction using t-SNE","metadata":{"_uuid":"a3cb5ff6-d2f2-4876-809a-a4bb919f8896","_cell_guid":"8df8632e-0cd4-440c-8727-9bcc694fc5f1","papermill":{"duration":0.041459,"end_time":"2023-12-17T05:30:17.769876","exception":false,"start_time":"2023-12-17T05:30:17.728417","status":"completed"},"tags":[],"trusted":true}},{"cell_type":"code","source":"# show feature space after t-SNE\n#plot_tsne(train_data.loc[\n#            :, feat_type_detector.num_columns\n#        ].fillna(0), target, perplexity=1000, random_state=0)","metadata":{"_uuid":"65cc789a-6c9f-414f-b070-008500d4633f","_cell_guid":"0545e21b-e18d-49a7-84b5-481e435e2273","collapsed":false,"papermill":{"duration":40.540988,"end_time":"2023-12-17T05:30:58.349735","exception":false,"start_time":"2023-12-17T05:30:17.808747","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-02-06T16:23:57.119925Z","iopub.execute_input":"2024-02-06T16:23:57.120271Z","iopub.status.idle":"2024-02-06T16:23:57.126981Z","shell.execute_reply.started":"2024-02-06T16:23:57.120242Z","shell.execute_reply":"2024-02-06T16:23:57.125349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Map of associations between categorical features","metadata":{"_uuid":"0f68f445-e2a8-4337-9e97-447a2650de52","_cell_guid":"feb5f1f1-b645-4fc5-8c76-019c3c77ddcb","papermill":{"duration":0.043861,"end_time":"2023-12-17T05:30:58.439156","exception":false,"start_time":"2023-12-17T05:30:58.395295","status":"completed"},"tags":[],"trusted":true}},{"cell_type":"code","source":"# show a heatmap of assocations between categorical variables\nif len(feat_type_detector.cat_columns) > 0:\n    theil_matrix = plot_theil_u_heatmap(eda_sample, feat_type_detector.cat_columns)\n    theil_matrix","metadata":{"_uuid":"7ce3f01f-1cd4-4ffc-bd18-a14767c291dc","_cell_guid":"0865f243-6041-43f3-ba85-fa4fb1d5feb5","collapsed":false,"papermill":{"duration":0.843514,"end_time":"2023-12-17T05:30:59.326612","exception":false,"start_time":"2023-12-17T05:30:58.483098","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-02-06T16:23:57.128509Z","iopub.execute_input":"2024-02-06T16:23:57.128954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Missing values","metadata":{"_uuid":"79852b53-0892-4618-a88a-e9a7d60fabb2","_cell_guid":"2e5a4bca-d796-4da7-9546-35c260c2a479","papermill":{"duration":0.047442,"end_time":"2023-12-17T05:30:59.419759","exception":false,"start_time":"2023-12-17T05:30:59.372317","status":"completed"},"tags":[],"trusted":true}},{"cell_type":"code","source":"# plot the percentage of Nulls for all features\nif eda_sample.loc[:, feat_type_detector.num_columns].isna().sum().sum() > 0:\n    plot_null_percentage(\n       train_data.loc[:, feat_type_detector.num_columns],\n        )\nelse:\n    print(\"This dataset does not have any missing values\")","metadata":{"_uuid":"60b5423a-6515-4229-8d39-0f709eaadf9d","_cell_guid":"fdc40754-0c04-462a-9163-1522bc1d0f62","collapsed":false,"papermill":{"duration":0.503336,"end_time":"2023-12-17T05:30:59.970197","exception":false,"start_time":"2023-12-17T05:30:59.466861","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Do we have columns with high cardinality?","metadata":{"_uuid":"f240d2d5-9580-4d10-9ff2-82b7741a66af","_cell_guid":"b167b428-16bb-489e-b2e9-250ccbd25937","papermill":{"duration":0.045154,"end_time":"2023-12-17T05:31:00.063546","exception":false,"start_time":"2023-12-17T05:31:00.018392","status":"completed"},"tags":[],"trusted":true}},{"cell_type":"code","source":"# detect columns with a very high share of unique values\nmany_unique_cols = check_unique_values(eda_sample, feat_type_detector.cat_columns, threshold=0.90)\nmany_unique_cols","metadata":{"_uuid":"84f6de56-2e7f-47c3-9920-fc9343f15ebe","_cell_guid":"d176cfab-d00e-4e20-8ba9-752bb02f2ee5","collapsed":false,"papermill":{"duration":0.059117,"end_time":"2023-12-17T05:31:00.168221","exception":false,"start_time":"2023-12-17T05:31:00.109104","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Leakage detection\n\nWith big data and complex pipelines data leakage can easily sneak in.\nTo detect leakage BlueCast offers two functions:","metadata":{"_uuid":"bef04a1a-50dd-4def-8478-0e09ad28d61d","_cell_guid":"f370c3b6-3c44-4a90-8a8b-81c686961ccd","papermill":{"duration":0.044606,"end_time":"2023-12-17T05:31:00.258219","exception":false,"start_time":"2023-12-17T05:31:00.213613","status":"completed"},"tags":[],"trusted":true}},{"cell_type":"code","source":"from bluecast.eda.data_leakage_checks import (\n    detect_categorical_leakage,\n    detect_leakage_via_correlation,\n)","metadata":{"_uuid":"dba130f9-62f9-4707-8dad-0af76cb09ce9","_cell_guid":"b5ebdcbc-dfd5-48f6-8427-301f685cff45","collapsed":false,"papermill":{"duration":0.053099,"end_time":"2023-12-17T05:31:00.356218","exception":false,"start_time":"2023-12-17T05:31:00.303119","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Leakage detection for numerical columns","metadata":{"_uuid":"7ad62441-1994-4dd5-b867-a035ebb961b1","_cell_guid":"908c29ca-4b27-4fed-9177-51b4f4b9a436","papermill":{"duration":0.045512,"end_time":"2023-12-17T05:31:00.446739","exception":false,"start_time":"2023-12-17T05:31:00.401227","status":"completed"},"tags":[],"trusted":true}},{"cell_type":"code","source":"# Detect leakage of numeric columns based on correlation\nnumresult = detect_leakage_via_correlation(\n        eda_sample.loc[:, feat_type_detector.num_columns], target, threshold=0.9 # target column is part of detected numerical columns here\n    )\nnumresult","metadata":{"_uuid":"d97e5f5a-50d6-4e3a-af99-3c4fd58f008b","_cell_guid":"f2b12201-f0bd-45f0-b19c-a880898040bd","collapsed":false,"papermill":{"duration":0.061327,"end_time":"2023-12-17T05:31:00.554105","exception":false,"start_time":"2023-12-17T05:31:00.492778","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Leakage detection for categorical columns","metadata":{"_uuid":"fbc9c79b-21b5-4a4b-8e16-b1dc0f6313f2","_cell_guid":"b5877d04-6624-435c-b964-e6729884acb8","papermill":{"duration":0.044869,"end_time":"2023-12-17T05:31:00.645232","exception":false,"start_time":"2023-12-17T05:31:00.600363","status":"completed"},"tags":[],"trusted":true}},{"cell_type":"code","source":"# Detect leakage of categorical columns based on Theil's U\nresult = detect_categorical_leakage(\n        eda_sample.loc[:, feat_type_detector.cat_columns + [target]], target, threshold=0.9\n    )\nresult","metadata":{"_uuid":"07349d69-be02-469e-9465-5451c178d2ea","_cell_guid":"cf75b535-38ad-4342-b71c-f69abfedff21","collapsed":false,"papermill":{"duration":0.071453,"end_time":"2023-12-17T05:31:00.761657","exception":false,"start_time":"2023-12-17T05:31:00.690204","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This could also be a false positive when these features are contant. We might neeed to drop these.","metadata":{"_uuid":"ec72a326-34f4-4d93-b769-fe4057793b1f","_cell_guid":"e527e01c-e66f-41c3-acdb-97de21b0c837","papermill":{"duration":0.044907,"end_time":"2023-12-17T05:31:00.852201","exception":false,"start_time":"2023-12-17T05:31:00.807294","status":"completed"},"tags":[],"trusted":true}},{"cell_type":"markdown","source":"# Check for data drift","metadata":{"_uuid":"e2f509b4-51ea-4939-84f1-04453ea06262","_cell_guid":"c906798d-b809-410a-ab87-6cd3b76d2011","trusted":true}},{"cell_type":"code","source":"from bluecast.monitoring.data_monitoring import DataDrift","metadata":{"_uuid":"6c449423-df90-450a-87ea-98d1eac996e5","_cell_guid":"1a570287-0752-4125-ba96-890c9d5c3bce","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_drift = DataDrift()","metadata":{"_uuid":"736b538d-4e5d-40b3-82d0-9969db4326f7","_cell_guid":"502b5b5f-4495-4bed-8a9d-648d7f511652","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data drift for numerical columns","metadata":{"_uuid":"0fbd83bf-4e96-4793-bdfc-992729d9168f","_cell_guid":"75ce5f0e-93a3-4f1d-a5fd-943431c4a68c","trusted":true}},{"cell_type":"code","source":"data_drift.kolmogorov_smirnov_test(eda_sample, X_submission)\ndata_drift.kolmogorov_smirnov_flags","metadata":{"_uuid":"20ac3b52-7da3-40ac-a18f-ddc16d047848","_cell_guid":"6dd3321a-4690-4d57-b788-911347a57584","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data drift for categorical columns","metadata":{"_uuid":"93251b97-4acf-4cdf-b321-60400a193432","_cell_guid":"177e51db-7536-4f7e-97b0-049ba5029982","trusted":true}},{"cell_type":"code","source":"data_drift.population_stability_index(eda_sample.drop(target, axis=1), X_submission)\ndata_drift.population_stability_index_flags","metadata":{"_uuid":"78b7b4d5-3df9-4b42-a3c5-5a50f586b125","_cell_guid":"ac0a0b92-30c6-498a-84d8-655d386a593a","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#for col, data_drift_flag in data_drift.kolmogorov_smirnov_flags.items():\n#    data_drift.qqplot_two_samples(train[col], test[col], x_label=f\"{col} from train\", y_label=f\"{col} from test\")","metadata":{"_uuid":"5c9e9afa-51a9-496c-a9bc-00a1eb7cc187","_cell_guid":"4b9be9cc-ee01-46d7-bc7c-c47e3126c125","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"toc\"></a>\n\n<a href=\"#toc\" style=\"background-color: #E1B12D; color: #ffffff; padding: 7px 10px; text-decoration: none; border-radius: 50px;\">Back to top</a><a id=\"toc\"></a>\n\n<a id=\"1.2\"></a>","metadata":{"_uuid":"7eac713b-7489-461e-b8e9-6417e1cebbe6","_cell_guid":"94ab1904-5368-4757-905a-0bf92902196b","trusted":true}},{"cell_type":"markdown","source":"# Building the pipeline in a few lines of code","metadata":{"_uuid":"2c3fd73b-a9ee-49d4-af68-c9c489b7a535","_cell_guid":"e5ad5f36-4683-42ab-839f-a500116777e6","papermill":{"duration":0.045726,"end_time":"2023-12-17T05:31:00.942985","exception":false,"start_time":"2023-12-17T05:31:00.897259","status":"completed"},"tags":[],"trusted":true}},{"cell_type":"code","source":"from bluecast.config.training_config import TrainingConfig, XgboostTuneParamsConfig\n\n# We give more depth\nxgboost_param_config = XgboostTuneParamsConfig()\nxgboost_param_config.steps_max = 1000\nxgboost_param_config.max_depth_max = 7\n\n# Create a custom training config and adjust general training parameters\ntrain_config = TrainingConfig()\ntrain_config.global_random_state = 5432\ntrain_config.hypertuning_cv_folds = 5\ntrain_config.hyperparameter_tuning_rounds = 200\ntrain_config.hyperparameter_tuning_max_runtime_secs = 60 * 60 * 9\n#train_config.enable_grid_search_fine_tuning = True # enable param refinement\ntrain_config.use_full_data_for_final_model = True\n#train_config.precise_cv_tuning = True\n#train_config.gridsearch_nb_parameters_per_grid = 5\n#train_config.cat_encoding_via_ml_algorithm = True\n#train_config.calculate_shap_values = False\n\nskf = RepeatedStratifiedKFold(n_splits=5, n_repeats=2, random_state=1987)","metadata":{"_uuid":"7632463f-8bd8-4748-9934-7cd18e8e45a8","_cell_guid":"0eb0062b-cf1c-4095-8ab7-c5f17f3e3974","collapsed":false,"papermill":{"duration":0.055277,"end_time":"2023-12-17T05:31:01.043513","exception":false,"start_time":"2023-12-17T05:31:00.988236","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-02-06T04:37:09.329397Z","iopub.execute_input":"2024-02-06T04:37:09.329884Z","iopub.status.idle":"2024-02-06T04:37:09.341888Z","shell.execute_reply.started":"2024-02-06T04:37:09.329849Z","shell.execute_reply":"2024-02-06T04:37:09.340086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"automl = BlueCast(\n        class_problem=\"binary\", # also multiclass is possible\n        #stratifier=skf,\n        conf_training=train_config,\n        conf_xgboost=xgboost_param_config,\n        #custom_in_fold_preprocessor=custom_preprocessor,\n        #custom_preprocessor=custom_preprocessor,\n        #ml_model=custom_model_tab,\n        target_column=target\n        )","metadata":{"_uuid":"ccdffa1b-bb6a-46b2-a63d-0aa192383aea","_cell_guid":"5fd9ba2b-dffd-435b-b3ad-e5a34a62d776","collapsed":false,"papermill":{"duration":0.053261,"end_time":"2023-12-17T05:31:01.142291","exception":false,"start_time":"2023-12-17T05:31:01.08903","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-02-06T04:37:22.963858Z","iopub.execute_input":"2024-02-06T04:37:22.964321Z","iopub.status.idle":"2024-02-06T04:37:22.972372Z","shell.execute_reply.started":"2024-02-06T04:37:22.964289Z","shell.execute_reply":"2024-02-06T04:37:22.969429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"automl.fit(train.sample(500000, random_state=100), target_col=target)","metadata":{"_uuid":"e8d4ce21-0495-4de2-a236-b5fdaff923ba","_cell_guid":"bdd94422-0a87-40dd-94c6-945ccbeb2fda","collapsed":false,"papermill":{"duration":16463.286806,"end_time":"2023-12-17T10:05:24.475794","exception":false,"start_time":"2023-12-17T05:31:01.188988","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-02-06T04:37:22.975318Z","iopub.execute_input":"2024-02-06T04:37:22.976487Z","iopub.status.idle":"2024-02-06T04:40:56.6746Z","shell.execute_reply.started":"2024-02-06T04:37:22.976445Z","shell.execute_reply":"2024-02-06T04:40:56.670995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"toc\"></a>\n\n<a href=\"#toc\" style=\"background-color: #E1B12D; color: #ffffff; padding: 7px 10px; text-decoration: none; border-radius: 50px;\">Back to top</a><a id=\"toc\"></a>\n\n<a id=\"1.2\"></a>","metadata":{"_uuid":"9bd8f2db-2dbf-48fd-93e4-ec0658a54bde","_cell_guid":"13c53fd2-2f98-448e-a4b5-089d4859604b","trusted":true}},{"cell_type":"markdown","source":"# Plot decision trees","metadata":{"_uuid":"69e5d99d-590d-4606-b7af-3a55d3df95d8","_cell_guid":"adab20d7-97f7-45f1-88bc-1dd34b3ddc87","papermill":{"duration":0.086162,"end_time":"2023-12-17T10:05:24.652597","exception":false,"start_time":"2023-12-17T10:05:24.566435","status":"completed"},"tags":[],"trusted":true}},{"cell_type":"code","source":"plot_tree(automl.ml_model.model)\nfig = plt.gcf()\nfig.set_size_inches(150, 80)\nplt.show()","metadata":{"_uuid":"1f059146-72e6-43ca-bf5a-c6328e7f917f","_cell_guid":"5ce38182-d36e-48ac-8195-eaed951c5708","collapsed":false,"papermill":{"duration":27.96353,"end_time":"2023-12-17T10:05:52.699392","exception":false,"start_time":"2023-12-17T10:05:24.735862","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-02-06T04:40:56.676269Z","iopub.status.idle":"2024-02-06T04:40:56.676915Z","shell.execute_reply.started":"2024-02-06T04:40:56.676554Z","shell.execute_reply":"2024-02-06T04:40:56.676578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predict on new data","metadata":{"_uuid":"af616f03-66b5-4bf6-886d-8c38d8ff51c3","_cell_guid":"b7b0cfd9-0707-4902-8608-4fdfef239270","papermill":{"duration":0.226997,"end_time":"2023-12-17T10:05:53.163223","exception":false,"start_time":"2023-12-17T10:05:52.936226","status":"completed"},"tags":[],"trusted":true}},{"cell_type":"code","source":"probs, classes = automl.predict(X_submission)","metadata":{"_uuid":"dcd299d7-b1b1-472c-89f2-e4df3b263291","_cell_guid":"471009c9-cd0f-4cd4-8810-a3e81d3b9fd8","collapsed":false,"papermill":{"duration":0.905294,"end_time":"2023-12-17T10:05:54.27796","exception":false,"start_time":"2023-12-17T10:05:53.372666","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-02-06T04:40:56.678286Z","iopub.status.idle":"2024-02-06T04:40:56.678865Z","shell.execute_reply.started":"2024-02-06T04:40:56.678559Z","shell.execute_reply":"2024-02-06T04:40:56.678585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Don't lose your progress!\n\nBlueCast offers simple utilities to save and load your pipeline (including the tracker)","metadata":{"_uuid":"3784986d-7b38-411b-9bd7-52df76cff828","_cell_guid":"8c84ed01-1c0c-4254-85fb-babebcd5b19a","papermill":{"duration":0.212888,"end_time":"2023-12-17T10:06:01.063902","exception":false,"start_time":"2023-12-17T10:06:00.851014","status":"completed"},"tags":[],"trusted":true}},{"cell_type":"code","source":"# save pipeline including tracker\nsave_to_production(automl, \"/kaggle/working/\", \"bluecast_cv_pipeline\")\n\n# in production or for further experiments this can be loaded again\nautoml_loaded = load_for_production(\"/kaggle/working/\", \"bluecast_cv_pipeline\")","metadata":{"_uuid":"bb4df005-781c-4495-a04c-cd674f140109","_cell_guid":"a26b0b57-78a9-47fe-bcc6-27525bed81dc","collapsed":false,"papermill":{"duration":0.790559,"end_time":"2023-12-17T10:06:02.06785","exception":false,"start_time":"2023-12-17T10:06:01.277291","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-02-06T04:40:56.69073Z","iopub.status.idle":"2024-02-06T04:40:56.691386Z","shell.execute_reply.started":"2024-02-06T04:40:56.691084Z","shell.execute_reply":"2024-02-06T04:40:56.691109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission time","metadata":{"_uuid":"564a207c-5e20-457d-a76d-64e046eb59b5","_cell_guid":"c05cd2f9-df9f-416e-b962-3e02f8b7a06a","papermill":{"duration":0.237528,"end_time":"2023-12-17T10:06:02.553176","exception":false,"start_time":"2023-12-17T10:06:02.315648","status":"completed"},"tags":[],"trusted":true}},{"cell_type":"code","source":"submission = pd.DataFrame({\n    \"case_id\": data_submission[\"case_id\"].to_numpy(),\n    \"score\": probs\n}).set_index('case_id')\nsubmission.to_csv(\"./submission.csv\")","metadata":{"_uuid":"d429b25f-a347-446e-99a5-1f6f83b4c070","_cell_guid":"4484c665-52b5-4b3a-bac4-ed0222fe999b","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-02-06T04:40:56.693625Z","iopub.status.idle":"2024-02-06T04:40:56.694167Z","shell.execute_reply.started":"2024-02-06T04:40:56.693921Z","shell.execute_reply":"2024-02-06T04:40:56.693941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"_uuid":"b93a832f-a5eb-4017-9e32-b9a0186ad29a","_cell_guid":"568db178-ce4e-4e14-a4a5-48bf67fed63c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-02-06T04:40:56.700349Z","iopub.status.idle":"2024-02-06T04:40:56.701237Z","shell.execute_reply.started":"2024-02-06T04:40:56.700851Z","shell.execute_reply":"2024-02-06T04:40:56.700876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"toc\"></a>\n\n<a href=\"#toc\" style=\"background-color: #E1B12D; color: #ffffff; padding: 7px 10px; text-decoration: none; border-radius: 50px;\">Back to top</a><a id=\"toc\"></a>\n\n<a id=\"1.2\"></a>","metadata":{"_uuid":"096fcf6f-39e1-46e8-baa4-9a61dce0d0f1","_cell_guid":"e2518101-ed32-40d1-9f2a-db9d9c27f3a0","trusted":true}}]}