{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":50160,"databundleVersionId":7602123,"sourceType":"competition"},{"sourceId":7591933,"sourceType":"datasetVersion","datasetId":4419271},{"sourceId":7595068,"sourceType":"datasetVersion","datasetId":4420820},{"sourceId":7595084,"sourceType":"datasetVersion","datasetId":4420830},{"sourceId":7595590,"sourceType":"datasetVersion","datasetId":4421149}],"dockerImageVersionId":30646,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport gc\nfrom glob import glob\nfrom pathlib import Path\nfrom datetime import datetime\n\nimport numpy as np\nimport pandas as pd\nimport polars as pl\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.model_selection import TimeSeriesSplit, GroupKFold, StratifiedGroupKFold\nfrom sklearn.base import BaseEstimator, RegressorMixin\n\nimport joblib\n\nimport lightgbm as lgb\n\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)","metadata":{"_uuid":"3d4e0809-3518-4855-a9a9-20cbdfbb1995","_cell_guid":"fce2eea7-274e-488c-a753-ed2248628fe9","collapsed":false,"execution":{"iopub.status.busy":"2024-02-09T08:41:28.715743Z","iopub.execute_input":"2024-02-09T08:41:28.716179Z","iopub.status.idle":"2024-02-09T08:41:34.237389Z","shell.execute_reply.started":"2024-02-09T08:41:28.71614Z","shell.execute_reply":"2024-02-09T08:41:34.236177Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Pipeline:\n    @staticmethod\n    def set_table_dtypes(df):\n        for col in df.columns:\n            if col in [\"case_id\", \"WEEK_NUM\", \"num_group1\", \"num_group2\"]:\n                df = df.with_columns(pl.col(col).cast(pl.Int64))\n            elif col in [\"date_decision\"]:\n                df = df.with_columns(pl.col(col).cast(pl.Date))\n            elif col[-1] in (\"P\", \"A\"):\n                df = df.with_columns(pl.col(col).cast(pl.Float64))\n            elif col[-1] in (\"M\",):\n                df = df.with_columns(pl.col(col).cast(pl.String))\n            elif col[-1] in (\"D\",):\n                df = df.with_columns(pl.col(col).cast(pl.Date))            \n\n        return df\n    \n    @staticmethod\n    def handle_dates(df):\n        for col in df.columns:\n            if col[-1] in (\"D\",):\n                df = df.with_columns(pl.col(col) - pl.col(\"date_decision\"))\n                df = df.with_columns(pl.col(col).dt.total_days())\n                \n        df = df.drop(\"date_decision\", \"MONTH\")\n\n        return df\n    \n    @staticmethod\n    def filter_cols(df):\n        for col in df.columns:\n            if col not in [\"target\", \"case_id\", \"WEEK_NUM\"]:\n                isnull = df[col].is_null().mean()\n\n                if isnull > 0.95:\n                    df = df.drop(col)\n\n        for col in df.columns:\n            if (col not in [\"target\", \"case_id\", \"WEEK_NUM\"]) & (df[col].dtype == pl.String):\n                freq = df[col].n_unique()\n\n                if (freq == 1) | (freq > 200):\n                    df = df.drop(col)\n\n        return df","metadata":{"_uuid":"85b79683-b55f-452c-a2bc-dd449f375d5a","_cell_guid":"fe21a36e-5d9f-4949-802c-b7c39e047949","collapsed":false,"execution":{"iopub.status.busy":"2024-02-09T08:41:34.23976Z","iopub.execute_input":"2024-02-09T08:41:34.240628Z","iopub.status.idle":"2024-02-09T08:41:34.263001Z","shell.execute_reply.started":"2024-02-09T08:41:34.240581Z","shell.execute_reply":"2024-02-09T08:41:34.261274Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Aggregator:\n    @staticmethod\n    def num_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"P\", \"A\")]\n\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n\n        return expr_max\n\n    @staticmethod\n    def date_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"D\",)]\n\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n\n        return expr_max\n\n    @staticmethod\n    def str_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"M\",)]\n        \n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n\n        return expr_max\n\n    @staticmethod\n    def other_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"T\", \"L\")]\n        \n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n\n        return expr_max\n    \n    @staticmethod\n    def count_expr(df):\n        cols = [col for col in df.columns if \"num_group\" in col]\n\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n\n        return expr_max\n\n    @staticmethod\n    def get_exprs(df):\n        exprs = Aggregator.num_expr(df) + \\\n                Aggregator.date_expr(df) + \\\n                Aggregator.str_expr(df) + \\\n                Aggregator.other_expr(df) + \\\n                Aggregator.count_expr(df)\n\n        return exprs","metadata":{"_uuid":"d60bcb63-d0f9-40df-881f-a003d88470c5","_cell_guid":"d05b2fcd-098c-461f-aff7-331899607836","collapsed":false,"execution":{"iopub.status.busy":"2024-02-09T08:41:34.265643Z","iopub.execute_input":"2024-02-09T08:41:34.266651Z","iopub.status.idle":"2024-02-09T08:41:34.285147Z","shell.execute_reply.started":"2024-02-09T08:41:34.266574Z","shell.execute_reply":"2024-02-09T08:41:34.283979Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_file(path, depth=None):\n    df = pl.read_parquet(path)\n    df = df.pipe(Pipeline.set_table_dtypes)\n    \n    if depth in [1, 2]:\n        df = df.group_by(\"case_id\").agg(Aggregator.get_exprs(df))\n    \n    return df\n\ndef read_files(regex_path, depth=None):\n    chunks = []\n    for path in glob(str(regex_path)):\n        chunks.append(pl.read_parquet(path).pipe(Pipeline.set_table_dtypes))\n        \n    df = pl.concat(chunks, how=\"vertical_relaxed\")\n    if depth in [1, 2]:\n        df = df.group_by(\"case_id\").agg(Aggregator.get_exprs(df))\n    \n    return df","metadata":{"_uuid":"fcd92928-ade1-4081-9386-0f05dcdba917","_cell_guid":"4a622e74-29fe-492e-9cbe-632c21376cb4","collapsed":false,"execution":{"iopub.status.busy":"2024-02-09T08:41:34.289334Z","iopub.execute_input":"2024-02-09T08:41:34.290481Z","iopub.status.idle":"2024-02-09T08:41:34.302351Z","shell.execute_reply.started":"2024-02-09T08:41:34.290434Z","shell.execute_reply":"2024-02-09T08:41:34.300984Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_eng(df_base, depth_0, depth_1, depth_2):\n    df_base = (\n        df_base\n        .with_columns(\n            month_decision = pl.col(\"date_decision\").dt.month(),\n            weekday_decision = pl.col(\"date_decision\").dt.weekday(),\n        )\n    )\n        \n    for i, df in enumerate(depth_0 + depth_1 + depth_2):\n        df_base = df_base.join(df, how=\"left\", on=\"case_id\", suffix=f\"_{i}\")\n        \n    df_base = df_base.pipe(Pipeline.handle_dates)\n    \n    return df_base","metadata":{"_uuid":"6e3af296-672f-4f79-97a7-cc850b6a8f78","_cell_guid":"81b6c66d-adc9-4643-b990-9ed1dea185f5","collapsed":false,"execution":{"iopub.status.busy":"2024-02-09T08:41:34.304813Z","iopub.execute_input":"2024-02-09T08:41:34.305792Z","iopub.status.idle":"2024-02-09T08:41:34.315508Z","shell.execute_reply.started":"2024-02-09T08:41:34.305726Z","shell.execute_reply":"2024-02-09T08:41:34.31414Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def to_pandas(df_data, cat_cols=None):\n    df_data = df_data.to_pandas()\n    \n    if cat_cols is None:\n        cat_cols = list(df_data.select_dtypes(\"object\").columns)\n    \n    df_data[cat_cols] = df_data[cat_cols].astype(\"category\")\n    \n    return df_data, cat_cols","metadata":{"_uuid":"3ab0c14b-0d36-45e3-ac81-c5698e06fc56","_cell_guid":"a80cf721-e117-4e3e-9590-e6f214fab917","collapsed":false,"execution":{"iopub.status.busy":"2024-02-09T08:41:34.317867Z","iopub.execute_input":"2024-02-09T08:41:34.318823Z","iopub.status.idle":"2024-02-09T08:41:34.331008Z","shell.execute_reply.started":"2024-02-09T08:41:34.318779Z","shell.execute_reply":"2024-02-09T08:41:34.329712Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Configuration","metadata":{"_uuid":"32f85615-437a-466e-a49d-b35a9f79ccc9","_cell_guid":"55328b79-25d4-44f2-9653-3877d2d3f760","trusted":true}},{"cell_type":"code","source":"ROOT            = Path(\"/kaggle/input/home-credit-credit-risk-model-stability\")\nTRAIN_DIR       = ROOT / \"parquet_files\" / \"train\"\nTEST_DIR        = ROOT / \"parquet_files\" / \"test\"","metadata":{"_uuid":"1bbb017c-f7ad-442d-a2a7-acfd79aa857b","_cell_guid":"bdde1632-10b7-473f-9162-cf75d3013125","collapsed":false,"execution":{"iopub.status.busy":"2024-02-09T08:41:34.33339Z","iopub.execute_input":"2024-02-09T08:41:34.334305Z","iopub.status.idle":"2024-02-09T08:41:34.341539Z","shell.execute_reply.started":"2024-02-09T08:41:34.334244Z","shell.execute_reply":"2024-02-09T08:41:34.33995Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Train Files Read & Feature Engineering","metadata":{"_uuid":"c1b42b9e-92e1-47bf-b5ee-e0d5dbff822a","_cell_guid":"37061613-6aeb-4e3a-8402-61ee02e134a3","trusted":true}},{"cell_type":"code","source":"data_store = {\n    \"df_base\": read_file(TRAIN_DIR / \"train_base.parquet\"),\n    \"depth_0\": [\n        read_file(TRAIN_DIR / \"train_static_cb_0.parquet\"),\n        read_files(TRAIN_DIR / \"train_static_0_*.parquet\"),\n    ],\n    \"depth_1\": [\n        read_files(TRAIN_DIR / \"train_applprev_1_*.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_a_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_b_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_c_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_credit_bureau_b_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_other_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_person_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_deposit_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_debitcard_1.parquet\", 1),\n    ],\n    \"depth_2\": [\n        read_file(TRAIN_DIR / \"train_credit_bureau_b_2.parquet\", 2),\n    ]\n}","metadata":{"_uuid":"c00c8bd3-6eca-44c9-ab8b-feda8a5dee2f","_cell_guid":"d3a8268f-543a-46d7-b5f5-9064116d0cda","collapsed":false,"execution":{"iopub.status.busy":"2024-02-09T08:41:34.34538Z","iopub.execute_input":"2024-02-09T08:41:34.347389Z","iopub.status.idle":"2024-02-09T08:42:09.99477Z","shell.execute_reply.started":"2024-02-09T08:41:34.347051Z","shell.execute_reply":"2024-02-09T08:42:09.993456Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = feature_eng(**data_store)\n\nprint(\"train data shape:\\t\", df_train.shape)","metadata":{"_uuid":"eafe8ae8-f8ee-4346-b77f-d718e1f7a589","_cell_guid":"694377c6-59b0-44ab-aa7d-a2341da45bf2","collapsed":false,"execution":{"iopub.status.busy":"2024-02-09T08:42:09.996391Z","iopub.execute_input":"2024-02-09T08:42:09.996795Z","iopub.status.idle":"2024-02-09T08:42:19.128505Z","shell.execute_reply.started":"2024-02-09T08:42:09.996762Z","shell.execute_reply":"2024-02-09T08:42:19.127327Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Test Files Read & Feature Engineering","metadata":{"_uuid":"442dc518-f559-4896-b602-491240698062","_cell_guid":"cc4e3c09-dd48-4023-99ab-a50858a446d1","trusted":true}},{"cell_type":"code","source":"data_store = {\n    \"df_base\": read_file(TEST_DIR / \"test_base.parquet\"),\n    \"depth_0\": [\n        read_file(TEST_DIR / \"test_static_cb_0.parquet\"),\n        read_files(TEST_DIR / \"test_static_0_*.parquet\"),\n    ],\n    \"depth_1\": [\n        read_files(TEST_DIR / \"test_applprev_1_*.parquet\", 1),\n        read_file(TEST_DIR / \"test_tax_registry_a_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_tax_registry_b_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_tax_registry_c_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_credit_bureau_b_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_other_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_person_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_deposit_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_debitcard_1.parquet\", 1),\n    ],\n    \"depth_2\": [\n        read_file(TEST_DIR / \"test_credit_bureau_b_2.parquet\", 2),\n    ]\n}","metadata":{"_uuid":"8b8dfbcf-fa4f-43bb-b962-7ab3854cbe02","_cell_guid":"917f2da4-3bc8-4246-96de-ae8dc56f0950","collapsed":false,"execution":{"iopub.status.busy":"2024-02-09T08:42:19.132754Z","iopub.execute_input":"2024-02-09T08:42:19.133236Z","iopub.status.idle":"2024-02-09T08:42:19.590392Z","shell.execute_reply.started":"2024-02-09T08:42:19.133185Z","shell.execute_reply":"2024-02-09T08:42:19.588678Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = feature_eng(**data_store)\n\nprint(\"test data shape:\\t\", df_test.shape)","metadata":{"_uuid":"d9f842ff-96d7-41d0-bccc-be4b6e609da0","_cell_guid":"658167ea-3659-4a84-b18d-e3a5bcf423b8","collapsed":false,"execution":{"iopub.status.busy":"2024-02-09T08:42:19.59204Z","iopub.execute_input":"2024-02-09T08:42:19.592454Z","iopub.status.idle":"2024-02-09T08:42:19.633684Z","shell.execute_reply.started":"2024-02-09T08:42:19.592423Z","shell.execute_reply":"2024-02-09T08:42:19.632688Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = df_train.pipe(Pipeline.filter_cols)\ndf_test = df_test.select([col for col in df_train.columns if col != \"target\"])\n\nprint(\"train data shape:\\t\", df_train.shape)\nprint(\"test data shape:\\t\", df_test.shape)","metadata":{"_uuid":"d071bbec-856e-4fb1-8bd2-06240ec55952","_cell_guid":"8a0218e6-ca3e-49f3-bc67-a0a9e9aa00ac","collapsed":false,"execution":{"iopub.status.busy":"2024-02-09T08:42:19.634911Z","iopub.execute_input":"2024-02-09T08:42:19.635747Z","iopub.status.idle":"2024-02-09T08:42:22.268265Z","shell.execute_reply.started":"2024-02-09T08:42:19.635714Z","shell.execute_reply":"2024-02-09T08:42:22.265892Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train, cat_cols = to_pandas(df_train)\ndf_test, cat_cols = to_pandas(df_test, cat_cols)","metadata":{"_uuid":"745d2ca5-7346-4745-b09b-4356b8a482a4","_cell_guid":"97d13ea2-2c6e-4b5f-8a58-71fe875276a8","collapsed":false,"execution":{"iopub.status.busy":"2024-02-09T08:42:22.270242Z","iopub.execute_input":"2024-02-09T08:42:22.271479Z","iopub.status.idle":"2024-02-09T08:42:38.60713Z","shell.execute_reply.started":"2024-02-09T08:42:22.271438Z","shell.execute_reply":"2024-02-09T08:42:38.605387Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del data_store\n\ngc.collect()","metadata":{"_uuid":"adfae797-3e81-4bbf-81c9-f8ab6fac8d31","_cell_guid":"ca1f9f35-6703-44ba-a7d4-014f2e9e29db","collapsed":false,"execution":{"iopub.status.busy":"2024-02-09T08:42:38.609162Z","iopub.execute_input":"2024-02-09T08:42:38.609887Z","iopub.status.idle":"2024-02-09T08:42:38.745334Z","shell.execute_reply.started":"2024-02-09T08:42:38.609851Z","shell.execute_reply":"2024-02-09T08:42:38.743856Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### EDA","metadata":{"_uuid":"fe8eb3dd-5f1c-4414-91bb-ad558a80c811","_cell_guid":"776e6cf5-b08f-4c8a-9552-0f33956c3a82","trusted":true}},{"cell_type":"code","source":"print(\"Train is duplicated:\\t\", df_train[\"case_id\"].duplicated().any())\nprint(\"Train Week Range:\\t\", (df_train[\"WEEK_NUM\"].min(), df_train[\"WEEK_NUM\"].max()))\n\nprint()\n\nprint(\"Test is duplicated:\\t\", df_test[\"case_id\"].duplicated().any())\nprint(\"Test Week Range:\\t\", (df_test[\"WEEK_NUM\"].min(), df_test[\"WEEK_NUM\"].max()))","metadata":{"_uuid":"9b78a140-c7f3-4ab2-8e61-c03964eab442","_cell_guid":"20cb5b38-c69d-4ee1-b4ce-135345db13d8","collapsed":false,"execution":{"iopub.status.busy":"2024-02-09T08:42:38.746993Z","iopub.execute_input":"2024-02-09T08:42:38.748034Z","iopub.status.idle":"2024-02-09T08:42:38.792725Z","shell.execute_reply.started":"2024-02-09T08:42:38.747986Z","shell.execute_reply":"2024-02-09T08:42:38.791387Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.lineplot(\n    data=df_train,\n    x=\"WEEK_NUM\",\n    y=\"target\",\n)\nplt.show()","metadata":{"_uuid":"448294ef-bd26-4691-81f1-9d27a9228e4e","_cell_guid":"4a288dcb-71bb-4a25-ba00-a77ab967b16c","collapsed":false,"execution":{"iopub.status.busy":"2024-02-09T08:42:38.794529Z","iopub.execute_input":"2024-02-09T08:42:38.795313Z","iopub.status.idle":"2024-02-09T08:42:58.264452Z","shell.execute_reply.started":"2024-02-09T08:42:38.795268Z","shell.execute_reply":"2024-02-09T08:42:58.263001Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Training","metadata":{"_uuid":"daa125da-2fc4-453b-bd21-00a34df5c721","_cell_guid":"b264d2b3-638b-4303-8157-3f6535c68b3e","trusted":true}},{"cell_type":"code","source":"class VotingModel(BaseEstimator, RegressorMixin):\n    def __init__(self, estimators):\n        super().__init__()\n        self.estimators = estimators\n        \n    def fit(self, X, y=None):\n        return self\n    \n    def predict(self, X):\n        y_preds = [estimator.predict(X) for estimator in self.estimators]\n        return np.mean(y_preds, axis=0)\n    \n    def predict_proba(self, X):\n        y_preds = [estimator.predict_proba(X) for estimator in self.estimators]\n        return np.mean(y_preds, axis=0)","metadata":{"_uuid":"73653d31-9590-46c7-93a4-1504696a1c2e","_cell_guid":"b18f4c7c-b341-40e2-a13d-1b45e0407d39","collapsed":false,"execution":{"iopub.status.busy":"2024-02-09T08:42:58.266275Z","iopub.execute_input":"2024-02-09T08:42:58.267462Z","iopub.status.idle":"2024-02-09T08:42:58.277117Z","shell.execute_reply.started":"2024-02-09T08:42:58.267411Z","shell.execute_reply":"2024-02-09T08:42:58.275941Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import roc_auc_score\n\nX = df_train.drop(columns=[\"target\", \"case_id\",\"WEEK_NUM\"])\ny = df_train[\"target\"]\nweeks = df_train[\"WEEK_NUM\"]\n\ncv = StratifiedGroupKFold(n_splits=5, shuffle=False)\n\nparams = {\n    \"boosting_type\": \"gbdt\",\n    \"objective\": \"binary\",\n    \"metric\": \"auc\",\n    \"max_depth\": 8,\n    \"learning_rate\": 0.05,\n    \"n_estimators\": 1000,\n    \"colsample_bytree\": 0.8, \n    \"colsample_bynode\": 0.8,\n    \"verbose\": -1,\n    \"random_state\": 42,\n    #\"device\": \"gpu\",\n}\n\nfitted_models = []\ncv_scores = []  \n\nfor idx_train, idx_valid in cv.split(X, y, groups=weeks):\n    X_train, y_train = X.iloc[idx_train], y.iloc[idx_train]\n    X_valid, y_valid = X.iloc[idx_valid], y.iloc[idx_valid]\n\n    print(\"Valid week range: \", (weeks.iloc[idx_valid].min(), weeks.iloc[idx_valid].max()))\n\n    model = lgb.LGBMClassifier(**params)\n    model.fit(\n        X_train, y_train,\n        eval_set=[(X_valid, y_valid)],\n        callbacks=[lgb.log_evaluation(50), lgb.early_stopping(50)]\n    )\n\n    fitted_models.append(model)\n\n    y_pred_valid = model.predict_proba(X_valid)[:, 1]\n    auc_score = roc_auc_score(y_valid, y_pred_valid)\n    cv_scores.append(auc_score)\n\nmodel = VotingModel(fitted_models)\nprint(\"CV AUC scores: \", cv_scores)\nprint(\"Average CV AUC score: \", sum(cv_scores) / len(cv_scores))","metadata":{"_uuid":"a529d6b7-2327-420c-894a-4f5d32a3611a","_cell_guid":"de469088-ac5e-4d7f-991b-ee9efa6fba09","collapsed":false,"execution":{"iopub.status.busy":"2024-02-09T08:42:58.278308Z","iopub.execute_input":"2024-02-09T08:42:58.279491Z","iopub.status.idle":"2024-02-09T08:46:34.848731Z","shell.execute_reply.started":"2024-02-09T08:42:58.279451Z","shell.execute_reply":"2024-02-09T08:46:34.847396Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = df_test.drop(columns=[\"WEEK_NUM\"])\nX_test = X_test.set_index(\"case_id\")\n\nlgb_pred = pd.Series(model.predict_proba(X_test)[:, 1], index=X_test.index)","metadata":{"_uuid":"0e2a8853-b861-43ac-90dd-713291be8125","_cell_guid":"9d635443-58ce-4935-a3be-78b67b7fa0a6","collapsed":false,"execution":{"iopub.status.busy":"2024-02-09T08:46:34.850627Z","iopub.execute_input":"2024-02-09T08:46:34.850999Z","iopub.status.idle":"2024-02-09T08:46:35.113379Z","shell.execute_reply.started":"2024-02-09T08:46:34.850968Z","shell.execute_reply":"2024-02-09T08:46:35.110857Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del model","metadata":{"_uuid":"99f0ed24-d223-4770-905a-1deabe98fb0b","_cell_guid":"89535208-be8e-4483-97a1-17a469d7ca90","collapsed":false,"execution":{"iopub.status.busy":"2024-02-09T08:46:35.115611Z","iopub.execute_input":"2024-02-09T08:46:35.116202Z","iopub.status.idle":"2024-02-09T08:46:35.121496Z","shell.execute_reply.started":"2024-02-09T08:46:35.116154Z","shell.execute_reply":"2024-02-09T08:46:35.120372Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **NN**","metadata":{"_uuid":"0a1ebfd3-fa7e-48eb-8949-cedbd95de3a0","_cell_guid":"de68cbf6-1add-4cf7-b3a8-b2a8599ad0ce","trusted":true}},{"cell_type":"code","source":"def fill_na(df, fill_value_numeric=0, fill_value_categorical='Missing', fill_value_default='Unknown'):\n    \"\"\"\n    Fills missing values in a DataFrame.\n\n    Parameters:\n    df (pd.DataFrame): The DataFrame to fill missing values in.\n    fill_value_numeric (int or float): The value to fill missing values with in numeric columns.\n    fill_value_categorical (str): The value to fill missing values with in categorical columns.\n    fill_value_default (str): The value to fill missing values with in other types of columns.\n\n    Returns:\n    pd.DataFrame: DataFrame with missing values filled.\n    \"\"\"\n    for col in df.columns:\n        if df[col].dtype.name == 'category':\n            # Add a new category for missing values and fill with it\n            df[col] = df[col].cat.add_categories([fill_value_categorical]).fillna(fill_value_categorical)\n        elif pd.api.types.is_numeric_dtype(df[col]):\n            # Fill numeric columns with the specified numeric value\n            df[col] = df[col].fillna(fill_value_numeric)\n        else:\n            # Fill other types of columns with the specified default value\n            df[col] = df[col].fillna(fill_value_default)\n    return df\n\nX_train = fill_na(X)\nX_test = fill_na(X_test)","metadata":{"_uuid":"1ab3a078-6a2a-4e0b-aafb-3762744d3c77","_cell_guid":"c97eb19a-1570-4cfc-a4c2-a7ff8176ee50","collapsed":false,"execution":{"iopub.status.busy":"2024-02-09T08:46:35.123637Z","iopub.execute_input":"2024-02-09T08:46:35.124961Z","iopub.status.idle":"2024-02-09T08:46:38.128887Z","shell.execute_reply.started":"2024-02-09T08:46:35.124906Z","shell.execute_reply":"2024-02-09T08:46:38.127398Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del X\n\ngc.collect()","metadata":{"_uuid":"1bbb10ad-45de-4cb2-81af-a5a2544b214c","_cell_guid":"f293ba9e-5fb7-4180-99e9-15a8bc13a9bf","collapsed":false,"execution":{"iopub.status.busy":"2024-02-09T08:46:38.1308Z","iopub.execute_input":"2024-02-09T08:46:38.131217Z","iopub.status.idle":"2024-02-09T08:46:38.272728Z","shell.execute_reply.started":"2024-02-09T08:46:38.131183Z","shell.execute_reply":"2024-02-09T08:46:38.271396Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_cols = [col for col in X_train.columns if X_train[col].dtype == 'category']\nnum_cols = [col for col in X_train.columns if col not in cat_cols]","metadata":{"_uuid":"108e8255-5aba-42e2-ac83-97f398ee83da","_cell_guid":"62b71f4f-44fd-402a-9a8d-f47c54f019dd","collapsed":false,"execution":{"iopub.status.busy":"2024-02-09T08:46:38.274493Z","iopub.execute_input":"2024-02-09T08:46:38.275345Z","iopub.status.idle":"2024-02-09T08:46:38.300214Z","shell.execute_reply.started":"2024-02-09T08:46:38.275302Z","shell.execute_reply":"2024-02-09T08:46:38.299017Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nfrom sklearn.preprocessing import LabelEncoder\n\n\nlabel_encoders = {}\nfor col in cat_cols:\n    le = LabelEncoder()\n    all_values = np.concatenate([X_train[col],X_test[col]], axis=0).astype(str)\n    le.fit(all_values)\n    X_train[col] = le.transform(X_train[col].astype(str))\n    X_test[col] = le.transform(X_test[col].astype(str))\n    label_encoders[col] = le\n\nX_test_combined = np.hstack([X_test[cat_cols].values, X_test[num_cols].values])\nX_train_combined = np.hstack([X_train[cat_cols].values, X_train[num_cols].values])\n\nX_test = X_test_combined.astype('float32')\nX_train = X_train_combined.astype('float32')","metadata":{"_uuid":"1b1946f6-81a1-4bfa-8dff-02d8ebd316b6","_cell_guid":"556cfebd-8f42-4617-8042-e5e57d3ff174","collapsed":false,"execution":{"iopub.status.busy":"2024-02-09T08:46:38.301843Z","iopub.execute_input":"2024-02-09T08:46:38.302909Z","iopub.status.idle":"2024-02-09T08:48:47.005431Z","shell.execute_reply.started":"2024-02-09T08:46:38.302858Z","shell.execute_reply":"2024-02-09T08:48:47.004195Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del X_test_combined,X_train_combined\n\ngc.collect()","metadata":{"_uuid":"b75f08e3-d6eb-4c8a-809c-8c6454718547","_cell_guid":"6978c19a-6a54-4be5-98da-25c44bacdc50","collapsed":false,"execution":{"iopub.status.busy":"2024-02-09T08:48:47.007152Z","iopub.execute_input":"2024-02-09T08:48:47.007558Z","iopub.status.idle":"2024-02-09T08:48:56.311938Z","shell.execute_reply.started":"2024-02-09T08:48:47.007522Z","shell.execute_reply":"2024-02-09T08:48:56.310013Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train = y","metadata":{"_uuid":"90395a3d-2a34-42f5-bf12-5972f1e1828e","_cell_guid":"680f8a03-851a-450c-ba76-a9f1352d6a22","collapsed":false,"execution":{"iopub.status.busy":"2024-02-09T08:48:56.313716Z","iopub.execute_input":"2024-02-09T08:48:56.314255Z","iopub.status.idle":"2024-02-09T08:48:56.330228Z","shell.execute_reply.started":"2024-02-09T08:48:56.314205Z","shell.execute_reply":"2024-02-09T08:48:56.32916Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"To save time, we use hold-one-out cv at the moment.","metadata":{"_uuid":"ab19d214-81f0-45a2-a03a-3526c0f1b857","_cell_guid":"edb6a4bf-21e2-4240-ae74-e452b019f8ea","trusted":true}},{"cell_type":"code","source":"week_threshold = 70\n\ntrain_indices = df_train[df_train.WEEK_NUM < week_threshold].index\nvalid_indices = df_train[df_train.WEEK_NUM >= week_threshold].index\n\nX_valid = X_train[valid_indices]\ny_valid = df_train.loc[valid_indices, 'target']\n\nX_train = X_train[train_indices]\ny_train = df_train.loc[train_indices, 'target']","metadata":{"_uuid":"4cc4cec3-117c-425a-823c-5df09d83e294","_cell_guid":"4d428cb5-8ba8-45b5-b98a-28c8b647bf99","collapsed":false,"execution":{"iopub.status.busy":"2024-02-09T08:50:08.937956Z","iopub.execute_input":"2024-02-09T08:50:08.938446Z","iopub.status.idle":"2024-02-09T08:50:11.313129Z","shell.execute_reply.started":"2024-02-09T08:50:08.938411Z","shell.execute_reply":"2024-02-09T08:50:11.311692Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.callbacks import EarlyStopping,LearningRateScheduler\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Input, Dense,BatchNormalization, Dropout\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.metrics import AUC\nimport tensorflow as tf\n\ninput_layer = Input(shape=(X_train.shape[1],), name='input_layer')\n\nbatch_norm_input = BatchNormalization(name='batch_norm_input')(input_layer)\n\ndense_layer_1 = Dense(64, name='dense1')(batch_norm_input)\nbatch_norm_1 = BatchNormalization(name='batch_norm1')(dense_layer_1)\ndropout_1 = Dropout(0.5, name='dropout1')(batch_norm_1)\nx = tf.keras.layers.Activation('swish')(dropout_1)\n\ndense_layer_2 = Dense(32, name='dense2')(x)\nbatch_norm_2 = BatchNormalization(name='batch_norm2')(dense_layer_2)\ndropout_2 = Dropout(0.5, name='dropout2')(batch_norm_2)\noutput = tf.keras.layers.Activation('swish')(dropout_2)\n\noutput_layer = Dense(1, activation='sigmoid', name='output')(output)\n\nmodel = Model(inputs=input_layer, outputs=output_layer)\n\nmodel.compile(optimizer=Adam(learning_rate=1e-3), loss='binary_crossentropy', metrics=['AUC'])\n\n\nES = EarlyStopping(\n    monitor='val_loss',  \n    patience=3,         \n    restore_best_weights=True \n)\n\nhistory = model.fit(\n   X_train, y_train, \n    validation_data=(X_valid, y_valid),\n    epochs=15, \n    batch_size=64,\n    callbacks=[ES],\n    verbose=1\n)","metadata":{"_uuid":"91f1ba31-90f1-416b-bf04-52789c42fe1f","_cell_guid":"d2aacdce-1721-4d8a-b00f-e37c0ef4005d","collapsed":false,"execution":{"iopub.status.busy":"2024-02-09T08:50:13.886405Z","iopub.execute_input":"2024-02-09T08:50:13.88684Z","iopub.status.idle":"2024-02-09T08:52:56.207303Z","shell.execute_reply.started":"2024-02-09T08:50:13.886806Z","shell.execute_reply":"2024-02-09T08:52:56.205157Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nn_pred = model.predict(X_test).ravel()","metadata":{"_uuid":"4297fdc5-9ec7-4141-b0bb-973ea6f152da","_cell_guid":"914d6aa2-51e2-47b2-aabf-170b47438d51","collapsed":false,"execution":{"iopub.status.busy":"2024-02-09T08:48:57.962167Z","iopub.status.idle":"2024-02-09T08:48:57.963204Z","shell.execute_reply.started":"2024-02-09T08:48:57.962921Z","shell.execute_reply":"2024-02-09T08:48:57.962945Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"W1 = 0.95\nW = [W1,1-W1]\ny_pred = W[0]*lgb_pred + W[1]*nn_pred","metadata":{"_uuid":"b4c0251f-a4d2-4819-9723-89883203e6f1","_cell_guid":"403a49c0-fd7c-4ed8-bc4b-07f5c05efe71","collapsed":false,"execution":{"iopub.status.busy":"2024-02-09T08:48:57.965028Z","iopub.status.idle":"2024-02-09T08:48:57.965552Z","shell.execute_reply.started":"2024-02-09T08:48:57.965333Z","shell.execute_reply":"2024-02-09T08:48:57.965353Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Submission","metadata":{"_uuid":"61d0d329-44db-4da8-b5f5-c40869f6395b","_cell_guid":"91ee6e19-6b32-4cc1-808a-27b5855cfc98","trusted":true}},{"cell_type":"code","source":"df_subm = pd.read_csv(ROOT / \"sample_submission.csv\")\ndf_subm = df_subm.set_index(\"case_id\")\n\ndf_subm[\"score\"] = y_pred","metadata":{"_uuid":"ccab19a3-1c2d-473b-9977-475d22c236cb","_cell_guid":"e10d15f6-f3aa-4997-840a-92d4e952004b","collapsed":false,"execution":{"iopub.status.busy":"2024-02-09T08:48:57.967262Z","iopub.status.idle":"2024-02-09T08:48:57.967732Z","shell.execute_reply.started":"2024-02-09T08:48:57.96751Z","shell.execute_reply":"2024-02-09T08:48:57.967532Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Check null: \", df_subm[\"score\"].isnull().any())","metadata":{"_uuid":"52cfb604-ce05-493c-b6e0-0cb22fd79b0b","_cell_guid":"d260d05b-ff7f-42a4-81cc-ca1e1f4e0632","collapsed":false,"execution":{"iopub.status.busy":"2024-02-09T08:48:57.970231Z","iopub.status.idle":"2024-02-09T08:48:57.971225Z","shell.execute_reply.started":"2024-02-09T08:48:57.970943Z","shell.execute_reply":"2024-02-09T08:48:57.970967Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_subm.head()","metadata":{"_uuid":"311b9882-bd14-4dbf-96d6-32ecc96e8fa8","_cell_guid":"97bde275-325d-4d0d-81f9-4dba3e678fbc","collapsed":false,"execution":{"iopub.status.busy":"2024-02-09T08:48:57.973057Z","iopub.status.idle":"2024-02-09T08:48:57.973588Z","shell.execute_reply.started":"2024-02-09T08:48:57.973375Z","shell.execute_reply":"2024-02-09T08:48:57.973395Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_subm.to_csv(\"submission.csv\")","metadata":{"_uuid":"2900bdf9-a337-4bbd-8216-0cab5d589d7e","_cell_guid":"4a29ae2b-38f0-4b8f-aef0-6808b1a91366","collapsed":false,"execution":{"iopub.status.busy":"2024-02-09T08:48:57.976394Z","iopub.status.idle":"2024-02-09T08:48:57.977115Z","shell.execute_reply.started":"2024-02-09T08:48:57.976751Z","shell.execute_reply":"2024-02-09T08:48:57.976778Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]}]}