{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.14"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"},{"sourceId":9997028,"sourceType":"datasetVersion","datasetId":6106113}],"dockerImageVersionId":30761,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false},"papermill":{"default_parameters":{},"duration":6693.17838,"end_time":"2024-12-10T22:09:36.779049","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-12-10T20:18:03.600669","version":"2.6.0"},"colab":{"provenance":[{"file_id":"1dR7OQn9lN5beldkirnwLdkaYoG4LUNlx","timestamp":1732282140660}]}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"3a22f8b2","cell_type":"markdown","source":"","metadata":{"papermill":{"duration":0.012607,"end_time":"2024-12-10T20:18:06.399677","exception":false,"start_time":"2024-12-10T20:18:06.387070","status":"completed"},"tags":[]},"attachments":{}},{"id":"2a1a3289","cell_type":"markdown","source":"# **FOREWORD**\n\nThis kernel is used to test notebook blends for kernels in experiment 9 and experiment 7. This is my candidate 1 for the final submission <br>\n\n### **EXPERIMENTS**\n\n|Experiment Number| CV score| LB score|\n|-----------------| :-:     | :-:     |\n|MLV9-4           |0.463429 | 0.471   |\n|MLV9-5           |0.466009 | 0.470   |\n|MLV9-7           |0.466455 | 0.466   |\n|MLV7-2           |0.468296 | 0.466   |\n|MLV7-24          |0.463159 | 0.470   |\n\n\n**Overall CV = 0.469299**\n\n\n","metadata":{"papermill":{"duration":0.011079,"end_time":"2024-12-10T20:18:06.422173","exception":false,"start_time":"2024-12-10T20:18:06.411094","status":"completed"},"tags":[]}},{"id":"0228b516","cell_type":"markdown","source":"# **COMMON SCRIPTS**","metadata":{"papermill":{"duration":0.011225,"end_time":"2024-12-10T20:18:06.444745","exception":false,"start_time":"2024-12-10T20:18:06.433520","status":"completed"},"tags":[]}},{"id":"1113aeec","cell_type":"code","source":"%%time \n\nimport os, joblib, random, torch, warnings, optuna\nfrom tqdm import tqdm\nfrom pathlib import Path\nfrom concurrent.futures import ThreadPoolExecutor\nfrom typing import Dict\nfrom colorama import Fore, Style, Back\nfrom pprint import pprint\nfrom collections.abc import Callable\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom gc import collect\n\nwarnings.filterwarnings('ignore')\n\nimport numpy as np\nimport pandas as pd\n\nfrom sklearn.metrics import (make_scorer, \ncohen_kappa_score, mean_squared_error, confusion_matrix, ConfusionMatrixDisplay)\n\nfrom sklearn.compose import ColumnTransformer, make_column_selector\nfrom sklearn.model_selection import StratifiedKFold, cross_val_score, cross_val_predict, PredefinedSplit as PDS\nfrom sklearn.preprocessing import OrdinalEncoder\nfrom sklearn.ensemble import VotingRegressor\nfrom sklearn.impute import SimpleImputer, KNNImputer\nfrom category_encoders import TargetEncoder\nfrom sklearn.pipeline import Pipeline\nfrom scipy.optimize import minimize\nfrom sklearn.inspection import permutation_importance\nfrom sklearn.base import clone\n\nimport lightgbm as lgb\nfrom lightgbm import LGBMRegressor as LGBMR\nfrom catboost import CatBoostRegressor as CBR\nfrom xgboost import XGBRegressor as XGBR\n\n\nSEED  = 42\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nseed_everything(42)\n\n# Color printing\ndef PrintColor(text: str, color = Fore.BLUE, style = Style.BRIGHT):\n    \"Prints color outputs using colorama using a text F-string\"\n    print(style + color + text + Style.RESET_ALL)\n    \nprint(f\"\\n---> Imports done\")\n\nKAPPA_SCORER = \\\nmake_scorer(\n    cohen_kappa_score,\n    greater_is_better=True,\n    weights ='quadratic',\n)\n\nmyscorer = \\\nmake_scorer(\n    mean_squared_error,\n    greater_is_better= False,\n    squared =False,\n)\n\ndef ScoreMetric(ytrue, ypreds):\n    \"Scoring metric for the competition\"\n    \n    score = \\\n    cohen_kappa_score(\n        np.uint8(np.round(ytrue)),\n        np.uint8(np.round(ypreds)),\n        weights = \"quadratic\"\n    )\n    return score\n\n\nclass MyLGBMR(lgb.LGBMRegressor):\n    '''\n    Custom LightGBM Regressor\n\n    It optimizes threshold values during fitting.\n    Main goal is preventing overfit on validation data.\n    '''\n\n    @staticmethod\n    def ScoreMetric(ytrue, ypreds)-> float:\n        \"Scoring metric for the competition\"\n\n        return cohen_kappa_score(\n            np.uint8(np.round(ytrue,0)),\n            np.uint8(np.round(ypreds,0)),\n            weights = \"quadratic\"\n        )\n\n    def _threshold_rounder(self, y_pred, thresholds):\n        output = np.zeros_like(y_pred, dtype=int)\n\n        for i in range(len(thresholds)):\n            output += (y_pred >= thresholds[i]).astype(int)\n        return output\n\n    def _eval_preds(self, thresholds, y_true, y_pred):\n        y_pred = self._threshold_rounder(y_pred, thresholds)\n        score  = self.ScoreMetric(y_true, y_pred)\n        return -score\n\n    def fit(\n        self, X, y, verbose = False, **kwargs\n    ):\n        super().fit(X, y, **kwargs)\n        y_pred           = super().predict(X, **kwargs)\n        self.base_score_ = self.ScoreMetric(y, y_pred)\n\n        self.n_classes_ = len(np.unique(y))\n        init_th = np.linspace(0, self.n_classes_ - 1, self.n_classes_) + 0.5\n        init_th = init_th[0: -1]\n\n        if verbose :\n            pprint(init_th)\n\n        self.optimizer = \\\n        minimize(\n            self._eval_preds,\n            x0     = init_th,\n            args   = (y, y_pred),\n            method = 'Nelder-Mead',\n        )\n\n        self.optimized_score_ = -self._eval_preds(self.optimizer.x, y, y_pred)\n        return self\n\n    def predict(\n        self,\n        X : np.ndarray | pd.DataFrame,\n        base_preds: bool = False,\n        **kwargs\n    ):\n        y_pred = super().predict(X, **kwargs)\n\n        if base_preds:\n            return y_pred\n        else:\n            return self._threshold_rounder(y_pred, self.optimizer.x)\n\n    def get_optimization_info(self)-> Dict:\n        \"Provides the scores before and after the optimization and thresholds\"\n\n        return {\n            'base_score'     : self.base_score_,\n            'optimized_score': self.optimized_score_,\n            'thresholds'     : self.optimizer.x\n        }\n\nclass MyCBR(CBR):\n    '''\n    Custom Catboost Regressor\n\n    It optimizes threshold values during fitting.\n    Main goal is preventing overfit on validation data.\n    '''\n\n    @staticmethod\n    def ScoreMetric(ytrue, ypreds)-> float:\n        \"Scoring metric for the competition\"\n\n        return cohen_kappa_score(\n            np.uint8(np.round(ytrue,0)),\n            np.uint8(np.round(ypreds,0)),\n            weights = \"quadratic\"\n        )\n\n    def _threshold_rounder(self, y_pred, thresholds):\n        output = np.zeros_like(y_pred, dtype=int)\n\n        for i in range(len(thresholds)):\n            output += (y_pred >= thresholds[i]).astype(int)\n        return output\n\n    def _eval_preds(self, thresholds, y_true, y_pred):\n        y_pred = self._threshold_rounder(y_pred, thresholds)\n        score  = self.ScoreMetric(y_true, y_pred)\n        return -score\n\n    def fit(\n        self, X, y, verbose = False, **kwargs\n    ):\n        super().fit(X, y, **kwargs)\n        y_pred           = super().predict(X, **kwargs)\n        self.base_score_ = self.ScoreMetric(y, y_pred)\n\n        self.n_classes_ = len(np.unique(y))\n        init_th = np.linspace(0, self.n_classes_ - 1, self.n_classes_) + 0.5\n        init_th = init_th[0: -1]\n\n        if verbose :\n            pprint(init_th)\n\n        self.optimizer = \\\n        minimize(\n            self._eval_preds,\n            x0     = init_th,\n            args   = (y, y_pred),\n            method = 'Nelder-Mead',\n        )\n\n        self.optimized_score_ = -self._eval_preds(self.optimizer.x, y, y_pred)\n        return self\n\n    def predict(\n        self,\n        X : np.ndarray | pd.DataFrame,\n        base_preds: bool = False,\n        **kwargs\n    ):\n        y_pred = super().predict(X, **kwargs)\n\n        if base_preds:\n            return y_pred\n        else:\n            return self._threshold_rounder(y_pred, self.optimizer.x)\n\n    def get_optimization_info(self)-> Dict:\n        \"Provides the scores before and after the optimization and thresholds\"\n\n        return {\n            'base_score'     : self.base_score_,\n            'optimized_score': self.optimized_score_,\n            'thresholds'     : self.optimizer.x\n        }\n\nclass MyXGBR(XGBR):\n    '''\n    Custom XGBRegressor\n\n    It optimizes threshold values during fitting.\n    Main goal is preventing overfit on validation data.\n    '''\n\n    @staticmethod\n    def ScoreMetric(ytrue, ypreds)-> float:\n        \"Scoring metric for the competition\"\n\n        return cohen_kappa_score(\n            np.uint8(np.round(ytrue,0)),\n            np.uint8(np.round(ypreds,0)),\n            weights = \"quadratic\"\n        )\n\n    def _threshold_rounder(self, y_pred, thresholds):\n        output = np.zeros_like(y_pred, dtype=int)\n\n        for i in range(len(thresholds)):\n            output += (y_pred >= thresholds[i]).astype(int)\n        return output\n\n    def _eval_preds(self, thresholds, y_true, y_pred):\n        y_pred = self._threshold_rounder(y_pred, thresholds)\n        score  = self.ScoreMetric(y_true, y_pred)\n        return -score\n\n    def fit(\n        self, X, y, verbose = False, **kwargs\n    ):\n        super().fit(X, y, **kwargs)\n        y_pred           = super().predict(X, **kwargs)\n        self.base_score_ = self.ScoreMetric(y, y_pred)\n\n        self.n_classes_ = len(np.unique(y))\n        init_th = np.linspace(0, self.n_classes_ - 1, self.n_classes_) + 0.5\n        init_th = init_th[0: -1]\n\n        if verbose :\n            pprint(init_th)\n\n        self.optimizer = \\\n        minimize(\n            self._eval_preds,\n            x0     = init_th,\n            args   = (y, y_pred),\n            method = 'Nelder-Mead',\n        )\n\n        self.optimized_score_ = -self._eval_preds(self.optimizer.x, y, y_pred)\n        return self\n\n    def predict(\n        self,\n        X : np.ndarray | pd.DataFrame,\n        base_preds: bool = False,\n        **kwargs\n    ):\n        y_pred = super().predict(X, **kwargs)\n\n        if base_preds:\n            return y_pred\n        else:\n            return self._threshold_rounder(y_pred, self.optimizer.x)\n\n    def get_optimization_info(self)-> Dict:\n        \"Provides the scores before and after the optimization and thresholds\"\n\n        return {\n            'base_score'     : self.base_score_,\n            'optimized_score': self.optimized_score_,\n            'thresholds'     : self.optimizer.x\n        }\n\nclass ModelTrainer:\n    \"This class trains the provided model on the train-test data and returns the predictions and fitted models\"\n\n    def __init__(\n        self,\n        drop_cols : list,\n        ntop      : int = 50,\n        verbose   : bool = True,\n        test_preds_req : bool = True,\n    ):\n        self.drop_cols = drop_cols\n        self.ntop      = ntop\n        self.verbose   = verbose\n        self.test_preds_req = test_preds_req\n\n    def MakeOfflineModel(\n        self,\n        X, y, ygrp, Xtest,\n        model            : Callable,\n        method           : str,\n        ftreimp_plot_req : bool = True,\n        **kwargs,\n    ):\n        \"\"\"\n        This function trains the provided model on the dataset and cross-validates appropriately\n\n        Inputs-\n        X, y, ygrp       - training data components (Xtrain, ytrain, fold_nb)\n        Xtest            - test data (optional)\n        model            - model object for training\n        method           - model method label\n        ftreimp_plot_req - boolean flag to plot tree feature importances\n\n        Returns-\n        base_oof, adj_oof, mdl_preds - prediction arrays\n        fitted_models                - fitted model list for test set\n        ftreimp                      - feature importances across selected features\n        thresholds                   - all fold level threshold arrays\n        \"\"\"\n\n        PrintColor(f\"\\n ===== {method.upper()} =====  \\n\")\n\n        mdl_preds     = np.zeros(len(Xtest))\n        base_oof      = np.zeros(len(X))\n        adj_oof       = np.zeros(len(X))\n        thresholds    = {}\n        fitted_models = []\n\n        scores, bscores, ftreimp = 0,0,0\n\n        cv = PDS(ygrp[0 : len(X)])\n        n_splits = ygrp.nunique()\n\n        for fold_nb, (train_idx, dev_idx) in enumerate(cv.split(X, y)):\n\n            PrintColor(f\"------- FOLD {fold_nb} ------- \", color = Fore.BLACK)\n            \n            Xtr  = X.iloc[train_idx]\n            ytr  = y.iloc[train_idx]\n            Xdev = X.iloc[dev_idx]\n            ydev = y.iloc[dev_idx]\n\n            model.fit(Xtr, ytr)\n            fitted_models.append(model)\n\n            thresholds[f\"Fold{fold_nb}\"] = model[\"M\"].optimizer.x\n\n            try:\n                ftreimp += model[\"M\"].feature_importances_\n            except:\n                pass\n\n            base_preds = model.predict(Xdev, **{\"base_preds\" : True})\n            opt_preds  = model.predict(Xdev, **{\"base_preds\" : False})\n            base_score = model[\"M\"].ScoreMetric(ydev, base_preds)\n            opt_score  = model[\"M\"].ScoreMetric(ydev, opt_preds)\n            scores    += opt_score / n_splits\n            bscores   += base_score/ n_splits\n\n            base_oof[dev_idx] = base_preds\n            adj_oof[dev_idx]  = opt_preds\n\n            if self.verbose:\n                PrintColor(\n                    f\"---> OOF | Base = {base_score :.6f} Optimized = {opt_score :.6f}\",\n                    color = Fore.RED\n                )\n                \n                PrintColor(\n                    f\"\\n---> Optimization information\", \n                    color = Fore.MAGENTA,\n                )\n                pprint(model[\"M\"].get_optimization_info())\n                print()\n\n            if self.test_preds_req :\n                try:\n                    mdl_preds = \\\n                    mdl_preds + \\\n                    model.predict(\n                        Xtest.drop(self.drop_cols, axis=1, errors = \"ignore\"),\n                        base_preds = False\n                    )\n\n                except:\n                    mdl_preds = mdl_preds + model.predict(Xtest, base_preds = False)\n            print()\n\n        PrintColor(f\"---> Overall scores - \", color = Fore.RED)\n        PrintColor(f\"---> Optimized OOF = {scores:.6f} | Base OOF = {bscores  :.6f}\")\n\n        mdl_preds = mdl_preds / n_splits\n        ftreimp   = pd.Series(ftreimp, index = Xdev.columns)\n\n        if ftreimp_plot_req :\n            self.PlotFtreImp(ftreimp, method,)\n        else:\n            pass\n\n        return (base_oof, adj_oof, mdl_preds, fitted_models, ftreimp, thresholds,)\n\n    def PlotFtreImp(\n            self,\n            ftreimp     : pd.Series,\n            method      : str,\n            title_specs : dict = {'fontsize': 12, 'fontweight' : 'bold','color': '#992600'},\n            **params,\n        ):\n            \"This method plots the feature importances for the model provided\"\n\n            print()\n\n            with sns.axes_style(\"white\"):\n                fig, ax = plt.subplots(1, 1, figsize = (25, 7.5))\n\n                ftreimp.sort_values(ascending = False).\\\n                head(self.ntop).\\\n                plot.bar(ax = ax, color = \"tab:blue\")\n                ax.set_title(f\"Feature Importances - {method}\", **title_specs)\n\n                plt.tight_layout()\n                plt.show()\n\n            print()\n","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"jupyter":{"source_hidden":true},"papermill":{"duration":8.247656,"end_time":"2024-12-10T20:18:14.704839","exception":false,"start_time":"2024-12-10T20:18:06.457183","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"8eeeba9a","cell_type":"code","source":"%%time \n\ndef process_file(filename, dirname):\n    df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    df.drop('step', axis=1, inplace=True)\n    \n    return df.describe().values.reshape(-1), filename.split('=')[1]\n\ndef load_time_series(dirname):\n    ids = os.listdir(dirname)\n    \n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n    \n    stats, indexes = zip(*results)\n    df = pd.DataFrame(stats, columns=[f\"stat_{i}\" for i in range(len(stats[0]))])\n    df['id'] = indexes\n    \n    return df\n\ndef make_sec_ftre(\n    X : pd.DataFrame, \n    label : str = \"Train\",\n)-> pd.DataFrame:\n    \"This function uses the csv customer/ participant file and extracts features\"\n\n    PrintColor(\n        f\"\\n{'-' * 10} {label.upper()} PREPROCESSING {'-' * 10}\\n\", \n        color = Fore.RED\n    )\n    \n    df = X.copy()\n    \n    PrintColor(f\"---> Physical section features = {df.shape}\", color = Fore.CYAN)\n    df.loc[df[\"Physical-Weight\"] < 10, \"Physical-Weight\"] = np.nan\n    df.loc[df[\"Physical-Height\"] < 5,  \"Physical-Height\"] = np.nan\n    \n    for col in ['Physical-BMI', 'BIA-BIA_BMI']:\n        df.loc[df[col] <= 5, col] = np.nan\n        \n    for col in [\"Physical-Systolic_BP\", \"Physical-Diastolic_BP\"]:\n        df.loc[df[col] <= 0, col] = np.nan\n        \n    df[\"Imp_BMI\"]    = df['BIA-BIA_BMI'].fillna(df['Physical-BMI'])\n    df[\"Change_BMI\"] = df['BIA-BIA_BMI'] - df['Physical-BMI']\n    \n    df[\"Grp_PBMI\"]   = \\\n    np.select(\n        [df['Physical-BMI'].isna() == True, \n         df['Physical-BMI'] < 18.5, \n         df['Physical-BMI'] < 25, \n         df['Physical-BMI'] < 30, \n         df['Physical-BMI'] < 35, \n         df['Physical-BMI'] < 40\n        ],\n        [\"NA\", \"Underweight\", \"Normal\", \"Overweight\", \"Obese1\", \"Obese2\"],\n        \"Obese3\" \n    )\n    df[\"Grp_PBMI\"] = df[\"Grp_PBMI\"]\n    \n    df[\"Grp_BIABMI\"]   = \\\n    np.select(\n        [df['BIA-BIA_BMI'].isna() == True, \n         df['BIA-BIA_BMI'] < 18.5, \n         df['BIA-BIA_BMI'] < 25, \n         df['BIA-BIA_BMI'] < 30, \n         df['BIA-BIA_BMI'] < 35, \n         df['BIA-BIA_BMI'] < 40\n        ],\n        [\"NA\", \"Underweight\", \"Normal\", \"Overweight\", \"Obese1\", \"Obese2\"],\n        \"Obese3\"  \n    )\n    df[\"Grp_BIABMI\"] = df[\"Grp_BIABMI\"]\n    \n    df[\"Grp_Imp_BMI\"]   = \\\n    np.select(\n        [df['Imp_BMI'].isna() == True,\n         df['Imp_BMI'] < 18.5, \n         df['Imp_BMI'] < 25, \n         df['Imp_BMI'] < 30, \n         df['Imp_BMI'] < 35, \n         df['Imp_BMI'] < 40\n        ],\n        [\"NA\", \"Underweight\", \"Normal\", \"Overweight\", \"Obese1\", \"Obese2\"],\n        \"Obese3\" \n    )\n    df[\"Grp_Imp_BMI\"] = df[\"Grp_Imp_BMI\"]\n\n    df[\"Ratio_WaistHeight\"] = (df[\"Physical-Waist_Circumference\"] / df[\"Physical-Height\"]).replace([np.inf, -1*np.inf], np.nan)\n    df[\"Diff_BP\"]           = df[\"Physical-Systolic_BP\"] - df[\"Physical-Diastolic_BP\"]\n    \n    df.loc[df[\"Diff_BP\"] < 0, [\"Physical-Systolic_BP\", \"Physical-Diastolic_BP\"]] = \\\n    df.loc[df[\"Diff_BP\"] < 0, [\"Physical-Diastolic_BP\", \"Physical-Systolic_BP\"]].values\n    \n    df[\"Diff_BP\"]  = df[\"Physical-Systolic_BP\"] - df[\"Physical-Diastolic_BP\"]\n    df[\"Grp_BP\"] = \\\n    np.select(\n        [(df[\"Physical-Systolic_BP\"].isna() == True) | (df[\"Physical-Diastolic_BP\"].isna() == True),\n         (df[\"Physical-Systolic_BP\"] < 90) | (df[\"Physical-Diastolic_BP\"] < 60) ,\n         (df[\"Physical-Systolic_BP\"].between(90, 119, inclusive = \"both\")) & (df[\"Physical-Diastolic_BP\"].between(60, 79, inclusive = \"both\")),\n         (df[\"Physical-Systolic_BP\"].between(120,129, inclusive = \"both\")) & (df[\"Physical-Diastolic_BP\"] < 80),\n         (df[\"Physical-Systolic_BP\"].between(130,139, inclusive = \"both\")) | (df[\"Physical-Diastolic_BP\"].between(80, 89, inclusive = \"both\")), \n         (df[\"Physical-Systolic_BP\"] >= 140) | (df[\"Physical-Diastolic_BP\"] >= 90),\n        ],\n        [\"NA\", \"Low\", \"Normal\", \"Elevated\", \"High1\", \"High2\"],\n        \"High3\",\n    )\n    df[\"Grp_BP\"] = df[\"Grp_BP\"].astype(\"category\")\n\n    df[\"Grp_HeartRate\"] = \\\n        np.select(\n            [df[\"Physical-HeartRate\"].isna() == True,\n             df[\"Physical-HeartRate\"] < 60,\n             df[\"Physical-HeartRate\"].between(60, 100, inclusive = \"left\"),\n            ], \n            [\"NA\", \"Bradycardia\", \"Normal\"], \"Tachycardia\"\n        )\n    df[\"Grp_HeartRate\"] = df[\"Grp_HeartRate\"].astype(\"category\")\n            \n    for col in df.filter(regex = \"Grp_\", axis = 1).columns:\n        if df[col].dtype == \"category\":\n            pass\n        else:\n            try:\n                df[col] = df[col].astype(\"category\")\n            except:\n                pass\n\n    df = df.replace([np.inf, -1*np.inf], np.nan)\n    PrintColor(f\"---> All features complete = {df.shape} -- {label}\\n\", color = Fore.RED)\n    return df\n","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"jupyter":{"source_hidden":true},"papermill":{"duration":0.035712,"end_time":"2024-12-10T20:18:14.752320","exception":false,"start_time":"2024-12-10T20:18:14.716608","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"1da98675","cell_type":"code","source":"%%time \n\ntarget_col  = \"sii\"\nmethod      = \"LGBM1R\"\nn_refits    = 100\n\n# Data storage for all OOF predictions:-\nAll_OOF_Preds = {}\nAll_Mdl_Preds = {}","metadata":{"papermill":{"duration":0.021566,"end_time":"2024-12-10T20:18:14.785917","exception":false,"start_time":"2024-12-10T20:18:14.764351","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"56ed7284","cell_type":"code","source":"%%time \n\n# Loading the time series dataset\nroot     = Path('/kaggle/input/child-mind-institute-problematic-internet-use')\nts_train = load_time_series(root / \"series_train.parquet\")\nts_test  = load_time_series(root / \"series_test.parquet\")\n\nPrintColor(f\"\\n\\n---> Shape = {ts_train.shape} {ts_test.shape}\")\n\n# Loading the submission file:-\ndf_subm  = \\\npd.read_csv(\n    '/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv', \n    index_col='id'\n)","metadata":{"papermill":{"duration":88.384528,"end_time":"2024-12-10T20:19:43.182292","exception":false,"start_time":"2024-12-10T20:18:14.797764","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"829faec7-9f11-42d1-a299-cbcab7497f95","cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"adabe8ac","cell_type":"markdown","source":"# **MODEL1 EXPERIMENT V9_4**","metadata":{"papermill":{"duration":0.029847,"end_time":"2024-12-10T20:19:43.243310","exception":false,"start_time":"2024-12-10T20:19:43.213463","status":"completed"},"tags":[]}},{"id":"dda7e71c","cell_type":"code","source":"%%time \n\nn_neighbors = 2\nversion_nb  = \"V9_4\"","metadata":{"papermill":{"duration":0.03912,"end_time":"2024-12-10T20:19:43.313307","exception":false,"start_time":"2024-12-10T20:19:43.274187","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"bd945b3b","cell_type":"markdown","source":"## **DATA LOADS**","metadata":{"papermill":{"duration":0.030061,"end_time":"2024-12-10T20:19:43.373468","exception":false,"start_time":"2024-12-10T20:19:43.343407","status":"completed"},"tags":[]}},{"id":"857751b5","cell_type":"code","source":"%%time \n\n# Preparing the dataset for the inference\ndf_train = pd.read_csv(root / 'train.csv')\ndf_test  = pd.read_csv(root / 'test.csv')\n\n# Making new features:-\ndf_train = make_sec_ftre(df_train, \"Train\", )\nprint()\ndf_test  = make_sec_ftre(df_test,  \"Test\", )\n\ntime_series_cols = ts_train.columns.tolist()\ntime_series_cols.remove(\"id\")\n\ndf_train = pd.merge(df_train, ts_train, how=\"left\", on='id')\ndf_test  = pd.merge(df_test, ts_test,   how=\"left\", on='id')\ndf_train = df_train.set_index('id')\ndf_test  = df_test.set_index('id')\n\ntrain_   = df_train.loc[df_train[\"sii\"].isna()]\ndf_train = df_train.dropna(subset= [\"sii\"])\n\nfeature_cols = list(df_test.drop(\"id\", axis = 1, errors = \"ignore\").columns)\ncat_cols     = [c for c in feature_cols if \"Season\" in c]\nnum_cols     = list(df_test.select_dtypes(np.number).columns)\nnum_cols     = sorted(list(set(num_cols).intersection(set(feature_cols))))\n\nPrintColor(f\"\\n\\n---> Shape = {df_train.shape} {df_test.shape}\")\n\n# Preparing the targets\ntargets_df = \\\n(df_train.\n filter(regex = r\"PCIAT|sii\", axis=1).\n iloc[:, 1:].\n dropna(subset = \"sii\", axis=0).\n fillna(0).\n astype(np.uint8)\n)\n\nPrintColor(f\"\\n---> Targets \\n\")\ntargets = np.array(targets_df.columns)[0: 20]\nwith np.printoptions(linewidth = 150):\n    print(np.array(targets))\n\nprint()\n","metadata":{"papermill":{"duration":0.220682,"end_time":"2024-12-10T20:19:43.623864","exception":false,"start_time":"2024-12-10T20:19:43.403182","status":"completed"},"tags":[],"trusted":true,"_kg_hide-input":true},"outputs":[],"execution_count":null},{"id":"e7f739ce","cell_type":"markdown","source":"## **FEATURE ENGINEERING**","metadata":{"papermill":{"duration":0.029661,"end_time":"2024-12-10T20:19:43.683823","exception":false,"start_time":"2024-12-10T20:19:43.654162","status":"completed"},"tags":[]}},{"id":"d6dd1f94","cell_type":"code","source":"%%time \n\nartefacts = {}\n\nX     = df_train[feature_cols]\ny     = df_train[target_col]\nXtest = df_test[feature_cols]\ncv    = StratifiedKFold(5, shuffle=True, random_state=SEED)\n\nygrp  = np.zeros(len(X))\nfor fold_nb, (_, dev_idx) in enumerate(cv.split(X, y)):\n    ygrp[dev_idx] = fold_nb\n\nygrp = pd.Series(ygrp, name = \"fold_nb\", dtype = np.uint8)\n\nxform = SimpleImputer(strategy = \"mean\")\nlen_train = len(X)\n\nx               = xform.fit_transform(pd.concat([X[num_cols], train_[num_cols]]))[0 : len_train]\nX[num_cols]     = x\nXtest[num_cols] = xform.transform(Xtest[num_cols])\ndel x\n\nartefacts[\"Imputer\"] = xform\n\nxform = \\\nOrdinalEncoder(dtype=np.int32,\n               handle_unknown='use_encoded_value',\n               unknown_value=-1,\n               encoded_missing_value=-2,\n              )\n\nx = xform.fit_transform(pd.concat([X[cat_cols], train_[cat_cols]]))[0 : len_train]\n\nX[cat_cols]     = x\nXtest[cat_cols] = xform.transform(Xtest[cat_cols])\n\nartefacts[\"Encoder\"] = xform\njoblib.dump(artefacts, f\"Artefacts_{version_nb}.joblib\")\n\nPrintColor(f\"\\n\\n---> Shape = {X.shape} {Xtest.shape}\")\n","metadata":{"papermill":{"duration":0.146226,"end_time":"2024-12-10T20:19:43.860042","exception":false,"start_time":"2024-12-10T20:19:43.713816","status":"completed"},"tags":[],"trusted":true,"_kg_hide-input":true},"outputs":[],"execution_count":null},{"id":"be25eed0","cell_type":"markdown","source":"## **MODEL TRAINING**","metadata":{"papermill":{"duration":0.030052,"end_time":"2024-12-10T20:19:43.920512","exception":false,"start_time":"2024-12-10T20:19:43.890460","status":"completed"},"tags":[]}},{"id":"41d6ad4f","cell_type":"code","source":"%%time \n\n# Initializing storage elements\nOOF_Preds     = {}\nAdj_OOF_Preds = {}\nFittedModels  = {}\nFtreImp       = {}\n\nOnlineModels  = {}\nOnl_Preds     = {}\nOnl_Mdl_Preds = {}\n\nlgb_params = \\\n{\n'objective'       : 'l2',\n'verbosity'       : -1,\n'n_iter'          : 200,\n'lambda_l1'       : 0.005116829730239727,\n'lambda_l2'       : 0.0011520776712645852,\n'learning_rate'   : 0.02376367323636638,\n'max_depth'       : 5,\n'num_leaves'      : 207,\n'colsample_bytree': 0.7759862336963801,\n'colsample_bynode': 0.5110355095943208,\n'bagging_fraction': 0.5485770314992224,\n'bagging_freq'    : 7,\n'min_data_in_leaf': 78,\n}\n\ndrop_cols = [\"Source\", \"id\", \"Id\", \"Label\", target_col, \"fold_nb\"]\nmd        = ModelTrainer(drop_cols = drop_cols, ntop  = 50, verbose = True)\nX         = X.drop(drop_cols, axis=1, errors = \"ignore\")\nXtest     = Xtest.drop(drop_cols, axis=1, errors = \"ignore\")\ncat_mdl_cols = list(Xtest.select_dtypes(\"category\").columns)\n\nprint(f\"\\n{'=' * 20} {method.upper()} MODEL TRAINING {'=' * 20}\\n\")\n\nfor mytarget in tqdm(targets_df.columns[0: 20]):\n    PrintColor(\n        f\"\\n ======== CURRENT TARGET = {mytarget} ======== \\n\"\n    )\n\n    model = Pipeline(steps = [(\"M\", MyLGBMR(random_state = 42, **lgb_params))])\n\n    PrintColor(\n        f\"\\n ======== OFFLINE MODELS - CV ======== \\n\", color = Fore.CYAN\n    )\n    (base_oof, adj_oof, mdl_preds, fitted_models, ftreimp, thresholds) = \\\n    md.MakeOfflineModel(\n        X,\n        targets_df[mytarget],\n        ygrp,\n        Xtest,\n        model  = model,\n        method = method,\n        ftreimp_plot_req = False,\n    )\n\n    FittedModels[mytarget] = fitted_models\n    FtreImp[mytarget]      = pd.Series(ftreimp, index = X.columns)\n    OOF_Preds[mytarget]    = base_oof\n    Adj_OOF_Preds[mytarget]= adj_oof   \n\n    PrintColor(\n        f\"\\n ======== FULL REFIT MODELS ======== \\n\", color = Fore.CYAN\n    )\n\n    reg_ = \\\n    VotingRegressor(\n        [(f\"{method}_{mystate}\" , MyLGBMR(random_state = mystate, **lgb_params))\n         for mystate in tqdm(list(range(0, n_refits, 1))) \n        ]\n    )\n    \n    onl_model = Pipeline(steps = [(\"M\", reg_)])\n    onl_model.fit(\n        X.drop(drop_cols, axis=1, errors = \"ignore\"), \n        targets_df[mytarget]\n    )\n\n    OnlineModels[mytarget]   = onl_model\n    Onl_Preds[mytarget]      = onl_model.predict(X)\n    Onl_Mdl_Preds[mytarget]  = onl_model.predict(Xtest)\n","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"papermill":{"duration":1410.184048,"end_time":"2024-12-10T20:43:14.136946","exception":false,"start_time":"2024-12-10T20:19:43.952898","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"0c95f0ec","cell_type":"markdown","source":"## **SUBMISSION**","metadata":{"papermill":{"duration":0.044159,"end_time":"2024-12-10T20:43:14.226028","exception":false,"start_time":"2024-12-10T20:43:14.181869","status":"completed"},"tags":[]}},{"id":"317c5431","cell_type":"code","source":"%%time \n\n# Data storage and final CV score\npciat_total = pd.DataFrame(Adj_OOF_Preds).sum(axis=1)\nsii_proxy = \\\nnp.select(\n    [pciat_total <= 30, \n     pciat_total  < 50, \n     pciat_total  < 80\n    ],\n    [0,1,2],\n    3\n)\nscore = ScoreMetric(targets_df[target_col], sii_proxy)\nPrintColor(f\"\\n---> Final OOF score = {score :.6f} \\n\")\n\nPrintColor(f\"---> Prediction counts\")\nprint(np.bincount(sii_proxy))\n\nprint()\nfig, ax = plt.subplots(1,1, figsize = (4,4))\nConfusionMatrixDisplay.from_predictions(\n    targets_df[\"sii\"], \n    sii_proxy, \n    cmap = 'Blues',\n    colorbar= False,\n    text_kw = {\"fontweight\" : \"bold\", \"fontsize\" : 14},\n    ax = ax\n)\nax.set_title(\n    f\"Confusion matrix - {target_col}\\n\", \n    **{\"color\"      : \"maroon\", \n       \"fontweight\" : \"bold\", \n       \"fontsize\"   : 10,\n      }\n)\nplt.tight_layout()\nplt.show()\n\nprint();\n\n# Storing the OOF predictions in the master storage \nAll_OOF_Preds[\"model1\"] = sii_proxy\n\n# Storing the relevant files:-\n(pd.DataFrame(Adj_OOF_Preds).\n assign(sii = sii_proxy).\n to_csv(f\"Adj_OOF_Preds_LGBM1RML{version_nb}.csv\")\n)\n\n(pd.DataFrame(OOF_Preds).to_csv(f\"OOF_Preds_LGBM1RML{version_nb}.csv\"))\n\njoblib.dump(OnlineModels,  f\"OnlineModels_LGBM1RML{version_nb}.joblib\")\njoblib.dump(fitted_models, f\"FittedModels_LGBM1RML{version_nb}.joblib\")\n\n!ls\n\n# Submission\npciat_total = pd.DataFrame(Onl_Mdl_Preds).sum(axis=1)\nsii_proxy = \\\nnp.select(\n    [pciat_total <= 30, \n     pciat_total < 50, \n     pciat_total < 80\n    ],\n    [0,1,2],\n    3\n)\n\ndf_subm[\"model1\"]  = sii_proxy","metadata":{"papermill":{"duration":14.254469,"end_time":"2024-12-10T20:43:28.525191","exception":false,"start_time":"2024-12-10T20:43:14.270722","status":"completed"},"tags":[],"trusted":true,"_kg_hide-input":true},"outputs":[],"execution_count":null},{"id":"acec328e","cell_type":"markdown","source":"","metadata":{"papermill":{"duration":0.044057,"end_time":"2024-12-10T20:43:28.614291","exception":false,"start_time":"2024-12-10T20:43:28.570234","status":"completed"},"tags":[]}},{"id":"f1f62a28","cell_type":"markdown","source":"# **MODEL2 - EXPERIMENT MLV9_5**","metadata":{"papermill":{"duration":0.044122,"end_time":"2024-12-10T20:43:28.703296","exception":false,"start_time":"2024-12-10T20:43:28.659174","status":"completed"},"tags":[]}},{"id":"e54292e8","cell_type":"code","source":"%%time \n\nn_neighbors = 3\nversion_nb  = \"V9_5\"","metadata":{"papermill":{"duration":0.056827,"end_time":"2024-12-10T20:43:28.804550","exception":false,"start_time":"2024-12-10T20:43:28.747723","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"9482c9a4","cell_type":"markdown","source":"## **DATA LOAD**","metadata":{"papermill":{"duration":0.045225,"end_time":"2024-12-10T20:43:28.895654","exception":false,"start_time":"2024-12-10T20:43:28.850429","status":"completed"},"tags":[]}},{"id":"762ecaa3","cell_type":"code","source":"%%time \n\n# Preparing the dataset for the inference\ndf_train = pd.read_csv(root / 'train.csv')\ndf_test  = pd.read_csv(root / 'test.csv')\n\n# Making new features:-\ndf_train = make_sec_ftre(df_train, \"Train\", )\nprint()\ndf_test  = make_sec_ftre(df_test,  \"Test\", )\n\ntime_series_cols = ts_train.columns.tolist()\ntime_series_cols.remove(\"id\")\n\ndf_train = pd.merge(df_train, ts_train, how=\"left\", on='id')\ndf_test  = pd.merge(df_test, ts_test,   how=\"left\", on='id')\ndf_train = df_train.set_index('id')\ndf_test  = df_test.set_index('id')\n\ntrain_   = df_train.loc[df_train[\"sii\"].isna()]\ndf_train = df_train.dropna(subset= [\"sii\"])\n\nfeature_cols = list(df_test.drop(\"id\", axis = 1, errors = \"ignore\").columns)\ncat_cols     = [c for c in feature_cols if \"Season\" in c]\nnum_cols     = list(df_test.select_dtypes(np.number).columns)\nnum_cols     = sorted(list(set(num_cols).intersection(set(feature_cols))))\n\nPrintColor(f\"\\n\\n---> Shape = {df_train.shape} {df_test.shape}\")\n\n# Preparing the targets\ntargets_df = \\\n(df_train.\n filter(regex = r\"PCIAT|sii\", axis=1).\n iloc[:, 1:].\n dropna(subset = \"sii\", axis=0).\n fillna(0).\n astype(np.uint8)\n)\n\nPrintColor(f\"\\n---> Targets \\n\")\ntargets = np.array(targets_df.columns)[0: 20]\nwith np.printoptions(linewidth = 150):\n    print(np.array(targets))\n\nprint()\n","metadata":{"papermill":{"duration":0.230652,"end_time":"2024-12-10T20:43:29.171614","exception":false,"start_time":"2024-12-10T20:43:28.940962","status":"completed"},"tags":[],"trusted":true,"_kg_hide-input":true},"outputs":[],"execution_count":null},{"id":"c3ffbe1e","cell_type":"markdown","source":"## **FEATURE ENGINEERING**","metadata":{"papermill":{"duration":0.045248,"end_time":"2024-12-10T20:43:29.262640","exception":false,"start_time":"2024-12-10T20:43:29.217392","status":"completed"},"tags":[]}},{"id":"0a48a6ad","cell_type":"code","source":"%%time \n\nartefacts = {}\n\nX     = df_train[feature_cols]\ny     = df_train[target_col]\nXtest = df_test[feature_cols]\ncv    = StratifiedKFold(5, shuffle=True, random_state=SEED)\n\nygrp  = np.zeros(len(X))\nfor fold_nb, (_, dev_idx) in enumerate(cv.split(X, y)):\n    ygrp[dev_idx] = fold_nb\n\nygrp = pd.Series(ygrp, name = \"fold_nb\", dtype = np.uint8)\n\ncat_cols        = list(X.select_dtypes(exclude = np.number).columns)\nX[cat_cols]     = X[cat_cols].astype(\"category\")\nXtest[cat_cols] = Xtest[cat_cols].astype(\"category\")\n\nct = \\\nColumnTransformer(\n    [(\"Imp\", KNNImputer(n_neighbors = n_neighbors), num_cols ),\n     (\"Enc\", OrdinalEncoder(dtype=np.int32,\n                            handle_unknown='use_encoded_value',\n                            unknown_value=-1,\n                            encoded_missing_value=-2,\n                           ),\n      cat_cols,\n     )\n    ],\n    verbose_feature_names_out = False,\n    verbose = False,\n    remainder = \"passthrough\",\n)\n\nPrintColor(f\"\\n\\n---> Shape = {X.shape} {Xtest.shape}\")\n","metadata":{"papermill":{"duration":0.102424,"end_time":"2024-12-10T20:43:29.409957","exception":false,"start_time":"2024-12-10T20:43:29.307533","status":"completed"},"tags":[],"trusted":true,"_kg_hide-input":true},"outputs":[],"execution_count":null},{"id":"e746d05a","cell_type":"markdown","source":"## **MODEL TRAINING**","metadata":{"papermill":{"duration":0.044585,"end_time":"2024-12-10T20:43:29.499772","exception":false,"start_time":"2024-12-10T20:43:29.455187","status":"completed"},"tags":[]}},{"id":"d3e81dfb","cell_type":"code","source":"%%time \n\n# Initializing storage elements\nOOF_Preds     = {}\nAdj_OOF_Preds = {}\nFittedModels  = {}\nFtreImp       = {}\n\nOnlineModels  = {}\nOnl_Preds     = {}\nOnl_Mdl_Preds = {}\n\nlgb_params = \\\n{\n'objective'       : 'l2',\n'verbosity'       : -1,\n'n_iter'          : 200,\n'lambda_l1'       : 0.005116829730239727,\n'lambda_l2'       : 0.0011520776712645852,\n'learning_rate'   : 0.02376367323636638,\n'max_depth'       : 5,\n'num_leaves'      : 207,\n'colsample_bytree': 0.7759862336963801,\n'colsample_bynode': 0.5110355095943208,\n'bagging_fraction': 0.5485770314992224,\n'bagging_freq'    : 7,\n'min_data_in_leaf': 78,\n}\n\ndrop_cols    = [\"Source\", \"id\", \"Id\", \"Label\", target_col, \"fold_nb\"]\nmd           = ModelTrainer(drop_cols = drop_cols, ntop  = 50, verbose = True)\nX            = X.drop(drop_cols, axis=1, errors = \"ignore\")\nXtest        = Xtest.drop(drop_cols, axis=1, errors = \"ignore\")\ncat_mdl_cols = list(Xtest.select_dtypes(\"category\").columns)\n\nprint(f\"\\n{'=' * 20} {method.upper()} MODEL TRAINING {'=' * 20}\\n\")\n\nfor mytarget in tqdm(targets_df.columns[0: 20]):\n    PrintColor(\n        f\"\\n ======== CURRENT TARGET = {mytarget} ======== \\n\"\n    )\n\n    model = \\\n    Pipeline(\n        steps = \\\n        [(\"CT\", ct),\n         (\"M\", MyLGBMR(random_state = 42, **lgb_params))\n        ]\n    )\n\n    PrintColor(\n        f\"\\n ======== OFFLINE MODELS - CV ======== \\n\", color = Fore.CYAN\n    )\n    (base_oof, adj_oof, mdl_preds, fitted_models, ftreimp, thresholds) = \\\n    md.MakeOfflineModel(\n        X,\n        targets_df[mytarget],\n        ygrp,\n        Xtest,\n        model  = model,\n        method = method,\n        ftreimp_plot_req = False,\n    )\n\n    FittedModels[mytarget] = fitted_models\n    FtreImp[mytarget]      = pd.Series(ftreimp, index = X.columns)\n    OOF_Preds[mytarget]    = base_oof\n    Adj_OOF_Preds[mytarget]= adj_oof   \n\n    PrintColor(\n        f\"\\n ======== FULL REFIT MODELS ======== \\n\", color = Fore.CYAN\n    )\n\n    reg_ = \\\n    VotingRegressor(\n        [(f\"{method}_{mystate}\" , MyLGBMR(random_state = mystate, **lgb_params))\n         for mystate in tqdm(list(range(0, n_refits, 1))) \n        ]\n    )\n    \n    onl_model = Pipeline(steps = [(\"CT\", ct), (\"M\", reg_)])\n    onl_model.fit(\n        X.drop(drop_cols, axis=1, errors = \"ignore\"), \n        targets_df[mytarget]\n    )\n\n    OnlineModels[mytarget]   = onl_model\n    Onl_Preds[mytarget]      = onl_model.predict(X)\n    Onl_Mdl_Preds[mytarget]  = onl_model.predict(Xtest)\n\nprint()\n","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"papermill":{"duration":2358.513962,"end_time":"2024-12-10T21:22:48.058751","exception":false,"start_time":"2024-12-10T20:43:29.544789","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"c9589f82","cell_type":"markdown","source":"## **SUBMISSION**","metadata":{"papermill":{"duration":0.059758,"end_time":"2024-12-10T21:22:48.179651","exception":false,"start_time":"2024-12-10T21:22:48.119893","status":"completed"},"tags":[]}},{"id":"eb832ae3","cell_type":"code","source":"%%time \n\n# Data storage and final CV score\npciat_total = pd.DataFrame(Adj_OOF_Preds).sum(axis=1)\nsii_proxy = \\\nnp.select(\n    [pciat_total <= 30, \n     pciat_total  < 50, \n     pciat_total  < 80\n    ],\n    [0,1,2],\n    3\n)\nscore = ScoreMetric(targets_df[target_col], sii_proxy)\nPrintColor(f\"\\n---> Final OOF score = {score :.6f} \\n\")\n\nPrintColor(f\"---> Prediction counts\")\nprint(np.bincount(sii_proxy))\n\nprint()\nfig, ax = plt.subplots(1,1, figsize = (4,4))\nConfusionMatrixDisplay.from_predictions(\n    targets_df[\"sii\"], \n    sii_proxy, \n    cmap = 'Blues',\n    colorbar= False,\n    text_kw = {\"fontweight\" : \"bold\", \"fontsize\" : 14},\n    ax = ax\n)\nax.set_title(\n    f\"Confusion matrix - {target_col}\\n\", \n    **{\"color\"      : \"maroon\", \n       \"fontweight\" : \"bold\", \n       \"fontsize\"   : 10,\n      }\n)\nplt.tight_layout()\nplt.show()\n\nprint();\n\n# Storing the OOF predictions in the master storage \nAll_OOF_Preds[\"model2\"] = sii_proxy\n\n# Storing the relevant files:-\n(pd.DataFrame(Adj_OOF_Preds).\n assign(sii = sii_proxy).\n to_csv(f\"Adj_OOF_Preds_LGBM1RML{version_nb}.csv\")\n)\n\n(pd.DataFrame(OOF_Preds).to_csv(f\"OOF_Preds_LGBM1RML{version_nb}.csv\"))\njoblib.dump(OnlineModels,  f\"OnlineModels_LGBM1RML{version_nb}.joblib\")\njoblib.dump(fitted_models, f\"FittedModels_LGBM1RML{version_nb}.joblib\")\n\n!ls\n\n# Submission\npciat_total = pd.DataFrame(Onl_Mdl_Preds).sum(axis=1)\nsii_proxy = \\\nnp.select(\n    [pciat_total <= 30, \n     pciat_total < 50, \n     pciat_total < 80\n    ],\n    [0,1,2],\n    3\n)\n\ndf_subm[\"model2\"]  = sii_proxy\n\nprint()","metadata":{"papermill":{"duration":12.541917,"end_time":"2024-12-10T21:23:00.782756","exception":false,"start_time":"2024-12-10T21:22:48.240839","status":"completed"},"tags":[],"trusted":true,"_kg_hide-input":true},"outputs":[],"execution_count":null},{"id":"455dd643","cell_type":"markdown","source":"","metadata":{"papermill":{"duration":0.059169,"end_time":"2024-12-10T21:23:00.902551","exception":false,"start_time":"2024-12-10T21:23:00.843382","status":"completed"},"tags":[]}},{"id":"b3f17d11","cell_type":"markdown","source":"# **MODEL3 - EXPERIMENT 9_7**","metadata":{"papermill":{"duration":0.230267,"end_time":"2024-12-10T21:23:01.192411","exception":false,"start_time":"2024-12-10T21:23:00.962144","status":"completed"},"tags":[]}},{"id":"2e1ca76a","cell_type":"code","source":"%%time \n\nn_neighbors = 4\nversion_nb  = \"V9_7\"","metadata":{"papermill":{"duration":0.069282,"end_time":"2024-12-10T21:23:01.320428","exception":false,"start_time":"2024-12-10T21:23:01.251146","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"2fa3296e","cell_type":"markdown","source":"## **DATA LOAD**","metadata":{"papermill":{"duration":0.059132,"end_time":"2024-12-10T21:23:01.443964","exception":false,"start_time":"2024-12-10T21:23:01.384832","status":"completed"},"tags":[]}},{"id":"63879676","cell_type":"code","source":"%%time \n\n# Preparing the dataset for the inference\ndf_train = pd.read_csv(root / 'train.csv')\ndf_test  = pd.read_csv(root / 'test.csv')\n\n\n# Making new features:-\ndf_train = make_sec_ftre(df_train, \"Train\", )\nprint()\ndf_test  = make_sec_ftre(df_test,  \"Test\", )\n\ntime_series_cols = ts_train.columns.tolist()\ntime_series_cols.remove(\"id\")\n\ndf_train = pd.merge(df_train, ts_train, how=\"left\", on='id')\ndf_test  = pd.merge(df_test, ts_test,   how=\"left\", on='id')\ndf_train = df_train.set_index('id')\ndf_test  = df_test.set_index('id')\n\ntrain_   = df_train.loc[df_train[\"sii\"].isna()]\ndf_train = df_train.dropna(subset= [\"sii\"])\n\nfeature_cols = list(df_test.drop(\"id\", axis = 1, errors = \"ignore\").columns)\ncat_cols     = [c for c in feature_cols if \"Season\" in c]\nnum_cols     = list(df_test.select_dtypes(np.number).columns)\nnum_cols     = sorted(list(set(num_cols).intersection(set(feature_cols))))\n\nPrintColor(f\"\\n\\n---> Shape = {df_train.shape} {df_test.shape}\")\n\n# Preparing the targets\ntargets_df = \\\n(df_train.\n filter(regex = r\"PCIAT|sii\", axis=1).\n iloc[:, 1:].\n dropna(subset = \"sii\", axis=0).\n fillna(0).\n astype(np.uint8)\n)\n\nPrintColor(f\"\\n---> Targets \\n\")\ntargets = np.array(targets_df.columns)[0: 20]\nwith np.printoptions(linewidth = 150):\n    print(np.array(targets))\n\nprint()\n","metadata":{"papermill":{"duration":0.255055,"end_time":"2024-12-10T21:23:01.758365","exception":false,"start_time":"2024-12-10T21:23:01.503310","status":"completed"},"tags":[],"trusted":true,"_kg_hide-input":true},"outputs":[],"execution_count":null},{"id":"4c5cb9aa","cell_type":"markdown","source":"## **FEATURE ENGINEERING**","metadata":{"papermill":{"duration":0.059927,"end_time":"2024-12-10T21:23:01.877775","exception":false,"start_time":"2024-12-10T21:23:01.817848","status":"completed"},"tags":[]}},{"id":"c9df2784","cell_type":"code","source":"%%time \n\nartefacts = {}\n\nX     = df_train[feature_cols]\ny     = df_train[target_col]\nXtest = df_test[feature_cols]\ncv    = StratifiedKFold(5, shuffle=True, random_state=SEED)\n\nygrp  = np.zeros(len(X))\nfor fold_nb, (_, dev_idx) in enumerate(cv.split(X, y)):\n    ygrp[dev_idx] = fold_nb\n\nygrp = pd.Series(ygrp, name = \"fold_nb\", dtype = np.uint8)\n\ncat_cols        = list(X.select_dtypes(exclude = np.number).columns)\nX[cat_cols]     = X[cat_cols].astype(\"category\")\nXtest[cat_cols] = Xtest[cat_cols].astype(\"category\")\n\nct = \\\nColumnTransformer(\n    [(\"Imp\", SimpleImputer(strategy = \"mean\"), num_cols ),\n     (\"Enc\", TargetEncoder(), cat_cols)\n    ],\n    verbose_feature_names_out = False,\n    verbose = False,\n    remainder = \"passthrough\",\n)\n\nPrintColor(f\"\\n\\n---> Shape = {X.shape} {Xtest.shape}\")","metadata":{"papermill":{"duration":0.099843,"end_time":"2024-12-10T21:23:02.037013","exception":false,"start_time":"2024-12-10T21:23:01.937170","status":"completed"},"tags":[],"trusted":true,"_kg_hide-input":true},"outputs":[],"execution_count":null},{"id":"cfdb3142","cell_type":"markdown","source":"## **MODEL TRAINING**","metadata":{"papermill":{"duration":0.062121,"end_time":"2024-12-10T21:23:02.160025","exception":false,"start_time":"2024-12-10T21:23:02.097904","status":"completed"},"tags":[]}},{"id":"e0dbba51","cell_type":"code","source":"%%time \n\n# Initializing storage elements\nOOF_Preds     = {}\nAdj_OOF_Preds = {}\nFittedModels  = {}\nFtreImp       = {}\n\nOnlineModels  = {}\nOnl_Preds     = {}\nOnl_Mdl_Preds = {}\n\nlgb_params = \\\n{\n'objective'       : 'l2',\n'verbosity'       : -1,\n'n_iter'          : 200,\n'lambda_l1'       : 0.005116829730239727,\n'lambda_l2'       : 0.0011520776712645852,\n'learning_rate'   : 0.02376367323636638,\n'max_depth'       : 5,\n'num_leaves'      : 207,\n'colsample_bytree': 0.7759862336963801,\n'colsample_bynode': 0.5110355095943208,\n'bagging_fraction': 0.5485770314992224,\n'bagging_freq'    : 7,\n'min_data_in_leaf': 78,\n}\n\ndrop_cols    = [\"Source\", \"id\", \"Id\", \"Label\", target_col, \"fold_nb\"]\nmd           = ModelTrainer(drop_cols = drop_cols, ntop  = 50, verbose = True)\nX            = X.drop(drop_cols, axis=1, errors = \"ignore\")\nXtest        = Xtest.drop(drop_cols, axis=1, errors = \"ignore\")\ncat_mdl_cols = list(Xtest.select_dtypes(\"category\").columns)\n\nprint(f\"\\n{'=' * 20} {method.upper()} MODEL TRAINING {'=' * 20}\\n\")\n\nfor mytarget in tqdm(targets_df.columns[0: 20]):\n    PrintColor(\n        f\"\\n ======== CURRENT TARGET = {mytarget} ======== \\n\"\n    )\n\n    model = \\\n    Pipeline(\n        steps = \\\n        [(\"CT\", ct),\n         (\"M\", MyLGBMR(random_state = 42, **lgb_params))\n        ]\n    )\n\n    PrintColor(\n        f\"\\n ======== OFFLINE MODELS - CV ======== \\n\", color = Fore.CYAN\n    )\n    (base_oof, adj_oof, mdl_preds, fitted_models, ftreimp, thresholds) = \\\n    md.MakeOfflineModel(\n        X,\n        targets_df[mytarget],\n        ygrp,\n        Xtest,\n        model  = model,\n        method = method,\n        ftreimp_plot_req = False,\n    )\n\n    FittedModels[mytarget] = fitted_models\n    FtreImp[mytarget]      = pd.Series(ftreimp, index = X.columns)\n    OOF_Preds[mytarget]    = base_oof\n    Adj_OOF_Preds[mytarget]= adj_oof   \n\n    PrintColor(\n        f\"\\n ======== FULL REFIT MODELS ======== \\n\", color = Fore.CYAN\n    )\n\n    reg_ = \\\n    VotingRegressor(\n        [(f\"{method}_{mystate}\" , MyLGBMR(random_state = mystate, **lgb_params))\n         for mystate in tqdm(list(range(0, n_refits, 1))) \n        ]\n    )\n    \n    onl_model = Pipeline(steps = [(\"CT\", ct), (\"M\", reg_)])\n    onl_model.fit(\n        X.drop(drop_cols, axis=1, errors = \"ignore\"), \n        targets_df[mytarget]\n    )\n\n    OnlineModels[mytarget]   = onl_model\n    Onl_Preds[mytarget]      = onl_model.predict(X)\n    Onl_Mdl_Preds[mytarget]  = onl_model.predict(Xtest)\n","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"papermill":{"duration":1345.207029,"end_time":"2024-12-10T21:45:27.426721","exception":false,"start_time":"2024-12-10T21:23:02.219692","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"354699df","cell_type":"markdown","source":"## **SUBMISSION**","metadata":{"papermill":{"duration":0.075345,"end_time":"2024-12-10T21:45:27.577289","exception":false,"start_time":"2024-12-10T21:45:27.501944","status":"completed"},"tags":[]}},{"id":"ddc646ea","cell_type":"code","source":"%%time \n\n# Data storage and final CV score\npciat_total = pd.DataFrame(Adj_OOF_Preds).sum(axis=1)\nsii_proxy = \\\nnp.select(\n    [pciat_total <= 30, \n     pciat_total  < 50, \n     pciat_total  < 80\n    ],\n    [0,1,2],\n    3\n)\nscore = ScoreMetric(targets_df[target_col], sii_proxy)\nPrintColor(f\"\\n---> Final OOF score = {score :.6f} \\n\")\n\nPrintColor(f\"---> Prediction counts\")\nprint(np.bincount(sii_proxy))\n\nprint()\nfig, ax = plt.subplots(1,1, figsize = (4,4))\nConfusionMatrixDisplay.from_predictions(\n    targets_df[\"sii\"], \n    sii_proxy, \n    cmap = 'Blues',\n    colorbar= False,\n    text_kw = {\"fontweight\" : \"bold\", \"fontsize\" : 14},\n    ax = ax\n)\nax.set_title(\n    f\"Confusion matrix - {target_col}\\n\", \n    **{\"color\"      : \"maroon\", \n       \"fontweight\" : \"bold\", \n       \"fontsize\"   : 10,\n      }\n)\nplt.tight_layout()\nplt.show()\n\nprint();\n\n# Storing the OOF predictions in the master storage \nAll_OOF_Preds[\"model3\"] = sii_proxy\n\n# Storing the relevant files:-\n(pd.DataFrame(Adj_OOF_Preds).\n assign(sii = sii_proxy).\n to_csv(f\"Adj_OOF_Preds_LGBM1RML{version_nb}.csv\")\n)\n\n(pd.DataFrame(OOF_Preds).to_csv(f\"OOF_Preds_LGBM1RML{version_nb}.csv\"))\njoblib.dump(OnlineModels,  f\"OnlineModels_LGBM1RML{version_nb}.joblib\")\njoblib.dump(fitted_models, f\"FittedModels_LGBM1RML{version_nb}.joblib\")\n\n!ls\n\n# Submission\npciat_total = pd.DataFrame(Onl_Mdl_Preds).sum(axis=1)\nsii_proxy = \\\nnp.select(\n    [pciat_total <= 30, \n     pciat_total < 50, \n     pciat_total < 80\n    ],\n    [0,1,2],\n    3\n)\n\ndf_subm[\"model3\"]  = sii_proxy\n","metadata":{"papermill":{"duration":13.059047,"end_time":"2024-12-10T21:45:40.711119","exception":false,"start_time":"2024-12-10T21:45:27.652072","status":"completed"},"tags":[],"trusted":true,"_kg_hide-input":true},"outputs":[],"execution_count":null},{"id":"34381dc9","cell_type":"code","source":"","metadata":{"papermill":{"duration":0.076634,"end_time":"2024-12-10T21:45:40.863021","exception":false,"start_time":"2024-12-10T21:45:40.786387","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"fec8ae8a","cell_type":"markdown","source":"# **MODEL4 - EXPERIMENT 7_2**","metadata":{"papermill":{"duration":0.075814,"end_time":"2024-12-10T21:45:41.017321","exception":false,"start_time":"2024-12-10T21:45:40.941507","status":"completed"},"tags":[]}},{"id":"31b6276d","cell_type":"code","source":"%%time \n\nversion_nb = \"V7_2\"","metadata":{"papermill":{"duration":0.08576,"end_time":"2024-12-10T21:45:41.180101","exception":false,"start_time":"2024-12-10T21:45:41.094341","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"64c64623","cell_type":"markdown","source":"## **DATA LOAD**","metadata":{"papermill":{"duration":0.073297,"end_time":"2024-12-10T21:45:41.328021","exception":false,"start_time":"2024-12-10T21:45:41.254724","status":"completed"},"tags":[]}},{"id":"d8f8a6f1","cell_type":"code","source":"%%time \n\nroot     = Path('/kaggle/input/child-mind-institute-problematic-internet-use')\ndf_train = pd.read_csv(root / 'train.csv')\ndf_test  = pd.read_csv(root / 'test.csv')\n\ntime_series_cols = ts_train.columns.tolist()\ntime_series_cols.remove(\"id\")\n\ndf_train = pd.merge(df_train, ts_train, how=\"left\", on='id')\ndf_test  = pd.merge(df_test, ts_test, how=\"left\", on='id')\ndf_train = df_train.set_index('id')\ndf_test  = df_test.set_index('id')\n\ncat_cols     = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', \n                'Fitness_Endurance-Season', 'FGC-Season', 'BIA-Season', 'PAQ_A-Season', \n                'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season'\n               ]\n\nnum_cols     = ['Basic_Demos-Age', 'Basic_Demos-Sex', 'CGAS-CGAS_Score', 'Physical-BMI', \n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Max_Stage', 'Fitness_Endurance-Time_Mins', \n                'Fitness_Endurance-Time_Sec', 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', \n                'FGC-FGC_GSND', 'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', \n                'FGC-FGC_PU', 'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', \n                'FGC-FGC_SRR', 'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone', \n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI', 'BIA-BIA_BMR', \n                'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM', 'BIA-BIA_FFMI', 'BIA-BIA_FMI', \n                'BIA-BIA_Fat', 'BIA-BIA_Frame_num', 'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', \n                'BIA-BIA_SMM', 'BIA-BIA_TBW', 'PAQ_A-PAQ_A_Total', 'PAQ_C-PAQ_C_Total',\n                'SDS-SDS_Total_Raw', 'SDS-SDS_Total_T', 'PreInt_EduHx-computerinternet_hoursday'\n               ]\n\ntabular_cols = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex', \n                'CGAS-Season', 'CGAS-CGAS_Score', 'Physical-Season', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference', \n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Season', 'Fitness_Endurance-Max_Stage', 'Fitness_Endurance-Time_Mins',\n                'Fitness_Endurance-Time_Sec', 'FGC-Season', 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', \n                'FGC-FGC_GSND', 'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU', \n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR', 'FGC-FGC_SRR_Zone',\n                'FGC-FGC_TL', 'FGC-FGC_TL_Zone', 'BIA-Season', 'BIA-BIA_Activity_Level_num', \n                'BIA-BIA_BMC', 'BIA-BIA_BMI', 'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', \n                'BIA-BIA_FFM', 'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM', 'BIA-BIA_TBW', \n                'PAQ_A-Season', 'PAQ_A-PAQ_A_Total', 'PAQ_C-Season', 'PAQ_C-PAQ_C_Total', \n                'SDS-Season', 'SDS-SDS_Total_Raw', 'SDS-SDS_Total_T', 'PreInt_EduHx-Season', \n                'PreInt_EduHx-computerinternet_hoursday'\n               ]\n\nfeature_cols = tabular_cols + time_series_cols\nnum_cols     = num_cols + time_series_cols\ndf_train     = df_train.dropna(subset= [\"sii\"])\n\nPrintColor(f\"\\n\\n---> Shape = {df_train.shape} {df_test.shape}\")","metadata":{"papermill":{"duration":0.175386,"end_time":"2024-12-10T21:45:41.579314","exception":false,"start_time":"2024-12-10T21:45:41.403928","status":"completed"},"tags":[],"trusted":true,"_kg_hide-input":true},"outputs":[],"execution_count":null},{"id":"1a60a08b","cell_type":"markdown","source":"## **FEATURE ENGINEERING**","metadata":{"papermill":{"duration":0.105039,"end_time":"2024-12-10T21:45:41.758941","exception":false,"start_time":"2024-12-10T21:45:41.653902","status":"completed"},"tags":[]}},{"id":"ea218ea5","cell_type":"code","source":"%%time \n\nX     = df_train[feature_cols]\ny     = df_train[target_col]\nXtest = df_test[feature_cols]\ncv    = StratifiedKFold(5, shuffle=True, random_state=SEED)\n\nygrp  = np.zeros(len(X))\nfor fold_nb, (_, dev_idx) in enumerate(cv.split(X, y)):\n    ygrp[dev_idx] = fold_nb\n\nygrp = pd.Series(ygrp, name = \"fold_nb\", dtype = np.uint8)\n\nxform = SimpleImputer(strategy = \"mean\")\nX[num_cols]     = xform.fit_transform(X[num_cols])\nXtest[num_cols] = xform.transform(Xtest[num_cols])\n\nxform = \\\nOrdinalEncoder(dtype=np.int32,\n               handle_unknown='use_encoded_value',\n               unknown_value=-1,\n               encoded_missing_value=-2,\n              )\nX[cat_cols]     = xform.fit_transform(X[cat_cols])\nXtest[cat_cols] = xform.transform(Xtest[cat_cols])\n\nPrintColor(f\"\\n\\n---> Shape = {X.shape} {Xtest.shape}\")","metadata":{"papermill":{"duration":0.168678,"end_time":"2024-12-10T21:45:42.001641","exception":false,"start_time":"2024-12-10T21:45:41.832963","status":"completed"},"tags":[],"trusted":true,"_kg_hide-input":true},"outputs":[],"execution_count":null},{"id":"c89f62f6","cell_type":"markdown","source":"## **MODEL TRAINING**","metadata":{"papermill":{"duration":0.074854,"end_time":"2024-12-10T21:45:42.154635","exception":false,"start_time":"2024-12-10T21:45:42.079781","status":"completed"},"tags":[]}},{"id":"b54da7b2","cell_type":"code","source":"%%time \n\nparams = {\n    'objective'       : 'l2',\n    'verbosity'       : -1,\n    'n_iter'          : 200,\n    'lambda_l1'       : 0.005116829730239727,\n    'lambda_l2'       : 0.0011520776712645852,\n    'learning_rate'   : 0.02376367323636638,\n    'max_depth'       : 5,\n    'num_leaves'      : 207,\n    'colsample_bytree': 0.7759862336963801,\n    'colsample_bynode': 0.5110355095943208,\n    'bagging_fraction': 0.5485770314992224,\n    'bagging_freq'    : 7,\n    'min_data_in_leaf': 78,\n}\n\nprint(f\"\\n{'=' * 20} {method.upper()} MODEL TRAINING {'=' * 20}\\n\")\ndrop_cols = [\"Source\", \"id\", \"Id\", \"Label\", target_col, \"fold_nb\"]\n\nmd    = ModelTrainer(drop_cols = drop_cols, ntop  = 50, verbose = True)\nmodel = Pipeline(steps = [(\"M\", MyLGBMR(random_state = 42, **params))])\n\n(base_oof, adj_oof, mdl_preds, fitted_models, ftreimp, thresholds) = \\\nmd.MakeOfflineModel(\n    X,\n    y,\n    ygrp,\n    Xtest,\n    model  = model,\n    method = method,\n    ftreimp_plot_req = True,\n)\n\nreg_ = \\\nVotingRegressor([\n    ('lgb_0', MyLGBMR(**params, random_state=12)),\n    ('lgb_1', MyLGBMR(**params, random_state=22)),\n    ('lgb_2', MyLGBMR(**params, random_state=32)),\n    ('lgb_3', MyLGBMR(**params, random_state=42)),\n    ('lgb_4', MyLGBMR(**params, random_state=52)),\n    ('lgb_5', MyLGBMR(**params, random_state=62)),\n    ('lgb_6', MyLGBMR(**params, random_state=72)),\n    ('lgb_7', MyLGBMR(**params, random_state=82)),\n    ('lgb_8', MyLGBMR(**params, random_state=92)),\n    ('lgb_9', MyLGBMR(**params, random_state=102)),\n    ('lgb_10', MyLGBMR(**params, random_state=777)),\n    ('lgb_11', MyLGBMR(**params, random_state=500)),\n    ('lgb_12', MyLGBMR(**params, random_state=1000)),\n    ('lgb_13', MyLGBMR(**params, random_state=5000)),\n]\n)\n\nmodel = Pipeline(steps = [(\"M\", reg_)])\nmodel.fit(X, y)\n\n","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"papermill":{"duration":10.914433,"end_time":"2024-12-10T21:45:53.144326","exception":false,"start_time":"2024-12-10T21:45:42.229893","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"4340ee10","cell_type":"markdown","source":"## **SUBMISSION**","metadata":{"papermill":{"duration":0.078446,"end_time":"2024-12-10T21:45:53.302651","exception":false,"start_time":"2024-12-10T21:45:53.224205","status":"completed"},"tags":[]}},{"id":"3f57bce4","cell_type":"code","source":"%%time \n\ndf_subm[\"model4\"] = model.predict(Xtest)\ndf_subm[\"model4\"] = np.uint8(df_subm[\"model4\"].round())\n\nprint()\nfig, ax = plt.subplots(1,1, figsize = (4,4))\nConfusionMatrixDisplay.from_predictions(\n    y, \n    adj_oof, \n    cmap = 'Blues',\n    colorbar= False,\n    text_kw = {\"fontweight\" : \"bold\", \"fontsize\" : 14},\n    ax = ax\n)\nax.set_title(\n    f\"Confusion matrix - {target_col}\\n\", \n    **{\"color\"      : \"maroon\", \n       \"fontweight\" : \"bold\", \n       \"fontsize\"   : 10,\n      }\n)\nplt.tight_layout()\nplt.show()\n\nprint();\n\n# Storing the OOF predictions in the master storage \nAll_OOF_Preds[\"model4\"] = adj_oof\n\n# Storing the relevant datasets\n(pd.DataFrame(adj_oof, columns = [\"sii\"]).\n to_csv(f\"Adj_OOF_Preds_LGBM1RML{version_nb}.csv\")\n)\n\n(pd.DataFrame(base_oof).to_csv(f\"OOF_Preds_LGBM1RML{version_nb}.csv\"))\n\njoblib.dump(model,         f\"OnlineModels_LGBM1RML{version_nb}.joblib\")\njoblib.dump(fitted_models, f\"FittedModels_LGBM1RML{version_nb}.joblib\")\n\nprint()\n!ls","metadata":{"papermill":{"duration":1.884401,"end_time":"2024-12-10T21:45:55.265219","exception":false,"start_time":"2024-12-10T21:45:53.380818","status":"completed"},"tags":[],"trusted":true,"_kg_hide-input":true},"outputs":[],"execution_count":null},{"id":"310926db","cell_type":"markdown","source":"","metadata":{"papermill":{"duration":0.079018,"end_time":"2024-12-10T21:45:55.423366","exception":false,"start_time":"2024-12-10T21:45:55.344348","status":"completed"},"tags":[]}},{"id":"fd3aeaf7","cell_type":"markdown","source":"# **MODEL 5 - EXPERIMENT 7_24**","metadata":{"papermill":{"duration":0.079061,"end_time":"2024-12-10T21:45:55.636833","exception":false,"start_time":"2024-12-10T21:45:55.557772","status":"completed"},"tags":[]}},{"id":"cb7a1ab9","cell_type":"code","source":"%%time \n\nversion_nb = \"MLV7_24\"","metadata":{"papermill":{"duration":0.089296,"end_time":"2024-12-10T21:45:55.804365","exception":false,"start_time":"2024-12-10T21:45:55.715069","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"2931cc00","cell_type":"markdown","source":"## **DATA LOAD**","metadata":{"papermill":{"duration":0.080648,"end_time":"2024-12-10T21:45:55.964885","exception":false,"start_time":"2024-12-10T21:45:55.884237","status":"completed"},"tags":[]}},{"id":"b0da1696","cell_type":"code","source":"%%time \n\ndef extract_stats(data):\n    return [\n        data.mean(), \n        data.std(), \n        data.max(), \n        data.min(), \n        data.diff().mean(), \n        data.diff().std(),\n    ]\n\ndef make_stats(df):\n    \"\"\"\n    Source - https://www.kaggle.com/code/mehrankazeminia/3-cmi-end-to-end\n    \"\"\"\n    \n    df[\"hours\"]  = df[\"time_of_day\"] // (3_600 * 1_000_000_000)\n    df[\"l2_xyz\"] = df[\"X\"] ** 2 +  df[\"Y\"] ** 2 + df[\"Z\"] ** 2\n    df[\"l1_xyz\"] = df[\"X\"].abs()+  df[\"Y\"].abs()+ df[\"Z\"].abs() \n    \n    features = [\n        df[\"non-wear_flag\"].mean(),\n        df[\"enmo\"][df[\"enmo\"] >= 0.05].sum(),\n    ]\n\n    # Mask1 - by day/ night/ sunrise-sunset\n    night   = ((df[\"hours\"] >= 22)  | (df[\"hours\"] <= 6))\n    day     = ((df[\"hours\"] <= 20) & (df[\"hours\"] >= 7))\n    no_mask = np.ones(len(df), dtype=bool)\n    \n    keys  = [\"enmo\", \"anglez\", \"light\", \"battery_voltage\"]\n    masks = [no_mask, night, day]\n      \n    # Iterate over keys and masks to generate the statistics\n    for key in keys:\n        for mask in masks:\n            filtered_data = df.loc[mask, key]\n            features.extend(extract_stats(filtered_data))\n\n    return features\n\ndef process_file(filename, dirname):\n    df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    df.drop('step', axis=1, inplace=True)\n    return make_stats(df), filename.split('=')[1]\n\ndef load_time_series(dirname) -> pd.DataFrame:\n    ids = os.listdir(dirname)\n    \n    with ThreadPoolExecutor() as executor:\n        results = \\\n        list(\n            tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), \n                 total=len(ids))\n        )\n    \n    stats, indexes = zip(*results)\n    df = pd.DataFrame(stats, columns=[f\"stat_{i}\" for i in range(len(stats[0]))])\n    df['id'] = indexes\n    return df","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"papermill":{"duration":0.097671,"end_time":"2024-12-10T21:45:56.144866","exception":false,"start_time":"2024-12-10T21:45:56.047195","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"7a76366a","cell_type":"code","source":"%%time \n\nroot     = Path('/kaggle/input/child-mind-institute-problematic-internet-use')\ndf_train = pd.read_csv(root / 'train.csv')\ndf_test  = pd.read_csv(root / 'test.csv')\n\nts_train = load_time_series(root / \"series_train.parquet\")\nts_test  = load_time_series(root / \"series_test.parquet\")\n\n# Making new features:-\ndf_train = make_sec_ftre(df_train, \"Train\", )\nprint()\ndf_test  = make_sec_ftre(df_test,  \"Test\", )\n\ntime_series_cols = ts_train.columns.tolist()\ntime_series_cols.remove(\"id\")\n\ndf_train = pd.merge(df_train, ts_train, how=\"left\", on='id')\ndf_test  = pd.merge(df_test, ts_test, how=\"left\", on='id')\ndf_train = df_train.set_index('id')\ndf_test  = df_test.set_index('id')\ndf_train = df_train.dropna(subset= [\"sii\"])\n\nfeature_cols = list(df_test.drop(\"id\", axis = 1, errors = \"ignore\").columns)\ncat_cols     = [c for c in feature_cols if \"Season\" in c]\nnum_cols     = list(df_test.select_dtypes(np.number).columns)\nnum_cols     = sorted(list(set(num_cols).intersection(set(feature_cols))))\n\nPrintColor(f\"\\n\\n---> Shape = {df_train.shape} {df_test.shape}\")\n\ntargets_df = \\\n(df_train.\n filter(regex = r\"PCIAT|sii\", axis=1).\n iloc[:, 1:].\n dropna(subset = \"sii\", axis=0).\n fillna(0).\n astype(np.uint8)\n)\n\nPrintColor(f\"\\n---> Targets \\n\")\nwith np.printoptions(linewidth = 150):\n    print(np.array(targets_df.columns))\n","metadata":{"papermill":{"duration":93.533187,"end_time":"2024-12-10T21:47:29.756241","exception":false,"start_time":"2024-12-10T21:45:56.223054","status":"completed"},"tags":[],"trusted":true,"_kg_hide-input":true},"outputs":[],"execution_count":null},{"id":"f3f60d0f","cell_type":"markdown","source":"## **FEATURE ENGINEERING**","metadata":{"papermill":{"duration":0.097879,"end_time":"2024-12-10T21:47:29.956451","exception":false,"start_time":"2024-12-10T21:47:29.858572","status":"completed"},"tags":[]}},{"id":"cd0d3ac3","cell_type":"code","source":"%%time \n\nX     = df_train[feature_cols]\ny     = df_train[target_col]\nXtest = df_test[feature_cols]\ncv    = StratifiedKFold(5, shuffle=True, random_state=SEED)\n\nygrp  = np.zeros(len(X))\nfor fold_nb, (_, dev_idx) in enumerate(cv.split(X, y)):\n    ygrp[dev_idx] = fold_nb\n\nygrp = pd.Series(ygrp, name = \"fold_nb\", dtype = np.uint8)\n\nxform = SimpleImputer(strategy = \"mean\")\nX[num_cols]     = xform.fit_transform(X[num_cols])\nXtest[num_cols] = xform.transform(Xtest[num_cols])\n\nxform = \\\nOrdinalEncoder(dtype=np.int32,\n               handle_unknown='use_encoded_value',\n               unknown_value=-1,\n               encoded_missing_value=-2,\n              )\nX[cat_cols]     = xform.fit_transform(X[cat_cols])\nXtest[cat_cols] = xform.transform(Xtest[cat_cols])\n\nPrintColor(f\"\\n\\n---> Shape = {X.shape} {Xtest.shape}\")\n","metadata":{"papermill":{"duration":0.189584,"end_time":"2024-12-10T21:47:30.248998","exception":false,"start_time":"2024-12-10T21:47:30.059414","status":"completed"},"tags":[],"trusted":true,"_kg_hide-input":true},"outputs":[],"execution_count":null},{"id":"3b48ec06","cell_type":"markdown","source":"## **MODEL TRAINING**","metadata":{"papermill":{"duration":0.096863,"end_time":"2024-12-10T21:47:30.442981","exception":false,"start_time":"2024-12-10T21:47:30.346118","status":"completed"},"tags":[]}},{"id":"97ecd73f","cell_type":"code","source":"%%time \n\n# Initializing storage elements\nOOF_Preds     = {}\nAdj_OOF_Preds = {}\nFittedModels  = {}\nFtreImp       = {}\n\nOnlineModels  = {}\nOnl_Preds     = {}\nOnl_Mdl_Preds = {}\n\ndrop_cols = [\"Source\", \"id\", \"Id\", \"Label\", target_col, \"fold_nb\"]\nmd        = ModelTrainer(drop_cols = drop_cols, ntop  = 50, verbose = True)\nX         = X.drop(drop_cols, axis=1, errors = \"ignore\")\nXtest     = Xtest.drop(drop_cols, axis=1, errors = \"ignore\")\ncat_mdl_cols = list(Xtest.select_dtypes(\"category\").columns)\n\nlgb_params = \\\n{\n'objective'       : 'l2',\n'verbosity'       : -1,\n'n_iter'          : 200,\n'lambda_l1'       : 0.005116829730239727,\n'lambda_l2'       : 0.0011520776712645852,\n'learning_rate'   : 0.02376367323636638,\n'max_depth'       : 5,\n'num_leaves'      : 207,\n'colsample_bytree': 0.7759862336963801,\n'colsample_bynode': 0.5110355095943208,\n'bagging_fraction': 0.5485770314992224,\n'bagging_freq'    : 7,\n'min_data_in_leaf': 78,\n}\n\n\nprint(f\"\\n{'=' * 20} {method.upper()} MODEL TRAINING {'=' * 20}\\n\")\n\nfor mytarget in tqdm(targets_df.columns[0: 20]):\n    PrintColor(\n        f\"\\n ======== CURRENT TARGET = {mytarget} ======== \\n\"\n    )\n\n    model = Pipeline(steps = [(\"M\", MyLGBMR(random_state = 42, **lgb_params))])\n\n    PrintColor(\n        f\"\\n ======== OFFLINE MODELS - CV ======== \\n\", color = Fore.CYAN\n    )\n    (base_oof, adj_oof, mdl_preds, fitted_models, ftreimp, thresholds) = \\\n    md.MakeOfflineModel(\n        X,\n        targets_df[mytarget],\n        ygrp,\n        Xtest,\n        model  = model,\n        method = method,\n        ftreimp_plot_req = False,\n    )\n\n    FittedModels[mytarget] = fitted_models\n    FtreImp[mytarget]      = pd.Series(ftreimp, index = X.columns)\n    OOF_Preds[mytarget]    = base_oof\n    Adj_OOF_Preds[mytarget]= adj_oof   \n\n    PrintColor(\n        f\"\\n ======== FULL REFIT MODELS ======== \\n\", color = Fore.CYAN\n    )\n\n    reg_ = \\\n    VotingRegressor(\n        [(f\"{method}_{mystate}\" , MyLGBMR(random_state = mystate, **lgb_params))\n         for mystate in tqdm(list(range(0, n_refits, 1))) \n        ]\n    )\n    \n    onl_model = Pipeline(steps = [(\"M\", reg_)])\n    onl_model.fit(\n        X.drop(drop_cols, axis=1, errors = \"ignore\"), \n        targets_df[mytarget]\n    )\n\n    OnlineModels[mytarget]   = onl_model\n    Onl_Preds[mytarget]      = onl_model.predict(X)\n    Onl_Mdl_Preds[mytarget]  = onl_model.predict(Xtest)\n","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"papermill":{"duration":1304.775824,"end_time":"2024-12-10T22:09:15.317181","exception":false,"start_time":"2024-12-10T21:47:30.541357","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"ca974e3f","cell_type":"markdown","source":"## **SUBMISSION**","metadata":{"papermill":{"duration":0.110562,"end_time":"2024-12-10T22:09:15.538897","exception":false,"start_time":"2024-12-10T22:09:15.428335","status":"completed"},"tags":[]}},{"id":"73bc5c46","cell_type":"code","source":"%%time \n\n# Data storage and final CV score\npciat_total = pd.DataFrame(Adj_OOF_Preds).sum(axis=1)\nsii_proxy = \\\nnp.select(\n    [pciat_total <= 30, \n     pciat_total  < 50, \n     pciat_total  < 80\n    ],\n    [0,1,2],\n    3\n)\nscore = ScoreMetric(targets_df[target_col], sii_proxy)\nPrintColor(f\"\\n---> Final OOF score = {score :.6f} \\n\")\n\n# Storing the OOF predictions in the master storage \nAll_OOF_Preds[\"model5\"] = sii_proxy\n\npd.DataFrame(FtreImp).to_csv(f\"FtreImp_LGBM1RML{version_nb}.csv\")\n\n(pd.DataFrame(Adj_OOF_Preds).\n assign(sii = sii_proxy).\n to_csv(f\"Adj_OOF_Preds_LGBM1RML{version_nb}.csv\")\n)\n\n(pd.DataFrame(OOF_Preds).to_csv(f\"OOF_Preds_LGBM1RML{version_nb}.csv\"))\npd.DataFrame(Onl_Preds).to_csv(f\"Onl_Preds_LGBM1RML{version_nb}.csv\")\njoblib.dump(FittedModels, f\"FittedModels_LGBM1RML{version_nb}.joblib\")\njoblib.dump(OnlineModels, f\"OnlineModels_LGBM1RML{version_nb}.joblib\")\n\nPrintColor(f\"---> Prediction counts\")\nprint(np.bincount(sii_proxy))\n\nprint()\nfig, ax = plt.subplots(1,1, figsize = (4,4))\nConfusionMatrixDisplay.from_predictions(\n    targets_df[\"sii\"], \n    sii_proxy, \n    cmap = 'Blues',\n    colorbar= False,\n    text_kw = {\"fontweight\" : \"bold\", \"fontsize\" : 14},\n    ax = ax\n)\nax.set_title(\n    f\"Confusion matrix - {target_col}\\n\", \n    **{\"color\"      : \"maroon\", \n       \"fontweight\" : \"bold\", \n       \"fontsize\"   : 10,\n      }\n)\nplt.tight_layout()\nplt.show()\n\n\npciat_total = pd.DataFrame(Onl_Mdl_Preds).sum(axis=1)\nsii_proxy = \\\nnp.select(\n    [pciat_total <= 30, pciat_total < 50, pciat_total < 80],\n    [0,1,2],\n    3\n)\n\ndf_subm[\"model5\"]  = sii_proxy\n\n!ls\nprint()","metadata":{"papermill":{"duration":13.680289,"end_time":"2024-12-10T22:09:29.334046","exception":false,"start_time":"2024-12-10T22:09:15.653757","status":"completed"},"tags":[],"trusted":true,"_kg_hide-input":true},"outputs":[],"execution_count":null},{"id":"9e4d624c","cell_type":"markdown","source":"","metadata":{"papermill":{"duration":0.110109,"end_time":"2024-12-10T22:09:29.557986","exception":false,"start_time":"2024-12-10T22:09:29.447877","status":"completed"},"tags":[]}},{"id":"21c5ad10","cell_type":"markdown","source":"# **FINAL SUBMISSION**","metadata":{"papermill":{"duration":0.110637,"end_time":"2024-12-10T22:09:29.778810","exception":false,"start_time":"2024-12-10T22:09:29.668173","status":"completed"},"tags":[]}},{"id":"e9b79c1f","cell_type":"code","source":"%%time \n\nfinal_oof = \\\nnp.uint8(\n        np.round(\n            pd.DataFrame(All_OOF_Preds).mean(axis=1).values,0\n        )\n)\n\nscore = ScoreMetric(targets_df[target_col], final_oof)\nPrintColor(\n    f\"\\n\\n---> Final blend score = {score :.6f}\"\n)\n\nPrintColor(f\"\\n---> Predictions - final OOF\")\nprint(np.bincount(final_oof))\nprint()\n\ndf_subm[target_col] = \\\nnp.uint8(\n    np.round(\n        df_subm.drop(target_col, axis=1).mean(axis = 1).values,\n        0\n    )\n)\n\nPrintColor(f\"\\n---> Final submission file\\n\")\ndisplay(\n   df_subm[[target_col]].head(10) \n)\n\ndf_subm[[target_col]].to_csv(f\"submission.csv\")\nprint()","metadata":{"papermill":{"duration":0.1463,"end_time":"2024-12-10T22:09:30.038764","exception":false,"start_time":"2024-12-10T22:09:29.892464","status":"completed"},"tags":[],"trusted":true,"_kg_hide-input":true},"outputs":[],"execution_count":null},{"id":"9f3ae6fa","cell_type":"code","source":"print()\n!ls \n\nprint()\n!head submission.csv","metadata":{"papermill":{"duration":2.916124,"end_time":"2024-12-10T22:09:33.070337","exception":false,"start_time":"2024-12-10T22:09:30.154213","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null}]}