{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"},{"sourceId":9982818,"sourceType":"datasetVersion","datasetId":6143022}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nfrom colorama import Fore, Style\n\nfrom lightgbm import LGBMRegressor\nfrom catboost import CatBoostRegressor\nfrom xgboost import XGBRegressor\n\nfrom sklearn.base import clone\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.impute import KNNImputer\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import cohen_kappa_score\nfrom scipy.optimize import minimize\nfrom sklearn.ensemble import VotingRegressor\n\n# from scipy.optimize import minimize\nfrom concurrent.futures import ThreadPoolExecutor\nfrom tqdm import tqdm\n\n","metadata":{"_uuid":"df25e8ef-eff5-4e3f-9fe4-f58279a00a26","_cell_guid":"17c47664-8074-4fb5-915b-96e458d74d15","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-12-12T09:35:06.325284Z","iopub.execute_input":"2024-12-12T09:35:06.325602Z","iopub.status.idle":"2024-12-12T09:35:10.546332Z","shell.execute_reply.started":"2024-12-12T09:35:06.325574Z","shell.execute_reply":"2024-12-12T09:35:10.545551Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Mapping - Dropping features not in test sets","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/child-mind-institute-problematic-internet-use/train.csv\")\ndf = df.dropna(subset=['sii']).reset_index() # keeping labeled values only\ntest = pd.read_csv(\"/kaggle/input/child-mind-institute-problematic-internet-use/test.csv\")\nseason_mapping = {\n    'Winter': -1,\n    'Spring': -0.5,\n    'Summer': 0.5,\n    'Fall': 1\n}\n# mapping non-string values\ndf = df.replace(season_mapping)\ntest = test.replace(season_mapping)\n\n# dropping questions not in test dataset\ntest_missing_columns = set(df.columns) - set(test.columns)\nfor col in test_missing_columns:\n    if col != 'sii':  # Retain the target column for training\n        df.drop(columns=col, inplace=True)\n# for later use\ntrain_ids = df['id']\ntest_ids = test['id']\ntrain_labels = df['sii']\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T09:35:10.547743Z","iopub.execute_input":"2024-12-12T09:35:10.548262Z","iopub.status.idle":"2024-12-12T09:35:10.667040Z","shell.execute_reply.started":"2024-12-12T09:35:10.548234Z","shell.execute_reply":"2024-12-12T09:35:10.666110Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df['sii'].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T09:35:10.668165Z","iopub.execute_input":"2024-12-12T09:35:10.668437Z","iopub.status.idle":"2024-12-12T09:35:10.683183Z","shell.execute_reply.started":"2024-12-12T09:35:10.668410Z","shell.execute_reply":"2024-12-12T09:35:10.682251Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# K-Nearest neighboors missing data imputation","metadata":{}},{"cell_type":"code","source":"featureCols = sorted(list(set(df.columns) - set(['sii', 'id'])))\n# featureCols","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T09:35:10.685092Z","iopub.execute_input":"2024-12-12T09:35:10.685377Z","iopub.status.idle":"2024-12-12T09:35:10.693061Z","shell.execute_reply.started":"2024-12-12T09:35:10.685343Z","shell.execute_reply":"2024-12-12T09:35:10.692340Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dropCols = []\nfor column in featureCols:\n    if (df[column].isnull().sum() > 1300):\n        dropCols.append(column)\ndropCols\ndf = df.drop(dropCols, axis=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T09:35:10.693968Z","iopub.execute_input":"2024-12-12T09:35:10.694229Z","iopub.status.idle":"2024-12-12T09:35:10.711914Z","shell.execute_reply.started":"2024-12-12T09:35:10.694206Z","shell.execute_reply":"2024-12-12T09:35:10.711226Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"featureCols = sorted(list(set(df.columns) - set(['sii', 'id'])))\ntrain = pd.DataFrame(df, columns=featureCols)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T09:35:10.713020Z","iopub.execute_input":"2024-12-12T09:35:10.713358Z","iopub.status.idle":"2024-12-12T09:35:10.719119Z","shell.execute_reply.started":"2024-12-12T09:35:10.713331Z","shell.execute_reply":"2024-12-12T09:35:10.718320Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"imputer = KNNImputer(n_neighbors=5)  # k=4\n# # test 1 0.439\n# imputed_data = imputer.fit_transform(df[featureCols])\n# train = pd.DataFrame(imputed_data, columns=featureCols)\n# # train['sii'] = df['sii']\n# test_imputed = imputer.fit_transform(test[featureCols])\n# test = pd.DataFrame(test_imputed, columns=featureCols)\n# test\n# test 2\ndata = pd.concat([train, test], axis=0, ignore_index=True)\nimputed_data = imputer.fit_transform(data[featureCols])\ndata = pd.DataFrame(imputed_data, columns=featureCols)\ndata\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T09:35:10.720336Z","iopub.execute_input":"2024-12-12T09:35:10.720741Z","iopub.status.idle":"2024-12-12T09:35:12.488428Z","shell.execute_reply.started":"2024-12-12T09:35:10.720702Z","shell.execute_reply":"2024-12-12T09:35:12.487469Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.shape[0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T09:35:12.489616Z","iopub.execute_input":"2024-12-12T09:35:12.489945Z","iopub.status.idle":"2024-12-12T09:35:12.495850Z","shell.execute_reply.started":"2024-12-12T09:35:12.489918Z","shell.execute_reply":"2024-12-12T09:35:12.494894Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = data.head(train.shape[0])\ntest = data.drop(train.index, axis=0).reset_index()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T09:35:12.496819Z","iopub.execute_input":"2024-12-12T09:35:12.497126Z","iopub.status.idle":"2024-12-12T09:35:12.507243Z","shell.execute_reply.started":"2024-12-12T09:35:12.497092Z","shell.execute_reply":"2024-12-12T09:35:12.506431Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T09:35:12.510796Z","iopub.execute_input":"2024-12-12T09:35:12.511045Z","iopub.status.idle":"2024-12-12T09:35:12.540160Z","shell.execute_reply.started":"2024-12-12T09:35:12.511021Z","shell.execute_reply":"2024-12-12T09:35:12.539240Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T09:35:12.541248Z","iopub.execute_input":"2024-12-12T09:35:12.541566Z","iopub.status.idle":"2024-12-12T09:35:12.575268Z","shell.execute_reply.started":"2024-12-12T09:35:12.541527Z","shell.execute_reply":"2024-12-12T09:35:12.574358Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['sii'] = df['sii']\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T09:35:12.576294Z","iopub.execute_input":"2024-12-12T09:35:12.576615Z","iopub.status.idle":"2024-12-12T09:35:12.582412Z","shell.execute_reply.started":"2024-12-12T09:35:12.576590Z","shell.execute_reply":"2024-12-12T09:35:12.581522Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T09:35:12.583775Z","iopub.execute_input":"2024-12-12T09:35:12.584438Z","iopub.status.idle":"2024-12-12T09:35:12.623892Z","shell.execute_reply.started":"2024-12-12T09:35:12.584397Z","shell.execute_reply":"2024-12-12T09:35:12.623099Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Training function using QWK metric and threshold optimization","metadata":{}},{"cell_type":"code","source":"def extract_features(df):\n    # df[\"Feat_0\"] = df[\"Physical-BMI\"] * df[\"Basic_Demos-Age\"]\n    # df[\"Feat_1\"] = df[\"PreInt_EduHx-computerinternet_hoursday\"] * df[\"Basic_Demos-Age\"]\n    # df[\"Feat_2\"] = df[\"Physical-BMI\"] * df[\"PreInt_EduHx-computerinternet_hoursday\"]\n    # df[\"Feat_3\"] = df[\"BIA-BIA_Fat\"] / df[\"BIA-BIA_BMI\"]\n    # df[\"Feat_4\"] = df[\"BIA-BIA_FFMI\"] / df[\"BIA-BIA_Fat\"]\n    # df[\"Feat_5\"] = df[\"BIA-BIA_FMI\"] / df[\"BIA-BIA_Fat\"]\n    # df[\"Feat_6\"] = df[\"BIA-BIA_LST\"] / df[\"BIA-BIA_TBW\"]\n    # df[\"Feat_7\"] = df[\"BIA-BIA_Fat\"] * df[\"BIA-BIA_BMR\"]\n    # df[\"Feat_8\"] = df[\"BIA-BIA_Fat\"] * df[\"BIA-BIA_DEE\"]\n    # df[\"Feat_9\"] = df[\"BIA-BIA_BMR\"] / df[\"Physical-Weight\"]\n    # df[\"Feat_10\"] = df[\"BIA-BIA_DEE\"] / df[\"Physical-Weight\"]\n    # df[\"Feat_11\"] = df[\"BIA-BIA_SMM\"] / df[\"Physical-Height\"]\n    # df[\"Feat_12\"] = df[\"BIA-BIA_SMM\"] / df[\"BIA-BIA_FMI\"]\n    # df[\"Feat_13\"] = df[\"BIA-BIA_TBW\"] / df[\"Physical-Weight\"]\n    # df[\"Feat_14\"] = df[\"BIA-BIA_ICW\"] / df[\"BIA-BIA_TBW\"]\n    # df[\"Feat_15\"] = df[\"Physical-BMI\"] * df[\"Physical-HeartRate\"]\n    df[\"Feat_0\"] = df[\"Physical-Height\"] * df[\"PAQ_C-PAQ_C_Total\"]\n    df[\"Feat_1\"] = df[\"FGC-FGC_TL_Zone\"] * df[\"Physical-Height\"]\n    df[\"Feat_2\"] = df[\"PreInt_EduHx-computerinternet_hoursday\"] * df[\"BIA-BIA_Activity_Level_num\"]\n    # df[\"Feat_3\"] = df[\"Fitness_Endurance-Time_Sec\"] / df[\"PreInt_EduHx-computerinternet_hoursday\"]\n    df[\"Feat_4\"] = df[\"CGAS-CGAS_Score\"] / df[\"FGC-FGC_CU_Zone\"]\n    df[\"Feat_5\"] = df[\"Basic_Demos-Age\"] / df[\"FGC-FGC_SRR_Zone\"]\n    df[\"Feat_7\"] = df[\"PAQ_C-PAQ_C_Total\"] * df[\"BIA-BIA_Frame_num\"]\n    # df[\"Feat_9\"] = df[\"FGC-FGC_GSD\"] / df[\"SDS-SDS_Total_Raw\"]\n    # df[\"Feat_10\"] = df[\"PAQ_A-PAQ_A_Total\"] / df[\"PreInt_EduHx-computerinternet_hoursday\"]\n    df[\"Feat_11\"] = df[\"BIA-BIA_LDM\"] / df[\"PreInt_EduHx-computerinternet_hoursday\"]\n    df[\"Feat_14\"] = df[\"BIA-BIA_BMI\"] / df[\"SDS-SDS_Total_Raw\"]\n    df[\"Feat_15\"] = df[\"Physical-Height\"] * df[\"SDS-SDS_Total_T\"]\n    df[\"Feat_16\"] = df[\"Physical-Height\"] * df[\"Physical-Height\"]\n    df[\"Feat_17\"] = df[\"FGC-FGC_SRL_Zone\"] / df[\"Physical-Weight\"]\n    df[\"Feat_18\"] = df[\"Basic_Demos-Sex\"] * df[\"Basic_Demos-Sex\"]\n    # df[\"Feat_19\"] = df[\"FGC-FGC_GSND_Zone\"] / df[\"BIA-BIA_Fat\"]\n\n    return df\n\ntrain = extract_features(train)\ntest = extract_features(test)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T09:35:12.625779Z","iopub.execute_input":"2024-12-12T09:35:12.626315Z","iopub.status.idle":"2024-12-12T09:35:12.647605Z","shell.execute_reply.started":"2024-12-12T09:35:12.626277Z","shell.execute_reply":"2024-12-12T09:35:12.646629Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# qwk score\ndef quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\n# threshold rounder\ndef threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n# prediction evaluation using qwk function\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n    return -quadratic_weighted_kappa(y_true, rounded_p)\n\ndef TrainML(model_class, train, test_data, featureCols) -> list[float]:\n    X = train[featureCols]\n    y = train['sii']\n\n    # Identify rows with NaN values in X\n\n    # Fill NaN and infinite values in X and test_data\n    X = X.fillna(0)\n    X = X.replace([np.inf, -np.inf], 0)\n    # test_data = test_data.fillna(0)\n    test_data = test_data.replace([np.inf, -np.inf], 0)\n\n    scaler = StandardScaler()\n    scaler.fit(X)\n    X = pd.DataFrame(scaler.transform(X), columns=X.columns)\n    test_data = test_data[featureCols]\n    test_data = pd.DataFrame(scaler.transform(test_data), columns=test_data.columns)\n    # nan_mask = test_data.isnull().any(axis=1)\n    # nan_indices = nan_mask[nan_mask].index\n    # print(test_data)\n    # print(nan_indices)\n    n_splits = 5\n    random_state = 42\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=random_state)\n\n    oof_non_rounded = np.zeros(len(y), dtype=float)\n    oof_rounded = np.zeros(len(y), dtype=int)\n    test_preds = np.zeros((len(test_data), n_splits))\n\n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n        X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n        model = clone(model_class)\n        model.fit(X_train, y_train)\n\n        y_train_pred = model.predict(X_train)\n        y_val_pred = model.predict(X_val)\n\n        oof_non_rounded[test_idx] = y_val_pred\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n        oof_rounded[test_idx] = y_val_pred_rounded\n\n        test_preds[:, fold] = model.predict(test_data)\n\n    KappaOptimizer = minimize(evaluate_predictions,\n                              x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n                              method='Nelder-Mead')\n    assert KappaOptimizer.success, \"Optimization did not converge.\"\n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOptimizer.x)\n\n    tpm = test_preds.mean(axis=1)\n    tp_rounded = threshold_Rounder(tpm, KappaOptimizer.x)\n\n    # Inject NaN predictions for rows with NaN in the original train\n    predictions = np.array(tp_rounded.tolist())\n    print(predictions)\n    # for idx in nan_indices:\n    #     predictions[idx] = np.nan\n\n    return predictions.tolist()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T09:35:12.648887Z","iopub.execute_input":"2024-12-12T09:35:12.649210Z","iopub.status.idle":"2024-12-12T09:35:12.663864Z","shell.execute_reply.started":"2024-12-12T09:35:12.649182Z","shell.execute_reply":"2024-12-12T09:35:12.662695Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Parameters","metadata":{}},{"cell_type":"code","source":"LGBM_params = {\n    'learning_rate': 0.046,\n    'max_depth': 12,\n    'num_leaves': 478,\n    'min_data_in_leaf': 13,\n    'feature_fraction': 0.893,\n    'bagging_fraction': 0.784,\n    'bagging_freq': 4,\n    'lambda_l1': 10,  \n    'lambda_l2': 0.01\n}\n\nXGB_Params = {\n    'learning_rate': 0.05,\n    'max_depth': 6,\n    'n_estimators': 400,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 1,  \n    'reg_lambda': 5,  \n    'random_state': 42,\n    'tree_method': 'exact'\n}\n\n\nCatBoost_Params = {\n    'learning_rate': 0.05,\n    'depth': 6,\n    'iterations': 400,\n    'random_seed': 42,\n    'verbose': 0,\n    'l2_leaf_reg': 10  \n}\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T09:35:12.665032Z","iopub.execute_input":"2024-12-12T09:35:12.665658Z","iopub.status.idle":"2024-12-12T09:35:12.680830Z","shell.execute_reply.started":"2024-12-12T09:35:12.665620Z","shell.execute_reply":"2024-12-12T09:35:12.680096Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# !pip install --no-index --no-deps /kaggle/input/pytorchtransformer/tab_transformer_pytorch-0.3.0-py3-none-any.whl\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T09:35:12.681970Z","iopub.execute_input":"2024-12-12T09:35:12.682371Z","iopub.status.idle":"2024-12-12T09:35:12.696150Z","shell.execute_reply.started":"2024-12-12T09:35:12.682331Z","shell.execute_reply":"2024-12-12T09:35:12.695206Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from pytorch_tabnet.tab_model import TabNetRegressor\n# import torch\n# from sklearn.base import BaseEstimator, RegressorMixin\n# from sklearn.impute import SimpleImputer\n# from sklearn.model_selection import train_test_split\n# from pytorch_tabnet.callbacks import Callback\n# import os\n# import torch\n# from pytorch_tabnet.callbacks import Callback\n\n# class TabNetWrapper(BaseEstimator, RegressorMixin):\n#     def __init__(self, **kwargs):\n#         self.model = TabNetRegressor(**kwargs)\n#         self.kwargs = kwargs\n#         self.imputer = SimpleImputer(strategy='median')\n#         self.best_model_path = 'best_tabnet_model.pt'\n        \n#     def fit(self, X, y):\n#         # Handle missing values\n#         X_imputed = self.imputer.fit_transform(X)\n        \n#         if hasattr(y, 'values'):\n#             y = y.values\n            \n#         # Create internal validation set\n#         X_train, X_valid, y_train, y_valid = train_test_split(\n#             X_imputed, \n#             y, \n#             test_size=0.2,\n#             random_state=42\n#         )\n                \n#         # Train TabNet model\n#         history = self.model.fit(\n#             X_train=X_train,\n#             y_train=y_train.reshape(-1, 1),\n#             eval_set=[(X_valid, y_valid.reshape(-1, 1))],\n#             eval_name=['valid'],\n#             eval_metric=['mse'],\n#             max_epochs=500,\n#             patience=50,\n#             batch_size=1024,\n#             virtual_batch_size=128,\n#             num_workers=0,\n#             drop_last=False,\n#             callbacks=[\n#                 TabNetPretrainedModelCheckpoint(\n#                     filepath=self.best_model_path,\n#                     monitor='valid_mse',\n#                     mode='min',\n#                     save_best_only=True,\n#                     verbose=True\n#                 )\n#             ]\n#         )\n#                 # Load the best model\n#         if os.path.exists(self.best_model_path):\n#             self.model.load_model(self.best_model_path)\n#             os.remove(self.best_model_path)  # Remove temporary file\n        \n#         return self\n#         def predict(self, X):\n#             X_imputed = self.imputer.transform(X)\n#             return self.model.predict(X_imputed).flatten()\n    \n#     def __deepcopy__(self, memo):\n#         # Add deepcopy support for scikit-learn\n#         cls = self.__class__\n#         result = cls.__new__(cls)\n#         memo[id(self)] = result\n#         for k, v in self.__dict__.items():\n#             setattr(result, k, deepcopy(v, memo))\n#         return result\n\n# TabNet_Params = {\n#     'n_d': 64,              # Width of the decision prediction layer\n#     'n_a': 64,              # Width of the attention embedding for each step\n#     'n_steps': 5,           # Number of steps in the architecture\n#     'gamma': 1.5,           # Coefficient for feature selection regularization\n#     'n_independent': 2,     # Number of independent GLU layer in each GLU block\n#     'n_shared': 2,          # Number of shared GLU layer in each GLU block\n#     'lambda_sparse': 1e-4,  # Sparsity regularization\n#     'optimizer_fn': torch.optim.Adam,\n#     'optimizer_params': dict(lr=2e-2, weight_decay=1e-5),\n#     'mask_type': 'entmax',\n#     'scheduler_params': dict(mode=\"min\", patience=10, min_lr=1e-5, factor=0.5),\n#     'scheduler_fn': torch.optim.lr_scheduler.ReduceLROnPlateau,\n#     'verbose': 1,\n#     'device_name': 'cuda' if torch.cuda.is_available() else 'cpu'\n# }\n\n# class TabNetPretrainedModelCheckpoint(Callback):\n#     def __init__(self, filepath, monitor='val_loss', mode='min', \n#                  save_best_only=True, verbose=1):\n#         super().__init__()  # Initialize parent class\n#         self.filepath = filepath\n#         self.monitor = monitor\n#         self.mode = mode\n#         self.save_best_only = save_best_only\n#         self.verbose = verbose\n#         self.best = float('inf') if mode == 'min' else -float('inf')\n        \n#     def on_train_begin(self, logs=None):\n#         self.model = self.trainer  # Use trainer itself as model\n        \n#     def on_epoch_end(self, epoch, logs=None):\n#         logs = logs or {}\n#         current = logs.get(self.monitor)\n#         if current is None:\n#             return\n\n#     # Check if current metric is better than best\n#     if (self.mode == 'min' and current < self.best) or \\\n#        (self.mode == 'max' and current > self.best):\n#         if self.verbose:\n#             print(f'\\nEpoch {epoch}: {self.monitor} improved from {self.best:.4f} to {current:.4f}')\n#         self.best = current\n#         if self.save_best_only:\n#             self.model.save_model(self.filepath)  # Save the entire model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T09:35:12.697504Z","iopub.execute_input":"2024-12-12T09:35:12.697872Z","iopub.status.idle":"2024-12-12T09:35:12.710257Z","shell.execute_reply.started":"2024-12-12T09:35:12.697844Z","shell.execute_reply":"2024-12-12T09:35:12.709395Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model decleration","metadata":{}},{"cell_type":"code","source":"Light = LGBMRegressor(**LGBM_params, random_state=42, verbose=-1, n_estimators=300)\nXGB_Model = XGBRegressor(**XGB_Params)\nCatBoost_Model = CatBoostRegressor(**CatBoost_Params)\n\nvoting_model = VotingRegressor(estimators=[\n    ('lightgbm', Light),\n    ('xgboost', XGB_Model),\n    ('catboost', CatBoost_Model)],\n     weights=[0.3, 0.5, 0.2]\n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T09:35:12.711271Z","iopub.execute_input":"2024-12-12T09:35:12.711657Z","iopub.status.idle":"2024-12-12T09:35:12.727535Z","shell.execute_reply.started":"2024-12-12T09:35:12.711626Z","shell.execute_reply":"2024-12-12T09:35:12.726689Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# XGboost model","metadata":{}},{"cell_type":"code","source":"# xgb_preds = TrainML(model_class=XGB_Model, test_data=test)\n# sub = pd.DataFrame({\n    \n#     'id'   : test_ids,\n    \n#     'sii': xgb_preds\n# })\n# xgb_preds\n# sub","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T09:35:12.728693Z","iopub.execute_input":"2024-12-12T09:35:12.729445Z","iopub.status.idle":"2024-12-12T09:35:12.738607Z","shell.execute_reply.started":"2024-12-12T09:35:12.729406Z","shell.execute_reply":"2024-12-12T09:35:12.737832Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Light Gradient Boosting Machine model","metadata":{}},{"cell_type":"code","source":"# lgbm_preds = TrainML(model_class=Light, test_data=test)\n# sub = pd.DataFrame({\n    \n#     'id'   : test_ids,\n    \n#     'sii': lgbm_preds\n# })\n\n# sub","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T09:35:12.739532Z","iopub.execute_input":"2024-12-12T09:35:12.739823Z","iopub.status.idle":"2024-12-12T09:35:12.749479Z","shell.execute_reply.started":"2024-12-12T09:35:12.739796Z","shell.execute_reply":"2024-12-12T09:35:12.748741Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Catboost model","metadata":{}},{"cell_type":"code","source":"# cat_preds = TrainML(model_class=CatBoost_Model, test_data=test)\n# sub = pd.DataFrame({\n    \n#     'id'   : test_ids,\n    \n#     'sii': cat_preds\n# })\n\n# sub","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T09:35:12.750407Z","iopub.execute_input":"2024-12-12T09:35:12.750731Z","iopub.status.idle":"2024-12-12T09:35:12.762536Z","shell.execute_reply.started":"2024-12-12T09:35:12.750695Z","shell.execute_reply":"2024-12-12T09:35:12.761791Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<h1>Handle Time Series Data</h1>","metadata":{}},{"cell_type":"code","source":"def process_file(filename, dirname):\n    df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    df.drop('step', axis=1, inplace=True)\n    return df.describe().values.reshape(-1), filename.split('=')[1]\n\ndef load_time_series(dirname) -> pd.DataFrame:\n    ids = os.listdir(dirname)\n    \n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n    \n    stats, indexes = zip(*results)\n    \n    df = pd.DataFrame(stats, columns=[f\"stat_{i}\" for i in range(len(stats[0]))])\n    df['id'] = indexes\n    return df\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T09:35:12.763488Z","iopub.execute_input":"2024-12-12T09:35:12.763716Z","iopub.status.idle":"2024-12-12T09:35:12.776001Z","shell.execute_reply.started":"2024-12-12T09:35:12.763693Z","shell.execute_reply":"2024-12-12T09:35:12.775181Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_ts = load_time_series('/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet')\ntest_ts = load_time_series('/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet')\ntrain_ts","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T09:35:12.777121Z","iopub.execute_input":"2024-12-12T09:35:12.777425Z","iopub.status.idle":"2024-12-12T09:36:27.784705Z","shell.execute_reply.started":"2024-12-12T09:35:12.777400Z","shell.execute_reply":"2024-12-12T09:36:27.783827Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T09:36:27.785792Z","iopub.execute_input":"2024-12-12T09:36:27.786078Z","iopub.status.idle":"2024-12-12T09:36:27.813862Z","shell.execute_reply.started":"2024-12-12T09:36:27.786035Z","shell.execute_reply":"2024-12-12T09:36:27.813092Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_with_ts = pd.concat([train, train_ts], axis=1, join='inner')\ntest_with_ts = pd.concat([test, test_ts], axis=1, join='inner')\n\ntrain_with_ts","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T09:36:27.814969Z","iopub.execute_input":"2024-12-12T09:36:27.815675Z","iopub.status.idle":"2024-12-12T09:36:27.844267Z","shell.execute_reply.started":"2024-12-12T09:36:27.815625Z","shell.execute_reply":"2024-12-12T09:36:27.843529Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Voting regressor model - ensembler","metadata":{}},{"cell_type":"code","source":"# test_with_ts","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T09:36:27.845528Z","iopub.execute_input":"2024-12-12T09:36:27.845887Z","iopub.status.idle":"2024-12-12T09:36:27.849960Z","shell.execute_reply.started":"2024-12-12T09:36:27.845848Z","shell.execute_reply":"2024-12-12T09:36:27.849041Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # featureCols\n# normal_preds = TrainML(voting_model, train, test, featureCols)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T09:36:27.855027Z","iopub.execute_input":"2024-12-12T09:36:27.855374Z","iopub.status.idle":"2024-12-12T09:36:27.860968Z","shell.execute_reply.started":"2024-12-12T09:36:27.855346Z","shell.execute_reply":"2024-12-12T09:36:27.860230Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# normal_preds","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T09:36:27.861897Z","iopub.execute_input":"2024-12-12T09:36:27.862235Z","iopub.status.idle":"2024-12-12T09:36:27.871889Z","shell.execute_reply.started":"2024-12-12T09:36:27.862198Z","shell.execute_reply":"2024-12-12T09:36:27.871244Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def TrainML(model_class, train, test_data, featureCols) -> list[int]:\nnots_featureCols = sorted(list(set(train.columns) - set(['sii', 'id'])))\n# featureCols\nnots_preds = TrainML(voting_model, train, test, nots_featureCols)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T09:36:27.872724Z","iopub.execute_input":"2024-12-12T09:36:27.873030Z","iopub.status.idle":"2024-12-12T09:36:52.977496Z","shell.execute_reply.started":"2024-12-12T09:36:27.872989Z","shell.execute_reply":"2024-12-12T09:36:52.976477Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"nots_sub = pd.DataFrame({\n    \n    'id'   : test_ids,\n    'sii': nots_preds\n})\n\n\n# nots_sub.to_csv('submission.csv', index=False)\nnots_sub","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T09:36:52.978798Z","iopub.execute_input":"2024-12-12T09:36:52.979455Z","iopub.status.idle":"2024-12-12T09:36:52.988854Z","shell.execute_reply.started":"2024-12-12T09:36:52.979412Z","shell.execute_reply":"2024-12-12T09:36:52.987941Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ts_sub = pd.DataFrame({\n    \n#     'id'   : test_ids,\n#     'sii': ts_preds\n# })\n\n\n# # ts_sub.to_csv('submission.csv', index=False)\n# ts_sub","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T09:36:52.990183Z","iopub.execute_input":"2024-12-12T09:36:52.990531Z","iopub.status.idle":"2024-12-12T09:36:53.439730Z","shell.execute_reply.started":"2024-12-12T09:36:52.990493Z","shell.execute_reply":"2024-12-12T09:36:53.438470Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def TrainML(model_class, train, test_data, featureCols) -> list[int]:\nts_featureCols = sorted(list(set(train_with_ts.columns) - set(['sii', 'id'])))\n# featureCols\nts_preds = TrainML(voting_model, train_with_ts, test_with_ts, ts_featureCols)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T09:37:16.744629Z","iopub.execute_input":"2024-12-12T09:37:16.745529Z","iopub.status.idle":"2024-12-12T09:37:59.167114Z","shell.execute_reply.started":"2024-12-12T09:37:16.745494Z","shell.execute_reply":"2024-12-12T09:37:59.166098Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# nots_preds","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T09:36:53.442283Z","iopub.status.idle":"2024-12-12T09:36:53.442746Z","shell.execute_reply.started":"2024-12-12T09:36:53.442511Z","shell.execute_reply":"2024-12-12T09:36:53.442534Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# vote_preds = TrainML(model_class=voting_model, test_data=test)\nts_sub = pd.DataFrame({\n    \n    'id'   : test_with_ts['id'],\n    'sii': ts_preds\n})\n\n\nts_sub.to_csv('submission.csv', index=False)\nts_sub","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T09:38:06.132561Z","iopub.execute_input":"2024-12-12T09:38:06.133262Z","iopub.status.idle":"2024-12-12T09:38:06.144491Z","shell.execute_reply.started":"2024-12-12T09:38:06.133228Z","shell.execute_reply":"2024-12-12T09:38:06.143666Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub1 = ts_sub.sort_values(by='id').reset_index(drop=True)\nsub2 = nots_sub.sort_values(by='id').reset_index(drop=True)\n# sub3 = sub3.sort_values(by='id').reset_index(drop=True)\n\ncombined = pd.DataFrame({\n    'id': sub2['id'],\n    'sii_1': sub1['sii'],\n    'sii_2': sub2['sii'],\n    # 'sii_3': sub3['sii']\n})\ndef majority_vote(df):\n    \"\"\"\n    This function checks the 'sii_1' column in the DataFrame.\n    If 'sii_1' is not NaN, it uses its value; otherwise, it defaults to 'sii_2'.\n\n    Parameters:\n        df (pd.DataFrame): Input DataFrame with 'id', 'sii_1', and 'sii_2'.\n\n    Returns:\n        pd.DataFrame: DataFrame containing 'id' and 'sii' (final values based on the logic).\n    \"\"\"\n    df['sii'] = df['sii_1'].combine_first(df['sii_2'])\n    return df[['id', 'sii']]\nfinal_sub = majority_vote(combined)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T09:38:07.093384Z","iopub.execute_input":"2024-12-12T09:38:07.093734Z","iopub.status.idle":"2024-12-12T09:38:07.106079Z","shell.execute_reply.started":"2024-12-12T09:38:07.093702Z","shell.execute_reply":"2024-12-12T09:38:07.105145Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_sub.to_csv('submission.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T09:38:12.068528Z","iopub.execute_input":"2024-12-12T09:38:12.069390Z","iopub.status.idle":"2024-12-12T09:38:12.074332Z","shell.execute_reply.started":"2024-12-12T09:38:12.069356Z","shell.execute_reply":"2024-12-12T09:38:12.073504Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_sub","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T09:38:17.019949Z","iopub.execute_input":"2024-12-12T09:38:17.020856Z","iopub.status.idle":"2024-12-12T09:38:17.031589Z","shell.execute_reply.started":"2024-12-12T09:38:17.020820Z","shell.execute_reply":"2024-12-12T09:38:17.030452Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}