{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.14"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"},{"sourceId":10161923,"sourceType":"datasetVersion","datasetId":6275040},{"sourceId":10162319,"sourceType":"datasetVersion","datasetId":6275333}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"\n**Adding FT-Transformer**","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport re\nfrom sklearn.base import clone\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.model_selection import StratifiedKFold\nfrom scipy.optimize import minimize\nfrom concurrent.futures import ThreadPoolExecutor\nfrom tqdm import tqdm\n\nfrom sklearn.preprocessing import StandardScaler\nimport matplotlib.pyplot as plt\nfrom keras.models import Model\nfrom keras.layers import Input, Dense\nfrom keras.optimizers import Adam\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, Dataset\n\nfrom colorama import Fore, Style\nfrom IPython.display import clear_output\nimport warnings\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.ensemble import VotingRegressor, RandomForestRegressor, GradientBoostingRegressor\nfrom sklearn.impute import SimpleImputer, KNNImputer\nfrom sklearn.pipeline import Pipeline\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None\n\nSEED = 42\nn_splits = 5","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2024-12-15T17:14:35.967802Z","iopub.status.busy":"2024-12-15T17:14:35.967446Z","iopub.status.idle":"2024-12-15T17:14:53.983921Z","shell.execute_reply":"2024-12-15T17:14:53.983103Z","shell.execute_reply.started":"2024-12-15T17:14:35.967767Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"featuresCols_1 = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-Season', 'CGAS-CGAS_Score', 'Physical-Season', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Season', 'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-Season', 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone', 'BIA-Season',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-Season', 'PAQ_A-PAQ_A_Total', 'PAQ_C-Season',\n                'PAQ_C-PAQ_C_Total', 'SDS-Season', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T', 'PreInt_EduHx-Season',\n                'PreInt_EduHx-computerinternet_hoursday', 'sii']","metadata":{"execution":{"iopub.execute_input":"2024-12-15T17:14:53.986735Z","iopub.status.busy":"2024-12-15T17:14:53.985741Z","iopub.status.idle":"2024-12-15T17:14:53.991972Z","shell.execute_reply":"2024-12-15T17:14:53.991033Z","shell.execute_reply.started":"2024-12-15T17:14:53.986691Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def process_file(filename, dirname):\n    df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    df.drop('step', axis=1, inplace=True)\n    \n    return df.describe().values.reshape(-1), filename.split('=')[1]","metadata":{"execution":{"iopub.execute_input":"2024-12-15T17:14:53.993425Z","iopub.status.busy":"2024-12-15T17:14:53.993055Z","iopub.status.idle":"2024-12-15T17:14:54.004712Z","shell.execute_reply":"2024-12-15T17:14:54.003864Z","shell.execute_reply.started":"2024-12-15T17:14:53.993387Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_time_series(dirname):\n    ids = os.listdir(dirname)\n    \n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n    \n    stats, indexes = zip(*results)\n    df = pd.DataFrame(stats, columns=[f\"stat_{i}\" for i in range(len(stats[0]))])\n    df['id'] = indexes\n    \n    return df","metadata":{"execution":{"iopub.execute_input":"2024-12-15T17:14:54.00713Z","iopub.status.busy":"2024-12-15T17:14:54.00682Z","iopub.status.idle":"2024-12-15T17:14:54.014279Z","shell.execute_reply":"2024-12-15T17:14:54.013458Z","shell.execute_reply.started":"2024-12-15T17:14:54.007095Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_ts  = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")","metadata":{"execution":{"iopub.execute_input":"2024-12-15T17:14:54.01553Z","iopub.status.busy":"2024-12-15T17:14:54.015264Z","iopub.status.idle":"2024-12-15T17:16:06.002395Z","shell.execute_reply":"2024-12-15T17:16:06.001598Z","shell.execute_reply.started":"2024-12-15T17:14:54.015506Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"display(train_ts.head())","metadata":{"execution":{"iopub.execute_input":"2024-12-15T17:16:06.00371Z","iopub.status.busy":"2024-12-15T17:16:06.003428Z","iopub.status.idle":"2024-12-15T17:16:06.073299Z","shell.execute_reply":"2024-12-15T17:16:06.072515Z","shell.execute_reply.started":"2024-12-15T17:16:06.003682Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"time_series_cols = train_ts.columns.tolist()\ntime_series_cols.remove(\"id\")","metadata":{"execution":{"iopub.execute_input":"2024-12-15T17:16:06.074989Z","iopub.status.busy":"2024-12-15T17:16:06.074531Z","iopub.status.idle":"2024-12-15T17:16:06.079333Z","shell.execute_reply":"2024-12-15T17:16:06.078367Z","shell.execute_reply.started":"2024-12-15T17:16:06.074931Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train  = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest   = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\nprint(\"Train data shape: \", train.shape)\nprint(\"Test data shape: \", test.shape)\ndisplay(train.head())","metadata":{"execution":{"iopub.execute_input":"2024-12-15T17:16:06.080747Z","iopub.status.busy":"2024-12-15T17:16:06.080401Z","iopub.status.idle":"2024-12-15T17:16:06.198196Z","shell.execute_reply":"2024-12-15T17:16:06.197367Z","shell.execute_reply.started":"2024-12-15T17:16:06.080719Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print('Columns missing in test:')\nprint([f for f in train.columns if f not in test.columns])","metadata":{"execution":{"iopub.execute_input":"2024-12-15T17:16:06.199768Z","iopub.status.busy":"2024-12-15T17:16:06.199365Z","iopub.status.idle":"2024-12-15T17:16:06.204857Z","shell.execute_reply":"2024-12-15T17:16:06.204029Z","shell.execute_reply.started":"2024-12-15T17:16:06.199727Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.merge(train, train_ts, how=\"left\", on='id')\ntest = pd.merge(test, test_ts, how=\"left\", on='id')","metadata":{"execution":{"iopub.execute_input":"2024-12-15T17:16:06.208427Z","iopub.status.busy":"2024-12-15T17:16:06.207962Z","iopub.status.idle":"2024-12-15T17:16:06.232315Z","shell.execute_reply":"2024-12-15T17:16:06.231523Z","shell.execute_reply.started":"2024-12-15T17:16:06.208398Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = train.drop('id', axis=1)\ntest  = test .drop('id', axis=1) ","metadata":{"execution":{"iopub.execute_input":"2024-12-15T17:16:06.233655Z","iopub.status.busy":"2024-12-15T17:16:06.233319Z","iopub.status.idle":"2024-12-15T17:16:06.242139Z","shell.execute_reply":"2024-12-15T17:16:06.241295Z","shell.execute_reply.started":"2024-12-15T17:16:06.233616Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"featuresCols = featuresCols_1 + time_series_cols\ntrain = train[featuresCols]","metadata":{"execution":{"iopub.execute_input":"2024-12-15T17:16:06.243554Z","iopub.status.busy":"2024-12-15T17:16:06.24322Z","iopub.status.idle":"2024-12-15T17:16:06.253433Z","shell.execute_reply":"2024-12-15T17:16:06.252606Z","shell.execute_reply.started":"2024-12-15T17:16:06.243525Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = train.dropna(subset='sii')","metadata":{"execution":{"iopub.execute_input":"2024-12-15T17:16:06.254708Z","iopub.status.busy":"2024-12-15T17:16:06.254437Z","iopub.status.idle":"2024-12-15T17:16:06.262842Z","shell.execute_reply":"2024-12-15T17:16:06.262077Z","shell.execute_reply.started":"2024-12-15T17:16:06.254681Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.execute_input":"2024-12-15T17:16:06.26439Z","iopub.status.busy":"2024-12-15T17:16:06.264016Z","iopub.status.idle":"2024-12-15T17:16:06.38579Z","shell.execute_reply":"2024-12-15T17:16:06.38505Z","shell.execute_reply.started":"2024-12-15T17:16:06.264362Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cat_c = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', 'Fitness_Endurance-Season', \n          'FGC-Season', 'BIA-Season', 'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season']\n\nlen(cat_c)","metadata":{"execution":{"iopub.execute_input":"2024-12-15T17:16:06.387111Z","iopub.status.busy":"2024-12-15T17:16:06.386883Z","iopub.status.idle":"2024-12-15T17:16:06.392592Z","shell.execute_reply":"2024-12-15T17:16:06.391641Z","shell.execute_reply.started":"2024-12-15T17:16:06.387087Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def update(df):\n    for c in cat_c: \n        df[c] = df[c].fillna('Missing')\n        df[c] = df[c].astype('category')\n    return df","metadata":{"execution":{"iopub.execute_input":"2024-12-15T17:16:06.393808Z","iopub.status.busy":"2024-12-15T17:16:06.393533Z","iopub.status.idle":"2024-12-15T17:16:06.400409Z","shell.execute_reply":"2024-12-15T17:16:06.399821Z","shell.execute_reply.started":"2024-12-15T17:16:06.393783Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train[cat_c].info()","metadata":{"execution":{"iopub.execute_input":"2024-12-15T17:16:06.401688Z","iopub.status.busy":"2024-12-15T17:16:06.40143Z","iopub.status.idle":"2024-12-15T17:16:06.424877Z","shell.execute_reply":"2024-12-15T17:16:06.424077Z","shell.execute_reply.started":"2024-12-15T17:16:06.401663Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = update(train)\ntest = update(test)\n\ntrain.shape, test.shape","metadata":{"execution":{"iopub.execute_input":"2024-12-15T17:16:06.426023Z","iopub.status.busy":"2024-12-15T17:16:06.425767Z","iopub.status.idle":"2024-12-15T17:16:06.450885Z","shell.execute_reply":"2024-12-15T17:16:06.450216Z","shell.execute_reply.started":"2024-12-15T17:16:06.425997Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_mapping(column, dataset):\n    unique_values = dataset[column].unique()\n    return {value: idx for idx, value in enumerate(unique_values)}","metadata":{"execution":{"iopub.execute_input":"2024-12-15T17:16:06.45192Z","iopub.status.busy":"2024-12-15T17:16:06.451687Z","iopub.status.idle":"2024-12-15T17:16:06.456232Z","shell.execute_reply":"2024-12-15T17:16:06.455141Z","shell.execute_reply.started":"2024-12-15T17:16:06.451896Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for col in cat_c:\n    mapping_train = create_mapping(col, train)\n    mapping_test = create_mapping(col, test)\n    \n    train[col] = train[col].replace(mapping_train).astype(int)\n    test[col] = test[col].replace(mapping_test).astype(int)\n\nprint(f'Train Shape : {train.shape} || Test Shape : {test.shape}')","metadata":{"execution":{"iopub.execute_input":"2024-12-15T17:16:06.457916Z","iopub.status.busy":"2024-12-15T17:16:06.457274Z","iopub.status.idle":"2024-12-15T17:16:06.498921Z","shell.execute_reply":"2024-12-15T17:16:06.49808Z","shell.execute_reply.started":"2024-12-15T17:16:06.457887Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n","metadata":{"execution":{"iopub.execute_input":"2024-12-15T17:16:06.500077Z","iopub.status.busy":"2024-12-15T17:16:06.499838Z","iopub.status.idle":"2024-12-15T17:16:06.503998Z","shell.execute_reply":"2024-12-15T17:16:06.503112Z","shell.execute_reply.started":"2024-12-15T17:16:06.500053Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))","metadata":{"execution":{"iopub.execute_input":"2024-12-15T17:16:06.505867Z","iopub.status.busy":"2024-12-15T17:16:06.505224Z","iopub.status.idle":"2024-12-15T17:16:06.511719Z","shell.execute_reply":"2024-12-15T17:16:06.511018Z","shell.execute_reply.started":"2024-12-15T17:16:06.505827Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def evaluate_predictions(thresholds, y_true, oof_non_rounded):\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n    return -quadratic_weighted_kappa(y_true, rounded_p)","metadata":{"execution":{"iopub.execute_input":"2024-12-15T17:16:06.513651Z","iopub.status.busy":"2024-12-15T17:16:06.512872Z","iopub.status.idle":"2024-12-15T17:16:06.523327Z","shell.execute_reply":"2024-12-15T17:16:06.52262Z","shell.execute_reply.started":"2024-12-15T17:16:06.513616Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def TrainML(model_class, test_data):\n    X = train.drop(['sii'], axis=1)\n    y = train['sii']\n\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n    \n    train_S = []\n    test_S = []\n    \n    oof_non_rounded = np.zeros(len(y), dtype=float) \n    oof_rounded = np.zeros(len(y), dtype=int) \n    test_preds = np.zeros((len(test_data), n_splits))\n\n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n        X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n\n        model = clone(model_class)\n        model.fit(X_train, y_train)\n\n        y_train_pred = model.predict(X_train)\n        y_val_pred = model.predict(X_val)\n\n        oof_non_rounded[test_idx] = y_val_pred\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n        oof_rounded[test_idx] = y_val_pred_rounded\n\n        train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n        val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n        train_S.append(train_kappa)\n        test_S.append(val_kappa)\n        \n        test_preds[:, fold] = model.predict(test_data)\n        \n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n        clear_output(wait=True)\n\n    print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n    KappaOPtimizer = minimize(evaluate_predictions,\n                              x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n                              method='Nelder-Mead')\n    assert KappaOPtimizer.success, \"Optimization did not converge.\"\n    \n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n    tpm = test_preds.mean(axis=1)\n    tpTuned = threshold_Rounder(tpm, KappaOPtimizer.x)\n    \n    submission = pd.DataFrame({\n        'id': sample['id'],\n        'sii': tpTuned\n    })\n\n    return submission","metadata":{"execution":{"iopub.execute_input":"2024-12-15T17:16:06.524861Z","iopub.status.busy":"2024-12-15T17:16:06.524531Z","iopub.status.idle":"2024-12-15T17:16:06.53532Z","shell.execute_reply":"2024-12-15T17:16:06.534483Z","shell.execute_reply.started":"2024-12-15T17:16:06.524825Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Model parameters for LightGBM\nParams = {\n    'learning_rate': 0.046,\n    'max_depth': 12,\n    'num_leaves': 478,\n    'min_data_in_leaf': 13,\n    'feature_fraction': 0.893,\n    'bagging_fraction': 0.784,\n    'bagging_freq': 4,\n    'lambda_l1': 10,  # Increased from 6.59\n    'lambda_l2': 0.01  # Increased from 2.68e-06\n}\n\n\n# XGBoost parameters\nXGB_Params = {\n    'learning_rate': 0.05,\n    'max_depth': 6,\n    'n_estimators': 200,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 1,  # Increased from 0.1\n    'reg_lambda': 5,  # Increased from 1\n    'random_state': SEED\n}\n\n\nCatBoost_Params = {\n    'learning_rate': 0.05,\n    'depth': 6,\n    'iterations': 200,\n    'random_seed': SEED,\n    'cat_features': cat_c,\n    'verbose': 0,\n    'l2_leaf_reg': 10  # Increase this value\n}","metadata":{"execution":{"iopub.execute_input":"2024-12-15T17:16:06.536537Z","iopub.status.busy":"2024-12-15T17:16:06.536274Z","iopub.status.idle":"2024-12-15T17:16:06.549767Z","shell.execute_reply":"2024-12-15T17:16:06.54896Z","shell.execute_reply.started":"2024-12-15T17:16:06.536512Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Light = LGBMRegressor(**Params, random_state=SEED, verbose=-1, n_estimators=300)\nXGB_Model = XGBRegressor(**XGB_Params)\nCatBoost_Model = CatBoostRegressor(**CatBoost_Params)\n\nvoting_model = VotingRegressor(estimators=[\n    ('lightgbm', Light),\n    ('xgboost', XGB_Model),\n    ('catboost', CatBoost_Model)\n])","metadata":{"execution":{"iopub.execute_input":"2024-12-15T17:16:06.551523Z","iopub.status.busy":"2024-12-15T17:16:06.550718Z","iopub.status.idle":"2024-12-15T17:16:06.560855Z","shell.execute_reply":"2024-12-15T17:16:06.560063Z","shell.execute_reply.started":"2024-12-15T17:16:06.551484Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Submission1 = TrainML(voting_model, test)\n\nSubmission1 ","metadata":{"execution":{"iopub.execute_input":"2024-12-15T17:16:06.562006Z","iopub.status.busy":"2024-12-15T17:16:06.561772Z","iopub.status.idle":"2024-12-15T17:16:50.515947Z","shell.execute_reply":"2024-12-15T17:16:50.515109Z","shell.execute_reply.started":"2024-12-15T17:16:06.561983Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Model 2**\n\nThis models introduce Simple Imputer for handling missing datas","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\nfeaturesCols_2 = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-Season', 'CGAS-CGAS_Score', 'Physical-Season', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Season', 'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-Season', 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone', 'BIA-Season',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-Season', 'PAQ_A-PAQ_A_Total', 'PAQ_C-Season',\n                'PAQ_C-PAQ_C_Total', 'SDS-Season', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T', 'PreInt_EduHx-Season',\n                'PreInt_EduHx-computerinternet_hoursday', 'sii']\n\ncat_c = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', \n          'Fitness_Endurance-Season', 'FGC-Season', 'BIA-Season', \n          'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season']\n\ntrain_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\n\ntime_series_cols = train_ts.columns.tolist()\ntime_series_cols.remove(\"id\")\n\ntrain = pd.merge(train, train_ts, how=\"left\", on='id')\ntest = pd.merge(test, test_ts, how=\"left\", on='id')\n\ntrain = train.drop('id', axis=1)\ntest = test.drop('id', axis=1)\n\nfeaturesCols = featuresCols_2 + time_series_cols\n\ntrain = train[featuresCols]\ntrain = train.dropna(subset='sii')\n\ndef update(df):\n    global cat_c\n    for c in cat_c: \n        df[c] = df[c].fillna('Missing')\n        df[c] = df[c].astype('category')\n    return df\n\ntrain = update(train)\ntest = update(test)\n\ndef create_mapping(column, dataset):\n    unique_values = dataset[column].unique()\n    return {value: idx for idx, value in enumerate(unique_values)}\n\nfor col in cat_c:\n    mapping = create_mapping(col, train)\n    mappingTe = create_mapping(col, test)\n    \n    train[col] = train[col].replace(mapping).astype(int)\n    test[col] = test[col].replace(mappingTe).astype(int)\n\ndef quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n    return -quadratic_weighted_kappa(y_true, rounded_p)\n\ndef TrainML(model_class, test_data):\n    X = train.drop(['sii'], axis=1)\n    y = train['sii']\n\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n    \n    train_S = []\n    test_S = []\n    \n    oof_non_rounded = np.zeros(len(y), dtype=float) \n    oof_rounded = np.zeros(len(y), dtype=int) \n    test_preds = np.zeros((len(test_data), n_splits))\n\n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n        X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n\n        model = clone(model_class)\n        model.fit(X_train, y_train)\n\n        y_train_pred = model.predict(X_train)\n        y_val_pred = model.predict(X_val)\n\n        oof_non_rounded[test_idx] = y_val_pred\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n        oof_rounded[test_idx] = y_val_pred_rounded\n\n        train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n        val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n        train_S.append(train_kappa)\n        test_S.append(val_kappa)\n        \n        test_preds[:, fold] = model.predict(test_data)\n        \n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n        clear_output(wait=True)\n\n    print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n    KappaOPtimizer = minimize(evaluate_predictions,\n                              x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n                              method='Nelder-Mead')\n    assert KappaOPtimizer.success, \"Optimization did not converge.\"\n    \n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n    tpm = test_preds.mean(axis=1)\n    tpTuned = threshold_Rounder(tpm, KappaOPtimizer.x)\n    \n    submission = pd.DataFrame({\n        'id': sample['id'],\n        'sii': tpTuned\n    })\n\n    return submission\n\nimputer = SimpleImputer(strategy='median')\n\nensemble = VotingRegressor(estimators=[\n    ('lgb', Pipeline(steps=[('imputer', imputer), ('regressor', LGBMRegressor(random_state=SEED))])),\n    ('xgb', Pipeline(steps=[('imputer', imputer), ('regressor', XGBRegressor(random_state=SEED))])),\n    ('cat', Pipeline(steps=[('imputer', imputer), ('regressor', CatBoostRegressor(random_state=SEED, silent=True))])),\n    ('rf', Pipeline(steps=[('imputer', imputer), ('regressor', RandomForestRegressor(random_state=SEED))])),\n    ('gb', Pipeline(steps=[('imputer', imputer), ('regressor', GradientBoostingRegressor(random_state=SEED))]))\n])\n\nSubmission2 = TrainML(ensemble, test)\n\nSubmission2","metadata":{"execution":{"iopub.execute_input":"2024-12-15T17:16:50.519818Z","iopub.status.busy":"2024-12-15T17:16:50.519516Z","iopub.status.idle":"2024-12-15T17:19:57.71015Z","shell.execute_reply":"2024-12-15T17:19:57.709252Z","shell.execute_reply.started":"2024-12-15T17:16:50.519791Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Model 3**\n\n* This models use AutoEncoder to reduce dimension of ts dataset\n* This models use KNNImputer to fill missing data\n* This models use FT-Transformer\n","metadata":{}},{"cell_type":"code","source":"class AutoEncoder(nn.Module):\n    def __init__(self, input_dim, encoding_dim):\n        super(AutoEncoder, self).__init__()\n        self.encoder = nn.Sequential(\n            nn.Linear(input_dim, encoding_dim*3),\n            nn.ReLU(),\n            nn.Linear(encoding_dim*3, encoding_dim*2),\n            nn.ReLU(),\n            nn.Linear(encoding_dim*2, encoding_dim),\n            nn.ReLU()\n        )\n        self.decoder = nn.Sequential(\n            nn.Linear(encoding_dim, input_dim*2),\n            nn.ReLU(),\n            nn.Linear(input_dim*2, input_dim*3),\n            nn.ReLU(),\n            nn.Linear(input_dim*3, input_dim),\n            nn.Sigmoid()\n        )\n        \n    def forward(self, x):\n        encoded = self.encoder(x)\n        decoded = self.decoder(encoded)\n        return decoded\n\n\ndef perform_autoencoder(df, encoding_dim=50, epochs=50, batch_size=32):\n    scaler = StandardScaler()\n    df_scaled = scaler.fit_transform(df)\n    \n    data_tensor = torch.FloatTensor(df_scaled)\n    \n    input_dim = data_tensor.shape[1]\n    autoencoder = AutoEncoder(input_dim, encoding_dim)\n    \n    criterion = nn.MSELoss()\n    optimizer = optim.Adam(autoencoder.parameters())\n    \n    for epoch in range(epochs):\n        for i in range(0, len(data_tensor), batch_size):\n            batch = data_tensor[i : i + batch_size]\n            optimizer.zero_grad()\n            reconstructed = autoencoder(batch)\n            loss = criterion(reconstructed, batch)\n            loss.backward()\n            optimizer.step()\n            \n        if (epoch + 1) % 10 == 0:\n            print(f'Epoch [{epoch + 1}/{epochs}], Loss: {loss.item():.4f}]')\n                 \n    with torch.no_grad():\n        encoded_data = autoencoder.encoder(data_tensor).numpy()\n        \n    df_encoded = pd.DataFrame(encoded_data, columns=[f'Enc_{i + 1}' for i in range(encoded_data.shape[1])])\n    \n    return df_encoded\n","metadata":{"execution":{"iopub.execute_input":"2024-12-15T17:19:57.711651Z","iopub.status.busy":"2024-12-15T17:19:57.711334Z","iopub.status.idle":"2024-12-15T17:19:57.725121Z","shell.execute_reply":"2024-12-15T17:19:57.724247Z","shell.execute_reply.started":"2024-12-15T17:19:57.711621Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## FT-Transformer ","metadata":{}},{"cell_type":"code","source":"!pip install --no-index --no-deps /kaggle/input/pytorch/einops-0.8.0-py3-none-any.whl","metadata":{"execution":{"iopub.execute_input":"2024-12-15T17:19:57.726696Z","iopub.status.busy":"2024-12-15T17:19:57.726311Z","iopub.status.idle":"2024-12-15T17:20:00.047587Z","shell.execute_reply":"2024-12-15T17:20:00.046723Z","shell.execute_reply.started":"2024-12-15T17:19:57.726657Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install --no-index --no-deps /kaggle/input/pytorchtransformer/tab_transformer_pytorch-0.3.0-py3-none-any.whl","metadata":{"execution":{"iopub.execute_input":"2024-12-15T17:20:00.049164Z","iopub.status.busy":"2024-12-15T17:20:00.048879Z","iopub.status.idle":"2024-12-15T17:20:01.89429Z","shell.execute_reply":"2024-12-15T17:20:01.893234Z","shell.execute_reply.started":"2024-12-15T17:20:00.049136Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tab_transformer_pytorch import FTTransformer\nfrom sklearn.base import BaseEstimator, RegressorMixin\nfrom sklearn.utils.validation import check_X_y, check_array\nfrom sklearn.model_selection import train_test_split\nfrom torch.utils.data import DataLoader, TensorDataset","metadata":{"execution":{"iopub.execute_input":"2024-12-15T17:20:01.89603Z","iopub.status.busy":"2024-12-15T17:20:01.895766Z","iopub.status.idle":"2024-12-15T17:20:01.910237Z","shell.execute_reply":"2024-12-15T17:20:01.909614Z","shell.execute_reply.started":"2024-12-15T17:20:01.896004Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class FTTransformerWrapper(BaseEstimator, RegressorMixin):\n    def __init__(self, \n                 dim=32, \n                 depth=6, \n                 heads=8, \n                 attn_dropout=0.1, \n                 ff_dropout=0.1,\n                 max_epochs=10,\n                 patience=10,\n                 lr=1e-4, \n                 batch_size=64):\n        # Model hyperparameters\n        self.dim = dim\n        self.depth = depth\n        self.heads = heads\n        self.attn_dropout = attn_dropout\n        self.ff_dropout = ff_dropout\n        \n        # Training parameters\n        self.max_epochs = max_epochs\n        self.patience = patience\n        self.lr = lr\n        self.batch_size = batch_size\n        \n        # Additional setup\n        self.imputer = SimpleImputer(strategy='median')\n        \n        # Device selection\n        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n        \n        # Model placeholders\n        self.model = None\n        self._categorical_columns = None\n        self._continuous_columns = None\n\n    def _clear_cuda_memory(self):\n        \"\"\"Clear CUDA memory safely\"\"\"\n        if torch.cuda.is_available():\n            torch.cuda.empty_cache()\n            torch.cuda.synchronize()\n\n    def _prepare_data(self, X, y=None, is_train=True):\n        \"\"\"Prepare data for training or prediction\"\"\"\n        # Impute missing values\n        X_imputed = self.imputer.fit_transform(X) if is_train else self.imputer.transform(X)\n        \n        # Identify categorical and continuous columns\n        all_cols = X.columns.tolist()\n\n        # Categorical columns (update this list with your specific categorical features)\n        cat_cols = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', \n                    'Fitness_Endurance-Season', 'FGC-Season', 'BIA-Season', \n                    'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season']\n        \n        self._categorical_columns = [col for col in all_cols if col in cat_cols]\n        self._continuous_columns = [\n            c for c in all_cols \n            if (c not in self._categorical_columns) and (c != 'sii')\n        ]\n        \n        # Convert to tensors\n        X_categ = torch.tensor(\n            X_imputed[:, [all_cols.index(c) for c in self._categorical_columns]], \n            dtype=torch.long, \n            device=self.device\n        )\n        X_cont = torch.tensor(\n            X_imputed[:, [all_cols.index(c) for c in self._continuous_columns]], \n            dtype=torch.float32, \n            device=self.device\n        )\n        \n        # Process target if provided\n        if y is not None:\n            y = y.values if hasattr(y, 'values') else y\n            y_tensor = torch.tensor(y, dtype=torch.float32, device=self.device)\n            return X_categ, X_cont, y_tensor\n        \n        return X_categ, X_cont\n\n    def fit(self, X, y):\n        \"\"\"Fit the FTTransformer model\"\"\"\n        # Clear any existing CUDA memory\n        self._clear_cuda_memory()\n        \n        # Split data into train and validation sets\n        X_train, X_valid, y_train, y_valid = train_test_split(\n            X, y, test_size=0.2, random_state=42\n        )\n        \n        # Prepare data\n        X_train_categ, X_train_cont, y_train_t = self._prepare_data(X_train, y_train)\n        X_valid_categ, X_valid_cont, y_valid_t = self._prepare_data(X_valid, y_valid)\n        \n        # Determine categories for categorical features\n        categories = [X[col].nunique() for col in self._categorical_columns]\n        \n        # Initialize model\n        self.model = FTTransformer(\n            categories=categories,\n            num_continuous=len(self._continuous_columns),\n            dim=self.dim,\n            dim_out=1,\n            depth=self.depth,\n            heads=self.heads,\n            attn_dropout=self.attn_dropout,\n            ff_dropout=self.ff_dropout,\n        ).to(self.device)\n        \n        # Loss and optimizer\n        criterion = nn.MSELoss()\n        optimizer = optim.Adam(self.model.parameters(), lr=self.lr)\n        \n        # Learning rate scheduler\n        scheduler = optim.lr_scheduler.ReduceLROnPlateau(\n            optimizer, mode='min', patience=10, factor=0.5, min_lr=1e-5\n        )\n        \n        # Prepare DataLoaders\n        train_dataset = TensorDataset(X_train_categ, X_train_cont, y_train_t)\n        valid_dataset = TensorDataset(X_valid_categ, X_valid_cont, y_valid_t)\n        \n        train_loader = DataLoader(train_dataset, batch_size=self.batch_size, shuffle=True)\n        valid_loader = DataLoader(valid_dataset, batch_size=self.batch_size, shuffle=False)\n        \n        # Training loop\n        best_val_loss = float('inf')\n        epochs_no_improve = 0\n        \n        for epoch in range(self.max_epochs):\n            # Training phase\n            self.model.train()\n            train_loss = self._train_epoch(train_loader, criterion, optimizer)\n            \n            # Validation phase\n            self.model.eval()\n            val_loss = self._validate(valid_loader, criterion)\n            \n            # Learning rate scheduling\n            scheduler.step(val_loss)\n            \n            # Early stopping\n            if val_loss < best_val_loss:\n                best_val_loss = val_loss\n                epochs_no_improve = 0\n            else:\n                epochs_no_improve += 1\n            \n            # Print progress\n            print(f\"Epoch {epoch+1}/{self.max_epochs}, \"\n                  f\"Train Loss: {train_loss:.4f}, \"\n                  f\"Val Loss: {val_loss:.4f}\")\n            \n            # Periodic memory cleanup\n            if epoch % 10 == 0:\n                self._clear_cuda_memory()\n            \n            # Early stopping condition\n            if epochs_no_improve >= self.patience:\n                print(f\"Early stopping triggered after {epoch+1} epochs\")\n                break\n        \n        # Final memory cleanup\n        self._clear_cuda_memory()\n        \n        return self\n\n    def _train_epoch(self, train_loader, criterion, optimizer):\n        \"\"\"Train for a single epoch\"\"\"\n        total_loss = 0.0\n        \n        for batch_categ, batch_cont, batch_labels in train_loader:\n            optimizer.zero_grad()\n            outputs = self.model(batch_categ, batch_cont)\n            loss = criterion(outputs.squeeze(), batch_labels)\n            \n            loss.backward()\n            optimizer.step()\n            \n            total_loss += loss.item()\n            \n            # Clear batch memory\n            del batch_categ, batch_cont, batch_labels, outputs, loss\n            self._clear_cuda_memory()\n        \n        return total_loss / len(train_loader)\n\n    def _validate(self, valid_loader, criterion):\n        \"\"\"Validate the model\"\"\"\n        total_loss = 0.0\n        \n        with torch.no_grad():\n            for batch_categ, batch_cont, batch_labels in valid_loader:\n                outputs = self.model(batch_categ, batch_cont)\n                loss = criterion(outputs.squeeze(), batch_labels)\n                \n                total_loss += loss.item()\n                \n                # Clear batch memory\n                del batch_categ, batch_cont, batch_labels, outputs, loss\n                self._clear_cuda_memory()\n        \n        return total_loss / len(valid_loader)\n\n    def predict(self, X):\n        \"\"\"Make predictions with batch processing\"\"\"\n        if self.model is None:\n            raise ValueError(\"Model has not been fitted yet.\")\n        \n        # Prepare prediction dataset\n        X_categ, X_cont = self._prepare_data(X, is_train=False)\n        \n        # Create DataLoader for batch processing\n        pred_dataset = TensorDataset(X_categ, X_cont)\n        pred_loader = DataLoader(\n            pred_dataset, \n            batch_size=self.batch_size, \n            shuffle=False\n        )\n        \n        # Initialize prediction container\n        all_preds = []\n        \n        # Prediction in batches\n        self.model.eval()\n        with torch.no_grad():\n            for batch_categ, batch_cont in pred_loader:\n                batch_logits = self.model(batch_categ, batch_cont)\n                batch_preds = batch_logits.squeeze().cpu().numpy()\n                all_preds.append(batch_preds)\n                \n                # Clear batch tensors\n                del batch_categ, batch_cont, batch_logits\n                self._clear_cuda_memory()\n        \n        # Combine predictions\n        preds = np.concatenate(all_preds)\n        \n        # Post-processing\n        preds_cont = np.clip(np.round(preds), 0, 3).astype(int)\n        \n        # Final memory cleanup\n        del X_categ, X_cont, all_preds\n        self._clear_cuda_memory()\n        \n        return preds_cont","metadata":{"execution":{"iopub.execute_input":"2024-12-15T17:20:01.912092Z","iopub.status.busy":"2024-12-15T17:20:01.911762Z","iopub.status.idle":"2024-12-15T17:20:01.935551Z","shell.execute_reply":"2024-12-15T17:20:01.934764Z","shell.execute_reply.started":"2024-12-15T17:20:01.912051Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\nfeaturesCols_3 = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-Season', 'CGAS-CGAS_Score', 'Physical-Season', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Season', 'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-Season', 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone', 'BIA-Season',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-Season', 'PAQ_A-PAQ_A_Total', 'PAQ_C-Season',\n                'PAQ_C-PAQ_C_Total', 'SDS-Season', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T', 'PreInt_EduHx-Season',\n                'PreInt_EduHx-computerinternet_hoursday', 'sii']\n\ncat_c = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', \n          'Fitness_Endurance-Season', 'FGC-Season', 'BIA-Season', \n          'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season']\n\n\n\ntrain = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\n\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\n\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\n\ntrain_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\n\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\n\n\ndf_train = train_ts.drop('id', axis=1)\n\ndf_test = test_ts.drop('id', axis=1)\n\n\ntrain_ts_encoded = perform_autoencoder(df_train, encoding_dim=60, epochs=100, batch_size=32)\n\ntest_ts_encoded = perform_autoencoder(df_test, encoding_dim=60, epochs=100, batch_size=32)\n\n\ntime_series_cols = train_ts_encoded.columns.tolist()\n\ntrain_ts_encoded[\"id\"]=train_ts[\"id\"]\n\ntest_ts_encoded['id']=test_ts[\"id\"]\n\n\ntrain = pd.merge(train, train_ts_encoded, how=\"left\", on='id')\n\ntest = pd.merge(test, test_ts_encoded, how=\"left\", on='id')\n\n\nimputer = KNNImputer(n_neighbors=5)\n\nnumeric_cols = train.select_dtypes(include=['float64', 'int64']).columns\n\nimputed_data = imputer.fit_transform(train[numeric_cols])\n\ntrain_imputed = pd.DataFrame(imputed_data, columns=numeric_cols)\n\ntrain_imputed['sii'] = train_imputed['sii'].round().astype(int)\n\nfor col in train.columns:\n\n    if col not in numeric_cols:\n\n        train_imputed[col] = train[col]\n\n        \ntrain = train_imputed\n\n\ntrain = train.drop('id', axis=1)\n\ntest  = test .drop('id', axis=1)  \n\n\nfeaturesCols = featuresCols_3 + time_series_cols\n\ntrain = train[featuresCols]\ntrain = train.dropna(subset='sii')\n\ndef update(df):\n    global cat_c\n    for c in cat_c: \n        df[c] = df[c].fillna('Missing')\n        df[c] = df[c].astype('category')\n    return df\n\ntrain = update(train)\ntest = update(test)\n\ndef create_mapping(column, dataset):\n    unique_values = dataset[column].unique()\n    return {value: idx for idx, value in enumerate(unique_values)}\n\nfor col in cat_c:\n    mapping = create_mapping(col, train)\n    mappingTe = create_mapping(col, test)\n    \n    train[col] = train[col].replace(mapping).astype(int)\n    test[col] = test[col].replace(mappingTe).astype(int)\n\ndef quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n    return -quadratic_weighted_kappa(y_true, rounded_p)\n\ndef TrainML(model_class, test_data):\n    X = train.drop(['sii'], axis=1)\n    y = train['sii']\n\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n    \n    train_S = []\n    test_S = []\n    \n    oof_non_rounded = np.zeros(len(y), dtype=float) \n    oof_rounded = np.zeros(len(y), dtype=int) \n    test_preds = np.zeros((len(test_data), n_splits))\n\n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n        X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n\n        model = clone(model_class)\n        model.fit(X_train, y_train)\n\n        y_train_pred = model.predict(X_train)\n        y_val_pred = model.predict(X_val)\n\n        oof_non_rounded[test_idx] = y_val_pred\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n        oof_rounded[test_idx] = y_val_pred_rounded\n\n        train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n        val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n        train_S.append(train_kappa)\n        test_S.append(val_kappa)\n        \n        test_preds[:, fold] = model.predict(test_data)\n        \n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n        clear_output(wait=True)\n\n    print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n    KappaOPtimizer = minimize(evaluate_predictions,\n                              x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n                              method='Nelder-Mead')\n    assert KappaOPtimizer.success, \"Optimization did not converge.\"\n    \n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n    tpm = test_preds.mean(axis=1)\n    tpTuned = threshold_Rounder(tpm, KappaOPtimizer.x)\n    \n    submission = pd.DataFrame({\n        'id': sample['id'],\n        'sii': tpTuned\n    })\n\n    return submission\n\n\nft_transformer = FTTransformerWrapper(\n    dim=32,\n    depth=6,\n    heads=8,\n    attn_dropout=0.3,\n    ff_dropout=0.3,\n    max_epochs=100,\n    lr=0.001,\n)\n\nParams = {\n    'learning_rate': 0.046,\n    'max_depth': 12,\n    'num_leaves': 478,\n    'min_data_in_leaf': 13,\n    'feature_fraction': 0.893,\n    'bagging_fraction': 0.784,\n    'bagging_freq': 4,\n    'lambda_l1': 10,  # Increased from 6.59\n    'lambda_l2': 0.01,  # Increased from 2.68e-06\n    'device': 'cpu'\n\n}\n\n\n# XGBoost parameters\nXGB_Params = {\n    'learning_rate': 0.05,\n    'max_depth': 6,\n    'n_estimators': 200,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 1,  # Increased from 0.1\n    'reg_lambda': 5,  # Increased from 1\n    'random_state': SEED,\n    'tree_method': 'gpu_hist',\n\n}\n\n\nCatBoost_Params = {\n    'learning_rate': 0.05,\n    'depth': 6,\n    'iterations': 200,\n    'random_seed': SEED,\n    'verbose': 0,\n    'l2_leaf_reg': 10,  # Increase this value\n    'task_type': 'GPU'\n\n}\n\nLight = LGBMRegressor(**Params, random_state=SEED, verbose=-1, n_estimators=300)\nXGB_Model = XGBRegressor(**XGB_Params)\nCatBoost_Model = CatBoostRegressor(**CatBoost_Params)\n\n# Combine models using Voting Regressor\nvoting_model = VotingRegressor(estimators=[\n    ('lightgbm', Light),\n    ('xgboost', XGB_Model),\n    ('catboost', CatBoost_Model),\n    ('ft_transformer', ft_transformer)],\n    weights=[5.0,4.0,4.0,4.0])\n\nSubmission3 = TrainML(voting_model, test)\n\nSubmission3","metadata":{"execution":{"iopub.execute_input":"2024-12-15T17:20:01.93713Z","iopub.status.busy":"2024-12-15T17:20:01.93684Z","iopub.status.idle":"2024-12-15T17:28:39.41854Z","shell.execute_reply":"2024-12-15T17:28:39.417611Z","shell.execute_reply.started":"2024-12-15T17:20:01.937105Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub1 = Submission1\nsub2 = Submission2\nsub3 = Submission3\n\nsub1 = sub1.sort_values(by='id').reset_index(drop=True)\nsub2 = sub2.sort_values(by='id').reset_index(drop=True)\nsub3 = sub3.sort_values(by='id').reset_index(drop=True)\n\ncombined = pd.DataFrame({\n    'id': sub1['id'],\n    'sii_1': sub1['sii'],\n    'sii_2': sub2['sii'],\n    'sii_3': sub3['sii']\n})\n\ndef majority_vote(row):\n    return row.mode()[0]\n\ncombined['final_sii'] = combined[['sii_1', 'sii_2', 'sii_3']].apply(majority_vote, axis=1)\n\nfinal_submission = combined[['id', 'final_sii']].rename(columns={'final_sii': 'sii'})\n\nfinal_submission.to_csv('submission.csv', index=False)\n\nprint(\"Majority voting completed and saved to 'Final_Submission.csv'\")","metadata":{"execution":{"iopub.execute_input":"2024-12-15T17:28:39.420374Z","iopub.status.busy":"2024-12-15T17:28:39.419944Z","iopub.status.idle":"2024-12-15T17:28:39.440448Z","shell.execute_reply":"2024-12-15T17:28:39.439451Z","shell.execute_reply.started":"2024-12-15T17:28:39.420331Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_submission","metadata":{"execution":{"iopub.execute_input":"2024-12-15T17:28:39.442127Z","iopub.status.busy":"2024-12-15T17:28:39.441548Z","iopub.status.idle":"2024-12-15T17:28:39.450098Z","shell.execute_reply":"2024-12-15T17:28:39.449175Z","shell.execute_reply.started":"2024-12-15T17:28:39.442082Z"}},"outputs":[],"execution_count":null}]}