{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.14"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30805,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true},"papermill":{"default_parameters":{},"duration":114.924774,"end_time":"2024-10-05T05:01:35.769707","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-10-05T04:59:40.844933","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport random\nimport optuna\nfrom sklearn.base import clone\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.model_selection import StratifiedKFold\nfrom scipy.optimize import minimize\nfrom concurrent.futures import ThreadPoolExecutor\nfrom tqdm import tqdm\nfrom colorama import Fore, Style\nfrom IPython.display import clear_output\nimport warnings\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.ensemble import VotingRegressor\nfrom sklearn.impute import SimpleImputer\nfrom imblearn.over_sampling import SMOTE\nfrom sklearn.decomposition import PCA\nfrom sklearn.model_selection import cross_val_score\nfrom lightgbm import LGBMRegressor","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T17:35:37.463726Z","iopub.execute_input":"2024-12-13T17:35:37.464545Z","iopub.status.idle":"2024-12-13T17:35:37.471078Z","shell.execute_reply.started":"2024-12-13T17:35:37.464509Z","shell.execute_reply":"2024-12-13T17:35:37.470160Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Set random seeds for reproducibility\nSEED = 42\nnp.random.seed(SEED)\nrandom.seed(SEED)\n\n# Suppress warnings\nwarnings.filterwarnings('ignore')\n\n# Pandas option for displaying all columns\npd.options.display.max_columns = None\n\n# Constants\nn_splits = 5\n\ndef process_file(filename, dirname):\n    df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    df.drop('step', axis=1, inplace=True)\n    return df.describe().values.reshape(-1), filename.split('=')[1]\n\ndef load_time_series(dirname) -> pd.DataFrame:\n    ids = os.listdir(dirname)\n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n    stats, indexes = zip(*results)\n    df = pd.DataFrame(stats, columns=[f\"stat_{i}\" for i in range(len(stats[0]))])\n    df['id'] = indexes\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T17:35:37.472462Z","iopub.execute_input":"2024-12-13T17:35:37.472734Z","iopub.status.idle":"2024-12-13T17:35:37.486235Z","shell.execute_reply.started":"2024-12-13T17:35:37.472694Z","shell.execute_reply":"2024-12-13T17:35:37.485549Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load datasets\ntrain = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\n# Load time series data\ntrain_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\n\n# Merge datasets\ntime_series_cols = train_ts.columns.tolist()\ntime_series_cols.remove(\"id\")\ntrain = pd.merge(train, train_ts, how=\"left\", on='id')\ntest = pd.merge(test, test_ts, how=\"left\", on='id')\ntrain = train.drop('id', axis=1)\ntest = test.drop('id', axis=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T17:35:37.487053Z","iopub.execute_input":"2024-12-13T17:35:37.487300Z","iopub.status.idle":"2024-12-13T17:36:44.936045Z","shell.execute_reply.started":"2024-12-13T17:35:37.487276Z","shell.execute_reply":"2024-12-13T17:36:44.935107Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define feature columns\nfeaturesCols = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex', 'CGAS-Season', 'CGAS-CGAS_Score',\n                'Physical-Season', 'Physical-BMI', 'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP', 'Fitness_Endurance-Season',\n                'Fitness_Endurance-Max_Stage', 'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec', 'FGC-Season',\n                'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND', 'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone',\n                'FGC-FGC_PU', 'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR', 'FGC-FGC_SRR_Zone',\n                'FGC-FGC_TL', 'FGC-FGC_TL_Zone', 'BIA-Season', 'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM', 'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat',\n                'BIA-BIA_Frame_num', 'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM', 'BIA-BIA_TBW',\n                'PAQ_A-Season', 'PAQ_A-PAQ_A_Total', 'PAQ_C-Season', 'PAQ_C-PAQ_C_Total', 'SDS-Season', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T', 'PreInt_EduHx-Season', 'PreInt_EduHx-computerinternet_hoursday', 'sii']\nfeaturesCols += time_series_cols\ntrain = train[featuresCols]\ntrain = train.dropna(subset=['sii'])\n\ncat_c = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', 'Fitness_Endurance-Season', 'FGC-Season',\n         'BIA-Season', 'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T17:36:44.937783Z","iopub.execute_input":"2024-12-13T17:36:44.938057Z","iopub.status.idle":"2024-12-13T17:36:44.948830Z","shell.execute_reply.started":"2024-12-13T17:36:44.938031Z","shell.execute_reply":"2024-12-13T17:36:44.948172Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def update(df):\n    global cat_c\n    for c in cat_c:\n        df[c] = df[c].fillna('Missing')\n        df[c] = df[c].astype('category')\n    return df\n\ntrain = update(train)\ntest = update(test)\n\ndef create_mapping(column, dataset):\n    unique_values = dataset[column].unique()\n    return {value: idx for idx, value in enumerate(unique_values)}\n\nfor col in cat_c:\n    mapping = create_mapping(col, train)\n    train[col] = train[col].replace(mapping).astype(int)\n    test[col] = test[col].replace(create_mapping(col, test)).astype(int)\n\ndef quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n    return -quadratic_weighted_kappa(y_true, rounded_p)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T17:36:44.949793Z","iopub.execute_input":"2024-12-13T17:36:44.950136Z","iopub.status.idle":"2024-12-13T17:36:45.009081Z","shell.execute_reply.started":"2024-12-13T17:36:44.950090Z","shell.execute_reply":"2024-12-13T17:36:45.008512Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def TrainML(model_class, test_data):\n    X = train.drop(['sii'], axis=1).values\n    y = train['sii'].values\n\n    imputer = SimpleImputer(strategy='mean')\n    X = imputer.fit_transform(X)\n    test_data = imputer.transform(test_data)\n\n    #smote = SMOTE(random_state=SEED)\n    #X_res, y_res = smote.fit_resample(X, y)\n    \n    X_res, y_res = X, y\n\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n    train_S = []\n    test_S = []\n    oof_non_rounded = np.zeros(len(y_res), dtype=float)\n    oof_rounded = np.zeros(len(y_res), dtype=int)\n    test_preds = np.zeros((len(test_data), n_splits))\n\n    for fold, (train_idx, val_idx) in enumerate(SKF.split(X_res, y_res)):\n        X_train, X_val = X_res[train_idx], X_res[val_idx]\n        y_train, y_val = y_res[train_idx], y_res[val_idx]\n\n        model = clone(model_class)\n        model.fit(X_train, y_train)\n        y_train_pred = model.predict(X_train)\n        y_val_pred = model.predict(X_val)\n\n        oof_non_rounded[val_idx] = y_val_pred\n        y_val_pred_rounded = np.round(y_val_pred).astype(int)\n        oof_rounded[val_idx] = y_val_pred_rounded\n\n        train_kappa = quadratic_weighted_kappa(y_train, np.round(y_train_pred).astype(int))\n        val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n        train_S.append(train_kappa)\n        test_S.append(val_kappa)\n\n        test_preds[:, fold] = model.predict(test_data)\n\n        print(f\"Fold {fold + 1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n        clear_output(wait=True)\n\n    print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n    # Optimize thresholds with Nelder-Mead method\n    KappaOptimizer = minimize(evaluate_predictions, x0=[0.5, 1.5, 2.5], args=(y_res, oof_non_rounded), method='Nelder-Mead')\n    assert KappaOptimizer.success, \"Optimization did not converge.\"\n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOptimizer.x)\n    tKappa = quadratic_weighted_kappa(y_res, oof_tuned)\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n    tpm = test_preds.mean(axis=1)\n    tpTuned = threshold_Rounder(tpm, KappaOptimizer.x)\n    submission = pd.DataFrame({\n        'id': sample['id'],\n        'sii': tpTuned\n    })\n\n    return submission","metadata":{"execution":{"iopub.status.busy":"2024-12-13T17:36:45.010042Z","iopub.execute_input":"2024-12-13T17:36:45.010305Z","iopub.status.idle":"2024-12-13T17:36:45.019920Z","shell.execute_reply.started":"2024-12-13T17:36:45.010281Z","shell.execute_reply":"2024-12-13T17:36:45.019067Z"},"papermill":{"duration":0.048348,"end_time":"2024-10-05T05:01:19.304949","exception":false,"start_time":"2024-10-05T05:01:19.256601","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Model parameters for LightGBM\nParams = {\n    'learning_rate': 0.046,\n    'max_depth': 12,\n    'num_leaves': 478,\n    'min_data_in_leaf': 13,\n    'feature_fraction': 0.893,\n    'bagging_fraction': 0.784,\n    'bagging_freq': 4,\n    'lambda_l1': 10,\n    'lambda_l2': 0.01,\n    'device': 'gpu'\n}\n\n# XGBoost parameters\nXGB_Params = {\n    'learning_rate': 0.05,\n    'max_depth': 6,\n    'n_estimators': 200,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 1,\n    'reg_lambda': 5,\n    'random_state': SEED,\n    'tree_method': 'hist',\n    'device': 'cuda'\n}\n\n# CatBoost parameters including specification of categorical features\nCatBoost_Params = {\n    'learning_rate': 0.05,\n    'depth': 6,\n    'iterations': 200,\n    'random_seed': SEED,\n    'cat_features': cat_c,  # Important: ensure this parameter is configured correctly\n    'verbose': 0,\n    'l2_leaf_reg': 100,\n    'task_type': 'GPU'\n}\n\n# Create model instances\nLight = LGBMRegressor(**Params, verbose=-1, n_estimators=200)\nXGB_Model = XGBRegressor(**XGB_Params)\nCatBoost_Model = CatBoostRegressor(**CatBoost_Params)\n\n# Combine models using Voting Regressor\nvoting_model = VotingRegressor(estimators=[\n    ('lightgbm', Light),\n    ('xgboost', XGB_Model),\n    #('catboost', CatBoost_Model)\n], weights=[20, 10,])\n\n# Train the ensemble model\nSubmission = TrainML(voting_model, test.values)","metadata":{"execution":{"iopub.status.busy":"2024-12-13T17:44:48.138732Z","iopub.execute_input":"2024-12-13T17:44:48.139450Z","iopub.status.idle":"2024-12-13T17:44:56.829356Z","shell.execute_reply.started":"2024-12-13T17:44:48.139412Z","shell.execute_reply":"2024-12-13T17:44:56.828451Z"},"papermill":{"duration":15.421112,"end_time":"2024-10-05T05:01:34.755017","exception":false,"start_time":"2024-10-05T05:01:19.333905","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Save submission\nSubmission.to_csv('submission.csv', index=False)\nprint(Submission['sii'].value_counts())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T17:45:02.122260Z","iopub.execute_input":"2024-12-13T17:45:02.123077Z","iopub.status.idle":"2024-12-13T17:45:02.129902Z","shell.execute_reply.started":"2024-12-13T17:45:02.123042Z","shell.execute_reply":"2024-12-13T17:45:02.128978Z"}},"outputs":[],"execution_count":null}]}