{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"},{"sourceId":7453542,"sourceType":"datasetVersion","datasetId":921302}],"dockerImageVersionId":30805,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport polars as pl\nimport pandas as pd\nfrom sklearn.base import clone\nfrom copy import deepcopy\nimport optuna\nfrom scipy.optimize import minimize\nimport os\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport re\nfrom colorama import Fore, Style\n\nfrom tqdm import tqdm\nfrom IPython.display import clear_output\nfrom concurrent.futures import ThreadPoolExecutor\n\nimport warnings\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None\n\nimport lightgbm as lgb\nfrom catboost import CatBoostRegressor, CatBoostClassifier\nfrom xgboost import XGBRegressor\nfrom sklearn.ensemble import VotingRegressor\nfrom sklearn.model_selection import *\nfrom sklearn.metrics import *\n\nn_splits = 5\nSEED = 42","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-18T14:49:35.463456Z","iopub.execute_input":"2024-12-18T14:49:35.463704Z","iopub.status.idle":"2024-12-18T14:49:41.386297Z","shell.execute_reply.started":"2024-12-18T14:49:35.463678Z","shell.execute_reply":"2024-12-18T14:49:41.385625Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Loading","metadata":{}},{"cell_type":"markdown","source":"## Load tabular dataset","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T14:49:41.387913Z","iopub.execute_input":"2024-12-18T14:49:41.388848Z","iopub.status.idle":"2024-12-18T14:49:41.469736Z","shell.execute_reply.started":"2024-12-18T14:49:41.388807Z","shell.execute_reply":"2024-12-18T14:49:41.468726Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T14:49:41.470727Z","iopub.execute_input":"2024-12-18T14:49:41.470994Z","iopub.status.idle":"2024-12-18T14:49:41.620894Z","shell.execute_reply.started":"2024-12-18T14:49:41.470952Z","shell.execute_reply":"2024-12-18T14:49:41.619919Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['id'].head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T14:49:41.622532Z","iopub.execute_input":"2024-12-18T14:49:41.622772Z","iopub.status.idle":"2024-12-18T14:49:41.628895Z","shell.execute_reply.started":"2024-12-18T14:49:41.622749Z","shell.execute_reply":"2024-12-18T14:49:41.627951Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Load timeseries data","metadata":{}},{"cell_type":"markdown","source":"1. **process_file**: This function process file timeseries, extract general information in the file like count, mean, std, min, 25%, 50%, 75% and max of each features and then the features matrix is flattened to a vector to represent the data in the file\n2. **load_time_series** Format and load all timeseries files after processed in a folder.","metadata":{}},{"cell_type":"code","source":"def process_file(filename, dirname):\n    df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    df.drop('step', axis=1, inplace=True)\n    return df.describe().values.reshape(-1), filename.split('=')[1]\n\ndef load_time_series(dirname) -> pd.DataFrame:\n    ids = os.listdir(dirname)\n    \n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n    \n    stats, indexes = zip(*results)\n    \n    df = pd.DataFrame(stats, columns=[f\"Stat_{i}\" for i in range(len(stats[0]))])\n    df['id'] = indexes\n    \n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T14:49:41.629907Z","iopub.execute_input":"2024-12-18T14:49:41.630315Z","iopub.status.idle":"2024-12-18T14:49:41.639444Z","shell.execute_reply.started":"2024-12-18T14:49:41.630282Z","shell.execute_reply":"2024-12-18T14:49:41.638669Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\ntime_series_cols = train_ts.columns.tolist()\ntime_series_cols.remove(\"id\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T14:49:41.640417Z","iopub.execute_input":"2024-12-18T14:49:41.640648Z","iopub.status.idle":"2024-12-18T14:50:51.147864Z","shell.execute_reply.started":"2024-12-18T14:49:41.640625Z","shell.execute_reply":"2024-12-18T14:50:51.146991Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_ts.head(5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T14:50:51.149033Z","iopub.execute_input":"2024-12-18T14:50:51.149327Z","iopub.status.idle":"2024-12-18T14:50:51.211788Z","shell.execute_reply.started":"2024-12-18T14:50:51.149293Z","shell.execute_reply":"2024-12-18T14:50:51.210941Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Time series data is then merged to the tabular data","metadata":{}},{"cell_type":"code","source":"train = pd.merge(train, train_ts, how=\"left\", on='id')\ntest = pd.merge(test, test_ts, how=\"left\", on='id')\n\ntrain = train.drop('id', axis=1)\ntest = test.drop('id', axis=1)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T14:50:51.212985Z","iopub.execute_input":"2024-12-18T14:50:51.213272Z","iopub.status.idle":"2024-12-18T14:50:51.242182Z","shell.execute_reply.started":"2024-12-18T14:50:51.213228Z","shell.execute_reply":"2024-12-18T14:50:51.241407Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T14:50:51.243137Z","iopub.execute_input":"2024-12-18T14:50:51.243377Z","iopub.status.idle":"2024-12-18T14:50:51.347988Z","shell.execute_reply.started":"2024-12-18T14:50:51.243354Z","shell.execute_reply":"2024-12-18T14:50:51.347137Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Filtering","metadata":{}},{"cell_type":"markdown","source":"Select the columns which is present in test data to train","metadata":{}},{"cell_type":"code","source":"featuresCols = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-Season', 'CGAS-CGAS_Score', 'Physical-Season', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Season', 'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-Season', 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone', 'BIA-Season',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-Season', 'PAQ_A-PAQ_A_Total', 'PAQ_C-Season',\n                'PAQ_C-PAQ_C_Total', 'SDS-Season', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T', 'PreInt_EduHx-Season',\n                'PreInt_EduHx-computerinternet_hoursday', 'sii']\n\nfeaturesCols += time_series_cols\n\ntrain = train[featuresCols]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T14:50:51.350936Z","iopub.execute_input":"2024-12-18T14:50:51.351442Z","iopub.status.idle":"2024-12-18T14:50:51.357815Z","shell.execute_reply.started":"2024-12-18T14:50:51.351415Z","shell.execute_reply":"2024-12-18T14:50:51.357002Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Drop the NaN sii value","metadata":{}},{"cell_type":"code","source":"train = train.dropna(subset='sii')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T14:50:51.358779Z","iopub.execute_input":"2024-12-18T14:50:51.359020Z","iopub.status.idle":"2024-12-18T14:50:51.371696Z","shell.execute_reply.started":"2024-12-18T14:50:51.358996Z","shell.execute_reply":"2024-12-18T14:50:51.370907Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Fill the missing categorical data with \"Missing\" ","metadata":{}},{"cell_type":"code","source":"cat_c = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', 'Fitness_Endurance-Season', \n          'FGC-Season', 'BIA-Season', 'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season']\n\ndef update(df):\n    for c in cat_c: \n        df[c] = df[c].fillna('Missing')\n        df[c] = df[c].astype('category')\n    return df\n        \ntrain = update(train)\ntest = update(test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T14:50:51.372592Z","iopub.execute_input":"2024-12-18T14:50:51.372839Z","iopub.status.idle":"2024-12-18T14:50:51.399338Z","shell.execute_reply.started":"2024-12-18T14:50:51.372802Z","shell.execute_reply":"2024-12-18T14:50:51.398723Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Create a mapping from string to integer to push data to the model (Use one hot encode instead)","metadata":{}},{"cell_type":"code","source":"def create_mapping(column, dataset):\n    unique_values = dataset[column].unique()\n    return {value: idx for idx, value in enumerate(unique_values)}\n\nfor col in cat_c:\n    mapping_train = create_mapping(col, train)\n    mapping_test = create_mapping(col, test)\n    \n    train[col] = train[col].replace(mapping_train).astype(int)\n    test[col] = test[col].replace(mapping_test).astype(int)\n\nprint(f'Train Shape : {train.shape} || Test Shape : {test.shape}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T14:50:51.400030Z","iopub.execute_input":"2024-12-18T14:50:51.400247Z","iopub.status.idle":"2024-12-18T14:50:51.437594Z","shell.execute_reply.started":"2024-12-18T14:50:51.400225Z","shell.execute_reply":"2024-12-18T14:50:51.436914Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T14:50:51.438375Z","iopub.execute_input":"2024-12-18T14:50:51.438577Z","iopub.status.idle":"2024-12-18T14:50:51.452575Z","shell.execute_reply.started":"2024-12-18T14:50:51.438556Z","shell.execute_reply":"2024-12-18T14:50:51.451732Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T14:50:51.453594Z","iopub.execute_input":"2024-12-18T14:50:51.453868Z","iopub.status.idle":"2024-12-18T14:50:51.545365Z","shell.execute_reply.started":"2024-12-18T14:50:51.453822Z","shell.execute_reply":"2024-12-18T14:50:51.544591Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['sii'].head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T14:50:51.546372Z","iopub.execute_input":"2024-12-18T14:50:51.546614Z","iopub.status.idle":"2024-12-18T14:50:51.552426Z","shell.execute_reply.started":"2024-12-18T14:50:51.546591Z","shell.execute_reply":"2024-12-18T14:50:51.551520Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Training Function","metadata":{}},{"cell_type":"markdown","source":"**quadratic_weighted_kappa**: calculate QWK value","metadata":{}},{"cell_type":"code","source":"def quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T14:50:51.553317Z","iopub.execute_input":"2024-12-18T14:50:51.553527Z","iopub.status.idle":"2024-12-18T14:50:51.561179Z","shell.execute_reply.started":"2024-12-18T14:50:51.553505Z","shell.execute_reply":"2024-12-18T14:50:51.560390Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**threshold_Rounder**: Turn the sii from PCIAT_Total to categorical ","metadata":{}},{"cell_type":"code","source":"def threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T14:50:51.562153Z","iopub.execute_input":"2024-12-18T14:50:51.562396Z","iopub.status.idle":"2024-12-18T14:50:51.571335Z","shell.execute_reply.started":"2024-12-18T14:50:51.562373Z","shell.execute_reply":"2024-12-18T14:50:51.570662Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**evaluate_predictions**: this function evaluate the prediction of the model by first turn integer prediction values to categorical values and then calculate QWK from it and the true labels.","metadata":{}},{"cell_type":"code","source":"def evaluate_predictions(thresholds, y_true, oof_non_rounded):\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n    return -quadratic_weighted_kappa(y_true, rounded_p)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T14:50:51.572225Z","iopub.execute_input":"2024-12-18T14:50:51.572484Z","iopub.status.idle":"2024-12-18T14:50:51.583190Z","shell.execute_reply.started":"2024-12-18T14:50:51.572461Z","shell.execute_reply":"2024-12-18T14:50:51.582574Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**TrainML**: Train the model using K-Fold, The model is regression model, predict a real value represent how bad the patient was. The value may not explicitly different, so we re-define the threshold to make it split more accurate","metadata":{}},{"cell_type":"code","source":"def TrainML(model_class, test_data):\n    \n    X = train.drop(['sii'], axis=1)\n    y = train['sii']\n\n    # Apply K-Fold\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n    \n    train_S = []\n    test_S = []\n    \n    oof_non_rounded = np.zeros(len(y), dtype=float) \n    oof_rounded = np.zeros(len(y), dtype=int) \n    test_preds = np.zeros((len(test_data), n_splits))\n\n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n        # Train model\n        X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n\n        model = clone(model_class)\n        model.fit(X_train, y_train)\n\n        y_train_pred = model.predict(X_train)\n        y_val_pred = model.predict(X_val)\n\n        # Round to integer values\n        oof_non_rounded[test_idx] = y_val_pred\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n        oof_rounded[test_idx] = y_val_pred_rounded\n\n        train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n        val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n        train_S.append(train_kappa)\n        test_S.append(val_kappa)\n\n        #Predict with test dataset\n        test_preds[:, fold] = model.predict(test_data)\n        \n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n        clear_output(wait=True)\n\n    print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n    # Using optimizer to find the best threshold\n    KappaOPtimizer = minimize(evaluate_predictions,\n                              x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n                              method='Nelder-Mead') # Nelder-Mead | # Powell\n    assert KappaOPtimizer.success, \"Optimization did not converge.\"\n\n    # Use the threshold retrive from the optimizer to predict again to evaluate\n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n    # Use the threshold retrive from the optimizer to predict test\n    tpm = test_preds.mean(axis=1)\n    tpTuned = threshold_Rounder(tpm, KappaOPtimizer.x)\n\n    # Create submition\n    submission = pd.DataFrame({\n        'id': sample['id'],\n        'sii': tpTuned\n    })\n\n    return submission,model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T14:50:51.584325Z","iopub.execute_input":"2024-12-18T14:50:51.584567Z","iopub.status.idle":"2024-12-18T14:50:51.595626Z","shell.execute_reply.started":"2024-12-18T14:50:51.584544Z","shell.execute_reply":"2024-12-18T14:50:51.594805Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Create model and train the model","metadata":{}},{"cell_type":"code","source":"Params7 = {'learning_rate': 0.1, 'max_depth': 10, 'num_leaves': 100, 'min_data_in_leaf': 10,\n           'feature_fraction': 0.5, 'bagging_fraction': 0.5, 'bagging_freq': 2, \n           'lambda_l1': 1, 'lambda_l2': 1e-04} # CV : 0.4094 | LB : 0.471\n\nLight = lgb.LGBMRegressor(**Params7, verbose=-1, n_estimators=200, random_state=SEED)\nSubmission,model = TrainML(Light,test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T14:50:51.596640Z","iopub.execute_input":"2024-12-18T14:50:51.597458Z","iopub.status.idle":"2024-12-18T14:50:57.901553Z","shell.execute_reply.started":"2024-12-18T14:50:51.597432Z","shell.execute_reply":"2024-12-18T14:50:57.900868Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Features important","metadata":{}},{"cell_type":"code","source":"feature_importance_df = pd.DataFrame({\n    'Feature': model.booster_.feature_name(),\n    'Importance': model.booster_.feature_importance(importance_type='gain')\n})","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T14:50:57.902454Z","iopub.execute_input":"2024-12-18T14:50:57.902719Z","iopub.status.idle":"2024-12-18T14:50:57.907535Z","shell.execute_reply.started":"2024-12-18T14:50:57.902676Z","shell.execute_reply":"2024-12-18T14:50:57.906789Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"feature_importance_df = feature_importance_df.sort_values(by='Importance', ascending=False)\n\nplt.figure(figsize=(20, 40))\nsns.barplot(x='Importance', y='Feature', data=feature_importance_df.head(100)) \nplt.title(\"Top Feature Importance\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T14:50:57.908487Z","iopub.execute_input":"2024-12-18T14:50:57.908713Z","iopub.status.idle":"2024-12-18T14:50:59.144571Z","shell.execute_reply.started":"2024-12-18T14:50:57.908690Z","shell.execute_reply":"2024-12-18T14:50:59.143766Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Submit model","metadata":{}},{"cell_type":"code","source":"Submission.to_csv('submission.csv', index=False)\nprint(Submission['sii'].value_counts())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T14:50:59.145698Z","iopub.execute_input":"2024-12-18T14:50:59.146048Z","iopub.status.idle":"2024-12-18T14:50:59.155506Z","shell.execute_reply.started":"2024-12-18T14:50:59.145998Z","shell.execute_reply":"2024-12-18T14:50:59.154663Z"}},"outputs":[],"execution_count":null}]}