{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"},{"sourceId":7453542,"sourceType":"datasetVersion","datasetId":921302}],"dockerImageVersionId":30804,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Overview","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport seaborn as sns\nimport warnings\nfrom xgboost import XGBRegressor\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.pipeline import Pipeline\nfrom xgboost import XGBClassifier\n\nimport os\nimport re\nfrom tqdm import tqdm\nimport polars as pl\nimport polars.selectors as cs\nimport matplotlib.pyplot as plt\nfrom matplotlib.ticker import MaxNLocator, FormatStrFormatter, PercentFormatter\nimport seaborn as sns\nimport plotly.subplots as sp\nimport plotly.express as px\nfrom concurrent.futures import ThreadPoolExecutor\nfrom colorama import Fore, Style\nfrom IPython.display import clear_output\nfrom IPython.display import display\nimport warnings\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None\n\nfrom sklearn.base import clone, BaseEstimator, RegressorMixin\nfrom sklearn.ensemble import RandomForestClassifier, RandomForestRegressor\nfrom sklearn.ensemble import StackingRegressor\nfrom sklearn.linear_model import Ridge\nfrom sklearn.experimental import enable_iterative_imputer\nfrom sklearn.impute import IterativeImputer\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix, accuracy_score, precision_score, recall_score, f1_score, roc_curve, auc\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.model_selection import StratifiedKFold\nfrom scipy.optimize import minimize\nfrom sklearn.ensemble import VotingRegressor, RandomForestRegressor, GradientBoostingRegressor, HistGradientBoostingRegressor, ExtraTreesRegressor\nfrom sklearn.impute import SimpleImputer, KNNImputer\nfrom sklearn.pipeline import Pipeline\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.model_selection import GridSearchCV\n\n!pip -q install /kaggle/input/pytorchtabnet/pytorch_tabnet-4.1.0-py3-none-any.whl\nfrom pytorch_tabnet.tab_model import TabNetRegressor\nfrom sklearn.base import BaseEstimator, RegressorMixin\nfrom sklearn.impute import SimpleImputer, KNNImputer\nfrom sklearn.model_selection import train_test_split\nfrom pytorch_tabnet.callbacks import Callback\nimport os\nimport torch\n\nfrom pytorch_tabnet.tab_model import TabNetRegressor\nfrom pytorch_tabnet.callbacks import Callback\n\nfrom keras.models import Model\nfrom keras.layers import Input, Dense\nfrom keras.optimizers import Adam\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nimport pytorch_tabnet","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:35:16.164357Z","iopub.execute_input":"2024-12-19T14:35:16.164719Z","iopub.status.idle":"2024-12-19T14:36:11.284501Z","shell.execute_reply.started":"2024-12-19T14:35:16.164682Z","shell.execute_reply":"2024-12-19T14:36:11.283054Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Load data","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\ndata_dict = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/data_dictionary.csv')\ntrain.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:11.286643Z","iopub.execute_input":"2024-12-19T14:36:11.28784Z","iopub.status.idle":"2024-12-19T14:36:11.462326Z","shell.execute_reply.started":"2024-12-19T14:36:11.287783Z","shell.execute_reply":"2024-12-19T14:36:11.460779Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:11.463951Z","iopub.execute_input":"2024-12-19T14:36:11.464323Z","iopub.status.idle":"2024-12-19T14:36:11.5259Z","shell.execute_reply.started":"2024-12-19T14:36:11.46429Z","shell.execute_reply":"2024-12-19T14:36:11.52435Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Target","metadata":{}},{"cell_type":"code","source":"train_cols = set(train.columns)\ntest_cols = set(test.columns)\ncolumns_not_in_test = sorted(list(train_cols - test_cols))\n\ncolumns_to_exclude = ['PCIAT-PCIAT_Total', 'PCIAT-Season', 'sii']\nquestion_columns = [\n    col for col in columns_not_in_test if col not in columns_to_exclude\n]\n\nquestion_columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:11.527649Z","iopub.execute_input":"2024-12-19T14:36:11.528051Z","iopub.status.idle":"2024-12-19T14:36:11.53899Z","shell.execute_reply.started":"2024-12-19T14:36:11.528016Z","shell.execute_reply":"2024-12-19T14:36:11.537324Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\nThe task is to predict the internet addiction status (SII) of a group of people. In the training dataset, participants answer 20 questions (PCIAT-PCIAT_1, PCIAT-PCIAT_2, ..., PCIAT-PCIAT_20), where each question is rated from 0 to 5, with 0 indicating no severity and 5 indicating the highest severity. The total score from these 20 questions is calculated in the column PCIAT-PCIAT_Total and mapped to the target column, SII, which has values ranging from 0 to 3. This is the column that needs to be predicted.","metadata":{}},{"cell_type":"markdown","source":"## Filter rows with ambigious sii","metadata":{}},{"cell_type":"markdown","source":"\nThere are many survey participants who did not answer some questions, or there were errors in data collection. These rows could cause the total score to change when those questions are answered, making the SII unclear. Filter out the rows where the SII is unclear","metadata":{}},{"cell_type":"code","source":"def recalculate_sii(row):\n    if pd.isna(row['PCIAT-PCIAT_Total']):\n        return np.nan\n    max_possible = row['PCIAT-PCIAT_Total'] + row[question_columns].isna().sum() * 5\n    if row['PCIAT-PCIAT_Total'] <= 30 and max_possible <= 30:\n        return 0\n    elif 31 <= row['PCIAT-PCIAT_Total'] <= 49 and max_possible <= 49:\n        return 1\n    elif 50 <= row['PCIAT-PCIAT_Total'] <= 79 and max_possible <= 79:\n        return 2\n    elif row['PCIAT-PCIAT_Total'] >= 80 and max_possible >= 80:\n        return 3\n    return np.nan\n\ntrain['recalc_sii'] = train.apply(recalculate_sii, axis=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:11.54474Z","iopub.execute_input":"2024-12-19T14:36:11.545286Z","iopub.status.idle":"2024-12-19T14:36:13.153404Z","shell.execute_reply.started":"2024-12-19T14:36:11.545235Z","shell.execute_reply":"2024-12-19T14:36:13.151996Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mismatch_rows = train[\n    (train['recalc_sii'] != train['sii']) & train['sii'].notna()\n]\n\nmismatch_rows[question_columns + ['recalc_sii']].style.map(\n    lambda x: 'background-color: #FFC0CB' if pd.isna(x) else ''\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:13.155507Z","iopub.execute_input":"2024-12-19T14:36:13.156009Z","iopub.status.idle":"2024-12-19T14:36:13.235736Z","shell.execute_reply.started":"2024-12-19T14:36:13.155968Z","shell.execute_reply":"2024-12-19T14:36:13.234405Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"These rows have ambigious sii values => Drop","metadata":{}},{"cell_type":"code","source":"train['sii'] = train['recalc_sii']\ntrain = train.drop(mismatch_rows.index)\n\ntrain[columns_not_in_test + ['recalc_sii']]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:13.23787Z","iopub.execute_input":"2024-12-19T14:36:13.238346Z","iopub.status.idle":"2024-12-19T14:36:13.288434Z","shell.execute_reply.started":"2024-12-19T14:36:13.238305Z","shell.execute_reply":"2024-12-19T14:36:13.28683Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"na_total_rows = train[train['sii'].isna()]\nna_total_rows","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:13.291173Z","iopub.execute_input":"2024-12-19T14:36:13.292217Z","iopub.status.idle":"2024-12-19T14:36:13.382993Z","shell.execute_reply.started":"2024-12-19T14:36:13.292143Z","shell.execute_reply":"2024-12-19T14:36:13.381259Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\nWe decided to drop all rows where the SII cannot be calculated, as no questions were answered","metadata":{}},{"cell_type":"code","source":"train = train.dropna(subset=['PCIAT-PCIAT_Total'])\ntrain","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:13.385357Z","iopub.execute_input":"2024-12-19T14:36:13.386514Z","iopub.status.idle":"2024-12-19T14:36:13.501553Z","shell.execute_reply.started":"2024-12-19T14:36:13.386451Z","shell.execute_reply":"2024-12-19T14:36:13.499702Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"For the remaining rows, fill the unanswered questions with the mode of the corresponding column.","metadata":{}},{"cell_type":"code","source":"for column in question_columns:\n    if train[column].isna().any():\n        mode_value = train[column].mode()[0]\n        train[column] = train[column].fillna(mode_value)\n\ntrain[columns_not_in_test + ['recalc_sii']]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:13.503769Z","iopub.execute_input":"2024-12-19T14:36:13.504376Z","iopub.status.idle":"2024-12-19T14:36:13.570331Z","shell.execute_reply.started":"2024-12-19T14:36:13.504324Z","shell.execute_reply":"2024-12-19T14:36:13.568803Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.drop(columns='recalc_sii', inplace=True)\n\ntrain[columns_not_in_test]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:13.572527Z","iopub.execute_input":"2024-12-19T14:36:13.573879Z","iopub.status.idle":"2024-12-19T14:36:13.622814Z","shell.execute_reply.started":"2024-12-19T14:36:13.573816Z","shell.execute_reply":"2024-12-19T14:36:13.621139Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Helper functions","metadata":{}},{"cell_type":"code","source":"def calculate_stats(data, columns):\n    if isinstance(columns, str):\n        columns = [columns]\n\n    stats = []\n    for col in columns:\n        if data[col].dtype in ['object', 'category']:\n            counts = data[col].value_counts(dropna=False, sort=False)\n            percents = data[col].value_counts(normalize=True, dropna=False, sort=False) * 100\n            formatted = counts.astype(str) + ' (' + percents.round(2).astype(str) + '%)'\n            stats_col = pd.DataFrame({'count (%)': formatted})\n            stats.append(stats_col)\n        else:\n            stats_col = data[col].describe().to_frame().transpose()\n            stats_col['missing'] = data[col].isnull().sum()\n            stats_col.index.name = col\n            stats.append(stats_col)\n\n    return pd.concat(stats, axis=0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:13.625041Z","iopub.execute_input":"2024-12-19T14:36:13.625685Z","iopub.status.idle":"2024-12-19T14:36:13.637085Z","shell.execute_reply.started":"2024-12-19T14:36:13.625625Z","shell.execute_reply":"2024-12-19T14:36:13.635344Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- **quadratic_weigthed_kappa**: Hàm tính điểm có trọng số. Đây là hàm được dùng tính điểm trong yêu cầu đề bài\n- **threshold_Rounder**: Hàm làm tròn dựa trên thresholds đã được customize. Thay vì làm tròn ở 0.5, 1.5 và 2.5, chúng ta có thể tự tính ngưỡng làm tròn sao cho mô hình hoạt động tốt nhất.\n- **evaluate_predictions**: Hàm này đánh giá điểm của mô hình dựa trên thresholds, sử dụng hàm threshold_Rounder để làm tròn giá trị dự đoán, sau đó tính điểm Kappa. Dùng để tối ưu hóa trong quá trình cải thiện mô hình.\n- **trainML**: Hàm huấn luyện mô hình với StratifiedKFold. Fit từng fold, tính ngưỡng làm tròn tối ưu cho sau khi huấn luyện qua tất cả các fold, và dự đoán trên tập test, trả về kết quả dự đoán và ô hình đã fit","metadata":{}},{"cell_type":"code","source":"def quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n    return -quadratic_weighted_kappa(y_true, rounded_p)\n\ndef TrainML(model_class, train_data, test_data):\n    \n    X = train_data.drop(['sii'], axis=1)\n    y = train_data['sii']\n\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n    \n    train_S = []\n    test_S = []\n    \n    oof_non_rounded = np.zeros(len(y), dtype=float) \n    oof_rounded = np.zeros(len(y), dtype=int) \n    test_preds = np.zeros((len(test_data), n_splits))\n\n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n        X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n\n        model = clone(model_class)\n        model.fit(X_train, y_train)\n\n        y_train_pred = model.predict(X_train)\n        y_val_pred = model.predict(X_val)\n\n        oof_non_rounded[test_idx] = y_val_pred\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n        oof_rounded[test_idx] = y_val_pred_rounded\n\n        train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n        val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n        train_S.append(train_kappa)\n        test_S.append(val_kappa)\n        \n        test_preds[:, fold] = model.predict(test_data)\n        \n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n        clear_output(wait=True)\n\n    print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n    KappaOPtimizer = minimize(evaluate_predictions,\n                              x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n                              method='Nelder-Mead') # Nelder-Mead | # Powell\n    assert KappaOPtimizer.success, \"Optimization did not converge.\"\n    \n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n    tpm = test_preds.mean(axis=1)\n    tpTuned = threshold_Rounder(tpm, KappaOPtimizer.x)\n    \n    submission = pd.DataFrame({\n        'id': sample['id'],\n        'sii': tpTuned\n    })\n\n    return submission,model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:13.639135Z","iopub.execute_input":"2024-12-19T14:36:13.639653Z","iopub.status.idle":"2024-12-19T14:36:13.662718Z","shell.execute_reply.started":"2024-12-19T14:36:13.639605Z","shell.execute_reply":"2024-12-19T14:36:13.661251Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#  Data cleaning","metadata":{}},{"cell_type":"markdown","source":"The group's approach will be to assess the importance of each column with respect to SII to decide whether to keep or discard it. Next, we will examine its relationship with other columns in the training dataset, primarily age, sex, and season, as these columns already contain sufficient information. We will use these columns to predict missing values for the important ones. Then, we will analyze the relationship of these columns with the PCIAT-PCIAT_Total column to identify any notable patterns. ","metadata":{}},{"cell_type":"markdown","source":"Finally, columns that cannot be filled due to insufficient relationships with reliable columns will be imputed using KNN, with all columns in the dataset being used for this step.","metadata":{}},{"cell_type":"markdown","source":"## Age","metadata":{}},{"cell_type":"markdown","source":"Add Age Group columns","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n\ndef apply_age_group(df):\n    df['Age Group'] = pd.cut(\n        df['Basic_Demos-Age'],\n        bins=[4, 12, 18, 22],\n        labels=['Children', 'Adolescents', 'Adults'],\n    )\n    return df\n\ntrain = apply_age_group(train)\ntest = apply_age_group(test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:13.664644Z","iopub.execute_input":"2024-12-19T14:36:13.665184Z","iopub.status.idle":"2024-12-19T14:36:13.689933Z","shell.execute_reply.started":"2024-12-19T14:36:13.665144Z","shell.execute_reply":"2024-12-19T14:36:13.688297Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\nle = LabelEncoder()\ntrain['Age_Group_Label'] = le.fit_transform(train['Age Group'])\ntest['Age_Group_Label'] = le.fit_transform(test['Age Group'])\ntrain[['Age Group', 'Age_Group_Label']]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:13.69213Z","iopub.execute_input":"2024-12-19T14:36:13.692647Z","iopub.status.idle":"2024-12-19T14:36:13.71687Z","shell.execute_reply.started":"2024-12-19T14:36:13.692598Z","shell.execute_reply":"2024-12-19T14:36:13.715077Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"calculate_stats(train, ['Basic_Demos-Age'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:13.719417Z","iopub.execute_input":"2024-12-19T14:36:13.720042Z","iopub.status.idle":"2024-12-19T14:36:13.748634Z","shell.execute_reply.started":"2024-12-19T14:36:13.719986Z","shell.execute_reply":"2024-12-19T14:36:13.746738Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## CGAS score","metadata":{}},{"cell_type":"markdown","source":"###  CGAS-Season","metadata":{}},{"cell_type":"code","source":"calculate_stats(train, 'CGAS-Season')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:13.751097Z","iopub.execute_input":"2024-12-19T14:36:13.75169Z","iopub.status.idle":"2024-12-19T14:36:13.776876Z","shell.execute_reply.started":"2024-12-19T14:36:13.751635Z","shell.execute_reply":"2024-12-19T14:36:13.775323Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Fill missing season values by ratio of each season's occurences in train data","metadata":{}},{"cell_type":"code","source":"season_counts = train['CGAS-Season'].value_counts(normalize=True)\n\ntrain['CGAS-Season'] = train['CGAS-Season'].apply(\n    lambda x: np.random.choice(season_counts.index, p=season_counts.values) if pd.isna(x) else x\n)\ntest['CGAS-Season'] = test['CGAS-Season'].apply(\n    lambda x: np.random.choice(season_counts.index, p=season_counts.values) if pd.isna(x) else x\n)\ncalculate_stats(train, 'CGAS-Season')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:13.77901Z","iopub.execute_input":"2024-12-19T14:36:13.779689Z","iopub.status.idle":"2024-12-19T14:36:13.832493Z","shell.execute_reply.started":"2024-12-19T14:36:13.779491Z","shell.execute_reply":"2024-12-19T14:36:13.830739Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### CGAS Score","metadata":{}},{"cell_type":"code","source":"calculate_stats(train, 'CGAS-CGAS_Score')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:13.834573Z","iopub.execute_input":"2024-12-19T14:36:13.835053Z","iopub.status.idle":"2024-12-19T14:36:13.860491Z","shell.execute_reply.started":"2024-12-19T14:36:13.835008Z","shell.execute_reply":"2024-12-19T14:36:13.859129Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Drop the outlier with CGAS score = 999","metadata":{}},{"cell_type":"code","source":"train.loc[train['CGAS-CGAS_Score'] == 999, 'CGAS-CGAS_Score'] = np.nan","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:13.862168Z","iopub.execute_input":"2024-12-19T14:36:13.862614Z","iopub.status.idle":"2024-12-19T14:36:13.871076Z","shell.execute_reply.started":"2024-12-19T14:36:13.862566Z","shell.execute_reply":"2024-12-19T14:36:13.869292Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"calculate_stats(train, 'CGAS-CGAS_Score')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:13.873264Z","iopub.execute_input":"2024-12-19T14:36:13.873851Z","iopub.status.idle":"2024-12-19T14:36:13.902211Z","shell.execute_reply.started":"2024-12-19T14:36:13.873796Z","shell.execute_reply":"2024-12-19T14:36:13.900645Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Check relationshoip with age and sex","metadata":{}},{"cell_type":"code","source":"sns.scatterplot(data=train, x='Basic_Demos-Age', y='CGAS-CGAS_Score', palette='viridis')\nplt.title(\"Relationship between Age and CGAS Score\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:13.904104Z","iopub.execute_input":"2024-12-19T14:36:13.904579Z","iopub.status.idle":"2024-12-19T14:36:14.252028Z","shell.execute_reply.started":"2024-12-19T14:36:13.90451Z","shell.execute_reply":"2024-12-19T14:36:14.25009Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sns.boxplot(data=train, x='Age Group', y='CGAS-CGAS_Score')\nplt.title(\"CGAS Score Distribution by Age Group\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:14.254238Z","iopub.execute_input":"2024-12-19T14:36:14.254984Z","iopub.status.idle":"2024-12-19T14:36:14.527708Z","shell.execute_reply.started":"2024-12-19T14:36:14.254929Z","shell.execute_reply":"2024-12-19T14:36:14.525901Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"There shouldn't be data for adults. As definition: Children's Global Assessment Scale - Numeric scale used by mental health clinicians to rate the general functioning of youths under the age of 18. Not sure how to handle that so we will just leave it there","metadata":{}},{"cell_type":"code","source":"sns.boxplot(data=train, x='Basic_Demos-Sex', y='CGAS-CGAS_Score')\nplt.title(\"CGAS Score Distribution by Sex\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:14.530452Z","iopub.execute_input":"2024-12-19T14:36:14.531068Z","iopub.status.idle":"2024-12-19T14:36:14.787989Z","shell.execute_reply.started":"2024-12-19T14:36:14.531012Z","shell.execute_reply":"2024-12-19T14:36:14.786165Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"age_group_cgas_stats = train.groupby('Age Group')['CGAS-CGAS_Score'].describe()\nage_group_cgas_stats","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:14.790199Z","iopub.execute_input":"2024-12-19T14:36:14.790687Z","iopub.status.idle":"2024-12-19T14:36:14.829256Z","shell.execute_reply.started":"2024-12-19T14:36:14.790648Z","shell.execute_reply":"2024-12-19T14:36:14.82765Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- The CGAS score distribution differs between the adults group and the children/adolescents groups. However, the adults group has very few samples, and the CGAS test was not designed for adults. Therefore, this information is not usable.\n- The CGAS score distribution is similar in the children and adolescents groups.\n\n=> Conclusion: Age cannot be used to predict the CGAS score.","metadata":{}},{"cell_type":"code","source":"sex_cgas_stats = train.groupby('Basic_Demos-Sex')['CGAS-CGAS_Score'].describe()\nsex_cgas_stats","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:14.831095Z","iopub.execute_input":"2024-12-19T14:36:14.831532Z","iopub.status.idle":"2024-12-19T14:36:14.862151Z","shell.execute_reply.started":"2024-12-19T14:36:14.831493Z","shell.execute_reply":"2024-12-19T14:36:14.86061Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The CGAS distribution is similar for both males and females.\n\nConclusion: Sex cannot be used to predict the CGAS score.","metadata":{}},{"cell_type":"markdown","source":"### Final conclusions: Should not touch the CGAS score yet since the relationship with other columns seems unclear","metadata":{}},{"cell_type":"markdown","source":"Relationship with PCIAT Total:","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\nvalid_data = train.dropna(subset=['CGAS-CGAS_Score', 'PCIAT-PCIAT_Total'])\n\nplt.figure(figsize=(10, 6))\nsns.scatterplot(\n    data=valid_data, \n    x='CGAS-CGAS_Score', \n    y='PCIAT-PCIAT_Total',\n    hue=valid_data['PCIAT-PCIAT_Total'] > 80,  # PCIAT severe group\n    alpha=0.3\n)\n\nplt.title(\"Scatter Plot of CGAS Score vs PCIAT Total\", fontsize=16)\nplt.xlabel(\"CGAS-CGAS_Score\", fontsize=12)\nplt.ylabel(\"PCIAT-PCIAT_Total (PIU Severity)\", fontsize=12)\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:14.863957Z","iopub.execute_input":"2024-12-19T14:36:14.86439Z","iopub.status.idle":"2024-12-19T14:36:15.34515Z","shell.execute_reply.started":"2024-12-19T14:36:14.864355Z","shell.execute_reply":"2024-12-19T14:36:15.343245Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"There are no participants with the highest SII scores (3 or severely problematic internet use) who have good CGAS scores (81-100).\n=> Create a weighted function that focuses on the CGAS > 80 group (since this group does not have severe PIU). Use an inverse sigmoid function to assign higher weights to CGAS > 80.","metadata":{}},{"cell_type":"code","source":"def sigmoid_weight_cgas_high(cgas, a=0.1, b=80):\n    return 1 / (1 + np.exp(-a * (cgas - b)))\n\n# Add below\n# train['CGAS_Weight'] = train['CGAS-CGAS_Score'].apply(sigmoid_weight_cgas_high)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:15.358763Z","iopub.execute_input":"2024-12-19T14:36:15.359242Z","iopub.status.idle":"2024-12-19T14:36:15.365816Z","shell.execute_reply.started":"2024-12-19T14:36:15.359205Z","shell.execute_reply":"2024-12-19T14:36:15.364384Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The CGAS score column will be weighted with this information.","metadata":{}},{"cell_type":"markdown","source":"## Physical Measures","metadata":{}},{"cell_type":"code","source":"physical_columns = [\n 'Physical-BMI',\n 'Physical-Height',\n 'Physical-Weight',\n 'Physical-Waist_Circumference',\n 'Physical-Diastolic_BP',\n 'Physical-HeartRate',\n 'Physical-Systolic_BP'\n]\n\nwh_cols = [\n    'Physical-BMI', 'Physical-Height',\n    'Physical-Weight', 'Physical-Waist_Circumference'\n]\n\nheart_cols = [\n 'Physical-Diastolic_BP',\n 'Physical-HeartRate',\n 'Physical-Systolic_BP'\n]\n\ncalculate_stats(train, wh_cols)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:15.367773Z","iopub.execute_input":"2024-12-19T14:36:15.368201Z","iopub.status.idle":"2024-12-19T14:36:15.408625Z","shell.execute_reply.started":"2024-12-19T14:36:15.368163Z","shell.execute_reply":"2024-12-19T14:36:15.407126Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"(train[wh_cols] == 0).sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:15.410341Z","iopub.execute_input":"2024-12-19T14:36:15.410754Z","iopub.status.idle":"2024-12-19T14:36:15.42292Z","shell.execute_reply.started":"2024-12-19T14:36:15.410718Z","shell.execute_reply":"2024-12-19T14:36:15.421368Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Filter zero values","metadata":{}},{"cell_type":"code","source":"train[wh_cols] = train[wh_cols].replace(0, np.nan)\ntest[wh_cols] = test[wh_cols].replace(0, np.nan)\ncalculate_stats(train, wh_cols)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:15.424515Z","iopub.execute_input":"2024-12-19T14:36:15.424997Z","iopub.status.idle":"2024-12-19T14:36:15.464877Z","shell.execute_reply.started":"2024-12-19T14:36:15.424959Z","shell.execute_reply":"2024-12-19T14:36:15.462836Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Since other columns can be inferred from weight and height, we will first focus on filling the missing values in the weight and height columns. Once these are complete, we will use them to predict the remaining physical attributes.","metadata":{}},{"cell_type":"markdown","source":"Encode physical seasons:","metadata":{}},{"cell_type":"code","source":"encoded_season_train = pd.get_dummies(train, columns=['Basic_Demos-Enroll_Season'], prefix='Season', drop_first=False)\nencoded_season_test = pd.get_dummies(test, columns=['Basic_Demos-Enroll_Season'], prefix='Season', drop_first=False)\ntrain = train.join(encoded_season_train[['Season_Fall', 'Season_Spring', 'Season_Summer', 'Season_Winter']])\ntest = test.join(encoded_season_test[['Season_Fall', 'Season_Spring', 'Season_Summer', 'Season_Winter']])\ntrain.columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:15.466716Z","iopub.execute_input":"2024-12-19T14:36:15.467178Z","iopub.status.idle":"2024-12-19T14:36:15.50439Z","shell.execute_reply.started":"2024-12-19T14:36:15.467142Z","shell.execute_reply":"2024-12-19T14:36:15.502669Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Check the relationship between enroll season and w, h","metadata":{}},{"cell_type":"code","source":"train.groupby('Basic_Demos-Enroll_Season')[['Physical-Weight', 'Physical-Height']].mean()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:15.506058Z","iopub.execute_input":"2024-12-19T14:36:15.506446Z","iopub.status.idle":"2024-12-19T14:36:15.524153Z","shell.execute_reply.started":"2024-12-19T14:36:15.506413Z","shell.execute_reply":"2024-12-19T14:36:15.522367Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.groupby(['Basic_Demos-Enroll_Season', 'Basic_Demos-Sex'])[['Physical-Weight', 'Physical-Height']].mean()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:15.526217Z","iopub.execute_input":"2024-12-19T14:36:15.526713Z","iopub.status.idle":"2024-12-19T14:36:15.54803Z","shell.execute_reply.started":"2024-12-19T14:36:15.526649Z","shell.execute_reply":"2024-12-19T14:36:15.545933Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- Weight:\nFor each season, females tend to have lower or similar weight compared to males.\nExample:\nFall: 87.21 (female) vs. 90.43 (male)\nWinter: 92.12 (female) ~ 91.98 (male)\nWeight tends to be higher in Summer and Winter and lower in Fall and Spring, but the difference between seasons is minimal (1–5 kg).\n\n- Height:\nMales are consistently taller than females across all seasons, but the difference is relatively small (0.1–1 cm).\nExample:\nSpring: 55.18 (female) vs. 56.15 (male)\nWinter: 56.16 (female) vs. 56.39 (male)\nSeasonal variation in height is negligible, with a maximum difference of around 1 cm between seasons.\n\n=> Conclusion:\nWeight: There are slight differences across seasons and genders, particularly with higher weights in Summer and Winter. However, these variations are minor and could be influenced by other factors like age or lifestyle.\nHeight: Seasonal effects on height are almost insignificant, as the average difference between seasons is less than 1 cm.","metadata":{}},{"cell_type":"markdown","source":"However there is very little data so we will still be using all 3 three columns season, age and sex to predict missing values for w and h","metadata":{}},{"cell_type":"markdown","source":"Convert to cm and kg","metadata":{}},{"cell_type":"code","source":"lbs_to_kg = 0.453592\ninches_to_cm = 2.54\n\ndef process_physical_BMI(df):\n    df['Physical-Weight'] = df['Physical-Weight'] * lbs_to_kg\n    df['Physical-Height'] = df['Physical-Height'] * inches_to_cm\n    df['Physical-Waist_Circumference'] = df['Physical-Waist_Circumference'] * inches_to_cm\n    \n    df['Physical-BMI'] = np.where(\n        df['Physical-Weight'].notna() & df['Physical-Height'].notna(),\n        df['Physical-Weight'] / ((df['Physical-Height'] / 100) ** 2),\n        np.nan\n    )\n    \n    return df\n\ntrain = process_physical_BMI(train)\ntest = process_physical_BMI(test)\n\ncalculate_stats(train, wh_cols)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:15.550131Z","iopub.execute_input":"2024-12-19T14:36:15.550746Z","iopub.status.idle":"2024-12-19T14:36:15.598411Z","shell.execute_reply.started":"2024-12-19T14:36:15.550691Z","shell.execute_reply":"2024-12-19T14:36:15.596905Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Fill missing data for w and h using knn. We will only use Season_Fall and Season_Winter, as these are the two season columns with the highest height and weight differences among participants. This is to avoid potential bias from using seasons with smaller differences in weight and height, which could lead to less accurate predictions.","metadata":{}},{"cell_type":"code","source":"from sklearn.impute import KNNImputer\n\n# Chọn 2 mùa có chiều cao cân nặng chênh nhau nhiều nhất\nimputer = KNNImputer(n_neighbors=10)\n\nselected_features = ['Basic_Demos-Age', 'Season_Fall', 'Season_Winter', 'Basic_Demos-Sex', 'Physical-Weight', 'Physical-Height']\n\nimputed_data = imputer.fit_transform(train[selected_features])\ntrain_imputed = pd.DataFrame(imputed_data, columns=selected_features)\ntrain = train.drop(columns=selected_features).reset_index()\n\nimputed_test_data = imputer.transform(test[selected_features])\ntest_imputed = pd.DataFrame(imputed_test_data, columns=selected_features)\ntest = test.drop(columns=selected_features).reset_index()\n\ntrain = pd.concat([train, train_imputed], axis=1)\ntest = pd.concat([test, test_imputed], axis=1)\n\ncalculate_stats(train, ['Physical-Weight', 'Physical-Height'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:15.600503Z","iopub.execute_input":"2024-12-19T14:36:15.601886Z","iopub.status.idle":"2024-12-19T14:36:15.771293Z","shell.execute_reply.started":"2024-12-19T14:36:15.601822Z","shell.execute_reply":"2024-12-19T14:36:15.769828Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Recalculate BMI from w and h","metadata":{}},{"cell_type":"code","source":"train['Physical-BMI'] = train.apply(\n    lambda row: row['Physical-Weight'] / (row['Physical-Height'] / 100) ** 2 \n    if pd.isnull(row['Physical-BMI']) else row['Physical-BMI'], axis=1\n)\ntest['Physical-BMI'] = test.apply(\n    lambda row: row['Physical-Weight'] / (row['Physical-Height'] / 100) ** 2 \n    if pd.isnull(row['Physical-BMI']) else row['Physical-BMI'], axis=1\n)\ncalculate_stats(train, ['Physical-BMI'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:15.772974Z","iopub.execute_input":"2024-12-19T14:36:15.773317Z","iopub.status.idle":"2024-12-19T14:36:15.845183Z","shell.execute_reply.started":"2024-12-19T14:36:15.773287Z","shell.execute_reply":"2024-12-19T14:36:15.843675Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Compare with the BIA-BIA_BMI column, which is also the BMI index measured in the same year.","metadata":{}},{"cell_type":"code","source":"bmi_ratio = train['Physical-BMI'] / train['BIA-BIA_BMI']\ncolor = (bmi_ratio < 0.8) | (bmi_ratio > 1.2)  # red if difference > 30%\n\nplt.scatter(\n    train['Physical-BMI'],\n    train['BIA-BIA_BMI'],\n    s=6,\n    c=color,\n    cmap='coolwarm'\n)\nplt.gca().set_aspect('equal')\nplt.xlabel('Physical-BMI')\nplt.ylabel('BIA-BIA_BMI')\nplt.title('Physical-BMI vs VIA-BIA_BMI')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:15.847138Z","iopub.execute_input":"2024-12-19T14:36:15.847691Z","iopub.status.idle":"2024-12-19T14:36:16.135391Z","shell.execute_reply.started":"2024-12-19T14:36:15.84763Z","shell.execute_reply":"2024-12-19T14:36:16.133811Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The red points indicate that one of the two columns is incorrect, as BMI does not change significantly within a year. Replace the BIA-BMI of these points with the newly calculated Physical-BMI.","metadata":{}},{"cell_type":"code","source":"train.loc[color, 'BIA-BIA_BMI'] = train.loc[color, 'Physical-BMI']\nplt.scatter(\n    train['Physical-BMI'],\n    train['BIA-BIA_BMI'],\n    s=6,\n    c=color,\n    cmap='coolwarm'\n)\nplt.gca().set_aspect('equal')\nplt.xlabel('Physical-BMI')\nplt.ylabel('BIA-BIA_BMI')\nplt.title('Physical-BMI vs VIA-BIA_BMI')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:16.137467Z","iopub.execute_input":"2024-12-19T14:36:16.138073Z","iopub.status.idle":"2024-12-19T14:36:16.461319Z","shell.execute_reply.started":"2024-12-19T14:36:16.138022Z","shell.execute_reply":"2024-12-19T14:36:16.459648Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Waist is important because sitting a lot leads to a bigger belly. :))\nThere are many missing values for Waist (2239), so KNN will not be used.\nWaist Circumference (cm) = a × BMI + b × Height (cm) + c\n=> Using regression model","metadata":{}},{"cell_type":"code","source":"waist_data = train.dropna(subset=['Physical-Waist_Circumference'])\n\nX = waist_data[['Physical-BMI', 'Physical-Weight']]\ny = waist_data['Physical-Waist_Circumference']\n\nfrom sklearn.linear_model import LinearRegression\nwaist_model = LinearRegression()\nwaist_model.fit(X, y)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:16.463155Z","iopub.execute_input":"2024-12-19T14:36:16.463515Z","iopub.status.idle":"2024-12-19T14:36:16.486162Z","shell.execute_reply.started":"2024-12-19T14:36:16.463483Z","shell.execute_reply":"2024-12-19T14:36:16.484408Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Dự đoán các giá trị bị thiếu","metadata":{}},{"cell_type":"code","source":"missing_waist = train['Physical-Waist_Circumference'].isnull()\ntrain.loc[missing_waist, 'Physical-Waist_Circumference'] = waist_model.predict(\n    train.loc[missing_waist, ['Physical-BMI', 'Physical-Weight']]\n)\n\nmissing_waist_test = test['Physical-Waist_Circumference'].isnull()\ntest.loc[missing_waist_test, 'Physical-Waist_Circumference'] = waist_model.predict(\n    test.loc[missing_waist_test, ['Physical-BMI', 'Physical-Weight']]\n)\n\ncalculate_stats(train, wh_cols)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:16.488341Z","iopub.execute_input":"2024-12-19T14:36:16.48892Z","iopub.status.idle":"2024-12-19T14:36:16.535135Z","shell.execute_reply.started":"2024-12-19T14:36:16.488864Z","shell.execute_reply":"2024-12-19T14:36:16.533657Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Blood Pressure và Heart Rate","metadata":{}},{"cell_type":"code","source":"bp_hr_cols = [\n    'Physical-Diastolic_BP', 'Physical-Systolic_BP',\n    'Physical-HeartRate'\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:16.537296Z","iopub.execute_input":"2024-12-19T14:36:16.537889Z","iopub.status.idle":"2024-12-19T14:36:16.544371Z","shell.execute_reply.started":"2024-12-19T14:36:16.53783Z","shell.execute_reply":"2024-12-19T14:36:16.54282Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"(train[bp_hr_cols] < 50).sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:16.545992Z","iopub.execute_input":"2024-12-19T14:36:16.546364Z","iopub.status.idle":"2024-12-19T14:36:16.564324Z","shell.execute_reply.started":"2024-12-19T14:36:16.54633Z","shell.execute_reply":"2024-12-19T14:36:16.562651Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train[train['Physical-Systolic_BP'] <= train['Physical-Diastolic_BP']][bp_hr_cols]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:16.56611Z","iopub.execute_input":"2024-12-19T14:36:16.566506Z","iopub.status.idle":"2024-12-19T14:36:16.587171Z","shell.execute_reply.started":"2024-12-19T14:36:16.566472Z","shell.execute_reply":"2024-12-19T14:36:16.585472Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Filter rows where heartrate < 50 and systolic < diastolic.","metadata":{}},{"cell_type":"code","source":"train[bp_hr_cols] = train[bp_hr_cols].replace(0, np.nan)\ntrain.loc[train['Physical-Systolic_BP'] <= train['Physical-Diastolic_BP'], bp_hr_cols] = np.nan","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:16.589028Z","iopub.execute_input":"2024-12-19T14:36:16.589512Z","iopub.status.idle":"2024-12-19T14:36:16.609729Z","shell.execute_reply.started":"2024-12-19T14:36:16.589461Z","shell.execute_reply":"2024-12-19T14:36:16.607833Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Not sure what to do with this data since it can be varied so we will leave it there","metadata":{}},{"cell_type":"markdown","source":"## Sleep Disturbance Scale","metadata":{}},{"cell_type":"markdown","source":"We expect this data to be important since us ourselves experience severe sleep disturbance from this course ","metadata":{}},{"cell_type":"code","source":"SDS_columns = ['SDS-Season', 'SDS-SDS_Total_Raw', 'SDS-SDS_Total_T']\n\nSDS_number_columns = ['SDS-SDS_Total_Raw', 'SDS-SDS_Total_T']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:16.611521Z","iopub.execute_input":"2024-12-19T14:36:16.612182Z","iopub.status.idle":"2024-12-19T14:36:16.623403Z","shell.execute_reply.started":"2024-12-19T14:36:16.612142Z","shell.execute_reply":"2024-12-19T14:36:16.62197Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"calculate_stats(train, 'SDS-Season')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:16.625567Z","iopub.execute_input":"2024-12-19T14:36:16.626026Z","iopub.status.idle":"2024-12-19T14:36:16.648841Z","shell.execute_reply.started":"2024-12-19T14:36:16.625989Z","shell.execute_reply":"2024-12-19T14:36:16.647437Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"calculate_stats(train, SDS_number_columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:16.65112Z","iopub.execute_input":"2024-12-19T14:36:16.651613Z","iopub.status.idle":"2024-12-19T14:36:16.683886Z","shell.execute_reply.started":"2024-12-19T14:36:16.651534Z","shell.execute_reply":"2024-12-19T14:36:16.682602Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(18, 5))\n\n# SDS-SDS_Total_Raw\nplt.subplot(1, 3, 2)\nsns.histplot(train['SDS-SDS_Total_Raw'].dropna(), bins=20, kde=True)\nplt.title('SDS-SDS_Total_Raw')\nplt.xlabel('Value')\n\n# SDS-SDS_Total_T\nplt.subplot(1, 3, 3)\nsns.histplot(train['SDS-SDS_Total_T'].dropna(), bins=20, kde=True)\nplt.title('SDS-SDS_Total_T')\nplt.xlabel('Value')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:16.68569Z","iopub.execute_input":"2024-12-19T14:36:16.687316Z","iopub.status.idle":"2024-12-19T14:36:17.443286Z","shell.execute_reply.started":"2024-12-19T14:36:16.687256Z","shell.execute_reply":"2024-12-19T14:36:17.441605Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.title('Sleep disturbance scale: conversion from raw to t score')\nplt.scatter(train['SDS-SDS_Total_Raw'],\n            train['SDS-SDS_Total_T'],\n            color='brown')\nplt.xlabel('SDS-SDS_Total_Raw')\nplt.ylabel('SDS-SDS_Total_T')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:17.445232Z","iopub.execute_input":"2024-12-19T14:36:17.445834Z","iopub.status.idle":"2024-12-19T14:36:17.737954Z","shell.execute_reply.started":"2024-12-19T14:36:17.445778Z","shell.execute_reply":"2024-12-19T14:36:17.736365Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"There are many SDS_Total_Raw values rounded to the same value as SDS_Total_T. Since the SDS_Total_Raw column contains more information than SDS_Total_T, drop SDS_Total_T should be fine.","metadata":{}},{"cell_type":"code","source":"train = train.drop(columns=['SDS-SDS_Total_T'])\ntest = test.drop(columns=['SDS-SDS_Total_T'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:17.739563Z","iopub.execute_input":"2024-12-19T14:36:17.739944Z","iopub.status.idle":"2024-12-19T14:36:17.752088Z","shell.execute_reply.started":"2024-12-19T14:36:17.739911Z","shell.execute_reply":"2024-12-19T14:36:17.750361Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Check extreme sleep disturbance cases","metadata":{}},{"cell_type":"code","source":"cols=['Basic_Demos-Age', 'Age Group', 'SDS-SDS_Total_Raw', 'PCIAT-PCIAT_Total']\ntrain[train['SDS-SDS_Total_Raw'] > 70][cols]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:17.754057Z","iopub.execute_input":"2024-12-19T14:36:17.754536Z","iopub.status.idle":"2024-12-19T14:36:17.788973Z","shell.execute_reply.started":"2024-12-19T14:36:17.7545Z","shell.execute_reply":"2024-12-19T14:36:17.787696Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train[train['SDS-SDS_Total_Raw'] > 70].groupby('Age Group').size()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:17.791961Z","iopub.execute_input":"2024-12-19T14:36:17.792663Z","iopub.status.idle":"2024-12-19T14:36:17.812002Z","shell.execute_reply.started":"2024-12-19T14:36:17.792594Z","shell.execute_reply":"2024-12-19T14:36:17.809954Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"There are 28 children experiencing severe sleep disturbance, which might not be true since they are... children? We will remove these values (capping technique to avoid extreme values)","metadata":{}},{"cell_type":"code","source":"train['SDS-SDS_Total_Raw'] = train.apply(\n    lambda row: np.nan if row['Age Group'] == 'Children' and row['SDS-SDS_Total_Raw'] > 70 else row['SDS-SDS_Total_Raw'], \n    axis=1\n)\ntrain[train['SDS-SDS_Total_Raw'] > 70][cols]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:17.81422Z","iopub.execute_input":"2024-12-19T14:36:17.815812Z","iopub.status.idle":"2024-12-19T14:36:17.88133Z","shell.execute_reply.started":"2024-12-19T14:36:17.815733Z","shell.execute_reply":"2024-12-19T14:36:17.880073Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Check the relationship with PIU","metadata":{}},{"cell_type":"code","source":"valid_data = train.dropna(subset=['SDS-SDS_Total_Raw', 'PCIAT-PCIAT_Total'])\n\nplt.figure(figsize=(10, 6))\nsns.scatterplot(\n    data=valid_data, \n    x='SDS-SDS_Total_Raw', \n    y='PCIAT-PCIAT_Total',\n    hue=valid_data['PCIAT-PCIAT_Total'] > 80,  # PCIAT severe group\n    alpha=0.3\n)\n\nplt.title(\"Scatter Plot of SDS Total Raw vs PCIAT Total\", fontsize=16)\nplt.xlabel(\"SDS-SDS_Total_Raw (Sleep Disturbance Score)\", fontsize=12)\nplt.ylabel(\"PCIAT-PCIAT_Total (PIU Severity)\", fontsize=12)\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:17.883066Z","iopub.execute_input":"2024-12-19T14:36:17.883577Z","iopub.status.idle":"2024-12-19T14:36:18.388418Z","shell.execute_reply.started":"2024-12-19T14:36:17.88351Z","shell.execute_reply":"2024-12-19T14:36:18.386896Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- Not so clear linear trend\n\n- The data is concentrated in the lower left corner. This indicates that most of the samples have low SDS and PCIAT values, meaning the majority of survey participants do not have severe sleep disturbances or heavy Internet addiction.\n\n- When SDS < 35, there is no PCIAT > 80 (SII = 3). This suggests that individuals with low sleep disturbances generally do not have severe Internet addiction. This reinforces the relationship between sleep disturbances and Internet addiction: sleep disturbances could be a predictor of severe Internet addiction.\n\n=> Conclusion: Add a new column for SDS_Weight.","metadata":{}},{"cell_type":"markdown","source":"SDS-Season","metadata":{}},{"cell_type":"code","source":"season_stats = train.groupby('SDS-Season')['SDS-SDS_Total_Raw'].agg(['mean', 'median'])\nseason_stats","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:18.390196Z","iopub.execute_input":"2024-12-19T14:36:18.390672Z","iopub.status.idle":"2024-12-19T14:36:18.411088Z","shell.execute_reply.started":"2024-12-19T14:36:18.390626Z","shell.execute_reply":"2024-12-19T14:36:18.409474Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"SDS is higher in winter","metadata":{}},{"cell_type":"markdown","source":"Fill SDS-Season with the ratio of each season's occurrence.","metadata":{}},{"cell_type":"code","source":"season_counts = train['SDS-Season'].value_counts(normalize=True)\n\ntrain['SDS-Season'] = train['SDS-Season'].apply(\n    lambda x: np.random.choice(season_counts.index, p=season_counts.values) if pd.isna(x) else x\n)\ntest['SDS-Season'] = test['SDS-Season'].apply(\n    lambda x: np.random.choice(season_counts.index, p=season_counts.values) if pd.isna(x) else x\n)\n\ncalculate_stats(train, 'SDS-Season')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:18.413629Z","iopub.execute_input":"2024-12-19T14:36:18.414167Z","iopub.status.idle":"2024-12-19T14:36:18.460148Z","shell.execute_reply.started":"2024-12-19T14:36:18.414118Z","shell.execute_reply":"2024-12-19T14:36:18.458852Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Check relationship between Age Sex and SDS score","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(8, 6))\nsns.boxplot(x='Basic_Demos-Sex', y='SDS-SDS_Total_Raw', data=train)\nplt.title('Distribution of SDS by Sex')\nplt.xlabel('Sex (0 = Male, 1 = Female)')\nplt.ylabel('SDS-SDS_Total_Raw')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:18.462175Z","iopub.execute_input":"2024-12-19T14:36:18.462819Z","iopub.status.idle":"2024-12-19T14:36:18.723041Z","shell.execute_reply.started":"2024-12-19T14:36:18.46276Z","shell.execute_reply":"2024-12-19T14:36:18.721381Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"It looks same to us...","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(8, 6))\nsns.boxplot(x='Age Group', y='SDS-SDS_Total_Raw', data=train)\nplt.title('Boxplot of SDS by Age Group')\nplt.xlabel('Age Group')\nplt.ylabel('SDS-SDS_Total_Raw')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:18.724586Z","iopub.execute_input":"2024-12-19T14:36:18.725061Z","iopub.status.idle":"2024-12-19T14:36:19.016991Z","shell.execute_reply.started":"2024-12-19T14:36:18.725023Z","shell.execute_reply":"2024-12-19T14:36:19.01564Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- Adults have no outliers with high SDS, but this age group has very little data.\n- Adolescents have the highest median SDS, which makes sense.","metadata":{}},{"cell_type":"markdown","source":"\n=> Use Age and SDS-Season to predict the SDS score","metadata":{}},{"cell_type":"markdown","source":"Encode season columns","metadata":{}},{"cell_type":"code","source":"encoded_season_df = pd.get_dummies(train, columns=['SDS-Season'], prefix='SDS-Season', drop_first=False)\ntrain = train.join(encoded_season_df[['SDS-Season_Fall', 'SDS-Season_Spring', 'SDS-Season_Summer', 'SDS-Season_Winter']])\n\nencoded_season_df = pd.get_dummies(test, columns=['SDS-Season'], prefix='SDS-Season', drop_first=False)\ntest = test.join(encoded_season_df[['SDS-Season_Fall', 'SDS-Season_Spring', 'SDS-Season_Summer', 'SDS-Season_Winter']])\n\ntrain.columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:19.019133Z","iopub.execute_input":"2024-12-19T14:36:19.0197Z","iopub.status.idle":"2024-12-19T14:36:19.051626Z","shell.execute_reply.started":"2024-12-19T14:36:19.019648Z","shell.execute_reply":"2024-12-19T14:36:19.049681Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Fill missing SDS-SDS_Total_Raw values with Age SDS-season by KNN","metadata":{}},{"cell_type":"code","source":"X_train_SDS_Score = train[['Basic_Demos-Age', 'SDS-Season_Fall', 'SDS-Season_Spring', 'SDS-Season_Winter']]\n\ny_train_SDS_Score = train['SDS-SDS_Total_Raw']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:19.053913Z","iopub.execute_input":"2024-12-19T14:36:19.054395Z","iopub.status.idle":"2024-12-19T14:36:19.063023Z","shell.execute_reply.started":"2024-12-19T14:36:19.054355Z","shell.execute_reply":"2024-12-19T14:36:19.061113Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sds_imputer = KNNImputer(n_neighbors=7)\n\ndata_with_sds = pd.concat([X_train_SDS_Score, y_train_SDS_Score], axis=1)\n\nfilled_data = sds_imputer.fit_transform(data_with_sds)\n\ncolumns = data_with_sds.columns.tolist()\nfilled_df = pd.DataFrame(filled_data, columns=columns)\n\ntrain['SDS-SDS_Total_Raw'] = filled_df['SDS-SDS_Total_Raw']\ncalculate_stats(train, ['SDS-SDS_Total_Raw'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:19.065331Z","iopub.execute_input":"2024-12-19T14:36:19.065922Z","iopub.status.idle":"2024-12-19T14:36:19.190213Z","shell.execute_reply.started":"2024-12-19T14:36:19.06586Z","shell.execute_reply":"2024-12-19T14:36:19.188346Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Fill test data with trained imputer","metadata":{}},{"cell_type":"code","source":"X_test_SDS_Score = test[['Basic_Demos-Age', 'SDS-Season_Fall', 'SDS-Season_Spring', 'SDS-Season_Winter']]\ny_test_SDS_Score = test['SDS-SDS_Total_Raw']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:19.191978Z","iopub.execute_input":"2024-12-19T14:36:19.196103Z","iopub.status.idle":"2024-12-19T14:36:19.21068Z","shell.execute_reply.started":"2024-12-19T14:36:19.196013Z","shell.execute_reply":"2024-12-19T14:36:19.208811Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_test_with_sds = pd.concat([X_test_SDS_Score, y_test_SDS_Score], axis=1)\nfilled_test_data = sds_imputer.transform(data_test_with_sds)\nfilled_test_df = pd.DataFrame(filled_test_data, columns=data_test_with_sds.columns)\ntest['SDS-SDS_Total_Raw'] = filled_test_df['SDS-SDS_Total_Raw']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:19.212493Z","iopub.execute_input":"2024-12-19T14:36:19.213169Z","iopub.status.idle":"2024-12-19T14:36:19.233831Z","shell.execute_reply.started":"2024-12-19T14:36:19.2131Z","shell.execute_reply":"2024-12-19T14:36:19.231982Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Add SDS_Weight","metadata":{}},{"cell_type":"code","source":"def sigmoid_weight(sds, a=0.1, b=35):\n    return 1 / (1 + np.exp(a * (sds - b)))\n\ntrain['SDS_Weight'] = train['SDS-SDS_Total_Raw'].apply(sigmoid_weight)\ntest['SDS_Weight'] = test['SDS-SDS_Total_Raw'].apply(sigmoid_weight)\ntrain[['SDS_Weight', 'SDS-SDS_Total_Raw']]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:19.235276Z","iopub.execute_input":"2024-12-19T14:36:19.235783Z","iopub.status.idle":"2024-12-19T14:36:19.26492Z","shell.execute_reply.started":"2024-12-19T14:36:19.235728Z","shell.execute_reply":"2024-12-19T14:36:19.263715Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Internet Use","metadata":{}},{"cell_type":"markdown","source":"This should be important too","metadata":{}},{"cell_type":"code","source":"calculate_stats(train, 'PreInt_EduHx-Season')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:19.266883Z","iopub.execute_input":"2024-12-19T14:36:19.267352Z","iopub.status.idle":"2024-12-19T14:36:19.288774Z","shell.execute_reply.started":"2024-12-19T14:36:19.267316Z","shell.execute_reply":"2024-12-19T14:36:19.287115Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"calculate_stats(train, ['PreInt_EduHx-computerinternet_hoursday'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:19.29065Z","iopub.execute_input":"2024-12-19T14:36:19.291104Z","iopub.status.idle":"2024-12-19T14:36:19.315593Z","shell.execute_reply.started":"2024-12-19T14:36:19.291062Z","shell.execute_reply":"2024-12-19T14:36:19.313997Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Relationshiop with age and sex","metadata":{}},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 2, figsize=(18, 5))\n\n# Hours of Internet Use by Age\nsns.boxplot(y=train['Basic_Demos-Age'], x=train['PreInt_EduHx-computerinternet_hoursday'], ax=axes[0], palette=\"Set3\")\naxes[0].set_title('Hours of Internet Use by Age')\naxes[0].set_ylabel('Age')\naxes[0].set_xlabel('Hours per Day Group')\n\n# Hours of Internet Use by Age Group\nsns.boxplot(y='PreInt_EduHx-computerinternet_hoursday', x='Age Group', data=train, ax=axes[1], palette=\"Set3\")\naxes[1].set_title('Internet Hours by Age Group')\naxes[1].set_ylabel('Hours per Day (Numeric)')\naxes[1].set_xlabel('Age Group')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:19.317086Z","iopub.execute_input":"2024-12-19T14:36:19.317458Z","iopub.status.idle":"2024-12-19T14:36:20.439054Z","shell.execute_reply.started":"2024-12-19T14:36:19.317425Z","shell.execute_reply":"2024-12-19T14:36:20.437605Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"stats = train.groupby(['Basic_Demos-Sex', 'PreInt_EduHx-computerinternet_hoursday']\n).size().unstack(fill_value=0)\nstats_prop = stats.div(stats.sum(axis=1), axis=0) * 100\n\nstats = stats.astype(str) +' (' + stats_prop.round(1).astype(str) + '%)'\nstats","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:20.440709Z","iopub.execute_input":"2024-12-19T14:36:20.44111Z","iopub.status.idle":"2024-12-19T14:36:20.464303Z","shell.execute_reply.started":"2024-12-19T14:36:20.441074Z","shell.execute_reply":"2024-12-19T14:36:20.462938Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- Older age leads to more internet use => create a new feature combining age and internet use.\n- Sex is not useful","metadata":{}},{"cell_type":"markdown","source":"Relationship with sii","metadata":{}},{"cell_type":"code","source":"stats = train.groupby(\n    ['sii', 'PreInt_EduHx-computerinternet_hoursday']\n).size().unstack(fill_value=0)\nstats_prop = stats.div(stats.sum(axis=1), axis=0) * 100\n\nstats = stats.astype(str) +' (' + stats_prop.round(1).astype(str) + '%)'\nstats","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:20.466065Z","iopub.execute_input":"2024-12-19T14:36:20.46659Z","iopub.status.idle":"2024-12-19T14:36:20.490832Z","shell.execute_reply.started":"2024-12-19T14:36:20.466517Z","shell.execute_reply":"2024-12-19T14:36:20.489174Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(8, 6))\nsns.boxplot(\n    x='PreInt_EduHx-computerinternet_hoursday', y='PCIAT-PCIAT_Total',\n    data=train,\n    hue='Age Group', palette=\"Set3\"\n)\nplt.title('PCIAT_Total vs Hours of Internet Use by Age Group')\nplt.ylabel('PCIAT_Total')\nplt.xlabel('Hours per Day Group')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:20.492689Z","iopub.execute_input":"2024-12-19T14:36:20.493258Z","iopub.status.idle":"2024-12-19T14:36:21.075056Z","shell.execute_reply.started":"2024-12-19T14:36:20.493214Z","shell.execute_reply":"2024-12-19T14:36:21.073305Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- Many people use the internet for <1 hour/day but still have SII = 2 or 3\n=> Check other indicators to see if there are any factors contributing to PIU that are unrelated to internet use.\n- Many people use the internet for >3 hours/day but still have SII = 0 or 1\nCheck these two groups.","metadata":{}},{"cell_type":"markdown","source":"High internet use low sii group:","metadata":{}},{"cell_type":"code","source":"high_sii_high_internet = train[(train['sii'] >= 2) & (train['PreInt_EduHx-computerinternet_hoursday'] >= 2)]\nlow_sii_high_internet = train[(train['sii'] < 2) & (train['PreInt_EduHx-computerinternet_hoursday'] >= 2)]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:21.077118Z","iopub.execute_input":"2024-12-19T14:36:21.077648Z","iopub.status.idle":"2024-12-19T14:36:21.091438Z","shell.execute_reply.started":"2024-12-19T14:36:21.077602Z","shell.execute_reply":"2024-12-19T14:36:21.089313Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"high_sii_means = [\n    high_sii_high_internet['Basic_Demos-Age'].mean(),\n    high_sii_high_internet['Basic_Demos-Sex'].mean(),\n    high_sii_high_internet['Physical-Weight'].mean(),\n    high_sii_high_internet['Physical-Height'].mean(),\n    high_sii_high_internet['SDS-SDS_Total_Raw'].mean(),\n    high_sii_high_internet['CGAS-CGAS_Score'].mean(),\n    len(high_sii_high_internet),\n]\nlow_sii_means = [\n    low_sii_high_internet['Basic_Demos-Age'].mean(),\n    low_sii_high_internet['Basic_Demos-Sex'].mean(),\n    low_sii_high_internet['Physical-Weight'].mean(),\n    low_sii_high_internet['Physical-Height'].mean(),\n    low_sii_high_internet['SDS-SDS_Total_Raw'].mean(),\n    low_sii_high_internet['CGAS-CGAS_Score'].mean(),\n    len(low_sii_high_internet),\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:21.093335Z","iopub.execute_input":"2024-12-19T14:36:21.09385Z","iopub.status.idle":"2024-12-19T14:36:21.109193Z","shell.execute_reply.started":"2024-12-19T14:36:21.093811Z","shell.execute_reply":"2024-12-19T14:36:21.107318Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"labels = ['Age', 'Sex (0=Male)', 'Weight', 'Height', 'SDS', 'CGAS', 'Total Individuals']\n\nfig, axes = plt.subplots(1, 7, figsize=(20, 5), sharey=False)\n\nfor i, ax in enumerate(axes):\n    ax.bar(['High SII', 'Low SII'], [high_sii_means[i], low_sii_means[i]], color=['blue', 'orange'])\n    ax.set_title(labels[i])\n    ax.set_ylabel('Value' if i != 6 else 'Count')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:21.11138Z","iopub.execute_input":"2024-12-19T14:36:21.113057Z","iopub.status.idle":"2024-12-19T14:36:22.366139Z","shell.execute_reply.started":"2024-12-19T14:36:21.11297Z","shell.execute_reply":"2024-12-19T14:36:22.364474Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- There is a large difference in quantity between the groups.\n- The high SII group is older, which makes sense => Create a new feature combining age and internet hours.\n- Higher w, h, and SDS may just reflect age.\n- There is a trend where males have higher SII => Create a new feature combining age and sex (we ran out of time).","metadata":{}},{"cell_type":"markdown","source":"Low internet high piu:","metadata":{}},{"cell_type":"code","source":"high_sii_low_internet = train[(train['sii'] >= 2) & (train['PreInt_EduHx-computerinternet_hoursday'] < 2)]\nlow_sii_low_internet = train[(train['sii'] < 2) & (train['PreInt_EduHx-computerinternet_hoursday'] < 2)]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:22.368038Z","iopub.execute_input":"2024-12-19T14:36:22.368471Z","iopub.status.idle":"2024-12-19T14:36:22.382419Z","shell.execute_reply.started":"2024-12-19T14:36:22.368433Z","shell.execute_reply":"2024-12-19T14:36:22.380943Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"high_sii_means = [\n    high_sii_low_internet['Basic_Demos-Age'].mean(),\n    high_sii_low_internet['Basic_Demos-Sex'].mean(),\n    high_sii_low_internet['Physical-Weight'].mean(),\n    high_sii_low_internet['Physical-Height'].mean(),\n    high_sii_low_internet['SDS-SDS_Total_Raw'].mean(),\n    high_sii_low_internet['CGAS-CGAS_Score'].mean(),\n    len(high_sii_low_internet)\n]\nlow_sii_means = [\n    low_sii_low_internet['Basic_Demos-Age'].mean(),\n    low_sii_low_internet['Basic_Demos-Sex'].mean(),\n    low_sii_low_internet['Physical-Weight'].mean(),\n    low_sii_low_internet['Physical-Height'].mean(),\n    low_sii_low_internet['SDS-SDS_Total_Raw'].mean(),\n    low_sii_low_internet['CGAS-CGAS_Score'].mean(),\n    len(low_sii_low_internet)\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:22.384692Z","iopub.execute_input":"2024-12-19T14:36:22.385232Z","iopub.status.idle":"2024-12-19T14:36:22.397195Z","shell.execute_reply.started":"2024-12-19T14:36:22.385182Z","shell.execute_reply":"2024-12-19T14:36:22.39551Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"labels = ['Age', 'Sex (1=Male)', 'Weight', 'Height', 'SDS_Total_Raw', 'CGAS_Score', 'Total Individuals']\n\ngroups = ['High SII', 'Low SII']\ndata = [high_sii_means, low_sii_means]\n\nfig, axes = plt.subplots(1, 7, figsize=(24, 6), sharey=False)\n\nfor i, ax in enumerate(axes):\n    ax.bar(groups, [data[0][i], data[1][i]], color=['blue', 'orange'])\n    ax.set_title(labels[i])\n    ax.set_ylabel('Value' if i != 6 else 'Count')\n\nplt.title('Low Internet')\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:22.399437Z","iopub.execute_input":"2024-12-19T14:36:22.399942Z","iopub.status.idle":"2024-12-19T14:36:23.653488Z","shell.execute_reply.started":"2024-12-19T14:36:22.399902Z","shell.execute_reply":"2024-12-19T14:36:23.651649Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- The number of people with low SII is higher than those with high SII, which makes sense.\n- The number of people with high SII is lower than those with high internet use, which makes sense.","metadata":{}},{"cell_type":"markdown","source":"Conclusion: In both cases of high SII with low internet use and high SII with high internet use, the majority are male. A feature from sex and internet use might be useful. And of course a feature from age and internet use too","metadata":{}},{"cell_type":"markdown","source":"### PreInt_EduHx-Season","metadata":{}},{"cell_type":"markdown","source":"Fill missing season cols","metadata":{}},{"cell_type":"code","source":"season_counts = train['PreInt_EduHx-Season'].value_counts(normalize=True)\n\ntrain['PreInt_EduHx-Season'] = train['PreInt_EduHx-Season'].apply(lambda x: np.random.choice(season_counts.index, p=season_counts.values) if pd.isna(x) else x)\ntest['PreInt_EduHx-Season'] = test['PreInt_EduHx-Season'].apply(lambda x: np.random.choice(season_counts.index, p=season_counts.values) if pd.isna(x) else x)\n\ncalculate_stats(train, 'PreInt_EduHx-Season')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:23.655679Z","iopub.execute_input":"2024-12-19T14:36:23.656176Z","iopub.status.idle":"2024-12-19T14:36:23.683238Z","shell.execute_reply.started":"2024-12-19T14:36:23.656132Z","shell.execute_reply":"2024-12-19T14:36:23.681707Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"###  PreInt_EduHx-computerinternet_hoursday","metadata":{}},{"cell_type":"markdown","source":"We will be using Logistic Regression to predict internet use hours","metadata":{}},{"cell_type":"code","source":"features = ['Basic_Demos-Age', 'Basic_Demos-Sex', 'SDS-SDS_Total_Raw', 'PreInt_EduHx-Season']\ntarget = 'PreInt_EduHx-computerinternet_hoursday'\n\ntrain_data = train[train[target].notna()]\n\nX_train = train_data[features]\ny_train = train_data[target]\n\npreprocessor = ColumnTransformer(\n    transformers=[\n        ('season', OneHotEncoder(), ['PreInt_EduHx-Season']),  # Mùa\n        ('num', 'passthrough', ['Basic_Demos-Age', 'Basic_Demos-Sex', 'SDS-SDS_Total_Raw'])  # Các cột số\n    ])\n\nmodel = Pipeline(steps=[\n    ('preprocessor', preprocessor),\n    ('classifier', LogisticRegression(max_iter=1000, multi_class='ovr'))\n])\n\nmodel.fit(X_train, y_train)\n\nX_missing = train[train[target].isna()][features]\npredicted_values = model.predict(X_missing)\n\ntrain.loc[train[target].isna(), target] = predicted_values\n\nX_missing_test = test[test[target].isna()][features]\npredicted_values_test = model.predict(X_missing_test)\n\ntest.loc[test[target].isna(), target] = predicted_values_test\n\ncalculate_stats(train, ['PreInt_EduHx-computerinternet_hoursday'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:23.684913Z","iopub.execute_input":"2024-12-19T14:36:23.68531Z","iopub.status.idle":"2024-12-19T14:36:24.001362Z","shell.execute_reply.started":"2024-12-19T14:36:23.685277Z","shell.execute_reply":"2024-12-19T14:36:23.999697Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## PAQ","metadata":{}},{"cell_type":"code","source":"PAQ_Adolescents_columns = ['PAQ_A-Season', 'PAQ_A-PAQ_A_Total']\ncalculate_stats(train, 'PAQ_A-Season')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:24.007157Z","iopub.execute_input":"2024-12-19T14:36:24.00833Z","iopub.status.idle":"2024-12-19T14:36:24.044214Z","shell.execute_reply.started":"2024-12-19T14:36:24.008247Z","shell.execute_reply":"2024-12-19T14:36:24.042598Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"calculate_stats(train, 'PAQ_A-PAQ_A_Total')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:24.051873Z","iopub.execute_input":"2024-12-19T14:36:24.052656Z","iopub.status.idle":"2024-12-19T14:36:24.077269Z","shell.execute_reply.started":"2024-12-19T14:36:24.052589Z","shell.execute_reply":"2024-12-19T14:36:24.075411Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"PAQ_Children_columns = ['PAQ_C-Season', 'PAQ_C-PAQ_C_Total']\ncalculate_stats(train, 'PAQ_C-Season')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:24.078971Z","iopub.execute_input":"2024-12-19T14:36:24.079358Z","iopub.status.idle":"2024-12-19T14:36:24.09433Z","shell.execute_reply.started":"2024-12-19T14:36:24.079323Z","shell.execute_reply":"2024-12-19T14:36:24.092786Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"calculate_stats(train, 'PAQ_C-PAQ_C_Total')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:24.096125Z","iopub.execute_input":"2024-12-19T14:36:24.096564Z","iopub.status.idle":"2024-12-19T14:36:24.126036Z","shell.execute_reply.started":"2024-12-19T14:36:24.096485Z","shell.execute_reply":"2024-12-19T14:36:24.124159Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Not sure what to do with these columns since they have the highest missing values count","metadata":{}},{"cell_type":"markdown","source":"##  Bio-electric Impedance Analysis","metadata":{}},{"cell_type":"code","source":"bia_data_dict = data_dict[data_dict['Instrument'] == 'Bio-electric Impedance Analysis']\ncategorical_columns = bia_data_dict[bia_data_dict['Type'] == 'categorical int']['Field'].tolist()\ncontinuous_columns = bia_data_dict[bia_data_dict['Type'] == 'float']['Field'].tolist()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:24.128221Z","iopub.execute_input":"2024-12-19T14:36:24.128944Z","iopub.status.idle":"2024-12-19T14:36:24.140662Z","shell.execute_reply.started":"2024-12-19T14:36:24.128881Z","shell.execute_reply":"2024-12-19T14:36:24.139264Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(24, 20))\n\nfor idx, col in enumerate(continuous_columns):\n    plt.subplot(4, 4, idx + 1)\n    sns.histplot(train[col].dropna(), bins=20, kde=True)\n    plt.title(data_dict[data_dict['Field'] == col]['Description'].values[0])\n    plt.xlabel('Value')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:24.142584Z","iopub.execute_input":"2024-12-19T14:36:24.143036Z","iopub.status.idle":"2024-12-19T14:36:28.83471Z","shell.execute_reply.started":"2024-12-19T14:36:24.142998Z","shell.execute_reply":"2024-12-19T14:36:28.833242Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Only the BMI column seems reliable, the rest should be discarded","metadata":{}},{"cell_type":"code","source":"for col in continuous_columns:\n    if (col != 'BIA-BIA_BMI'):\n        train = train.drop(columns=col)\n        test = test.drop(columns=col)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:28.83673Z","iopub.execute_input":"2024-12-19T14:36:28.837246Z","iopub.status.idle":"2024-12-19T14:36:28.875082Z","shell.execute_reply.started":"2024-12-19T14:36:28.837195Z","shell.execute_reply":"2024-12-19T14:36:28.873803Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Time series","metadata":{}},{"cell_type":"code","source":"def process_file(filename, dirname):\n    df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    df.drop('step', axis=1, inplace=True)\n    return df.describe().values.reshape(-1), filename.split('=')[1]\n\ndef load_time_series(dirname) -> pd.DataFrame:\n    ids = os.listdir(dirname)\n\n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n\n    stats, indexes = zip(*results)\n\n    df = pd.DataFrame(stats, columns=[f\"stat_{i}\" for i in range(len(stats[0]))])\n    df['id'] = indexes\n    return df\n\ntrain_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\n\ntime_series_cols = train_ts.columns.tolist()\ntime_series_cols.remove(\"id\")\n\ntrain = pd.merge(train, train_ts, how=\"left\", on='id')\ntest = pd.merge(test, test_ts, how=\"left\", on='id')\n\ntrain = train.drop('id', axis=1)\ntest = test.drop('id', axis=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:36:28.876963Z","iopub.execute_input":"2024-12-19T14:36:28.877344Z","iopub.status.idle":"2024-12-19T14:38:14.550486Z","shell.execute_reply.started":"2024-12-19T14:36:28.87731Z","shell.execute_reply":"2024-12-19T14:38:14.549275Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Other features","metadata":{}},{"cell_type":"markdown","source":"We will fill other missing values by KNN for this notebook","metadata":{}},{"cell_type":"code","source":"imputer = KNNImputer(n_neighbors=10)\n\nfeature_numeric_cols = train.select_dtypes(include=['float64', 'int64', 'bool']).columns\n\ncolumns_to_exclude = question_columns + ['PCIAT-PCIAT_Total', 'sii']\n\nfeature_numeric_cols = [col for col in feature_numeric_cols if col not in columns_to_exclude]\n\nimputed_train_data = imputer.fit_transform(train[feature_numeric_cols])\ntrain_imputed = pd.DataFrame(imputed_train_data, columns=feature_numeric_cols)\n\nfor col in train.columns:\n    if col not in feature_numeric_cols:\n        train_imputed[col] = train[col]\n\nimputed_test_data = imputer.transform(test[feature_numeric_cols])\ntest_imputed = pd.DataFrame(imputed_test_data, columns=feature_numeric_cols)\n\nfor col in test.columns:\n    if col not in feature_numeric_cols:\n        test_imputed[col] = test[col]\n\ntrain = train_imputed\ntest = test_imputed\n\ntrain.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:38:14.553762Z","iopub.execute_input":"2024-12-19T14:38:14.554277Z","iopub.status.idle":"2024-12-19T14:38:23.736311Z","shell.execute_reply.started":"2024-12-19T14:38:14.554241Z","shell.execute_reply":"2024-12-19T14:38:23.734909Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature engineering","metadata":{}},{"cell_type":"markdown","source":"Encode unencoded season columns and drop String columns","metadata":{}},{"cell_type":"code","source":"encoded_season_cols = ['Basic_Demos-Enroll_Season', 'SDS-Season']\nnon_encoded_season_cols = ['CGAS-Season', 'Physical-Season', 'Fitness_Endurance-Season', \n          'FGC-Season', 'BIA-Season', 'PAQ_A-Season', 'PAQ_C-Season', 'PreInt_EduHx-Season']\n\ndef remove_encoded_cols(df):\n    return df.drop(columns=encoded_season_cols + ['Age Group', 'index'])\n\ntrain = remove_encoded_cols(train)\ntest = remove_encoded_cols(test)\n\ndef fillna_season(df):\n    for c in non_encoded_season_cols: \n        df[c] = df[c].fillna('Missing')\n        df[c] = df[c].astype('category')\n    return df\n        \ntrain = fillna_season(train)\ntest = fillna_season(test)\n\ndef create_mapping(column, dataset):\n    unique_values = dataset[column].unique()\n    return {value: idx for idx, value in enumerate(unique_values)}\n\nfor col in non_encoded_season_cols:\n    mapping_train = create_mapping(col, train)\n    mapping_test = create_mapping(col, test)\n    \n    train[col] = train[col].replace(mapping_train).astype(int)\n    test[col] = test[col].replace(mapping_test).astype(int)\n\n\ntrain = train.drop(columns=question_columns + ['PCIAT-Season', 'PCIAT-PCIAT_Total'])\n\nprint(f'Train Shape : {train.shape} || Test Shape : {test.shape}')\ntrain.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:38:23.738571Z","iopub.execute_input":"2024-12-19T14:38:23.739156Z","iopub.status.idle":"2024-12-19T14:38:23.914644Z","shell.execute_reply.started":"2024-12-19T14:38:23.739099Z","shell.execute_reply":"2024-12-19T14:38:23.913281Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\nThis step is crucial as it creates new features by combining existing variables, capturing more complex relationships in the data. These engineered features, such as interactions between age, BMI, CGAS scores, and internet usage, are key for improving the predictive power of the model.","metadata":{}},{"cell_type":"code","source":"def feature_engineering(df):\n    #Weight\n    df['CGAS_Weight'] = df['CGAS-CGAS_Score'].apply(sigmoid_weight_cgas_high)\n    df['SDS_Score_Weighted'] = df['SDS-SDS_Total_Raw'] * df['SDS_Weight']\n    df['CGAS_Score_Weighted'] = df['CGAS-CGAS_Score'] * df['CGAS_Weight']\n    df = df.drop(columns=['SDS-SDS_Total_Raw', 'SDS_Weight', 'CGAS-CGAS_Score', 'CGAS_Weight', 'Age_Group_Label'])\n    \n    #Age\n    df['Internet_Hours_Age'] = df['PreInt_EduHx-computerinternet_hoursday'] * df['Basic_Demos-Age']\n    df['Physical-Waist_Age'] = df['Basic_Demos-Age'] * df['Physical-Waist_Circumference']\n    df['BMI_Age'] = df['Physical-BMI'] * df['Basic_Demos-Age']\n    df['Physical-Height_Age'] = df['Basic_Demos-Age'] * df['Physical-Height']\n\n    #SDS\n    df['SDS_BMI'] = df['Physical-BMI'] * df['SDS_Score_Weighted']\n    df['CGAS_SDS'] = df['CGAS_Score_Weighted'] * df['SDS_Score_Weighted']\n    df['CGAS_Endurance_Mins'] = df['CGAS_Score_Weighted'] * df['Fitness_Endurance-Time_Mins']\n    df['SDS_Activity'] = df['BIA-BIA_Activity_Level_num'] * df['SDS_Score_Weighted']\n    df['SDS_InternetHours'] = df['SDS_Score_Weighted'] * df['PreInt_EduHx-computerinternet_hoursday']\n\n    df['BMI_Systolic_BP'] = df['Physical-BMI'] * df['Physical-Systolic_BP']\n    df['Age_Systolic_BP'] = df['Basic_Demos-Age'] * df['Physical-Systolic_BP']\n    df['PreInt_Systolic_BP'] = df['Physical-Systolic_BP'] * df['PreInt_EduHx-computerinternet_hoursday']\n    df['PAQ_A_Activity'] = df['BIA-BIA_Activity_Level_num'] * df['PAQ_A-PAQ_A_Total']\n    df['Activity_CU_PU'] = df['BIA-BIA_Activity_Level_num'] * df['FGC-FGC_CU'] * df['FGC-FGC_PU']\n\n    #FGC\n    df['FGC_CU_PU'] = df['FGC-FGC_CU'] * df['FGC-FGC_PU']\n    df['FGC_CU_PU_Age'] = df['FGC-FGC_CU'] * df['FGC-FGC_PU'] * df['Basic_Demos-Age']\n    df['FGC_GSND_GSD'] = df['FGC-FGC_GSND'] * df['FGC-FGC_GSD']\n    df['FGC_GSND_GSD_Age'] = df['FGC-FGC_GSND'] * df['FGC-FGC_GSD'] * df['Basic_Demos-Age']\n    df['CGAS_CU_PU'] = df['CGAS_Score_Weighted'] * df['FGC-FGC_CU'] * df['FGC-FGC_PU']\n    df['PreInt_FGC_CU_PU'] = df['PreInt_EduHx-computerinternet_hoursday'] * df['FGC-FGC_CU'] * df['FGC-FGC_PU']\n    df['Endurance_CU_PU'] = df['Fitness_Endurance-Time_Mins'] * df['FGC-FGC_CU'] * df['FGC-FGC_PU']\n    return df\n\ntrain = feature_engineering(train)\ntest = feature_engineering(test)\n\nnew_features = ['Internet_Hours_Age', 'Physical-Waist_Age', 'BMI_Age', 'Physical-Height_Age', 'SDS_InternetHours', 'SDS_BMI', 'CGAS_SDS', 'CGAS_Endurance_Mins', 'SDS_Activity', 'BMI_Systolic_BP', 'Age_Systolic_BP', 'PreInt_Systolic_BP', 'PAQ_A_Activity', 'Activity_CU_PU', 'FGC_CU_PU', 'FGC_CU_PU_Age', 'FGC_GSND_GSD', 'FGC_GSND_GSD_Age', 'CGAS_CU_PU', 'PreInt_FGC_CU_PU', 'Endurance_CU_PU', 'CGAS_Weight', 'SDS_Score_Weighted', 'CGAS_Score_Weighted']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:38:23.916276Z","iopub.execute_input":"2024-12-19T14:38:23.916687Z","iopub.status.idle":"2024-12-19T14:38:23.968511Z","shell.execute_reply.started":"2024-12-19T14:38:23.916651Z","shell.execute_reply":"2024-12-19T14:38:23.966966Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Parameters Setting","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import Ridge\nfrom sklearn.svm import SVR\n\nSEED = 42\nn_splits = 5","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:38:23.97086Z","iopub.execute_input":"2024-12-19T14:38:23.971405Z","iopub.status.idle":"2024-12-19T14:38:23.978307Z","shell.execute_reply.started":"2024-12-19T14:38:23.971354Z","shell.execute_reply":"2024-12-19T14:38:23.976949Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"LGB_Params = {\n    'learning_rate': 0.046,\n    'max_depth': 12,\n    'num_leaves': 478,\n    'min_data_in_leaf': 13,\n    'feature_fraction': 0.893,\n    'bagging_fraction': 0.784,\n    'bagging_freq': 4,\n    'lambda_l1': 10,\n    'lambda_l2': 0.01\n}\n\n# LGB_Params.update({\n#     'max_depth': 8,\n#     'num_leaves': 150,\n#     'min_data_in_leaf': 30,\n#     'learning_rate': 0.01,\n#     'feature_fraction': 0.7,\n#     'bagging_fraction': 0.7,\n# })\n\nRF_Params = {\n    'n_estimators': 200,\n    'max_depth': 6,\n    'max_features': 0.8,\n    'min_samples_split': 2,\n    'min_samples_leaf': 1,\n    'bootstrap': True,\n    'random_state': SEED\n}\n\nCatBoost_Params = {\n        'learning_rate': 0.05,\n        'depth': 6,\n        'iterations': 200,\n        'random_seed': SEED,\n        'verbose': 0,\n        'l2_leaf_reg': 10,\n        #'task_type': 'GPU'\n}\n\n# CatBoost_Params.update({\n#     'learning_rate': 0.005,  # Giảm learning rate\n#     'depth': 8,  # Tăng depth để học sâu hơn\n#     'iterations': 500,  # Tăng số vòng lặp\n#     'l2_leaf_reg': 5,  # Giảm regularization\n#     'loss_function': 'RMSE'  # Thử nghiệm với loss khác, nếu phù hợp\n# })\n\n# CatBoost_Params = {\n#     'learning_rate': 0.01,  # Giảm learning rate để mô hình học từ từ hơn\n#     'depth': 7,  # Tăng chiều sâu để mô hình học tốt hơn\n#     'iterations': 500,  # Tăng số lượng iterations\n#     'random_seed': SEED,\n#     'verbose': 0,\n#     'l2_leaf_reg': 15,  # Tăng regularization để giảm overfitting\n#     'task_type': 'GPU',\n#     'border_count': 128,  # Tăng số lượng biên để cải thiện phân loại\n# }\n\nXGB_Params = {\n    'learning_rate': 0.05,\n    'max_depth': 6,\n    'n_estimators': 200,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 1,  \n    'reg_lambda': 3,  \n    'random_state': SEED,\n    #'tree_method': 'gpu_hist',\n}\n\nXGB_Params.update({\n    'learning_rate': 0.01,\n    'max_depth': 5,\n    'n_estimators': 500,\n    'reg_alpha': 5,\n    'reg_lambda': 10,\n})\n\nTabNet_Params = {\n    'n_d': 64, \n    'n_a': 64,  \n    'n_steps': 5, \n    'gamma': 1.5,\n    'n_independent': 2,\n    'n_shared': 2,\n    'lambda_sparse': 1e-4, \n    'optimizer_fn': torch.optim.Adam,\n    'optimizer_params': dict(lr=2e-2, weight_decay=1e-5),\n    'mask_type': 'entmax',\n    'scheduler_params': dict(mode=\"min\", patience=10, min_lr=1e-5, factor=0.5),\n    'scheduler_fn': torch.optim.lr_scheduler.ReduceLROnPlateau,\n    'verbose': 1,\n    'device_name': 'cuda' if torch.cuda.is_available() else 'cpu'\n}\n\nRidge_Params = {\n    'alpha': 1.0, \n    'solver': 'auto', \n    'random_state': SEED \n}\n\nSVR_Params = {\n    'C': 1.0,\n    'kernel': 'rbf',\n    'epsilon': 0.1,\n    'degree': 3,\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:38:23.980086Z","iopub.execute_input":"2024-12-19T14:38:23.980418Z","iopub.status.idle":"2024-12-19T14:38:23.996443Z","shell.execute_reply.started":"2024-12-19T14:38:23.980387Z","shell.execute_reply":"2024-12-19T14:38:23.994757Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model definition","metadata":{}},{"cell_type":"code","source":"LGB_Model = LGBMRegressor(**LGB_Params, random_state=SEED, verbose=-1, n_estimators=300)\nCatBoost_Model = CatBoostRegressor(**CatBoost_Params)\nXGB_Model = XGBRegressor(**XGB_Params)\n# TabNet_Model = TabNetWrapper(**TabNet_Params) \nRidge_Model = Ridge(**Ridge_Params)\nSVR_Model = SVR(**SVR_Params)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:38:23.998508Z","iopub.execute_input":"2024-12-19T14:38:23.99901Z","iopub.status.idle":"2024-12-19T14:38:24.019623Z","shell.execute_reply.started":"2024-12-19T14:38:23.998973Z","shell.execute_reply":"2024-12-19T14:38:24.018235Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Tabnet","metadata":{}},{"cell_type":"code","source":"class TabNetWrapper(BaseEstimator, RegressorMixin):\n    def __init__(self, **kwargs):\n        self.model = TabNetRegressor(**kwargs)\n        self.kwargs = kwargs\n        self.imputer = KNNImputer(n_neighbors=5)\n        #self.imputer = SimpleImputer(strategy='median')\n        self.best_model_path = 'best_tabnet_model.pt'\n\n    def fit(self, X, y):\n        X_imputed = self.imputer.fit_transform(X)\n\n        if hasattr(y, 'values'):\n            y = y.values\n\n        X_train, X_valid, y_train, y_valid = train_test_split(\n            X_imputed,\n            y,\n            test_size=0.2,\n            random_state=42\n        )\n\n        # Train TabNet model\n        history = self.model.fit(\n            X_train=X_train,\n            y_train=y_train.reshape(-1, 1),\n            eval_set=[(X_valid, y_valid.reshape(-1, 1))],\n            eval_name=['valid'],\n            eval_metric=['mse', 'mae', 'rmse'],\n            max_epochs=500,\n            patience=50,\n            batch_size=1024,\n            virtual_batch_size=128,\n            num_workers=0,\n            drop_last=False,\n            callbacks=[\n                TabNetPretrainedModelCheckpoint(\n                    filepath=self.best_model_path,\n                    monitor='valid_mse',\n                    mode='min',\n                    save_best_only=True,\n                    verbose=True\n                )\n            ]\n        )\n\n        # Load the best model\n        if os.path.exists(self.best_model_path):\n            self.model.load_model(self.best_model_path)\n            os.remove(self.best_model_path)  # Remove temporary file\n\n        return self\n\n    def predict(self, X):\n        X_imputed = self.imputer.transform(X)\n        return self.model.predict(X_imputed).flatten()\n\n    def __deepcopy__(self, memo):\n        cls = self.__class__\n        result = cls.__new__(cls)\n        memo[id(self)] = result\n        for k, v in self.__dict__.items():\n            setattr(result, k, deepcopy(v, memo))\n        return result\n\nclass TabNetPretrainedModelCheckpoint(Callback):\n    def __init__(self, filepath, monitor='val_loss', mode='min',\n                 save_best_only=True, verbose=1):\n        super().__init__()\n        self.filepath = filepath\n        self.monitor = monitor\n        self.mode = mode\n        self.save_best_only = save_best_only\n        self.verbose = verbose\n        self.best = float('inf') if mode == 'min' else -float('inf')\n\n    def on_train_begin(self, logs=None):\n        self.model = self.trainer\n\n    def on_epoch_end(self, epoch, logs=None):\n        logs = logs or {}\n        current = logs.get(self.monitor)\n        if current is None:\n            return\n\n        if (self.mode == 'min' and current < self.best) or \\\n           (self.mode == 'max' and current > self.best):\n            if self.verbose:\n                print(f'\\nEpoch {epoch}: {self.monitor} improved from {self.best:.4f} to {current:.4f}')\n            self.best = current\n            if self.save_best_only:\n                self.model.save_model(self.filepath)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:38:24.021915Z","iopub.execute_input":"2024-12-19T14:38:24.02235Z","iopub.status.idle":"2024-12-19T14:38:24.043519Z","shell.execute_reply.started":"2024-12-19T14:38:24.022305Z","shell.execute_reply":"2024-12-19T14:38:24.041637Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"TabNet_Model = TabNetWrapper(**TabNet_Params)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:38:24.04523Z","iopub.execute_input":"2024-12-19T14:38:24.045717Z","iopub.status.idle":"2024-12-19T14:38:24.064335Z","shell.execute_reply.started":"2024-12-19T14:38:24.045679Z","shell.execute_reply":"2024-12-19T14:38:24.062709Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Train and predict","metadata":{}},{"cell_type":"code","source":"print(train.shape)\ntrain.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:38:24.066259Z","iopub.execute_input":"2024-12-19T14:38:24.066735Z","iopub.status.idle":"2024-12-19T14:38:24.205932Z","shell.execute_reply.started":"2024-12-19T14:38:24.066695Z","shell.execute_reply":"2024-12-19T14:38:24.204032Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(test.shape)\ntest.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:38:24.217499Z","iopub.execute_input":"2024-12-19T14:38:24.217995Z","iopub.status.idle":"2024-12-19T14:38:24.354046Z","shell.execute_reply.started":"2024-12-19T14:38:24.217956Z","shell.execute_reply":"2024-12-19T14:38:24.352494Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Show features imporatance for XGB, LGB and CatBoost models","metadata":{}},{"cell_type":"code","source":"def show_feature_importance(model, model_type='xgb'):\n    if model_type == 'xgb':  # XGBoost\n        booster = model.get_booster()\n        feature_importance = booster.get_score(importance_type='gain')\n        feature_importance_df = pd.DataFrame({\n            'Feature': list(feature_importance.keys()),\n            'Importance': list(feature_importance.values())\n        })\n\n    elif model_type == 'lgb':  # LightGBM\n        feature_importance_df = pd.DataFrame({\n            'Feature': model.feature_name_,\n            'Importance': model.feature_importances_\n        })\n\n    elif model_type == 'catboost':  # CatBoost\n        feature_importance = model.get_feature_importance()\n        feature_importance_df = pd.DataFrame({\n            'Feature': model.feature_names_,\n            'Importance': feature_importance\n        })\n\n    else:\n        raise ValueError(\"model_type must be 'xgb', 'lgb', or 'catboost'\")\n    \n    # Sắp xếp theo tầm quan trọng\n    feature_importance_df = feature_importance_df.sort_values(by='Importance', ascending=False)\n\n    # Visualization\n    plt.figure(figsize=(20, 40))\n    sns.barplot(x='Importance', y='Feature', data=feature_importance_df)\n    plt.title(\"Top Feature Importance\")\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:38:24.356005Z","iopub.execute_input":"2024-12-19T14:38:24.356408Z","iopub.status.idle":"2024-12-19T14:38:24.365969Z","shell.execute_reply.started":"2024-12-19T14:38:24.356369Z","shell.execute_reply":"2024-12-19T14:38:24.364487Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:38:24.368019Z","iopub.execute_input":"2024-12-19T14:38:24.368535Z","iopub.status.idle":"2024-12-19T14:38:24.389212Z","shell.execute_reply.started":"2024-12-19T14:38:24.368478Z","shell.execute_reply":"2024-12-19T14:38:24.387142Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## XGB","metadata":{}},{"cell_type":"code","source":"Submission1,new_XGB_Model = TrainML(XGB_Model, train, test)\nshow_feature_importance(new_XGB_Model, model_type='xgb')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:38:24.390867Z","iopub.execute_input":"2024-12-19T14:38:24.391236Z","iopub.status.idle":"2024-12-19T14:38:46.921438Z","shell.execute_reply.started":"2024-12-19T14:38:24.391203Z","shell.execute_reply":"2024-12-19T14:38:46.91951Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Submission1","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:38:46.923535Z","iopub.execute_input":"2024-12-19T14:38:46.924091Z","iopub.status.idle":"2024-12-19T14:38:46.939987Z","shell.execute_reply.started":"2024-12-19T14:38:46.92404Z","shell.execute_reply":"2024-12-19T14:38:46.938233Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## LGB","metadata":{}},{"cell_type":"code","source":"Submission2,new_LGB_Model = TrainML(LGB_Model, train, test)\nshow_feature_importance(new_LGB_Model, model_type='lgb')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:38:46.942169Z","iopub.execute_input":"2024-12-19T14:38:46.942718Z","iopub.status.idle":"2024-12-19T14:39:01.867973Z","shell.execute_reply.started":"2024-12-19T14:38:46.942675Z","shell.execute_reply":"2024-12-19T14:39:01.866628Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Submission2","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:39:01.869792Z","iopub.execute_input":"2024-12-19T14:39:01.870168Z","iopub.status.idle":"2024-12-19T14:39:01.883487Z","shell.execute_reply.started":"2024-12-19T14:39:01.870134Z","shell.execute_reply":"2024-12-19T14:39:01.881918Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## CatBoost","metadata":{}},{"cell_type":"code","source":"Submission3,new_Cat_Model = TrainML(CatBoost_Model, train, test)\nshow_feature_importance(new_Cat_Model, model_type='catboost')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:39:01.885365Z","iopub.execute_input":"2024-12-19T14:39:01.885837Z","iopub.status.idle":"2024-12-19T14:39:21.362203Z","shell.execute_reply.started":"2024-12-19T14:39:01.885777Z","shell.execute_reply":"2024-12-19T14:39:21.360406Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Submission3","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:39:21.364692Z","iopub.execute_input":"2024-12-19T14:39:21.365445Z","iopub.status.idle":"2024-12-19T14:39:21.384629Z","shell.execute_reply.started":"2024-12-19T14:39:21.365366Z","shell.execute_reply":"2024-12-19T14:39:21.382877Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Tabnet","metadata":{}},{"cell_type":"markdown","source":"We are not using this anymore since bad results","metadata":{}},{"cell_type":"code","source":"# sample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\n# Submission4,new_TabNet_Model = TrainML(TabNet_Model, test)\n# show_feature_importance(new_TabNet_Model, model_type='tabnet')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:39:21.386668Z","iopub.execute_input":"2024-12-19T14:39:21.387256Z","iopub.status.idle":"2024-12-19T14:39:21.397209Z","shell.execute_reply.started":"2024-12-19T14:39:21.387213Z","shell.execute_reply":"2024-12-19T14:39:21.395459Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Submission4","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:39:21.399042Z","iopub.execute_input":"2024-12-19T14:39:21.399612Z","iopub.status.idle":"2024-12-19T14:39:21.416523Z","shell.execute_reply.started":"2024-12-19T14:39:21.39953Z","shell.execute_reply":"2024-12-19T14:39:21.415036Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Ensemble Model","metadata":{}},{"cell_type":"markdown","source":"Models are combined using a Stacking Regressor, and a Voting Regressor is used to combine the stacked model for ensemble predictions","metadata":{}},{"cell_type":"code","source":"ridge_pipeline = Pipeline([\n    ('imputer', SimpleImputer(strategy='median')),  \n    ('ridge', Ridge(alpha=1.0))\n])\n\nsvr_pipeline = Pipeline([\n    ('imputer', SimpleImputer(strategy='mean')),\n    ('svr', SVR(C=1.0, kernel='rbf'))\n])\n\nestimators = [\n    ('catboost', CatBoost_Model),\n    ('lightgbm', LGB_Model),\n    ('xgboost', XGB_Model),\n    # ('tabnet', TabNet_Model),\n    # ('ridge', ridge_pipeline),\n    # ('svr', svr_pipeline)\n]\n\n# StackingRegressor\nstacking_model = StackingRegressor(\n    estimators=estimators,\n    # final_estimator=GradientBoostingRegressor(n_estimators=50, learning_rate=0.1),\n    # cv=5  \n)\n\nensemble_model = VotingRegressor(\n    estimators=[\n        # ('catboost', CatBoost_Model),\n        # ('lightgbm', LGB_Model),\n        # ('xgboost', XGB_Model),\n        ('stacking', stacking_model)\n    ],\n    # weights=[1, 3, 2, 3] \n)\n\nfinal_submission,new_ensemble_model = TrainML(ensemble_model, train, test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:39:21.418489Z","iopub.execute_input":"2024-12-19T14:39:21.419047Z","iopub.status.idle":"2024-12-19T14:44:47.484404Z","shell.execute_reply.started":"2024-12-19T14:39:21.419001Z","shell.execute_reply":"2024-12-19T14:44:47.48298Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_submission","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:44:47.486217Z","iopub.execute_input":"2024-12-19T14:44:47.486602Z","iopub.status.idle":"2024-12-19T14:44:47.500893Z","shell.execute_reply.started":"2024-12-19T14:44:47.486565Z","shell.execute_reply":"2024-12-19T14:44:47.499166Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"final_submission[['id', 'sii']].to_csv('submission.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T14:44:47.502625Z","iopub.execute_input":"2024-12-19T14:44:47.503045Z","iopub.status.idle":"2024-12-19T14:44:47.520056Z","shell.execute_reply.started":"2024-12-19T14:44:47.503007Z","shell.execute_reply":"2024-12-19T14:44:47.518281Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Credit","metadata":{}},{"cell_type":"markdown","source":"EDA from https://www.kaggle.com/code/antoninadolgorukova/cmi-piu-features-eda\n\nTime series data EDA and threshold tuning methods from https://www.kaggle.com/code/ambrosm/piu-eda-which-makes-sense","metadata":{}}]}