{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **EDA on Key Features** - Problematic Internet Use","metadata":{}},{"cell_type":"code","source":"# Packages \n# Data Processing \nimport numpy as np \nimport pandas as pd \n# Visualization \nimport matplotlib.pyplot as plt \nplt.rcParams['figure.dpi'] = 200 \nimport seaborn as sns \n# Statistics \nimport math \nfrom scipy import stats \nfrom scipy.stats import norm \nfrom scipy.stats import chi2_contingency\n# File Path \nimport os \nfor dirname, _, filenames in os.walk('/kaggle/input'): \n    for filename in filenames: \n        print(os.path.join(dirname, filename))\n \nimport re\nfrom colorama import Fore, Style\nfrom tqdm import tqdm\nfrom IPython.display import clear_output\nfrom concurrent.futures import ThreadPoolExecutor\nimport polars as pl\nfrom sklearn.base import clone\nfrom copy import deepcopy\nimport optuna\nfrom scipy.optimize import minimize","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-10-27T03:05:15.488815Z","iopub.execute_input":"2024-10-27T03:05:15.489296Z","iopub.status.idle":"2024-10-27T03:05:16.643251Z","shell.execute_reply.started":"2024-10-27T03:05:15.489249Z","shell.execute_reply":"2024-10-27T03:05:16.641942Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# version check\nprint(f\"numpy version: {np.__version__}\")\nprint(f\"pandas version: {pd.__version__}\")\n\n# Ignore Warning\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n# setting\npath_root = \"/kaggle/input/\"\nseed = 394\n\n# pandas display setting\npd.set_option('display.max_rows', 20)\npd.set_option('display.max_columns', 200)","metadata":{"execution":{"iopub.status.busy":"2024-10-27T03:05:16.645264Z","iopub.execute_input":"2024-10-27T03:05:16.645664Z","iopub.status.idle":"2024-10-27T03:05:16.653265Z","shell.execute_reply.started":"2024-10-27T03:05:16.645622Z","shell.execute_reply":"2024-10-27T03:05:16.651844Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\n\n# Preprocess\n\ndef process_file(filename, dirname):\n    df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    df.drop('step', axis=1, inplace=True)\n    return df.describe().values.reshape(-1), filename.split('=')[1]\n\ndef load_time_series(dirname) -> pd.DataFrame:\n    ids = os.listdir(dirname)\n    \n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n    \n    stats, indexes = zip(*results)\n    \n    df = pd.DataFrame(stats, columns=[f\"Stat_{i}\" for i in range(len(stats[0]))])\n    df['id'] = indexes\n    \n    return df\n\ntrain = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\ntrain_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\ntime_series_cols = train_ts.columns.tolist()\ntime_series_cols.remove(\"id\")\n\ntrain = pd.merge(train, train_ts, how=\"left\", on='id')\ntest = pd.merge(test, test_ts, how=\"left\", on='id')\n\ntrain = train.drop('id', axis=1)\ntest = test.drop('id', axis=1)\n\nfeaturesCols = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-Season', 'CGAS-CGAS_Score', 'Physical-Season', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Season', 'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-Season', 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone', 'BIA-Season',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-Season', 'PAQ_A-PAQ_A_Total', 'PAQ_C-Season',\n                'PAQ_C-PAQ_C_Total', 'SDS-Season', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T', 'PreInt_EduHx-Season',\n                'PreInt_EduHx-computerinternet_hoursday', 'sii']\n\nfeaturesCols += time_series_cols\n\ntrain = train[featuresCols]\ntrain = train.dropna(subset='sii')\n\ncat_c = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', 'Fitness_Endurance-Season', \n          'FGC-Season', 'BIA-Season', 'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season']\n\ndef update(df):\n    for c in cat_c: \n        df[c] = df[c].fillna('Missing')\n        df[c] = df[c].astype('category')\n    return df\n        \ntrain = update(train)\ntest = update(test)\n\ndef create_mapping(column, dataset):\n    unique_values = dataset[column].unique()\n    return {value: idx for idx, value in enumerate(unique_values)}\n\nfor col in cat_c:\n    mapping_train = create_mapping(col, train)\n    mapping_test = create_mapping(col, test)\n    \n    train[col] = train[col].replace(mapping_train).astype(int)\n    test[col] = test[col].replace(mapping_test).astype(int)\n\nprint(f'Train Shape : {train.shape} || Test Shape : {test.shape}')\n\ndf_train = train.copy()\ndf_test = test.copy()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-10-27T03:05:16.655400Z","iopub.execute_input":"2024-10-27T03:05:16.655876Z","iopub.status.idle":"2024-10-27T03:06:54.371548Z","shell.execute_reply.started":"2024-10-27T03:05:16.655814Z","shell.execute_reply":"2024-10-27T03:06:54.369622Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 1. Overview","metadata":{}},{"cell_type":"code","source":"print(df_train.shape)\nprint(df_test.shape)","metadata":{"execution":{"iopub.status.busy":"2024-10-27T03:06:54.375492Z","iopub.execute_input":"2024-10-27T03:06:54.376066Z","iopub.status.idle":"2024-10-27T03:06:54.382926Z","shell.execute_reply.started":"2024-10-27T03:06:54.376002Z","shell.execute_reply":"2024-10-27T03:06:54.381523Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"display(df_train.head())\ndisplay(df_train.tail())","metadata":{"execution":{"iopub.status.busy":"2024-10-27T03:06:54.384519Z","iopub.execute_input":"2024-10-27T03:06:54.384992Z","iopub.status.idle":"2024-10-27T03:06:54.645848Z","shell.execute_reply.started":"2024-10-27T03:06:54.384946Z","shell.execute_reply":"2024-10-27T03:06:54.644585Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"display(df_test.head())\ndisplay(df_test.tail())","metadata":{"execution":{"iopub.status.busy":"2024-10-27T03:06:54.647390Z","iopub.execute_input":"2024-10-27T03:06:54.647828Z","iopub.status.idle":"2024-10-27T03:06:54.873946Z","shell.execute_reply.started":"2024-10-27T03:06:54.647773Z","shell.execute_reply":"2024-10-27T03:06:54.872640Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train.columns","metadata":{"execution":{"iopub.status.busy":"2024-10-27T03:06:54.875477Z","iopub.execute_input":"2024-10-27T03:06:54.875906Z","iopub.status.idle":"2024-10-27T03:06:54.883760Z","shell.execute_reply.started":"2024-10-27T03:06:54.875862Z","shell.execute_reply":"2024-10-27T03:06:54.882521Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# descriptive statistics\ndisplay(df_train.describe().round(3).T) # numerical","metadata":{"execution":{"iopub.status.busy":"2024-10-27T03:06:54.885266Z","iopub.execute_input":"2024-10-27T03:06:54.885700Z","iopub.status.idle":"2024-10-27T03:06:55.209473Z","shell.execute_reply.started":"2024-10-27T03:06:54.885643Z","shell.execute_reply":"2024-10-27T03:06:55.207640Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 2. Data Cleaning","metadata":{}},{"cell_type":"code","source":"# subsetting\ndf_train = train[[\n    'Basic_Demos-Age', 'Basic_Demos-Sex',\n    'SDS-SDS_Total_Raw', \n    'PreInt_EduHx-computerinternet_hoursday', \n    'PAQ_C-PAQ_C_Total', 'PAQ_A-PAQ_A_Total',\n    'BIA-BIA_SMM', 'BIA-BIA_BMI',\n    'Physical-Height', 'Physical-Weight', 'Physical-BMI',\n    'CGAS-CGAS_Score', \n    'FGC-FGC_CU',\n    'sii'\n]].copy()","metadata":{"execution":{"iopub.status.busy":"2024-10-27T05:51:17.543358Z","iopub.execute_input":"2024-10-27T05:51:17.544668Z","iopub.status.idle":"2024-10-27T05:51:17.557070Z","shell.execute_reply.started":"2024-10-27T05:51:17.544612Z","shell.execute_reply":"2024-10-27T05:51:17.554901Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# missing values\ndf_train.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2024-10-27T05:51:17.586906Z","iopub.execute_input":"2024-10-27T05:51:17.587382Z","iopub.status.idle":"2024-10-27T05:51:17.602380Z","shell.execute_reply.started":"2024-10-27T05:51:17.587336Z","shell.execute_reply":"2024-10-27T05:51:17.600453Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test[[\n    'Basic_Demos-Age', 'Basic_Demos-Sex',\n    'SDS-SDS_Total_Raw', \n    'PreInt_EduHx-computerinternet_hoursday', \n    'PAQ_C-PAQ_C_Total', 'PAQ_A-PAQ_A_Total',\n    'BIA-BIA_SMM', \n    'Physical-Height', 'Physical-Weight', 'Physical-BMI', \n    'CGAS-CGAS_Score', \n    'FGC-FGC_CU'\n]].isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2024-10-27T05:51:17.637859Z","iopub.execute_input":"2024-10-27T05:51:17.638300Z","iopub.status.idle":"2024-10-27T05:51:17.651855Z","shell.execute_reply.started":"2024-10-27T05:51:17.638258Z","shell.execute_reply":"2024-10-27T05:51:17.650407Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# visualize missing values\nplt.figure(figsize = (4, 4), facecolor = \"white\")\n\nsns.heatmap(\n    df_train.isnull(), vmin = 0, vmax = 1\n)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-27T05:51:17.684963Z","iopub.execute_input":"2024-10-27T05:51:17.685391Z","iopub.status.idle":"2024-10-27T05:51:18.292373Z","shell.execute_reply.started":"2024-10-27T05:51:17.685351Z","shell.execute_reply":"2024-10-27T05:51:18.290798Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# duplicated rows\ndf_train.loc[df_train.duplicated()]","metadata":{"execution":{"iopub.status.busy":"2024-10-27T05:51:18.295002Z","iopub.execute_input":"2024-10-27T05:51:18.295532Z","iopub.status.idle":"2024-10-27T05:51:18.316091Z","shell.execute_reply.started":"2024-10-27T05:51:18.295474Z","shell.execute_reply":"2024-10-27T05:51:18.314915Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 3. Univariate Analysis","metadata":{}},{"cell_type":"code","source":"def summary_numerical_dist(df_data, col, q_min, q_max):\n    \n    fig = plt.figure(figsize = (8, 4), facecolor = \"white\")\n\n    layout_plot = (2, 2)\n    num_subplot = 4\n    axes = [None for _ in range(num_subplot)]\n    list_shape_subplot = [[(0, 0), (0, 1), (1, 0), (1, 1)], [1, 1, 1, 1], [1, 1, 1, 1]]\n    for i in range(num_subplot):\n        axes[i] = plt.subplot2grid(\n            layout_plot, list_shape_subplot[0][i],\n            rowspan = list_shape_subplot[1][i],\n            colspan = list_shape_subplot[2][i]\n        )\n\n    sns.histplot(data = df_data, x = col, kde = True, ax = axes[0])\n    stats.probplot(x = df_data[col], dist = stats.norm, plot = axes[1])\n    sns.boxplot(data = df_data, x = col, ax = axes[2])\n    pts = df_data[col].quantile(q = np.arange(q_min, q_max, 0.01))\n    sns.lineplot(x = pts.index, y = pts, ax = axes[3])\n    axes[3].grid(True)\n\n    list_title = [\"Histogram\", \"QQ plot\", \"Boxplot\", \"Outlier\"]\n    for i in range(num_subplot):\n        axes[i].set_title(list_title[i])\n    plt.suptitle(f\"Distribution of: {col}\", fontsize = 15)\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-27T05:51:18.317332Z","iopub.execute_input":"2024-10-27T05:51:18.317811Z","iopub.status.idle":"2024-10-27T05:51:18.330884Z","shell.execute_reply.started":"2024-10-27T05:51:18.317762Z","shell.execute_reply":"2024-10-27T05:51:18.329195Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def summary_categorical_dist(df_data, col):\n    \n    fig = plt.figure(figsize = (8, 4), facecolor = \"white\")\n\n    layout_plot = (1, 2)\n    num_subplot = 2\n    axes = [None for _ in range(num_subplot)]\n    list_shape_subplot = [[(0, 0), (0, 1)], [1, 1], [1, 1]]\n    for i in range(num_subplot):\n        axes[i] = plt.subplot2grid(\n            layout_plot, list_shape_subplot[0][i],\n            rowspan = list_shape_subplot[1][i],\n            colspan = list_shape_subplot[2][i]\n        )\n    \n    count = df_data[col].value_counts().sort_index()\n    \n    sns.countplot(data = df_data, y = col, order = count.index, ax = axes[0])\n    axes[1].pie(data = df_data, x = count, labels = count.index, autopct = '%1.1f%%', startangle = 90)\n    \n    list_title = [\"Counts\", \"Proportions\"]\n    for i in range(num_subplot):\n        axes[i].set_title(list_title[i])\n    plt.suptitle(f\"Distribution of: {col}\", fontsize = 15)\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-27T05:51:18.334088Z","iopub.execute_input":"2024-10-27T05:51:18.334639Z","iopub.status.idle":"2024-10-27T05:51:18.347095Z","shell.execute_reply.started":"2024-10-27T05:51:18.334582Z","shell.execute_reply":"2024-10-27T05:51:18.345928Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# descriptive statistics\ndisplay(df_train.describe().round(3).T) # numerical","metadata":{"execution":{"iopub.status.busy":"2024-10-27T05:51:18.349079Z","iopub.execute_input":"2024-10-27T05:51:18.349483Z","iopub.status.idle":"2024-10-27T05:51:18.415779Z","shell.execute_reply.started":"2024-10-27T05:51:18.349437Z","shell.execute_reply":"2024-10-27T05:51:18.414374Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"summary_numerical_dist(df_train, 'SDS-SDS_Total_Raw', .95, 1)","metadata":{"execution":{"iopub.status.busy":"2024-10-27T05:51:18.417518Z","iopub.execute_input":"2024-10-27T05:51:18.417963Z","iopub.status.idle":"2024-10-27T05:51:19.501967Z","shell.execute_reply.started":"2024-10-27T05:51:18.417918Z","shell.execute_reply":"2024-10-27T05:51:19.500722Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"summary_numerical_dist(df_train, 'PreInt_EduHx-computerinternet_hoursday', .95, 1)","metadata":{"execution":{"iopub.status.busy":"2024-10-27T05:51:19.503340Z","iopub.execute_input":"2024-10-27T05:51:19.503758Z","iopub.status.idle":"2024-10-27T05:51:20.709462Z","shell.execute_reply.started":"2024-10-27T05:51:19.503712Z","shell.execute_reply":"2024-10-27T05:51:20.707805Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"summary_numerical_dist(df_train, 'Basic_Demos-Age', .95, 1)","metadata":{"execution":{"iopub.status.busy":"2024-10-27T05:51:20.711113Z","iopub.execute_input":"2024-10-27T05:51:20.711555Z","iopub.status.idle":"2024-10-27T05:51:21.735331Z","shell.execute_reply.started":"2024-10-27T05:51:20.711508Z","shell.execute_reply":"2024-10-27T05:51:21.734035Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"summary_numerical_dist(df_train, 'PAQ_C-PAQ_C_Total', .95, 1)","metadata":{"execution":{"iopub.status.busy":"2024-10-27T05:51:21.736986Z","iopub.execute_input":"2024-10-27T05:51:21.737470Z","iopub.status.idle":"2024-10-27T05:51:22.742744Z","shell.execute_reply.started":"2024-10-27T05:51:21.737402Z","shell.execute_reply":"2024-10-27T05:51:22.741390Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"summary_numerical_dist(df_train, 'BIA-BIA_SMM', .95, 1)","metadata":{"execution":{"iopub.status.busy":"2024-10-27T05:51:22.748089Z","iopub.execute_input":"2024-10-27T05:51:22.748659Z","iopub.status.idle":"2024-10-27T05:51:25.796049Z","shell.execute_reply.started":"2024-10-27T05:51:22.748596Z","shell.execute_reply":"2024-10-27T05:51:25.794012Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"summary_numerical_dist(df_train, 'Physical-Height', .95, 1)","metadata":{"execution":{"iopub.status.busy":"2024-10-27T05:51:25.797672Z","iopub.execute_input":"2024-10-27T05:51:25.798257Z","iopub.status.idle":"2024-10-27T05:51:26.847727Z","shell.execute_reply.started":"2024-10-27T05:51:25.798194Z","shell.execute_reply":"2024-10-27T05:51:26.845160Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"summary_numerical_dist(df_train, 'Physical-Weight', .95, 1)","metadata":{"execution":{"iopub.status.busy":"2024-10-27T05:51:26.850046Z","iopub.execute_input":"2024-10-27T05:51:26.850721Z","iopub.status.idle":"2024-10-27T05:51:28.164654Z","shell.execute_reply.started":"2024-10-27T05:51:26.850639Z","shell.execute_reply":"2024-10-27T05:51:28.162811Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"summary_numerical_dist(df_train, 'Physical-BMI', .95, 1)","metadata":{"execution":{"iopub.status.busy":"2024-10-27T05:51:28.167476Z","iopub.execute_input":"2024-10-27T05:51:28.168510Z","iopub.status.idle":"2024-10-27T05:51:29.341709Z","shell.execute_reply.started":"2024-10-27T05:51:28.168439Z","shell.execute_reply":"2024-10-27T05:51:29.340028Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"summary_numerical_dist(df_train, 'CGAS-CGAS_Score', .95, 1)","metadata":{"execution":{"iopub.status.busy":"2024-10-27T05:51:29.343931Z","iopub.execute_input":"2024-10-27T05:51:29.344511Z","iopub.status.idle":"2024-10-27T05:51:30.417718Z","shell.execute_reply.started":"2024-10-27T05:51:29.344437Z","shell.execute_reply":"2024-10-27T05:51:30.416287Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"summary_numerical_dist(df_train, 'FGC-FGC_CU', .95, 1)","metadata":{"execution":{"iopub.status.busy":"2024-10-27T05:51:30.419614Z","iopub.execute_input":"2024-10-27T05:51:30.420270Z","iopub.status.idle":"2024-10-27T05:51:31.592994Z","shell.execute_reply.started":"2024-10-27T05:51:30.420136Z","shell.execute_reply":"2024-10-27T05:51:31.591414Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"summary_categorical_dist(df_train, 'Basic_Demos-Sex')","metadata":{"execution":{"iopub.status.busy":"2024-10-27T05:51:31.595234Z","iopub.execute_input":"2024-10-27T05:51:31.595823Z","iopub.status.idle":"2024-10-27T05:51:32.063338Z","shell.execute_reply.started":"2024-10-27T05:51:31.595753Z","shell.execute_reply":"2024-10-27T05:51:32.062139Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"summary_categorical_dist(df_train, 'sii')","metadata":{"execution":{"iopub.status.busy":"2024-10-27T05:51:32.064828Z","iopub.execute_input":"2024-10-27T05:51:32.065227Z","iopub.status.idle":"2024-10-27T05:51:32.506312Z","shell.execute_reply.started":"2024-10-27T05:51:32.065182Z","shell.execute_reply":"2024-10-27T05:51:32.504696Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 4. Relationship Analysis","metadata":{}},{"cell_type":"code","source":"# correlation matrix\nplt.figure(figsize = (8, 4), facecolor = \"white\")\n\n# plotting\nsns.heatmap(\n    data = df_train.corr(numeric_only = True),\n    cmap = \"vlag\",\n    vmin = -1, vmax = 1,\n    linecolor = \"white\", linewidth = 0.5,\n    annot = True,\n    fmt = \".2f\"\n)\n\nplt.title('Correlation Heatmap')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-27T05:51:32.507914Z","iopub.execute_input":"2024-10-27T05:51:32.508316Z","iopub.status.idle":"2024-10-27T05:51:33.678366Z","shell.execute_reply.started":"2024-10-27T05:51:32.508275Z","shell.execute_reply":"2024-10-27T05:51:33.677160Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 5. Deep Inspection","metadata":{}},{"cell_type":"markdown","source":"## 5.1. Demographic Features - Age and Sex","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize = (6, 2), facecolor = \"white\")\n\nsns.violinplot(\n    data = df_train, \n    x = 'Basic_Demos-Age', y = 'sii',\n    orient = 'h'\n)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-27T05:51:33.679901Z","iopub.execute_input":"2024-10-27T05:51:33.680371Z","iopub.status.idle":"2024-10-27T05:51:34.005947Z","shell.execute_reply.started":"2024-10-27T05:51:33.680315Z","shell.execute_reply":"2024-10-27T05:51:34.004753Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize = (6, 3), facecolor = \"white\")\n\nsns.lineplot(\n    data = df_train, \n    x = 'Basic_Demos-Age', y = 'sii',\n    marker = 'o',\n    hue = 'Basic_Demos-Sex'\n)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-27T05:51:34.007764Z","iopub.execute_input":"2024-10-27T05:51:34.008275Z","iopub.status.idle":"2024-10-27T05:51:35.117327Z","shell.execute_reply.started":"2024-10-27T05:51:34.008216Z","shell.execute_reply":"2024-10-27T05:51:35.115857Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"temp_crosstab = pd.crosstab(\n    df_train['Basic_Demos-Sex'], df_train['sii'],\n    margins = True,\n    normalize = False\n)\n\nprint(f\"chi2 test p-value = {chi2_contingency(temp_crosstab)[1]}\")\ndisplay(temp_crosstab)","metadata":{"execution":{"iopub.status.busy":"2024-10-27T05:51:35.118937Z","iopub.execute_input":"2024-10-27T05:51:35.119417Z","iopub.status.idle":"2024-10-27T05:51:35.172877Z","shell.execute_reply.started":"2024-10-27T05:51:35.119358Z","shell.execute_reply":"2024-10-27T05:51:35.171639Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Summary:\n- positive(+) correlation with age\n    - increse until around 13~14\n    - no interaction with sex\n- correlation with sex\n    - more probability when male","metadata":{}},{"cell_type":"markdown","source":"## 5.2. Sleep Disturbance Scale","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize = (6, 2), facecolor = \"white\")\n\nsns.boxplot(\n    data = df_train, \n    x = 'SDS-SDS_Total_Raw', y = 'sii',\n    orient = 'h'\n)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-27T05:51:35.174529Z","iopub.execute_input":"2024-10-27T05:51:35.174944Z","iopub.status.idle":"2024-10-27T05:51:35.400698Z","shell.execute_reply.started":"2024-10-27T05:51:35.174901Z","shell.execute_reply":"2024-10-27T05:51:35.399276Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train[\"log_SDS-SDS_Total_Raw\"] = np.log(df_train[\"SDS-SDS_Total_Raw\"])","metadata":{"execution":{"iopub.status.busy":"2024-10-27T05:51:35.402020Z","iopub.execute_input":"2024-10-27T05:51:35.402441Z","iopub.status.idle":"2024-10-27T05:51:35.410136Z","shell.execute_reply.started":"2024-10-27T05:51:35.402398Z","shell.execute_reply":"2024-10-27T05:51:35.408738Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize = (6, 2), facecolor = \"white\")\n\nsns.boxplot(\n    data = df_train, \n    x = 'log_SDS-SDS_Total_Raw', y = 'sii',\n    orient = 'h'\n)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-27T05:51:35.412049Z","iopub.execute_input":"2024-10-27T05:51:35.412477Z","iopub.status.idle":"2024-10-27T05:51:35.679756Z","shell.execute_reply.started":"2024-10-27T05:51:35.412431Z","shell.execute_reply":"2024-10-27T05:51:35.678545Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize = (6, 2), facecolor = \"white\")\n\nsns.boxplot(\n    data = df_train, \n    x = 'log_SDS-SDS_Total_Raw', y = 'sii',\n    orient = 'h',\n    hue = 'Basic_Demos-Sex'\n)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-27T05:51:35.681430Z","iopub.execute_input":"2024-10-27T05:51:35.681959Z","iopub.status.idle":"2024-10-27T05:51:36.160909Z","shell.execute_reply.started":"2024-10-27T05:51:35.681898Z","shell.execute_reply":"2024-10-27T05:51:36.159699Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Summary:\n- positive(+) correlation with SDS\n    - skewness can be fixed by log transformation\n    - no clear interaction with age and sex","metadata":{}},{"cell_type":"markdown","source":"## 5.3. Hours of using computer/internet","metadata":{}},{"cell_type":"code","source":"temp_crosstab = pd.crosstab(\n    df_train['PreInt_EduHx-computerinternet_hoursday'], df_train['sii'],\n    margins = True,\n    normalize = False\n)\n\nprint(f\"chi2 test p-value = {chi2_contingency(temp_crosstab)[1]}\")\ndisplay(temp_crosstab)","metadata":{"execution":{"iopub.status.busy":"2024-10-27T05:51:36.162601Z","iopub.execute_input":"2024-10-27T05:51:36.163102Z","iopub.status.idle":"2024-10-27T05:51:36.207052Z","shell.execute_reply.started":"2024-10-27T05:51:36.163048Z","shell.execute_reply":"2024-10-27T05:51:36.205755Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train['is_Adolescent'] = np.where(df_train['Basic_Demos-Age'] >= 14, 1, 0)","metadata":{"execution":{"iopub.status.busy":"2024-10-27T05:51:36.208675Z","iopub.execute_input":"2024-10-27T05:51:36.209100Z","iopub.status.idle":"2024-10-27T05:51:36.217089Z","shell.execute_reply.started":"2024-10-27T05:51:36.209055Z","shell.execute_reply":"2024-10-27T05:51:36.215335Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize = (6, 4), facecolor = \"white\")\n\nsns.violinplot(\n    data = df_train, \n    x = 'PreInt_EduHx-computerinternet_hoursday', y = 'sii',\n    orient = 'v',\n    hue = 'is_Adolescent'\n)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-27T05:51:36.225113Z","iopub.execute_input":"2024-10-27T05:51:36.225715Z","iopub.status.idle":"2024-10-27T05:51:36.685098Z","shell.execute_reply.started":"2024-10-27T05:51:36.225633Z","shell.execute_reply":"2024-10-27T05:51:36.683866Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Summary:\n- this is categorical feature\n- positive(+) correlation\n    - interaction with age(adolescent)\n    - distribution trend if not adolescent: 0 < 1, 2 < 3\n    - distribution trend if adolescent: 0, 1 < 2 < 3","metadata":{}},{"cell_type":"markdown","source":"## 5.4. Physical Activity Questionnaire","metadata":{}},{"cell_type":"markdown","source":"https://www.prismsports.org/UserFiles/file/PAQ_manual_ScoringandPDF.pdf\n\n> Physical Activity Questionnaire for Older Children (**PAQ-C**): \n> The PAQ-C is appropriate for elementary school-aged children (grades 4-8; approximately ages **8-14**) who are currently in the school system and have recess as a regular part of their school week.  \n\n> Physical Activity Questionnaire for Adolescents (**PAQ-A**): \n> The PAQ-A is appropriate for high school students (grades 9-12; approximately ages **14-20**) who are currently in the school system.\n\nThat is why I set the 14 as adolescent.","metadata":{}},{"cell_type":"code","source":"df_train.groupby(['is_Adolescent']).agg({\n    'PAQ_C-PAQ_C_Total': 'count',\n    'PAQ_A-PAQ_A_Total': 'count'\n})","metadata":{"execution":{"iopub.status.busy":"2024-10-27T05:51:36.686816Z","iopub.execute_input":"2024-10-27T05:51:36.687309Z","iopub.status.idle":"2024-10-27T05:51:36.704051Z","shell.execute_reply.started":"2024-10-27T05:51:36.687255Z","shell.execute_reply":"2024-10-27T05:51:36.702808Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize = (6, 2), facecolor = \"white\")\n\nsns.boxplot(\n    data = df_train, \n    x = 'PAQ_C-PAQ_C_Total', y = 'sii',\n    orient = 'h',\n    hue = 'Basic_Demos-Sex'\n)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-27T05:51:36.705530Z","iopub.execute_input":"2024-10-27T05:51:36.705920Z","iopub.status.idle":"2024-10-27T05:51:37.167653Z","shell.execute_reply.started":"2024-10-27T05:51:36.705878Z","shell.execute_reply":"2024-10-27T05:51:37.166217Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize = (6, 2), facecolor = \"white\")\n\nsns.boxplot(\n    data = df_train, \n    x = 'PAQ_A-PAQ_A_Total', y = 'sii',\n    orient = 'h',\n    hue = 'Basic_Demos-Sex'\n)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-27T05:51:37.169430Z","iopub.execute_input":"2024-10-27T05:51:37.170509Z","iopub.status.idle":"2024-10-27T05:51:37.666921Z","shell.execute_reply.started":"2024-10-27T05:51:37.170443Z","shell.execute_reply":"2024-10-27T05:51:37.665649Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Summary:\n- interaction with sex and age(adolescent)\n    - when children, dropping PAQ score is key indicators for female\n    - when adolescent, dropping PAQ score is key indicators for male","metadata":{}},{"cell_type":"markdown","source":"## 5.5. Physical Features - Height, Weight, BMI","metadata":{}},{"cell_type":"markdown","source":"### 5.5.1. Zero Weight?","metadata":{}},{"cell_type":"code","source":"df_train.loc[df_train['Physical-Weight'] == 0]","metadata":{"execution":{"iopub.status.busy":"2024-10-27T05:51:37.668637Z","iopub.execute_input":"2024-10-27T05:51:37.669180Z","iopub.status.idle":"2024-10-27T05:51:37.699576Z","shell.execute_reply.started":"2024-10-27T05:51:37.669107Z","shell.execute_reply":"2024-10-27T05:51:37.697781Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Zero weight is unacceptable.  \nSo I will try substitution using a linear model.","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import LinearRegression\nfrom sklearn.metrics import mean_squared_error, r2_score","metadata":{"execution":{"iopub.status.busy":"2024-10-27T05:51:37.701452Z","iopub.execute_input":"2024-10-27T05:51:37.702018Z","iopub.status.idle":"2024-10-27T05:51:37.709537Z","shell.execute_reply.started":"2024-10-27T05:51:37.701972Z","shell.execute_reply":"2024-10-27T05:51:37.708279Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def make_linear_model(df_data, y):\n    \n    df_data = df_data.dropna()\n    df_X = df_data.drop([y], axis = 1)\n    df_y = df_data[y]\n\n    model = LinearRegression()\n    model.fit(df_X, df_y)\n    \n    print(\"R^2:\", r2_score(df_y, model.predict(df_X)))\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2024-10-27T06:03:28.996193Z","iopub.execute_input":"2024-10-27T06:03:28.996779Z","iopub.status.idle":"2024-10-27T06:03:29.007344Z","shell.execute_reply.started":"2024-10-27T06:03:28.996727Z","shell.execute_reply":"2024-10-27T06:03:29.004810Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_substitute_weight = make_linear_model(df_train[['Basic_Demos-Age', 'Basic_Demos-Sex', 'Physical-Weight']], 'Physical-Weight')","metadata":{"execution":{"iopub.status.busy":"2024-10-27T06:03:29.013366Z","iopub.execute_input":"2024-10-27T06:03:29.013986Z","iopub.status.idle":"2024-10-27T06:03:29.048064Z","shell.execute_reply.started":"2024-10-27T06:03:29.013932Z","shell.execute_reply":"2024-10-27T06:03:29.044667Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for row in df_train.loc[df_train['Physical-Weight'] == 0].index:\n    temp_weight = model_substitute_weight.predict(df_train.loc[df_train.index == row][['Basic_Demos-Age', 'Basic_Demos-Sex']])[0]\n    df_train.loc[row, 'Physical-Weight'] = temp_weight","metadata":{"execution":{"iopub.status.busy":"2024-10-27T06:03:29.051706Z","iopub.execute_input":"2024-10-27T06:03:29.053898Z","iopub.status.idle":"2024-10-27T06:03:29.064006Z","shell.execute_reply.started":"2024-10-27T06:03:29.053662Z","shell.execute_reply":"2024-10-27T06:03:29.061491Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train.loc[df_train['Physical-Weight'] == 0]","metadata":{"execution":{"iopub.status.busy":"2024-10-27T06:03:29.066773Z","iopub.execute_input":"2024-10-27T06:03:29.067244Z","iopub.status.idle":"2024-10-27T06:03:29.098446Z","shell.execute_reply.started":"2024-10-27T06:03:29.067193Z","shell.execute_reply":"2024-10-27T06:03:29.096444Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 5.5.2. Age as Confounder","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize = (6, 2), facecolor = \"white\")\n\nsns.boxplot(\n    data = df_train, \n    x = 'Physical-Height', y = 'sii',\n    orient = 'h'\n)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-27T06:03:29.101048Z","iopub.execute_input":"2024-10-27T06:03:29.103428Z","iopub.status.idle":"2024-10-27T06:03:29.349441Z","shell.execute_reply.started":"2024-10-27T06:03:29.103329Z","shell.execute_reply":"2024-10-27T06:03:29.348075Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize = (6, 2), facecolor = \"white\")\n\nsns.boxplot(\n    data = df_train, \n    x = 'Physical-BMI', y = 'sii',\n    orient = 'h'\n)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-27T06:03:29.351162Z","iopub.execute_input":"2024-10-27T06:03:29.351887Z","iopub.status.idle":"2024-10-27T06:03:29.643347Z","shell.execute_reply.started":"2024-10-27T06:03:29.351825Z","shell.execute_reply":"2024-10-27T06:03:29.641775Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize = (4, 4), facecolor = \"white\")\n\nsns.regplot(\n    data = df_train, \n    x = 'Basic_Demos-Age', y = 'Physical-Height'\n)\nsns.regplot(\n    data = df_train, \n    x = 'Basic_Demos-Age', y = 'Physical-BMI'\n)\n\nplt.ylabel('(kg/m^2 or lb)')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-27T06:03:29.646787Z","iopub.execute_input":"2024-10-27T06:03:29.647331Z","iopub.status.idle":"2024-10-27T06:03:30.408731Z","shell.execute_reply.started":"2024-10-27T06:03:29.647281Z","shell.execute_reply":"2024-10-27T06:03:30.407349Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Age is strongly correlated with height.  \nSo, what will happen when we control the age?","metadata":{}},{"cell_type":"code","source":"model_get_height_resid_by_age = make_linear_model(df_train[['Basic_Demos-Age', 'Physical-Height']], 'Physical-Height')","metadata":{"execution":{"iopub.status.busy":"2024-10-27T06:03:30.410424Z","iopub.execute_input":"2024-10-27T06:03:30.410866Z","iopub.status.idle":"2024-10-27T06:03:30.427068Z","shell.execute_reply.started":"2024-10-27T06:03:30.410819Z","shell.execute_reply":"2024-10-27T06:03:30.425412Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train['resid_Height_by_Age'] = df_train['Physical-Height'] - model_get_height_resid_by_age.predict(df_train[['Basic_Demos-Age']])\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2024-10-27T06:03:30.429783Z","iopub.execute_input":"2024-10-27T06:03:30.430215Z","iopub.status.idle":"2024-10-27T06:03:30.464545Z","shell.execute_reply.started":"2024-10-27T06:03:30.430170Z","shell.execute_reply":"2024-10-27T06:03:30.463090Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize = (6, 2), facecolor = \"white\")\n\nsns.boxplot(\n    data = df_train, \n    x = 'resid_Height_by_Age', y = 'sii',\n    orient = 'h'\n)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-27T06:03:30.467964Z","iopub.execute_input":"2024-10-27T06:03:30.469753Z","iopub.status.idle":"2024-10-27T06:03:30.730854Z","shell.execute_reply.started":"2024-10-27T06:03:30.469656Z","shell.execute_reply":"2024-10-27T06:03:30.728482Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The trend has disappeared.  \nSo it can be concluded that the trend according to Height can be entirely by Age.  \nAnd what about BMI?","metadata":{}},{"cell_type":"code","source":"model_get_BMI_resid_by_age = make_linear_model(df_train[['Basic_Demos-Age', 'Physical-BMI']], 'Physical-BMI')","metadata":{"execution":{"iopub.status.busy":"2024-10-27T06:03:30.733056Z","iopub.execute_input":"2024-10-27T06:03:30.733493Z","iopub.status.idle":"2024-10-27T06:03:30.756870Z","shell.execute_reply.started":"2024-10-27T06:03:30.733449Z","shell.execute_reply":"2024-10-27T06:03:30.754674Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train['resid_BMI_by_Age'] = df_train['Physical-BMI'] - model_get_BMI_resid_by_age.predict(df_train[['Basic_Demos-Age']])\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2024-10-27T06:03:30.761888Z","iopub.execute_input":"2024-10-27T06:03:30.762390Z","iopub.status.idle":"2024-10-27T06:03:30.800790Z","shell.execute_reply.started":"2024-10-27T06:03:30.762339Z","shell.execute_reply":"2024-10-27T06:03:30.799325Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize = (6, 2), facecolor = \"white\")\n\nsns.boxplot(\n    data = df_train, \n    x = 'resid_BMI_by_Age', y = 'sii',\n    orient = 'h'\n)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-27T06:03:30.802439Z","iopub.execute_input":"2024-10-27T06:03:30.802862Z","iopub.status.idle":"2024-10-27T06:03:31.096015Z","shell.execute_reply.started":"2024-10-27T06:03:30.802819Z","shell.execute_reply":"2024-10-27T06:03:31.094581Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"After controling the Age, the part that is explained by BMI remains.","metadata":{}},{"cell_type":"markdown","source":"### 5.5.3. Interaction","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize = (6, 2), facecolor = \"white\")\n\nsns.boxplot(\n    data = df_train, \n    x = 'resid_BMI_by_Age', y = 'sii',\n    orient = 'h',\n    hue = 'is_Adolescent'\n)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-27T06:09:09.637279Z","iopub.execute_input":"2024-10-27T06:09:09.637985Z","iopub.status.idle":"2024-10-27T06:09:10.111938Z","shell.execute_reply.started":"2024-10-27T06:09:09.637934Z","shell.execute_reply":"2024-10-27T06:09:10.110608Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize = (6, 2), facecolor = \"white\")\n\nsns.boxplot(\n    data = df_train, \n    x = 'resid_BMI_by_Age', y = 'sii',\n    orient = 'h',\n    hue = 'Basic_Demos-Sex'\n)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-27T06:09:43.106585Z","iopub.execute_input":"2024-10-27T06:09:43.107068Z","iopub.status.idle":"2024-10-27T06:09:43.612542Z","shell.execute_reply.started":"2024-10-27T06:09:43.107023Z","shell.execute_reply":"2024-10-27T06:09:43.610987Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Summary:\n- rows that have zero weight has been substituted(for further use)\n- confounding effect of Age\n    - positive correlation with Height can be entirely explained by Age\n- interaction with sex and age\n    - for children and female, BMI above the ordinary level may be considered a sign of greater risk","metadata":{}},{"cell_type":"markdown","source":"## 5.6. Bio-electric Impedance Analysis","metadata":{}},{"cell_type":"markdown","source":"### 5.6.1. Outlier Correction","metadata":{}},{"cell_type":"code","source":"# outlier correction\ndf_train.loc[df_train['BIA-BIA_SMM'] > 500]","metadata":{"execution":{"iopub.status.busy":"2024-10-27T06:03:31.097885Z","iopub.execute_input":"2024-10-27T06:03:31.098439Z","iopub.status.idle":"2024-10-27T06:03:31.118731Z","shell.execute_reply.started":"2024-10-27T06:03:31.098377Z","shell.execute_reply":"2024-10-27T06:03:31.117188Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train.loc[3205 ,'BIA-BIA_SMM'] = 36.0769\ndf_train.loc[3205 ,'BIA-BIA_BMI'] = 16.972291\ndf_train.loc[3511 ,'BIA-BIA_SMM'] = 82.3028","metadata":{"execution":{"iopub.status.busy":"2024-10-27T06:03:31.120416Z","iopub.execute_input":"2024-10-27T06:03:31.120855Z","iopub.status.idle":"2024-10-27T06:03:31.131609Z","shell.execute_reply.started":"2024-10-27T06:03:31.120811Z","shell.execute_reply":"2024-10-27T06:03:31.130103Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train[(df_train.index == 3205) | (df_train.index == 3511)]","metadata":{"execution":{"iopub.status.busy":"2024-10-27T06:03:31.133432Z","iopub.execute_input":"2024-10-27T06:03:31.133868Z","iopub.status.idle":"2024-10-27T06:03:31.167495Z","shell.execute_reply.started":"2024-10-27T06:03:31.133824Z","shell.execute_reply":"2024-10-27T06:03:31.165590Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"summary_numerical_dist(df_train, 'BIA-BIA_SMM', .95, 1)","metadata":{"execution":{"iopub.status.busy":"2024-10-27T06:03:31.169561Z","iopub.execute_input":"2024-10-27T06:03:31.170869Z","iopub.status.idle":"2024-10-27T06:03:32.320590Z","shell.execute_reply.started":"2024-10-27T06:03:31.170801Z","shell.execute_reply":"2024-10-27T06:03:32.319347Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# descriptive statistics\ndisplay(df_train.describe().round(3).T) # numerical","metadata":{"execution":{"iopub.status.busy":"2024-10-27T06:03:32.322061Z","iopub.execute_input":"2024-10-27T06:03:32.322465Z","iopub.status.idle":"2024-10-27T06:03:32.395927Z","shell.execute_reply.started":"2024-10-27T06:03:32.322421Z","shell.execute_reply":"2024-10-27T06:03:32.394586Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 5.6.2. Measurement Error?","metadata":{}},{"cell_type":"code","source":"temp_df = df_train[['Physical-BMI', 'BIA-BIA_BMI']]\ntemp_df['BMI_Physical_minus_BIA'] = temp_df['Physical-BMI'] - temp_df['BIA-BIA_BMI']\ntemp_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-10-27T06:03:32.397637Z","iopub.execute_input":"2024-10-27T06:03:32.398052Z","iopub.status.idle":"2024-10-27T06:03:32.416701Z","shell.execute_reply.started":"2024-10-27T06:03:32.398009Z","shell.execute_reply":"2024-10-27T06:03:32.414136Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"display(temp_df.describe().round(3).T) # numerical","metadata":{"execution":{"iopub.status.busy":"2024-10-27T06:03:32.419613Z","iopub.execute_input":"2024-10-27T06:03:32.420373Z","iopub.status.idle":"2024-10-27T06:03:32.451175Z","shell.execute_reply.started":"2024-10-27T06:03:32.420304Z","shell.execute_reply":"2024-10-27T06:03:32.449777Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"need further investigation","metadata":{}},{"cell_type":"markdown","source":"### 5.6.3. Make by Ratio!","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize = (6, 2), facecolor = \"white\")\n\nsns.boxplot(\n    data = df_train, \n    x = 'BIA-BIA_SMM', y = 'sii',\n    orient = 'h'\n)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-27T06:03:32.452806Z","iopub.execute_input":"2024-10-27T06:03:32.453222Z","iopub.status.idle":"2024-10-27T06:03:32.705625Z","shell.execute_reply.started":"2024-10-27T06:03:32.453169Z","shell.execute_reply":"2024-10-27T06:03:32.704177Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"positive relationship with SMM is counterintuitive.  \nIt is reasonable to use SMM as a ratio to body weight.","metadata":{}},{"cell_type":"code","source":"df_train['SMM_per_Weight'] = df_train['BIA-BIA_SMM'] / df_train['Physical-Weight']","metadata":{"execution":{"iopub.status.busy":"2024-10-27T06:03:32.707493Z","iopub.execute_input":"2024-10-27T06:03:32.707940Z","iopub.status.idle":"2024-10-27T06:03:32.719612Z","shell.execute_reply.started":"2024-10-27T06:03:32.707894Z","shell.execute_reply":"2024-10-27T06:03:32.717608Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train.loc[df_train['SMM_per_Weight'] > 1]","metadata":{"execution":{"iopub.status.busy":"2024-10-27T06:03:32.722001Z","iopub.execute_input":"2024-10-27T06:03:32.722775Z","iopub.status.idle":"2024-10-27T06:03:32.763934Z","shell.execute_reply.started":"2024-10-27T06:03:32.722699Z","shell.execute_reply":"2024-10-27T06:03:32.762232Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"'SMM / Weight' cannot be over 1 theorically.  \nSo I will condider these rows as measurement errors.","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize = (6, 2), facecolor = \"white\")\n\nsns.boxplot(\n    data = df_train.loc[df_train['SMM_per_Weight'] < 1], \n    x = 'SMM_per_Weight', y = 'sii',\n    orient = 'h'\n)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-27T06:03:32.765517Z","iopub.execute_input":"2024-10-27T06:03:32.765954Z","iopub.status.idle":"2024-10-27T06:03:33.081985Z","shell.execute_reply.started":"2024-10-27T06:03:32.765908Z","shell.execute_reply":"2024-10-27T06:03:33.080413Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"now it shows intuitive result(slight negative correlation).","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize = (6, 2), facecolor = \"white\")\n\nsns.boxplot(\n    data = df_train.loc[df_train['SMM_per_Weight'] < 1], \n    x = 'SMM_per_Weight', y = 'sii',\n    orient = 'h',\n    hue = 'is_Adolescent'\n)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-27T06:03:33.084872Z","iopub.execute_input":"2024-10-27T06:03:33.085418Z","iopub.status.idle":"2024-10-27T06:03:33.617719Z","shell.execute_reply.started":"2024-10-27T06:03:33.085367Z","shell.execute_reply":"2024-10-27T06:03:33.616001Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize = (6, 2), facecolor = \"white\")\n\nsns.boxplot(\n    data = df_train.loc[df_train['SMM_per_Weight'] < 1], \n    x = 'SMM_per_Weight', y = 'sii',\n    orient = 'h',\n    hue = 'Basic_Demos-Sex'\n)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-27T06:03:33.619236Z","iopub.execute_input":"2024-10-27T06:03:33.619718Z","iopub.status.idle":"2024-10-27T06:03:34.093899Z","shell.execute_reply.started":"2024-10-27T06:03:33.619575Z","shell.execute_reply":"2024-10-27T06:03:34.091791Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Summary:\n- need to correct extreme outliers\n- making by ratio is reasonable\n- interaction with sex\n    - if male, there is no significant trend\n    - but if female, negative(-) correlation is clear","metadata":{}},{"cell_type":"markdown","source":"## 5.7. Children's Global Assessment Scale","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize = (6, 2), facecolor = \"white\")\n\nsns.boxplot(\n    data = df_train, \n    x = 'CGAS-CGAS_Score', y = 'sii',\n    orient = 'h'\n)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-27T06:23:17.506900Z","iopub.execute_input":"2024-10-27T06:23:17.507925Z","iopub.status.idle":"2024-10-27T06:23:17.766887Z","shell.execute_reply.started":"2024-10-27T06:23:17.507841Z","shell.execute_reply":"2024-10-27T06:23:17.765550Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize = (6, 2), facecolor = \"white\")\n\nsns.boxplot(\n    data = df_train, \n    x = 'CGAS-CGAS_Score', y = 'sii',\n    orient = 'h',\n    hue = 'is_Adolescent'\n)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-27T06:24:57.024748Z","iopub.execute_input":"2024-10-27T06:24:57.025845Z","iopub.status.idle":"2024-10-27T06:24:57.538607Z","shell.execute_reply.started":"2024-10-27T06:24:57.025790Z","shell.execute_reply":"2024-10-27T06:24:57.536652Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize = (6, 2), facecolor = \"white\")\n\nsns.boxplot(\n    data = df_train, \n    x = 'CGAS-CGAS_Score', y = 'sii',\n    orient = 'h',\n    hue = 'Basic_Demos-Sex'\n)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-27T06:25:09.430010Z","iopub.execute_input":"2024-10-27T06:25:09.430494Z","iopub.status.idle":"2024-10-27T06:25:09.905049Z","shell.execute_reply.started":"2024-10-27T06:25:09.430447Z","shell.execute_reply":"2024-10-27T06:25:09.903583Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Summary:\n- negative(-) correlation\n- interaction with sex\n    - if female, negative(-) correlation is more clear","metadata":{}}]}