{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 0. Import Library","metadata":{}},{"cell_type":"code","source":"# standard\nimport numpy as np\nimport pandas as pd\nimport time\n\n# plots\nimport matplotlib.pyplot as plt\nimport missingno as msno\nimport seaborn as sns\nimport plotly\nimport plotly.express as px\n\n# statistics\nfrom scipy import stats\nfrom sklearn.metrics import cohen_kappa_score\n\n# H2O\nimport h2o\nfrom h2o.estimators import H2OGradientBoostingEstimator","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-05T01:49:13.206674Z","iopub.execute_input":"2024-12-05T01:49:13.207253Z","iopub.status.idle":"2024-12-05T01:49:16.975661Z","shell.execute_reply.started":"2024-12-05T01:49:13.207217Z","shell.execute_reply":"2024-12-05T01:49:16.974887Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# configs\npd.set_option('display.max_columns', None) # we want to display all columns in this notebook\npd.set_option('display.max_rows', 100) # increase rows to be displayed\npd.set_option('display.max_colwidth', None) # show full cell contents\n\n# random seed\nmy_random_seed = 111\n\n# aesthetics\ndefault_color_1 = 'darkblue'\ndefault_color_2 = 'darkgreen'\ndefault_color_3 = 'darkred'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T01:49:16.977442Z","iopub.execute_input":"2024-12-05T01:49:16.977987Z","iopub.status.idle":"2024-12-05T01:49:16.983556Z","shell.execute_reply.started":"2024-12-05T01:49:16.977950Z","shell.execute_reply":"2024-12-05T01:49:16.982304Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 1. Load Data","metadata":{}},{"cell_type":"code","source":"# load data\ndf_train = pd.read_csv('../input/child-mind-institute-problematic-internet-use/train.csv')\ndf_test = pd.read_csv('../input/child-mind-institute-problematic-internet-use/test.csv')\ndf_sub = pd.read_csv('../input/child-mind-institute-problematic-internet-use/sample_submission.csv')\ndf_dict = pd.read_csv('../input/child-mind-institute-problematic-internet-use/data_dictionary.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T01:52:23.419279Z","iopub.execute_input":"2024-12-05T01:52:23.419659Z","iopub.status.idle":"2024-12-05T01:52:23.476049Z","shell.execute_reply.started":"2024-12-05T01:52:23.419622Z","shell.execute_reply":"2024-12-05T01:52:23.475181Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# show data dictionary\ndf_dict","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T01:49:17.088483Z","iopub.execute_input":"2024-12-05T01:49:17.088790Z","iopub.status.idle":"2024-12-05T01:49:17.118233Z","shell.execute_reply.started":"2024-12-05T01:49:17.088759Z","shell.execute_reply":"2024-12-05T01:49:17.117257Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T01:52:31.750815Z","iopub.execute_input":"2024-12-05T01:52:31.751182Z","iopub.status.idle":"2024-12-05T01:52:31.888877Z","shell.execute_reply.started":"2024-12-05T01:52:31.751150Z","shell.execute_reply":"2024-12-05T01:52:31.887775Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# structure of data - train\ndf_test.head(20)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T01:49:17.239485Z","iopub.execute_input":"2024-12-05T01:49:17.239804Z","iopub.status.idle":"2024-12-05T01:49:17.318918Z","shell.execute_reply.started":"2024-12-05T01:49:17.239772Z","shell.execute_reply":"2024-12-05T01:49:17.317896Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T01:49:17.320126Z","iopub.execute_input":"2024-12-05T01:49:17.320463Z","iopub.status.idle":"2024-12-05T01:49:17.443515Z","shell.execute_reply.started":"2024-12-05T01:49:17.320415Z","shell.execute_reply":"2024-12-05T01:49:17.442467Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train[\"sii\"].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T01:49:17.444958Z","iopub.execute_input":"2024-12-05T01:49:17.445263Z","iopub.status.idle":"2024-12-05T01:49:17.464989Z","shell.execute_reply.started":"2024-12-05T01:49:17.445232Z","shell.execute_reply":"2024-12-05T01:49:17.463976Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"msno.bar(df_train.iloc[:, :82], sort='ascending') ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T01:49:17.466442Z","iopub.execute_input":"2024-12-05T01:49:17.466895Z","iopub.status.idle":"2024-12-05T01:49:20.145405Z","shell.execute_reply.started":"2024-12-05T01:49:17.466827Z","shell.execute_reply":"2024-12-05T01:49:20.144339Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_cols = set(df_train.columns)\ntest_cols = set(df_test.columns)\ncolumns_not_in_test = sorted(list(train_cols - test_cols))\ndf_dict[df_dict['Field'].isin(columns_not_in_test)]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T01:49:20.148691Z","iopub.execute_input":"2024-12-05T01:49:20.149044Z","iopub.status.idle":"2024-12-05T01:49:20.167273Z","shell.execute_reply.started":"2024-12-05T01:49:20.149010Z","shell.execute_reply":"2024-12-05T01:49:20.166112Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pciat_min_max = df_train.groupby('sii')['PCIAT-PCIAT_Total'].agg(['min', 'max'])\npciat_min_max = pciat_min_max.rename(\n    columns={'min': 'Minimum PCIAT total Score', 'max': 'Maximum total PCIAT Score'}\n)\npciat_min_max","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T01:49:20.168732Z","iopub.execute_input":"2024-12-05T01:49:20.169887Z","iopub.status.idle":"2024-12-05T01:49:20.190599Z","shell.execute_reply.started":"2024-12-05T01:49:20.169812Z","shell.execute_reply":"2024-12-05T01:49:20.189671Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_with_sii = df_train[df_train['sii'].notna()][columns_not_in_test]\ntrain_with_sii[train_with_sii.isna().any(axis=1)].head(20).style.applymap(\n    lambda x: 'background-color: #FFC0CB' if pd.isna(x) else ''\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T01:49:20.191732Z","iopub.execute_input":"2024-12-05T01:49:20.192391Z","iopub.status.idle":"2024-12-05T01:49:20.284749Z","shell.execute_reply.started":"2024-12-05T01:49:20.192357Z","shell.execute_reply":"2024-12-05T01:49:20.283763Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Try to drop some data having the ratio of missing features greater than 70%","metadata":{}},{"cell_type":"code","source":"threshold = 1 * len(df_train)\ncolumns_with_data = df_train.columns[df_train.isnull().sum() < threshold]\ntrain_data = df_train[columns_with_data]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T01:49:20.286189Z","iopub.execute_input":"2024-12-05T01:49:20.286805Z","iopub.status.idle":"2024-12-05T01:49:20.299862Z","shell.execute_reply.started":"2024-12-05T01:49:20.286756Z","shell.execute_reply":"2024-12-05T01:49:20.298967Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T01:49:20.301165Z","iopub.execute_input":"2024-12-05T01:49:20.301559Z","iopub.status.idle":"2024-12-05T01:49:20.324346Z","shell.execute_reply.started":"2024-12-05T01:49:20.301510Z","shell.execute_reply":"2024-12-05T01:49:20.323352Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Check the corelation between PCIAT-total and sii","metadata":{}},{"cell_type":"code","source":"PCIAT_cols = [f'PCIAT-PCIAT_{i+1:02d}' for i in range(20)]\nrecalc_total_score = train_with_sii[PCIAT_cols].sum(\n    axis=1, skipna=True\n)\n(recalc_total_score == train_with_sii['PCIAT-PCIAT_Total']).all()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T01:49:20.325591Z","iopub.execute_input":"2024-12-05T01:49:20.325992Z","iopub.status.idle":"2024-12-05T01:49:20.336627Z","shell.execute_reply.started":"2024-12-05T01:49:20.325958Z","shell.execute_reply":"2024-12-05T01:49:20.335502Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Tất cả các PICAT bị miss tính bằng 0, dựa vào tổng PCIAT đưa ra label sii","metadata":{}},{"cell_type":"code","source":"pciat_min_max = df_train.groupby('sii')['PCIAT-PCIAT_Total'].agg(['min', 'max'])\npciat_min_max = pciat_min_max.rename(\n    columns={'min': 'Minimum PCIAT total Score', 'max': 'Maximum total PCIAT Score'}\n)\npciat_min_max","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T01:49:20.338017Z","iopub.execute_input":"2024-12-05T01:49:20.338428Z","iopub.status.idle":"2024-12-05T01:49:20.352849Z","shell.execute_reply.started":"2024-12-05T01:49:20.338383Z","shell.execute_reply":"2024-12-05T01:49:20.351440Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Thử tính lại sii, với giả định tất cả các PCIAT chưa được trả lời đạt max, nếu như PCIAT_old và PCIAT_new nằm cùng 1 ngưỡng sii thì mới công nhận sii chuẩn, còn không đưa ra not a number","metadata":{}},{"cell_type":"code","source":"def recalculate_sii(row):\n    if pd.isna(row['PCIAT-PCIAT_Total']):\n        return np.nan\n    max_possible = row['PCIAT-PCIAT_Total'] + row[PCIAT_cols].isna().sum() * 5\n    if row['PCIAT-PCIAT_Total'] <= 30 and max_possible <= 30:\n        return 0\n    elif 31 <= row['PCIAT-PCIAT_Total'] <= 49 and max_possible <= 49:\n        return 1\n    elif 50 <= row['PCIAT-PCIAT_Total'] <= 79 and max_possible <= 79:\n        return 2\n    elif row['PCIAT-PCIAT_Total'] >= 80 and max_possible >= 80:\n        return 3\n    return np.nan\n\ndf_train['recalc_sii'] = df_train.apply(recalculate_sii, axis=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T01:49:20.354398Z","iopub.execute_input":"2024-12-05T01:49:20.354722Z","iopub.status.idle":"2024-12-05T01:49:21.898284Z","shell.execute_reply.started":"2024-12-05T01:49:20.354691Z","shell.execute_reply":"2024-12-05T01:49:21.897195Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mismatch_rows = df_train[\n    (df_train['recalc_sii'] != df_train['sii']) & df_train['sii'].notna()\n]\n\nmismatch_rows[PCIAT_cols + [\n    'PCIAT-PCIAT_Total', 'sii', 'recalc_sii'\n]].style.applymap(\n    lambda x: 'background-color: #FFC0CB' if pd.isna(x) else ''\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T01:49:21.899642Z","iopub.execute_input":"2024-12-05T01:49:21.899986Z","iopub.status.idle":"2024-12-05T01:49:21.924870Z","shell.execute_reply.started":"2024-12-05T01:49:21.899955Z","shell.execute_reply":"2024-12-05T01:49:21.923719Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mismatch_rows.shape\ntrain = df_train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T01:49:21.926306Z","iopub.execute_input":"2024-12-05T01:49:21.926693Z","iopub.status.idle":"2024-12-05T01:49:21.933354Z","shell.execute_reply.started":"2024-12-05T01:49:21.926658Z","shell.execute_reply":"2024-12-05T01:49:21.932239Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Ta có 17 bộ dữ liệu có nguy cơ sai","metadata":{}},{"cell_type":"code","source":"train['sii'] = train['recalc_sii']\ntrain['complete_resp_total'] = train['PCIAT-PCIAT_Total'].where(\n    train[PCIAT_cols].notna().all(axis=1), np.nan\n)\n\nsii_map = {0: '0 (None)', 1: '1 (Mild)', 2: '2 (Moderate)', 3: '3 (Severe)'}\ntrain['sii'] = train['sii'].map(sii_map).fillna('Missing')\n\nsii_order = ['Missing', '0 (None)', '1 (Mild)', '2 (Moderate)', '3 (Severe)']\ntrain['sii'] = pd.Categorical(train['sii'], categories=sii_order, ordered=True)\n\ntrain.drop(columns='recalc_sii', inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T01:49:21.934557Z","iopub.execute_input":"2024-12-05T01:49:21.934870Z","iopub.status.idle":"2024-12-05T01:49:21.958235Z","shell.execute_reply.started":"2024-12-05T01:49:21.934811Z","shell.execute_reply":"2024-12-05T01:49:21.957301Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sii_counts = train['sii'].value_counts().reset_index()\ntotal = sii_counts['count'].sum()\nsii_counts['percentage'] = (sii_counts['count'] / total) * 100\n\nfig, axes = plt.subplots(1, 2, figsize=(14, 5))\n\n# SII\nsns.barplot(x='sii', y='count', data=sii_counts, palette='Blues_d', ax=axes[0])\naxes[0].set_title('Distribution of Severity Impairment Index (sii)', fontsize=14)\nfor p in axes[0].patches:\n    height = p.get_height()\n    percentage = sii_counts.loc[sii_counts['count'] == height, 'percentage'].values[0]\n    axes[0].text(\n        p.get_x() + p.get_width() / 2,\n        height + 5, f'{int(height)} ({percentage:.1f}%)',\n        ha=\"center\", fontsize=12\n    )\n\n# PCIAT_Total for complete responses\nsns.histplot(train['complete_resp_total'].dropna(), bins=20, ax=axes[1])\naxes[1].set_title('Distribution of PCIAT_Total', fontsize=14)\naxes[1].set_xlabel('PCIAT_Total for Complete PCIAT Responses')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T01:49:21.959484Z","iopub.execute_input":"2024-12-05T01:49:21.960156Z","iopub.status.idle":"2024-12-05T01:49:22.426180Z","shell.execute_reply.started":"2024-12-05T01:49:21.960123Z","shell.execute_reply":"2024-12-05T01:49:22.425082Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def calculate_stats(data, columns):\n    if isinstance(columns, str):\n        columns = [columns]\n\n    stats = []\n    for col in columns:\n        if data[col].dtype in ['object', 'category']:\n            counts = data[col].value_counts(dropna=False, sort=False)\n            percents = data[col].value_counts(normalize=True, dropna=False, sort=False) * 100\n            formatted = counts.astype(str) + ' (' + percents.round(2).astype(str) + '%)'\n            stats_col = pd.DataFrame({'count (%)': formatted})\n            stats.append(stats_col)\n        else:\n            stats_col = data[col].describe().to_frame().transpose()\n            stats_col['missing'] = data[col].isnull().sum()\n            stats_col.index.name = col\n            stats.append(stats_col)\n\n    return pd.concat(stats, axis=0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T01:49:22.427464Z","iopub.execute_input":"2024-12-05T01:49:22.427785Z","iopub.status.idle":"2024-12-05T01:49:22.435164Z","shell.execute_reply.started":"2024-12-05T01:49:22.427753Z","shell.execute_reply":"2024-12-05T01:49:22.433996Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Phân tích dữ liệu ở tuổi và giới tính","metadata":{}},{"cell_type":"code","source":"train['Age Group'] = pd.cut(\n    train['Basic_Demos-Age'],\n    bins=[4, 12, 18, 22],\n    labels=['Children (5-12)', 'Adolescents (13-18)', 'Adults (19-22)']\n)\ncalculate_stats(train, 'Age Group')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T01:49:22.436428Z","iopub.execute_input":"2024-12-05T01:49:22.437058Z","iopub.status.idle":"2024-12-05T01:49:22.461094Z","shell.execute_reply.started":"2024-12-05T01:49:22.437025Z","shell.execute_reply":"2024-12-05T01:49:22.459945Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sex_map = {0: 'Male', 1: 'Female'}\ntrain['Basic_Demos-Sex'] = train['Basic_Demos-Sex'].map(sex_map)\ncalculate_stats(train, 'Basic_Demos-Sex')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T01:49:22.462122Z","iopub.execute_input":"2024-12-05T01:49:22.462396Z","iopub.status.idle":"2024-12-05T01:49:22.474498Z","shell.execute_reply.started":"2024-12-05T01:49:22.462366Z","shell.execute_reply":"2024-12-05T01:49:22.473401Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"stats = train.groupby(['Age Group', 'sii']).size().unstack(fill_value=0)\nfig, axes = plt.subplots(1, len(stats), figsize=(18, 5))\n\nfor i, age_group in enumerate(stats.index):\n    group_counts = stats.loc[age_group] / stats.loc[age_group].sum()\n    axes[i].pie(\n        group_counts, labels=group_counts.index, autopct='%1.1f%%',\n        startangle=90, colors=sns.color_palette(\"Set3\"),\n        labeldistance=1.05, pctdistance=0.80\n    )\n    axes[i].set_title(f'SII Distribution for {age_group}')\n    axes[i].axis('equal')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T01:49:22.475783Z","iopub.execute_input":"2024-12-05T01:49:22.476111Z","iopub.status.idle":"2024-12-05T01:49:23.003781Z","shell.execute_reply.started":"2024-12-05T01:49:22.476081Z","shell.execute_reply":"2024-12-05T01:49:23.002805Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Internet use","metadata":{}},{"cell_type":"code","source":"train['PreInt_EduHx-computerinternet_hoursday'].unique()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T01:49:23.005106Z","iopub.execute_input":"2024-12-05T01:49:23.005491Z","iopub.status.idle":"2024-12-05T01:49:23.013310Z","shell.execute_reply.started":"2024-12-05T01:49:23.005446Z","shell.execute_reply":"2024-12-05T01:49:23.012253Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"param_map = {0: '< 1h/day', 1: '~ 1h/day', 2: '~ 2hs/day', 3: '> 3hs/day'}\ntrain['internet_use_encoded'] = train[\n    'PreInt_EduHx-computerinternet_hoursday'\n].map(param_map).fillna('Missing')\n\nparam_ord = ['Missing', '< 1h/day', '~ 1h/day', '~ 2hs/day', '> 3hs/day']\ntrain['internet_use_encoded'] = pd.Categorical(\n    train['internet_use_encoded'], categories=param_ord,\n    ordered=True\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T01:49:23.014510Z","iopub.execute_input":"2024-12-05T01:49:23.014861Z","iopub.status.idle":"2024-12-05T01:49:23.027599Z","shell.execute_reply.started":"2024-12-05T01:49:23.014801Z","shell.execute_reply":"2024-12-05T01:49:23.026588Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"calculate_stats(train, 'PreInt_EduHx-computerinternet_hoursday')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T01:49:23.031566Z","iopub.execute_input":"2024-12-05T01:49:23.031934Z","iopub.status.idle":"2024-12-05T01:49:23.053073Z","shell.execute_reply.started":"2024-12-05T01:49:23.031897Z","shell.execute_reply":"2024-12-05T01:49:23.052055Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}