{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport polars as pl\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:29.355211Z","iopub.execute_input":"2024-11-21T22:08:29.355722Z","iopub.status.idle":"2024-11-21T22:08:29.362815Z","shell.execute_reply.started":"2024-11-21T22:08:29.355664Z","shell.execute_reply":"2024-11-21T22:08:29.360948Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#
Read data dictionary
","metadata":{}},{"cell_type":"code","source":"data_dict = pd.read_csv(\"/kaggle/input/child-mind-institute-problematic-internet-use/data_dictionary.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:29.365264Z","iopub.execute_input":"2024-11-21T22:08:29.366527Z","iopub.status.idle":"2024-11-21T22:08:29.381023Z","shell.execute_reply.started":"2024-11-21T22:08:29.366467Z","shell.execute_reply":"2024-11-21T22:08:29.379551Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(data_dict.shape)\ndata_dict.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:29.383241Z","iopub.execute_input":"2024-11-21T22:08:29.383716Z","iopub.status.idle":"2024-11-21T22:08:29.398801Z","shell.execute_reply.started":"2024-11-21T22:08:29.383665Z","shell.execute_reply":"2024-11-21T22:08:29.397437Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(data_dict['Type'].unique())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:29.402025Z","iopub.execute_input":"2024-11-21T22:08:29.402515Z","iopub.status.idle":"2024-11-21T22:08:29.412489Z","shell.execute_reply.started":"2024-11-21T22:08:29.402465Z","shell.execute_reply":"2024-11-21T22:08:29.411206Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print('Categorical count: ', data_dict[data_dict['Type'].str.contains('categorical')].shape[0])\nprint('String count: ', data_dict[data_dict['Type'].str.contains('str')].shape[0])\nprint('Float count: ', data_dict[data_dict['Type'].str.contains('float')].shape[0])\nprint('Int count: ', data_dict[data_dict['Type'].str.contains('int') & ~data_dict['Type'].str.contains('categorical')].shape[0])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:29.413679Z","iopub.execute_input":"2024-11-21T22:08:29.414055Z","iopub.status.idle":"2024-11-21T22:08:29.430014Z","shell.execute_reply.started":"2024-11-21T22:08:29.414021Z","shell.execute_reply":"2024-11-21T22:08:29.428422Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_dict[data_dict['Field'] == 'PCIAT-PCIAT_Total']['Value Labels'].iloc[0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:29.431576Z","iopub.execute_input":"2024-11-21T22:08:29.431981Z","iopub.status.idle":"2024-11-21T22:08:29.447843Z","shell.execute_reply.started":"2024-11-21T22:08:29.431946Z","shell.execute_reply":"2024-11-21T22:08:29.446130Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#
EDA For series_train/series_test
","metadata":{}},{"cell_type":"code","source":"actigraphy = pd.read_parquet('/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet/id=0417c91e/part-0.parquet')\nactigraphy","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:29.449756Z","iopub.execute_input":"2024-11-21T22:08:29.450292Z","iopub.status.idle":"2024-11-21T22:08:29.518126Z","shell.execute_reply.started":"2024-11-21T22:08:29.450227Z","shell.execute_reply":"2024-11-21T22:08:29.516828Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#
EDA for train/test
","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:29.519552Z","iopub.execute_input":"2024-11-21T22:08:29.520014Z","iopub.status.idle":"2024-11-21T22:08:29.574449Z","shell.execute_reply.started":"2024-11-21T22:08:29.519968Z","shell.execute_reply":"2024-11-21T22:08:29.573182Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:29.575739Z","iopub.execute_input":"2024-11-21T22:08:29.576116Z","iopub.status.idle":"2024-11-21T22:08:29.596533Z","shell.execute_reply.started":"2024-11-21T22:08:29.576082Z","shell.execute_reply":"2024-11-21T22:08:29.595243Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:29.600605Z","iopub.execute_input":"2024-11-21T22:08:29.601089Z","iopub.status.idle":"2024-11-21T22:08:29.617857Z","shell.execute_reply.started":"2024-11-21T22:08:29.601051Z","shell.execute_reply":"2024-11-21T22:08:29.616542Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Train shape: {test.shape}\")\ntest.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:29.619314Z","iopub.execute_input":"2024-11-21T22:08:29.619742Z","iopub.status.idle":"2024-11-21T22:08:29.660050Z","shell.execute_reply.started":"2024-11-21T22:08:29.619696Z","shell.execute_reply":"2024-11-21T22:08:29.658728Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Train shape: {train.shape}\")\ntrain.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:29.661560Z","iopub.execute_input":"2024-11-21T22:08:29.662041Z","iopub.status.idle":"2024-11-21T22:08:29.696538Z","shell.execute_reply.started":"2024-11-21T22:08:29.661994Z","shell.execute_reply":"2024-11-21T22:08:29.695426Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Các cột bên test ít hơn => kiểm tra các cột còn thiếu","metadata":{}},{"cell_type":"code","source":"train_cols = set(train.columns)\ntest_cols = set(test.columns)\ncolumns_not_in_test = sorted(list(train_cols - test_cols))\ndata_dict[data_dict['Field'].isin(columns_not_in_test)]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:29.697892Z","iopub.execute_input":"2024-11-21T22:08:29.698237Z","iopub.status.idle":"2024-11-21T22:08:29.715377Z","shell.execute_reply.started":"2024-11-21T22:08:29.698204Z","shell.execute_reply":"2024-11-21T22:08:29.714036Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"=>> Parent-Child Internet Addiction Test (PCIAT) không có trong test","metadata":{}},{"cell_type":"markdown","source":"##
Parent-Child Internet Addiction Test (PCIAT)
","metadata":{}},{"cell_type":"code","source":"train_with_sii = train[train['sii'].notna()][columns_not_in_test]\ntrain_with_sii[train_with_sii.isna().any(axis=1)].head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:29.716688Z","iopub.execute_input":"2024-11-21T22:08:29.717173Z","iopub.status.idle":"2024-11-21T22:08:29.754092Z","shell.execute_reply.started":"2024-11-21T22:08:29.717124Z","shell.execute_reply":"2024-11-21T22:08:29.752844Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Một số câu hỏi Kiểm tra chứng nghiện Internet có thể bị người trả lời bỏ qua (thiếu giá trị trong các cột PCIAT-PCIAT_01 đến PCIAT-PCIAT_20), nhưng điểm SII vẫn được lấy từ tổng của các giá trị không phải NA, dẫn đến các giá trị SII có khả năng không hợp lệ (tất nhiên trừ khi một số câu trả lời bị cắt bỏ sau khi dữ liệu được thu thập, chỉ để cung cấp thêm một chút thông tin về thử thách)","metadata":{}},{"cell_type":"code","source":"PCIAT_cols = [f'PCIAT-PCIAT_{i+1:02d}' for i in range(20)]\nrecalc_total_score = train_with_sii[PCIAT_cols].sum(\n axis=1, skipna=True\n)\n(recalc_total_score == train_with_sii['PCIAT-PCIAT_Total']).all()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:29.755913Z","iopub.execute_input":"2024-11-21T22:08:29.756413Z","iopub.status.idle":"2024-11-21T22:08:29.768883Z","shell.execute_reply.started":"2024-11-21T22:08:29.756345Z","shell.execute_reply":"2024-11-21T22:08:29.767637Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def recalculate_sii(row):\n if pd.isna(row['PCIAT-PCIAT_Total']):\n return np.nan\n max_possible = row['PCIAT-PCIAT_Total'] + row[PCIAT_cols].isna().sum() * 5\n if row['PCIAT-PCIAT_Total'] <= 30 and max_possible <= 30:\n return 0\n elif 31 <= row['PCIAT-PCIAT_Total'] <= 49 and max_possible <= 49:\n return 1\n elif 50 <= row['PCIAT-PCIAT_Total'] <= 79 and max_possible <= 79:\n return 2\n elif row['PCIAT-PCIAT_Total'] >= 80 and max_possible >= 80:\n return 3\n return np.nan\n\ntrain['recalc_sii'] = train.apply(recalculate_sii, axis=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:29.770450Z","iopub.execute_input":"2024-11-21T22:08:29.770945Z","iopub.status.idle":"2024-11-21T22:08:31.318524Z","shell.execute_reply.started":"2024-11-21T22:08:29.770897Z","shell.execute_reply":"2024-11-21T22:08:31.317423Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mismatch_rows = train[\n (train['recalc_sii'] != train['sii']) & train['sii'].notna()\n]\n\nmismatch_rows[PCIAT_cols + [\n 'PCIAT-PCIAT_Total', 'sii', 'recalc_sii'\n]].style.applymap(\n lambda x: 'background-color: #FFC0CB' if pd.isna(x) else ''\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:31.319942Z","iopub.execute_input":"2024-11-21T22:08:31.320292Z","iopub.status.idle":"2024-11-21T22:08:31.347564Z","shell.execute_reply.started":"2024-11-21T22:08:31.320258Z","shell.execute_reply":"2024-11-21T22:08:31.346327Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:31.349211Z","iopub.execute_input":"2024-11-21T22:08:31.349715Z","iopub.status.idle":"2024-11-21T22:08:31.382227Z","shell.execute_reply.started":"2024-11-21T22:08:31.349660Z","shell.execute_reply":"2024-11-21T22:08:31.381081Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['sii'] = train['recalc_sii']\ntrain['complete_resp_total'] = train['PCIAT-PCIAT_Total'].where(\n train[PCIAT_cols].notna().all(axis=1), np.nan\n)\n\nsii_map = {0: '0 (None)', 1: '1 (Mild)', 2: '2 (Moderate)', 3: '3 (Severe)'}\ntrain['sii'] = train['sii'].map(sii_map).fillna('Missing')\n\nsii_order = ['Missing', '0 (None)', '1 (Mild)', '2 (Moderate)', '3 (Severe)']\ntrain['sii'] = pd.Categorical(train['sii'], categories=sii_order, ordered=True)\n\ntrain.drop(columns='recalc_sii', inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:31.383518Z","iopub.execute_input":"2024-11-21T22:08:31.383923Z","iopub.status.idle":"2024-11-21T22:08:31.403015Z","shell.execute_reply.started":"2024-11-21T22:08:31.383887Z","shell.execute_reply":"2024-11-21T22:08:31.401822Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train[['sii', 'complete_resp_total']].head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:31.404339Z","iopub.execute_input":"2024-11-21T22:08:31.404680Z","iopub.status.idle":"2024-11-21T22:08:31.419657Z","shell.execute_reply.started":"2024-11-21T22:08:31.404646Z","shell.execute_reply":"2024-11-21T22:08:31.418421Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sii_counts = train['sii'].value_counts().reset_index()\ntotal = sii_counts['count'].sum()\nsii_counts['percentage'] = (sii_counts['count'] / total) * 100\n\nfig, axes = plt.subplots(1, 2, figsize=(14, 7))\n\n# SII\nsns.barplot(x='sii', y='count', data=sii_counts, palette='Blues_d', ax=axes[0])\naxes[0].set_title('Distribution of Severity Impairment Index (sii)', fontsize=14)\nfor p in axes[0].patches:\n height = p.get_height()\n percentage = sii_counts.loc[sii_counts['count'] == height, 'percentage'].values[0]\n axes[0].text(\n p.get_x() + p.get_width() / 2,\n height + 5, f'{int(height)} ({percentage:.1f}%)',\n ha=\"center\", fontsize=12\n )\n\n# PCIAT_Total for complete responses\nsns.histplot(train['complete_resp_total'].dropna(), bins=40, ax=axes[1])\naxes[1].set_title('Distribution of PCIAT_Total', fontsize=14)\naxes[1].set_xlabel('PCIAT_Total for Complete PCIAT Responses')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:31.421125Z","iopub.execute_input":"2024-11-21T22:08:31.421498Z","iopub.status.idle":"2024-11-21T22:08:32.120230Z","shell.execute_reply.started":"2024-11-21T22:08:31.421465Z","shell.execute_reply":"2024-11-21T22:08:32.118968Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"=>> 40% số người tham gia không bị ảnh hưởng bởi việc sử dụng Internet, 31% không được đánh giá và chỉ thiểu số (~10%) bị suy giảm ở mức độ từ 2 đến 3.","metadata":{}},{"cell_type":"markdown","source":"##
Age and Sex
","metadata":{}},{"cell_type":"code","source":"train['Basic_Demos-Age'].isna().sum() ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:32.121851Z","iopub.execute_input":"2024-11-21T22:08:32.122305Z","iopub.status.idle":"2024-11-21T22:08:32.132100Z","shell.execute_reply.started":"2024-11-21T22:08:32.122256Z","shell.execute_reply":"2024-11-21T22:08:32.130750Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['Basic_Demos-Sex'].isna().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:32.133688Z","iopub.execute_input":"2024-11-21T22:08:32.134102Z","iopub.status.idle":"2024-11-21T22:08:32.145565Z","shell.execute_reply.started":"2024-11-21T22:08:32.134053Z","shell.execute_reply":"2024-11-21T22:08:32.144225Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def calculate_stats(data, columns):\n if isinstance(columns, str):\n columns = [columns]\n\n stats = []\n for col in columns:\n if data[col].dtype in ['object', 'category']:\n counts = data[col].value_counts(dropna=False, sort=False)\n percents = data[col].value_counts(normalize=True, dropna=False, sort=False) * 100\n formatted = counts.astype(str) + ' (' + percents.round(2).astype(str) + '%)'\n stats_col = pd.DataFrame({'count (%)': formatted})\n stats.append(stats_col)\n else:\n stats_col = data[col].describe().to_frame().transpose()\n stats_col['missing'] = data[col].isnull().sum()\n stats_col.index.name = col\n stats.append(stats_col)\n\n return pd.concat(stats, axis=0)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:32.147129Z","iopub.execute_input":"2024-11-21T22:08:32.147568Z","iopub.status.idle":"2024-11-21T22:08:32.164433Z","shell.execute_reply.started":"2024-11-21T22:08:32.147531Z","shell.execute_reply":"2024-11-21T22:08:32.162961Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['Age Group'] = pd.cut(\n train['Basic_Demos-Age'],\n bins=[4, 12, 18, 22],\n labels=['Children (5-12)', 'Adolescents (13-18)', 'Adults (19-22)']\n)\ncalculate_stats(train, 'Age Group')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:32.166175Z","iopub.execute_input":"2024-11-21T22:08:32.167102Z","iopub.status.idle":"2024-11-21T22:08:32.190823Z","shell.execute_reply.started":"2024-11-21T22:08:32.167041Z","shell.execute_reply":"2024-11-21T22:08:32.189274Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, axes = plt.subplots(3, 1, figsize=(5, 10))\n\n# SII by Age\nsns.boxplot(y=train['Basic_Demos-Age'], x=train['sii'], ax=axes[0], palette=\"Set3\")\naxes[0].set_title('SII by Age')\naxes[0].set_ylabel('Age')\naxes[0].set_xlabel('SII')\n\n# Complete PCIAT Responses by Age Group\nsns.boxplot(\n x='Age Group', y='complete_resp_total',\n data=train, palette=\"Set3\", ax=axes[1]\n)\naxes[1].set_title('Complete PCIAT Responses by Age Group')\naxes[1].set_ylabel('PCIAT_Total for Complete Responses')\naxes[1].set_xlabel('Age Group')\n\n# PCIAT_Total by Sex\nsns.histplot(\n data=train, x='complete_resp_total',\n hue='Basic_Demos-Sex', multiple='stack',\n palette=\"Set3\", bins=20, ax=axes[2]\n)\naxes[2].set_title('PCIAT_Total Distribution by Sex')\naxes[2].set_xlabel('PCIAT_Total for Complete Responses')\naxes[2].set_ylabel('Frequency')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:32.192536Z","iopub.execute_input":"2024-11-21T22:08:32.193110Z","iopub.status.idle":"2024-11-21T22:08:33.050571Z","shell.execute_reply.started":"2024-11-21T22:08:32.193057Z","shell.execute_reply":"2024-11-21T22:08:33.049270Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"stats = train.groupby(['Age Group', 'sii']).size().unstack(fill_value=0)\nfig, axes = plt.subplots(len(stats), 1, figsize=(5, 12))\n\nfor i, age_group in enumerate(stats.index):\n group_counts = stats.loc[age_group] / stats.loc[age_group].sum()\n axes[i].pie(\n group_counts, labels=group_counts.index, autopct='%1.1f%%',\n startangle=90, colors=sns.color_palette(\"Set3\"),\n labeldistance=1.05, pctdistance=0.80\n )\n axes[i].set_title(f'SII Distribution for {age_group}')\n axes[i].axis('equal')\n\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:33.051928Z","iopub.execute_input":"2024-11-21T22:08:33.052276Z","iopub.status.idle":"2024-11-21T22:08:33.558130Z","shell.execute_reply.started":"2024-11-21T22:08:33.052243Z","shell.execute_reply":"2024-11-21T22:08:33.556809Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"=> sự phân bổ SII cho trẻ em và người lớn nghiêng về các giá trị thấp hơn (không có và nhẹ), trong khi đó, đối với thanh thiếu niên, sự phân bổ cân bằng hơn giữa các loại không có, nhẹ và trung bình.","metadata":{}},{"cell_type":"code","source":"stats = train.groupby(['Age Group', 'sii']).size().unstack(fill_value=0)\nstats_prop = stats.div(stats.sum(axis=1), axis=0) * 100\n\nstats = stats.astype(str) +' (' + stats_prop.round(1).astype(str) + '%)'\nstats","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:33.566240Z","iopub.execute_input":"2024-11-21T22:08:33.566733Z","iopub.status.idle":"2024-11-21T22:08:33.590950Z","shell.execute_reply.started":"2024-11-21T22:08:33.566607Z","shell.execute_reply":"2024-11-21T22:08:33.589727Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"stats = train[train['sii'] != 'Missing'].groupby(\n ['Age Group', 'sii']\n).size().unstack(fill_value=0)\nstats_prop = stats.div(stats.sum(axis=1), axis=0) * 100\n\nstats = stats.astype(str) +' (' + stats_prop.round(1).astype(str) + '%)'\nstats","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:33.592242Z","iopub.execute_input":"2024-11-21T22:08:33.592591Z","iopub.status.idle":"2024-11-21T22:08:33.617088Z","shell.execute_reply.started":"2024-11-21T22:08:33.592559Z","shell.execute_reply":"2024-11-21T22:08:33.615956Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"=> Số lượng thanh thiếu niên thấp hơn nhiều so với trẻ em và số lượng người lớn tham gia cũng cực kỳ thấp (tổng cộng 88 người và chỉ có 36 người mắc SII). Vì vậy, có thể có những mối quan hệ mà ta không thể nhìn thấy với cỡ mẫu không đồng đều như vậy và sự thiếu đại diện của các trường hợp nghiêm trọng.","metadata":{}},{"cell_type":"markdown","source":"##
Internet Use
","metadata":{}},{"cell_type":"code","source":"data = train[train['PreInt_EduHx-computerinternet_hoursday'].notna()]\nage_range = data['Basic_Demos-Age']\nprint(\n f\"Age range for participants with measured PreInt_EduHx-computerinternet_hoursday data:\"\n f\" {age_range.min()} - {age_range.max()} years\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:33.618408Z","iopub.execute_input":"2024-11-21T22:08:33.618765Z","iopub.status.idle":"2024-11-21T22:08:33.627308Z","shell.execute_reply.started":"2024-11-21T22:08:33.618731Z","shell.execute_reply":"2024-11-21T22:08:33.626035Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['PreInt_EduHx-computerinternet_hoursday'].unique()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:33.628924Z","iopub.execute_input":"2024-11-21T22:08:33.629413Z","iopub.status.idle":"2024-11-21T22:08:33.646894Z","shell.execute_reply.started":"2024-11-21T22:08:33.629363Z","shell.execute_reply":"2024-11-21T22:08:33.645747Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"param_map = {0: '< 1h/day', 1: '~ 1h/day', 2: '~ 2hs/day', 3: '> 3hs/day'}\ntrain['internet_use_encoded'] = train['PreInt_EduHx-computerinternet_hoursday'].map(param_map).fillna('Missing')\n\nparam_ord = ['Missing', '< 1h/day', '~ 1h/day', '~ 2hs/day', '> 3hs/day']\ntrain['internet_use_encoded'] = pd.Categorical(\n train['internet_use_encoded'], categories=param_ord,\n ordered=True\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:33.648352Z","iopub.execute_input":"2024-11-21T22:08:33.648770Z","iopub.status.idle":"2024-11-21T22:08:33.670325Z","shell.execute_reply.started":"2024-11-21T22:08:33.648733Z","shell.execute_reply":"2024-11-21T22:08:33.668973Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['internet_use_encoded']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:33.671988Z","iopub.execute_input":"2024-11-21T22:08:33.672470Z","iopub.status.idle":"2024-11-21T22:08:33.693020Z","shell.execute_reply.started":"2024-11-21T22:08:33.672420Z","shell.execute_reply":"2024-11-21T22:08:33.691888Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"calculate_stats(train, 'PreInt_EduHx-Season')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:33.694965Z","iopub.execute_input":"2024-11-21T22:08:33.695331Z","iopub.status.idle":"2024-11-21T22:08:33.714538Z","shell.execute_reply.started":"2024-11-21T22:08:33.695297Z","shell.execute_reply":"2024-11-21T22:08:33.713141Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, axes = plt.subplots(3, 1, figsize=(5, 14))\n\n# Hours of Internet Use\nax1 = sns.countplot(x='internet_use_encoded', data=train, palette=\"Set3\", ax=axes[0])\naxes[0].set_title('Distribution of Hours of Internet Use')\naxes[0].set_xlabel('Hours per Day Group')\naxes[0].set_ylabel('Count')\n\ntotal = len(train['internet_use_encoded'])\nfor p in ax1.patches:\n count = int(p.get_height())\n percentage = '{:.1f}%'.format(100 * count / total)\n ax1.annotate(f'{count} ({percentage})', (p.get_x() + p.get_width() / 2., p.get_height()), \n ha='center', va='baseline', fontsize=10, color='black', xytext=(0, 5), \n textcoords='offset points')\n\n# Hours of Internet Use by Age\nsns.boxplot(y=train['Basic_Demos-Age'], x=train['internet_use_encoded'], ax=axes[1], palette=\"Set3\")\naxes[1].set_title('Hours of Internet Use by Age')\naxes[1].set_ylabel('Age')\naxes[1].set_xlabel('Hours per Day Group')\n\n# Hours of Internet Use (numeric) by Age Group\nsns.boxplot(y='PreInt_EduHx-computerinternet_hoursday', x='Age Group', data=train, ax=axes[2], palette=\"Set3\")\naxes[2].set_title('Internet Hours by Age Group')\naxes[2].set_ylabel('Hours per Day (Numeric)')\naxes[2].set_xlabel('Age Group')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:33.716290Z","iopub.execute_input":"2024-11-21T22:08:33.716815Z","iopub.status.idle":"2024-11-21T22:08:34.521253Z","shell.execute_reply.started":"2024-11-21T22:08:33.716743Z","shell.execute_reply":"2024-11-21T22:08:34.519768Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"stats = train.groupby(\n ['Age Group', 'internet_use_encoded']\n).size().unstack(fill_value=0)\nfig, axes = plt.subplots(len(stats),1, figsize=(5, 14))\n\nfor i, age_group in enumerate(stats.index):\n group_counts = stats.loc[age_group] / stats.loc[age_group].sum()\n axes[i].pie(group_counts, labels=group_counts.index, autopct='%1.1f%%',\n startangle=90, colors=sns.color_palette(\"Set3\"), labeldistance=1.1)\n axes[i].set_title(f'Distribution of Hours of Internet Use\\n{age_group}')\n axes[i].axis('equal')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:34.523232Z","iopub.execute_input":"2024-11-21T22:08:34.523746Z","iopub.status.idle":"2024-11-21T22:08:35.054726Z","shell.execute_reply.started":"2024-11-21T22:08:34.523693Z","shell.execute_reply":"2024-11-21T22:08:35.053496Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"stats = train.groupby(['Basic_Demos-Sex', 'internet_use_encoded']\n).size().unstack(fill_value=0)\nstats_prop = stats.div(stats.sum(axis=1), axis=0) * 100\n\nstats = stats.astype(str) +' (' + stats_prop.round(1).astype(str) + '%)'\nstats","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:35.056357Z","iopub.execute_input":"2024-11-21T22:08:35.056765Z","iopub.status.idle":"2024-11-21T22:08:35.082352Z","shell.execute_reply.started":"2024-11-21T22:08:35.056728Z","shell.execute_reply":"2024-11-21T22:08:35.080636Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- Dữ liệu sử dụng Internet bị thiếu đối với 16,6% số người tham gia, trong khi 38,5% cho biết sử dụng Internet ít hơn một giờ mỗi ngày\n- Tương tự như dữ liệu SII, biểu đồ hình hộp cho thấy mức sử dụng Internet hàng ngày cao hơn có liên quan đến độ tuổi lớn hơn, với sự chồng chéo đáng kể về độ tuổi trong mỗi danh mục sử dụng Internet. Nhưng ở đây cả cách biểu diễn theo phân loại và số học về số giờ dành cho trực tuyến đều cho thấy mối quan hệ tuyến tính nhất quán.\n- Biểu đồ hình tròn cho các nhóm tuổi được căn chỉnh tốt và hiển thị giống nhau.Việc tạo ra một tính năng tương tác giữa việc sử dụng internet và độ tuổi có thể hữu ích cho việc lập mô hình.\n- Việc sử dụng Internet khá giống nhau đối với cả hai giới.","metadata":{}},{"cell_type":"markdown","source":"###
Internet Use with SII
","metadata":{}},{"cell_type":"code","source":"sii_reported = train[train['sii'] != \"Missing\"]\nsii_reported.loc[:, 'sii'] = sii_reported['sii'].cat.remove_unused_categories()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:35.084651Z","iopub.execute_input":"2024-11-21T22:08:35.085046Z","iopub.status.idle":"2024-11-21T22:08:35.097466Z","shell.execute_reply.started":"2024-11-21T22:08:35.085010Z","shell.execute_reply":"2024-11-21T22:08:35.096095Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"stats = sii_reported.groupby(\n ['internet_use_encoded', 'sii']\n).size().unstack(fill_value=0)\nstats_prop = stats.div(stats.sum(axis=1), axis=0) * 100\n\nstats = stats.astype(str) +' (' + stats_prop.round(1).astype(str) + '%)'\nstats","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:35.099112Z","iopub.execute_input":"2024-11-21T22:08:35.099541Z","iopub.status.idle":"2024-11-21T22:08:35.132968Z","shell.execute_reply.started":"2024-11-21T22:08:35.099481Z","shell.execute_reply":"2024-11-21T22:08:35.131583Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig = plt.figure(figsize=(12, 10))\ngs = fig.add_gridspec(2, 1, height_ratios=[1, 1.5])\n\n# SII vs Hours of Internet Use\nax1 = fig.add_subplot(gs[0, 0])\nsns.boxplot(\n x='sii', y='PreInt_EduHx-computerinternet_hoursday',\n data=sii_reported,\n ax=ax1, palette=\"Set3\"\n)\nax1.set_title('SII vs Hours of Internet Use')\nax1.set_ylabel('Hours per Day')\nax1.set_xlabel('SII')\n\n\n# SII vs Hours of Internet Use by Age Group (Full width)\nax3 = fig.add_subplot(gs[1, :])\nsns.boxplot(\n x='internet_use_encoded', y='complete_resp_total',\n data=sii_reported,\n hue='Age Group', ax=ax3, palette=\"Set3\"\n)\nax3.set_title('PCIAT_Total vs Hours of Internet Use by Age Group')\nax3.set_ylabel('PCIAT_Total for Complete PCIAT Responses')\nax3.set_xlabel('Hours per Day Group')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:35.134907Z","iopub.execute_input":"2024-11-21T22:08:35.135400Z","iopub.status.idle":"2024-11-21T22:08:36.217112Z","shell.execute_reply.started":"2024-11-21T22:08:35.135351Z","shell.execute_reply":"2024-11-21T22:08:36.215890Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"stats = sii_reported.groupby(\n ['sii', 'internet_use_encoded']\n).size().unstack(fill_value=0)\nfig, axes = plt.subplots(1, len(stats), figsize=(18, 5))\n\nfor i, sii_group in enumerate(stats.index):\n group_counts = stats.loc[sii_group] / stats.loc[sii_group].sum()\n axes[i].pie(\n group_counts, labels=group_counts.index, autopct='%1.1f%%',\n startangle=90, colors=sns.color_palette(\"Set3\"), labeldistance=1.1\n )\n axes[i].set_title(f'Hours of using computer/internet\\n for SII = {sii_group}')\n axes[i].axis('equal')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:36.218603Z","iopub.execute_input":"2024-11-21T22:08:36.218985Z","iopub.status.idle":"2024-11-21T22:08:36.813483Z","shell.execute_reply.started":"2024-11-21T22:08:36.218948Z","shell.execute_reply":"2024-11-21T22:08:36.812307Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"stats = sii_reported.groupby(\n ['sii', 'internet_use_encoded']\n).size().unstack(fill_value=0)\nstats_prop = stats.div(stats.sum(axis=1), axis=0) * 100\n\nstats = stats.astype(str) +' (' + stats_prop.round(1).astype(str) + '%)'\nstats","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:36.815000Z","iopub.execute_input":"2024-11-21T22:08:36.815340Z","iopub.status.idle":"2024-11-21T22:08:36.839090Z","shell.execute_reply.started":"2024-11-21T22:08:36.815306Z","shell.execute_reply":"2024-11-21T22:08:36.837722Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- Trong biểu đồ hình hộp, mặc dù có sự trùng lặp đáng kể giữa các danh mục sử dụng Internet và SII khác nhau, chúng tôi thấy xu hướng tích cực giữa suy giảm PIU và việc sử dụng Internet, với những người có điểm SII cao hơn dành nhiều thời gian trực tuyến hơn (sẽ thật lạ nếu điều này không xảy ra).\n\n- Biểu đồ hình tròn cũng cho thấy có một tỷ lệ đáng kể người tham gia (tổng cộng 83 người), ở mọi lứa tuổi, dành rất ít thời gian trực tuyến (dưới 1 giờ mỗi ngày) nhưng có điểm SII cao (20,7% với SII 2 - vừa phải suy giảm và 14,7% có SII = 3 - suy giảm nghiêm trọng).","metadata":{}},{"cell_type":"markdown","source":"##
Demographics
","metadata":{}},{"cell_type":"markdown","source":"##
Physical Health and Fitness (objective measurements)
","metadata":{}},{"cell_type":"markdown","source":"##
Behavioral (subjective reported)
","metadata":{}},{"cell_type":"markdown","source":"#
Feature Selection
","metadata":{}},{"cell_type":"code","source":"featuresCols = ['Basic_Demos-Age', 'Basic_Demos-Sex',\n 'CGAS-CGAS_Score', 'Physical-BMI',\n 'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n 'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n 'Fitness_Endurance-Max_Stage',\n 'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n 'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n 'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n 'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone',\n 'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n 'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n 'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n 'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n 'BIA-BIA_TBW', 'PAQ_A-PAQ_A_Total',\n 'PAQ_C-PAQ_C_Total', 'SDS-SDS_Total_Raw',\n 'SDS-SDS_Total_T',\n 'PreInt_EduHx-computerinternet_hoursday', 'sii']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:36.841107Z","iopub.execute_input":"2024-11-21T22:08:36.841454Z","iopub.status.idle":"2024-11-21T22:08:36.847601Z","shell.execute_reply.started":"2024-11-21T22:08:36.841422Z","shell.execute_reply":"2024-11-21T22:08:36.846366Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#
Model Training and Evaluation
","metadata":{}},{"cell_type":"markdown","source":"\r- **Model**\n - **NN**\n - **LightGBM**\n - **XGBoost**\n - **CatBoost**\n - **Voting Regressor**\n- **Cross-Validation**: K-Fold phân tầng được sử dụng để phân chia dữ liệu thành các tập huấn luyện và xác thực, đảm bảo phân bổ lớp cân bằng trong mỗi lần.\n- **Quadratic Weighted Kappa (QWK)**: Hiệu suất của các mô hình được đánh giá bằng cách sử dụng QWK, đo lường sự phù hợp giữa giá trị dự đoán và giá trị thực tế, có tính đến tính chất thứ tự của biến mục tiêu.\n- **Threshold Optimization**: Hàm `minimize` từ `scipy.optimize` được sử dụng để tinh chỉnh các ngưỡng quyết định ánh xạ các dự đoán liên tục thành các danh mục riêng biệt (None, Mild, Moderate, Severe).\nSevere).","metadata":{}},{"cell_type":"markdown","source":"##
NN
","metadata":{}},{"cell_type":"code","source":"DATA_DIR = \"../input/child-mind-institute-problematic-internet-use/\"\n\ntrain = pl.read_csv(DATA_DIR + \"train.csv\")\ntest = pl.read_csv(DATA_DIR + \"test.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:36.849399Z","iopub.execute_input":"2024-11-21T22:08:36.849916Z","iopub.status.idle":"2024-11-21T22:08:36.889997Z","shell.execute_reply.started":"2024-11-21T22:08:36.849864Z","shell.execute_reply":"2024-11-21T22:08:36.888637Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"TARGET_COLS = [\"PCIAT-PCIAT_01\", \"PCIAT-PCIAT_02\", \"PCIAT-PCIAT_03\", \"PCIAT-PCIAT_04\", \"PCIAT-PCIAT_05\",\n \"PCIAT-PCIAT_06\", \"PCIAT-PCIAT_07\", \"PCIAT-PCIAT_08\", \"PCIAT-PCIAT_09\", \"PCIAT-PCIAT_10\", \"PCIAT-PCIAT_11\",\n \"PCIAT-PCIAT_12\", \"PCIAT-PCIAT_13\", \"PCIAT-PCIAT_14\", \"PCIAT-PCIAT_15\", \"PCIAT-PCIAT_16\", \"PCIAT-PCIAT_17\",\n \"PCIAT-PCIAT_18\", \"PCIAT-PCIAT_19\", \"PCIAT-PCIAT_20\", \"PCIAT-PCIAT_Total\", \"sii\"]\n\nFEATURE_COLS = [\"Basic_Demos-Enroll_Season\", \"Basic_Demos-Age\", \"Basic_Demos-Sex\", \"CGAS-Season\", \"CGAS-CGAS_Score\",\n \"Physical-Season\", \"Physical-BMI\", \"Physical-Height\", \"Physical-Weight\", \"Physical-Waist_Circumference\",\n \"Physical-Diastolic_BP\", \"Physical-HeartRate\", \"Physical-Systolic_BP\", \"Fitness_Endurance-Season\",\n \"Fitness_Endurance-Max_Stage\", \"Fitness_Endurance-Time_Mins\", \"Fitness_Endurance-Time_Sec\", \"FGC-Season\",\n \"FGC-FGC_CU\", \"FGC-FGC_CU_Zone\", \"FGC-FGC_GSND\", \"FGC-FGC_GSND_Zone\", \"FGC-FGC_GSD\", \"FGC-FGC_GSD_Zone\",\n \"FGC-FGC_PU\", \"FGC-FGC_PU_Zone\", \"FGC-FGC_SRL\", \"FGC-FGC_SRL_Zone\", \"FGC-FGC_SRR\", \"FGC-FGC_SRR_Zone\", \"FGC-FGC_TL\",\n \"FGC-FGC_TL_Zone\", \"BIA-Season\", \"BIA-BIA_Activity_Level_num\", \"BIA-BIA_BMC\", \"BIA-BIA_BMI\", \"BIA-BIA_BMR\",\n \"BIA-BIA_DEE\", \"BIA-BIA_ECW\", \"BIA-BIA_FFM\", \"BIA-BIA_FFMI\", \"BIA-BIA_FMI\", \"BIA-BIA_Fat\", \"BIA-BIA_Frame_num\",\n \"BIA-BIA_ICW\", \"BIA-BIA_LDM\", \"BIA-BIA_LST\", \"BIA-BIA_SMM\", \"BIA-BIA_TBW\", \"PAQ_A-Season\", \"PAQ_A-PAQ_A_Total\",\n \"PAQ_C-Season\", \"PAQ_C-PAQ_C_Total\", \"SDS-Season\", \"SDS-SDS_Total_Raw\", \"SDS-SDS_Total_T\", \"PreInt_EduHx-Season\",\n \"PreInt_EduHx-computerinternet_hoursday\"]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:36.891961Z","iopub.execute_input":"2024-11-21T22:08:36.892450Z","iopub.status.idle":"2024-11-21T22:08:36.900401Z","shell.execute_reply.started":"2024-11-21T22:08:36.892400Z","shell.execute_reply":"2024-11-21T22:08:36.899156Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Preprocessing","metadata":{}},{"cell_type":"code","source":"import polars.selectors as cs\n\ntrain_test = pl.concat([train, test], how=\"diagonal\")\n\nIS_TEST = test.height <= 100\n\ntrain_test = train_test.with_columns(cs.string().cast(pl.Categorical).fill_null(\"NAN\"))\n\ntrain = train_test[:train.height]\ntest = train_test[train.height:]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:36.902002Z","iopub.execute_input":"2024-11-21T22:08:36.902492Z","iopub.status.idle":"2024-11-21T22:08:36.928946Z","shell.execute_reply.started":"2024-11-21T22:08:36.902440Z","shell.execute_reply":"2024-11-21T22:08:36.927154Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Feature and Target Selection","metadata":{}},{"cell_type":"code","source":"train_without_null = train.drop_nulls(subset=TARGET_COLS)\n\nX = train_without_null.select(FEATURE_COLS)\nX_test = test.select(FEATURE_COLS)\ny = train_without_null.select(TARGET_COLS)\n\nX = X.to_pandas()\nX_test = X_test.to_pandas()\ny = y.to_pandas()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:36.930012Z","iopub.execute_input":"2024-11-21T22:08:36.930343Z","iopub.status.idle":"2024-11-21T22:08:36.952847Z","shell.execute_reply.started":"2024-11-21T22:08:36.930307Z","shell.execute_reply":"2024-11-21T22:08:36.951659Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Handling Missing Values","metadata":{}},{"cell_type":"code","source":"num_features = X.select_dtypes(include=[np.number]).columns.tolist()\nX[num_features] = X[num_features].fillna(X[num_features].mean())\nX_test[num_features] = X_test[num_features].fillna(X[num_features].mean())\n\ncat_features = [col for col in FEATURE_COLS if col not in num_features]\nfor col in cat_features:\n X[col] = X[col].astype('category')\n X_test[col] = X_test[col].astype('category')\n\n if \"NAN\" not in X[col].cat.categories:\n X[col] = X[col].cat.add_categories(\"NAN\")\n if \"NAN\" not in X_test[col].cat.categories:\n X_test[col] = X_test[col].cat.add_categories(\"NAN\")\n\n X[col] = X[col].fillna(\"NAN\")\n X_test[col] = X_test[col].fillna(\"NAN\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:36.954370Z","iopub.execute_input":"2024-11-21T22:08:36.954837Z","iopub.status.idle":"2024-11-21T22:08:37.028086Z","shell.execute_reply.started":"2024-11-21T22:08:36.954764Z","shell.execute_reply":"2024-11-21T22:08:37.026877Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Encoding Categorical Variables","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder, StandardScaler\n\nlabel_encoders = {}\nfor col in cat_features:\n le = LabelEncoder()\n le.fit(pd.concat([X[col], X_test[col]], axis=0).astype(str))\n X[col] = le.transform(X[col].astype(str))\n X_test[col] = le.transform(X_test[col].astype(str))\n label_encoders[col] = le","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:37.029469Z","iopub.execute_input":"2024-11-21T22:08:37.029832Z","iopub.status.idle":"2024-11-21T22:08:37.074846Z","shell.execute_reply.started":"2024-11-21T22:08:37.029776Z","shell.execute_reply":"2024-11-21T22:08:37.073705Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"scaler = StandardScaler()\nscaler.fit(pd.concat([X[num_features], X_test[num_features]], axis=0))\nX[num_features] = scaler.transform(X[num_features])\nX_test[num_features] = scaler.transform(X_test[num_features])\n\nX_np = X.values\nX_test_np = X_test.values\ny_np = y.values","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:37.076523Z","iopub.execute_input":"2024-11-21T22:08:37.077094Z","iopub.status.idle":"2024-11-21T22:08:37.114072Z","shell.execute_reply.started":"2024-11-21T22:08:37.077041Z","shell.execute_reply":"2024-11-21T22:08:37.112904Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Optimized Rounder Class","metadata":{}},{"cell_type":"code","source":"# class OptimizedRounder:\n# \"\"\"\n# Optimizes rounding of continuous predictions into discrete class labels by maximizing\n# the Quadratic Weighted Kappa (QWK) score using Optuna optimization.\n# \"\"\"\n\n# def __init__(self, n_classes: int, n_trials: int = 100):\n# \"\"\"\n# Initializes the OptimizedRounder.\n\n# Args:\n# n_classes (int): The number of discrete class labels to predict.\n# n_trials (int): The number of optimization trials to run. Defaults to 100.\n# \"\"\"\n# self.n_classes = n_classes\n# self.labels = np.arange(n_classes)\n# self.n_trials = n_trials\n# self.metric = partial(cohen_kappa_score, weights=\"quadratic\")\n\n# def fit(self, y_pred: NDArray[np.float_], y_true: NDArray[np.int_]) -> None:\n# \"\"\"\n# Optimizes thresholds that round continuous predictions to the nearest class.\n\n# Args:\n# y_pred (NDArray[np.float_]): Continuous predictions from the model.\n# y_true (NDArray[np.int_]): True target labels for comparison.\n# \"\"\"\n# y_pred = self._normalize(y_pred)\n\n# def objective(trial: optuna.Trial) -> float:\n# thresholds = []\n# for i in range(self.n_classes - 1):\n# low = thresholds[-1] if i > 0 else y_pred.min()\n# high = y_pred.max()\n# th = trial.suggest_float(f\"threshold_{i}\", low, high)\n# thresholds.append(th)\n# try:\n# y_pred_rounded = np.digitize(y_pred, thresholds)\n# except ValueError:\n# return -100\n# return self.metric(y_true, y_pred_rounded)\n\n# optuna.logging.disable_default_handler()\n# study = optuna.create_study(direction=\"maximize\")\n# study.optimize(objective, n_trials=self.n_trials)\n\n# self.thresholds = [study.best_params[f\"threshold_{i}\"] for i in range(self.n_classes - 1)]\n\n# def predict(self, y_pred: NDArray[np.float_]) -> NDArray[np.int_]:\n# \"\"\"\n# Rounds continuous predictions using the optimized thresholds.\n\n# Args:\n# y_pred (NDArray[np.float_]): Continuous predictions.\n\n# Returns:\n# NDArray[np.int_]: Rounded class labels.\n# \"\"\"\n# assert hasattr(self, \"thresholds\"), \"You must call fit() before predict()\"\n# y_pred = self._normalize(y_pred)\n# return np.digitize(y_pred, self.thresholds)\n\n# def _normalize(self, y: NDArray[np.float_]) -> NDArray[np.float_]:\n# \"\"\"\n# Normalizes predictions to a range [0, n_classes - 1].\n\n# Args:\n# y (NDArray[np.float_]): Predictions to normalize.\n\n# Returns:\n# NDArray[np.float_]: Normalized predictions.\n# \"\"\"\n# return (y - y.min()) / (y.max() - y.min()) * (self.n_classes - 1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:37.115449Z","iopub.execute_input":"2024-11-21T22:08:37.116397Z","iopub.status.idle":"2024-11-21T22:08:37.181368Z","shell.execute_reply.started":"2024-11-21T22:08:37.116340Z","shell.execute_reply":"2024-11-21T22:08:37.179812Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Neural network","metadata":{}},{"cell_type":"code","source":"# class MultiTargetRegressionModel(nn.Module):\n# \"\"\"\n# Neural network model for multi-target regression using embeddings for categorical features\n# and fully connected layers for regression.\n# \"\"\"\n\n# def __init__(self, embedding_sizes, num_numerical_features, output_size):\n# \"\"\"\n# Initializes the model architecture.\n\n# Args:\n# embedding_sizes: List of tuples containing the number of categories and embedding sizes for each categorical feature.\n# num_numerical_features: The number of numerical input features.\n# output_size: The number of target variables to predict.\n# \"\"\"\n# super(MultiTargetRegressionModel, self).__init__()\n# self.embeddings = nn.ModuleList(\n# [nn.Embedding(num_embeddings=categories, embedding_dim=size) for categories, size in embedding_sizes])\n# self.embedding_output_size = sum([size for _, size in embedding_sizes])\n# self.fc1 = nn.Linear(self.embedding_output_size + num_numerical_features, 128)\n# self.fc2 = nn.Linear(128, 64)\n# self.output = nn.Linear(64, output_size)\n# self.dropout = nn.Dropout(0.1)\n# self.relu = nn.ReLU()\n\n# def forward(self, x_categorical, x_numerical):\n# \"\"\"\n# Forward pass of the model.\n\n# Args:\n# x_categorical: Categorical input features.\n# x_numerical: Numerical input features.\n\n# Returns:\n# Predicted values for all targets.\n# \"\"\"\n# x = [emb(x_categorical[:, i]) for i, emb in enumerate(self.embeddings)]\n# x = torch.cat(x, 1)\n# x = torch.cat([x, x_numerical], 1)\n# x = self.dropout(self.relu(self.fc1(x)))\n# x = self.dropout(self.relu(self.fc2(x)))\n# x = self.output(x)\n# return x","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T22:08:37.182395Z","iopub.status.idle":"2024-11-21T22:08:37.182881Z","shell.execute_reply.started":"2024-11-21T22:08:37.182610Z","shell.execute_reply":"2024-11-21T22:08:37.182665Z"}},"outputs":[],"execution_count":null}]}