{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<div\r\n      style=\"\r\n        text-align: center;\r\n        font-weight: bold;\r\n        font-size: 32px;\r\n        font-family: Arial, Helvetica, sans-serif;\r\n        color: white;\r\n        background-color: rgb(84, 84, 84);\r\n        padding-top: 20px;\r\n        padding-bottom: 20px;\r\n        border-radius: 20px;\r\n      \"\r\n    >\r\n      CMI | Problematic Internet Use\r\n    </div>","metadata":{}},{"cell_type":"markdown","source":"# I. Import Library","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport optuna\nfrom concurrent.futures import ThreadPoolExecutor\nfrom tqdm import tqdm\nfrom scipy.optimize import minimize\nfrom IPython.display import clear_output\nimport concurrent.futures\nimport warnings\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom matplotlib import colors\nfrom colorama import Style, Fore\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\n\nfrom sklearn.base import clone\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.pipeline import Pipeline\n\nfrom sklearn.ensemble import RandomForestRegressor\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom lightgbm import LGBMRegressor\nfrom sklearn.ensemble import GradientBoostingRegressor\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.ensemble import VotingRegressor\n\n\nN_FOLD = 5\nSEED = 42\n\noptuna.logging.set_verbosity(optuna.logging.WARNING)\nwarnings.filterwarnings('ignore')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:29:20.879946Z","iopub.execute_input":"2024-12-19T15:29:20.880300Z","iopub.status.idle":"2024-12-19T15:29:29.275302Z","shell.execute_reply.started":"2024-12-19T15:29:20.880273Z","shell.execute_reply":"2024-12-19T15:29:29.274235Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# II. `csv` Data Exploration","metadata":{}},{"cell_type":"markdown","source":"## 1. Load & Preview Data","metadata":{}},{"cell_type":"code","source":"train_data = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv', index_col='id')\ntest_data = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv', index_col='id')\ndata_dict = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/data_dictionary.csv')\n\n\ntrain_data_col = train_data.columns.tolist()\ntest_data_col = test_data.columns.tolist()\nprint('Train shape and Test shape:')\ntrain_data.shape, test_data.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:29:29.276565Z","iopub.execute_input":"2024-12-19T15:29:29.277254Z","iopub.status.idle":"2024-12-19T15:29:29.379378Z","shell.execute_reply.started":"2024-12-19T15:29:29.277226Z","shell.execute_reply":"2024-12-19T15:29:29.378331Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:29:29.381277Z","iopub.execute_input":"2024-12-19T15:29:29.381590Z","iopub.status.idle":"2024-12-19T15:29:29.416033Z","shell.execute_reply.started":"2024-12-19T15:29:29.381565Z","shell.execute_reply":"2024-12-19T15:29:29.415010Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_data.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:29:29.417442Z","iopub.execute_input":"2024-12-19T15:29:29.417715Z","iopub.status.idle":"2024-12-19T15:29:29.440356Z","shell.execute_reply.started":"2024-12-19T15:29:29.417692Z","shell.execute_reply":"2024-12-19T15:29:29.439429Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_dict.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:29:29.441287Z","iopub.execute_input":"2024-12-19T15:29:29.441600Z","iopub.status.idle":"2024-12-19T15:29:29.462760Z","shell.execute_reply.started":"2024-12-19T15:29:29.441575Z","shell.execute_reply":"2024-12-19T15:29:29.461720Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2. Missing values in train & test dataset","metadata":{}},{"cell_type":"code","source":"print('Duplicates in dtrain:', train_data.duplicated().sum())\nprint('Duplicates in dtest:', test_data.duplicated().sum())\n\nprint('Percentage of null values in each column in train dataset')\n\nprint(train_data.isnull().mean() * 100)\n\nmissing_percentage = train_data.isnull().mean() * 100\n\n\nplt.figure(figsize=(19, 6))\nsns.barplot(x=missing_percentage.index, y=missing_percentage.values, palette='viridis')\nplt.xticks(rotation=45, ha='right')\nplt.xlabel('Columns')\nplt.ylabel('Percentage of Missing Values')\nplt.title('Percentage of Missing Values in Each Column')\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:29:29.463927Z","iopub.execute_input":"2024-12-19T15:29:29.464206Z","iopub.status.idle":"2024-12-19T15:29:30.571022Z","shell.execute_reply.started":"2024-12-19T15:29:29.464185Z","shell.execute_reply":"2024-12-19T15:29:30.569955Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print('Percentage of null values in each column in test dataset')\ntest_data.isnull().mean() * 100\n\nmissing_percentage = test_data.isnull().mean() * 100\n\n\nplt.figure(figsize=(19, 6))\nsns.barplot(x=missing_percentage.index, y=missing_percentage.values, palette='viridis')\nplt.xticks(rotation=45, ha='right')\nplt.xlabel('Columns')\nplt.ylabel('Percentage of Missing Values')\nplt.title('Percentage of Missing Values in Each Column')\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:29:30.572031Z","iopub.execute_input":"2024-12-19T15:29:30.572309Z","iopub.status.idle":"2024-12-19T15:29:31.379107Z","shell.execute_reply.started":"2024-12-19T15:29:30.572284Z","shell.execute_reply":"2024-12-19T15:29:31.378045Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"As we can see:\n- there are duplicates data in train dataset\n- on both train and test data, columns revelant to `Season` have high missing percentage, so we can drop these columns.","metadata":{}},{"cell_type":"markdown","source":"## 3. Train & Test differences\nLet's see what columns in train dataset but not in test dataset","metadata":{}},{"cell_type":"code","source":"columns_not_in_test = sorted(list(set(train_data_col) - set(test_data_col)))\ndata_dict[data_dict['Field'].isin(columns_not_in_test)]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:29:31.380267Z","iopub.execute_input":"2024-12-19T15:29:31.380693Z","iopub.status.idle":"2024-12-19T15:29:31.397473Z","shell.execute_reply.started":"2024-12-19T15:29:31.380656Z","shell.execute_reply":"2024-12-19T15:29:31.396229Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"As the result, the test dataset includes all feature columns in the train dataset (except 22 PCIAT scores and sii score)","metadata":{}},{"cell_type":"markdown","source":"## 4. Statistics of some features","metadata":{}},{"cell_type":"code","source":"features = test_data_col.copy()\nprint('#Features :', len(features))\n\n# Numerical Features\nnum_features = [f for f in features if train_data[f].dtype==float or f=='Basic_Demos-Age']\nprint('#numerical features :', len(num_features))\n\n# Categorical Features\ncat_features = [f for f in features if f not in num_features]\nprint('#categorical features :', len(cat_features))\n\n# Target Features\ntarget_col = [f for f in train_data_col if f not in test_data_col]\nprint('#target features :', len(target_col), '\\n')\n\n# Unique Number\nprint('Unique #values in each set\\n')\n# pd.set_option('display.max_rows', 500)\npd.DataFrame(data= {'Unique number in train': train_data[features].nunique(), \n                    'Unique number in test': test_data[features].nunique()}).sort_values(by=['Unique number in train']) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:29:31.400391Z","iopub.execute_input":"2024-12-19T15:29:31.400688Z","iopub.status.idle":"2024-12-19T15:29:31.449011Z","shell.execute_reply.started":"2024-12-19T15:29:31.400661Z","shell.execute_reply":"2024-12-19T15:29:31.448123Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Features groups","metadata":{}},{"cell_type":"code","source":"groups = data_dict.groupby('Instrument')['Field'].apply(list).to_dict()\n\nfor instrument, features in groups.items():\n    print(f\"{instrument}: {features}\\n\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:29:31.450728Z","iopub.execute_input":"2024-12-19T15:29:31.451058Z","iopub.status.idle":"2024-12-19T15:29:31.459494Z","shell.execute_reply.started":"2024-12-19T15:29:31.451031Z","shell.execute_reply":"2024-12-19T15:29:31.458627Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 5. Distribution of sii and PCIAT scores","metadata":{}},{"cell_type":"code","source":"train = train_data.copy()\nsii_map = {0: '0 (None)', 1: '1 (Mild)', 2: '2 (Moderate)', 3: '3 (Severe)'}\ntrain['sii'] = train['sii'].map(sii_map).fillna('Missing')\n\nsii_order = ['Missing', '0 (None)', '1 (Mild)', '2 (Moderate)', '3 (Severe)']\ntrain['sii'] = pd.Categorical(train['sii'], categories=sii_order, ordered=True)\n\nsii = train['sii'].value_counts().reset_index()\nsii_total = sii['count'].sum()\nsii['percentage'] = (sii['count'] / sii_total) * 100\n\nfig, axes = plt.subplots(1, 2, figsize=(14, 5))\n\n# SII\nsns.barplot(x='sii', y='count', data=sii, palette='viridis', ax=axes[0])\naxes[0].set_title('Distribution of Severity Impairment Index (sii)', fontsize=14)\nfor p in axes[0].patches:\n    height = p.get_height()\n    percentage = sii.loc[sii['count'] == height, 'percentage'].values[0]\n    axes[0].text(\n        p.get_x() + p.get_width() / 2,\n        height + 5, f'{int(height)} ({percentage:.1f}%)',\n        ha=\"center\", fontsize=12\n    )\n\n\nPCIAT_columns = [f'PCIAT-PCIAT_{i+1:02d}' for i in range(20)]\ntrain['complete_resp_total'] = train['PCIAT-PCIAT_Total'].where(\n    train[PCIAT_columns].notna().all(axis=1), np.nan\n)\n# PCIAT_Total for complete responses\nsns.histplot(train['complete_resp_total'].dropna(), bins=20, ax=axes[1])\naxes[1].set_title('Distribution of PCIAT_Total', fontsize=14)\naxes[1].set_xlabel('PCIAT_Total for Complete PCIAT Responses')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:29:31.460477Z","iopub.execute_input":"2024-12-19T15:29:31.460780Z","iopub.status.idle":"2024-12-19T15:29:31.926680Z","shell.execute_reply.started":"2024-12-19T15:29:31.460756Z","shell.execute_reply":"2024-12-19T15:29:31.925496Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 6. `sii` distribution for each Age group","metadata":{}},{"cell_type":"code","source":"train['Age Group'] = pd.cut(\n    train['Basic_Demos-Age'],\n    bins=[2, 5, 12, 18, 22],\n    labels=['Early Childhood (3-5)', 'Children (6-11)', 'Adolescents (12-18)', 'Adults (19-22)']\n)\n\nstats = train.groupby(['Age Group', 'sii']).size().unstack(fill_value=0)\nfig, axes = plt.subplots(1, len(stats), figsize=(18, 5))\n\nvalid_age_groups = [age_group for age_group in stats.index if stats.loc[age_group].sum() > 0]\n\nfor i, age_group in enumerate(valid_age_groups):\n    group_counts = stats.loc[age_group] / stats.loc[age_group].sum()\n    axes[i].pie(\n        group_counts, labels=group_counts.index, autopct='%1.1f%%',\n        startangle=90, colors=sns.color_palette(\"Pastel1\"),\n        labeldistance=1.05, pctdistance=0.80\n    )\n    axes[i].set_title(f'SII Distribution for {age_group}', fontsize=10)\n    axes[i].axis('equal')  # Ensures pie chart is a circle\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:29:31.927766Z","iopub.execute_input":"2024-12-19T15:29:31.928194Z","iopub.status.idle":"2024-12-19T15:29:32.533045Z","shell.execute_reply.started":"2024-12-19T15:29:31.928154Z","shell.execute_reply":"2024-12-19T15:29:32.531864Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 7. Seasons distribution","metadata":{}},{"cell_type":"code","source":"import seaborn as sns\n\nseason = train['Basic_Demos-Enroll_Season'].value_counts()\n\ncolors = sns.color_palette(\"pastel\", len(season))\n\nplt.pie(\n    season.values,\n    labels=season.index,\n    autopct='%1.1f%%',\n    startangle=90,\n    colors=colors\n)\nplt.title('Season of Enrollment')\nplt.axis('equal')  # Đảm bảo biểu đồ tròn không bị méo\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:29:32.534413Z","iopub.execute_input":"2024-12-19T15:29:32.534787Z","iopub.status.idle":"2024-12-19T15:29:32.664740Z","shell.execute_reply.started":"2024-12-19T15:29:32.534753Z","shell.execute_reply":"2024-12-19T15:29:32.663380Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- The enrollment is distributed relatively evenly across the four seasons (Fall, Spring, Summer, and Winter). While there are slight variations, no single season dominates the enrollment numbers. This suggests that students enroll throughout the year, and the institution might not experience significant enrollment peaks or lulls tied to specific seasons.\n- Spring has the highest percentage of enrollment at 28.5%, indicating it's the most popular time for students to begin or continue their studies. This could be due to various factors, such as academic calendar structures or student preferences.\n- Fall has the lowest percentage of enrollment at 21.9%. This is somewhat unexpected, as Fall is traditionally a common start time for academic years in many educational systems. This lower percentage might warrant further investigation to understand the underlying reasons.\n- Summer (24.5%) and Winter (25.2%) have very similar enrollment percentages. This suggests a consistent level of student engagement during these off-peak seasons, possibly driven by specialized programs, shorter courses, or students seeking to accelerate their studies.","metadata":{}},{"cell_type":"markdown","source":"## 8. Gender distribution","metadata":{}},{"cell_type":"code","source":"gender = train['Basic_Demos-Sex'].value_counts()\n\nplt.pie(\n    gender.values,  # Tần suất\n    labels=['Boys', 'Girls'],  # Nhãn\n    autopct='%1.1f%%',  # Hiển thị tỉ lệ phần trăm\n    startangle=90,  # Bắt đầu từ góc 90 độ\n    colors=sns.color_palette('pastel')[:2]  # Sử dụng màu pastel\n)\nplt.title('Gender of Participants')\nplt.axis('equal')  # Đảm bảo biểu đồ tròn không bị méo\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:29:32.665963Z","iopub.execute_input":"2024-12-19T15:29:32.666390Z","iopub.status.idle":"2024-12-19T15:29:32.801804Z","shell.execute_reply.started":"2024-12-19T15:29:32.666353Z","shell.execute_reply":"2024-12-19T15:29:32.800657Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from matplotlib.ticker import MaxNLocator\n\nfig, ax = plt.subplots(figsize=(8, 6))\n\nboys_data = train[train['Basic_Demos-Sex'] == 0]['Basic_Demos-Age'].value_counts().sort_index()\ngirls_data = train[train['Basic_Demos-Sex'] == 1]['Basic_Demos-Age'].value_counts().sort_index()\n\nages = sorted(set(boys_data.index).union(girls_data.index))\n\nboys_counts = [boys_data.get(age, 0) for age in ages]\ngirls_counts = [girls_data.get(age, 0) for age in ages]\n\nwidth = 0.4\n\nax.bar(np.array(ages) - width/2, boys_counts, width=width, color=sns.color_palette('pastel')[0], label='Boys')\nax.bar(np.array(ages) + width/2, girls_counts, width=width, color=sns.color_palette('pastel')[1], label='Girls')\n\nax.xaxis.set_major_locator(MaxNLocator(integer=True))  # Hiển thị giá trị nguyên trên trục x\nax.set_ylabel('Count')\nax.set_xlabel('Age')\nax.set_title('Age Distribution by Sex')\n\nax.legend()\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:29:32.802713Z","iopub.execute_input":"2024-12-19T15:29:32.803104Z","iopub.status.idle":"2024-12-19T15:29:33.215017Z","shell.execute_reply.started":"2024-12-19T15:29:32.803067Z","shell.execute_reply":"2024-12-19T15:29:33.213768Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 9. Internet Time Usage","metadata":{}},{"cell_type":"code","source":"param_map = {0: '< 1h/day', 1: '~ 1h/day', 2: '~ 2hs/day', 3: '> 3hs/day'}\ntrain['internet_use_encoded'] = train[\n    'PreInt_EduHx-computerinternet_hoursday'\n].map(param_map).fillna('Missing')\n\nparam_ord = ['Missing', '< 1h/day', '~ 1h/day', '~ 2hs/day', '> 3hs/day']\ntrain['internet_use_encoded'] = pd.Categorical(\n    train['internet_use_encoded'], categories=param_ord,\n    ordered=True\n)\n\nstats = train.groupby(\n    ['sii', 'internet_use_encoded']\n).size().unstack(fill_value=0)\nfig, axes = plt.subplots(1, len(stats), figsize=(18, 5))\n\nfor i, sii_group in enumerate(stats.index):\n    group_counts = stats.loc[sii_group] / stats.loc[sii_group].sum()\n    axes[i].pie(\n        group_counts, labels=group_counts.index, autopct='%1.1f%%',\n        startangle=90, colors=sns.color_palette(\"Set3\"), labeldistance=1.1\n    )\n    axes[i].set_title(f'Hours of using computer/internet\\n for SII = {sii_group}')\n    axes[i].axis('equal')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:29:33.216079Z","iopub.execute_input":"2024-12-19T15:29:33.216442Z","iopub.status.idle":"2024-12-19T15:29:34.049036Z","shell.execute_reply.started":"2024-12-19T15:29:33.216411Z","shell.execute_reply":"2024-12-19T15:29:34.048058Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(8, 5))\n\nsns.countplot(x='internet_use_encoded', data=train, palette=\"Set3\", ax=ax)\n\nax.set_title('Distribution of Hours of Internet Use')\nax.set_xlabel('Hours per Day Group')\nax.set_ylabel('Count')\n\ntotal = len(train['internet_use_encoded'])\n\nfor p in ax.patches:\n    count = int(p.get_height())  # Lấy chiều cao của cột\n    percentage = '{:.1f}%'.format(100 * count / total)  # Tính phần trăm\n    ax.annotate(f'{count} ({percentage})', (p.get_x() + p.get_width() / 2., p.get_height()), \n                ha='center', va='baseline', fontsize=10, color='black', xytext=(0, 5), \n                textcoords='offset points')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:29:34.050360Z","iopub.execute_input":"2024-12-19T15:29:34.050740Z","iopub.status.idle":"2024-12-19T15:29:34.364887Z","shell.execute_reply.started":"2024-12-19T15:29:34.050702Z","shell.execute_reply":"2024-12-19T15:29:34.363931Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 10. Correlation matrix","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Lọc các giá trị không null cho cột 'sii'\ntrain_with_sii = train[train['sii'].notnull()]\n\n# Tính toán ma trận tương quan\nplt.figure(figsize=(14, 30))\ncorr_matrix = train_with_sii[[\n    'PCIAT-PCIAT_Total', 'Basic_Demos-Age', 'Basic_Demos-Sex', 'Physical-BMI', \n    'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n    'Physical-Diastolic_BP', 'Physical-Systolic_BP', 'Physical-HeartRate',\n    'PreInt_EduHx-computerinternet_hoursday', 'SDS-SDS_Total_T', 'PAQ_A-PAQ_A_Total',\n    'PAQ_C-PAQ_C_Total', 'Fitness_Endurance-Max_Stage', 'Fitness_Endurance-Time_Mins','Fitness_Endurance-Time_Sec',\n    'FGC-FGC_CU', 'FGC-FGC_GSND','FGC-FGC_GSD','FGC-FGC_PU','FGC-FGC_SRL','FGC-FGC_SRR','FGC-FGC_TL','BIA-BIA_Activity_Level_num', \n    'BIA-BIA_BMC', 'BIA-BIA_BMI', 'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n    'BIA-BIA_FFMI','BIA-BIA_FMI', 'BIA-BIA_Fat','BIA-BIA_Frame_num','BIA-BIA_ICW','BIA-BIA_LDM','BIA-BIA_LST',\n    'BIA-BIA_SMM','BIA-BIA_TBW'\n]].corr()\n\nsii_corr = corr_matrix['PCIAT-PCIAT_Total'].drop('PCIAT-PCIAT_Total')\n\n# In ra các mối tương quan\nprint(sii_corr)\n\n# Vẽ biểu đồ cột ngang\nplt.figure(figsize=(8, 6))\nsii_corr.sort_values().plot(kind='barh', color=sns.color_palette(\"pastel\", len(sii_corr)))\nplt.title('Features with Correlation with PCIAT-PCIAT_Total')\nplt.xlabel('Correlation coefficient')\nplt.ylabel('Features')\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:29:34.365788Z","iopub.execute_input":"2024-12-19T15:29:34.366113Z","iopub.status.idle":"2024-12-19T15:29:34.902892Z","shell.execute_reply.started":"2024-12-19T15:29:34.366089Z","shell.execute_reply":"2024-12-19T15:29:34.901714Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# III. Time series parquet Exploration","metadata":{}},{"cell_type":"markdown","source":"## Load & Preview one participant data","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\npath = '/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet/id=00115b9f/part-0.parquet'\nseries_train = pd.read_parquet(path)\nseries_train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:29:34.903901Z","iopub.execute_input":"2024-12-19T15:29:34.904200Z","iopub.status.idle":"2024-12-19T15:29:35.036139Z","shell.execute_reply.started":"2024-12-19T15:29:34.904176Z","shell.execute_reply":"2024-12-19T15:29:35.034895Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"participant_id = path.split('/')[-2].split('=')[-1]\nparticipant_id","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:29:35.037158Z","iopub.execute_input":"2024-12-19T15:29:35.037513Z","iopub.status.idle":"2024-12-19T15:29:35.043408Z","shell.execute_reply.started":"2024-12-19T15:29:35.037481Z","shell.execute_reply":"2024-12-19T15:29:35.042573Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train[train['id'] == participant_id]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:29:35.044238Z","iopub.execute_input":"2024-12-19T15:29:35.044535Z","iopub.status.idle":"2024-12-19T15:29:35.074692Z","shell.execute_reply.started":"2024-12-19T15:29:35.044510Z","shell.execute_reply":"2024-12-19T15:29:35.073752Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Helper function","metadata":{}},{"cell_type":"code","source":"def plot_series_data(df, col='non-wear_flag',\n                     label='Worn (0 = Worn, 1 = Not Worn)',\n                     title='Non-Wear Flag',\n                     x_col='day_time', x_label='Day Relative to PCIAT + Time'):\n    plt.figure(figsize=(18, 12))\n    \n    pastel_colors = sns.color_palette(\"pastel\", 7)\n    \n    # X\n    plt.subplot(7, 1, 1)\n    plt.scatter(df[x_col], df['X'], label='X', color=pastel_colors[0], s=1)\n    plt.title('X (Acceleration along X axis)')\n    plt.ylabel('Acceleration (m/s²)')\n\n    # Y\n    plt.subplot(7, 1, 2)\n    plt.scatter(df[x_col], df['Y'], label='Y', color=pastel_colors[1], s=1)\n    plt.title('Y (Acceleration along Y axis)')\n    plt.ylabel('Acceleration (m/s²)')\n\n    # Z\n    plt.subplot(7, 1, 3)\n    plt.scatter(df[x_col], df['Z'], label='Z', color=pastel_colors[2], s=1)\n    plt.title('Z (Acceleration along Z axis)')\n    plt.ylabel('Acceleration (m/s²)')\n    \n    # ENMO\n    plt.subplot(7, 1, 4)\n    plt.scatter(df[x_col], df['enmo'], label='ENMO', color=pastel_colors[3], s=1)\n    plt.title('ENMO (Euclidean Norm Minus One)')\n    plt.ylabel('Movement Intensity')\n\n    # Angle Z\n    plt.subplot(7, 1, 5)\n    plt.scatter(df[x_col], df['anglez'], label='Angle Z', color=pastel_colors[4], s=1)\n    plt.title('Angle Z')\n    plt.ylabel('Angle (degrees)')\n\n    # Light\n    plt.subplot(7, 1, 6)\n    plt.scatter(df[x_col], df['light'], label='Light', color=pastel_colors[5], s=1)\n    plt.title('Ambient Light')\n    plt.ylabel('Light (lux)')\n\n    # Any other column\n    plt.subplot(7, 1, 7)\n    plt.scatter(df[x_col], df[col], label=col, color=pastel_colors[6], s=1)\n    plt.title(f'{title}')\n    plt.ylabel(f'{label}')\n    plt.xlabel(f'{x_label}')\n\n    plt.tight_layout()\n    plt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:29:35.075949Z","iopub.execute_input":"2024-12-19T15:29:35.076244Z","iopub.status.idle":"2024-12-19T15:29:35.092476Z","shell.execute_reply.started":"2024-12-19T15:29:35.076220Z","shell.execute_reply":"2024-12-19T15:29:35.091331Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_series_data(series_train, x_col='step', x_label='Step')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:29:35.093586Z","iopub.execute_input":"2024-12-19T15:29:35.093984Z","iopub.status.idle":"2024-12-19T15:29:36.520352Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Process data","metadata":{}},{"cell_type":"code","source":"TRAIN_PARQUET_PATH = \"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\"\nTEST_PARQUET_PATH = \"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\"\nTRAIN_CSV_PATH = \"/kaggle/input/child-mind-institute-problematic-internet-use/train.csv\"\nTEST_CSV_PATH = \"/kaggle/input/child-mind-institute-problematic-internet-use/test.csv\"\nSAMPLE_PATH = \"/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv\"\nDICTIONARY_PATH = \"/kaggle/input/child-mind-institute-problematic-internet-use/data_dictionary.csv\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:29:36.521572Z","iopub.execute_input":"2024-12-19T15:29:36.521854Z","iopub.status.idle":"2024-12-19T15:29:36.526564Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Time series data","metadata":{}},{"cell_type":"code","source":"def process_time_series(file_path: str) -> list:\n    df = pd.read_parquet(file_path)\n    df = df.drop(\"step\", axis=1)\n    return df.describe().values.flatten().tolist()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:29:36.527577Z","iopub.execute_input":"2024-12-19T15:29:36.527958Z","iopub.status.idle":"2024-12-19T15:29:36.542324Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_time_series(dir: str) -> pd.DataFrame:\n    all_parquet_folders = os.listdir(dir)\n    all_ids = [file.split(\"=\")[-1] for file in all_parquet_folders]\n    all_parquet_files = [\n        os.path.join(dir, folder_name, \"part-0.parquet\")\n        for folder_name in all_parquet_folders\n    ]\n\n    with ThreadPoolExecutor(max_workers=4) as excuter:\n        results = list(\n            tqdm(\n                excuter.map(process_time_series, all_parquet_files), total=len(all_ids)\n            )\n        )\n\n    df = pd.DataFrame(results)\n    df.columns = [f\"Stat_{i}\" for i in range(len(results[0]))]\n    df[\"id\"] = all_ids\n\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:29:36.543305Z","iopub.execute_input":"2024-12-19T15:29:36.543675Z","iopub.status.idle":"2024-12-19T15:29:36.559232Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# class AutoEncoder(nn.Module):\n#     def __init__(self, input_dim, encoding_dim):\n#         super(AutoEncoder, self).__init__()\n        \n#         self.encoder = nn.Sequential(\n#             nn.Linear(input_dim, encoding_dim*3),\n#             nn.ReLU(),\n#             nn.Linear(encoding_dim*3, encoding_dim*2),\n#             nn.ReLU(),\n#             nn.Linear(encoding_dim*2, encoding_dim),\n#             nn.ReLU()\n#         )\n        \n#         self.decoder = nn.Sequential(\n#             nn.Linear(encoding_dim, input_dim*2),\n#             nn.ReLU(),\n#             nn.Linear(input_dim*2, input_dim*3),\n#             nn.ReLU(),\n#             nn.Linear(input_dim*3, input_dim),\n#             nn.Sigmoid()\n#         )\n        \n#     def forward(self, x):\n#         encoded = self.encoder(x)\n#         decoded = self.decoder(encoded)\n#         return decoded","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:29:36.564220Z","iopub.execute_input":"2024-12-19T15:29:36.564532Z","iopub.status.idle":"2024-12-19T15:29:36.578641Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def encode_time_series(\n#     df: pd.DataFrame, encoding_dim: int, batch_size: int, epochs: int\n# ) -> pd.DataFrame:\n#     scaler = StandardScaler()\n#     df = scaler.fit_transform(df)\n#     data = torch.FloatTensor(df)\n#     input_dim = data.shape[1]\n#     autoencoder = AutoEncoder(input_dim, encoding_dim)\n\n#     criterion = nn.MSELoss()\n#     optimizer = optim.Adam(autoencoder.parameters())\n\n#     for epoch in range(epochs):\n#         for i in range(0, len(data), batch_size):\n#             batch = data[i : i + batch_size]\n#             optimizer.zero_grad()\n#             reconstructed = autoencoder(batch)\n#             loss = criterion(reconstructed, batch)\n#             loss.backward()\n#             optimizer.step()\n\n#         if (epoch + 1) % 10 == 0:\n#             print(f\"Epoch [{epoch + 1}/{epochs}], Loss: {loss.item():.4f}]\")\n\n#     with torch.no_grad():\n#         encoded_data = autoencoder.encoder(data).numpy()\n\n#     df_encoded = pd.DataFrame(\n#         encoded_data, columns=[f\"Enc_{i + 1}\" for i in range(encoded_data.shape[1])]\n#     )\n\n#     return df_encoded","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:29:36.581022Z","iopub.execute_input":"2024-12-19T15:29:36.581322Z","iopub.status.idle":"2024-12-19T15:29:36.599768Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## CSV data","metadata":{}},{"cell_type":"code","source":"# 10 columns (excluding PCIAT_Season)\nseason_cols = [\n    \"Basic_Demos-Enroll_Season\",\n    \"CGAS-Season\",\n    \"Physical-Season\",\n    \"Fitness_Endurance-Season\",\n    \"FGC-Season\",\n    \"BIA-Season\",\n    \"PAQ_A-Season\",\n    \"PAQ_C-Season\",\n    \"SDS-Season\",\n    \"PreInt_EduHx-Season\",\n]\n\ndef map_seasons(df: pd.DataFrame) -> pd.DataFrame:\n    df = df.drop(columns=[col for col in df.columns if \"PCIAT\" in col])\n\n    mapping = {\"Summer\": 0, \"Winter\": 1, \"Spring\": 2, \"Fall\": 3, \"Missing\": 4}\n\n    for col in season_cols:\n        df[col] = df[col].fillna(\"Missing\")\n        df[col] = df[col].map(mapping).astype(int)\n\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:29:36.600786Z","iopub.execute_input":"2024-12-19T15:29:36.601195Z","iopub.status.idle":"2024-12-19T15:29:36.613057Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature Engineering","metadata":{}},{"cell_type":"code","source":"def feature_engineering(df: pd.DataFrame) -> pd.DataFrame:\n    # season_cols = [col for col in df.columns if 'Season' in col]\n    # df = df.drop(season_cols, axis=1) \n    # df['BMI_Age'] = df['Physical-BMI'] * df['Basic_Demos-Age']\n    # df['Internet_Hours_Age'] = df['PreInt_EduHx-computerinternet_hoursday'] * df['Basic_Demos-Age']\n    # df['BMI_Internet_Hours'] = df['Physical-BMI'] * df['PreInt_EduHx-computerinternet_hoursday']\n    # df['BFP_BMI'] = df['BIA-BIA_Fat'] / df['BIA-BIA_BMI']\n    # df['FFMI_BFP'] = df['BIA-BIA_FFMI'] / df['BIA-BIA_Fat']\n    # df['FMI_BFP'] = df['BIA-BIA_FMI'] / df['BIA-BIA_Fat']\n    # df['LST_TBW'] = df['BIA-BIA_LST'] / df['BIA-BIA_TBW']\n    # df['BFP_BMR'] = df['BIA-BIA_Fat'] * df['BIA-BIA_BMR']\n    # df['BFP_DEE'] = df['BIA-BIA_Fat'] * df['BIA-BIA_DEE']\n    # df['BMR_Weight'] = df['BIA-BIA_BMR'] / df['Physical-Weight']\n    # df['DEE_Weight'] = df['BIA-BIA_DEE'] / df['Physical-Weight']\n    # df['SMM_Height'] = df['BIA-BIA_SMM'] / df['Physical-Height']\n    # df['Muscle_to_Fat'] = df['BIA-BIA_SMM'] / df['BIA-BIA_FMI']\n    # df['Hydration_Status'] = df['BIA-BIA_TBW'] / df['Physical-Weight']\n    # df['ICW_TBW'] = df['BIA-BIA_ICW'] / df['BIA-BIA_TBW']\n    # df['BMI_PHR'] = df['Physical-BMI'] * df['Physical-HeartRate']\n    \n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:29:36.613897Z","iopub.execute_input":"2024-12-19T15:29:36.614183Z","iopub.status.idle":"2024-12-19T15:29:36.634492Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Train model","metadata":{}},{"cell_type":"code","source":"def compute_qwk(y_pred: np.ndarray, y_true: np.ndarray) -> float:\n    return cohen_kappa_score(y_pred, y_true,weights=\"quadratic\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:29:36.635470Z","iopub.execute_input":"2024-12-19T15:29:36.635781Z","iopub.status.idle":"2024-12-19T15:29:36.651746Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def round_prediction(y_pred: np.ndarray, thresholds: list) -> np.ndarray:\n\n    return np.where(\n        y_pred < thresholds[0],\n        0,\n        np.where(y_pred < thresholds[1], 1, np.where(y_pred < thresholds[2], 2, 3)),\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:29:36.652804Z","iopub.execute_input":"2024-12-19T15:29:36.653201Z","iopub.status.idle":"2024-12-19T15:29:36.673915Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def evaluate_predictions(thresholds, y_true, y_pred):\n    rounded_pred = round_prediction(y_pred, thresholds)\n    return -compute_qwk(y_true, rounded_pred)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:29:36.674973Z","iopub.execute_input":"2024-12-19T15:29:36.675284Z","iopub.status.idle":"2024-12-19T15:29:36.691323Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_model(\n    train_set: pd.DataFrame, test_set: pd.DataFrame, model, test_id\n) -> pd.DataFrame:\n    x = train_set.drop(\"sii\", axis=1)\n    y = train_set[\"sii\"]\n\n    n_samples = len(y)\n\n    SKF = StratifiedKFold(n_splits=N_FOLD, shuffle=True, random_state=SEED)\n\n    train_kappa_scores = []\n    val_kappa_scores = []\n\n    non_rounded_train_pred = np.zeros(n_samples, dtype=float)\n    non_rounded_test_pred = np.zeros((len(test_set), N_FOLD), dtype=float)\n\n    for fold, (train_index, val_index) in enumerate(SKF.split(x, y)):\n        train_data, val_data = x.iloc[train_index], x.iloc[val_index]\n        train_label, val_label = y.iloc[train_index], y.iloc[val_index]\n\n        clone_model = clone(model)\n        clone_model.fit(train_data, train_label)\n\n        train_pred = clone_model.predict(train_data)\n        val_pred = clone_model.predict(val_data)\n\n        train_kappa_scores.append(\n            compute_qwk(train_pred.round(0).astype(int), train_label)\n        )\n\n        val_kappa_scores.append(\n            compute_qwk(val_pred.round(0).astype(int), val_label)\n        )\n\n        non_rounded_train_pred[val_index] = val_pred\n        non_rounded_test_pred[:, fold] = clone_model.predict(test_set)\n\n    kappa_optimizer = minimize(\n        evaluate_predictions,\n        x0=[0.5, 1.5, 2.5],\n        args=(y, non_rounded_train_pred),\n        method=\"Nelder-Mead\",\n    )\n\n    optimized_thresholds = kappa_optimizer.x\n    rounded_train_pred = round_prediction(non_rounded_train_pred, optimized_thresholds)\n    optimized_train_kappa_score = compute_qwk(rounded_train_pred, y)\n\n    print(\"Optimized train kappa score:\", optimized_train_kappa_score)\n\n    rounded_test_pred = round_prediction(\n        non_rounded_test_pred.mean(axis=1), optimized_thresholds\n    )\n\n    submisison = pd.DataFrame({\"id\": test_id, \"sii\": rounded_test_pred})\n\n    return (submisison, optimized_train_kappa_score)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:29:36.692494Z","iopub.execute_input":"2024-12-19T15:29:36.692851Z","iopub.status.idle":"2024-12-19T15:29:36.710699Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Submission 1","metadata":{}},{"cell_type":"markdown","source":"## Load data","metadata":{}},{"cell_type":"code","source":"# Load and encode time series data\ntrain_ts = load_time_series(TRAIN_PARQUET_PATH)\ntest_ts = load_time_series(TEST_PARQUET_PATH)\n# encoded_train_ts = encode_time_series(\n#     df=train_ts.drop(\"id\", axis=1), encoding_dim=60, batch_size=32, epochs=150\n# )\n# encoded_test_ts = encode_time_series(\n#     df=test_ts.drop(\"id\", axis=1), encoding_dim=60, batch_size=32, epochs=150\n# )\n# encoded_train_ts[\"id\"] = train_ts[\"id\"]\n# encoded_test_ts[\"id\"] = test_ts[\"id\"]\n\n# Load and process csv data\ntrain_csv = pd.read_csv(TRAIN_CSV_PATH)\ntest_csv = pd.read_csv(TEST_CSV_PATH)\nmapped_train_csv = map_seasons(train_csv.drop(\"id\", axis=1))\nmapped_test_csv = map_seasons(test_csv.drop(\"id\", axis=1))\n\nmapped_train_csv = mapped_train_csv.dropna(subset={\"sii\"})\n\nmapped_train_csv[\"id\"] = train_csv[\"id\"]\nmapped_test_csv[\"id\"] = test_csv[\"id\"]\n\n# Merge csv and time series data\ntrain = pd.merge(mapped_train_csv, train_ts, how=\"left\", on=\"id\")\ntest = pd.merge(mapped_test_csv, test_ts, how=\"left\", on=\"id\")\n\ntrain_id = train[\"id\"]\ntest_id = test[\"id\"]\n\ntrain = train.drop(\"id\", axis=1)\ntest = test.drop(\"id\", axis=1)\n\n# Do feature engineering\n# train = feature_engineering(train)\n# test = feature_engineering(test)\n\nif np.any(np.isinf(train)):\n    train = train.replace([np.inf, -np.inf], np.nan)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:29:36.711697Z","iopub.execute_input":"2024-12-19T15:29:36.712020Z","iopub.status.idle":"2024-12-19T15:30:56.480071Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Find optimal parameters for models","metadata":{}},{"cell_type":"code","source":"# def objective_xgb(trial):\n#     params = {\n#         'booster': trial.suggest_categorical('booster', ['gbtree', 'dart']),\n#         'max_depth': trial.suggest_int('max_depth', 3, 10),\n#         'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.3, log=True),\n#         'n_estimators': trial.suggest_int('n_estimators', 50, 500),\n#         'subsample': trial.suggest_float('subsample', 0.5, 1.0),\n#         'colsample_bytree': trial.suggest_float('colsample_bytree', 0.5, 1.0),\n#         'gamma': trial.suggest_float('gamma', 0, 5),\n#         'lambda': trial.suggest_float('lambda', 1e-3, 10, log=True),\n#     }\n    \n#     model = XGBRegressor(**params, verbosity=0, device=\"cuda\")\n#     _, metric = train_model(train, test, model, test_id)\n    \n#     return metric","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:30:56.480857Z","iopub.execute_input":"2024-12-19T15:30:56.481144Z","iopub.status.idle":"2024-12-19T15:30:56.484798Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def objective_catb(trial):\n#     params = {\n#         'depth': trial.suggest_int('depth', 3, 10),\n#         'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.3, log=True),\n#         'iterations': trial.suggest_int('iterations', 50, 500),\n#         'l2_leaf_reg': trial.suggest_float('l2_leaf_reg', 1, 10),\n#     }\n    \n#     model = CatBoostRegressor(**params, verbose=0, task_type='GPU')\n#     _, metric = train_model(train, test, model, test_id)\n    \n#     return metric","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:30:56.485671Z","iopub.execute_input":"2024-12-19T15:30:56.486001Z","iopub.status.idle":"2024-12-19T15:30:56.507768Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def objective_lgbm(trial):\n#     params = {\n#         'num_leaves': trial.suggest_int('num_leaves', 20, 300),\n#         'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.3, log=True),\n#         'n_estimators': trial.suggest_int('n_estimators', 50, 500),\n#         'min_child_samples': trial.suggest_int('min_child_samples', 5, 50),\n#         'subsample': trial.suggest_float('subsample', 0.5, 1.0),\n#         'colsample_bytree': trial.suggest_float('colsample_bytree', 0.5, 1.0),\n#         'reg_alpha': trial.suggest_float('reg_alpha', 0.0, 10.0),\n#         'reg_lambda': trial.suggest_float('reg_lambda', 0.0, 10.0),\n#     }\n    \n#     model = LGBMRegressor(**params, verbose=-1, device=\"GPU\")\n#     _, metric = train_model(train, test, model, test_id)\n    \n#     return metric","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:30:56.508727Z","iopub.execute_input":"2024-12-19T15:30:56.509026Z","iopub.status.idle":"2024-12-19T15:30:56.526220Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# study_xgb = optuna.create_study(direction='maximize')\n# study_catb = optuna.create_study(direction='maximize')\n# study_lgbm = optuna.create_study(direction='maximize')\n\n# def optimize_study(study, objective, n_trials=50):\n#     study.optimize(objective, n_trials=n_trials)\n\n# with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:\n#     futures = [\n#         executor.submit(optimize_study, study_xgb, objective_xgb),\n#         executor.submit(optimize_study, study_catb, objective_catb),\n#         executor.submit(optimize_study, study_lgbm, objective_lgbm)\n#     ]\n#     concurrent.futures.wait(futures)\n\n# BEST_PARAMS_XGB = study_xgb.best_params\n# BEST_PARAMS_CATB = study_catb.best_params\n# BEST_PARAMS_LGBM = study_lgbm.best_params\n\n# clear_output(wait=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:30:56.527188Z","iopub.execute_input":"2024-12-19T15:30:56.527496Z","iopub.status.idle":"2024-12-19T15:30:56.546160Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# print(BEST_PARAMS_XGB)\n# print(BEST_PARAMS_CATB)\n# print(BEST_PARAMS_LGBM)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:30:56.547117Z","iopub.execute_input":"2024-12-19T15:30:56.547413Z","iopub.status.idle":"2024-12-19T15:30:56.562248Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"XGB: {'booster': 'gbtree', 'max_depth': 5, 'learning_rate': 0.032806306822661946, 'n_estimators': 124, 'subsample': 0.7612200524957443, 'colsample_bytree': 0.7879462813404031, 'gamma': 2.27693382985497, 'lambda': 0.0037960352854142705}\n\nCATB: {'depth': 8, 'learning_rate': 0.02999839537419576, 'iterations': 492, 'l2_leaf_reg': 2.3005068330449827}\n\nLGBM: {'num_leaves': 290, 'learning_rate': 0.022351630623035076, 'n_estimators': 110, 'min_child_samples': 36, 'subsample': 0.64715318375847, 'colsample_bytree': 0.5964597906674066, 'reg_alpha': 1.5700746777921748, 'reg_lambda': 2.718790747208782}","metadata":{}},{"cell_type":"markdown","source":"## Train","metadata":{}},{"cell_type":"code","source":"BEST_PARAMS_XGB = {\n    \"booster\": \"dart\",\n    \"max_depth\": 3,\n    \"learning_rate\": 0.026326840847042142,\n    \"n_estimators\": 98,\n    \"subsample\": 0.8890660258651147,\n    \"colsample_bytree\": 0.5698858280617991,\n    \"gamma\": 3.7021400107634967,\n    \"lambda\": 1.5109673038871425,\n    \"verbose\": 0,\n    \"random_state\": SEED,\n}\n\nBEST_PARAMS_CATB = {\n    \"depth\": 7,\n    \"learning_rate\": 0.014654088407572447,\n    \"iterations\": 274,\n    \"l2_leaf_reg\": 4.957148716825033,\n    \"verbose\": 0,\n    \"random_state\": SEED,\n}\n\nBEST_PARAMS_LGBM = {\n    \"num_leaves\": 29,\n    \"learning_rate\": 0.014234944977835996,\n    \"n_estimators\": 246,\n    \"min_child_samples\": 49,\n    \"subsample\": 0.7674664773701858,\n    \"colsample_bytree\": 0.879948431560254,\n    \"reg_alpha\": 8.136293189993065,\n    \"reg_lambda\": 3.7353821130911413,\n    \"verbose\": -1,\n    \"random_state\": SEED,\n}\n\nBEST_PARAMS_RF = {\n    \"n_estimators\": 493,\n    \"max_depth\": 5,\n    \"min_samples_split\": 6,\n    \"min_samples_leaf\": 1,\n    \"max_features\": None,\n    \"bootstrap\": True,\n    \"random_state\": SEED,\n}\n\nBEST_PARAMS_GB = {\n    \"n_estimators\": 295,\n    \"learning_rate\": 0.012215329148983722,\n    \"max_depth\": 3,\n    \"min_samples_split\": 18,\n    \"min_samples_leaf\": 5,\n    \"subsample\": 0.599656091501263,\n    \"max_features\": None,\n    \"random_state\": SEED,\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:30:56.563229Z","iopub.execute_input":"2024-12-19T15:30:56.563544Z","iopub.status.idle":"2024-12-19T15:30:56.577924Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def objective_ensemble(trial):\n#     weights = {\n#         \"weights\": [\n#             trial.suggest_float(\"weight_xgb\", 0.0, 10.0),\n#             trial.suggest_float(\"weight_catb\", 0.0, 10.0),\n#             trial.suggest_float(\"weight_lgbm\", 0.0, 10.0),\n#         ]\n#     }\n    \n#     model = VotingRegressor(estimators=[\n#         (\"xgb\", XGBRegressor(**BEST_PARAMS_XGB, device=\"cuda\")),\n#         (\"catb\", CatBoostRegressor(**BEST_PARAMS_CATB, task_type=\"GPU\")),\n#         (\"lgbm\", LGBMRegressor(**BEST_PARAMS_LGBM, device=\"GPU\")),\n#     ], **weights)\n    \n#     _, metric = train_model(train, test, model, test_id)\n    \n#     return metric\n\n\n# study_ensemble = optuna.create_study(direction='maximize')\n\n# def optimize_study(study, objective, n_trials=50):\n#     study.optimize(objective, n_trials=n_trials)\n\n# with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:\n#     futures = [\n#         executor.submit(optimize_study, study_ensemble, objective_ensemble)\n#     ]\n#     concurrent.futures.wait(futures)\n\n# BEST_WEIGHTS = study_ensemble.best_params\n\n# clear_output(wait=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:30:56.578995Z","iopub.execute_input":"2024-12-19T15:30:56.579346Z","iopub.status.idle":"2024-12-19T15:30:56.597020Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# print(BEST_WEIGHTS)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:30:56.598033Z","iopub.execute_input":"2024-12-19T15:30:56.598341Z","iopub.status.idle":"2024-12-19T15:30:56.609954Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"{'weight_xgb': 0.29143842381072055, 'weight_catb': 0.7223070202458827, 'weight_lgbm': 0.0038376523038045}","metadata":{}},{"cell_type":"code","source":"imputer = SimpleImputer(strategy='median')\n\nensemble_model = VotingRegressor(estimators=[\n    ('lgb', Pipeline(steps=[('imputer', imputer), ('regressor', LGBMRegressor(**BEST_PARAMS_LGBM))])),\n    ('xgb', Pipeline(steps=[('imputer', imputer), ('regressor', XGBRegressor(**BEST_PARAMS_XGB))])),\n    ('cat', Pipeline(steps=[('imputer', imputer), ('regressor', CatBoostRegressor(**BEST_PARAMS_CATB))])),\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:30:56.610777Z","iopub.execute_input":"2024-12-19T15:30:56.611074Z","iopub.status.idle":"2024-12-19T15:30:56.627999Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission1, _ = train_model(train, test, ensemble_model, test_id)\nclear_output(wait=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:30:56.628991Z","iopub.execute_input":"2024-12-19T15:30:56.629343Z","iopub.status.idle":"2024-12-19T15:32:29.763049Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Submission 2","metadata":{}},{"cell_type":"code","source":"imputer = SimpleImputer(strategy='median')\n\nensemble_model = VotingRegressor(estimators=[\n    ('lgb', Pipeline(steps=[('imputer', imputer), ('regressor', LGBMRegressor(**BEST_PARAMS_LGBM))])),\n    ('xgb', Pipeline(steps=[('imputer', imputer), ('regressor', XGBRegressor(**BEST_PARAMS_XGB))])),\n    ('cat', Pipeline(steps=[('imputer', imputer), ('regressor', CatBoostRegressor(**BEST_PARAMS_CATB))])),\n    ('rf', Pipeline(steps=[('imputer', imputer), ('regressor', RandomForestRegressor(**BEST_PARAMS_RF))])),\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:32:29.764000Z","iopub.execute_input":"2024-12-19T15:32:29.764264Z","iopub.status.idle":"2024-12-19T15:32:29.769966Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission2, _ = train_model(train, test, ensemble_model, test_id)\nclear_output(wait=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:32:29.771181Z","iopub.execute_input":"2024-12-19T15:32:29.771542Z","iopub.status.idle":"2024-12-19T15:35:38.146961Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Submission 3","metadata":{}},{"cell_type":"code","source":"imputer = SimpleImputer(strategy='median')\n\nensemble_model = VotingRegressor(estimators=[\n    ('lgb', Pipeline(steps=[('imputer', imputer), ('regressor', LGBMRegressor(**BEST_PARAMS_LGBM))])),\n    ('xgb', Pipeline(steps=[('imputer', imputer), ('regressor', XGBRegressor(**BEST_PARAMS_XGB))])),\n    ('cat', Pipeline(steps=[('imputer', imputer), ('regressor', CatBoostRegressor(**BEST_PARAMS_CATB))])),\n    ('rf', Pipeline(steps=[('imputer', imputer), ('regressor', RandomForestRegressor(**BEST_PARAMS_RF))])),\n    ('gb', Pipeline(steps=[('imputer', imputer), ('regressor', GradientBoostingRegressor(**BEST_PARAMS_GB))]))\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:35:38.148712Z","iopub.execute_input":"2024-12-19T15:35:38.149167Z","iopub.status.idle":"2024-12-19T15:35:38.156712Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission3, _ = train_model(train, test, ensemble_model, test_id)\nclear_output(wait=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:35:38.158166Z","iopub.execute_input":"2024-12-19T15:35:38.158582Z","iopub.status.idle":"2024-12-19T15:39:19.620085Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Final submission","metadata":{}},{"cell_type":"code","source":"stacked_submission = pd.DataFrame({\n    \"id\": test_id,\n    \"sub1\": submission1[\"sii\"],\n    \"sub2\": submission2[\"sii\"],\n    \"sub3\": submission3[\"sii\"],\n})\n\ndef get_mode(row):\n    return row.mode()[0]\n\nfinal_submission = pd.DataFrame({\n    \"id\": stacked_submission[\"id\"],\n    \"sii\": stacked_submission.iloc[:, 1:].apply(get_mode, axis=1)\n})\n\nprint(final_submission)\n\nfinal_submission.to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T15:39:19.621507Z","iopub.execute_input":"2024-12-19T15:39:19.621829Z","iopub.status.idle":"2024-12-19T15:39:19.643732Z"}},"outputs":[],"execution_count":null}]}