{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30804,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-10T14:06:14.081070Z","iopub.execute_input":"2024-12-10T14:06:14.081509Z","iopub.status.idle":"2024-12-10T14:06:19.094870Z","shell.execute_reply.started":"2024-12-10T14:06:14.081471Z","shell.execute_reply":"2024-12-10T14:06:19.093106Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install xgboost\n!pip install catboost\n!pip install lightgbm\n!pip install dask[dataframe]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T14:06:19.097740Z","iopub.execute_input":"2024-12-10T14:06:19.098605Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nfrom sklearn.impute import KNNImputer\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.preprocessing import PowerTransformer\nfrom sklearn.preprocessing import RobustScaler \n\n# Sklearn estimators\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.ensemble import RandomForestClassifier\nfrom imblearn.ensemble import BalancedRandomForestClassifier, BalancedBaggingClassifier\nfrom sklearn.ensemble import GradientBoostingClassifier\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.ensemble import AdaBoostClassifier\nfrom xgboost import XGBClassifier\nfrom catboost import CatBoostClassifier\nfrom lightgbm import LGBMClassifier\nfrom sklearn.svm import SVC\nfrom sklearn.ensemble import VotingClassifier, StackingClassifier\n\n# Model evaluations\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, cohen_kappa_score\nfrom sklearn.decomposition import PCA\nfrom sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay\nfrom sklearn.metrics import classification_report\nfrom sklearn.model_selection import RandomizedSearchCV, GridSearchCV\nfrom imblearn.combine import SMOTETomek\nfrom imblearn.under_sampling import RandomUnderSampler\nfrom imblearn.pipeline import Pipeline\nfrom sklearn.base import BaseEstimator, ClassifierMixin\n# from sklearn.experimental import enable_iterative_imputer\n# from sklearn.impute import IterativeImputer\nfrom sklearn.impute import SimpleImputer\n\n\nimport dask.dataframe as dd\nimport os\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# pd.set_option('display.max_columns', None)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_dir = \"/kaggle/input/child-mind-institute-problematic-internet-use\"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_csv = pd.read_csv(f\"{data_dir}/train.csv\")\ntrain_ts = dd.read_parquet(f\"{data_dir}/series_train.parquet\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Aggregate duplicates\ntrain_grouped = train_ts.groupby('id').agg({\n    'X': 'mean',\n    'Y': 'mean',\n    'Z': 'mean',\n    'enmo': 'mean',\n    'anglez': 'mean',\n    'non-wear_flag': 'sum', \n    'light': 'mean',\n    'battery_voltage': 'mean',\n    'time_of_day': 'mean',\n    'weekday': 'mean',\n    'quarter': 'mean',\n    'relative_date_PCIAT': 'mean',\n}).compute()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Merge the aggregated data with the new features based on 'id'\ntrain_data = pd.merge(train_csv, train_grouped, how = 'left', on = 'id')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_data = train_data.copy()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Handle missing data","metadata":{}},{"cell_type":"code","source":"# lists of data to replace their NaNs\ncontinuous_data = [\"Basic_Demos-Age\", \"CGAS-CGAS_Score\", \"Physical-Diastolic_BP\", \"Physical-HeartRate\", \"Physical-Systolic_BP\",\n                   \"Fitness_Endurance-Max_Stage\", \"Fitness_Endurance-Time_Mins\", \"Fitness_Endurance-Time_Sec\", \"FGC-FGC_CU\",\n                   \"FGC-FGC_PU\", \"PCIAT-PCIAT_Total\", \"SDS-SDS_Total_Raw\", \"SDS-SDS_Total_T\", \"Physical-Waist_Circumference\",\n                   'X', 'Y', 'Z', 'enmo', 'anglez', 'light', 'battery_voltage', 'time_of_day', 'relative_date_PCIAT',\n                  \"Physical-BMI\", \"Physical-Height\", \"Physical-Weight\", \"FGC-FGC_GSND\", \"FGC-FGC_GSD\", \"FGC-FGC_SRL\", \"FGC-FGC_SRR\", \n                   \"FGC-FGC_TL\", \"BIA-BIA_BMC\", \"BIA-BIA_BMI\", \"BIA-BIA_BMR\", \"BIA-BIA_DEE\", \"BIA-BIA_ECW\", \"BIA-BIA_FFM\", \n                   \"BIA-BIA_FFMI\", \"BIA-BIA_FMI\", \"BIA-BIA_Fat\", \"BIA-BIA_ICW\", \"BIA-BIA_LDM\", \"BIA-BIA_LST\", \"BIA-BIA_SMM\", \n                   \"BIA-BIA_TBW\", \"PAQ_A-PAQ_A_Total\", \"PAQ_C-PAQ_C_Total\"]\n\ncategorical_data = [\"Basic_Demos-Enroll_Season\", \"Basic_Demos-Sex\", \"CGAS-Season\", \"Physical-Season\", \"Fitness_Endurance-Season\",\n                    \"FGC-Season\", \"FGC-FGC_CU_Zone\", \"FGC-FGC_GSND_Zone\", \"FGC-FGC_GSD_Zone\", \"FGC-FGC_PU_Zone\", \"FGC-FGC_SRL_Zone\",\n                    \"FGC-FGC_SRR_Zone\", \"FGC-FGC_TL_Zone\", \"BIA-Season\", \"BIA-BIA_Activity_Level_num\", \"BIA-BIA_Frame_num\", \"PAQ_A-Season\",\n                    \"PAQ_C-Season\", \"PCIAT-Season\", \"PCIAT-PCIAT_01\", \"PCIAT-PCIAT_02\", \"PCIAT-PCIAT_03\", \"PCIAT-PCIAT_04\", \"PCIAT-PCIAT_05\",\n                    \"PCIAT-PCIAT_06\", \"PCIAT-PCIAT_07\", \"PCIAT-PCIAT_08\", \"PCIAT-PCIAT_09\", \"PCIAT-PCIAT_10\", \"PCIAT-PCIAT_11\", \"PCIAT-PCIAT_12\",\n                    \"PCIAT-PCIAT_13\", \"PCIAT-PCIAT_14\", \"PCIAT-PCIAT_15\", \"PCIAT-PCIAT_16\", \"PCIAT-PCIAT_17\", \"PCIAT-PCIAT_18\", \"PCIAT-PCIAT_19\",\n                    \"PCIAT-PCIAT_20\", \"SDS-Season\", \"PreInt_EduHx-Season\", \"PreInt_EduHx-computerinternet_hoursday\", 'weekday', 'quarter', \n                    'non-wear_flag']","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Impute Categorical Data\ncat_imputer = SimpleImputer(strategy='most_frequent')\ndf_categorical_imputed = cat_imputer.fit_transform(df_data[categorical_data])\ndf_categorical_imputed = pd.DataFrame(df_categorical_imputed, columns=categorical_data)\ndf_data[categorical_data] = df_categorical_imputed","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Use median for selected continuous data\nfor col in continuous_data:\n    if col in df_data.columns:\n        new_value_median = df_data[col].median()\n        df_data[col] = df_data[col].fillna(new_value_median)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Fill in missing PCIAT-PCIAT_Total","metadata":{}},{"cell_type":"code","source":"df_data['PCIAT-PCIAT_Total'].unique()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Check if 'PCIAT-PCIAT_Total' has missing values\nif df_data['PCIAT-PCIAT_Total'].isna().any():\n    # Compute 'PCIAT-PCIAT_Total' only for rows where it is NaN\n    df_data.loc[df_data['PCIAT-PCIAT_Total'].isna(), 'PCIAT-PCIAT_Total'] = (\n        df_data.loc[df_data['PCIAT-PCIAT_Total'].isna(), [f\"PCIAT-PCIAT_{i:02}\" for i in range(1, 21)]]\n        .sum(axis=1)\n    )\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_data['PCIAT-PCIAT_Total'].unique()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(df_data['PCIAT-PCIAT_Total'].isna().any())","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Dynamically extract the range of PCIAT-PCIAT_Total for each class (sii value)","metadata":{}},{"cell_type":"code","source":"# Filter rows with valid data\nvalid_data = df_data[df_data['sii'].notna() & df_data['PCIAT-PCIAT_Total'].notna()]\n\n# Group by 'sii' and analyze the range of 'PCIAT-PCIAT_Total'\nsii_ranges = valid_data.groupby('sii')['PCIAT-PCIAT_Total'].agg(['min', 'max']).sort_index()\n\n# Display ranges (optional, for inspection)\nprint(\"Ranges of PCIAT-PCIAT_Total for each sii class:\")\nprint(sii_ranges)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Fill in missing sii","metadata":{}},{"cell_type":"code","source":"# Define thresholds dynamically based on observed ranges\nthresholds = []\nfor i in sii_ranges.index:\n    thresholds.append(sii_ranges.loc[i, 'max'])\n\n# Add a large upper bound to handle the highest range\nthresholds.append(float('inf'))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Map PCIAT-PCIAT_Total to sii dynamically\ndef map_pciat_to_sii_dynamic(total):\n    for i, upper_bound in enumerate(thresholds):\n        if total <= upper_bound:\n            return i  \n    # For unexpected cases\n    return None  ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Map to each sii missing\n# Apply the dynamic mapping to the dataset\ndf_data['sii'] = df_data['sii'].fillna(df_data['PCIAT-PCIAT_Total'].apply(map_pciat_to_sii_dynamic))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Cluster based, not the best since sii and PCIAT-PCIAT_Total have row-based relationships\n\n# from sklearn.preprocessing import StandardScaler\n# from sklearn.cluster import KMeans\n\n# # Select PCIAT features for clustering\n# pciat_features = [f\"PCIAT-PCIAT_{i:02}\" for i in range(1, 21)] \n# clustering_data = df_data[pciat_features]\n\n# # Handle missing values in PCIAT features\n# imputer = SimpleImputer(strategy=\"mean\")\n# clustering_data_imputed = imputer.fit_transform(clustering_data)\n\n# # Standardize the data\n# scaler = StandardScaler()\n# clustering_data_scaled = scaler.fit_transform(clustering_data_imputed)\n\n# # Apply k-means clustering\n# n_clusters = 4  # Choose the number of clusters based on domain knowledge or analysis\n# kmeans = KMeans(n_clusters=n_clusters, random_state=42)\n# df_data['cluster'] = kmeans.fit_predict(clustering_data_scaled)\n\n# # Map clusters to sii based on known values\n# # Analyze the sii distribution in each cluster\n# cluster_sii_mapping = (\n#     df_data[df_data['sii'].notna()]\n#     .groupby('cluster')['sii']\n#     .agg(lambda x: x.mode()[0])  # Use mode of sii in each cluster\n# )\n\n# # Impute missing sii values based on cluster\n# df_data['sii_imputed'] = df_data['cluster'].map(cluster_sii_mapping)\n\n# # Fill missing sii with imputed values\n# df_data['sii'] = df_data['sii'].fillna(df_data['sii_imputed'])\n\n# # Drop the temporary 'sii_imputed' and 'cluster' columns\n# df_data.drop(columns=['sii_imputed', 'cluster'], inplace=True)\n\n# # Check the results\n# print(df_data['sii'].isna().sum(), \"missing sii values remain.\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Drop columns","metadata":{}},{"cell_type":"code","source":"columns_to_drop = [\"id\", \"PCIAT-PCIAT_01\", \"PCIAT-PCIAT_02\", \"PCIAT-PCIAT_03\", \"PCIAT-PCIAT_04\", \"PCIAT-PCIAT_05\",\n                    \"PCIAT-PCIAT_06\", \"PCIAT-PCIAT_07\", \"PCIAT-PCIAT_08\", \"PCIAT-PCIAT_09\", \"PCIAT-PCIAT_10\", \"PCIAT-PCIAT_11\", \"PCIAT-PCIAT_12\",\n                    \"PCIAT-PCIAT_13\", \"PCIAT-PCIAT_14\", \"PCIAT-PCIAT_15\", \"PCIAT-PCIAT_16\", \"PCIAT-PCIAT_17\", \"PCIAT-PCIAT_18\", \"PCIAT-PCIAT_19\",\n                    \"PCIAT-PCIAT_20\", \"PCIAT-PCIAT_Total\"]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Drop columns\ndf_data.drop(columns_to_drop, axis = 1, inplace = True)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_data.duplicated().value_counts()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_duplicate = df_data[df_data.duplicated()]\ndf_duplicate.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# drop duplicated rows\ndf_data = df_data.drop_duplicates(keep='first')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# confirm if dropped\nduplicates_count = df_data.duplicated().sum()\nprint(f\"Number of exact duplicate rows: {duplicates_count}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_data['sii'].value_counts()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"seasons = []\n\n# Identify columns that are likely to contain season data\nfor label, value in df_data.items():\n    if pd.api.types.is_object_dtype(value):  # Only consider object (string) columns\n        seasons.append(label)\n\n# Map season values only for columns that contain valid season values\nfor col in seasons:\n    if col != \"id\":  # Avoid modifying the \"id\" column\n        valid_seasons = df_data[col].isin([\"Spring\", \"Winter\", \"Fall\", \"Summer\"]).any()\n        \n        if valid_seasons:  # Only map if the column contains valid season names\n            print(f\"Mapping values for {col}:\")\n            df_data[col] = df_data[col].map({\"Spring\": 0, \"Winter\": 1, \"Fall\": 2, \"Summer\": 3})","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"feature_to_encode = [\"CGAS-CGAS_Score\", \"Physical-Waist_Circumference\", \"Physical-Diastolic_BP\", \"Physical-HeartRate\",\n                     \"Physical-Systolic_BP\", \"Fitness_Endurance-Max_Stage\", \"Fitness_Endurance-Time_Mins\",\n                     \"Fitness_Endurance-Time_Sec\", \"FGC-FGC_CU\", \"FGC-FGC_CU_Zone\", \"FGC-FGC_GSND_Zone\", \"FGC-FGC_GSD_Zone\",\n                     \"FGC-FGC_PU\", \"FGC-FGC_PU_Zone\", \"FGC-FGC_SRL_Zone\", \"FGC-FGC_SRR_Zone\", \"FGC-FGC_TL\", \"FGC-FGC_TL_Zone\",\n                     \"BIA-BIA_Activity_Level_num\", \"BIA-BIA_Frame_num\", \"SDS-SDS_Total_Raw\",\n                     \"SDS-SDS_Total_T\", \"PreInt_EduHx-computerinternet_hoursday\", \"non-wear_flag\", \"sii\"]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for col in feature_to_encode:\n    if col in df_data.columns:\n        df_data[col] = (df_data[col]).astype('int64')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_data.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# The PCIAT-PCIAT_Total is a direct sum of the answers to the PCIAT-PCIAT_01 to PCIAT-PCIAT_20 questions. \n# The target variable sii is derived from PCIAT-PCIAT_Total. \n# If PCIAT-PCIAT_01 to PCIAT-PCIAT_20 and PCIAT-PCIAT_Total are not dropped, imputation would rely\n# on patterns in features or the test data, potentially \"leaking\" information about the target into \n# the training process. \n# This could lead to overly optimistic results during training and poor performance on unseen test data.\n# Using PCIAT variables as features to predict sii can lead to data leakage and overfitting","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Drop columns","metadata":{}},{"cell_type":"code","source":"df_data.columns","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Split data into train and validation sets","metadata":{}},{"cell_type":"code","source":"df_new = df_data.copy()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = df_new.drop(['sii'], axis = 1)\ny = df_new['sii']","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_new['sii'].value_counts()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Separate into train and test sets\nX_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size = 0.1, stratify = y, random_state = 104)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train.shape, X_valid.shape, y_train.shape, y_valid.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Change remaining object to numeric\ndef change_to_numeric(df):\n    # Select columns of type \"object\"\n    object_cols = df.select_dtypes(include=[\"object\"]).columns\n    # Convert object columns to numeric where possible\n    for col in object_cols:\n        df[col] = pd.to_numeric(df[col], errors='coerce')\n    return df\n\n# Apply the function\nX_train = change_to_numeric(X_train)\nX_valid = change_to_numeric(X_valid)\n\n# Now you can check the dtypes\nprint(X_train.dtypes)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Apply power transformation on X_train, X_valid and tet data because of outliers","metadata":{}},{"cell_type":"code","source":"pd.set_option('display.max_rows', 200)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# After transformation\nprint(\"NaN values in X_train\", np.isnan(X_train).sum())\nprint(\"NaN values in X_valid\", np.isnan(X_valid).sum())\nprint(\"Infinite values in X_train\", np.isinf(X_train).sum())\nprint(\"NaN values in X_valid\", np.isinf(X_valid).sum())","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Initialize PowerTransformer\ntransformer = PowerTransformer(method='yeo-johnson')\n\n# Apply power transformation to training data\nX_train_transformed = transformer.fit_transform(X_train)\n\n# Transform validation using the same transformer\nX_valid_transformed = transformer.transform(X_valid)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Do not use robust scaler or standard scaler after QuantileTransformer\n# from sklearn.preprocessing import QuantileTransformer\n\n# transformer = QuantileTransformer(output_distribution = 'normal')\n\n# # Apply quantile transformation to training data\n# X_train_transformed = transformer.fit_transform(X_train)\n\n# # Transform validation using the same transformer\n# X_valid_transformed = transformer.transform(X_valid)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"scaler = RobustScaler()\nX_train_scaled = scaler.fit_transform(X_train_transformed)\nX_valid_scaled = scaler.transform(X_valid_transformed)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# After transformation\nprint(\"NaN values in X_train\", np.isnan(X_train_scaled).sum())\nprint(\"NaN values in X_valid\", np.isnan(X_valid_scaled).sum())\nprint(\"Infinite values in X_train\", np.isinf(X_train_scaled).sum())\nprint(\"NaN values in X_valid\", np.isinf(X_valid_scaled).sum())","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train = X_train_scaled\nX_valid = X_valid_scaled","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Modelling","metadata":{}},{"cell_type":"code","source":"# Built-in metric from sklearn including cohen_kappa_score\ndef built_in_metrics(y_true, y_pred):\n    accuracy = accuracy_score(y_true, y_pred)\n    precision = precision_score(y_true, y_pred, average = \"weighted\")\n    recall = recall_score(y_true, y_pred, average = \"weighted\")\n    f1 = f1_score(y_true, y_pred, average = \"weighted\")\n    cohen_kappa = cohen_kappa_score(y_true, y_pred, weights = \"quadratic\")\n\n    return {\"Accuracy\": accuracy,\n            \"Precision\": precision,\n            \"Recall\": recall,\n            \"F1\": f1,\n            \"Cohen Kappa\": cohen_kappa}","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Multiclass Logistic Regression\nclf = LogisticRegression(multi_class = 'multinomial', class_weight=\"balanced\")\nclf.fit(X_train, y_train)\n\n# Predictions\ny_pred_train = clf.predict(X_train)\ny_pred_valid = clf.predict(X_valid)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Multiclass Logistic Regression\nclf = LogisticRegression(multi_class = 'multinomial', class_weight=\"balanced\")\nclf.fit(X_train, y_train)\n\n# Predictions\ny_pred_train = clf.predict(X_train)\ny_pred_valid = clf.predict(X_valid)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"training_built_in_scores = built_in_metrics(y_train, y_pred_train)\nvalidation_built_in_scores = built_in_metrics(y_valid, y_pred_valid)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pd.DataFrame({\"training Scores\": training_built_in_scores, \"Validation Scores\": validation_built_in_scores})","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Choose some good models after hyperparameters tuning locally and combining different models using stacking and voting classifiers","metadata":{}},{"cell_type":"code","source":"# Define the class weights obtained after calculation\nclass_weight = {0: 0.38, 1: 1.09, 2: 2.61, 3: 28.74}","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"rf_clf = RandomForestClassifier(n_estimators=100, class_weight = 'balanced', max_depth = None, max_features = 7,\n                                min_samples_split = 5)\ncatboost_clf = CatBoostClassifier(depth = 3, iterations = 300, l2_leaf_reg = 3, learning_rate = 0.2, silent = True)\nxgb_clf = XGBClassifier(n_estimators = 100, use_label_encoder = False, eval_metric = 'mlogloss', colsample_bytree = 0.8,\n                        learning_rate = 0.1, max_depth = 3, scale_pos_weight = 1)\nbrfc = BalancedRandomForestClassifier(class_weight = 'balanced_subsample', max_depth = None, max_features = 0.7, \n                                      min_samples_split = 2, n_estimators = 100)\ngb_clf = GradientBoostingClassifier(subsample = 0.8, n_estimators = 100, min_samples_split = 10, min_samples_leaf = 4, \n                                    max_features = 'log2', max_depth = 6, learning_rate = 0.2)\nlgbm_clf = LGBMClassifier(subsample = 0.6, reg_lambda = 0.5, reg_alpha = 0.5, num_leaves = 63, n_estimators = 100, min_child_weight = 0.01,\n                         min_child_samples = 20, max_depth = 7, learning_rate = 0.1, colsample_bytree = 0.6)\n\n# List of base classifiers\nclassifiers = [\n    ('rf', rf_clf),\n    ('gb', gb_clf),\n    ('catboost', catboost_clf),\n    ('lgbm', lgbm_clf),\n    ('xgb', xgb_clf),\n    ('Balance Random Forest', brfc)\n]\n\n# Create a Voting Classifier (Soft Voting)\nvoting_clf = VotingClassifier(estimators = classifiers, voting='soft') # , weights = [1, 1, 2, 1, 1, 1]\n\n# Alternatively, you can create a Stacking Classifier with Logistic Regression as the meta-model\nstacking_clf = StackingClassifier(estimators=classifiers, final_estimator=LogisticRegression())\n\n# Create the pipeline with the Voting or Stacking model\npipeline_voting = Pipeline([\n    ('voting', voting_clf)\n])\n\npipeline_stacking = Pipeline([\n    ('stacking', stacking_clf)\n])\n\n# Fit and evaluate each model\nscores_voting = {}\nscores_stacking = {}\n\n# Voting Classifier\npipeline_voting.fit(X_train, y_train)\ny_pred_train_voting = pipeline_voting.predict(X_train)\ny_pred_valid_voting = pipeline_voting.predict(X_valid)\n\n# Collect performance metrics for Voting Classifier\ntrain_score_voting = built_in_metrics(y_train, y_pred_train_voting)\nvalid_score_voting = built_in_metrics(y_valid, y_pred_valid_voting)\nscores_voting[\"Training Scores\"] = train_score_voting\nscores_voting[\"Validation Scores\"] = valid_score_voting\n\n# Stacking Classifier\npipeline_stacking.fit(X_train, y_train)\ny_pred_train_stacking = pipeline_stacking.predict(X_train)\ny_pred_valid_stacking = pipeline_stacking.predict(X_valid)\n\n# Collect performance metrics for Stacking Classifier\ntrain_score_stacking = built_in_metrics(y_train, y_pred_train_stacking)\nvalid_score_stacking = built_in_metrics(y_valid, y_pred_valid_stacking)\nscores_stacking[\"Training Scores\"] = train_score_stacking\nscores_stacking[\"Validation Scores\"] = valid_score_stacking\n\n# Display results\nvoting_results = pd.DataFrame(scores_voting)\nstacking_results = pd.DataFrame(scores_stacking)\n\n# Combine both results into a single DataFrame\nfinal_results2 = pd.concat([voting_results, stacking_results], axis=1)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_results2","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### With SMOTE","metadata":{}},{"cell_type":"code","source":"from imblearn.over_sampling import SMOTE\n\n# Initialize SMOTE and Classifiers\nsmote = SMOTE(random_state=104)\n\nrf_clf = RandomForestClassifier(n_estimators=100, random_state = 104)\ngb_clf = GradientBoostingClassifier(n_estimators=100, random_state = 104)\ncatboost_clf = CatBoostClassifier(random_state = 104, silent=True)\nlgbm_clf = LGBMClassifier(n_estimators=100, random_state = 104)\nxgb_clf = XGBClassifier(n_estimators=100, random_state = 104)\n\n# Create individual pipelines for each classifier with SMOTE\npipelines = {\n    \"Random Forest\": Pipeline([('smote', smote), ('rf', rf_clf)]),\n    \"Gradient Boosting\": Pipeline([('smote', smote), ('gb', gb_clf)]),\n    \"CatBoost\": Pipeline([('smote', smote), ('catboost', catboost_clf)]),\n    \"LightGBM\": Pipeline([('smote', smote), ('lgbm', lgbm_clf)]),\n    \"XGBoost\": Pipeline([('smote', smote), ('xgb', xgb_clf)]),\n}\n\n# Fit and evaluate each individual model\nindividual_scores = {}\nfor model_name, pipeline in pipelines.items():\n    pipeline.fit(X_train, y_train)\n\n    # Predict and evaluate on training and validation data\n    y_pred_train = pipeline.predict(X_train)\n    y_pred_valid = pipeline.predict(X_valid)\n\n    # Collect performance metrics\n    train_score = built_in_metrics(y_train, y_pred_train)\n    valid_score = built_in_metrics(y_valid, y_pred_valid)\n\n    # Store the scores\n    individual_scores[model_name] = {\n        \"Training Scores\": train_score,\n        \"Validation Scores\": valid_score\n    }\n\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(individual_scores[model_name])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pd.DataFrame(individual_scores[model_name])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# confusion matrix\nconf_matrix = confusion_matrix(y_valid, y_pred_valid_stacking)\n\n# Display the confusion matrix\ndisp = ConfusionMatrixDisplay(confusion_matrix = conf_matrix)\ndisp.plot(cmap='Blues', xticks_rotation='vertical')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Print classification report\nprint(classification_report(y_valid, y_pred_valid_stacking))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Ideal model","metadata":{}},{"cell_type":"code","source":"rf_clf = RandomForestClassifier(n_estimators=100, class_weight = 'balanced', max_depth = None, max_features = 7,\n                                min_samples_split = 5)\ncatboost_clf = CatBoostClassifier(depth = 3, iterations = 300, l2_leaf_reg = 3, learning_rate = 0.2, silent = True)\nxgb_clf = XGBClassifier(n_estimators = 100, use_label_encoder = False, eval_metric = 'mlogloss', colsample_bytree = 0.8,\n                        learning_rate = 0.1, max_depth = 3, scale_pos_weight = 1)\nbrfc = BalancedRandomForestClassifier(class_weight = 'balanced_subsample', max_depth = None, max_features = 0.7, \n                                      min_samples_split = 2, n_estimators = 100)\ngb_clf = GradientBoostingClassifier(subsample = 0.8, n_estimators = 100, min_samples_split = 10, min_samples_leaf = 4, \n                                    max_features = 'log2', max_depth = 6, learning_rate = 0.2)\nlgbm_clf = LGBMClassifier(subsample = 0.6, reg_lambda = 0.5, reg_alpha = 0.5, num_leaves = 63, n_estimators = 100, min_child_weight = 0.01,\n                         min_child_samples = 20, max_depth = 7, learning_rate = 0.1, colsample_bytree = 0.6)\n\n# List of base classifiers\nclassifiers = [\n    ('rf', rf_clf),\n    ('gb', gb_clf),\n    ('catboost', catboost_clf),\n    ('lgbm', lgbm_clf),\n    ('xgb', xgb_clf),\n    ('Balance Random Forest', brfc)\n]\n\nmeta_model = LogisticRegression()\n\n# ideal model (stacking)\nideal_model = StackingClassifier(estimators = classifiers, final_estimator = meta_model)\n\n# # Create the pipeline with the Voting or Stacking model\n# ideal_model_pipeline = Pipeline([('Ideal model', ideal_model)])\n\nideal_model_scores = {}\n\n# fit and predictions\nideal_model.fit(X_train, y_train)\ny_pred_train_ideal = ideal_model.predict(X_train)\ny_pred_valid_ideal = ideal_model.predict(X_valid)\n\n# metrics \ntrain_score_ideal = built_in_metrics(y_train, y_pred_train_ideal)\nvalid_score_ideal = built_in_metrics(y_valid, y_pred_valid_ideal)\n\nideal_model_scores[\"Training Scores\"] = train_score_ideal\nideal_model_scores[\"Validation Scores\"] = valid_score_ideal\n\nresults = pd.DataFrame(ideal_model_scores)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"results","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Make predictions on test data","metadata":{}},{"cell_type":"code","source":"test_csv = pd.read_csv(f\"{data_dir}/test.csv\")\ntest_ts = dd.read_parquet(f\"{data_dir}/series_test.parquet\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Aggregate duplicates\ntest_grouped = test_ts.groupby('id').agg({\n    'X': 'mean',\n    'Y': 'mean',\n    'Z': 'mean',\n    'enmo': 'mean',\n    'anglez': 'mean',\n    'non-wear_flag': 'sum', \n    'light': 'mean',\n    'battery_voltage': 'mean',\n    'time_of_day': 'mean',\n    'weekday': 'mean',\n    'quarter': 'mean',\n    'relative_date_PCIAT': 'mean',\n}).compute()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_ts.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_csv.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Merge the aggregated data with the new features based on 'id'\ndata_test = pd.merge(test_csv, test_grouped, how = 'left', on = 'id')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test = data_test.copy()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"missing_columns = set(df_data.columns) - set(df_test.columns)\nmissing_columns","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for col in missing_columns:\n    if col != \"sii\":  \n        if train_data[col].dtype == \"object\": \n            df_test[col] = 0\n        else:  \n            df_test[col] = 0","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Preprocessing test data","metadata":{}},{"cell_type":"code","source":"continuous_test_data = [\"Basic_Demos-Age\", \"CGAS-CGAS_Score\", \"Physical-Diastolic_BP\", \"Physical-HeartRate\", \"Physical-Systolic_BP\",\n                   \"Fitness_Endurance-Max_Stage\", \"Fitness_Endurance-Time_Mins\", \"Fitness_Endurance-Time_Sec\", \"FGC-FGC_CU\",\n                   \"FGC-FGC_PU\", \"PCIAT-PCIAT_Total\", \"SDS-SDS_Total_Raw\", \"SDS-SDS_Total_T\", \"Physical-Waist_Circumference\",\n                   'X', 'Y', 'Z', 'enmo', 'anglez', 'light', 'battery_voltage', 'time_of_day', 'relative_date_PCIAT',\n                  \"Physical-BMI\", \"Physical-Height\", \"Physical-Weight\", \"FGC-FGC_GSND\", \"FGC-FGC_GSD\", \"FGC-FGC_SRL\", \"FGC-FGC_SRR\", \n                   \"FGC-FGC_TL\", \"BIA-BIA_BMC\", \"BIA-BIA_BMI\", \"BIA-BIA_BMR\", \"BIA-BIA_DEE\", \"BIA-BIA_ECW\", \"BIA-BIA_FFM\", \n                   \"BIA-BIA_FFMI\", \"BIA-BIA_FMI\", \"BIA-BIA_Fat\", \"BIA-BIA_ICW\", \"BIA-BIA_LDM\", \"BIA-BIA_LST\", \"BIA-BIA_SMM\", \n                   \"BIA-BIA_TBW\", \"PAQ_A-PAQ_A_Total\", \"PAQ_C-PAQ_C_Total\"]\n\ncategorical_test_data = [\"Basic_Demos-Enroll_Season\", \"Basic_Demos-Sex\", \"CGAS-Season\", \"Physical-Season\", \"Fitness_Endurance-Season\",\n                    \"FGC-Season\", \"FGC-FGC_CU_Zone\", \"FGC-FGC_GSND_Zone\", \"FGC-FGC_GSD_Zone\", \"FGC-FGC_PU_Zone\", \"FGC-FGC_SRL_Zone\",\n                    \"FGC-FGC_SRR_Zone\", \"FGC-FGC_TL_Zone\", \"BIA-Season\", \"BIA-BIA_Activity_Level_num\", \"BIA-BIA_Frame_num\", \"PAQ_A-Season\",\n                    \"PAQ_C-Season\", \"PCIAT-Season\", \"SDS-Season\", \"PreInt_EduHx-Season\", \"PreInt_EduHx-computerinternet_hoursday\", 'weekday', 'quarter', \n                    'non-wear_flag']\n\n# Encode some features from floating to integer as shown in the data dictionary\nfeature_to_encode = [\"CGAS-CGAS_Score\", \"Physical-Waist_Circumference\", \"Physical-Diastolic_BP\", \"Physical-HeartRate\",\n                     \"Physical-Systolic_BP\", \"Fitness_Endurance-Max_Stage\", \"Fitness_Endurance-Time_Mins\",\n                     \"Fitness_Endurance-Time_Sec\", \"FGC-FGC_CU\", \"FGC-FGC_CU_Zone\", \"FGC-FGC_GSND_Zone\", \"FGC-FGC_GSD_Zone\",\n                     \"FGC-FGC_PU\", \"FGC-FGC_PU_Zone\", \"FGC-FGC_SRL_Zone\", \"FGC-FGC_SRR_Zone\", \"FGC-FGC_TL\", \"FGC-FGC_TL_Zone\",\n                     \"BIA-BIA_Activity_Level_num\", \"BIA-BIA_Frame_num\", \"SDS-SDS_Total_Raw\",\n                     \"SDS-SDS_Total_T\", \"PreInt_EduHx-computerinternet_hoursday\", \"non-wear_flag\", \"sii\"]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def data_preprocessing(data_test):\n    \"\"\"\n    Preprocess test data for prediction\n    \"\"\"\n    cat_imputer = SimpleImputer(strategy = 'most_frequent')\n    df_categorical_imputed = cat_imputer.fit_transform(data_test[categorical_test_data])\n    df_categorical_imputed = pd.DataFrame(df_categorical_imputed, columns = categorical_test_data)\n    data_test[categorical_test_data] = df_categorical_imputed\n    \n    # Use median for imputation of selected continuous data\n    for col in continuous_test_data:\n        if col in data_test.columns:\n            new_value_median = data_test[col].median()\n            data_test[col] = data_test[col].fillna(new_value_median)\n\n    # Drop columns\n    data_test.drop([\"id\"], axis = 1, inplace = True)\n\n    # Drop duplicates\n    data_test = data_test.drop_duplicates(keep='first')\n\n    seasons = []\n    \n    # Identify columns that are likely to contain season data\n    for label, value in data_test.items():\n        if pd.api.types.is_object_dtype(value): \n            seasons.append(label)\n    \n    # Map season values only for columns that contain valid season values\n    for col in seasons:\n        if col != \"id\": \n            valid_seasons = data_test[col].isin([\"Spring\", \"Winter\", \"Fall\", \"Summer\"]).any()\n            if valid_seasons:  \n                data_test[col] = data_test[col].map({\"Spring\": 0, \"Winter\": 1, \"Fall\": 2, \"Summer\": 3})\n\n    # Encode some features from floating to integer as shown in the data dictionary\n    for col in feature_to_encode:\n        if col in data_test.columns:\n            data_test[col] = (data_test[col]).astype('int64')\n\n    return data_test","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Pre-process the test data\ndf_test = data_preprocessing(df_test)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Save the feature columns used in training\nfeature_test_columns = [col for col in X.columns if col not in [\"id\", \"sii\"]]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Making sure the same order of columns is kept to avoid error during transformation\ndf_test = df_test[feature_test_columns]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Test Data Shape:\", df_test.shape)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test.info()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Change remaining object to numeric\ndef change_to_numeric(df):\n    # Select columns of type \"object\"\n    object_cols = df.select_dtypes(include=[\"object\"]).columns\n    # Convert object columns to numeric where possible\n    for col in object_cols:\n        df[col] = pd.to_numeric(df[col], errors='coerce')\n    return df\n\n# Apply the function\ndf_test = change_to_numeric(df_test)\n\n# Now you can check the dtypes\nprint(df_test.dtypes)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# After transformation\nprint(\"NaN values in df_test\", np.isnan(df_test).sum())\n\nprint(\"Infinite values in df_test\", np.isinf(df_test).sum())","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Power transformation\ndf_test_transformed = transformer.transform(df_test)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Robust scaler\ndf_test_scaled = scaler.transform(df_test_transformed)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Check for NaNs in numpy arrays\nprint(\"NaN values in train data:\", np.isnan(X_train).sum())\nprint(\"NaN values in test data:\", np.isnan(df_test_scaled).sum())\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"column_names = [f\"feature_{i}\" for i in range(df_test_scaled.shape[1])]\n\n# Convert the NumPy array to a pandas DataFrame\ndf_test_scaled_df = pd.DataFrame(df_test_scaled, columns=column_names)\n\n# Now check for NaN values\nnan_columns = df_test_scaled_df.columns[df_test_scaled_df.isna().any()].tolist()\n\n# Print columns with NaN values\nprint(\"Columns with NaN values:\", nan_columns)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Make predicts on test data\ntest_predictions = ideal_model.predict(df_test_scaled)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_predictions","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Save predictions for Kaggle evaluations\nsubmit = pd.DataFrame({\n    \"id\": data_test[\"id\"],\n    \"sii\": test_predictions\n})\n\nsubmit.to_csv(\"submission.csv\", index = False)\nprint(\"Submission file saved successfully!\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Reference","metadata":{}},{"cell_type":"markdown","source":"Adam Santorelli, Arianna Zuanazzi, Michael Leyden, Logan Lawler, Maggie Devkin, Yuki Kotani, and Gregory Kiar. Child Mind Institute — Problematic Internet Use. https://kaggle.com/competitions/child-mind-institute-problematic-internet-use, 2024. Kaggle.","metadata":{}}]}