˚LightGBM Modeling with Optuna: A Focus on Internet Use˚
","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"#Import Dependencies
","metadata":{}},{"cell_type":"code","source":"import pandas as pd \nimport optuna\nimport numpy as np \nimport matplotlib.pyplot as plt\nimport plotly.express as px\nimport seaborn as sns \nimport re\nimport math\nfrom io import StringIO\nfrom colorama import Fore, Style, init;\nfrom IPython.display import display, HTML\nfrom scipy.stats import skew \nimport plotly.graph_objects as go\nfrom plotly.subplots import make_subplots\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.ensemble import VotingClassifier, VotingRegressor\nfrom sklearn.model_selection import KFold, RepeatedStratifiedKFold, cross_val_score\nfrom sklearn.model_selection import RandomizedSearchCV\nfrom sklearn.metrics import cohen_kappa_score\nfrom tqdm import tqdm\nfrom functools import partial\nfrom lightgbm import LGBMRegressor\nfrom sklearn.preprocessing import LabelEncoder, MinMaxScaler , StandardScaler , QuantileTransformer, PowerTransformer\nimport xgboost as xgb\nimport lightgbm as lgb\nfrom sklearn.model_selection import cross_val_score, StratifiedKFold\nfrom catboost import CatBoostClassifier\nfrom sklearn.metrics import *\nfrom sklearn.svm import SVC\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.naive_bayes import MultinomialNB\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, ExtraTreesClassifier\nfrom xgboost import XGBClassifier\npd.set_option('display.max_columns', None)\n","metadata":{"execution":{"iopub.status.busy":"2024-09-21T18:11:44.516177Z","iopub.execute_input":"2024-09-21T18:11:44.516890Z","iopub.status.idle":"2024-09-21T18:11:50.145578Z","shell.execute_reply.started":"2024-09-21T18:11:44.516849Z","shell.execute_reply":"2024-09-21T18:11:50.144681Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Data Loading and Preprocessing
","metadata":{}},{"cell_type":"code","source":"# Load training and testing datasets\ntr_ds = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\nte_ds = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsubmission = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\n# Select specific feature columns for training\nfeatures = ['id', 'Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex', \n 'CGAS-Season', 'CGAS-CGAS_Score', 'Physical-Season', 'Physical-BMI',\n 'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n 'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n 'Fitness_Endurance-Season', 'Fitness_Endurance-Max_Stage', \n 'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec', \n 'FGC-Season', 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND', \n 'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU', \n 'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR', \n 'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone', 'BIA-Season',\n 'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n 'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM', \n 'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num', \n 'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM', \n 'BIA-BIA_TBW', 'PAQ_A-Season', 'PAQ_A-PAQ_A_Total', 'PAQ_C-Season',\n 'PAQ_C-PAQ_C_Total', 'SDS-Season', 'SDS-SDS_Total_Raw', \n 'SDS-SDS_Total_T', 'PreInt_EduHx-Season', \n 'PreInt_EduHx-computerinternet_hoursday', 'sii']\n\n# Select only the required columns for the train dataset\ntr_ds = tr_ds[features]\n\n# Remove rows from 'train' where 'sii' has missing values\ntr_ds = tr_ds.dropna(subset='sii')\n\n# Function to update dataframe by handling categorical columns and filling missing values\ndef update(df):\n # List of categorical columns\n cat_c = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', \n 'Fitness_Endurance-Season', 'FGC-Season', 'BIA-Season', \n 'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season']\n \n # List of columns to be treated as IDs\n id_col = ['id']\n \n # Convert 'id' column to 'category' type\n for c in id_col:\n df[c] = df[c].astype('category')\n \n # For each categorical column, fill missing values with 'Missing' and convert to 'category' type\n for c in cat_c:\n df[c] = df[c].fillna('Missing')\n df[c] = df[c].astype('category')\n \n return df\n\n# Update both training and testing datasets\ntr_ds = update(tr_ds)\nte_ds = update(te_ds)\n\n# Handling missing values\n# Drop columns with more than 70% missing values\ndef drop_high_missing_columns(df, threshold=0.7):\n # Calculate the percentage of missing values for each column\n missing_percent = df.isnull().mean()\n \n # Drop columns where missing values exceed the threshold (70%)\n df = df.drop(columns=missing_percent[missing_percent > threshold].index)\n return df\n\n# Impute missing values\ndef impute_missing_values(df):\n # For numeric columns, fill missing values with the column mean\n num_cols = df.select_dtypes(include=['float64', 'int64']).columns\n for col in num_cols:\n df[col] = df[col].fillna(df[col].mean())\n \n # For categorical columns, fill missing values with the most frequent value (mode)\n cat_cols = df.select_dtypes(include=['category']).columns\n for col in cat_cols:\n df[col] = df[col].fillna(df[col].mode()[0])\n \n return df\n\n# First drop columns with more than 70% missing values\ntr_ds = drop_high_missing_columns(tr_ds)\nte_ds = drop_high_missing_columns(te_ds)\n\n# Then impute missing values\ntr_ds = impute_missing_values(tr_ds)\nte_ds = impute_missing_values(te_ds)\n","metadata":{"execution":{"iopub.status.busy":"2024-09-21T18:11:58.684274Z","iopub.execute_input":"2024-09-21T18:11:58.685029Z","iopub.status.idle":"2024-09-21T18:11:58.876452Z","shell.execute_reply.started":"2024-09-21T18:11:58.684990Z","shell.execute_reply":"2024-09-21T18:11:58.875347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Basic Overview
","metadata":{}},{"cell_type":"code","source":"tr_ds.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-21T18:12:04.865789Z","iopub.execute_input":"2024-09-21T18:12:04.866457Z","iopub.status.idle":"2024-09-21T18:12:04.925087Z","shell.execute_reply.started":"2024-09-21T18:12:04.866416Z","shell.execute_reply":"2024-09-21T18:12:04.924101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"te_ds.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-21T18:12:24.778339Z","iopub.execute_input":"2024-09-21T18:12:24.778987Z","iopub.status.idle":"2024-09-21T18:12:24.828686Z","shell.execute_reply.started":"2024-09-21T18:12:24.778947Z","shell.execute_reply":"2024-09-21T18:12:24.827556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tr_ds.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2024-09-21T18:12:56.147599Z","iopub.execute_input":"2024-09-21T18:12:56.147976Z","iopub.status.idle":"2024-09-21T18:12:56.161881Z","shell.execute_reply.started":"2024-09-21T18:12:56.147939Z","shell.execute_reply":"2024-09-21T18:12:56.160890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"te_ds.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2024-09-21T18:12:33.127271Z","iopub.execute_input":"2024-09-21T18:12:33.127968Z","iopub.status.idle":"2024-09-21T18:12:33.139349Z","shell.execute_reply.started":"2024-09-21T18:12:33.127927Z","shell.execute_reply":"2024-09-21T18:12:33.138323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Model Building
","metadata":{}},{"cell_type":"code","source":"# Custom QWK metric for LightGBM\ndef quadratic_weighted_kappa(y_true, y_pred):\n return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\n# Optimizer to find the best thresholds for rounding\nclass OptimizedRounder:\n def __init__(self, n_classes, n_trials=100):\n self.n_classes = n_classes\n self.n_trials = n_trials\n self.metric = partial(cohen_kappa_score, weights=\"quadratic\")\n\n def fit(self, y_pred, y_true):\n def objective(trial):\n thresholds = []\n for i in range(self.n_classes - 1):\n th = trial.suggest_float(f\"threshold_{i}\", 0, 1)\n thresholds.append(th)\n y_pred_rounded = np.digitize(y_pred, thresholds)\n return self.metric(y_true, y_pred_rounded)\n\n study = optuna.create_study(direction=\"maximize\")\n study.optimize(objective, n_trials=self.n_trials)\n self.thresholds = [study.best_params[f\"threshold_{i}\"] for i in range(self.n_classes - 1)]\n\n def predict(self, y_pred):\n return np.digitize(y_pred, self.thresholds)\n\n# Model training with cross-validation and hyperparameter tuning\ndef train_model(X, y, cat_features, test_data, n_splits=5):\n skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=52)\n all_test_preds = []\n train_kappa_scores = []\n test_kappa_scores = []\n\n # Base parameters for LightGBM\n base_params = {\n 'objective': 'regression',\n 'metric': 'rmse',\n 'n_estimators': 1000,\n 'early_stopping_rounds': 50,\n }\n\n # Optuna hyperparameter optimization\n def objective(trial):\n params = {\n 'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.2),\n 'num_leaves': trial.suggest_int('num_leaves', 20, 150),\n 'max_depth': trial.suggest_int('max_depth', -1, 15),\n 'min_data_in_leaf': trial.suggest_int('min_data_in_leaf', 1, 50),\n 'feature_fraction': trial.suggest_float('feature_fraction', 0.6, 1.0),\n 'bagging_fraction': trial.suggest_float('bagging_fraction', 0.6, 1.0),\n 'bagging_freq': trial.suggest_int('bagging_freq', 1, 10),\n }\n fold_kappa_scores = []\n\n for fold, (train_idx, test_idx) in enumerate(skf.split(X, y)):\n X_train, X_test = X.iloc[train_idx], X.iloc[test_idx]\n y_train, y_test = y.iloc[train_idx], y.iloc[test_idx]\n\n train_data = lgb.Dataset(X_train, label=y_train, categorical_feature=cat_features)\n valid_data = lgb.Dataset(X_test, label=y_test, categorical_feature=cat_features, reference=train_data)\n\n model = lgb.train({**base_params, **params}, train_data, valid_sets=[valid_data])\n\n y_test_pred = model.predict(X_test)\n fold_kappa = quadratic_weighted_kappa(y_test, np.round(y_test_pred))\n fold_kappa_scores.append(fold_kappa)\n\n return np.mean(fold_kappa_scores)\n\n study = optuna.create_study(direction=\"maximize\")\n study.optimize(objective, n_trials=20) # You can increase n_trials for better results\n\n best_params = study.best_params\n print(\"Best hyperparameters:\", best_params)\n\n for fold, (train_idx, test_idx) in enumerate(tqdm(skf.split(X, y), desc=\"Training Folds\", total=n_splits)):\n X_train, X_test = X.iloc[train_idx], X.iloc[test_idx]\n y_train, y_test = y.iloc[train_idx], y.iloc[test_idx]\n\n train_data = lgb.Dataset(X_train, label=y_train, categorical_feature=cat_features)\n valid_data = lgb.Dataset(X_test, label=y_test, categorical_feature=cat_features, reference=train_data)\n\n model = lgb.train({**base_params, **best_params}, train_data, valid_sets=[valid_data])\n\n y_train_pred = model.predict(X_train)\n y_test_pred = model.predict(X_test)\n\n all_test_preds.append(model.predict(test_data.reindex(columns=X_train.columns, fill_value=0)))\n\n train_kappa = quadratic_weighted_kappa(y_train, np.round(y_train_pred))\n test_kappa = quadratic_weighted_kappa(y_test, np.round(y_test_pred))\n\n train_kappa_scores.append(train_kappa)\n test_kappa_scores.append(test_kappa)\n\n print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Test QWK: {test_kappa:.4f}\")\n\n print(\"\\n--- Final Mean Scores ---\")\n print(f\"Mean Train QWK: {np.mean(train_kappa_scores):.4f}\")\n print(f\"Mean Test QWK: {np.mean(test_kappa_scores):.4f}\")\n\n mean_test_preds = np.mean(all_test_preds, axis=0)\n\n return np.round(mean_test_preds).astype(int)\n\n# Prepare data\nX = tr_ds.drop(['sii'], axis=1)\ny = tr_ds['sii']\n\n# List of categorical feature column names\ncat_features = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', \n 'Fitness_Endurance-Season', 'FGC-Season', 'BIA-Season', \n 'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season']\n\n# Train the model and generate predictions for the test set\ntest_predictions = train_model(X, y, cat_features, te_ds)","metadata":{"execution":{"iopub.status.busy":"2024-09-21T18:13:12.589901Z","iopub.execute_input":"2024-09-21T18:13:12.590733Z","iopub.status.idle":"2024-09-21T18:13:31.101296Z","shell.execute_reply.started":"2024-09-21T18:13:12.590690Z","shell.execute_reply":"2024-09-21T18:13:31.100532Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Submission
","metadata":{}},{"cell_type":"code","source":"# Create a submission DataFrame\nsub = pd.DataFrame({\n 'id': submission['id'],\n 'sii': test_predictions\n})\n\nsub.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-09-21T18:13:43.202642Z","iopub.execute_input":"2024-09-21T18:13:43.203034Z","iopub.status.idle":"2024-09-21T18:13:43.211701Z","shell.execute_reply.started":"2024-09-21T18:13:43.202997Z","shell.execute_reply":"2024-09-21T18:13:43.210562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Thank You
","metadata":{}}]}