Child Mind Institute | Starter | Baseline
","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport polars as pl\nimport pandas as pd\nfrom sklearn.base import clone\nfrom copy import deepcopy\nimport optuna\nfrom scipy.optimize import minimize\n\nimport re\nfrom colorama import Fore, Style\n\nfrom tqdm import tqdm\nfrom IPython.display import clear_output\n\nimport warnings\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None\n\nimport lightgbm as lgb\nfrom catboost import CatBoostRegressor, CatBoostClassifier\nfrom xgboost import XGBRegressor\nfrom sklearn.ensemble import VotingRegressor\nfrom sklearn.model_selection import *\nfrom sklearn.metrics import *\nimport os\n\nSEED = 42\nn_splits = 5","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-09-23T11:58:43.756770Z","iopub.execute_input":"2024-09-23T11:58:43.757553Z","iopub.status.idle":"2024-09-23T11:58:43.770300Z","shell.execute_reply.started":"2024-09-23T11:58:43.757470Z","shell.execute_reply":"2024-09-23T11:58:43.768381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Basic Preprocess
","metadata":{}},{"cell_type":"code","source":"%%time\n\ntrain = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\ndef load_time_series(dirname) -> pd.DataFrame:\n ids = os.listdir(dirname)\n indexes = []\n stats = []\n for idname in tqdm(ids):\n df = pd.read_parquet(os.path.join(dirname, idname, 'part-0.parquet'))\n df.drop('step', axis=1, inplace=True)\n stats.append(df.describe().iloc[1:].values.reshape(-1))\n indexes.append(idname.split('=')[1])\n df = pd.DataFrame(stats, columns=[f\"stat_{i}\" for i in range(len(stats[0]))])\n df['id'] = indexes\n return df\n \ntrain_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\ntime_series_cols = train_ts.columns.tolist()\ntime_series_cols.remove(\"id\")\n\ntrain = pd.merge(train, train_ts, how=\"left\", on='id')\ntest = pd.merge(test, test_ts, how=\"left\", on='id')\n\ntrain = train.drop('id',axis=1)\ntest = test.drop('id',axis=1)\n\nfeaturesCols = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex',\n 'CGAS-Season', 'CGAS-CGAS_Score', 'Physical-Season', 'Physical-BMI',\n 'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n 'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n 'Fitness_Endurance-Season', 'Fitness_Endurance-Max_Stage',\n 'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n 'FGC-Season', 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n 'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n 'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n 'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone', 'BIA-Season',\n 'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n 'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n 'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n 'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n 'BIA-BIA_TBW', 'PAQ_A-Season', 'PAQ_A-PAQ_A_Total', 'PAQ_C-Season',\n 'PAQ_C-PAQ_C_Total', 'SDS-Season', 'SDS-SDS_Total_Raw',\n 'SDS-SDS_Total_T', 'PreInt_EduHx-Season',\n 'PreInt_EduHx-computerinternet_hoursday','sii']\n\nfeaturesCols += time_series_cols\n\ntrain = train[featuresCols]\ntrain = train.dropna(subset='sii')\n\ncat_c = ['Basic_Demos-Enroll_Season','CGAS-Season','Physical-Season','Fitness_Endurance-Season','FGC-Season',\n 'BIA-Season','PAQ_A-Season','PAQ_C-Season','SDS-Season','PreInt_EduHx-Season']\n\ndef update(df):\n global cat_c\n for c in cat_c : \n df[c] = df[c].fillna('Missing')\n df[c] = df[c].astype('category')\n \n return df\n \ntrain = update(train)\ntest = update(test)\n\ndef create_mapping(column, dataset):\n unique_values = dataset[column].unique()\n return {value: idx for idx, value in enumerate(unique_values)}\n\nfor col in cat_c:\n\n mapping = create_mapping(col, train)\n mappingTe = create_mapping(col, test)\n \n train[col] = train[col].replace(mapping).astype(int)\n test[col] = test[col].replace(mappingTe).astype(int)","metadata":{"execution":{"iopub.status.busy":"2024-09-23T11:58:50.780037Z","iopub.execute_input":"2024-09-23T11:58:50.780589Z","iopub.status.idle":"2024-09-23T12:03:17.651229Z","shell.execute_reply.started":"2024-09-23T11:58:50.780543Z","shell.execute_reply":"2024-09-23T12:03:17.650032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-23T12:03:17.653342Z","iopub.execute_input":"2024-09-23T12:03:17.653896Z","iopub.status.idle":"2024-09-23T12:03:17.803079Z","shell.execute_reply.started":"2024-09-23T12:03:17.653829Z","shell.execute_reply":"2024-09-23T12:03:17.801719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-23T12:03:17.804651Z","iopub.execute_input":"2024-09-23T12:03:17.805015Z","iopub.status.idle":"2024-09-23T12:03:17.944882Z","shell.execute_reply.started":"2024-09-23T12:03:17.804969Z","shell.execute_reply":"2024-09-23T12:03:17.943524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# adversarial validation\nif len(test) < len(train):\n sample_weight = np.ones(len(train))\nelse:\n adv_model = lgb.LGBMClassifier(random_state=SEED, verbose=-1, n_estimators=200)\n train_adv = train[[x for x in featuresCols if x != 'sii']].copy()\n test_adv = test[[x for x in featuresCols if x != 'sii']].sample(n=len(train), random_state=SEED)\n train_adv['Is_test'] = 0\n test_adv['Is_test'] = 1\n adv = pd.concat([train_adv, test_adv])\n adv_model.fit(adv.drop('Is_test', axis=1), adv['Is_test'])\n sample_weight = adv_model.predict_proba(train_adv.drop('Is_test', axis=1))[:, 1]","metadata":{"execution":{"iopub.status.busy":"2024-09-23T12:15:11.701954Z","iopub.execute_input":"2024-09-23T12:15:11.702416Z","iopub.status.idle":"2024-09-23T12:15:14.269126Z","shell.execute_reply.started":"2024-09-23T12:15:11.702372Z","shell.execute_reply":"2024-09-23T12:15:14.267769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Modeling | Single LGBM
","metadata":{}},{"cell_type":"code","source":"%%time\n\ndef quadratic_weighted_kappa(y_true, y_pred):\n return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n return np.where(oof_non_rounded < thresholds[0], 0,\n np.where(oof_non_rounded < thresholds[1], 1,\n np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n return -quadratic_weighted_kappa(y_true, rounded_p)\n\ndef TrainML(model_class, test_data, sample_weight, tune=False):\n \n X = train.drop(['sii'], axis=1)\n y = train['sii']\n\n SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n \n train_S = []\n test_S = []\n \n oof_non_rounded = np.zeros(len(y), dtype=float) \n oof_rounded = np.zeros(len(y), dtype=int) \n test_preds = np.zeros((len(test_data), n_splits))\n\n for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n\n model = clone(model_class)\n model.fit(X_train, y_train, sample_weight=sample_weight[train_idx])\n\n y_train_pred = model.predict(X_train)\n y_val_pred = model.predict(X_val)\n\n oof_non_rounded[test_idx] = y_val_pred\n y_val_pred_rounded = y_val_pred.round(0).astype(int)\n oof_rounded[test_idx] = y_val_pred_rounded\n\n train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n train_S.append(train_kappa)\n test_S.append(val_kappa)\n \n test_preds[:, fold] = model.predict(test_data)\n \n print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n clear_output(wait=True)\n\n print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n KappaOPtimizer = minimize(evaluate_predictions,\n x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n method='Nelder-Mead') # Nelder-Mead | # Powell\n assert KappaOPtimizer.success, \"Optimization did not converge.\"\n \n oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n \n if tune:\n return np.mean(test_S)\n\n tpm = test_preds.mean(axis=1)\n tpTuned = threshold_Rounder(tpm, KappaOPtimizer.x)\n \n submission = pd.DataFrame({\n 'id': sample['id'],\n 'sii': tpTuned\n })\n return submission","metadata":{"execution":{"iopub.status.busy":"2024-09-23T12:12:03.217827Z","iopub.execute_input":"2024-09-23T12:12:03.218424Z","iopub.status.idle":"2024-09-23T12:12:03.877586Z","shell.execute_reply.started":"2024-09-23T12:12:03.218373Z","shell.execute_reply":"2024-09-23T12:12:03.876049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time\n\n# # 自定义目标函数 Custom objective functions\n# def objective(trial):\n# # 定义需要调节的超参数空间 Define the hyperparameter space to be adjusted\n# params = {\n# 'learning_rate': trial.suggest_loguniform('learning_rate', 1e-4, 1e-1),\n# 'max_depth': trial.suggest_int('max_depth', 3, 12),\n# 'num_leaves': trial.suggest_int('num_leaves', 31, 512),\n# 'min_data_in_leaf': trial.suggest_int('min_data_in_leaf', 10, 100),\n# 'feature_fraction': trial.suggest_uniform('feature_fraction', 0.5, 1.0),\n# 'bagging_fraction': trial.suggest_uniform('bagging_fraction', 0.5, 1.0),\n# 'bagging_freq': trial.suggest_int('bagging_freq', 1, 7),\n# 'lambda_l1': trial.suggest_loguniform('lambda_l1', 1e-8, 10.0),\n# 'lambda_l2': trial.suggest_loguniform('lambda_l2', 1e-8, 10.0),\n# }\n\n# Light = lgb.LGBMRegressor(**params,random_state=SEED, verbose=-1,n_estimators=200)\n# return TrainML(Light, test, True)\n\n# # 创建一个Optuna Study Create an Optuna Study\n# study = optuna.create_study(direction='maximize') \n# study.optimize(objective, n_trials=100) # 设置进行100次搜索 Set 100 searches\n\n# # 输出最优超参数和最优结果 Output optimal hyperparameters and optimal results\n# print(\"Best trial:\")\n# trial = study.best_trial\n\n# print(f\" {trial.value}\")\n# print(\" Best hyperparameters: \", trial.params)\n\n# # Best trial:\n# # 0.48050601911401003\n# # Best hyperparameters: {'learning_rate': 0.01807927986490293, 'max_depth': 5, 'num_leaves': 309, 'min_data_in_leaf': 87, 'feature_fraction': 0.5258796792372337, 'bagging_fraction': 0.8961353601288653, 'bagging_freq': 3, 'lambda_l1': 0.0012794381687055592, 'lambda_l2': 4.7645510836117515e-06}\n\n# # Best trial:\n# # 0.40180590915921055\n# # Best hyperparameters: {'learning_rate': 0.09056475084257094, 'max_depth': 5, 'num_leaves': 429, 'min_data_in_leaf': 43, 'feature_fraction': 0.8916815865803562, 'bagging_fraction': 0.8276271977726875, 'bagging_freq': 6, 'lambda_l1': 0.07688998999378223, 'lambda_l2': 0.00026663802273190103}\n","metadata":{"execution":{"iopub.status.busy":"2024-09-23T04:27:51.065939Z","iopub.execute_input":"2024-09-23T04:27:51.066415Z","iopub.status.idle":"2024-09-23T04:41:31.419337Z","shell.execute_reply.started":"2024-09-23T04:27:51.066370Z","shell.execute_reply":"2024-09-23T04:41:31.418034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\n# Params = {'learning_rate': 0.07975474666326936, 'max_depth': 10, 'num_leaves': 207, 'min_data_in_leaf': 41,\n# 'feature_fraction': 0.6385678848225935, 'bagging_fraction': 0.9042038292349021, 'bagging_freq': 6, \n# 'lambda_l1': 9.920617415343463, 'lambda_l2': 4.351491475117983} # LB : 0.452\nParams = {'learning_rate': 0.09056475084257094, 'max_depth': 5, 'num_leaves': 429, 'min_data_in_leaf': 43, \n 'feature_fraction': 0.8916815865803562, 'bagging_fraction': 0.8276271977726875, 'bagging_freq': 6, \n 'lambda_l1': 0.07688998999378223, 'lambda_l2': 0.00026663802273190103}\n\n\nLight = lgb.LGBMRegressor(**Params,random_state=SEED, verbose=-1,n_estimators=200)\nSubmission = TrainML(Light,test, sample_weight)","metadata":{"execution":{"iopub.status.busy":"2024-09-23T12:12:21.229909Z","iopub.execute_input":"2024-09-23T12:12:21.230450Z","iopub.status.idle":"2024-09-23T12:12:29.348887Z","shell.execute_reply.started":"2024-09-23T12:12:21.230401Z","shell.execute_reply":"2024-09-23T12:12:29.347045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Submission
","metadata":{}},{"cell_type":"code","source":"%%time\n\nSubmission.to_csv('submission.csv', index=False)\nSubmission.head()\nprint(Submission['sii'].value_counts())","metadata":{"execution":{"iopub.status.busy":"2024-09-22T11:41:52.256728Z","iopub.execute_input":"2024-09-22T11:41:52.257259Z","iopub.status.idle":"2024-09-22T11:41:52.272558Z","shell.execute_reply.started":"2024-09-22T11:41:52.257205Z","shell.execute_reply":"2024-09-22T11:41:52.271248Z"},"trusted":true},"execution_count":null,"outputs":[]}]}