\n Child Mind Institute | NKwon \n
\n","metadata":{}},{"cell_type":"code","source":"# Trần Bình Minh , Khổng Mạnh Tuấn , Lê Quyết Chiến","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T04:53:56.739712Z","iopub.execute_input":"2024-12-20T04:53:56.740149Z","iopub.status.idle":"2024-12-20T04:53:56.744791Z","shell.execute_reply.started":"2024-12-20T04:53:56.740101Z","shell.execute_reply":"2024-12-20T04:53:56.743393Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport polars as pl\nimport pandas as pd\nfrom sklearn.base import clone\nfrom copy import deepcopy\nimport optuna\nfrom scipy.optimize import minimize\nimport os\nfrom scipy.stats import mode\n\nimport re\nfrom colorama import Fore, Style\nimport torch\nimport torch.nn as nn\n\nfrom tqdm import tqdm\nfrom IPython.display import clear_output\nfrom concurrent.futures import ThreadPoolExecutor\n\nimport warnings\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None\n\nimport lightgbm as lgb\nimport xgboost as xgb\nfrom xgboost import XGBRegressor\nfrom sklearn.model_selection import *\nfrom sklearn.metrics import *\n\nseed_list = [42, 2024, 1234]\nn_splits = 5","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T04:53:56.746235Z","iopub.execute_input":"2024-12-20T04:53:56.746597Z","iopub.status.idle":"2024-12-20T04:54:00.763204Z","shell.execute_reply.started":"2024-12-20T04:53:56.746566Z","shell.execute_reply":"2024-12-20T04:54:00.762168Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n Basic Progress\n
\n","metadata":{}},{"cell_type":"code","source":"#Feature Engineering\nclass AutoEncoder(nn.Module):\n def __init__(self, input_dim, encoding_dim):\n super(AutoEncoder, self).__init__()\n self.encoder = nn.Sequential(\n nn.Linear(input_dim, encoding_dim*3),\n nn.ReLU(),\n nn.Linear(encoding_dim*3, encoding_dim*2),\n nn.ReLU(),\n nn.Linear(encoding_dim*2, encoding_dim),\n nn.ReLU()\n )\n self.decoder = nn.Sequential(\n nn.Linear(encoding_dim, input_dim*2),\n nn.ReLU(),\n nn.Linear(input_dim*2, input_dim*3),\n nn.ReLU(),\n nn.Linear(input_dim*3, input_dim),\n nn.Sigmoid()\n )\n \n def forward(self, x):\n encoded = self.encoder(x)\n decoded = self.decoder(encoded)\n return decoded\n\n\ndef perform_autoencoder(df, encoding_dim=50, epochs=50, batch_size=32):\n scaler = StandardScaler()\n df_scaled = scaler.fit_transform(df)\n \n data_tensor = torch.FloatTensor(df_scaled)\n \n input_dim = data_tensor.shape[1]\n autoencoder = AutoEncoder(input_dim, encoding_dim)\n \n criterion = nn.MSELoss()\n optimizer = optim.Adam(autoencoder.parameters())\n \n for epoch in range(epochs):\n for i in range(0, len(data_tensor), batch_size):\n batch = data_tensor[i : i + batch_size]\n optimizer.zero_grad()\n reconstructed = autoencoder(batch)\n loss = criterion(reconstructed, batch)\n loss.backward()\n optimizer.step()\n \n if (epoch + 1) % 10 == 0:\n print(f'Epoch [{epoch + 1}/{epochs}], Loss: {loss.item():.4f}]')\n \n with torch.no_grad():\n encoded_data = autoencoder.encoder(data_tensor).numpy()\n \n df_encoded = pd.DataFrame(encoded_data, columns=[f'Enc_{i + 1}' for i in range(encoded_data.shape[1])])\n \n return df_encoded\n\ndef feature_engineering(df):\n season_cols = [col for col in df.columns if 'Season' in col]\n df = df.drop(season_cols, axis=1) \n df['BMI_Age'] = df['Physical-BMI'] * df['Basic_Demos-Age']\n df['Internet_Hours_Age'] = df['PreInt_EduHx-computerinternet_hoursday'] * df['Basic_Demos-Age']\n df['BMI_Internet_Hours'] = df['Physical-BMI'] * df['PreInt_EduHx-computerinternet_hoursday']\n df['BFP_BMI'] = df['BIA-BIA_Fat'] / df['BIA-BIA_BMI']\n df['FFMI_BFP'] = df['BIA-BIA_FFMI'] / df['BIA-BIA_Fat']\n df['FMI_BFP'] = df['BIA-BIA_FMI'] / df['BIA-BIA_Fat']\n df['LST_TBW'] = df['BIA-BIA_LST'] / df['BIA-BIA_TBW']\n df['BFP_BMR'] = df['BIA-BIA_Fat'] * df['BIA-BIA_BMR']\n df['BFP_DEE'] = df['BIA-BIA_Fat'] * df['BIA-BIA_DEE']\n df['BMR_Weight'] = df['BIA-BIA_BMR'] / df['Physical-Weight']\n df['DEE_Weight'] = df['BIA-BIA_DEE'] / df['Physical-Weight']\n df['SMM_Height'] = df['BIA-BIA_SMM'] / df['Physical-Height']\n df['Muscle_to_Fat'] = df['BIA-BIA_SMM'] / df['BIA-BIA_FMI']\n df['Hydration_Status'] = df['BIA-BIA_TBW'] / df['Physical-Weight']\n df['ICW_TBW'] = df['BIA-BIA_ICW'] / df['BIA-BIA_TBW']\n \n return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T04:54:00.764743Z","iopub.execute_input":"2024-12-20T04:54:00.765404Z","iopub.status.idle":"2024-12-20T04:54:00.779688Z","shell.execute_reply.started":"2024-12-20T04:54:00.765370Z","shell.execute_reply":"2024-12-20T04:54:00.778281Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\n\ndef process_file(filename, dirname):\n df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n df.drop('step', axis=1, inplace=True)\n return df.describe().values.reshape(-1), filename.split('=')[1]\n\ndef load_time_series(dirname) -> pd.DataFrame:\n ids = os.listdir(dirname)\n \n with ThreadPoolExecutor() as executor:\n results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n \n stats, indexes = zip(*results)\n \n df = pd.DataFrame(stats, columns=[f\"Stat_{i}\" for i in range(len(stats[0]))])\n df['id'] = indexes\n \n return df\n\ntrain = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\ntrain_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\ntime_series_cols = train_ts.columns.tolist()\ntime_series_cols.remove(\"id\")\n#train_ts_encoded = perform_autoencoder(train_ts, encoding_dim=60, epochs=100, batch_size=32)\n#test_ts_encoded = perform_autoencoder(test_ts, encoding_dim=60, epochs=100, batch_size=32)\n\n#time_series_cols = train_ts_encoded.columns.tolist()\n#train_ts_encoded[\"id\"]=train_ts[\"id\"]\n#test_ts_encoded['id']=test_ts[\"id\"]\ntrain = pd.merge(train, train_ts, how=\"left\", on='id')\ntest = pd.merge(test, test_ts, how=\"left\", on='id')\n#train = feature_engineering(train)\n#train = train.dropna(thresh=10, axis=0)\n#test = feature_engineering(test)\ntrain = train.drop('id', axis=1)\ntest = test.drop('id', axis=1)\n\nfeaturesCols = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex',\n 'CGAS-Season', 'CGAS-CGAS_Score', 'Physical-Season', 'Physical-BMI',\n 'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n 'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n 'Fitness_Endurance-Season', 'Fitness_Endurance-Max_Stage',\n 'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n 'FGC-Season', 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n 'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n 'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n 'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone', 'BIA-Season',\n 'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n 'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n 'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n 'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n 'BIA-BIA_TBW', 'PAQ_A-Season', 'PAQ_A-PAQ_A_Total', 'PAQ_C-Season',\n 'PAQ_C-PAQ_C_Total', 'SDS-Season', 'SDS-SDS_Total_Raw',\n 'SDS-SDS_Total_T', 'PreInt_EduHx-Season',\n 'PreInt_EduHx-computerinternet_hoursday', 'sii']\n\nfeaturesCols += time_series_cols\n\ntrain = train[featuresCols]\ntrain = train.dropna(subset='sii')\n\ncat_c = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', 'Fitness_Endurance-Season', \n 'FGC-Season', 'BIA-Season', 'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season']\n\ndef update(df):\n for c in cat_c: \n df[c] = df[c].fillna('Missing')\n df[c] = df[c].astype('category')\n return df\n \ntrain = update(train)\ntest = update(test)\n\ndef create_mapping(column, dataset):\n unique_values = dataset[column].unique()\n return {value: idx for idx, value in enumerate(unique_values)}\n\nfor col in cat_c:\n mapping_train = create_mapping(col, train)\n \n train[col] = train[col].replace(mapping_train).astype(int)\n test[col] = test[col].replace(mapping_train).astype(int)\n\nprint(f'Train Shape : {train.shape} || Test Shape : {test.shape}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T04:54:00.781324Z","iopub.execute_input":"2024-12-20T04:54:00.781624Z","iopub.status.idle":"2024-12-20T04:55:24.537689Z","shell.execute_reply.started":"2024-12-20T04:54:00.781597Z","shell.execute_reply":"2024-12-20T04:55:24.536517Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\n\ntrain.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T04:55:24.539320Z","iopub.execute_input":"2024-12-20T04:55:24.539730Z","iopub.status.idle":"2024-12-20T04:55:24.655564Z","shell.execute_reply.started":"2024-12-20T04:55:24.539697Z","shell.execute_reply":"2024-12-20T04:55:24.654311Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\n\ntest.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T04:55:24.656492Z","iopub.execute_input":"2024-12-20T04:55:24.656803Z","iopub.status.idle":"2024-12-20T04:55:24.771412Z","shell.execute_reply.started":"2024-12-20T04:55:24.656776Z","shell.execute_reply":"2024-12-20T04:55:24.770239Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n Modeling\n
\n","metadata":{}},{"cell_type":"code","source":"%%time\n\ndef quadratic_weighted_kappa(y_true, y_pred):\n return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n return np.where(oof_non_rounded < thresholds[0], 0,\n np.where(oof_non_rounded < thresholds[1], 1,\n np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n return -quadratic_weighted_kappa(y_true, rounded_p)\n\nX = train.drop(['sii'], axis=1)\ny = train['sii']\n\ndef TrainML(model_class, test_data, seed_list):\n\n SKF = StratifiedKFold(n_splits=n_splits, shuffle=False,)\n \n train_S = []\n test_S = []\n \n oof_non_rounded = np.zeros(len(y), dtype=float) \n oof_rounded = np.zeros(len(y), dtype=int) \n test_preds = np.zeros((len(test_data), n_splits))\n\n for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n \n random_seed = np.random.choice(seed_list)\n \n X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n\n model = clone(model_class)\n if hasattr(model, 'random_state'):\n model.set_params(random_state=random_seed)\n\n model.fit(X_train, y_train)\n\n y_train_pred = model.predict(X_train)\n y_val_pred = model.predict(X_val)\n\n oof_non_rounded[test_idx] = y_val_pred\n y_val_pred_rounded = y_val_pred.round(0).astype(int)\n oof_rounded[test_idx] = y_val_pred_rounded\n\n train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n train_S.append(train_kappa)\n test_S.append(val_kappa)\n \n test_preds[:, fold] = model.predict(test_data)\n \n print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n clear_output(wait=True)\n\n print(f\"Train : {np.mean(train_S):.4f}\")\n print(f\"Validation : {np.mean(test_S):.4f}\")\n\n KappaOPtimizer = minimize(evaluate_predictions,\n x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n method='Nelder-Mead') \n assert KappaOPtimizer.success, \"Optimization did not converge.\"\n \n oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n print(f\"----> || Optimized : {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n tpm = test_preds.mean(axis=1)\n tpTuned = threshold_Rounder(tpm, KappaOPtimizer.x)\n \n submission = pd.DataFrame({\n 'id': sample['id'],\n 'sii': tpTuned\n })\n\n return submission","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T04:55:24.774126Z","iopub.execute_input":"2024-12-20T04:55:24.774449Z","iopub.status.idle":"2024-12-20T04:55:24.787065Z","shell.execute_reply.started":"2024-12-20T04:55:24.774419Z","shell.execute_reply":"2024-12-20T04:55:24.786119Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\nParams_XGB = {\n 'learning_rate': 0.046, \n 'max_depth': 12, \n 'min_child_weight': 13, \n 'colsample_bytree': 0.893, \n 'subsample': 0.784, \n 'reg_alpha': 6.596, \n 'reg_lambda': 2.680e-06, \n 'n_estimators': 200,\n}\n\nXGB_Model = xgb.XGBRegressor(**Params_XGB)\nSubmission = TrainML(XGB_Model,test,seed_list)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T04:55:24.788280Z","iopub.execute_input":"2024-12-20T04:55:24.788583Z","iopub.status.idle":"2024-12-20T04:56:00.340118Z","shell.execute_reply.started":"2024-12-20T04:55:24.788557Z","shell.execute_reply":"2024-12-20T04:56:00.339048Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\n\nLatestParams = {'learning_rate': 0.03755757104848504, 'max_depth': 12, 'num_leaves': 18, 'min_data_in_leaf': 3, \n 'feature_fraction': 0.723690362968002, 'bagging_fraction': 0.688232590484764, 'bagging_freq': 5,\n 'lambda_l1': 0.18512987285245963, 'lambda_l2': 0.18435628737334625}\n\nLight = lgb.LGBMRegressor(**LatestParams, verbose=-1, n_estimators=200)\n\nSubmission1 = TrainML(Light,test,seed_list)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T04:56:00.341056Z","iopub.execute_input":"2024-12-20T04:56:00.341343Z","iopub.status.idle":"2024-12-20T04:56:08.921248Z","shell.execute_reply.started":"2024-12-20T04:56:00.341317Z","shell.execute_reply":"2024-12-20T04:56:08.920194Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n Submission\n
\n","metadata":{}},{"cell_type":"code","source":"#print(Submission1['sii'].value_counts())\n#Submission1.to_csv('submission.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T04:56:08.922266Z","iopub.execute_input":"2024-12-20T04:56:08.922539Z","iopub.status.idle":"2024-12-20T04:56:08.926489Z","shell.execute_reply.started":"2024-12-20T04:56:08.922514Z","shell.execute_reply":"2024-12-20T04:56:08.925251Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(Submission['sii'].value_counts())\nSubmission.to_csv('submission.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T04:59:32.544576Z","iopub.execute_input":"2024-12-20T04:59:32.544952Z","iopub.status.idle":"2024-12-20T04:59:32.555257Z","shell.execute_reply.started":"2024-12-20T04:59:32.544923Z","shell.execute_reply":"2024-12-20T04:59:32.554078Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#sub1 = Submission1\n#sub = Submission\n\n#sub1 = sub1.sort_values(by='id').reset_index(drop=True)\n#sub = sub.sort_values(by='id').reset_index(drop=True)\n\n#combined = pd.DataFrame({\n #'id': sub1['id'],\n #'sii_1': sub1['sii'],\n #'sii': sub['sii'],\n#})\n\n#def majority_vote(row):\n #return row.mode()[0]\n\n#combined['final_sii'] = combined[['sii_1', 'sii_2']].apply(majority_vote, axis=1)\n\n#final_submission = combined[['id', 'final_sii']].rename(columns={'final_sii': 'sii'})\n\n#final_submission.to_csv('submission.csv', index=False)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}