{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Import libraries\nimport optuna\nimport pandas as pd\nimport numpy as np\nfrom sklearn import preprocessing\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.metrics import mean_squared_error\nfrom concurrent.futures import ThreadPoolExecutor\nfrom tqdm import tqdm\nfrom scipy.optimize import minimize\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        \nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.ensemble import StackingRegressor\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.linear_model import ElasticNet\n# from sklearn.linear_model import LinearRegression","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-10-09T14:12:46.517435Z","iopub.execute_input":"2024-10-09T14:12:46.518017Z","iopub.status.idle":"2024-10-09T14:12:47.399888Z","shell.execute_reply.started":"2024-10-09T14:12:46.517972Z","shell.execute_reply":"2024-10-09T14:12:47.398521Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_columns', 1000)","metadata":{"execution":{"iopub.status.busy":"2024-10-09T14:12:47.402191Z","iopub.execute_input":"2024-10-09T14:12:47.402589Z","iopub.status.idle":"2024-10-09T14:12:47.410922Z","shell.execute_reply.started":"2024-10-09T14:12:47.402547Z","shell.execute_reply":"2024-10-09T14:12:47.409773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load data\ntrain_df = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest_df = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')","metadata":{"execution":{"iopub.status.busy":"2024-10-09T14:12:47.412539Z","iopub.execute_input":"2024-10-09T14:12:47.413067Z","iopub.status.idle":"2024-10-09T14:12:47.471486Z","shell.execute_reply.started":"2024-10-09T14:12:47.413012Z","shell.execute_reply":"2024-10-09T14:12:47.470421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Shape of the data:\nprint(\"train_df :\", train_df.shape)\nprint(\"test_df :\", test_df.shape)","metadata":{"execution":{"iopub.status.busy":"2024-10-09T14:12:47.474182Z","iopub.execute_input":"2024-10-09T14:12:47.474565Z","iopub.status.idle":"2024-10-09T14:12:47.481223Z","shell.execute_reply.started":"2024-10-09T14:12:47.474525Z","shell.execute_reply":"2024-10-09T14:12:47.479876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Custom functions\ndef process_file(filename, dirname):\n    data = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    data.drop('step', axis=1, inplace=True)\n    return data.describe().values.reshape(-1), filename.split('=')[1]\n\ndef load_time_series(dirname):\n    ids = os.listdir(dirname)\n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n    stats, indexes = zip(*results)\n    data = pd.DataFrame(stats, columns=[f\"stat_{i}\" for i in range(len(stats[0]))])\n    data['id'] = indexes\n    return data","metadata":{"execution":{"iopub.status.busy":"2024-10-09T14:12:47.482357Z","iopub.execute_input":"2024-10-09T14:12:47.482759Z","iopub.status.idle":"2024-10-09T14:12:47.495661Z","shell.execute_reply.started":"2024-10-09T14:12:47.482711Z","shell.execute_reply":"2024-10-09T14:12:47.494295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load time series data\ntrain_parquet = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_parquet = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")","metadata":{"execution":{"iopub.status.busy":"2024-10-09T14:12:47.497269Z","iopub.execute_input":"2024-10-09T14:12:47.497783Z","iopub.status.idle":"2024-10-09T14:15:10.261727Z","shell.execute_reply.started":"2024-10-09T14:12:47.497721Z","shell.execute_reply":"2024-10-09T14:15:10.260682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Merge and preprocess data\ntrain_df = pd.merge(train_df, train_parquet, how=\"left\", on='id')\ntest_df = pd.merge(test_df, test_parquet, how=\"left\", on='id')","metadata":{"execution":{"iopub.status.busy":"2024-10-09T14:15:10.263221Z","iopub.execute_input":"2024-10-09T14:15:10.263735Z","iopub.status.idle":"2024-10-09T14:15:10.290978Z","shell.execute_reply.started":"2024-10-09T14:15:10.263677Z","shell.execute_reply":"2024-10-09T14:15:10.289709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_features = test_df.drop(columns=['id']).columns\ntest_id = test_df['id']\ntrain_df = train_df.dropna(subset='sii')","metadata":{"execution":{"iopub.status.busy":"2024-10-09T14:15:10.292907Z","iopub.execute_input":"2024-10-09T14:15:10.293402Z","iopub.status.idle":"2024-10-09T14:15:10.306833Z","shell.execute_reply.started":"2024-10-09T14:15:10.293347Z","shell.execute_reply":"2024-10-09T14:15:10.305536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"code","source":"# check missing values\ntrain_missing_values = train_df[base_features].isnull().sum().sort_values(ascending=False)\ntest_missing_values = test_df.isnull().sum().sort_values(ascending=False)","metadata":{"execution":{"iopub.status.busy":"2024-10-09T14:15:10.309102Z","iopub.execute_input":"2024-10-09T14:15:10.309594Z","iopub.status.idle":"2024-10-09T14:15:10.325658Z","shell.execute_reply.started":"2024-10-09T14:15:10.309538Z","shell.execute_reply":"2024-10-09T14:15:10.324513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_missing_values)","metadata":{"execution":{"iopub.status.busy":"2024-10-09T14:15:10.331419Z","iopub.execute_input":"2024-10-09T14:15:10.331864Z","iopub.status.idle":"2024-10-09T14:15:10.339073Z","shell.execute_reply.started":"2024-10-09T14:15:10.331821Z","shell.execute_reply":"2024-10-09T14:15:10.337906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Remove columns with too many misssing values\ntrain_missing_ratio = train_df[base_features].isnull().mean()\n\n# Remove columns with more than 50% missing values\nthreshold = 0.5\nbase_features = train_missing_ratio[train_missing_ratio < threshold].index","metadata":{"execution":{"iopub.status.busy":"2024-10-09T14:15:10.340313Z","iopub.execute_input":"2024-10-09T14:15:10.34081Z","iopub.status.idle":"2024-10-09T14:15:10.359808Z","shell.execute_reply.started":"2024-10-09T14:15:10.340756Z","shell.execute_reply":"2024-10-09T14:15:10.358458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = pd.concat([train_df[base_features], train_df['sii']], axis=1)\ntest_data = test_df[base_features]","metadata":{"execution":{"iopub.status.busy":"2024-10-09T14:15:10.361257Z","iopub.execute_input":"2024-10-09T14:15:10.361649Z","iopub.status.idle":"2024-10-09T14:15:10.37278Z","shell.execute_reply.started":"2024-10-09T14:15:10.361587Z","shell.execute_reply":"2024-10-09T14:15:10.37154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# combine train data and test data\nall_df = pd.concat([train_data, test_data], sort=False).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-10-09T14:15:10.37503Z","iopub.execute_input":"2024-10-09T14:15:10.375541Z","iopub.status.idle":"2024-10-09T14:15:10.392486Z","shell.execute_reply.started":"2024-10-09T14:15:10.375486Z","shell.execute_reply":"2024-10-09T14:15:10.391244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create new features\n# all_df['BMI_Age_sum'] = all_df['Physical-BMI'] + all_df['Basic_Demos-Age']\n# all_df['BMI_Age_mult'] = all_df['Physical-BMI'] * all_df['Basic_Demos-Age']\n# all_df['Physical_Mean'] = all_df[['Physical-Height', 'Physical-Weight', 'Physical-BMI']].mean(axis=1)\n# all_df['Physical_Std'] = all_df[['Physical-Height', 'Physical-Weight', 'Physical-BMI']].std(axis=1)\n# all_df['BMI_Rank'] = all_df['Physical-BMI'].rank()\n# all_df['HeartRate_Rank'] = all_df['Physical-HeartRate'].rank(ascending=False)","metadata":{"execution":{"iopub.status.busy":"2024-10-09T14:15:10.394516Z","iopub.execute_input":"2024-10-09T14:15:10.395284Z","iopub.status.idle":"2024-10-09T14:15:10.400988Z","shell.execute_reply.started":"2024-10-09T14:15:10.395228Z","shell.execute_reply":"2024-10-09T14:15:10.39943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"categorical_features = all_df.select_dtypes(include=['object']).columns.tolist()\nnumerical_features = all_df.select_dtypes(include=['float64', 'int64']).drop(columns=['sii']).columns.tolist()\nall_df[categorical_features] = all_df[categorical_features].fillna('missing')\nall_df[numerical_features] = all_df[numerical_features].fillna(all_df[numerical_features].median())","metadata":{"execution":{"iopub.status.busy":"2024-10-09T14:15:10.402792Z","iopub.execute_input":"2024-10-09T14:15:10.403521Z","iopub.status.idle":"2024-10-09T14:15:10.449715Z","shell.execute_reply.started":"2024-10-09T14:15:10.40345Z","shell.execute_reply":"2024-10-09T14:15:10.448404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Handling missing data - Categorical data\nle = preprocessing.LabelEncoder()\nfor col in categorical_features:\n    all_df[col] = le.fit_transform(all_df[col]).astype(int)\n\n# Handling missing data - numeric data \nscaler = StandardScaler()\nall_df[numerical_features] = scaler.fit_transform(all_df[numerical_features])","metadata":{"execution":{"iopub.status.busy":"2024-10-09T14:15:10.451089Z","iopub.execute_input":"2024-10-09T14:15:10.45145Z","iopub.status.idle":"2024-10-09T14:15:10.490885Z","shell.execute_reply.started":"2024-10-09T14:15:10.451412Z","shell.execute_reply":"2024-10-09T14:15:10.489698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-10-09T14:15:10.49291Z","iopub.execute_input":"2024-10-09T14:15:10.493388Z","iopub.status.idle":"2024-10-09T14:15:10.543875Z","shell.execute_reply.started":"2024-10-09T14:15:10.493332Z","shell.execute_reply":"2024-10-09T14:15:10.54238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df_le = all_df[~all_df['sii'].isnull()]\ntest_df_le = all_df[all_df['sii'].isnull()].drop(columns=['sii'])","metadata":{"execution":{"iopub.status.busy":"2024-10-09T14:15:10.545605Z","iopub.execute_input":"2024-10-09T14:15:10.546082Z","iopub.status.idle":"2024-10-09T14:15:10.561294Z","shell.execute_reply.started":"2024-10-09T14:15:10.54603Z","shell.execute_reply":"2024-10-09T14:15:10.56037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# select the features with the strongest correlation with 'sii'\n# correlation_matrix = train_df_le.corr()\n# target_corr = correlation_matrix['sii'].drop('sii')\n\n# threshold = 0.01\n# high_corr_features = target_corr[abs(target_corr) > threshold].index.tolist()","metadata":{"execution":{"iopub.status.busy":"2024-10-09T14:15:10.562991Z","iopub.execute_input":"2024-10-09T14:15:10.563722Z","iopub.status.idle":"2024-10-09T14:15:10.568408Z","shell.execute_reply.started":"2024-10-09T14:15:10.563671Z","shell.execute_reply":"2024-10-09T14:15:10.56746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df_le =  pd.concat([train_df_le[high_corr_features], train_df_le['sii']], axis=1)\n# test_df_le = test_df_le[high_corr_features]","metadata":{"execution":{"iopub.status.busy":"2024-10-09T14:15:10.570303Z","iopub.execute_input":"2024-10-09T14:15:10.571151Z","iopub.status.idle":"2024-10-09T14:15:10.579186Z","shell.execute_reply.started":"2024-10-09T14:15:10.57111Z","shell.execute_reply":"2024-10-09T14:15:10.57783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Modeling","metadata":{}},{"cell_type":"code","source":"X = train_df_le.drop(columns=['sii'])\ny = train_df_le['sii']","metadata":{"execution":{"iopub.status.busy":"2024-10-09T14:15:10.581191Z","iopub.execute_input":"2024-10-09T14:15:10.58167Z","iopub.status.idle":"2024-10-09T14:15:10.593949Z","shell.execute_reply.started":"2024-10-09T14:15:10.581597Z","shell.execute_reply":"2024-10-09T14:15:10.592776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# SEED = 42\n# X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2024-10-09T14:15:10.596013Z","iopub.execute_input":"2024-10-09T14:15:10.596529Z","iopub.status.idle":"2024-10-09T14:15:10.606052Z","shell.execute_reply.started":"2024-10-09T14:15:10.596463Z","shell.execute_reply":"2024-10-09T14:15:10.604849Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# LGBM\n# def lgb_objective(trial):\n#     param = {\n#         'objective': 'regression',\n#         'metric': 'rmse',\n#         'boosting_type': 'gbdt',\n#         'max_depth': trial.suggest_int('max_depth', 3, 15),\n#         'num_leaves': trial.suggest_int('num_leaves', 20, 300),\n#         'learning_rate': trial.suggest_loguniform('learning_rate', 1e-4, 1e-1),\n#         'feature_fraction': trial.suggest_uniform('feature_fraction', 0.4, 1.0),\n#         'bagging_fraction': trial.suggest_uniform('bagging_fraction', 0.4, 1.0),\n#         'lambda_l1': trial.suggest_loguniform('lambda_l1', 1e-8, 10.0),\n#         'lambda_l2': trial.suggest_loguniform('lambda_l2', 1e-8, 10.0),\n#         'random_state': SEED\n#     }\n#     model = LGBMRegressor(**param)\n#     model.fit(X_train, y_train)\n#     preds = model.predict(X_val)\n#     rmse = mean_squared_error(y_val, preds, squared=False)\n#     return rmse\n\n# study = optuna.create_study(direction='minimize')\n# study.optimize(lgb_objective, n_trials=50)\n# LGBM_Params = study.best_params","metadata":{"execution":{"iopub.status.busy":"2024-10-09T14:15:10.607752Z","iopub.execute_input":"2024-10-09T14:15:10.608496Z","iopub.status.idle":"2024-10-09T14:15:10.619023Z","shell.execute_reply.started":"2024-10-09T14:15:10.608453Z","shell.execute_reply":"2024-10-09T14:15:10.617852Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# XGB\n# def xgb_objective(trial):\n#     param = {\n#         'objective': 'reg:squarederror',\n#         'max_depth': trial.suggest_int('max_depth', 3, 15),\n#         'learning_rate': trial.suggest_loguniform('learning_rate', 1e-4, 1e-1),\n#         'subsample': trial.suggest_uniform('subsample', 0.5, 1.0),\n#         'colsample_bytree': trial.suggest_uniform('colsample_bytree', 0.5, 1.0),\n#         'alpha': trial.suggest_loguniform('alpha', 1e-8, 10.0),\n#         'lambda': trial.suggest_loguniform('lambda', 1e-8, 10.0),\n#         'random_state': SEED\n#     }\n#     model = XGBRegressor(**param)\n#     model.fit(X_train, y_train)\n#     preds = model.predict(X_val)\n#     rmse = mean_squared_error(y_val, preds, squared=False)\n#     return rmse\n\n# study = optuna.create_study(direction='minimize')\n# study.optimize(xgb_objective, n_trials=50)\n# XGB_Params = study.best_params","metadata":{"execution":{"iopub.status.busy":"2024-10-09T14:15:10.620648Z","iopub.execute_input":"2024-10-09T14:15:10.621057Z","iopub.status.idle":"2024-10-09T14:15:10.630129Z","shell.execute_reply.started":"2024-10-09T14:15:10.621015Z","shell.execute_reply":"2024-10-09T14:15:10.628924Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# CatBoost\n# def cat_objective(trial):\n#     param = {\n#         'depth': trial.suggest_int('depth', 4, 10),\n#         'learning_rate': trial.suggest_loguniform('learning_rate', 1e-4, 1e-1),\n#         'l2_leaf_reg': trial.suggest_loguniform('l2_leaf_reg', 1e-5, 10.0),\n#         'bagging_temperature': trial.suggest_uniform('bagging_temperature', 0, 1),\n#         'border_count': trial.suggest_int('border_count', 32, 255),\n#         'random_state': SEED\n#     }\n#     model = CatBoostRegressor(**param, silent=True)\n#     model.fit(X_train, y_train)\n#     preds = model.predict(X_val)\n#     rmse = mean_squared_error(y_val, preds, squared=False)\n#     return rmse\n\n# study = optuna.create_study(direction='minimize')\n# study.optimize(cat_objective, n_trials=20)\n# CatBoost_Params = study.best_params","metadata":{"execution":{"iopub.status.busy":"2024-10-09T14:15:10.631859Z","iopub.execute_input":"2024-10-09T14:15:10.632242Z","iopub.status.idle":"2024-10-09T14:15:10.645236Z","shell.execute_reply.started":"2024-10-09T14:15:10.632201Z","shell.execute_reply":"2024-10-09T14:15:10.643994Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SEED = 42\nLGBM_Params = {'learning_rate': 0.011747572224219955, 'n_estimators': 993,  'min_child_weight': 0.0025036281384857462, 'colsample_bytree': 0.6648896193058901, 'reg_alpha': 0.7153672744430527, 'reg_lambda': 0.12158717311465662}\nXGB_Params = {'learning_rate': 0.007356059931165658, 'n_estimators': 957, 'subsample': 0.6555266544650088, 'colsample_bytree': 0.7712019245727745}\nCatBoost_Params = {'iterations': 804, 'learning_rate': 0.007849710402582562, 'l2_leaf_reg': 7.31183636902306, 'subsample': 0.5630297785016092, 'random_strength': 1.7097065892440113, 'bagging_temperature': 0.026593521316435192, 'border_count': 12}","metadata":{"execution":{"iopub.status.busy":"2024-10-09T14:15:10.647404Z","iopub.execute_input":"2024-10-09T14:15:10.647924Z","iopub.status.idle":"2024-10-09T14:15:10.658031Z","shell.execute_reply.started":"2024-10-09T14:15:10.647868Z","shell.execute_reply":"2024-10-09T14:15:10.656716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define the base models\nbase_models = [\n    ('lgb', LGBMRegressor(**LGBM_Params)),\n    ('xgb', XGBRegressor(**XGB_Params)),\n    ('cat', CatBoostRegressor(**CatBoost_Params))\n]\nmeta_model = RandomForestRegressor(n_estimators=200, \n                                   max_depth=3, \n                                   min_samples_split=5, \n                                   min_samples_leaf=4, \n                                   max_features='sqrt', \n                                   max_samples=0.7, \n                                   random_state=SEED) # RandomForest\n# meta_model = LGBMRegressor(n_estimators=100, max_depth=7, random_state=SEED)\nstacking_model = StackingRegressor(estimators=base_models, final_estimator=meta_model)\n\n# Cross-validation and model training\nkf = StratifiedKFold(n_splits=10, shuffle=True, random_state=SEED)\npredictions = np.zeros(X.shape[0]) \ntest_predictions = np.zeros(test_df_le.shape[0])\nqwk_scores = []\n\n# Function to optimize the QWK score by adjusting thresholds\ndef evaluate_predictions(thresholds, y_true, y_pred):\n    thresholds = np.sort(thresholds)  # Ensure thresholds are in ascending order\n    y_pred_classes = np.digitize(y_pred, thresholds)\n    return -cohen_kappa_score(y_true, y_pred_classes, weights='quadratic')\n\nfor train_idx, valid_idx in kf.split(X, y):\n    X_train, X_valid = X.iloc[train_idx], X.iloc[valid_idx]\n    y_train, y_valid = y.iloc[train_idx], y.iloc[valid_idx]\n    \n    # Fit the stacking model\n    stacking_model.fit(X_train, y_train)\n\n    # Predict for validation set\n    preds = stacking_model.predict(X_valid)\n    predictions[valid_idx] += preds\n\n    # Optimize thresholds for QWK score using Nelder-Mead\n    initial_thresholds = [0.5, 1.5, 2.5]  # Initial guess for thresholds\n    KappaOptimizer = minimize(evaluate_predictions, x0=initial_thresholds, \n                              args=(y_valid, preds), method='Nelder-Mead')\n    best_thresholds = KappaOptimizer.x\n\n    # Apply optimized thresholds to validation predictions\n    ensemble_preds = np.digitize(predictions[valid_idx], np.sort(best_thresholds))\n\n    # Calculate QWK score for the optimized predictions\n    qwk_score = cohen_kappa_score(y.iloc[valid_idx], ensemble_preds, weights='quadratic')\n    qwk_scores.append(qwk_score)\n\n    # Predict for test set and store using optimized thresholds\n    test_preds = stacking_model.predict(test_df_le)\n    test_predictions += np.digitize(test_preds, np.sort(best_thresholds)) / kf.n_splits","metadata":{"execution":{"iopub.status.busy":"2024-10-09T14:35:47.842805Z","iopub.execute_input":"2024-10-09T14:35:47.843324Z","iopub.status.idle":"2024-10-09T14:38:09.091109Z","shell.execute_reply.started":"2024-10-09T14:35:47.843277Z","shell.execute_reply":"2024-10-09T14:38:09.089964Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Show the average QWK score across all folds\nprint(f\"Average QWK Score: {np.mean(qwk_scores)}\")","metadata":{"execution":{"iopub.status.busy":"2024-10-09T14:38:09.093185Z","iopub.execute_input":"2024-10-09T14:38:09.093588Z","iopub.status.idle":"2024-10-09T14:38:09.100004Z","shell.execute_reply.started":"2024-10-09T14:38:09.093543Z","shell.execute_reply":"2024-10-09T14:38:09.098696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Show the average QWK score across all folds\n# print(f\"Average QWK Score: {np.mean(qwk_scores)}\")\n\n# LGBMRegressor -Average QWK Score: 0.4472421358970992","metadata":{"execution":{"iopub.status.busy":"2024-10-09T14:17:29.921792Z","iopub.execute_input":"2024-10-09T14:17:29.922173Z","iopub.status.idle":"2024-10-09T14:17:29.931849Z","shell.execute_reply.started":"2024-10-09T14:17:29.922132Z","shell.execute_reply":"2024-10-09T14:17:29.930686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define the base models\n# base_models = [\n#     ('lgb', LGBMRegressor(**LGBM_Params)),\n#     ('xgb', XGBRegressor(**XGB_Params)),\n#     ('cat', CatBoostRegressor(**CatBoost_Params))\n# ]\n\n# # Define the meta-model\n# meta_model = RandomForestRegressor(n_estimators=100, max_depth=3)\n\n# # Create the stacking model\n# stacking_model = StackingRegressor(estimators=base_models, final_estimator=meta_model)\n\n# # Cross-Validation and Model Training\n# kf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n# predictions = np.zeros(X.shape[0]) \n# test_predictions = np.zeros(test_df_le.shape[0])\n# qwk_scores = []\n\n# for train_idx, valid_idx in kf.split(X, y):\n#     X_train, X_valid = X.iloc[train_idx], X.iloc[valid_idx]\n#     y_train, y_valid = y.iloc[train_idx], y.iloc[valid_idx]\n    \n#     # Fit the stacking model\n#     stacking_model.fit(X_train, y_train)\n\n#     # Predict for validation set\n#     preds = stacking_model.predict(X_valid)\n#     predictions[valid_idx] += preds\n    \n#     # Predict for test set\n#     test_predictions += stacking_model.predict(test_df_le) / kf.n_splits\n\n#     # Round predictions to the nearest integer class label\n#     ensemble_preds = np.round(predictions[valid_idx]).astype(int)\n    \n#     # Calculate QWK score\n#     qwk_score = cohen_kappa_score(y.iloc[valid_idx], ensemble_preds, weights='quadratic')\n#     qwk_scores.append(qwk_score)\n\n# # Show the score\n# print(f\"Average QWK Score: {np.mean(qwk_scores)}\")","metadata":{"execution":{"iopub.status.busy":"2024-10-09T14:17:29.93306Z","iopub.execute_input":"2024-10-09T14:17:29.933411Z","iopub.status.idle":"2024-10-09T14:17:29.944394Z","shell.execute_reply.started":"2024-10-09T14:17:29.933374Z","shell.execute_reply":"2024-10-09T14:17:29.943226Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Cross-Validation and Model Training\n# kf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n# predictions = np.zeros(X.shape[0]) \n# test_predictions = np.zeros(test_df_le.shape[0])\n# qwk_scores = []\n\n# for train_idx, valid_idx in kf.split(X, y):\n#     X_train, X_valid = X.iloc[train_idx], X.iloc[valid_idx]\n#     y_train, y_valid = y.iloc[train_idx], y.iloc[valid_idx]\n    \n#     lgb_model = LGBMRegressor(**LGBM_Params)\n#     xgb_model = XGBRegressor(**XGB_Params)\n#     cat_model = CatBoostRegressor(**CatBoost_Params)\n    \n#     model_predictions = []\n    \n#     for model in [lgb_model, xgb_model, cat_model]:\n#         model.fit(X_train, y_train)\n#         pred = model.predict(X_valid)\n#         model_predictions.append(pred)\n\n#     # Averaging -training data\n#     predictions[valid_idx] += np.mean(model_predictions, axis=0)\n    \n#     # Averaging -test data\n#     test_preds = np.array([model.predict(test_df_le) for model in [lgb_model, xgb_model, cat_model]])\n#     test_predictions += np.mean(test_preds, axis=0) / kf.n_splits\n\n#     ensemble_preds = np.round(predictions[valid_idx]).astype(int)\n    \n#     # Calculate QWK score\n#     qwk_score = cohen_kappa_score(y.iloc[valid_idx], ensemble_preds, weights='quadratic')\n#     qwk_scores.append(qwk_score)\n\n# # Show the score\n# print(f\"Average QWK Score: {np.mean(qwk_scores)}\")","metadata":{"execution":{"iopub.status.busy":"2024-10-09T14:17:29.946228Z","iopub.execute_input":"2024-10-09T14:17:29.9469Z","iopub.status.idle":"2024-10-09T14:17:29.961047Z","shell.execute_reply.started":"2024-10-09T14:17:29.946841Z","shell.execute_reply":"2024-10-09T14:17:29.959738Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Optimize thresholds\n# def optimize_thresholds(predictions, y):\n#     best_thresholds, best_score = np.zeros(3), -np.inf\n#     for t1 in np.linspace(0.1, 0.9, 50):\n#         for t2 in np.linspace(t1, 0.9, 50):\n#             for t3 in np.linspace(t2, 0.9, 50):\n#                 thresholds = [t1, t2, t3]\n#                 valid_preds = np.digitize(predictions[:, 1], thresholds)\n#                 score = cohen_kappa_score(y, valid_preds, weights='quadratic')\n#                 if score > best_score:\n#                     best_score = score\n#                     best_thresholds = thresholds\n#     return best_thresholds, best_score\n\n# best_thresholds, best_score = optimize_thresholds(predictions, y)\n# print(f\"Optimal thresholds: {best_thresholds}, Best QWK: {best_score}\")","metadata":{"execution":{"iopub.status.busy":"2024-10-09T14:17:29.962811Z","iopub.execute_input":"2024-10-09T14:17:29.963208Z","iopub.status.idle":"2024-10-09T14:17:29.978533Z","shell.execute_reply.started":"2024-10-09T14:17:29.963165Z","shell.execute_reply":"2024-10-09T14:17:29.977292Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Final predictions","metadata":{}},{"cell_type":"code","source":"# Prepare the test predictions for submission\ntest_pred_classes = np.digitize(test_predictions, np.sort(best_thresholds))\nsubmit_df = pd.DataFrame({'id': test_id, 'sii': test_pred_classes.astype(int)})\nsubmit_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-10-09T14:17:29.980526Z","iopub.execute_input":"2024-10-09T14:17:29.98099Z","iopub.status.idle":"2024-10-09T14:17:29.998843Z","shell.execute_reply.started":"2024-10-09T14:17:29.98094Z","shell.execute_reply":"2024-10-09T14:17:29.997564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}