{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":false,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-10-11T22:29:21.461667Z","iopub.execute_input":"2024-10-11T22:29:21.462584Z","iopub.status.idle":"2024-10-11T22:29:24.839438Z","shell.execute_reply.started":"2024-10-11T22:29:21.462537Z","shell.execute_reply":"2024-10-11T22:29:24.838271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\n\nfrom xgboost import XGBRegressor\nimport xgboost as xgb\nfrom lightgbm import LGBMRegressor,LGBMClassifier\nimport lightgbm as lgb\nfrom catboost import CatBoostRegressor\nfrom sklearn.ensemble import GradientBoostingRegressor\nfrom sklearn.ensemble import AdaBoostRegressor\nfrom sklearn.ensemble import VotingRegressor\n\nfrom sklearn.base import clone\nfrom sklearn.model_selection import KFold\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import cohen_kappa_score\nfrom scipy.optimize import minimize","metadata":{"execution":{"iopub.status.busy":"2024-10-11T22:29:24.841564Z","iopub.execute_input":"2024-10-11T22:29:24.842760Z","iopub.status.idle":"2024-10-11T22:29:29.123797Z","shell.execute_reply.started":"2024-10-11T22:29:24.842702Z","shell.execute_reply":"2024-10-11T22:29:29.122717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\", category=FutureWarning)\nfrom optuna.exceptions import ExperimentalWarning\n\n# Suppress FutureWarnings and Optuna ExperimentalWarnings\nwarnings.filterwarnings(\"ignore\", category=FutureWarning)\nwarnings.filterwarnings(\"ignore\", category=ExperimentalWarning)","metadata":{"execution":{"iopub.status.busy":"2024-10-11T22:29:29.125557Z","iopub.execute_input":"2024-10-11T22:29:29.126243Z","iopub.status.idle":"2024-10-11T22:29:29.284247Z","shell.execute_reply.started":"2024-10-11T22:29:29.126200Z","shell.execute_reply":"2024-10-11T22:29:29.283292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest_data = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample_data = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2024-10-11T22:29:29.286560Z","iopub.execute_input":"2024-10-11T22:29:29.286966Z","iopub.status.idle":"2024-10-11T22:29:29.376232Z","shell.execute_reply.started":"2024-10-11T22:29:29.286924Z","shell.execute_reply":"2024-10-11T22:29:29.375073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Shape of the data:\nprint(\"train_data :\", train_data.shape)\nprint(\"test_data :\", test_data.shape)\nprint(\"sample_submission_data :\", sample_data.shape)","metadata":{"execution":{"iopub.status.busy":"2024-10-11T22:29:29.377621Z","iopub.execute_input":"2024-10-11T22:29:29.378095Z","iopub.status.idle":"2024-10-11T22:29:29.383983Z","shell.execute_reply.started":"2024-10-11T22:29:29.378044Z","shell.execute_reply":"2024-10-11T22:29:29.382875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\nfrom IPython.display import clear_output\nfrom concurrent.futures import ThreadPoolExecutor\n\ndef process_file(filename, dirname):\n    data = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    data.drop('step', axis=1, inplace=True)\n    return data.describe().values.reshape(-1), filename.split('=')[1]\n\ndef load_time_series(dirname) -> pd.DataFrame:\n    ids = os.listdir(dirname)\n    \n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n    \n    stats, indexes = zip(*results)\n    data = pd.DataFrame(stats, columns=[f\"stat_{i}\" for i in range(len(stats[0]))])\n    data['id'] = indexes\n    \n    return data\n        \ntrain_parquet = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_parquet = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\ntime_series_cols = train_parquet.columns.tolist()\ntime_series_cols.remove(\"id\")","metadata":{"execution":{"iopub.status.busy":"2024-10-11T22:29:29.385317Z","iopub.execute_input":"2024-10-11T22:29:29.385734Z","iopub.status.idle":"2024-10-11T22:31:05.003523Z","shell.execute_reply.started":"2024-10-11T22:29:29.385694Z","shell.execute_reply":"2024-10-11T22:31:05.002261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = pd.merge(train_data, train_parquet, how=\"left\", on='id')\ntest_data = pd.merge(test_data, test_parquet, how=\"left\", on='id')","metadata":{"execution":{"iopub.status.busy":"2024-10-11T22:31:05.005732Z","iopub.execute_input":"2024-10-11T22:31:05.006216Z","iopub.status.idle":"2024-10-11T22:31:05.040500Z","shell.execute_reply.started":"2024-10-11T22:31:05.006161Z","shell.execute_reply":"2024-10-11T22:31:05.039460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = train_data.drop('id',axis=1)\ntest_data = test_data.drop('id',axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-10-11T22:31:05.042023Z","iopub.execute_input":"2024-10-11T22:31:05.042477Z","iopub.status.idle":"2024-10-11T22:31:05.056962Z","shell.execute_reply.started":"2024-10-11T22:31:05.042425Z","shell.execute_reply":"2024-10-11T22:31:05.055705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Shape of the data:\nprint(\"train_data :\", train_data.shape)\nprint(\"test_data :\", test_data.shape)","metadata":{"execution":{"iopub.status.busy":"2024-10-11T22:31:05.058706Z","iopub.execute_input":"2024-10-11T22:31:05.059317Z","iopub.status.idle":"2024-10-11T22:31:05.068950Z","shell.execute_reply.started":"2024-10-11T22:31:05.059240Z","shell.execute_reply":"2024-10-11T22:31:05.067721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Example columns in your train and test data\ntrain_columns = train_data.columns.tolist()  # Train columns including target\ntest_columns = test_data.columns.tolist()    # Test columns\n\n# Identify common feature columns between train and test (excluding target)\ncommon_columns = [col for col in train_columns if col in test_columns]\n\n# Include the target column explicitly in the final train set\ncommon_columns.append('sii')\n\n# Now, reduce the training data to only the common feature columns + target\ntrain_data = train_data[common_columns]\n\n# Print the resulting columns in the training data\nprint(\"Train data columns:\", len(train_data.columns))\nprint(\"Test data columns:\", len(test_data.columns))","metadata":{"execution":{"iopub.status.busy":"2024-10-11T22:31:05.073891Z","iopub.execute_input":"2024-10-11T22:31:05.074331Z","iopub.status.idle":"2024-10-11T22:31:05.088718Z","shell.execute_reply.started":"2024-10-11T22:31:05.074290Z","shell.execute_reply":"2024-10-11T22:31:05.087558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Shape of the data:\nprint(\"train_data :\", train_data.shape)\nprint(\"test_data :\", test_data.shape)","metadata":{"execution":{"iopub.status.busy":"2024-10-11T22:31:05.090249Z","iopub.execute_input":"2024-10-11T22:31:05.090705Z","iopub.status.idle":"2024-10-11T22:31:05.110263Z","shell.execute_reply.started":"2024-10-11T22:31:05.090616Z","shell.execute_reply":"2024-10-11T22:31:05.109118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.head()","metadata":{"execution":{"iopub.status.busy":"2024-10-11T22:31:05.112247Z","iopub.execute_input":"2024-10-11T22:31:05.112735Z","iopub.status.idle":"2024-10-11T22:31:05.161903Z","shell.execute_reply.started":"2024-10-11T22:31:05.112678Z","shell.execute_reply":"2024-10-11T22:31:05.160710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.isnull().sum().sort_values(ascending=False)","metadata":{"execution":{"iopub.status.busy":"2024-10-11T22:31:05.163440Z","iopub.execute_input":"2024-10-11T22:31:05.163893Z","iopub.status.idle":"2024-10-11T22:31:05.181995Z","shell.execute_reply.started":"2024-10-11T22:31:05.163851Z","shell.execute_reply":"2024-10-11T22:31:05.180888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Calculate missing values\nmissing_values = train_data.isnull().mean() * 100\n\n# Plot\nmissing_values.plot(kind='bar', figsize=(25, 5), color='skyblue')\nplt.title('Percentage of Missing Values by Feature')\nplt.ylabel('Percentage')\nplt.xlabel('Features')\nplt.xticks(rotation=45)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-09-28T21:21:19.151469Z","iopub.execute_input":"2024-09-28T21:21:19.151963Z","iopub.status.idle":"2024-09-28T21:21:20.969464Z","shell.execute_reply.started":"2024-09-28T21:21:19.151909Z","shell.execute_reply":"2024-09-28T21:21:20.968158Z"}}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nprint(train_data['sii'].value_counts())\nsns.countplot(x='sii', data=train_data)\nplt.xticks(rotation=60)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-11T22:31:05.183404Z","iopub.execute_input":"2024-10-11T22:31:05.183784Z","iopub.status.idle":"2024-10-11T22:31:05.495738Z","shell.execute_reply.started":"2024-10-11T22:31:05.183746Z","shell.execute_reply":"2024-10-11T22:31:05.494472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = train_data.dropna(subset='sii')\n","metadata":{"execution":{"iopub.status.busy":"2024-10-11T22:31:05.497156Z","iopub.execute_input":"2024-10-11T22:31:05.497569Z","iopub.status.idle":"2024-10-11T22:31:05.507610Z","shell.execute_reply.started":"2024-10-11T22:31:05.497529Z","shell.execute_reply":"2024-10-11T22:31:05.506243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Distribution of the data:\ntrain_data.drop(['id'],axis=1).hist(figsize=(25,25),color = 'skyblue', edgecolor='black')\nplt.show()","metadata":{}},{"cell_type":"code","source":"test_data.head()\ntest_data.isnull().sum().sort_values(ascending=False)","metadata":{"execution":{"iopub.status.busy":"2024-10-11T22:31:05.508975Z","iopub.execute_input":"2024-10-11T22:31:05.509380Z","iopub.status.idle":"2024-10-11T22:31:05.526986Z","shell.execute_reply.started":"2024-10-11T22:31:05.509342Z","shell.execute_reply":"2024-10-11T22:31:05.525789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data.info()","metadata":{"execution":{"iopub.status.busy":"2024-10-11T22:31:05.528578Z","iopub.execute_input":"2024-10-11T22:31:05.529078Z","iopub.status.idle":"2024-10-11T22:31:05.558819Z","shell.execute_reply.started":"2024-10-11T22:31:05.529007Z","shell.execute_reply":"2024-10-11T22:31:05.557352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Calculate missing values\nmissing_values = test_data.isnull().mean() * 100\n\n# Plot\nmissing_values.plot(kind='bar', figsize=(25, 5), color='skyblue')\nplt.title('Percentage of Missing Values by Feature')\nplt.ylabel('Percentage')\nplt.xlabel('Features')\nplt.xticks(rotation=45)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-09-28T21:21:44.147643Z","iopub.execute_input":"2024-09-28T21:21:44.14811Z","iopub.status.idle":"2024-09-28T21:21:45.743914Z","shell.execute_reply.started":"2024-09-28T21:21:44.148064Z","shell.execute_reply":"2024-09-28T21:21:45.742516Z"}}},{"cell_type":"code","source":"sample_data.head()","metadata":{"execution":{"iopub.status.busy":"2024-10-11T22:31:05.560628Z","iopub.execute_input":"2024-10-11T22:31:05.561188Z","iopub.status.idle":"2024-10-11T22:31:05.578736Z","shell.execute_reply.started":"2024-10-11T22:31:05.561135Z","shell.execute_reply":"2024-10-11T22:31:05.577443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_cols = list(train_data.select_dtypes(exclude=['object']).columns.difference(['sii']))\ncat_cols = list(train_data.select_dtypes(include=['object']).columns)\n\nnum_cols_test = list(test_data.select_dtypes(exclude=['object']).columns)\ncat_cols_test = list(test_data.select_dtypes(include=['object']).columns)\n\n#num_cols_test = [col for col in num_cols_test if col not in ['id']]","metadata":{"execution":{"iopub.status.busy":"2024-10-11T22:31:05.580229Z","iopub.execute_input":"2024-10-11T22:31:05.580614Z","iopub.status.idle":"2024-10-11T22:31:05.602896Z","shell.execute_reply.started":"2024-10-11T22:31:05.580573Z","shell.execute_reply":"2024-10-11T22:31:05.601411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in cat_cols:\n    train_data[col] = train_data[col].fillna('missing')\n    train_data[col] = train_data[col].astype('category')\n    \n    test_data[col] = test_data[col].fillna('missing')\n    test_data[col] = test_data[col].astype('category')\n","metadata":{"execution":{"iopub.status.busy":"2024-10-11T22:31:05.604310Z","iopub.execute_input":"2024-10-11T22:31:05.605327Z","iopub.status.idle":"2024-10-11T22:31:05.639379Z","shell.execute_reply.started":"2024-10-11T22:31:05.605283Z","shell.execute_reply":"2024-10-11T22:31:05.638305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in num_cols:\n    train_data[col] = train_data[col].fillna(train_data[col].median())\n    test_data[col] = test_data[col].fillna(test_data[col].median())","metadata":{"execution":{"iopub.status.busy":"2024-10-11T22:31:05.640783Z","iopub.execute_input":"2024-10-11T22:31:05.641151Z","iopub.status.idle":"2024-10-11T22:31:05.808407Z","shell.execute_reply.started":"2024-10-11T22:31:05.641111Z","shell.execute_reply":"2024-10-11T22:31:05.807135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(cat_cols_test),len(cat_cols)","metadata":{"execution":{"iopub.status.busy":"2024-10-11T22:31:05.810110Z","iopub.execute_input":"2024-10-11T22:31:05.810501Z","iopub.status.idle":"2024-10-11T22:31:05.818708Z","shell.execute_reply.started":"2024-10-11T22:31:05.810452Z","shell.execute_reply":"2024-10-11T22:31:05.817385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#  object datatype columns encoding:\n\nlabelencoder = LabelEncoder()\nfor col_name in cat_cols:\n    train_data[col_name]=labelencoder.fit_transform(train_data[col_name]).astype(int)\n        \nfor col_name in cat_cols_test:\n    test_data[col_name]=labelencoder.transform(test_data[col_name]).astype(int)","metadata":{"execution":{"iopub.status.busy":"2024-10-11T22:31:05.820278Z","iopub.execute_input":"2024-10-11T22:31:05.820663Z","iopub.status.idle":"2024-10-11T22:31:05.861145Z","shell.execute_reply.started":"2024-10-11T22:31:05.820603Z","shell.execute_reply":"2024-10-11T22:31:05.859944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scaler = StandardScaler()\ntrain_data[num_cols] = scaler.fit_transform(train_data[num_cols])\ntest_data[num_cols] = scaler.transform(test_data[num_cols])","metadata":{"execution":{"iopub.status.busy":"2024-10-11T22:31:05.862884Z","iopub.execute_input":"2024-10-11T22:31:05.864321Z","iopub.status.idle":"2024-10-11T22:31:05.948630Z","shell.execute_reply.started":"2024-10-11T22:31:05.864266Z","shell.execute_reply":"2024-10-11T22:31:05.947620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX = train_data.drop(['sii'], axis=1)\ny = train_data['sii']\ntest = test_data","metadata":{"execution":{"iopub.status.busy":"2024-10-11T22:31:05.950056Z","iopub.execute_input":"2024-10-11T22:31:05.950452Z","iopub.status.idle":"2024-10-11T22:31:06.094924Z","shell.execute_reply.started":"2024-10-11T22:31:05.950412Z","shell.execute_reply":"2024-10-11T22:31:06.093721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgb_params =  {'learning_rate': 0.04547397925995074, 'n_estimators': 900, 'max_depth': 8, 'num_leaves': 188, 'min_child_weight': 0.0001965169854605073, 'subsample': 0.7741513397601295, 'colsample_bytree': 0.5903852186222435, 'lambda_l1': 8.433330481031051, 'lambda_l2': 0.06739257527669937}\nxgb_params = {'learning_rate': 0.0055892408569577015, 'n_estimators': 812, 'max_depth': 9, 'min_child_weight': 7, 'subsample': 0.7431645176138364, 'colsample_bytree': 0.9334177520684833, 'gamma': 0.0005032400388186735, 'reg_alpha': 4.675280686446186, 'reg_lambda': 0.002782303028089819}\nlparams = {'learning_rate': 0.046,'max_depth': 12,'num_leaves': 478,'min_data_in_leaf': 13,'feature_fraction': 0.893,'bagging_fraction': 0.784,'bagging_freq': 4,'lambda_l1': 10,'lambda_l2': 0.01}\nxparams = {'learning_rate': 0.05,'max_depth': 6,'n_estimators': 200,'subsample': 0.8,'colsample_bytree': 0.8,'reg_alpha': 1,  'reg_lambda': 5, 'random_state': 42}\ncparams = {'learning_rate': 0.05,'depth': 6,'iterations': 200,'random_seed': 42,'l2_leaf_reg': 10}","metadata":{"execution":{"iopub.status.busy":"2024-10-11T22:31:06.096539Z","iopub.execute_input":"2024-10-11T22:31:06.097461Z","iopub.status.idle":"2024-10-11T22:31:06.108298Z","shell.execute_reply.started":"2024-10-11T22:31:06.097404Z","shell.execute_reply":"2024-10-11T22:31:06.106852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params_lgb = {'learning_rate': 0.060210090165748686, 'n_estimators': 676, 'max_depth': 5, 'num_leaves': 193, 'min_child_weight': 1.4676211795900709, 'subsample': 0.9176759029661259, 'colsample_bytree': 0.6228483814299844, 'lambda_l1': 5.972758940714118, 'lambda_l2': 0.08520209502101517}\nparams_xgb = {'learning_rate': 0.07655257571702724, 'n_estimators': 688, 'max_depth': 7, 'min_child_weight': 10, 'subsample': 0.8740939473627481, 'colsample_bytree': 0.9986562622011108, 'gamma': 0.005098593898531702, 'reg_alpha': 9.637641942675724, 'reg_lambda': 0.014395773764050573}\nparams_cat = {'iterations': 587, 'learning_rate': 0.055230940995657174, 'depth': 4, 'l2_leaf_reg': 0.00018791609018454318, 'subsample': 0.6500825893922675, 'colsample_bylevel': 0.9880985604359044, 'random_strength': 0.12043764855944512, 'bagging_temperature': 0.0008351502400011265, 'min_data_in_leaf': 21}\nparams_AB =  {'n_estimators': 940, 'learning_rate': 0.0028835511040035183, 'loss': 'exponential'}\nparams_DT = {'criterion': 'squared_error', 'splitter': 'random', 'max_depth': 10, 'min_samples_split': 5, 'min_samples_leaf': 8, 'max_features': 'auto', 'ccp_alpha': 0.002198265167300811}\nparams_gb = {'n_estimators': 923, 'learning_rate': 0.016016511561543028, 'max_depth': 5, 'min_samples_split': 10, 'min_samples_leaf': 15, 'subsample': 0.9179202110147108, 'max_features': 'auto'}","metadata":{"execution":{"iopub.status.busy":"2024-10-11T22:31:06.110078Z","iopub.execute_input":"2024-10-11T22:31:06.110471Z","iopub.status.idle":"2024-10-11T22:31:06.124959Z","shell.execute_reply.started":"2024-10-11T22:31:06.110431Z","shell.execute_reply":"2024-10-11T22:31:06.123903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n    return -quadratic_weighted_kappa(y_true, rounded_p)","metadata":{"execution":{"iopub.status.busy":"2024-10-11T22:31:06.131848Z","iopub.execute_input":"2024-10-11T22:31:06.132929Z","iopub.status.idle":"2024-10-11T22:31:06.145430Z","shell.execute_reply.started":"2024-10-11T22:31:06.132862Z","shell.execute_reply":"2024-10-11T22:31:06.144432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_splits = 10\ndef train_model(model_class, test_data):\n    \n    X = train_data.drop(['sii'], axis=1)\n    y = train_data['sii']\n    test = test_data\n\n    SKF = StratifiedKFold(n_splits=10, shuffle=True, random_state=42)\n    \n    models = []\n    train_pred = []\n    test_pred = []\n    \n    oof_non_rounded = np.zeros(len(y), dtype=float) \n    oof_rounded = np.zeros(len(y), dtype=int) \n    test_preds = np.zeros((len(test_data), n_splits))\n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n        X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n\n        model = clone(model_class)\n        model.fit(X_train, y_train)\n\n        y_train_pred = model.predict(X_train)\n        y_val_pred = model.predict(X_val)\n\n        oof_non_rounded[test_idx] = y_val_pred\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n        oof_rounded[test_idx] = y_val_pred_rounded\n\n        train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n        val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n        train_pred.append(train_kappa)\n        test_pred.append(val_kappa)\n        \n        test_preds[:, fold] = model.predict(test)\n        \n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n        clear_output(wait=True)\n        models.append(model)\n\n    print(f\"Mean Train QWK : {np.mean(train_pred):.4f}\")\n    print(f\"Mean Validation QWK : {np.mean(test_pred):.4f}\")\n\n    KappaOPtimizer = minimize(evaluate_predictions, x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), method='Nelder-Mead') \n    assert KappaOPtimizer.success, \"Optimization did not converge.\"\n    \n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n    print(f\"Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n    pred_mean = test_preds.mean(axis=1)\n    pred = threshold_Rounder(pred_mean, KappaOPtimizer.x)\n    \n    submission = pd.DataFrame({'id': sample_data['id'],'sii': pred})\n\n    return submission","metadata":{"execution":{"iopub.status.busy":"2024-10-11T22:31:06.146792Z","iopub.execute_input":"2024-10-11T22:31:06.147140Z","iopub.status.idle":"2024-10-11T22:31:06.165255Z","shell.execute_reply.started":"2024-10-11T22:31:06.147102Z","shell.execute_reply":"2024-10-11T22:31:06.164162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install colorama","metadata":{"execution":{"iopub.status.busy":"2024-10-11T22:31:06.166720Z","iopub.execute_input":"2024-10-11T22:31:06.167167Z","iopub.status.idle":"2024-10-11T22:31:44.217649Z","shell.execute_reply.started":"2024-10-11T22:31:06.167125Z","shell.execute_reply":"2024-10-11T22:31:44.215577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from colorama import Fore, Style, init","metadata":{"execution":{"iopub.status.busy":"2024-10-11T22:31:44.220326Z","iopub.execute_input":"2024-10-11T22:31:44.220794Z","iopub.status.idle":"2024-10-11T22:31:44.228104Z","shell.execute_reply.started":"2024-10-11T22:31:44.220742Z","shell.execute_reply":"2024-10-11T22:31:44.226920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.tree import DecisionTreeRegressor\nfrom sklearn.ensemble import RandomForestRegressor\n# Create model instances\nlgb_model = LGBMRegressor(**lparams, random_state=42, verbosity=-1)\nXGB_Model = XGBRegressor(**xparams)\nCatBoost_Model = CatBoostRegressor(**cparams,verbose=0)\nGB_model = GradientBoostingRegressor()\nAdaBoost_model = AdaBoostRegressor(**params_AB)\nDT_model = DecisionTreeRegressor()\nRF_model = RandomForestRegressor(n_estimators=100, random_state=42)\n# Combine models using Voting Regressor\nvotingRegressor = VotingRegressor(estimators=[('lightgbm', lgb_model),\n                                           ('xgboost', XGB_Model),\n                                           ('catboost', CatBoost_Model)])","metadata":{"execution":{"iopub.status.busy":"2024-10-11T22:31:44.230056Z","iopub.execute_input":"2024-10-11T22:31:44.230474Z","iopub.status.idle":"2024-10-11T22:31:44.249542Z","shell.execute_reply.started":"2024-10-11T22:31:44.230433Z","shell.execute_reply":"2024-10-11T22:31:44.248397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train the ensemble model\nSubmission = train_model(votingRegressor, test)\n\n# Saving submission:\nSubmission.to_csv('submission.csv', index=False)\nprint(Submission['sii'].value_counts())","metadata":{"execution":{"iopub.status.busy":"2024-10-11T22:31:44.251219Z","iopub.execute_input":"2024-10-11T22:31:44.251596Z","iopub.status.idle":"2024-10-11T22:33:00.729496Z","shell.execute_reply.started":"2024-10-11T22:31:44.251554Z","shell.execute_reply":"2024-10-11T22:33:00.728299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"import optuna\nfrom sklearn.ensemble import AdaBoostRegressor\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.model_selection import KFold\nimport numpy as np\n\n# Define the Quadratic Weighted Kappa function\ndef quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights=\"quadratic\")\n\n# Define the objective function for Optuna\ndef objective(trial):\n    # Define hyperparameters to tune\n    params = {\n        \"n_estimators\": trial.suggest_int(\"n_estimators\", 50, 1000),\n        \"learning_rate\": trial.suggest_loguniform(\"learning_rate\", 1e-4, 1.0),\n        \"loss\": trial.suggest_categorical(\"loss\", [\"linear\", \"square\", \"exponential\"]),\n    }\n\n    # Initialize AdaBoostRegressor\n    model = AdaBoostRegressor(**params)\n    \n    # K-fold cross-validation\n    kf = KFold(n_splits=5, shuffle=True, random_state=42)\n    \n    scores = []\n    \n    for train_index, test_index in kf.split(X):\n        X_train, X_test = X.iloc[train_index], X.iloc[test_index]\n        y_train, y_test = y.iloc[train_index], y.iloc[test_index]\n        \n        # Fit the model\n        model.fit(X_train, y_train)\n        \n        # Predict on the validation set\n        y_pred = np.round(model.predict(X_test)).astype(int)\n        \n        # Calculate the QWK score\n        qwk_score = quadratic_weighted_kappa(y_test, y_pred)\n        scores.append(qwk_score)\n    \n    # Return the mean QWK score across all folds\n    return np.mean(scores)\n\n# Create Optuna study and optimize\nstudy = optuna.create_study(direction=\"maximize\")\nstudy.optimize(objective, n_trials=100)\n\n# Print best parameters and QWK score\nprint(\"Best hyperparameters: \", study.best_params)\nprint(f\"Best QWK score: {study.best_value:.4f}\")","metadata":{"execution":{"iopub.status.busy":"2024-09-29T21:19:54.192798Z","iopub.execute_input":"2024-09-29T21:19:54.193976Z","iopub.status.idle":"2024-09-30T01:18:01.876075Z","shell.execute_reply.started":"2024-09-29T21:19:54.193918Z","shell.execute_reply":"2024-09-30T01:18:01.874115Z"},"_kg_hide-input":true}},{"cell_type":"markdown","source":"import optuna\nfrom sklearn.ensemble import GradientBoostingRegressor\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.model_selection import KFold\nimport numpy as np\n\n# Define the Quadratic Weighted Kappa function\ndef quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights=\"quadratic\")\n\n# Define the objective function for Optuna\ndef objective(trial):\n    # Define hyperparameters to tune\n    params = {\n        \"n_estimators\": trial.suggest_int(\"n_estimators\", 100, 1000),\n        \"learning_rate\": trial.suggest_loguniform(\"learning_rate\", 1e-4, 0.1),\n        \"max_depth\": trial.suggest_int(\"max_depth\", 3, 12),\n        \"min_samples_split\": trial.suggest_int(\"min_samples_split\", 2, 20),\n        \"min_samples_leaf\": trial.suggest_int(\"min_samples_leaf\", 1, 20),\n        \"subsample\": trial.suggest_uniform(\"subsample\", 0.5, 1.0),\n        \"max_features\": trial.suggest_categorical(\"max_features\", [\"auto\", \"sqrt\", \"log2\", None]),\n    }\n    \n    # Initialize GradientBoostingRegressor\n    model = GradientBoostingRegressor(**params)\n    \n    # K-fold cross-validation\n    kf = KFold(n_splits=5, shuffle=True, random_state=42)\n    \n    scores = []\n    \n    for train_index, test_index in kf.split(X):\n        X_train, X_test = X.iloc[train_index], X.iloc[test_index]\n        y_train, y_test = y.iloc[train_index], y.iloc[test_index]\n        \n        # Fit the model\n        model.fit(X_train, y_train)\n        \n        # Predict on the validation set\n        y_pred = np.round(model.predict(X_test)).astype(int)\n        \n        # Calculate the QWK score\n        qwk_score = quadratic_weighted_kappa(y_test, y_pred)\n        scores.append(qwk_score)\n    \n    # Return the mean QWK score across all folds\n    return np.mean(scores)\n\n# Create Optuna study and optimize\nstudy = optuna.create_study(direction=\"maximize\")\nstudy.optimize(objective, n_trials=100)\n\n# Print best parameters and QWK score\nprint(\"Best hyperparameters: \", study.best_params)\nprint(f\"Best QWK score: {study.best_value:.4f}\")","metadata":{"execution":{"iopub.status.busy":"2024-09-30T01:18:01.879428Z","iopub.execute_input":"2024-09-30T01:18:01.880028Z"},"_kg_hide-input":true}}]}