{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"},{"sourceId":9682055,"sourceType":"datasetVersion","datasetId":5918194}],"dockerImageVersionId":30787,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install tsflex --no-index --find-links=file:///kaggle/input/cmi-time-series-tools \n!pip install seglearn --no-index --find-links=file:///kaggle/input/cmi-time-series-tools  ","metadata":{"execution":{"iopub.status.busy":"2024-12-07T17:20:18.916413Z","iopub.execute_input":"2024-12-07T17:20:18.916661Z","iopub.status.idle":"2024-12-07T17:20:36.588309Z","shell.execute_reply.started":"2024-12-07T17:20:18.916634Z","shell.execute_reply":"2024-12-07T17:20:36.587204Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport polars as pl\nfrom os.path import join\nimport os\nfrom tqdm import tqdm\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import train_test_split, StratifiedKFold, KFold\nfrom sklearn.base import clone\nfrom sklearn.model_selection import GridSearchCV, RandomizedSearchCV\nfrom sklearn.neighbors import KNeighborsRegressor\n\nfrom seglearn.feature_functions import base_features, emg_features\nfrom tsflex.features import FeatureCollection, MultipleFeatureDescriptors\nfrom tsflex.features.integrations import seglearn_feature_dict_wrapper\n\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.metrics import make_scorer, cohen_kappa_score\nfrom sklearn.ensemble import VotingRegressor\nfrom sklearn.impute import SimpleImputer\nfrom IPython.display import clear_output\nfrom scipy.optimize import minimize\nfrom colorama import Fore, Style\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-12-07T17:20:36.590411Z","iopub.execute_input":"2024-12-07T17:20:36.590800Z","iopub.status.idle":"2024-12-07T17:20:40.390215Z","shell.execute_reply.started":"2024-12-07T17:20:36.590758Z","shell.execute_reply":"2024-12-07T17:20:40.389410Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"base_dir = \"/kaggle/input/child-mind-institute-problematic-internet-use\"\ntrain_dir = join(base_dir, \"train.csv\")\ntest_dir = join(base_dir, \"test.csv\")\nsample_dir = join(base_dir, \"sample_submission.csv\")\n\ntrain_parquet = join(base_dir, \"series_train.parquet/\")\ntest_parquet = join(base_dir, \"series_test.parquet/\")","metadata":{"execution":{"iopub.status.busy":"2024-12-07T17:20:40.391316Z","iopub.execute_input":"2024-12-07T17:20:40.391931Z","iopub.status.idle":"2024-12-07T17:20:40.396440Z","shell.execute_reply.started":"2024-12-07T17:20:40.391892Z","shell.execute_reply":"2024-12-07T17:20:40.395570Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class TSPipeline:\n    @staticmethod\n    def build_tsflex_features(df: pd.DataFrame):\n\n        df = df.to_pandas()\n        len = df.shape[0]\n\n        basic_feats = MultipleFeatureDescriptors(\n            functions=seglearn_feature_dict_wrapper(base_features()),\n            series_names=['X', 'Y', 'Z', 'enmo', 'anglez', 'light', 'battery_voltage'],\n            windows=[len],\n            strides=[len],\n        )\n        \n        emg_feats = emg_features()\n        del emg_feats['simple square integral']\n        \n        emg_feats = MultipleFeatureDescriptors(\n            functions=seglearn_feature_dict_wrapper(emg_feats),\n            series_names=['X', 'Y', 'Z', 'enmo', 'anglez', 'light', 'battery_voltage'],\n            windows=[len],\n            strides=[len],\n        )\n        \n        fc = FeatureCollection([basic_feats, emg_feats])\n        \n        df = fc.calculate(df,\n                          return_df=True, \n                          include_final_window=True, \n                          approve_sparsity=True, \n                          window_idx=\"begin\").astype(np.float32)\n\n        return df","metadata":{"execution":{"iopub.status.busy":"2024-12-07T17:20:40.397972Z","iopub.execute_input":"2024-12-07T17:20:40.398235Z","iopub.status.idle":"2024-12-07T17:20:40.412752Z","shell.execute_reply.started":"2024-12-07T17:20:40.398211Z","shell.execute_reply":"2024-12-07T17:20:40.411904Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def read_file(path) -> np.ndarray: \n    df = pl.read_parquet(path)\n    df = df.pipe(TSPipeline.build_tsflex_features)\n    return df.to_numpy().flatten()","metadata":{"execution":{"iopub.status.busy":"2024-12-07T17:20:40.413729Z","iopub.execute_input":"2024-12-07T17:20:40.413971Z","iopub.status.idle":"2024-12-07T17:20:40.426627Z","shell.execute_reply.started":"2024-12-07T17:20:40.413948Z","shell.execute_reply":"2024-12-07T17:20:40.425738Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_time_series(dir_path: str) -> dict:\n    time_series_store = {}\n    file_ind = os.listdir(dir_path)\n    for file_name in tqdm(file_ind):\n        file_id = file_name.split('=')[1]\n        file_path = os.path.join(dir_path + file_name, 'part-0.parquet')\n        df = read_file(file_path)\n        time_series_store[file_id] = df\n        \n    return time_series_store","metadata":{"execution":{"iopub.status.busy":"2024-12-07T17:20:40.427746Z","iopub.execute_input":"2024-12-07T17:20:40.428208Z","iopub.status.idle":"2024-12-07T17:20:40.439688Z","shell.execute_reply.started":"2024-12-07T17:20:40.428179Z","shell.execute_reply":"2024-12-07T17:20:40.438885Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_time_series_store = load_time_series(train_parquet)\ntrain_time_series_store = pd.DataFrame(train_time_series_store).T\ntrain_time_series_store.columns = [f'f{i}' for i in range(train_time_series_store.shape[1])]\ntrain_time_series_store = train_time_series_store.reset_index()\ntrain_time_series_store.rename(columns={'index': 'id'}, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-12-07T17:20:40.440567Z","iopub.execute_input":"2024-12-07T17:20:40.440793Z","iopub.status.idle":"2024-12-07T17:31:43.958213Z","shell.execute_reply.started":"2024-12-07T17:20:40.440770Z","shell.execute_reply":"2024-12-07T17:31:43.957337Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_time_series_store = load_time_series(test_parquet)\ntest_time_series_store = pd.DataFrame(test_time_series_store).T\ntest_time_series_store.columns = [f'f{i}' for i in range(test_time_series_store.shape[1])]\ntest_time_series_store = test_time_series_store.reset_index()\ntest_time_series_store.rename(columns={'index': 'id'}, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-12-07T17:31:43.959418Z","iopub.execute_input":"2024-12-07T17:31:43.959682Z","iopub.status.idle":"2024-12-07T17:31:44.756713Z","shell.execute_reply.started":"2024-12-07T17:31:43.959656Z","shell.execute_reply":"2024-12-07T17:31:44.755667Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv(train_dir)\ntest = pd.read_csv(test_dir)\nsample = pd.read_csv(sample_dir)","metadata":{"execution":{"iopub.status.busy":"2024-12-07T17:31:44.758153Z","iopub.execute_input":"2024-12-07T17:31:44.758445Z","iopub.status.idle":"2024-12-07T17:31:44.817349Z","shell.execute_reply.started":"2024-12-07T17:31:44.758418Z","shell.execute_reply":"2024-12-07T17:31:44.816687Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sii_count = train['sii'].isna().replace({True: 'NaN', False: 'Number'})\nsii_count.value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-12-07T17:31:44.820210Z","iopub.execute_input":"2024-12-07T17:31:44.820458Z","iopub.status.idle":"2024-12-07T17:31:44.833974Z","shell.execute_reply.started":"2024-12-07T17:31:44.820434Z","shell.execute_reply":"2024-12-07T17:31:44.833124Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"missing_percent = train.isnull().mean() * 100 \nmissing_percent = missing_percent.to_frame().reset_index() \nmissing_percent.columns = ['columns_name', 'missing_percentage'] \nmissing_percent = missing_percent[missing_percent['missing_percentage'] > 50] \nmissing_percent","metadata":{"execution":{"iopub.status.busy":"2024-12-07T17:31:44.835239Z","iopub.execute_input":"2024-12-07T17:31:44.835899Z","iopub.status.idle":"2024-12-07T17:31:44.853821Z","shell.execute_reply.started":"2024-12-07T17:31:44.835839Z","shell.execute_reply":"2024-12-07T17:31:44.853094Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = train.dropna(subset=['sii'])\n\nfeatures_cols = list(set(train.columns).intersection(set(test.columns)))  \n\ntest = test[features_cols]\ntrain = train[features_cols + ['sii']]\n\nnumeric_columns = train.select_dtypes(include = ['float64', 'int64']).columns\ncategorical_columns = train.select_dtypes(include=['category', 'object']).columns","metadata":{"execution":{"iopub.status.busy":"2024-12-07T17:31:44.854940Z","iopub.execute_input":"2024-12-07T17:31:44.855220Z","iopub.status.idle":"2024-12-07T17:31:44.867321Z","shell.execute_reply.started":"2024-12-07T17:31:44.855195Z","shell.execute_reply":"2024-12-07T17:31:44.866461Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class PreprocessingHelper(BaseEstimator, TransformerMixin):   \n    def __init__(self, filling: str): \n        self.attribute_deleted = []                \n        self.base_num_algoritm_predict = {}\n        self.base_cat_algoritm_predict = {}\n        self.mapping_cat_storage = {}\n        self.label_encoder = LabelEncoder()     \n        self.filling='mean'\n        \n    def base_num_algoritm(self, y) -> float:\n        return np.mean(y, axis = 0)\n\n    def base_cat_algoritm(self, y) -> str:\n        return y.mode()[0]     \n    \n    def fit(self, X, y=None):\n        X_num, X_cat, X_df = X   \n        new_cat_columns = []\n        missing_percent = X_df.isnull().mean() * 100 \n        missing_percent = missing_percent.to_frame().reset_index() \n        missing_percent.columns = ['columns_name', 'missing_percentage'] \n        missing_percent = missing_percent[missing_percent['missing_percentage'] > 50] \n        self.attribute_deleted  = list(missing_percent.columns_name)\n       \n        for columns in X_num:\n            if columns not in self.attribute_deleted:\n                feature_with_isna = X_num[columns].dropna()  \n                base_pred = self.base_num_algoritm(feature_with_isna) \n                self.base_num_algoritm_predict[columns] = base_pred\n                \n                if len(set(feature_with_isna)) <= 7:\n                    new_cat_columns.append(columns)\n\n        # с учетом колонок которые переходят в категориальные переменные\n        for columns in X_cat.columns.to_list() + new_cat_columns:\n            if columns not in self.attribute_deleted:\n                if self.filling == 'mean':\n                    feature_with_isna = X_df[columns].dropna()\n                    base_pred = self.base_cat_algoritm(feature_with_isna) \n                else:\n                    feature_with_isna = X_df[columns].fillna('missing')\n                    base_pred = 'missing'\n                  \n                self.base_cat_algoritm_predict[columns] = base_pred\n                feature_with_isna = self.label_encoder.fit_transform(feature_with_isna) \n                classes = self.label_encoder.classes_\n                encoded_values = self.label_encoder.transform(classes)\n                mapping_cat =  dict(zip(classes, encoded_values))\n                self.mapping_cat_storage[columns] = mapping_cat\n                      \n        return self\n    \n    def transform(self, X, training = True) -> pd.DataFrame:\n        \n        X_num, X_cat, X_df = X \n        if training:\n            X_num = X_num.columns.to_list()\n            X_cat = X_cat.columns.to_list()\n            \n        upd_attribute_deleted = [col for col in self.attribute_deleted if col in X_df.columns]\n        X_df = X_df.drop(upd_attribute_deleted, axis=1)\n        \n        X_new_cat_columns = []\n        \n        x_columns_isna = X_df.columns[X_df.isna().any()].tolist()      \n        \n        for columns in X_num:\n\n            if columns in x_columns_isna:\n                unic_column_meanings = X_df[columns].dropna()\n\n                if len(set(unic_column_meanings)) <= 7:\n                    \n                    if training:\n                        X_new_cat_columns.append(columns)\n                        X_num.remove(columns)\n                    \n                    for ind, _ in X_df.iterrows():\n                        if pd.isna(X_df.loc[ind, columns]):\n                            if self.filling == 'mean':\n                                X_df.loc[ind, columns] = self.base_cat_algoritm(unic_column_meanings)\n                            else:\n                                X_df.loc[ind, columns] = 'missing'\n                else:\n                    for ind, _ in X_df.iterrows():\n                        if pd.isna(X_df.loc[ind, columns]):\n                            X_df.loc[ind, columns] = self.base_num_algoritm_predict[columns]\n\n        for columns in X_cat:\n            if columns in x_columns_isna:\n                for ind, _ in X_df.iterrows():\n                    if pd.isna(X_df.loc[ind, columns]):\n                        X_df.loc[ind, columns] = self.base_cat_algoritm_predict[columns]\n\n        if training:\n            X_cat = [c for c in X_df.columns if c in X_cat and c not in ['id', 'sii']]\n            X_cat+=X_new_cat_columns\n\n        for columns in X_cat:\n            X_df[columns] = X_df[columns].replace(\n                self.mapping_cat_storage[columns])\n        \n        return X_df, X_num, X_cat","metadata":{"execution":{"iopub.status.busy":"2024-12-07T17:31:44.868770Z","iopub.execute_input":"2024-12-07T17:31:44.869196Z","iopub.status.idle":"2024-12-07T17:31:44.886097Z","shell.execute_reply.started":"2024-12-07T17:31:44.869158Z","shell.execute_reply":"2024-12-07T17:31:44.885325Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class DataFrameSelector(BaseEstimator, TransformerMixin):\n    \n    def __init__(self, numeric_columns: pd.Series, categorical_columns: pd.Series):\n        self.numeric_columns = numeric_columns\n        self.categorical_columns = categorical_columns\n        \n    def fit(self, X, y=None):\n        return self\n    \n    def transform(self, X: pd.DataFrame) -> tuple:\n        self.numeric_columns = [i for i in self.numeric_columns if i in X.columns]\n        self.categorical_columns = [i for i in self.categorical_columns if i in X.columns]\n        return X[self.numeric_columns], X[self.categorical_columns], X","metadata":{"execution":{"iopub.status.busy":"2024-12-07T17:31:44.887029Z","iopub.execute_input":"2024-12-07T17:31:44.887355Z","iopub.status.idle":"2024-12-07T17:31:44.895811Z","shell.execute_reply.started":"2024-12-07T17:31:44.887309Z","shell.execute_reply":"2024-12-07T17:31:44.895130Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"metadata_pipeline = Pipeline(steps=[\n    ('selector', DataFrameSelector(numeric_columns, categorical_columns)),\n    ('Preprocessing', PreprocessingHelper(filling = 'missing'))\n    ])","metadata":{"execution":{"iopub.status.busy":"2024-12-07T17:31:44.896691Z","iopub.execute_input":"2024-12-07T17:31:44.896935Z","iopub.status.idle":"2024-12-07T17:31:44.906976Z","shell.execute_reply.started":"2024-12-07T17:31:44.896911Z","shell.execute_reply":"2024-12-07T17:31:44.906156Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train, numeric_columns, categorical_columns = metadata_pipeline.fit_transform(train)\ntrain, numeric_columns, categorical_columns = metadata_pipeline.named_steps['Preprocessing'].transform(\n    (numeric_columns, categorical_columns, train), training = False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T17:31:44.907965Z","iopub.execute_input":"2024-12-07T17:31:44.908188Z","iopub.status.idle":"2024-12-07T17:31:58.680264Z","shell.execute_reply.started":"2024-12-07T17:31:44.908165Z","shell.execute_reply":"2024-12-07T17:31:58.679606Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test, _, _ = metadata_pipeline.named_steps['Preprocessing'].transform(\n    (numeric_columns, categorical_columns, test), training = False)","metadata":{"execution":{"iopub.status.busy":"2024-12-07T17:31:58.681493Z","iopub.execute_input":"2024-12-07T17:31:58.681775Z","iopub.status.idle":"2024-12-07T17:31:58.852995Z","shell.execute_reply.started":"2024-12-07T17:31:58.681748Z","shell.execute_reply":"2024-12-07T17:31:58.852296Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Merge datasets","metadata":{"execution":{"iopub.status.busy":"2024-12-07T17:31:58.854361Z","iopub.execute_input":"2024-12-07T17:31:58.854756Z","iopub.status.idle":"2024-12-07T17:31:58.858712Z","shell.execute_reply.started":"2024-12-07T17:31:58.854716Z","shell.execute_reply":"2024-12-07T17:31:58.857862Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"time_series_cols = train_time_series_store.columns.tolist()\ntime_series_cols.remove(\"id\")\n\ntrain = pd.merge(train, train_time_series_store, how=\"left\", on='id')\ntest = pd.merge(test, test_time_series_store, how=\"left\", on='id')\n\ntrain = train.drop('id', axis=1)\ntest = test.drop('id', axis=1)\n\ntrain[time_series_cols] = train[time_series_cols].fillna(value=0)  \ntest[time_series_cols] = test[time_series_cols].fillna(value=0)  ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T17:31:58.859735Z","iopub.execute_input":"2024-12-07T17:31:58.860092Z","iopub.status.idle":"2024-12-07T17:31:58.913810Z","shell.execute_reply.started":"2024-12-07T17:31:58.860066Z","shell.execute_reply":"2024-12-07T17:31:58.912955Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Scaling","metadata":{"execution":{"iopub.status.busy":"2024-12-07T17:31:58.914971Z","iopub.execute_input":"2024-12-07T17:31:58.915794Z","iopub.status.idle":"2024-12-07T17:31:58.919329Z","shell.execute_reply.started":"2024-12-07T17:31:58.915740Z","shell.execute_reply":"2024-12-07T17:31:58.918538Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"scaler = StandardScaler()\n\nnum_columns = [c for c in train.columns if c in numeric_columns and c not in ['sii']] \nnum_columns = num_columns + time_series_cols\n\ntrain_scalar = scaler.fit_transform(train[num_columns])  \ntest_scalar = scaler.transform(test[num_columns])\ntrain_scalar = pd.DataFrame(data = train_scalar, columns = num_columns)\ntest_scalar = pd.DataFrame(data = test_scalar, columns = num_columns)\n\nfor col in num_columns:\n    train.loc[:, col] = train_scalar[col].values\n    test.loc[:, col] = test_scalar[col].values","metadata":{"execution":{"iopub.status.busy":"2024-12-07T17:31:58.920370Z","iopub.execute_input":"2024-12-07T17:31:58.920667Z","iopub.status.idle":"2024-12-07T17:31:59.179445Z","shell.execute_reply.started":"2024-12-07T17:31:58.920641Z","shell.execute_reply":"2024-12-07T17:31:59.178497Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i in categorical_columns:\n    train[i] = train[i].astype(int) \n    test[i] = test[i].astype(int) ","metadata":{"execution":{"iopub.status.busy":"2024-12-07T17:31:59.180580Z","iopub.execute_input":"2024-12-07T17:31:59.180882Z","iopub.status.idle":"2024-12-07T17:31:59.197027Z","shell.execute_reply.started":"2024-12-07T17:31:59.180838Z","shell.execute_reply":"2024-12-07T17:31:59.196296Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Models Param","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T17:31:59.197952Z","iopub.execute_input":"2024-12-07T17:31:59.198201Z","iopub.status.idle":"2024-12-07T17:31:59.206384Z","shell.execute_reply.started":"2024-12-07T17:31:59.198177Z","shell.execute_reply":"2024-12-07T17:31:59.205484Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lgb_params = {\n    'learning_rate': np.linspace(0.01, 0.1, num=100).tolist(),  \n    'max_depth': [3, 5, 8, 10, 12],\n    'n_estimators': list(range(10, 400, 5)),\n    'num_leaves': list(range(10, 500, 5)),\n    'min_data_in_leaf': list(range(10, 50, 3)),\n    'feature_fraction': [round(x, 3) for x in np.linspace(0.3, 1.0, num=40)],  \n    'bagging_fraction': [round(x, 3) for x in np.linspace(0.3, 1.0, num=40)],  \n    'bagging_freq': list(range(2, 60, 2)),\n    'lambda_l1': np.linspace(0, 5, num=40).tolist(),\n    'lambda_l2': np.linspace(0, 5, num=40).tolist(),\n    'device': ['gpu']\n}","metadata":{"execution":{"iopub.status.busy":"2024-12-07T17:31:59.207363Z","iopub.execute_input":"2024-12-07T17:31:59.207602Z","iopub.status.idle":"2024-12-07T17:31:59.217595Z","shell.execute_reply.started":"2024-12-07T17:31:59.207578Z","shell.execute_reply":"2024-12-07T17:31:59.216750Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"xgb_params = {\n    'learning_rate': np.linspace(0.01, 0.1, num=100).tolist(),\n    'max_depth': [3, 5, 8, 10, 12],\n    'n_estimators': list(range(10, 350, 5)),\n    'subsample': np.arange(0.1, 1.01, 0.05).tolist(),\n    'colsample_bytree': np.arange(0.1, 1.01, 0.05).tolist(),\n    'reg_alpha': np.arange(0.1, 10, 0.5).tolist(), \n    'reg_lambda': np.arange(0.1, 10, 0.5).tolist(),\n    'random_state': [42],\n    'tree_method': ['hist', 'approx', 'gpu_hist'],\n    'device': ['cuda']\n}","metadata":{"execution":{"iopub.status.busy":"2024-12-07T17:31:59.218660Z","iopub.execute_input":"2024-12-07T17:31:59.219094Z","iopub.status.idle":"2024-12-07T17:31:59.226474Z","shell.execute_reply.started":"2024-12-07T17:31:59.219065Z","shell.execute_reply":"2024-12-07T17:31:59.225595Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cat_boost_params = {\n    'learning_rate': np.linspace(0.01, 0.1, num=100).tolist(),\n    'depth': [3, 4, 5, 6, 7],\n    'iterations': list(range(10, 250, 10)),\n    'random_seed': [42],\n    'verbose': [0],\n    'l2_leaf_reg': list(range(10, 50, 10)),\n    'task_type': ['GPU']\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T17:31:59.227443Z","iopub.execute_input":"2024-12-07T17:31:59.227676Z","iopub.status.idle":"2024-12-07T17:31:59.235924Z","shell.execute_reply.started":"2024-12-07T17:31:59.227653Z","shell.execute_reply":"2024-12-07T17:31:59.235147Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')","metadata":{"execution":{"iopub.status.busy":"2024-12-07T17:33:55.084041Z","iopub.execute_input":"2024-12-07T17:33:55.084733Z","iopub.status.idle":"2024-12-07T17:33:55.088626Z","shell.execute_reply.started":"2024-12-07T17:33:55.084698Z","shell.execute_reply":"2024-12-07T17:33:55.087759Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def search_optimal_param(X: pd.DataFrame, model, param: dict = {}, best = True):\n    \n    skfolds = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n    \n    x = X.copy()\n    y = x['sii']\n    _ = x.pop('sii') \n    \n    for num_fold, (train_index, val_index) in enumerate(skfolds.split(x, y)):\n        X_train, X_val = x.iloc[train_index], x.iloc[val_index]\n        y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n        \n        model_fold = clone(model)\n        kappa_scorer = make_scorer(cohen_kappa_score)\n        \n        if best:\n            search = model_fold\n        else:\n            search = RandomizedSearchCV(model_fold, param, scoring = kappa_scorer)\n        \n        search.fit(X_train, y_train)\n        predictions = search.predict(X_val)  \n        predictions = np.round(predictions)\n        \n        kappa_fold = quadratic_weighted_kappa(y_val, predictions)\n        \n        if num_fold == 0:\n            print('fold: ', num_fold+1)\n            best_model_score = kappa_fold\n            if not best:\n                best_model = search.best_estimator_  \n                best_params = search.best_params_\n                print('best_params:', best_params)\n            print('kappa: ', kappa_fold)\n            \n        elif kappa_fold > best_model_score:\n            print('fold: ', num_fold+1)\n            best_model_score = kappa_fold\n            if not best:\n                best_model = search.best_estimator_  \n                best_params = search.best_params_\n                print('best_params:', best_params)\n            print('kappa: ', kappa_fold)\n\n    if best:\n        return search, best_model_score\n    \n    return best_model, best_params, best_model_score","metadata":{"execution":{"iopub.status.busy":"2024-12-07T17:33:57.268512Z","iopub.execute_input":"2024-12-07T17:33:57.268820Z","iopub.status.idle":"2024-12-07T17:33:57.276962Z","shell.execute_reply.started":"2024-12-07T17:33:57.268796Z","shell.execute_reply":"2024-12-07T17:33:57.276050Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# kappa:  0.45099520721736674\nxgb_best_params = {'tree_method': 'hist', \n                   'subsample': 0.6000000000000002, \n                   'reg_lambda': 4.1, \n                   'reg_alpha': 9.6, \n                   'random_state': 42, \n                   'n_estimators': 260, \n                   'max_depth': 12, \n                   'learning_rate': 0.020909090909090912, \n                   'device': 'cuda', \n                   'colsample_bytree': 0.9500000000000003}\n\ntraning = False\nif traning:\n    xgb = XGBRegressor()\n    xgb_best_model = XGBRegressor(**xgb_best_params)\n    xgb_best_score = 0\n    \n    BOLD = '\\033[1m'\n    RESET = '\\033[0m'\n    \n    epoch = 5\n    for i in range(epoch):\n        print(f\"{BOLD}{'epoch: '}{i+1}{RESET}\")\n        if i == 0:\n            xgb_best_model, xgb_best_score = search_optimal_param(train, xgb_best_model)\n            print('xgb_best_score from default param', xgb_best_score)\n            \n        else:\n            xgb_batch_model, xgb_batch_params, xgb_batch_score = search_optimal_param(train, xgb, xgb_params, False)\n            if xgb_batch_score > xgb_best_score:\n                \n                xgb_best_model = xgb_batch_model\n                xgb_best_params = xgb_batch_params\n                xgb_best_score = xgb_batch_score\n                \n                print(f\"{BOLD}{'new best_batch kappa: ' + str(xgb_best_score)}{RESET}\")\n                print(f\"{BOLD}{'new_best_batch_params:'}{RESET}\", xgb_best_params)","metadata":{"execution":{"iopub.status.busy":"2024-12-07T18:03:38.868593Z","iopub.execute_input":"2024-12-07T18:03:38.869303Z","iopub.status.idle":"2024-12-07T18:03:38.875940Z","shell.execute_reply.started":"2024-12-07T18:03:38.869269Z","shell.execute_reply":"2024-12-07T18:03:38.874929Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# new best_batch kappa: 0.47338742951450963\nlgb_best_params = {'num_leaves': 270, \n                   'n_estimators': 320, \n                   'min_data_in_leaf': 40, \n                   'max_depth': 8, \n                   'learning_rate': 0.022727272727272728, \n                   'lambda_l2': 2.1794871794871793, \n                   'lambda_l1': 4.23076923076923, \n                   'feature_fraction': 0.874, \n                   'device': 'gpu', \n                   'bagging_freq': 28, \n                   'bagging_fraction': 0.318}\n\ntraning = False\nif traning:\n    lgb = LGBMRegressor(verbose=-1)\n    lgb_best_model = LGBMRegressor(**lgb_best_params, verbose=-1)\n    lgb_best_score = 0\n    \n    BOLD = '\\033[1m'\n    RESET = '\\033[0m'\n    \n    epoch = 5    \n    for i in range(epoch):\n        print(f\"{BOLD}{'epoch: '}{i+1}{RESET}\")\n        if i == 0:\n            lgb_best_model, lgb_best_score = search_optimal_param(train, lgb_best_model)\n            print('lgb_best_score from default param', lgb_best_score)\n        else:\n            lgb_batch_model, lgb_batch_params, lgb_batch_score = search_optimal_param(train, lgb, lgb_params, False)\n            if lgb_batch_score > lgb_best_score:\n                \n                lgb_best_model = lgb_batch_model\n                lgb_best_params = lgb_batch_params\n                lgb_best_score = lgb_batch_score\n                \n                print(f\"{BOLD}{'new best_batch kappa: ' + str(lgb_best_score)}{RESET}\")\n                print(f\"{BOLD}{'new_best_batch_params:'}{RESET}\", lgb_best_params)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T18:03:42.466259Z","iopub.execute_input":"2024-12-07T18:03:42.467123Z","iopub.status.idle":"2024-12-07T18:03:42.473966Z","shell.execute_reply.started":"2024-12-07T18:03:42.467088Z","shell.execute_reply":"2024-12-07T18:03:42.473090Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# new best_batch kappa: 0.43985147547391046\ncb_best_params = {'verbose': 0, \n                  'task_type': 'GPU', \n                  'random_seed': 42, \n                  'learning_rate': 0.08454545454545455, \n                  'l2_leaf_reg': 30, \n                  'iterations': 100, \n                  'depth': 5}\n\ntraning = False\nif traning:\n    cat_boost = CatBoostRegressor(cat_features = categorical_columns)\n    cb_best_model = CatBoostRegressor(**cb_best_params, cat_features = categorical_columns)\n    cb_best_score = 0\n    \n    BOLD = '\\033[1m'\n    RESET = '\\033[0m'\n    \n    epoch = 5     \n    for i in range(epoch):\n        print(f\"{BOLD}{'epoch: '}{i+1}{RESET}\")\n        if i == 0:\n            cb_best_model, cb_best_score = search_optimal_param(train, cb_best_model)\n            print('cb_best_score from default param', cb_best_score)\n        else:\n            cb_batch_model, cb_batch_params, cb_batch_score = search_optimal_param(train, cat_boost, cat_boost_params, False)\n            if cb_batch_score > cb_best_score:\n                \n                cb_best_model = cb_batch_model\n                cb_best_params = cb_batch_params\n                cb_best_score = cb_batch_score\n                \n                print(f\"{BOLD}{'new best_batch kappa: ' + str(cb_best_score)}{RESET}\")\n                print(f\"{BOLD}{'new_best_batch_params:'}{RESET}\", cb_best_params)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T18:03:45.542250Z","iopub.execute_input":"2024-12-07T18:03:45.542644Z","iopub.status.idle":"2024-12-07T18:03:45.549271Z","shell.execute_reply.started":"2024-12-07T18:03:45.542610Z","shell.execute_reply":"2024-12-07T18:03:45.548393Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"SEED = 42\nn_splits = 5","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T18:03:48.695107Z","iopub.execute_input":"2024-12-07T18:03:48.695461Z","iopub.status.idle":"2024-12-07T18:03:48.699729Z","shell.execute_reply.started":"2024-12-07T18:03:48.695430Z","shell.execute_reply":"2024-12-07T18:03:48.698756Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n    return -quadratic_weighted_kappa(y_true, rounded_p)","metadata":{"execution":{"iopub.status.busy":"2024-12-07T18:03:50.940419Z","iopub.execute_input":"2024-12-07T18:03:50.940987Z","iopub.status.idle":"2024-12-07T18:03:50.945529Z","shell.execute_reply.started":"2024-12-07T18:03:50.940956Z","shell.execute_reply":"2024-12-07T18:03:50.944670Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def TrainML(model_class, test_data):\n    \n    X = train.drop(['sii'], axis=1)    \n    y = train['sii']   \n\n    X_res, y_res = X, y\n\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n    train_S = []\n    test_S = []\n    oof_non_rounded = np.zeros(len(y_res), dtype=float)\n    oof_rounded = np.zeros(len(y_res), dtype=int)\n    test_preds = np.zeros((len(test_data), n_splits))\n\n    for fold, (train_idx, val_idx) in enumerate(SKF.split(X_res, y_res)):\n        X_train, X_val = X_res.iloc[train_idx], X_res.iloc[val_idx]\n        y_train, y_val = y_res.iloc[train_idx], y_res.iloc[val_idx]\n\n        model = clone(model_class)\n        model.fit(X_train, y_train)\n        y_train_pred = model.predict(X_train)\n        y_val_pred = model.predict(X_val)\n\n        oof_non_rounded[val_idx] = y_val_pred\n        y_val_pred_rounded = np.round(y_val_pred).astype(int)\n        oof_rounded[val_idx] = y_val_pred_rounded\n\n        train_kappa = quadratic_weighted_kappa(y_train, np.round(y_train_pred).astype(int))\n        val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n        train_S.append(train_kappa)\n        test_S.append(val_kappa)\n\n        test_preds[:, fold] = model.predict(test_data)\n\n        print(f\"Fold {fold + 1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n        clear_output(wait=True)\n\n    print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n    # Optimize thresholds with Nelder-Mead method\n    KappaOptimizer = minimize(evaluate_predictions, x0=[0.5, 1.5, 2.5], args=(y_res, oof_non_rounded), method='Nelder-Mead')\n    assert KappaOptimizer.success, \"Optimization did not converge.\"\n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOptimizer.x)\n    tKappa = quadratic_weighted_kappa(y_res, oof_tuned)\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n    tpm = test_preds.mean(axis=1)\n    tpTuned = threshold_Rounder(tpm, KappaOptimizer.x)\n    submission = pd.DataFrame({\n        'id': sample['id'],\n        'sii': tpTuned\n    })\n\n    return submission","metadata":{"execution":{"iopub.status.busy":"2024-12-07T18:03:54.236136Z","iopub.execute_input":"2024-12-07T18:03:54.236953Z","iopub.status.idle":"2024-12-07T18:03:54.246138Z","shell.execute_reply.started":"2024-12-07T18:03:54.236920Z","shell.execute_reply":"2024-12-07T18:03:54.245288Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create model instances\nLight = LGBMRegressor(**lgb_best_params, verbose=-1)  \nXGB_Model = XGBRegressor(**xgb_best_params)\nCatBoost_Model = CatBoostRegressor(**cb_best_params, cat_features = categorical_columns)\n\n# Combine models using Voting Regressor\nvoting_model = VotingRegressor(estimators=[\n    ('lightgbm', Light),\n    ('xgboost', XGB_Model),\n    ('catboost', CatBoost_Model)\n], weights=[20, 10, 20])","metadata":{"execution":{"iopub.status.busy":"2024-12-07T18:03:59.407175Z","iopub.execute_input":"2024-12-07T18:03:59.407753Z","iopub.status.idle":"2024-12-07T18:03:59.414872Z","shell.execute_reply.started":"2024-12-07T18:03:59.407720Z","shell.execute_reply":"2024-12-07T18:03:59.414038Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission = TrainML(voting_model, test)","metadata":{"execution":{"iopub.status.busy":"2024-12-07T18:04:01.915992Z","iopub.execute_input":"2024-12-07T18:04:01.916659Z","iopub.status.idle":"2024-12-07T18:04:34.075638Z","shell.execute_reply.started":"2024-12-07T18:04:01.916629Z","shell.execute_reply":"2024-12-07T18:04:34.074811Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Save submission\nsubmission.to_csv('submission.csv', index=False)\nprint(submission['sii'].value_counts())","metadata":{"execution":{"iopub.status.busy":"2024-12-07T18:04:36.197296Z","iopub.execute_input":"2024-12-07T18:04:36.198194Z","iopub.status.idle":"2024-12-07T18:04:36.205681Z","shell.execute_reply.started":"2024-12-07T18:04:36.198161Z","shell.execute_reply":"2024-12-07T18:04:36.204904Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2024-12-07T18:04:38.322558Z","iopub.execute_input":"2024-12-07T18:04:38.322914Z","iopub.status.idle":"2024-12-07T18:04:38.332004Z","shell.execute_reply.started":"2024-12-07T18:04:38.322882Z","shell.execute_reply":"2024-12-07T18:04:38.331000Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null}]}