{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.14"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"},{"sourceId":10360403,"sourceType":"datasetVersion","datasetId":6415368}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false},"papermill":{"default_parameters":{},"duration":544.948362,"end_time":"2024-12-18T21:53:00.290055","environment_variables":{},"exception":true,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-12-18T21:43:55.341693","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# =============================================================================\n# IMPORTS & SETTINGS\n# =============================================================================\nimport os\nimport random\nimport warnings\nimport joblib\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nfrom scipy import stats\nfrom scipy.optimize import minimize\nfrom concurrent.futures import ThreadPoolExecutor\nfrom tqdm import tqdm\n\nimport catboost as ctb\nimport lightgbm as lgb\nimport xgboost as xgb\n\nimport tensorflow as tf\nimport keras\nfrom keras import Model\nfrom keras.layers import Input, Dense\nfrom keras.optimizers import Adam\nfrom tensorflow.keras import layers, models, Sequential\nfrom tensorflow.keras import Model as keras_model\nfrom tensorflow.keras.layers import (\n    Embedding, Concatenate, Flatten, Dropout, BatchNormalization\n)\n\nfrom sklearn.model_selection import (\n    StratifiedKFold, KFold, train_test_split, RepeatedStratifiedKFold\n)\nfrom sklearn.metrics import (\n    cohen_kappa_score, mean_squared_error\n)\nfrom sklearn.preprocessing import (\n    LabelEncoder, StandardScaler, OneHotEncoder, MinMaxScaler\n)\nfrom sklearn.impute import KNNImputer\nfrom sklearn.tree import DecisionTreeRegressor\nfrom sklearn.cluster import KMeans, DBSCAN\nfrom sklearn.linear_model import LogisticRegression, Ridge, LinearRegression, Lasso\nfrom sklearn.ensemble import (\n    HistGradientBoostingClassifier, RandomForestClassifier\n)\nfrom keras.utils import set_random_seed\n\nimport absl.logging\nabsl.logging.set_verbosity(absl.logging.ERROR)\n\n# =============================================================================\n# SEED & WARNINGS\n# =============================================================================\nRND_SEED = 42\nwarnings.filterwarnings('ignore')\nrandom.seed(RND_SEED)\nnp.random.seed(RND_SEED)\ntf.random.set_seed(RND_SEED)\nset_random_seed(RND_SEED)\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'\nos.environ['PYTHONHASHSEED'] = str(RND_SEED)\nos.environ['TF_DETERMINISTIC_OPS'] = '1'\ntf.config.experimental.enable_op_determinism() ","metadata":{"_uuid":"c53fdf8b-4f54-4f47-a750-8d9ca098f429","_cell_guid":"cbc7fe03-41de-4ee1-909a-6589b37bc42a","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-01-05T17:49:39.820154Z","iopub.execute_input":"2025-01-05T17:49:39.820592Z","iopub.status.idle":"2025-01-05T17:49:44.971534Z","shell.execute_reply.started":"2025-01-05T17:49:39.820555Z","shell.execute_reply":"2025-01-05T17:49:44.970483Z"},"papermill":{"duration":29.826797,"end_time":"2024-12-18T21:45:14.670714","exception":false,"start_time":"2024-12-18T21:44:44.843917","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =============================================================================\n# DATA READING\n# =============================================================================\ntrain = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')","metadata":{"_uuid":"f3742379-2436-4c3d-a0c6-4dbdda192bcc","_cell_guid":"3c8ff3c7-d55a-41dc-9ccd-ba24f7020944","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-01-05T17:49:44.973337Z","iopub.execute_input":"2025-01-05T17:49:44.974141Z","iopub.status.idle":"2025-01-05T17:49:45.065417Z","shell.execute_reply.started":"2025-01-05T17:49:44.974102Z","shell.execute_reply":"2025-01-05T17:49:45.064221Z"},"papermill":{"duration":0.109619,"end_time":"2024-12-18T21:45:14.792067","exception":false,"start_time":"2024-12-18T21:45:14.682448","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =============================================================================\n# CONSTANTS\n# =============================================================================\nTARGET_sii = 'sii'\nTARGET_PC = 'PCIAT-PCIAT_Total'\nTARGET_binned = 'binned'\nTARGET_cols = [TARGET_sii, TARGET_binned, TARGET_PC]\n\nTHRESHOLD = [31, 50, 80 ]\n# Normalized thresholds for the “binned” version of PCIAT_Total\nTHRESHOLD_1 = [31*9/93, 50*9/93, 80*9/93]\n","metadata":{"_uuid":"874b9bf0-2f22-47e2-9f5b-49f6b1f9eaf1","_cell_guid":"7a18653d-2970-4998-88bb-66d629a85438","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-01-05T17:49:45.066996Z","iopub.execute_input":"2025-01-05T17:49:45.067348Z","iopub.status.idle":"2025-01-05T17:49:45.073498Z","shell.execute_reply.started":"2025-01-05T17:49:45.067314Z","shell.execute_reply":"2025-01-05T17:49:45.072220Z"},"papermill":{"duration":0.019506,"end_time":"2024-12-18T21:45:14.822903","exception":false,"start_time":"2024-12-18T21:45:14.803397","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Identify numeric and categorical columns\nall_cols = train.columns.to_list()\nfeature_cols = [\n    col for col in all_cols\n    if ('PCIAT' not in col) and (col not in ['sii', 'id', 'binned'])\n]\nnum_cols = train[feature_cols].select_dtypes(['number']).columns.to_list()\ncat_cols = [col for col in feature_cols if 'Season' in col]\n\n# Convert seasonal columns to numeric categories\ntrain[cat_cols] = train[cat_cols].replace({'Spring':1,'Summer':2,'Fall':3,'Winter':4})\ntest[cat_cols] = test[cat_cols].replace({'Spring':1,'Summer':2,'Fall':3,'Winter':4})\n\ntrain[cat_cols] = train[cat_cols].fillna(0).astype('int').astype('category')\ntest[cat_cols] = test[cat_cols].fillna(0).astype('int').astype('category')","metadata":{"_uuid":"abc72e95-7331-409d-9926-fc32157854ac","_cell_guid":"9af88d96-6d04-40af-a5b8-cd4eac7efb86","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-01-05T17:49:45.076040Z","iopub.execute_input":"2025-01-05T17:49:45.076541Z","iopub.status.idle":"2025-01-05T17:49:45.141786Z","shell.execute_reply.started":"2025-01-05T17:49:45.076496Z","shell.execute_reply":"2025-01-05T17:49:45.140740Z"},"papermill":{"duration":0.08225,"end_time":"2024-12-18T21:45:14.915410","exception":false,"start_time":"2024-12-18T21:45:14.833160","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =============================================================================\n# DATA ENCODING & SCALING FUNCTIONS\n# =============================================================================\ndef label_encode(df_train, df_test, cols):\n    train_le = df_train.copy()\n    test_le = df_test.copy()\n    cardinality = {}\n    encoder = {}\n    for col in cols:\n        le = LabelEncoder()\n        le.fit(train_le[col])\n        train_le[col] = le.transform(train_le[col])\n        test_le[col] = le.transform(test_le[col])\n        cardinality[col] = len(le.classes_)\n        encoder[col] = le\n    return train_le, test_le, cardinality, encoder\n\ndef scaler_encode(df_train, df_test, cols):\n    tr = df_train.copy()\n    ts = df_test.copy()\n    scaler = StandardScaler()\n    tr[cols] = scaler.fit_transform(tr[cols])\n    ts[cols] = scaler.transform(ts[cols])\n    return tr, ts\ndef one_hot_encode(df_train, df_test, cols):\n    df_oh = df_train.copy()\n    ts_oh = df_test.copy()\n    bin_cols = []\n    for col in cols:\n        ohe = OneHotEncoder(sparse_output=False, handle_unknown='ignore')\n        temp_df = ohe.fit_transform(df_oh[[col]])\n        temp_ts = ohe.transform(ts_oh[[col]])\n        feature_names = [f\"{col}_{category}\" for category in ohe.categories_[0]]\n        enc_df = pd.DataFrame(temp_df, columns=feature_names)\n        enc_ts = pd.DataFrame(temp_ts, columns=feature_names)\n        df_oh = pd.concat([enc_df, df_oh], axis=1).drop(columns=[col])\n        ts_oh = pd.concat([enc_ts, ts_oh], axis=1).drop(columns=[col])      \n        bin_cols.extend(feature_names)\n    return df_oh, ts_oh, bin_cols\n","metadata":{"_uuid":"5fe985cb-dbdb-4b03-baad-342469a1d432","_cell_guid":"13c17c0e-47cb-4620-b529-604f04f07135","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-01-05T17:49:45.143367Z","iopub.execute_input":"2025-01-05T17:49:45.143852Z","iopub.status.idle":"2025-01-05T17:49:45.155165Z","shell.execute_reply.started":"2025-01-05T17:49:45.143803Z","shell.execute_reply":"2025-01-05T17:49:45.153827Z"},"papermill":{"duration":0.026235,"end_time":"2024-12-18T21:45:14.951783","exception":false,"start_time":"2024-12-18T21:45:14.925548","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =============================================================================\n# PARQUET READING & PROCESSING FOR TIME SERIES\n# =============================================================================\n'''\nBelow, we read multiple parquet files. We first figure out the variance of time series columns\nthen pick those with higher variance to train KMeans. We cluster new data using that model.\n'''\n\n\ndef process_file_whole(filename, dirname):\n    \"\"\"\n    Reads a parquet file, removes null values, \n    and returns time series variance and other stats in a dict.\n    \"\"\"\n    df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    df.drop('step', axis=1, inplace=True)\n\n    # For collection == 0: gather stats\n    if collection == 0:\n        df.dropna(inplace=True)\n        n_ts = {\n            'size': len(df),\n            'var': df['X'].var() + df['Y'].var() + df['Z'].var() + df['enmo'].var(),\n            'non_wear': df['non-wear_flag'].sum(),\n            'id': filename.split('=')[1]\n        }\n        return n_ts\n    \n    # For collection == 1: read raw data\n    if collection == 1:\n        return df\n\ndef load_time_series_whole(dirname):\n    \"\"\"\n    If collection == 0, returns a dataframe of variance stats from all IDs.\n    If collection == 1, merges the selected IDs (id_call) data.\n    \"\"\"\n    ids = os.listdir(dirname)\n\n    if collection == 0:\n        with ThreadPoolExecutor() as executor:\n            results = list(tqdm(executor.map(\n                lambda fname: process_file_whole(fname, dirname), \n                ids), total=len(ids))\n            )\n        return pd.DataFrame(results)\n\n    elif collection == 1:\n        data_id = []\n        for id_ in id_call:\n            data_id.append(process_file_whole('id='+id_, dirname))\n        return pd.concat(data_id, ignore_index=True)\n\n# First pass: figure out variance\ncollection = 0\npr_data = load_time_series_whole(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\npr_data = pd.merge(pr_data, train[['sii','id']], how='left', on='id')\npr_data['non-wear_size'] = pr_data['non_wear'] / pr_data['size']\n\n# Pick top variance from each SII=0..3\nid_call = []\nfor j in range(4):\n    index_var = pr_data[pr_data['sii'] == j]['var'].nlargest(3).index\n    id_call.extend(pr_data.loc[index_var]['id'].values)\n\n# Second pass: read in those high variance files\ncollection = 1\nprq_train = load_time_series_whole(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\n\n# =============================================================================\n# KMeans CLUSTERING\n# =============================================================================\n# Prepare data for clustering\nprq_train['time-diff'] = (\n    prq_train['time_of_day'] - prq_train['time_of_day'].shift(1)\n).where(\n    prq_train['relative_date_PCIAT'] == prq_train['relative_date_PCIAT'].shift(1), \n    0\n)\n\nmovment_cols = ['X','Y','Z','enmo','anglez','light','time-diff']\nscaler = StandardScaler()\nX_scaled = scaler.fit_transform(prq_train[movment_cols])\n\nk = 15\nkmeans = KMeans(n_clusters=k, random_state=RND_SEED)\nkmeans.fit(X_scaled)","metadata":{"_uuid":"5c9af04d-2da9-43ee-9114-d04f3c13df85","_cell_guid":"919b7300-42f6-4e03-94bb-c0eb7859f681","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-01-05T17:49:45.156502Z","iopub.execute_input":"2025-01-05T17:49:45.157017Z","iopub.status.idle":"2025-01-05T17:52:21.368348Z","shell.execute_reply.started":"2025-01-05T17:49:45.156954Z","shell.execute_reply":"2025-01-05T17:52:21.366565Z"},"papermill":{"duration":159.649963,"end_time":"2024-12-18T21:47:54.612064","exception":false,"start_time":"2024-12-18T21:45:14.962101","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def extract_movment(parquet_data):\n    \"\"\"\n    Applies the trained KMeans model on a parquet dataframe,\n    returns the proportion of each cluster's 'time-diff' sum to the entire 'time-diff'.\n    \"\"\"\n    pq = parquet_data.copy()\n    pq['time-diff'] = (\n        pq['time_of_day'] - pq['time_of_day'].shift(1)\n    ).where(\n        pq['relative_date_PCIAT'] == pq['relative_date_PCIAT'].shift(1),\n        0\n    )\n    X = StandardScaler().fit_transform(pq[movment_cols])\n    clusters = kmeans.predict(X)\n    pq['cluster'] = clusters\n\n    mov = {}\n    for i in range(k):\n        cluster_df = pq[pq['cluster'] == i]\n        # Ratio of that cluster’s time-diff sum to the entire time-diff sum\n        mov[f'movement_{i+1}_mean'] = cluster_df['time-diff'].sum() / pq['time-diff'].sum()\n    return mov","metadata":{"_uuid":"e6effa4f-0602-4036-b1f5-1ca19681ed08","_cell_guid":"2a490d59-e534-4f76-8402-32fcd07a7ed8","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-01-05T17:52:21.370923Z","iopub.execute_input":"2025-01-05T17:52:21.371332Z","iopub.status.idle":"2025-01-05T17:52:21.381900Z","shell.execute_reply.started":"2025-01-05T17:52:21.371299Z","shell.execute_reply":"2025-01-05T17:52:21.379938Z"},"papermill":{"duration":0.032515,"end_time":"2024-12-18T21:47:54.665200","exception":false,"start_time":"2024-12-18T21:47:54.632685","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def process_file(filename, dirname):\n    \"\"\"\n    Reads a parquet file, \n    extracts movement clusters with extract_movment().\n    \"\"\"\n    df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    df.drop('step', axis=1, inplace=True)\n    df.dropna(inplace=True)\n    n_ts = extract_movment(df)\n    n_ts['id'] = filename.split('=')[1]\n    return n_ts\n\ndef load_time_series(dirname):\n    \"\"\"\n    Loads all parquet files in a folder, \n    extracts cluster-based movement features for each, \n    and returns them in a dataframe.\n    \"\"\"\n    ids = os.listdir(dirname)\n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(\n            lambda fname: process_file(fname, dirname),\n            ids), total=len(ids))\n        )\n    return pd.DataFrame(results)\n\n# Load and merge with train/test data\ntrain_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\n\ntrain_ts.fillna(0, inplace=True)\ntest_ts.fillna(0, inplace=True)\ntrain_ts = pd.merge(train, train_ts, how=\"left\", on='id')\ntest_ts = pd.merge(test, test_ts, how=\"left\", on='id')\n\n# Time-series feature columns\nts_cols = [col for col in train_ts.columns if 'movement' in col]\nnum_cols_ts = num_cols + ts_cols\nfeature_cols_ts = feature_cols + ts_cols","metadata":{"_uuid":"02072727-0be6-4858-9bcd-a9fb6872c7a3","_cell_guid":"5d9cb04c-748c-4e4b-b787-712fb53efc23","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-01-05T17:52:21.383568Z","iopub.execute_input":"2025-01-05T17:52:21.383911Z","iopub.status.idle":"2025-01-05T17:54:10.159311Z","shell.execute_reply.started":"2025-01-05T17:52:21.383881Z","shell.execute_reply":"2025-01-05T17:54:10.158134Z"},"papermill":{"duration":108.892052,"end_time":"2024-12-18T21:49:43.576475","exception":false,"start_time":"2024-12-18T21:47:54.684423","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =============================================================================\n# CASTING CATEGORICAL & IMPUTATION\n# =============================================================================\ntrain[cat_cols] = train[cat_cols].astype('int').astype('category')\ntest[cat_cols] = test[cat_cols].astype('int').astype('category')\n\n# Impute numeric features with KNN\nimputer = KNNImputer(n_neighbors=10)\ntrain_imputed = imputer.fit_transform(train[num_cols])\ntest_imputed = imputer.transform(test[num_cols])\n\ntrain[num_cols] = pd.DataFrame(train_imputed, columns=num_cols)\ntest[num_cols] = pd.DataFrame(test_imputed, columns=num_cols)\n\n# Impute numeric features for TS-extended data\nimputer_ts = KNNImputer(n_neighbors=10)\ntrain_imputed_ts = imputer_ts.fit_transform(train_ts[num_cols_ts])\ntest_imputed_ts = imputer_ts.transform(test_ts[num_cols_ts])\n\ntrain_ts[num_cols_ts] = pd.DataFrame(train_imputed_ts, columns=num_cols_ts)\ntest_ts[num_cols_ts] = pd.DataFrame(test_imputed_ts, columns=num_cols_ts)\n","metadata":{"_uuid":"17602594-ed51-429f-a11f-5b0da6398210","_cell_guid":"b61746fb-7467-4041-a0da-d0afd580ea70","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-01-05T17:54:10.160863Z","iopub.execute_input":"2025-01-05T17:54:10.161216Z","iopub.status.idle":"2025-01-05T17:54:22.803221Z","shell.execute_reply.started":"2025-01-05T17:54:10.161182Z","shell.execute_reply":"2025-01-05T17:54:22.802119Z"},"papermill":{"duration":0.068036,"end_time":"2024-12-18T21:49:43.684761","exception":false,"start_time":"2024-12-18T21:49:43.616725","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =============================================================================\n# SPLIT TRAIN INTO LABELED & UNLABELED\n# =============================================================================\ntrain_missing = train[train[TARGET_sii].isna()].copy().reset_index(drop=True)\ntrain_ts = train_ts.dropna(subset=[TARGET_sii]).reset_index(drop=True)\ntrain = train.dropna(subset=[TARGET_sii]).reset_index(drop=True)\n\n# Create “binned” column for the train\ntrain[TARGET_binned] = pd.cut(train[TARGET_PC], bins=10, labels=False)\n","metadata":{"_uuid":"e83f02f2-940a-4bdd-8e80-5b34672d6861","_cell_guid":"1b335b02-d99c-4f48-8c48-073e44fe2167","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-01-05T17:54:22.807094Z","iopub.execute_input":"2025-01-05T17:54:22.807513Z","iopub.status.idle":"2025-01-05T17:54:22.835912Z","shell.execute_reply.started":"2025-01-05T17:54:22.807475Z","shell.execute_reply":"2025-01-05T17:54:22.834714Z"},"papermill":{"duration":0.07517,"end_time":"2024-12-18T21:49:56.590941","exception":false,"start_time":"2024-12-18T21:49:56.515771","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# using sample weight idea from :https://www.kaggle.com/code/lennarthaupts/cmi-detecting-problematic-digital-behavior.\ndef label_weight(data):\n    bins = pd.cut(data, bins=10, labels=False)\n    bin_counts = bins.value_counts(normalize=True)\n    weight_map = (1 / bin_counts).to_dict()\n    weights = bins.map(weight_map)\n    return weights / weights.mean()","metadata":{"_uuid":"b9d81929-6bc6-40fb-9fa4-2dbc58bd96e7","_cell_guid":"390b8e46-7574-43cf-a4ef-5de5478f9fb8","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-01-05T17:54:22.837223Z","iopub.execute_input":"2025-01-05T17:54:22.837596Z","iopub.status.idle":"2025-01-05T17:54:22.843816Z","shell.execute_reply.started":"2025-01-05T17:54:22.837556Z","shell.execute_reply":"2025-01-05T17:54:22.842439Z"},"papermill":{"duration":0.049732,"end_time":"2024-12-18T21:49:56.680967","exception":false,"start_time":"2024-12-18T21:49:56.631235","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def quadratic_weighted_kappa(target, preds):\n    return cohen_kappa_score(target, preds, weights=\"quadratic\")\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n           np.where(oof_non_rounded < thresholds[1], 1,\n           np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\ndef multiple_rmse_scores(actual, predicted):\n    errors = []\n    for i in range(predicted.shape[1]):\n        errors.append(mean_squared_error(actual, predicted[:, i]))\n    return errors\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T17:54:22.844877Z","iopub.execute_input":"2025-01-05T17:54:22.845186Z","iopub.status.idle":"2025-01-05T17:54:22.860222Z","shell.execute_reply.started":"2025-01-05T17:54:22.845157Z","shell.execute_reply":"2025-01-05T17:54:22.858666Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =============================================================================\n# MODEL INIT CLASS\n# =============================================================================\n'''\nThis class performs cross-validation training for different models \nand returns out-of-fold predictions (oof) plus test predictions.\n'''\nclass model_init:\n    def __init__(self, model_class, params):\n        self.model_class = model_class\n        self.model_params = params\n    \n    def model_build(self):\n        \"\"\"\n        Builds a fresh model from the class and params given.\n        If it's a Keras model, we handle it differently.\n        \"\"\"\n        if (self.model_class != keras_model):\n            self.model = self.model_class(**self.model_params)\n        else:\n            self.model = self.model_class()\n    \n    def Train(self, df_train, df_test, final_target, n_split):\n        \"\"\"\n        Manages cross-validation for the specified model.\n        Returns OOF predictions, test predictions, overall QWK, and the model itself.\n        \"\"\"\n        if 'original' in df_train.columns:\n            self.train = df_train[df_train['original'] == 1].drop('original', axis=1).reset_index(drop=True)\n            self.plus = df_train[df_train['original'] == 0].drop('original', axis=1).reset_index(drop=True)\n        else:\n            self.train = df_train\n            self.plus = None\n        \n        self.test = df_test\n        self.target_col = final_target\n        self.n_split = n_split\n\n        return self.fit_cv()\n    \n    def fit(self, X_train, y_train, X_val, y_val):\n        \"\"\"\n        Fits a single fold of data.\n        Uses sample weighting for certain runs.\n        \"\"\"\n        if WH == 1:\n            weights = label_weight(y_train)\n        else:\n            weights = np.ones(len(y_train))\n\n        # Keras model\n        if isinstance(self.model, keras_model):\n            self.model = build_nn_model()  # builds the structure\n            self.model.compile(\n                optimizer=keras.optimizers.AdamW(\n                    learning_rate=learning_rate,\n                    weight_decay=0.01,\n                    beta_1=0.9,\n                    beta_2=0.999,\n                    epsilon=1e-07\n                ),\n                loss='mse',\n                metrics=['mse']\n            )\n            self.model.fit(\n                nn_enc(X_train),\n                y_train,\n                batch_size=BS,\n                epochs=epochs,\n                sample_weight=weights,\n                validation_data=(nn_enc(X_val), y_val),\n                verbose=0\n            )\n        else:\n            # Classic ML\n            self.model.fit(X_train, y_train, sample_weight=weights)\n\n    def predict(self, X):\n        \"\"\"\n        Predict either with a Keras model or a scikit-learn model.\n        \"\"\"\n        if isinstance(self.model, keras_model):\n            return self.model.predict(nn_enc(X), verbose=0, batch_size=BS).flatten()\n        else:\n            return self.model.predict(X)\n\n    def fit_cv(self):\n        \"\"\"\n        K-fold cross-validation: returns OOF predictions, test predictions, QWK, and model list.\n        \"\"\"\n        oof_pred = np.zeros(len(self.train))\n        test_pred = np.zeros(len(self.test))\n\n        kfold = StratifiedKFold(\n            n_splits=self.n_split, shuffle=True, random_state=RND_SEED\n        )\n\n        for folds, (idx_train, idx_val) in enumerate(\n            kfold.split(self.train, self.train[TARGET_sii])\n        ):\n            model_fold = []\n\n            if self.plus is None:\n                tr = self.train.loc[idx_train]\n            else:\n                tr = pd.concat(\n                    [self.train.loc[idx_train], self.plus],\n                    ignore_index=True\n                )\n            \n            X_train = tr.drop(TARGET_cols, axis=1)\n            y_train = tr[self.target_col]\n            X_val = self.train.loc[idx_val].drop(TARGET_cols, axis=1)\n            y_val = self.train.loc[idx_val][TARGET_sii]\n\n            self.model_build()\n            self.fit(X_train, y_train, X_val, y_val)\n            oof_pred[idx_val] = self.predict(X_val)\n            test_pred += self.predict(self.test) / self.n_split\n\n            # Round OOF predictions using threshold\n            rounded_oof_fold = threshold_Rounder(oof_pred[idx_val], THRESHOLD_1)\n            fold_scores_qwk = quadratic_weighted_kappa(y_val, rounded_oof_fold)\n            print(f\"\\nFold No:{folds+1} QWK_metrics : {fold_scores_qwk:.5f}\")\n\n            model_fold.append(self.model)\n\n        # Evaluate overall metrics\n        rounded_oof = threshold_Rounder(oof_pred, THRESHOLD_1)\n        overal_score = quadratic_weighted_kappa(self.train[TARGET_sii], rounded_oof)\n        overal_score_rmse = mean_squared_error(self.train[self.target_col], oof_pred)\n        print(f\"\\n-------Overall QWK Score: {overal_score:.5f}\")\n        print(f\"-------Overall rsme Score: {overal_score_rmse:.5f}\\n\")\n\n        return oof_pred, test_pred, overal_score, model_fold","metadata":{"_uuid":"81f7b5fa-f8ac-4e9c-ae9f-f2646af6b32c","_cell_guid":"819561b6-31b2-4276-8c18-b264318dbebb","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-01-05T17:54:22.862134Z","iopub.execute_input":"2025-01-05T17:54:22.862576Z","iopub.status.idle":"2025-01-05T17:54:22.886590Z","shell.execute_reply.started":"2025-01-05T17:54:22.862521Z","shell.execute_reply":"2025-01-05T17:54:22.885415Z"},"papermill":{"duration":0.064648,"end_time":"2024-12-18T21:49:56.785861","exception":false,"start_time":"2024-12-18T21:49:56.721213","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =============================================================================\n# NEURAL NETWORK UTILS\n# =============================================================================\ndef build_nn_model():\n    \"\"\"\n    Builds a base Keras model with embeddings for categorical features.\n    \"\"\"\n    if nn_model_type == 'base model':\n        input_categorical = []\n        embedding_layers = []\n\n        # Numeric inputs\n        cont_inputs = Input(shape=(len(num_cols_nn),))\n        keras.utils.set_random_seed(RND_SEED)\n\n        # Categorical embeddings\n        for col in cat_cols:\n            input_cat = Input(shape=(1,))\n            card = cat_card[col]\n            embed_dim = min(8, card // 2)  # rule of thumb\n            embedding = Embedding(input_dim=card, output_dim=embed_dim)(input_cat)\n            embedding = Flatten()(embedding)\n            input_categorical.append(input_cat)\n            embedding_layers.append(embedding)\n\n        # Combine all features\n        all_features = Concatenate()(\n            embedding_layers + [cont_inputs]\n        )\n\n        # Dense layers\n        nn_layer = Dense(256, activation='relu')(all_features)\n        nn_layer = Dropout(0.2, seed=RND_SEED)(nn_layer)\n        nn_layer = Dense(128, activation='relu')(nn_layer)\n        nn_layer = Dropout(0.2, seed=RND_SEED)(nn_layer)\n        nn_layer = Dense(64, activation='relu')(nn_layer)\n\n        # Final output\n        output = Dense(1)(nn_layer)\n        model = Model(inputs=[cont_inputs] + input_categorical, outputs=output)\n        return model\n\ndef nn_enc(df):\n    \"\"\"\n    Transforms the dataset into the input format expected by the base Keras model.\n    Returns a list: [numerical_cols] + [cat_col1, cat_col2, ...]\n    \"\"\"\n    return [df[num_cols_nn].values] + [df[col].values for col in cat_cols]\n","metadata":{"_uuid":"b3724477-64a3-46cd-ba25-575ac697497a","_cell_guid":"ced373a4-be6f-4061-8254-e1c3edd51ba2","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-01-05T17:54:22.888076Z","iopub.execute_input":"2025-01-05T17:54:22.888540Z","iopub.status.idle":"2025-01-05T17:54:22.906130Z","shell.execute_reply.started":"2025-01-05T17:54:22.888492Z","shell.execute_reply":"2025-01-05T17:54:22.904740Z"},"papermill":{"duration":0.05342,"end_time":"2024-12-18T21:49:56.972463","exception":false,"start_time":"2024-12-18T21:49:56.919043","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =============================================================================\n# HILL CLIMBING FOR ENSEMBLING\n# =============================================================================\nclass hill_climbing:\n    \"\"\"\n    Performs a hill-climbing approach to ensemble multiple model predictions \n    by searching for the best linear combination that lowers RMSE.\n    \"\"\"\n\n    def __init__(self, eval_metric, tol=1e-9, max_model=1000):\n        self.max_model = max_model\n        self.tol = tol\n        self.metric = eval_metric\n\n    def fit(self, train, target, score):\n        \"\"\"\n        Train: data frame with out-of-fold model predictions as columns\n        Target: real labels\n        Score: current best score\n        \"\"\"\n        best_score = 0\n        best_index = -1\n        indices = [best_index]\n        old_best_score = best_score\n        self.train = np.array(train)\n        self.target = np.array(target.values)\n\n        # Weight steps\n        start = -0.50\n        ww = np.arange(start, 0.51, 0.01)\n        nn = len(ww)\n        files = train.keys().to_list()\n\n        self.models = [best_index]\n        self.weights = []\n        metrics = [best_score]\n\n        # The final ensemble after each addition\n        best_ensemble = self.train[:, best_index]\n\n        for kk in range(10000):\n            best_score = score\n            best_index = -1\n            best_weight = 0\n\n            # Try adding each model and check improvement\n            for k, ff in enumerate(files):\n                new_model = self.train[:, k]  \n                m1 = np.repeat(best_ensemble[:, np.newaxis], nn, axis=1) * (1 - ww)\n                m2 = np.repeat(new_model[:, np.newaxis], nn, axis=1) * ww\n                mm = m1 + m2\n                new_rmse = self.metric(self.target, mm)\n                new_score = np.min(new_rmse).item()\n\n                if new_score < best_score:\n                    best_score = new_score\n                    best_index = k\n                    ii = np.argmin(new_rmse).item()\n                    best_weight = ww[ii].item()\n                    potential_ensemble = mm[:, ii]\n\n            # STOP CRITERIA\n            indices.append(best_index)\n            indices = list(np.unique(indices))\n            if len(indices) > self.max_model:\n                print(f'=> Reached {self.max_model} models')\n                indices = indices[:-1]\n                break\n            if abs(best_score - old_best_score) < self.tol:\n                print(f'=> Reached tolerance {self.tol}')\n                break\n\n            # Record new results\n            if best_index != -1:\n                print(\n                    kk,\n                    'New best rmse', best_score,\n                    f'adding \"{files[best_index]}\"',\n                    'weight', f'{best_weight:0.3f}'\n                )\n                self.models.append(best_index)\n                self.weights.append(best_weight)\n                metrics.append(best_score)\n                best_ensemble = potential_ensemble\n                old_best_score = best_score\n\n        # Combine weights\n        wgt = np.array([1])\n        for w in self.weights:\n            wgt = wgt * (1 - w)\n            wgt = np.concatenate([wgt, np.array([w])])\n\n        # Show final model weights\n        rows = []\n        t = 0\n        for m, w, s in zip(self.models, wgt, metrics):\n            if m == -1:\n                continue\n            name = files[m]\n            dd = {'weight': w, 'model': name}\n            rows.append(dd)\n            t += float(f'{w:.3f}')\n        self.cl_output = pd.DataFrame(rows)\n        self.cl_output = (\n            self.cl_output.groupby('model')\n            .agg('sum')\n            .reset_index()\n            .sort_values('weight', ascending=False)\n            .reset_index(drop=True)\n        )\n\n    def predict(self, test):\n        \"\"\"\n        Applies the discovered weights to the test set predictions.\n        \"\"\"\n        self.test_final = 0\n        for i, j in enumerate(self.cl_output.model):\n            self.test_final += test[j] * self.cl_output.loc[i]['weight']\n        return self.test_final","metadata":{"_uuid":"7d1021b4-6e1e-4060-9be2-406b0d479019","_cell_guid":"21f52c2b-9131-422e-8b07-9ae99b74a936","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-01-05T17:54:22.908432Z","iopub.execute_input":"2025-01-05T17:54:22.908941Z","iopub.status.idle":"2025-01-05T17:54:22.930148Z","shell.execute_reply.started":"2025-01-05T17:54:22.908892Z","shell.execute_reply":"2025-01-05T17:54:22.928923Z"},"papermill":{"duration":0.062334,"end_time":"2024-12-18T21:49:57.165389","exception":false,"start_time":"2024-12-18T21:49:57.103055","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =============================================================================\n# SEMI-SUPERVISED LEARNING STEP \n# =============================================================================\n'''\nBelow, we do a semi-supervised approach:\n1) Train multiple models on labeled data\n2) Use hill-climbing ensemble to label unlabeled data\n3) Merge them back\n4) Train again on the combined dataset\n'''\n\n\n\ntrain_nn = train[feature_cols].copy()\ntest_nn = train_missing[feature_cols].copy()\ntrain_nn[TARGET_cols] = train[TARGET_cols]\n\ntrain_nn, test_nn, cat_card, _ = label_encode(train_nn, test_nn, cat_cols)\ntrain_nn, test_nn = scaler_encode(train_nn, test_nn, num_cols)\nnum_cols_nn = num_cols\n\n# Keras model example\nWH = 0\nBS = 256\nepochs = 6\nlearning_rate = 3e-3\nnn_model_type = 'base model'\nnn_instance = keras.Model()\n\nmodel_nn = model_init(keras_model, None)\noof_nn, test_missing_nn, acc_nn, model_NN = model_nn.Train(\n    train_nn, test_nn, TARGET_PC, 5\n)\n\n# CatBoost example\ntrain_gb = train[feature_cols].copy()\ntest_gb = train_missing[feature_cols].copy()\ntrain_gb[TARGET_cols] = train[TARGET_cols]\n\nparam_cat = {\n    'objective': 'Tweedie:variance_power=1.5', \n    'iterations': 273, \n    'depth': 6,\n    'learning_rate': 0.03347776308515933,\n    'l2_leaf_reg': 0.0005342937261279777, \n    'subsample': 0.645614570099021, \n    'bagging_temperature': 0.6118528947223795, \n    'random_strength': 1.3957991126597662, \n    'colsample_bylevel': 0.6460723242676091,\n    'min_data_in_leaf': 37,\n    'random_state': RND_SEED,\n    'cat_features': cat_cols,\n    'verbose': 0\n}\nmodel_cat = model_init(ctb.CatBoostRegressor, param_cat)\noof_cat, test_missing_cat, acc_cat, model_cat = model_cat.Train(\n    train_gb, test_gb, TARGET_PC, n_split=5\n)\n\n# LightGBM example\nparam_lgb = {\n    'objective': 'tweedie', \n    'n_estimators': 597, \n    'max_depth': 3, \n    'learning_rate': 0.01019160829182289,\n    'subsample': 0.5261976292373335,\n    'colsample_bytree': 0.5351784713007832,\n    'tweedie_variance_power': 1.152466250299122,\n    'random_state': RND_SEED,\n    'verbosity': -1\n}\nmodel_lgb = model_init(lgb.LGBMRegressor, param_lgb)\noof_lgb, test_missing_lgb, acc_lgb, model_lgb = model_lgb.Train(\n    train_gb, test_gb, TARGET_PC, n_split=5\n)\n\n# Lasso example\nparam_lasso = {\n    'fit_intercept': True,\n    'precompute': True, \n    'copy_X': True, \n    'warm_start': True,\n    'selection': 'random',\n    'max_iter': 531, \n    'alpha': 0.0004496880541220534, \n    'tol': 1.4872791031412706e-05,\n    'random_state': RND_SEED\n}\nmodel_lasso = model_init(Lasso, param_lasso)\noof_lasso, test_missing_lasso, _, model_lasso = model_lasso.Train(\n    train_gb, test_gb, TARGET_PC, n_split=5\n)\n\n# XGBoost example\nparam_xgb = {\n    'objective': 'reg:squarederror',\n    'n_estimators': 327,\n    'max_depth': 3,\n    'learning_rate': 0.020155904604737717,\n    'subsample': 0.7712561640659066, \n    'colsample_bytree': 0.5466806743991856,\n    'gamma': 2.6288923622161713,\n    'reg_alpha': 9.468184374972093e-05,\n    'reg_lambda': 0.0003582157866941332,\n    'tweedie_variance_power': 1.1581422325461046,\n    'enable_categorical': True,\n    'random_state': RND_SEED,\n    'verbosity': 0\n}\nmodel_xgb = model_init(xgb.XGBRegressor, param_xgb)\noof_xgb, test_missing_xgb, acc_xgb, model_xgb = model_xgb.Train(\n    train_gb, test_gb, TARGET_PC, n_split=5\n)\n\n# Combine OOF predictions\noof_preds = pd.DataFrame({\n    'cat': oof_cat,\n    'lgb': oof_lgb,\n    'xgb': oof_xgb,\n    'nn': oof_nn,\n    'lasso': oof_lasso\n})\n\nmissing_pred = pd.DataFrame({\n    'cat': test_missing_cat,\n    'lgb': test_missing_lgb,\n    'xgb': test_missing_xgb,\n    'nn': test_missing_nn,\n    'lasso': test_missing_lasso\n})\n\n# Hill climbing ensemble on unlabeled portion\nhc = hill_climbing(eval_metric=multiple_rmse_scores)\nhc.fit(oof_preds, train[TARGET_PC], 400)\nens_missing_pred = hc.predict(missing_pred)\n\n# Convert predictions to integer SII classes\ntuned_ens_missing = threshold_Rounder(ens_missing_pred, THRESHOLD)\n\n# Merge back and re-train\ntrain_missing[TARGET_sii] = tuned_ens_missing.round().astype('int')\ntrain_missing[TARGET_PC] = ens_missing_pred\ntrain_missing[TARGET_binned] = pd.cut(ens_missing_pred.round(), bins=10, labels=False)\ntrain_missing['original'] = 0\n\ntrain['original'] = 1\ntrain_plus = pd.concat([train_missing, train], ignore_index=True)\n","metadata":{"_uuid":"3bbd3dbd-aebc-4b3d-8b06-ba786f200b95","_cell_guid":"913e90ca-0b73-415c-a44b-73164e63d8c7","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-01-05T17:54:22.932154Z","iopub.execute_input":"2025-01-05T17:54:22.932629Z","iopub.status.idle":"2025-01-05T17:55:15.030036Z","shell.execute_reply.started":"2025-01-05T17:54:22.932581Z","shell.execute_reply":"2025-01-05T17:55:15.028468Z"},"papermill":{"duration":52.027028,"end_time":"2024-12-18T21:50:49.415408","exception":false,"start_time":"2024-12-18T21:49:57.388380","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =============================================================================\n# FINAL VOTING EXAMPLE\n# =============================================================================\n'''\nAt the end, we do a final stage of training with multiple models \n(including time series features for some) and ensemble them via hard voting.\n'''","metadata":{"_uuid":"b062af1f-d31c-48a3-b35c-6fed8cafc4eb","_cell_guid":"4b36e2a7-f506-4dba-b665-aa673ffb1528","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-01-05T17:55:15.031522Z","iopub.execute_input":"2025-01-05T17:55:15.031880Z","iopub.status.idle":"2025-01-05T17:55:15.040066Z","shell.execute_reply.started":"2025-01-05T17:55:15.031847Z","shell.execute_reply":"2025-01-05T17:55:15.038712Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_nn = train_plus.copy()\ntest_nn = test.copy()\n\ntrain_nn,test_nn,cat_card,_ = label_encode(train_nn,test_nn, cat_cols)\ntrain_nn,test_nn = scaler_encode(train_nn,test_nn,num_cols)\nnum_cols_nn = num_cols\nWH = 1\nBS=256\nepochs = 6\nlearning_rate = 3e-3\nnn_model_type = 'base model'\nnn_instance = keras.Model()\nmodel_nn = model_init(keras_model,None)\noof_pred_nn,test_pred_nn,acc_nn,model_NN=model_nn.Train(train_nn,test_nn,TARGET_binned,5)","metadata":{"_uuid":"be1aad3d-1b9b-4261-bdea-09818ef04663","_cell_guid":"5b1155ba-c7bc-4044-a67f-1ff9846e7ace","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-01-05T17:55:15.041548Z","iopub.execute_input":"2025-01-05T17:55:15.041914Z","iopub.status.idle":"2025-01-05T17:55:49.656666Z","shell.execute_reply.started":"2025-01-05T17:55:15.041880Z","shell.execute_reply":"2025-01-05T17:55:49.655377Z"},"papermill":{"duration":33.874509,"end_time":"2024-12-18T21:51:25.092920","exception":false,"start_time":"2024-12-18T21:50:51.218411","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Preparing data with and without timeseries features\n\ntrain_gb_plus = train_plus[feature_cols].copy()\ntrain_gb_plus['original'] = train_plus['original']\ntrain_gb_plus[TARGET_cols] = train_plus[TARGET_cols]\ntest_gb = test[feature_cols].copy()\n\ntrain_gb_ts = train_ts[feature_cols_ts].copy()\ntrain_gb_ts[TARGET_cols] = train[TARGET_cols]\ntest_gb_ts = test_ts[feature_cols_ts]","metadata":{"_uuid":"52f5c256-e9b8-4b39-80fc-599dc97426c1","_cell_guid":"80463ec1-8015-4444-ab51-3a7fd1208955","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-01-05T17:55:49.658532Z","iopub.execute_input":"2025-01-05T17:55:49.659019Z","iopub.status.idle":"2025-01-05T17:55:49.680903Z","shell.execute_reply.started":"2025-01-05T17:55:49.658970Z","shell.execute_reply":"2025-01-05T17:55:49.679760Z"},"papermill":{"duration":0.073228,"end_time":"2024-12-18T21:51:25.209185","exception":false,"start_time":"2024-12-18T21:51:25.135957","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"WH = 1\ntrain_dt,test_dt,_ = one_hot_encode(train_gb_plus,test_gb,cat_cols)\nparam_dt={\n          'criterion': 'squared_error', \n          'splitter': 'best',\n          'max_depth': 5,\n          'min_samples_split': 100, \n          'min_samples_leaf': 5,\n          'min_impurity_decrease': 0.02,\n           'ccp_alpha': 0.01,\n          'random_state' : RND_SEED,\n          }\n\nmodel_dt = model_init(DecisionTreeRegressor,param_dt)\noof_pred_dt_plus,test_pred_dt_plus,acc_dt,model_dt_plus=model_dt.Train(train_dt,test_dt,TARGET_binned,n_split = 5)","metadata":{"_uuid":"a0de8e50-489d-46f8-86ab-a0e19fb4cb2b","_cell_guid":"0c64a056-839c-4574-a7ed-331bd3d9fa27","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-01-05T17:55:49.682246Z","iopub.execute_input":"2025-01-05T17:55:49.682589Z","iopub.status.idle":"2025-01-05T17:55:50.176451Z","shell.execute_reply.started":"2025-01-05T17:55:49.682556Z","shell.execute_reply":"2025-01-05T17:55:50.175251Z"},"papermill":{"duration":0.552595,"end_time":"2024-12-18T21:51:35.373461","exception":false,"start_time":"2024-12-18T21:51:34.820866","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_lr = train_gb_plus.copy()\ntest_lr = test_gb.copy()\ntrain_lr,test_lr = scaler_encode(train_lr,test_lr ,num_cols)\nWH=0\nparam_lr= {\n         'solver': 'newton-cg',\n          'penalty':'l2',\n         'class_weight' :'balanced',\n         'multi_class': 'multinomial',\n         'max_iter': 500,\n         'warm_start': True,\n        'verbose':0, \n    }\n        \nmodel_lr = model_init(LogisticRegression,param_lr)\noof_pred_lr_plus,test_pred_lr_plus,overal_score,model_lr_plus=model_lr.Train(train_lr,test_lr,TARGET_binned,n_split = 5)","metadata":{"_uuid":"fd1cc625-eee2-4436-b907-df5731814cc4","_cell_guid":"c21462ad-ccd0-4387-907b-cf24e9ad3165","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-01-05T17:55:50.178096Z","iopub.execute_input":"2025-01-05T17:55:50.178511Z","iopub.status.idle":"2025-01-05T17:55:56.476551Z","shell.execute_reply.started":"2025-01-05T17:55:50.178474Z","shell.execute_reply":"2025-01-05T17:55:56.475348Z"},"papermill":{"duration":8.824118,"end_time":"2024-12-18T21:51:44.246321","exception":false,"start_time":"2024-12-18T21:51:35.422203","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"WH = 1\nparam_lgb={'objective': 'tweedie',\n           'n_estimators': 576, \n           'max_depth': 3, \n           'learning_rate': 0.014504865871703278,\n           'subsample': 0.7724866369943665, \n           'colsample_bytree': 0.5717189909747162,\n           'random_state' : RND_SEED,\n           'verbosity': -1,\n          }\n\nmodel_lgb = model_init(lgb.LGBMRegressor,param_lgb)\noof_pred_lgb_plus,test_pred_lgb_plus,acc_lgb,model_lgb_plus=model_lgb.Train(train_gb_plus,test_gb,TARGET_binned,n_split = 5)","metadata":{"_uuid":"8e17d490-7d26-4ecb-b816-9ecbc21e3703","_cell_guid":"636e2bbf-c650-4313-98c1-ba20c3f4e7fd","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-01-05T17:55:56.478055Z","iopub.execute_input":"2025-01-05T17:55:56.478557Z","iopub.status.idle":"2025-01-05T17:55:59.606429Z","shell.execute_reply.started":"2025-01-05T17:55:56.478519Z","shell.execute_reply":"2025-01-05T17:55:59.604552Z"},"papermill":{"duration":4.17506,"end_time":"2024-12-18T21:51:48.505939","exception":false,"start_time":"2024-12-18T21:51:44.330879","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"WH = 1\nparam_cat={'objective': 'Tweedie:variance_power=1.5',\n           'iterations': 292,\n           'depth': 6,\n           'learning_rate': 0.0624982363237828, \n           'l2_leaf_reg': 0.019685934698040447, \n           'subsample': 0.5018495996455012,\n           'bagging_temperature': 0.8627252427715155,\n           'random_strength': 5.015068697208244, \n           'colsample_bylevel': 0.6124379138359408,\n           'min_data_in_leaf': 89,\n           'random_state': RND_SEED,\n            'cat_features' : cat_cols,\n            'verbose': 0\n            }\n\nmodel_cat = model_init(ctb.CatBoostRegressor,param_cat)\noof_pred_cat_plus,test_pred_cat_plus,acc_cat,model_cat_plus=model_cat.Train(train_gb_plus,test_gb,TARGET_binned,n_split = 5)","metadata":{"_uuid":"ea09f0cd-f044-42b9-81ae-b915881db018","_cell_guid":"f334a9d8-2774-4b30-91dd-b3bef7fa3cc3","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-01-05T17:55:59.608512Z","iopub.execute_input":"2025-01-05T17:55:59.609085Z","iopub.status.idle":"2025-01-05T17:56:15.364231Z","shell.execute_reply.started":"2025-01-05T17:55:59.609031Z","shell.execute_reply":"2025-01-05T17:56:15.362896Z"},"papermill":{"duration":18.256928,"end_time":"2024-12-18T21:52:06.833874","exception":false,"start_time":"2024-12-18T21:51:48.576946","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"WH = 1\nparam_lgb={'iterations': 1000,\n             'objective':'tweedie',\n            'max_depth': 10,\n            # # 'cat_features' : cat_cols,\n            'learning_rate': 0.04,\n            # 'bagging_fraction': 0.78,\n            # 'bagging_freq' : 20,\n            #  # 'border_count': 60,\n             'verbosity' : -1}\n\nmodel_lgb = model_init(lgb.LGBMRegressor,param_lgb)\noof_pred_lgb_ts,test_pred_lgb_ts,acc_lgb,model_lgb_ts=model_lgb.Train(train_gb_ts,test_gb_ts,TARGET_binned,n_split = 5)","metadata":{"_uuid":"8e19d8d8-cf80-44c1-80be-f2bf22ff86e6","_cell_guid":"8e1ef150-cccf-4d5e-9d3e-a0c5b32498c9","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-01-05T17:56:15.366272Z","iopub.execute_input":"2025-01-05T17:56:15.366661Z","iopub.status.idle":"2025-01-05T17:56:18.148702Z","shell.execute_reply.started":"2025-01-05T17:56:15.366629Z","shell.execute_reply":"2025-01-05T17:56:18.147416Z"},"papermill":{"duration":2.990465,"end_time":"2024-12-18T21:52:09.993982","exception":false,"start_time":"2024-12-18T21:52:07.003517","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"WH = 1\nparam_cat={'iterations': 500,\n            'depth': 6,\n            'learning_rate': 0.015,\n            'l2_leaf_reg': 0.002,\n            'random_strength': 0.46,\n            'cat_features' : cat_cols,\n            # 'border_count': 51,\n            # 'min_data_in_leaf': 13,\n            'verbose' : 0}\nmodel_cat = model_init(ctb.CatBoostRegressor,param_cat)\noof_pred_cat_ts,test_pred_cat_ts,acc_cat,model_cat=model_cat.Train(train_gb_ts,test_gb_ts,TARGET_binned,n_split = 5)","metadata":{"_uuid":"0a51b539-5aef-4e18-a3f2-b6c9c1850801","_cell_guid":"96f4f728-b0aa-41e7-bec0-3c3784483d18","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-01-05T17:56:18.150227Z","iopub.execute_input":"2025-01-05T17:56:18.150620Z","iopub.status.idle":"2025-01-05T17:56:53.036725Z","shell.execute_reply.started":"2025-01-05T17:56:18.150578Z","shell.execute_reply":"2025-01-05T17:56:53.035438Z"},"papermill":{"duration":35.954629,"end_time":"2024-12-18T21:52:45.998523","exception":false,"start_time":"2024-12-18T21:52:10.043894","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"WH = 1\nparam_xgb={'objective': 'reg:tweedie',\n           'n_estimators': 480,\n           'max_depth': 3, \n           'learning_rate': 0.025707833957860277,\n           'subsample': 0.5290418060840998,\n           'colsample_bytree': 0.9330880728874675,\n           'gamma': 3.005575058716044,\n           'reg_alpha': 0.006796578090758156,\n           'reg_lambda': 1.2087541473056957e-05,\n           'tweedie_variance_power': 1.9699098521619942,\n           'random_state' : RND_SEED,\n           'enable_categorical' : True,\n            'verbosity' : 0}\n\nmodel_xgb = model_init(xgb.XGBRegressor,param_xgb)\noof_pred_xgb_plus,test_pred_xgb_plus,acc_xgb,model_xgb_plus=model_xgb.Train(train_gb_plus,test_gb,TARGET_binned,n_split = 5)","metadata":{"_uuid":"786e7fd5-257d-4e6a-93ac-5f04fbfaa4c3","_cell_guid":"34b83753-365a-4dce-b096-f187260ae153","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-01-05T17:56:53.038288Z","iopub.execute_input":"2025-01-05T17:56:53.038835Z","iopub.status.idle":"2025-01-05T17:56:58.758839Z","shell.execute_reply.started":"2025-01-05T17:56:53.038786Z","shell.execute_reply":"2025-01-05T17:56:58.757692Z"},"papermill":{"duration":5.703535,"end_time":"2024-12-18T21:52:51.754008","exception":false,"start_time":"2024-12-18T21:52:46.050473","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"WH = 1\nparam_xgb={'iterations': 500,\n            'max_depth': 3,\n            'eta': 0.054,\n           'objective' : 'reg:tweedie',\n           'tweedie_variance_power':1.3,\n            'bagging_fraction': 0.8,\n           'enable_categorical' : True,\n            # 'bagging_freq' : 10,\n            # 'random_strength': 0.5482698471489474,\n             'border_count': 48,\n           'random_seed' : 4,\n            'verbosity' : 0}\n\nmodel_xgb = model_init(xgb.XGBRegressor,param_xgb)\noof_pred_xgb_ts,test_pred_xgb_ts,acc_xgb,model_xgb_ts=model_xgb.Train(train_gb_ts,test_gb_ts,TARGET_binned,n_split = 5)","metadata":{"_uuid":"4194e62b-25d0-49b8-848c-7c9d2282a900","_cell_guid":"384f7abc-3c3e-426e-af14-89ec8ba1b1e7","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-01-05T17:56:58.760352Z","iopub.execute_input":"2025-01-05T17:56:58.760753Z","iopub.status.idle":"2025-01-05T17:57:00.701281Z","shell.execute_reply.started":"2025-01-05T17:56:58.760719Z","shell.execute_reply":"2025-01-05T17:57:00.700027Z"},"papermill":{"duration":1.985326,"end_time":"2024-12-18T21:52:53.789480","exception":false,"start_time":"2024-12-18T21:52:51.804154","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"WH = 1\nparam_lasso = { 'fit_intercept': False,\n               'precompute': True,\n               'copy_X': False,\n               'warm_start': False,\n               'selection': 'cyclic',\n               'max_iter': 1022,\n               'alpha': 0.0005076318790591836, \n               'tol': 8.154903080917285e-05,\n               'random_state' : RND_SEED\n\n              }\n\nmodel_lasso = model_init(Lasso,param_lasso)\noof_pred_lasso,test_pred_lasso,acc_lgb,model_lasso=model_lasso.Train(train_gb_plus,test_gb,TARGET_binned,n_split = 5)","metadata":{"_uuid":"078b7ccc-c915-473e-8102-fab5862ac9aa","_cell_guid":"82914390-6861-4b85-a75c-22afe6cb361b","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-01-05T17:57:00.705921Z","iopub.execute_input":"2025-01-05T17:57:00.706302Z","iopub.status.idle":"2025-01-05T17:57:00.878038Z","shell.execute_reply.started":"2025-01-05T17:57:00.706267Z","shell.execute_reply":"2025-01-05T17:57:00.876588Z"},"papermill":{"duration":0.235374,"end_time":"2024-12-18T21:52:54.074480","exception":false,"start_time":"2024-12-18T21:52:53.839106","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ens_preds = pd.DataFrame()\ntest_preds = pd.DataFrame()\n\n# Example collection of out-of-fold predictions\nens_preds['cat_plus'] = oof_pred_cat_plus\nens_preds['cat_ts'] = oof_pred_cat_ts\nens_preds['lgb_plus'] = oof_pred_lgb_plus\nens_preds['lgb_ts'] = oof_pred_lgb_ts\nens_preds['xgb_plus'] = oof_pred_xgb_plus\nens_preds['xgb_ts'] = oof_pred_xgb_ts\nens_preds['nn'] = oof_pred_nn\nens_preds['lasso'] = oof_pred_lasso\nens_preds['dt_plus'] = oof_pred_dt_plus\nens_preds['lr_plus'] = oof_pred_lr_plus\n\ntest_preds['cat_plus'] = test_pred_cat_plus\ntest_preds['cat_ts'] = test_pred_cat_ts\ntest_preds['lgb_plus'] = test_pred_lgb_plus\ntest_preds['lgb_ts'] = test_pred_lgb_ts\ntest_preds['xgb_plus'] = test_pred_xgb_plus\ntest_preds['xgb_ts'] = test_pred_xgb_ts\ntest_preds['nn'] = test_pred_nn\ntest_preds['lasso'] = test_pred_lasso\ntest_preds['dt_plus'] = test_pred_dt_plus\ntest_preds['lr_plus'] = test_pred_lr_plus\n\n# Threshold them\ntuned_ens_preds = pd.DataFrame()\ntuned_test_preds = pd.DataFrame()\n\nfor col in ens_preds.keys():\n    tuned_ens_preds[col] = threshold_Rounder(ens_preds[col], THRESHOLD_1)\n    tuned_test_preds[col] = threshold_Rounder(test_preds[col], THRESHOLD_1)\n\n# Hard-voting (mode) ensemble\nvoted_oof = tuned_ens_preds.mode(axis=1).iloc[:, 0]\nfinal_test = tuned_test_preds.mode(axis=1).iloc[:, 0]\n\n# QWK\nkappa_score = cohen_kappa_score(train[TARGET_sii], voted_oof, weights='quadratic')\nprint(kappa_score)\n\n# Make final submission\nsubmission = pd.DataFrame({\n    'id': sample['id'],\n    'sii': final_test\n})\nsubmission.to_csv('submission.csv', index=False)","metadata":{"_uuid":"7f3bb99f-a090-427c-ad70-a5357d4b5640","_cell_guid":"ba634b3e-edb4-44d9-bce1-b5201efc45a5","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-01-05T17:57:00.879500Z","iopub.execute_input":"2025-01-05T17:57:00.879908Z","iopub.status.idle":"2025-01-05T17:57:01.635024Z","shell.execute_reply.started":"2025-01-05T17:57:00.879874Z","shell.execute_reply":"2025-01-05T17:57:01.633803Z"},"papermill":{"duration":0.087784,"end_time":"2024-12-18T21:52:54.484832","exception":false,"start_time":"2024-12-18T21:52:54.397048","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null}]}