{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30775,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# !pip install -U lightautoml","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-12-18T06:15:30.220320Z","iopub.execute_input":"2024-12-18T06:15:30.220796Z","iopub.status.idle":"2024-12-18T06:15:30.228680Z","shell.execute_reply.started":"2024-12-18T06:15:30.220756Z","shell.execute_reply":"2024-12-18T06:15:30.227011Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"DEMO = False\nRUN_OPTUNA = False\nRUN_AUTOML = False","metadata":{"execution":{"iopub.status.busy":"2024-12-18T06:15:30.236141Z","iopub.execute_input":"2024-12-18T06:15:30.236876Z","iopub.status.idle":"2024-12-18T06:15:30.247728Z","shell.execute_reply.started":"2024-12-18T06:15:30.236728Z","shell.execute_reply":"2024-12-18T06:15:30.246265Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import gc\nimport os\nimport sys\nimport logging\nfrom tqdm import tqdm\nfrom pathlib import Path\nfrom concurrent.futures import ThreadPoolExecutor\n\nimport numpy as np\nimport pandas as pd\nimport polars as pl\n\nfrom scipy.stats import ttest_1samp\nfrom scipy.optimize import minimize\nfrom sklearn.preprocessing import MinMaxScaler, OneHotEncoder\nfrom sklearn.model_selection import train_test_split\n\nimport torch\n\nimport optuna\nfrom optuna import Trial\nfrom optuna.samplers import TPESampler\n\nimport lightgbm as lgbm\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\n\nif RUN_AUTOML:\n    from lightautoml.automl.presets.tabular_presets import TabularAutoML, TabularUtilizedAutoML\n    from lightautoml.dataset.roles import DatetimeRole\n    from lightautoml.tasks import Task\n\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.impute import KNNImputer\n\nimport joblib\n\npd.set_option(\"future.no_silent_downcasting\", True)","metadata":{"execution":{"iopub.status.busy":"2024-12-18T06:15:30.249588Z","iopub.execute_input":"2024-12-18T06:15:30.249965Z","iopub.status.idle":"2024-12-18T06:15:34.317191Z","shell.execute_reply.started":"2024-12-18T06:15:30.249929Z","shell.execute_reply":"2024-12-18T06:15:34.313970Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"SEED = 42\n\nnp.random.seed(SEED)","metadata":{"execution":{"iopub.status.busy":"2024-12-18T06:15:34.323668Z","iopub.execute_input":"2024-12-18T06:15:34.328289Z","iopub.status.idle":"2024-12-18T06:15:34.336992Z","shell.execute_reply.started":"2024-12-18T06:15:34.328157Z","shell.execute_reply":"2024-12-18T06:15:34.334578Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Config","metadata":{}},{"cell_type":"code","source":"# global config\nN_FOLD = 5","metadata":{"execution":{"iopub.status.busy":"2024-12-18T06:15:34.341560Z","iopub.execute_input":"2024-12-18T06:15:34.342350Z","iopub.status.idle":"2024-12-18T06:15:34.366560Z","shell.execute_reply.started":"2024-12-18T06:15:34.342295Z","shell.execute_reply":"2024-12-18T06:15:34.365380Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Data","metadata":{}},{"cell_type":"code","source":"root = Path(\"/kaggle/input/child-mind-institute-problematic-internet-use/\")\n\ntrain_path = root / \"train.csv\"\ntest_path = root / \"test.csv\"\ntrain_series_path = root / \"series_train.parquet\"\ntest_series_path = root / \"series_test.parquet\"\nsubmission_path = root / \"sample_submission.csv\"","metadata":{"execution":{"iopub.status.busy":"2024-12-18T06:15:34.367990Z","iopub.execute_input":"2024-12-18T06:15:34.369453Z","iopub.status.idle":"2024-12-18T06:15:34.380034Z","shell.execute_reply.started":"2024-12-18T06:15:34.369396Z","shell.execute_reply":"2024-12-18T06:15:34.378374Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_id = \"id\"\nseries_id = \"step\"\ntarget_1 = \"sii\"\ntarget_2 = \"PCIAT-PCIAT_Total\"","metadata":{"execution":{"iopub.status.busy":"2024-12-18T06:15:34.381721Z","iopub.execute_input":"2024-12-18T06:15:34.382184Z","iopub.status.idle":"2024-12-18T06:15:34.393730Z","shell.execute_reply.started":"2024-12-18T06:15:34.382145Z","shell.execute_reply":"2024-12-18T06:15:34.392401Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_sii(x, threshold=[30, 50, 80]):\n    if np.isnan(x):\n        return x\n    elif x <= threshold[0]:\n        return 0\n    elif x < threshold[1]:\n        return 1\n    elif x < threshold[2]:\n        return 2\n    else:\n        return 3\n\ndef get_scaled_sii(x, threshold=[30.0, 50.0, 80.0, 100.0]):\n    if np.isnan(x):\n        return x\n    elif x <= threshold[0]: # [0, 30] -> [0, 1)\n        return min(x / threshold[0], 1 - 1e-5)\n    elif x < threshold[1]: # [30, 50) -> [1, 2)\n        return 1 + (x - threshold[0]) / (threshold[1] - threshold[0])\n    elif x < threshold[2]: # [50, 80) -> [2, 3)\n        return 2 + (x - threshold[1]) / (threshold[2] - threshold[1])\n    else: # [80, 100]\n        return 3 + (x - threshold[2]) / (threshold[3] - threshold[2])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T06:15:34.395270Z","iopub.execute_input":"2024-12-18T06:15:34.395785Z","iopub.status.idle":"2024-12-18T06:15:34.408411Z","shell.execute_reply.started":"2024-12-18T06:15:34.395721Z","shell.execute_reply":"2024-12-18T06:15:34.407205Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pciat_columns = [\"PCIAT-PCIAT_Total\", \"PCIAT-Season\"] + [f\"PCIAT-PCIAT_{i:02d}\" for i in range(1, 21)]\n\ndef read_csv(path, split):\n    df = pl.scan_csv(path)\n\n    if split == \"train\":\n        df = (\n            df.filter(pl.col(target_1).is_not_null())\n        )\n\n    raw_columns = df.collect_schema().names()\n    select_columns = [col for col in raw_columns if col not in pciat_columns and col != \"sii\"] + ([target_1, target_2] if split == \"train\" else [])\n\n    return df.select(select_columns)\n\n\ndef read_parquet(path):\n    df = (\n        pl.scan_parquet(path)\n        .filter(pl.col(\"non-wear_flag\")==0)\n        .sort(series_id)\n    )\n\n    raw_columns = df.collect_schema().names()\n    select_columns = [col for col in raw_columns if col not in [\"battery_voltage\", \"non-wear_flag\"]]\n\n    return df.select(select_columns)","metadata":{"execution":{"iopub.status.busy":"2024-12-18T06:15:34.409974Z","iopub.execute_input":"2024-12-18T06:15:34.410351Z","iopub.status.idle":"2024-12-18T06:15:34.424012Z","shell.execute_reply.started":"2024-12-18T06:15:34.410316Z","shell.execute_reply":"2024-12-18T06:15:34.422590Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"remove_target = [] # 0.4526\nremove_target += ['FGC-FGC_CU_Zone', 'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD_Zone',\n 'FGC-FGC_PU_Zone', 'FGC-FGC_SRR_Zone', 'FGC-FGC_TL_Zone', 'BIA-Season',\n 'BIA-BIA_FFM', 'BIA-BIA_Frame_num', 'PAQ_A-Season', 'min_enmo', 'min_anglez',\n 'diff_enmo', 'diff_anglez'] # 0.4541 / 0.4547\nremove_target += ['FGC-Season'] # 0.4479\n# remove_target += ['Basic_Demos-Enroll_Season', 'CGAS-Season'] # 0.4553\n# remove_target += ['FGC-FGC_SRL_Zone', 'PreInt_EduHx-Season'] # 0.4533\n\ndef preprocess_csv(df):\n    # remove features\n    raw_columns = df.collect_schema().names()\n    select_columns = [col for col in raw_columns if col not in remove_target]\n    df = df.select(select_columns)\n    \n    # add features\n    df = (\n        df.with_columns([\n            (pl.col(\"Physical-BMI\") * pl.col(\"Basic_Demos-Age\")).alias(\"BMI_Age\"),\n            (pl.col(\"PreInt_EduHx-computerinternet_hoursday\") * pl.col(\"Basic_Demos-Age\")).alias(\"Internet_Hours_Age\"),\n            (pl.col(\"Physical-BMI\") * pl.col(\"PreInt_EduHx-computerinternet_hoursday\")).alias(\"BMI_Internet_Hours\"),\n            (pl.col(\"BIA-BIA_Fat\") / pl.col(\"BIA-BIA_BMI\")).alias(\"BFP_BMI\"),\n            (pl.col(\"BIA-BIA_FFMI\") / pl.col(\"BIA-BIA_Fat\")).alias(\"FFMI_BFP\"),\n            (pl.col(\"BIA-BIA_FMI\") / pl.col(\"BIA-BIA_Fat\")).alias(\"FMI_BFP\"),\n            (pl.col(\"BIA-BIA_LST\") / pl.col(\"BIA-BIA_TBW\")).alias(\"LST_TBW\"),\n            (pl.col(\"BIA-BIA_Fat\") * pl.col(\"BIA-BIA_BMR\")).alias(\"BFP_BMR\"),\n            (pl.col(\"BIA-BIA_Fat\") * pl.col(\"BIA-BIA_DEE\")).alias(\"BFP_DEE\"),\n            (pl.col(\"BIA-BIA_BMR\") / pl.col(\"Physical-Weight\")).alias(\"BMR_Weight\"),\n            (pl.col(\"BIA-BIA_DEE\") / pl.col(\"Physical-Weight\")).alias(\"DEE_Weight\"),\n            (pl.col(\"BIA-BIA_SMM\") / pl.col(\"Physical-Height\")).alias(\"SMM_Height\"),\n            (pl.col(\"BIA-BIA_SMM\") / pl.col(\"BIA-BIA_FMI\")).alias(\"Muscle_to_Fat\"),\n            (pl.col(\"BIA-BIA_TBW\") / pl.col(\"Physical-Weight\")).alias(\"Hydration_Status\"),\n            (pl.col(\"BIA-BIA_ICW\") / pl.col(\"BIA-BIA_TBW\")).alias(\"ICW_TBW\"),\n        ])\n    )\n\n    return (\n        df.collect()\n        .to_pandas()\n        .set_index(sample_id)\n    )\n\n\ndef preprocess_parquet(df, squeeze):\n    df = (\n        df.collect()\n        .to_pandas()\n    )\n    \n    # scale features\n    scale_features = [\"anglez\"]\n    \n    scaler = MinMaxScaler()\n    df[scale_features] = scaler.fit_transform(df[scale_features])\n\n    # squeeze features\n    if squeeze:\n        squeeze_features = [\"X\", \"Y\", \"Z\", \"enmo\", \"anglez\", \"relative_date_PCIAT\"]\n        df = df[squeeze_features]\n\n        stacked_columns = [y + \"_\" + x for y in df.describe().index for x in df.columns]\n        df = df.describe().stack().values\n        df = pd.Series(df, stacked_columns)\n\n        for col in squeeze_features:\n            df[f\"diff_{col}\"] = df[f\"max_{col}\"] - df[f\"min_{col}\"]\n            \n        count_features = [col for col in df.index if \"count\" in col]\n        date_features = [col for col in df.index if (\"PCIAT\" in col and \"diff\" not in col)]\n\n        return df.drop(index=(count_features + date_features))\n    \n    return df.set_index(series_id)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T06:15:34.425758Z","iopub.execute_input":"2024-12-18T06:15:34.426255Z","iopub.status.idle":"2024-12-18T06:15:34.447199Z","shell.execute_reply.started":"2024-12-18T06:15:34.426204Z","shell.execute_reply":"2024-12-18T06:15:34.445809Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def merge_csv_and_parquet(df_csv, df_parquet):\n    return df_csv.join(df_parquet)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T06:15:34.448734Z","iopub.execute_input":"2024-12-18T06:15:34.449161Z","iopub.status.idle":"2024-12-18T06:15:34.467749Z","shell.execute_reply.started":"2024-12-18T06:15:34.449122Z","shell.execute_reply":"2024-12-18T06:15:34.466004Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"categorical_features = [\"Basic_Demos-Sex\",\n                        # \"FGC-FGC_CU_Zone\", \"FGC-FGC_GSND_Zone\", \"FGC-FGC_GSD_Zone\", \"FGC-FGC_PU_Zone\", \n                        # \"FGC-FGC_SRL_Zone\", \"FGC-FGC_SRR_Zone\", \"FGC-FGC_TL_Zone\", \n                        # \"BIA-BIA_Activity_Level_num\", \"BIA-BIA_Frame_num\",\n                        # \"PreInt_EduHx-computerinternet_hoursday\",\n                        \"Basic_Demos-Enroll_Season\", \"CGAS-Season\", \"Physical-Season\",\n                        \"Fitness_Endurance-Season\", \"FGC-Season\", \"BIA-Season\",\n                        \"PAQ_A-Season\", \"PAQ_C-Season\", \"SDS-Season\", \"PreInt_EduHx-Season\"]\n\n\ndef transform_csv(df_train, df_test):\n    def get_category_map(data):\n        return {v: i for i, v in enumerate(data.unique())}\n\n    for feat in categorical_features:\n        if feat not in df_train.columns:\n            continue\n        df_train[feat] = df_train[feat].fillna(\"\")\n        df_test[feat] = df_test[feat].fillna(\"\")\n        \n        mapping = get_category_map(pd.concat([df_train[feat], df_test[feat]]))\n\n        df_train[feat] = df_train[feat].replace(mapping).astype(int)\n        df_test[feat] = df_test[feat].replace(mapping).astype(int)\n    print(f\"[INFO] encode {len(categorical_features)}\")\n    \n    return df_train, df_test\n\ndef transform_parquet(df_train, df_test):\n    pass\n\ndef transform_df(df_train, df_test):\n    # imputer\n    numerical_features = list(filter(lambda x: x not in categorical_features, df_train.columns))\n    impute_features = [col for col in numerical_features if (col not in [sample_id, series_id, target_1, target_2] + [\"PAQ_C-PAQ_C_Total\", \"PAQ_A-PAQ_A_Total\"])]\n    print(f\"[INFO] impute {len(impute_features)}\")\n\n    df_train_impute = []\n    df_test_impute = []\n\n    train_indices = [\n        (df_train[\"PAQ_C-PAQ_C_Total\"].isna()) & (df_train[\"PAQ_A-PAQ_A_Total\"].isna()),\n        (df_train[\"PAQ_C-PAQ_C_Total\"].isna()) & (~df_train[\"PAQ_A-PAQ_A_Total\"].isna()),\n        (~df_train[\"PAQ_C-PAQ_C_Total\"].isna()) & (df_train[\"PAQ_A-PAQ_A_Total\"].isna()),\n        (~df_train[\"PAQ_C-PAQ_C_Total\"].isna()) & (~df_train[\"PAQ_A-PAQ_A_Total\"].isna()),\n    ]\n\n    test_indices = [\n        (df_test[\"PAQ_C-PAQ_C_Total\"].isna()) & (df_test[\"PAQ_A-PAQ_A_Total\"].isna()),\n        (df_test[\"PAQ_C-PAQ_C_Total\"].isna()) & (~df_test[\"PAQ_A-PAQ_A_Total\"].isna()),\n        (~df_test[\"PAQ_C-PAQ_C_Total\"].isna()) & (df_test[\"PAQ_A-PAQ_A_Total\"].isna()),\n        (~df_test[\"PAQ_C-PAQ_C_Total\"].isna()) & (~df_test[\"PAQ_A-PAQ_A_Total\"].isna()),\n    ]\n\n    for train_index, test_index in zip(train_indices, test_indices):\n        train_subset = df_train[train_index]\n        test_subset = df_test[test_index]\n\n        if train_subset.shape[0] < 100: # skip impute\n            df_train_impute.append(train_subset)\n            df_test_impute.append(test_subset)\n            continue\n\n        imputer = KNNImputer(n_neighbors=5, keep_empty_features=True)\n\n        train_subset[impute_features] = imputer.fit_transform(train_subset[impute_features])\n        test_subset[impute_features] = imputer.transform(test_subset[impute_features])\n\n        df_train_impute.append(train_subset)\n        df_test_impute.append(test_subset)\n\n    df_train = pd.concat(df_train_impute, axis=0)\n    df_test = pd.concat(df_test_impute, axis=0)\n\n    return df_train, df_test","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T06:55:44.445646Z","iopub.execute_input":"2024-12-18T06:55:44.446053Z","iopub.status.idle":"2024-12-18T06:55:44.462007Z","shell.execute_reply.started":"2024-12-18T06:55:44.446018Z","shell.execute_reply":"2024-12-18T06:55:44.460564Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def read_and_merge_parquets(path):\n    filenames = os.listdir(path)\n    ids = [filename.split(\"=\")[1] for filename in filenames]\n    \n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(lambda filename: preprocess_parquet(read_parquet(path / filename), squeeze=True), filenames), total=len(filenames)))\n\n    return pd.DataFrame(results, index=ids)\n\n\ndf_train = preprocess_csv(read_csv(train_path, \"train\"))\ndf_test = preprocess_csv(read_csv(test_path, \"test\"))\ndf_train, df_test = transform_csv(df_train, df_test)\n\ndf_series_train = read_and_merge_parquets(train_series_path)\ndf_series_test = read_and_merge_parquets(test_series_path)\n\ndf_train = merge_csv_and_parquet(df_train, df_series_train)\ndf_test = merge_csv_and_parquet(df_test, df_series_test)\ndel df_series_train, df_series_test\n\ndf_train, df_test = transform_df(df_train, df_test)\n\nprint(df_train.shape, df_test.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T06:55:50.999116Z","iopub.execute_input":"2024-12-18T06:55:50.999569Z","iopub.status.idle":"2024-12-18T06:57:30.624973Z","shell.execute_reply.started":"2024-12-18T06:55:50.999519Z","shell.execute_reply":"2024-12-18T06:57:30.623484Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Train","metadata":{}},{"cell_type":"code","source":"MODELS = []\nWEIGHTS = []","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T06:57:36.157100Z","iopub.execute_input":"2024-12-18T06:57:36.157576Z","iopub.status.idle":"2024-12-18T06:57:36.164651Z","shell.execute_reply.started":"2024-12-18T06:57:36.157523Z","shell.execute_reply":"2024-12-18T06:57:36.162895Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights=\"quadratic\")\n    \ndef threshold_rounder(y_pred, threshold):\n    return np.where(y_pred < threshold[0], 0,\n                    np.where(y_pred < threshold[1], 1,\n                             np.where(y_pred < threshold[2], 2, 3)))\n\ndef kappa_loss(threshold, y_true, y_pred):\n    y_pred = threshold_rounder(y_pred, threshold)\n    return -quadratic_weighted_kappa(y_true, y_pred)\n\n# y_true: int, y_pred: float\ndef kappa_loss_base_1(y_true, y_pred):\n    y_pred = y_pred.round(0).astype(int)\n    return (\"kappa_base\", -quadratic_weighted_kappa(y_true, y_pred), True)\n\n# y_true: int, y_pred: float\ndef kappa_loss_base_2(y_true, y_pred):\n    y_pred = [get_sii(x) for x in y_pred]\n    return (\"kappa_base\", -quadratic_weighted_kappa(y_true, y_pred), True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T06:57:37.162326Z","iopub.execute_input":"2024-12-18T06:57:37.162805Z","iopub.status.idle":"2024-12-18T06:57:37.172813Z","shell.execute_reply.started":"2024-12-18T06:57:37.162766Z","shell.execute_reply":"2024-12-18T06:57:37.171365Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class EnsembleModel():\n    def __init__(self, estimators, target, threshold=[]):\n        self.estimators = estimators\n        self.target = target\n        self.threshold = threshold\n\n    def predict(self, X):\n        if len(threshold):\n            y_pred = [m.predict(X) for m in self.estimators]\n            y_pred = np.mean(y_pred, axis=0)\n            return threshold_rounder(y_pred, self.threshold)   \n\n    def predict_raw(self, X):\n        y_pred = [m.predict(X) for m in self.estimators]\n        y_pred = np.mean(y_pred, axis=0)\n        return y_pred","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T06:57:38.344540Z","iopub.execute_input":"2024-12-18T06:57:38.345002Z","iopub.status.idle":"2024-12-18T06:57:38.352740Z","shell.execute_reply.started":"2024-12-18T06:57:38.344925Z","shell.execute_reply":"2024-12-18T06:57:38.351447Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"* Optuna","metadata":{}},{"cell_type":"code","source":"if RUN_OPTUNA:\n    train_sample, val_sample = train_test_split(df_train, test_size=0.1, random_state=SEED)\n\n    # for target_1\n    X_train = train_sample.drop([target_1, target_2], axis=1)\n    y_train = train_sample[target_1]\n\n    X_val = val_sample.drop([target_1, target_2], axis=1)\n    y_val = val_sample[target_1]\n    \n    def objective(trial):\n        lgbm_param = {\n            \"n_estimators\": trial.suggest_int(\"n_estimators\", 100, 300, step=50),\n            \"learning_rate\": trial.suggest_float(\"learning_rate\", 0.0001, 0.01),\n            \"max_depth\": trial.suggest_int(\"max_depth\", 5, 10, step=1),\n            \"num_leaves\": trial.suggest_int(\"num_leaves\", 100, 500, step=1, log=True),\n            \"min_child_samples\": trial.suggest_int(\"min_child_samples\", 5, 30), # 13\n            \"min_child_weight\": trial.suggest_float(\"min_child_weight\", 1e-10, 1e-3, log=True),\n            \"colsample_bytree\": trial.suggest_float(\"colsample_bytree\", 0.3, 0.9, log=True),\n            \"subsample\": trial.suggest_float(\"subsample\", 0.3, 0.9, log=True),\n            \"subsample_freq\": trial.suggest_int(\"subsample_freq\", 0, 10),\n            \"reg_alpha\": trial.suggest_float(\"reg_alpha\", 0.01, 10, log=True),\n            \"reg_lambda\": trial.suggest_float(\"reg_lambda\", 0.001, 0.1, log=True),\n            \"random_state\": SEED,\n            \"verbose\": -1\n        }\n    \n        model = LGBMRegressor(**lgbm_param)\n        model.fit(X_train, y_train, eval_set=(X_val, y_val), eval_metric=kappa_loss_base_1, categorical_feature=categorical_features)\n    \n        y_pred = model.predict(X_val)\n        error = -cohen_kappa_score(y_val, y_pred.round(0).astype(int), weights=\"quadratic\")\n        \n        return error\n        \n    optuna.logging.get_logger(\"optuna\").addHandler(logging.StreamHandler(sys.stdout))\n    study = optuna.create_study(\n        sampler=optuna.samplers.TPESampler(seed=SEED),\n        pruner=optuna.pruners.HyperbandPruner()\n    )\n    study.optimize(objective, n_trials=30)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T06:57:38.892323Z","iopub.execute_input":"2024-12-18T06:57:38.893450Z","iopub.status.idle":"2024-12-18T06:57:38.908183Z","shell.execute_reply.started":"2024-12-18T06:57:38.893403Z","shell.execute_reply":"2024-12-18T06:57:38.906070Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if RUN_OPTUNA:\n    train_sample, val_sample = train_test_split(df_train, test_size=0.1, random_state=SEED)\n\n    # for target_2\n    X_train = train_sample.drop([target_1, target_2], axis=1)\n    y_train = train_sample[target_2]\n    sii_train = train_sample[target_1]\n    print(X_train.shape[0])\n\n    X_val = val_sample.drop([target_1, target_2], axis=1)\n    y_val = val_sample[target_2]\n    sii_val = val_sample[target_1]\n    \n    def objective(trial):\n        lgbm_param = {\n            \"n_estimators\": trial.suggest_int(\"n_estimators\", 30, 300, step=50),\n            \"learning_rate\": trial.suggest_float(\"learning_rate\", 0.0001, 0.01),\n            \"max_depth\": trial.suggest_int(\"max_depth\", 5, 20, step=1),\n            \"num_leaves\": trial.suggest_int(\"num_leaves\", 20, 500, step=1),\n            \"min_child_samples\": trial.suggest_int(\"min_child_samples\", 5, 50), # 13\n            \"min_child_weight\": trial.suggest_float(\"min_child_weight\", 1e-10, 1e-2),\n            \"colsample_bytree\": trial.suggest_float(\"colsample_bytree\", 0.1, 0.9),\n            \"subsample\": trial.suggest_float(\"subsample\", 0.3, 0.9),\n            \"subsample_freq\": trial.suggest_int(\"subsample_freq\", 0, 10),\n            \"reg_alpha\": trial.suggest_float(\"reg_alpha\", 0.01, 10),\n            \"reg_lambda\": trial.suggest_float(\"reg_lambda\", 0.001, 0.1),\n            \"random_state\": SEED,\n            \"verbose\": -1\n        }\n    \n        model = LGBMRegressor(**lgbm_param)\n        model.fit(X_train, y_train, eval_set=(X_val, y_val), eval_metric=\"l2\", categorical_feature=categorical_features)\n    \n        y_pred = model.predict(X_val)\n        error = -cohen_kappa_score(sii_val, [get_sii(y) for y in y_pred], weights=\"quadratic\")\n        \n        return error\n        \n    optuna.logging.get_logger(\"optuna\").addHandler(logging.StreamHandler(sys.stdout))\n    study = optuna.create_study(\n        sampler=optuna.samplers.TPESampler(seed=SEED),\n        pruner=optuna.pruners.HyperbandPruner()\n    )\n    study.optimize(objective, n_trials=5000)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T06:57:38.930896Z","iopub.execute_input":"2024-12-18T06:57:38.931399Z","iopub.status.idle":"2024-12-18T06:57:38.943698Z","shell.execute_reply.started":"2024-12-18T06:57:38.931357Z","shell.execute_reply":"2024-12-18T06:57:38.942307Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"* Single Tabular","metadata":{}},{"cell_type":"code","source":"lgbm_params = {\n    \"learning_rate\": 0.046,\n    \"max_depth\": 12,\n    \"num_leaves\": 478,\n    \"min_child_samples\": 13,\n    \"colsample_bytree\": 0.893,\n    \"subsample\": 0.784,\n    \"subsample_freq\": 4,\n    \"reg_alpha\": 10,\n    \"reg_lambda\": 0.01,\n    \"verbose\": -1\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T06:57:38.993457Z","iopub.execute_input":"2024-12-18T06:57:38.994713Z","iopub.status.idle":"2024-12-18T06:57:38.999978Z","shell.execute_reply.started":"2024-12-18T06:57:38.994666Z","shell.execute_reply":"2024-12-18T06:57:38.998746Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_1(model_params, train_params):\n    train_score_list = []\n    val_score_list = []\n    \n    oof_pred = np.zeros(df_train.shape[0], dtype=float)\n    \n    model_list = []\n\n    skf = StratifiedKFold(n_splits=N_FOLD, shuffle=True, random_state=SEED)\n    \n    X = df_train.drop([target_1, target_2], axis=1)\n    y = df_train[target_1]\n    \n    for fold, (train_idx, val_idx) in enumerate(tqdm(skf.split(X, y.apply(get_sii)))):\n        X_train = X.iloc[train_idx]\n        y_train = y.iloc[train_idx]\n\n        X_val = X.iloc[val_idx]\n        y_val = y.iloc[val_idx]\n        \n        model = LGBMRegressor(**model_params, random_state=SEED)\n        model.fit(X_train, y_train, eval_set=(X_val, y_val), eval_metric=kappa_loss_base_1, **train_params)\n        \n        y_pred = model.predict(X_train)\n        train_score = quadratic_weighted_kappa(y_train, y_pred.round(0).astype(int))\n\n        y_pred = model.predict(X_val)\n        val_score = quadratic_weighted_kappa(y_val, y_pred.round(0).astype(int))\n        oof_pred[val_idx] = y_pred\n\n        print(f\"[fold {fold}] train score: {train_score}\")\n        print(f\"[fold {fold}] val score: {val_score}\")\n\n        train_score_list.append(train_score)\n        val_score_list.append(val_score)\n        \n        model_list.append(model)\n\n    # val score\n    best_idx = np.argmax(val_score_list)\n    mean_score = np.mean(val_score_list)\n    # p = ttest_1samp(val_score_list, mean_score)[1] \n    print(f\"best val score: Model {best_idx}, val: {val_score_list[best_idx]}\")\n    print(f\"mean val score: {mean_score}\")\n\n    # tuned score\n    kappa_opt = minimize(kappa_loss, x0=[0.5, 1.5, 2.5], args=(y, oof_pred), method=\"Nelder-Mead\")\n    assert kappa_opt.success, \"Optimization did not converge.\"\n\n    tuned_score = -kappa_loss(kappa_opt.x, y, oof_pred)\n    print(f\"tuned score: {tuned_score}, threshold: {kappa_opt.x}\")\n\n    return model_list, kappa_opt.x","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T06:57:39.043140Z","iopub.execute_input":"2024-12-18T06:57:39.043619Z","iopub.status.idle":"2024-12-18T06:57:39.055972Z","shell.execute_reply.started":"2024-12-18T06:57:39.043579Z","shell.execute_reply":"2024-12-18T06:57:39.054472Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_params = {\n    \"categorical_feature\": list(filter(lambda x: x in df_train.columns, categorical_features))\n}\n\nlgbm_models, threshold = train_1(lgbm_params, train_params)\n\nMODELS.append(EnsembleModel(lgbm_models, target=target_1, threshold=threshold))\nWEIGHTS.append(1.0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T06:57:39.094826Z","iopub.execute_input":"2024-12-18T06:57:39.095300Z","iopub.status.idle":"2024-12-18T06:57:43.707726Z","shell.execute_reply.started":"2024-12-18T06:57:39.095260Z","shell.execute_reply":"2024-12-18T06:57:43.706644Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train(regressor, model_params, train_params):\n    train_score_list = []\n    val_score_list = []\n    \n    oof_pred = np.zeros(df_train.shape[0], dtype=float)\n    \n    model_list = []\n\n    skf = StratifiedKFold(n_splits=N_FOLD, shuffle=True, random_state=SEED)\n    \n    X = df_train.drop([target_1, target_2], axis=1)\n    y = df_train[target_1]\n    \n    for fold, (train_idx, val_idx) in enumerate(tqdm(skf.split(X, y.apply(get_sii)))):\n        X_train = X.iloc[train_idx]\n        y_train = y.iloc[train_idx]\n\n        X_val = X.iloc[val_idx]\n        y_val = y.iloc[val_idx]\n        \n        model = regressor(**model_params, random_state=SEED)\n        # model.fit(X_train, y_train, eval_set=(X_val, y_val), eval_metric=kappa_loss_base_1, **train_params)\n        model.fit(X_train, y_train)\n        \n        y_pred = model.predict(X_train)\n        train_score = quadratic_weighted_kappa(y_train, y_pred.round(0).astype(int))\n\n        y_pred = model.predict(X_val)\n        val_score = quadratic_weighted_kappa(y_val, y_pred.round(0).astype(int))\n        oof_pred[val_idx] = y_pred\n\n        print(f\"[fold {fold}] train score: {train_score}\")\n        print(f\"[fold {fold}] val score: {val_score}\")\n\n        train_score_list.append(train_score)\n        val_score_list.append(val_score)\n        \n        model_list.append(model)\n\n    # val score\n    best_idx = np.argmax(val_score_list)\n    mean_score = np.mean(val_score_list)\n    # p = ttest_1samp(val_score_list, mean_score)[1] \n    print(f\"best val score: Model {best_idx}, val: {val_score_list[best_idx]}\")\n    print(f\"mean val score: {mean_score}\")\n\n    # tuned score\n    kappa_opt = minimize(kappa_loss, x0=[0.5, 1.5, 2.5], args=(y, oof_pred), method=\"Nelder-Mead\")\n    assert kappa_opt.success, \"Optimization did not converge.\"\n\n    tuned_score = -kappa_loss(kappa_opt.x, y, oof_pred)\n    print(f\"tuned score: {tuned_score}, threshold: {kappa_opt.x}\")\n\n    return model_list, kappa_opt.x\n\n\ncb_params = {\n    'learning_rate': 0.05,\n    'depth': 6,\n    'iterations': 200,\n    'verbose': 0,\n    'l2_leaf_reg': 10,  # Increase this value\n}\n\nxgb_params = {\n    'learning_rate': 0.05,\n    'max_depth': 6,\n    'n_estimators': 200,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 1,  # Increased from 0.1\n    'reg_lambda': 5,  # Increased from 1\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T06:57:43.709491Z","iopub.execute_input":"2024-12-18T06:57:43.709865Z","iopub.status.idle":"2024-12-18T06:57:43.722298Z","shell.execute_reply.started":"2024-12-18T06:57:43.709832Z","shell.execute_reply":"2024-12-18T06:57:43.721086Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# cb_models, threshold = train(CatBoostRegressor, cb_params, train_params)\n\n# MODELS.append(EnsembleModel(lgbm_models, target=target_1, threshold=threshold))\n# WEIGHTS.append(1.0)\n\n# xbg_models, threshold = train(XGBRegressor, xgb_params, train_params)\n\n# MODELS.append(EnsembleModel(lgbm_models, target=target_1, threshold=threshold))\n# WEIGHTS.append(1.0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T06:57:43.723851Z","iopub.execute_input":"2024-12-18T06:57:43.724346Z","iopub.status.idle":"2024-12-18T06:57:43.740496Z","shell.execute_reply.started":"2024-12-18T06:57:43.724296Z","shell.execute_reply":"2024-12-18T06:57:43.739248Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lgbm_params2 = {\n    \"n_estimators\": 280,\n    \"learning_rate\": 0.00797759249109509,\n    \"max_depth\": 5,\n    \"num_leaves\": 306,\n    \"min_child_samples\": 13,\n    \"min_child_weight\": 0.0023448043225863698,\n    \"colsample_bytree\": 0.2105019668032641,\n    \"subsample\": 0.34575754434626743,\n    \"subsample_freq\": 7,\n    \"reg_alpha\": 6.951987691926444,\n    \"reg_lambda\": 0.054756259022914325,  # Increased from 2.68e-06\n    \"verbose\": -1\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T06:57:43.743538Z","iopub.execute_input":"2024-12-18T06:57:43.744101Z","iopub.status.idle":"2024-12-18T06:57:43.753171Z","shell.execute_reply.started":"2024-12-18T06:57:43.744050Z","shell.execute_reply":"2024-12-18T06:57:43.751985Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_2(model_params, train_params):\n    train_score_list = []\n    val_score_list = []\n    \n    oof_pred = np.zeros(df_train.shape[0], dtype=float)\n    \n    model_list = []\n\n    skf = StratifiedKFold(n_splits=N_FOLD, shuffle=True)\n    \n    X = df_train.drop([target_1, target_2], axis=1)\n    X_sii = df_train[target_1]\n    y = df_train[target_2]\n    \n    for fold, (train_idx, val_idx) in enumerate(tqdm(skf.split(X, y.apply(get_sii)))):\n        X_train = X.iloc[train_idx]\n        y_train = y.iloc[train_idx]\n\n        X_val = X.iloc[val_idx]\n        y_val = y.iloc[val_idx]\n        \n        model = LGBMRegressor(**model_params, random_state=SEED)\n        model.fit(X_train, y_train, eval_set=(X_val, y_val), eval_metric=\"l2\", **train_params)\n        \n        y_pred = model.predict(X_train)\n        print(y_pred)\n        train_score = quadratic_weighted_kappa([get_sii(y) for y in y_train], [get_sii(y) for y in y_pred])\n\n        y_pred = model.predict(X_val)\n        val_score = quadratic_weighted_kappa([get_sii(y) for y in y_val], [get_sii(y) for y in y_pred])\n        oof_pred[val_idx] = y_pred\n\n        print(f\"[fold {fold}] train score: {train_score}\")\n        print(f\"[fold {fold}] val score: {val_score}\")\n\n        train_score_list.append(train_score)\n        val_score_list.append(val_score)\n        \n        model_list.append(model)\n\n    # val score\n    best_idx = np.argmax(val_score_list)\n    mean_score = np.mean(val_score_list)\n    # p = ttest_1samp(val_score_list, mean_score)[1] \n    print(f\"best val score: Model {best_idx}, val: {val_score_list[best_idx]}\")\n    print(f\"mean val score: {mean_score}\")\n\n    # tuned score\n    kappa_opt = minimize(kappa_loss, x0=[30, 50, 80], args=(X_sii, oof_pred), method=\"Nelder-Mead\")\n    assert kappa_opt.success, \"Optimization did not converge.\"\n\n    tuned_score = -kappa_loss(kappa_opt.x, X_sii, oof_pred)\n    print(f\"tuned score: {tuned_score}, threshold: {kappa_opt.x}\")\n\n    return model_list, kappa_opt.x","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T06:57:43.754627Z","iopub.execute_input":"2024-12-18T06:57:43.754986Z","iopub.status.idle":"2024-12-18T06:57:43.773144Z","shell.execute_reply.started":"2024-12-18T06:57:43.754952Z","shell.execute_reply":"2024-12-18T06:57:43.771926Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# train_params = {\n#     \"categorical_feature\": categorical_features\n# }\n\n# lgbm_models, threshold = train_2(lgbm_params2, train_params)\n\n# MODELS.append(EnsembleModel(lgbm_models, target=target_2, threshold=[30, 50, 80]))\n# WEIGHTS.append(1.0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T06:57:43.774604Z","iopub.execute_input":"2024-12-18T06:57:43.775021Z","iopub.status.idle":"2024-12-18T06:57:43.790027Z","shell.execute_reply.started":"2024-12-18T06:57:43.774964Z","shell.execute_reply":"2024-12-18T06:57:43.788802Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"* LightAutoML","metadata":{}},{"cell_type":"code","source":"# AUTOML_CFG = {\n#     \"TASK\": \"REG\", # [\"CLF\", \"REG\"]\n#     \"LOSS\": \"QUANTILE\", # [\"MSE\", \"RMSLE\", \"QUANTILE\"]\n#     \"USE_UTILIZED\": True\n# }","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T06:57:43.791727Z","iopub.execute_input":"2024-12-18T06:57:43.792239Z","iopub.status.idle":"2024-12-18T06:57:43.808537Z","shell.execute_reply.started":"2024-12-18T06:57:43.792172Z","shell.execute_reply":"2024-12-18T06:57:43.807292Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# N_THREADS = 4\n# TEST_SIZE = 0.2\n# N_FOLDS = 5\n# TIMEOUT = 60 if DEMO else 3600\n# m = [\"lgb\"]\n\n# torch.set_num_threads(N_THREADS)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T06:57:43.810144Z","iopub.execute_input":"2024-12-18T06:57:43.810659Z","iopub.status.idle":"2024-12-18T06:57:43.820354Z","shell.execute_reply.started":"2024-12-18T06:57:43.810625Z","shell.execute_reply":"2024-12-18T06:57:43.819201Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# if AUTOML_CFG[\"TASK\"] == \"REG\":\n#     task_params = {\"name\": \"reg\", \"metric\": quadratic_weighted_kappa}\n    \n#     if AUTOML_CFG[\"LOSS\"] == \"MSE\":\n#         task_params[\"loss\"] = \"mse\"\n#     elif AUTOML_CFG[\"LOSS\"] == \"RMSLE\":\n#         task_params[\"loss\"] = \"rmsle\"\n#     elif AUTOML_CFG[\"LOSS\"] == \"QUANTILE\":\n#         task_params[\"loss\"] = \"quantile\"\n#         task_params[\"loss_params\"] = {\"q\": 0.9}\n# else:\n#     task_params = {\"name\": \"multiclass\", \"loss\": \"crossentropy\", \"metric\": quadratic_weighted_kappa}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T06:57:43.822164Z","iopub.execute_input":"2024-12-18T06:57:43.822565Z","iopub.status.idle":"2024-12-18T06:57:43.832295Z","shell.execute_reply.started":"2024-12-18T06:57:43.822493Z","shell.execute_reply":"2024-12-18T06:57:43.831033Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# task = Task(**task_params)\n\n# roles = {\n#     \"target\": target\n# }","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T06:57:43.835984Z","iopub.execute_input":"2024-12-18T06:57:43.836772Z","iopub.status.idle":"2024-12-18T06:57:43.844474Z","shell.execute_reply.started":"2024-12-18T06:57:43.836734Z","shell.execute_reply":"2024-12-18T06:57:43.843320Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# if AUTOML_CFG[\"USE_UTILIZED\"]:\n#     automl = TabularUtilizedAutoML(\n#                 task = task, \n#                 timeout = TIMEOUT,\n#                 cpu_limit = N_THREADS,\n#                 gpu_ids = None,\n#                 max_runs_per_config = 1,\n#                 general_params = {\"use_algos\": MODELS},\n#                 reader_params = {\"n_jobs\": N_THREADS, \"cv\": N_FOLDS, \"random_state\": SEED},\n#                 # lgb_params = {\"default_params\": lgb_params, \"freeze_defaults\": True},\n#             )\n# else:\n#     automl = TabularAutoML(\n#                 task = task, \n#                 timeout = TIMEOUT,\n#                 cpu_limit = N_THREADS,\n#                 gpu_ids = None,\n#                 general_params = {\"use_algos\": m},\n#                 reader_params = {\"n_jobs\": N_THREADS, \"cv\": N_FOLDS, \"random_state\": SEED},\n#                 # lgb_params = {\"default_params\": lgb_params, \"freeze_defaults\": True},\n#             )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T06:57:43.846056Z","iopub.execute_input":"2024-12-18T06:57:43.847094Z","iopub.status.idle":"2024-12-18T06:57:43.858050Z","shell.execute_reply.started":"2024-12-18T06:57:43.847042Z","shell.execute_reply":"2024-12-18T06:57:43.856820Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if RUN_AUTOML:\n    oof_pred = automl.fit_predict(df_train, roles=roles, verbose=4)\n    score_metric(df_train[target], oof_pred.data)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T06:57:43.859618Z","iopub.execute_input":"2024-12-18T06:57:43.860118Z","iopub.status.idle":"2024-12-18T06:57:43.876268Z","shell.execute_reply.started":"2024-12-18T06:57:43.860068Z","shell.execute_reply":"2024-12-18T06:57:43.874893Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Analyze Model","metadata":{}},{"cell_type":"code","source":"# Analyze AutoML\nif RUN_AUTOML:\n    print(automl.create_model_str_desc())\n\n    %%time\n\n    fast_fi = automl.get_feature_scores(\"fast\")\n    \n    # Run only automl is trained enough\n    if fast_fi is not None:\n        fast_fi.set_index(\"Feature\")[\"Importance\"].plot.bar(figsize = (30, 10), grid = True)\n    \n        drop_features = fast_fi[fast_fi[\"Importance\"] == 0]\n        drop_features[\"Feature\"].values\n        print(drop_features[\"Feature\"].values)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T06:57:43.877850Z","iopub.execute_input":"2024-12-18T06:57:43.879176Z","iopub.status.idle":"2024-12-18T06:57:43.890809Z","shell.execute_reply.started":"2024-12-18T06:57:43.879118Z","shell.execute_reply":"2024-12-18T06:57:43.889616Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# union = pd.Series([False] * len(MODELS[0].estimators[0].feature_name_))\n# inter = pd.Series([False] * len(MODELS[0].estimators[0].feature_name_))\n\n# for idx, model in enumerate(MODELS):\n#     for estimator in model.estimators:\n#         lgbm.plot_importance(estimator, max_num_features=30)\n#         union = union | (estimator.feature_importances_ == 0)\n#         inter = inter & (estimator.feature_importances_ == 0)\n\n# print(pd.Series(estimator.feature_name_)[union].values)\n# print(pd.Series(estimator.feature_name_)[inter].values)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T06:57:43.893486Z","iopub.execute_input":"2024-12-18T06:57:43.893904Z","iopub.status.idle":"2024-12-18T06:57:43.904991Z","shell.execute_reply.started":"2024-12-18T06:57:43.893869Z","shell.execute_reply":"2024-12-18T06:57:43.903824Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Save","metadata":{}},{"cell_type":"code","source":"for idx, model in enumerate(MODELS):\n    joblib.dump(model, f\"model_{idx}.pkl\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T06:57:44.698223Z","iopub.execute_input":"2024-12-18T06:57:44.698655Z","iopub.status.idle":"2024-12-18T06:57:44.767609Z","shell.execute_reply.started":"2024-12-18T06:57:44.698618Z","shell.execute_reply":"2024-12-18T06:57:44.766366Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Submit","metadata":{}},{"cell_type":"code","source":"test_pred_list = [model.predict(df_test) for model in MODELS]\n# test_pred_list = [model.predict_raw(df_test) for model in MODELS]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T06:57:45.479817Z","iopub.execute_input":"2024-12-18T06:57:45.480253Z","iopub.status.idle":"2024-12-18T06:57:45.503105Z","shell.execute_reply.started":"2024-12-18T06:57:45.480215Z","shell.execute_reply":"2024-12-18T06:57:45.501848Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# test_pred = np.zeros(test_pred_list[0].shape)\n# for i in range(len(MODELS)):\n#     test_pred += test_pred_list[i] * WEIGHTS[i]\nfrom scipy import stats\ntest_pred = stats.mode(np.asarray(test_pred_list))[0]\n\ntest_pred = pd.DataFrame(test_pred, index=df_test.index, columns=[target_1])\ntest_pred[target_1] = test_pred[target_1].astype(int)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T06:57:46.036878Z","iopub.execute_input":"2024-12-18T06:57:46.037304Z","iopub.status.idle":"2024-12-18T06:57:46.052181Z","shell.execute_reply.started":"2024-12-18T06:57:46.037271Z","shell.execute_reply":"2024-12-18T06:57:46.050607Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission = pl.read_csv(submission_path)\n\ntest_pred = test_pred.reindex(index=submission[\"id\"])\nsubmission = submission.with_columns(sii=test_pred.values[:, 0])\n\nsubmission.write_csv(\"submission.csv\")\nsubmission.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T06:57:47.308742Z","iopub.execute_input":"2024-12-18T06:57:47.309188Z","iopub.status.idle":"2024-12-18T06:57:47.323838Z","shell.execute_reply.started":"2024-12-18T06:57:47.309151Z","shell.execute_reply":"2024-12-18T06:57:47.322170Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}