{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":96164,"databundleVersionId":12993472,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# XGBoost Ensemble. An end-to-end solution.\n\nThis notebook covers an end-to-end approach to the DRW Crypto competition. From data preprocessing to the final model, this notebook tries different configurations involving data scaling, outlier filtering, feature selection, and model configuration. Then, performs the final prediction with the best configuration found.\n\n**To ensure that no future picking is performed, this notebook is restricted to the official competition input data. Moreover, feature selection is performed from scratch, with no prefab subset of features.**","metadata":{}},{"cell_type":"code","source":"!pip install polars==1.31.0\n\nimport os\nimport time\nimport json\nimport pickle\nimport builtins\nimport numpy as np\nimport polars as pl\nimport pandas as pd\nimport xgboost as xgb\nfrom scipy.stats import pearsonr\nfrom sklearn.model_selection import ParameterGrid","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-07-23T05:40:42.042463Z","iopub.execute_input":"2025-07-23T05:40:42.042766Z","iopub.status.idle":"2025-07-23T05:40:52.872647Z","shell.execute_reply.started":"2025-07-23T05:40:42.042741Z","shell.execute_reply":"2025-07-23T05:40:52.871442Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# The ConfigGrid class\n\nCreate the ConfigGrid class that will be used to encapsulate the configuration of the different executions.","metadata":{}},{"cell_type":"code","source":"CWD = \"/home/rayllon/_inception/Kaggle/DRW_Crypto/xgboost/main_xgboost_ensemble_results\"\n\n\n# %%\nclass ConfigGrid:\n    \"\"\"\n    * add_extra_features: Filter outliers below 1% and above 99%\n    * filter_outliers → Percentile threshold for filtering features based on avg. feature importance of 3\n      xgb of filter_features_model_n_rounds epochs each (only on training set)\n    * filter_features: Number of rounds for the feature importance model\n    * filter_features_model_n_rounds: Number of rounds for the feature importance model\n    * scaler: Options: \"standard\", \"minmax\", \"robust\"\n    * n_models: Number of models to train in the ensemble\n    * eastop_patience: Early stopping patience each model in the ensemble\n    * model_n_rounds: Number of rounds for each model in the ensemble\n    * base_params: Base parameters for the XGBoost model\n    * params: Fine tuning parameters for the XGBoost model\n    * train_data_path: Path to the training data\n    * models_save_dir: Directory to save the trained models\n    \"\"\"\n\n    def __init__(\n        self,\n        add_extra_features,\n        filter_outliers,\n        filter_features,\n        filter_features_model_n_rounds,\n        scaler,\n        n_models,\n        eastop_patience,\n        model_n_rounds,\n        base_params,\n        params,\n        train_data_path,\n        models_save_dir,\n    ):\n        self.ADD_EXTRA_FEATURES = add_extra_features\n        self.FILTER_OUTLIERS = filter_outliers\n        self.FILTER_FEATURES = filter_features\n        self.FILTER_FEATURES_MODEL_N_ROUNDS = filter_features_model_n_rounds\n        self.SCALER = scaler\n        self.N_MODELS = n_models\n        self.EASTOP_PATIENCE = eastop_patience\n        self.MODEL_N_ROUNDS = model_n_rounds\n        self.BASE_PARAMS = base_params\n        self.PARAMS = params\n        self.TRAIN_DATA_PATH = train_data_path\n        self.MODELS_SAVE_DIR = models_save_dir\n\n    def stream_configs(self):\n        config_grid_dict_copy = self.__dict__.copy()\n        base_params = config_grid_dict_copy.pop(\"BASE_PARAMS\")\n        params = config_grid_dict_copy.pop(\"PARAMS\")\n        config_dict = {**config_grid_dict_copy, **base_params, **params}\n\n        for key, value in config_dict.items():\n            if not isinstance(value, list):\n                config_dict[key] = [value]\n\n        grid = ParameterGrid(config_dict)\n        for config in grid:\n            config_obj = ConfigGrid(\n                add_extra_features=config[\"ADD_EXTRA_FEATURES\"],\n                filter_outliers=config[\"FILTER_OUTLIERS\"],\n                filter_features=config[\"FILTER_FEATURES\"],\n                filter_features_model_n_rounds=config[\"FILTER_FEATURES_MODEL_N_ROUNDS\"],\n                scaler=config[\"SCALER\"],\n                n_models=config[\"N_MODELS\"],\n                eastop_patience=config[\"EASTOP_PATIENCE\"],\n                model_n_rounds=config[\"MODEL_N_ROUNDS\"],\n                base_params={k: v for k, v in config.items() if k in base_params},\n                params={k: v for k, v in config.items() if k in params},\n                train_data_path=config[\"TRAIN_DATA_PATH\"],\n                models_save_dir=config[\"MODELS_SAVE_DIR\"],\n            )\n            yield config_obj\n\n    def get_config_id(self):\n        # id example: config_xgbensemble__extraFeatures__filterOutliers_1_99__filterFeatures_25_250_rounds__scaler_standard__nModels_10__eastopPatience_250__modelNRounds_2500\n        filter_outliers_id_test = (\n            f\"filterOutliers_{self.FILTER_OUTLIERS[0]}_{self.FILTER_OUTLIERS[1]}\"\n            if self.FILTER_OUTLIERS is not None\n            else \"filterOutliers_None\"\n        )\n        return f\"config_xgbensemble__extraFeatures_{self.ADD_EXTRA_FEATURES}__{filter_outliers_id_test}__filterFeatures_{self.FILTER_FEATURES}_{self.FILTER_FEATURES_MODEL_N_ROUNDS}_rounds__scaler_{self.SCALER}__nModels_{self.N_MODELS}__eastopPatience_{self.EASTOP_PATIENCE}__modelNRounds_{self.MODEL_N_ROUNDS}\"","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-07-23T05:40:52.874071Z","iopub.execute_input":"2025-07-23T05:40:52.874639Z","iopub.status.idle":"2025-07-23T05:40:52.885075Z","shell.execute_reply.started":"2025-07-23T05:40:52.874586Z","shell.execute_reply":"2025-07-23T05:40:52.884041Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature engineering\n\nDefine a function that extends the training dataset with a set of additional features, commonly employed in market forecasting tasks. ","metadata":{}},{"cell_type":"code","source":"# %%\ndef add_features_polars(df, test=False):\n    # Original features\n    df = df.with_columns((df[\"bid_qty\"] * df[\"ask_qty\"]).alias(\"bid_ask_interaction\"))\n\n    df = df.with_columns(\n        ((df[\"buy_qty\"] + df[\"sell_qty\"]) / (df[\"bid_qty\"] + df[\"ask_qty\"] + 1e-10)).alias(\n            \"trade_depth_utilisation\"\n        )\n    )\n    df = df.with_columns((df[\"buy_qty\"] / (df[\"bid_qty\"] + 1e-10)).alias(\"side_depth_consumption_buy\"))\n    df = df.with_columns((df[\"sell_qty\"] / (df[\"ask_qty\"] + 1e-10)).alias(\"side_depth_consumption_sell\"))\n\n    df = df.with_columns((df[\"bid_qty\"] * df[\"buy_qty\"]).alias(\"bid_buy_interaction\"))\n    df = df.with_columns((df[\"bid_qty\"] * df[\"sell_qty\"]).alias(\"bid_sell_interaction\"))\n    df = df.with_columns((df[\"ask_qty\"] * df[\"buy_qty\"]).alias(\"ask_buy_interaction\"))\n    df = df.with_columns((df[\"ask_qty\"] * df[\"sell_qty\"]).alias(\"ask_sell_interaction\"))\n\n    df = df.with_columns((df[\"sell_qty\"] * df[\"volume\"]).alias(\"volume_weighted_sell\"))\n    df = df.with_columns((df[\"buy_qty\"] / (df[\"sell_qty\"] + 1e-10)).alias(\"buy_sell_ratio\"))\n    df = df.with_columns(\n        (df[\"buy_qty\"] / (df[\"sell_qty\"] + df[\"buy_qty\"] + 1e-10)).alias(\"taker_buy_sell_fraction\")\n    )\n    df = df.with_columns((df[\"sell_qty\"] / (df[\"volume\"] + 1e-10)).alias(\"selling_pressure\"))\n    df = df.with_columns((np.log1p(df[\"volume\"])).alias(\"log_volume\"))\n\n    df = df.with_columns(\n        (np.abs(df[\"buy_qty\"] - df[\"sell_qty\"]) / (df[\"volume\"] + 1e-10)).alias(\"effective_spread_proxy\")\n    )\n    df = df.with_columns(\n        ((df[\"bid_qty\"] - df[\"ask_qty\"]) / (df[\"bid_qty\"] + df[\"ask_qty\"] + 1e-10)).alias(\"bid_ask_imbalance\")\n    )\n    df = df.with_columns(\n        ((df[\"buy_qty\"] - df[\"sell_qty\"]) / (df[\"buy_qty\"] + df[\"sell_qty\"] + 1e-10)).alias(\n            \"order_flow_imbalance\"\n        )\n    )\n    df = df.with_columns(((df[\"bid_qty\"] + df[\"ask_qty\"]) / (df[\"volume\"] + 1e-10)).alias(\"liquidity_ratio\"))\n\n    # === NEW MICROSTRUCTURE FEATURES ===\n\n    # Price Pressure Indicators\n    df = df.with_columns((df[\"buy_qty\"] - df[\"sell_qty\"]).alias(\"net_order_flow\"))\n    df = df.with_columns((df[\"net_order_flow\"] / (df[\"volume\"] + 1e-10)).alias(\"normalized_net_flow\"))\n    df = df.with_columns((df[\"buy_qty\"] / (df[\"volume\"] + 1e-10)).alias(\"buying_pressure\"))\n    df = df.with_columns((df[\"buy_qty\"] * df[\"volume\"]).alias(\"volume_weighted_buy\"))\n\n    # Liquidity Depth Measures\n    df = df.with_columns((df[\"bid_qty\"] + df[\"ask_qty\"]).alias(\"total_depth\"))\n    df = df.with_columns(\n        ((df[\"bid_qty\"] - df[\"ask_qty\"]) / (df[\"total_depth\"] + 1e-10)).alias(\"depth_imbalance\")\n    )\n    df = df.with_columns(\n        (np.abs(df[\"bid_qty\"] - df[\"ask_qty\"]) / (df[\"total_depth\"] + 1e-10)).alias(\"relative_spread\")\n    )\n    df = df.with_columns((np.log1p(df[\"total_depth\"])).alias(\"log_depth\"))\n\n    # Order Flow Toxicity Proxies\n    df = df.with_columns((np.abs(df[\"net_order_flow\"]) / (df[\"volume\"] + 1e-10)).alias(\"kyle_lambda\"))\n    df = df.with_columns((np.abs(df[\"order_flow_imbalance\"]) * df[\"volume\"]).alias(\"flow_toxicity\"))\n    df = df.with_columns(\n        ((df[\"buy_qty\"] + df[\"sell_qty\"]) / (df[\"total_depth\"] + 1e-10)).alias(\"aggressive_flow_ratio\")\n    )\n\n    # Market Activity Indicators\n    df = df.with_columns((df[\"volume\"] / (df[\"total_depth\"] + 1e-10)).alias(\"volume_depth_ratio\"))\n    df = df.with_columns(\n        ((df[\"buy_qty\"] + df[\"sell_qty\"]) / (df[\"volume\"] + 1e-10)).alias(\"activity_intensity\")\n    )\n    df = df.with_columns((np.log1p(df[\"buy_qty\"])).alias(\"log_buy_qty\"))\n    df = df.with_columns((np.log1p(df[\"sell_qty\"])).alias(\"log_sell_qty\"))\n    df = df.with_columns((np.log1p(df[\"bid_qty\"])).alias(\"log_bid_qty\"))\n    df = df.with_columns((np.log1p(df[\"ask_qty\"])).alias(\"log_ask_qty\"))\n\n    # Microstructure Volatility Proxies\n    df = df.with_columns(\n        (2 * np.abs(df[\"net_order_flow\"]) / (df[\"volume\"] + 1e-10)).alias(\"realized_spread_proxy\")\n    )\n    df = df.with_columns((df[\"net_order_flow\"] / (df[\"total_depth\"] + 1e-10)).alias(\"price_impact_proxy\"))\n    df = df.with_columns((np.abs(df[\"depth_imbalance\"])).alias(\"quote_volatility_proxy\"))\n\n    # Complex Interaction Terms\n    df = df.with_columns((df[\"net_order_flow\"] * df[\"total_depth\"]).alias(\"flow_depth_interaction\"))\n    df = df.with_columns((df[\"order_flow_imbalance\"] * df[\"volume\"]).alias(\"imbalance_volume_interaction\"))\n    df = df.with_columns((df[\"total_depth\"] * df[\"volume\"]).alias(\"depth_volume_interaction\"))\n    df = df.with_columns((np.abs(df[\"buy_qty\"] - df[\"sell_qty\"])).alias(\"buy_sell_spread\"))\n    df = df.with_columns((np.abs(df[\"bid_qty\"] - df[\"ask_qty\"])).alias(\"bid_ask_spread\"))\n\n    # Information Asymmetry Measures\n    df = df.with_columns(\n        (df[\"net_order_flow\"] / (df[\"bid_qty\"] + df[\"ask_qty\"] + 1e-10)).alias(\"trade_informativeness\")\n    )\n    df = df.with_columns((df[\"buy_sell_spread\"] / (df[\"volume\"] + 1e-10)).alias(\"execution_shortfall_proxy\"))\n    df = df.with_columns(\n        (df[\"net_order_flow\"] / (df[\"total_depth\"] + 1e-10) * df[\"volume\"]).alias(\"adverse_selection_proxy\")\n    )\n\n    # Market Efficiency Indicators\n    df = df.with_columns((df[\"volume\"] / (df[\"buy_qty\"] + df[\"sell_qty\"] + 1e-10)).alias(\"fill_probability\"))\n    df = df.with_columns(\n        ((df[\"buy_qty\"] + df[\"sell_qty\"]) / (df[\"total_depth\"] + 1e-10)).alias(\"execution_rate\")\n    )\n    df = df.with_columns((df[\"volume\"] / (df[\"bid_ask_spread\"] + 1e-10)).alias(\"market_efficiency\"))\n\n    # Non-linear Transformations\n    df = df.with_columns((np.sqrt(df[\"volume\"])).alias(\"sqrt_volume\"))\n    df = df.with_columns((np.sqrt(df[\"total_depth\"])).alias(\"sqrt_depth\"))\n    df = df.with_columns((df[\"volume\"] ** 2).alias(\"volume_squared\"))\n    df = df.with_columns((df[\"order_flow_imbalance\"] ** 2).alias(\"imbalance_squared\"))\n\n    # Relative Measures\n    df = df.with_columns((df[\"bid_qty\"] / (df[\"total_depth\"] + 1e-10)).alias(\"bid_ratio\"))\n    df = df.with_columns((df[\"bid_qty\"] / (df[\"ask_qty\"] + 1e-10)).alias(\"bid_ask_depth_ratio\"))\n    df = df.with_columns((df[\"ask_qty\"] / (df[\"total_depth\"] + 1e-10)).alias(\"ask_ratio\"))\n    df = df.with_columns((df[\"buy_qty\"] / (df[\"buy_qty\"] + df[\"sell_qty\"] + 1e-10)).alias(\"buy_ratio\"))\n    df = df.with_columns((df[\"sell_qty\"] / (df[\"buy_qty\"] + df[\"sell_qty\"] + 1e-10)).alias(\"sell_ratio\"))\n\n    # Market Stress Indicators\n    df = df.with_columns(\n        (df[\"buy_qty\"] + df[\"sell_qty\"]) / (df[\"total_depth\"] + 1e-10).alias(\"liquidity_consumption\")\n    )\n    df = df.with_columns(\n        (df[\"volume\"] / (df[\"total_depth\"] + 1e-10) * np.abs(df[\"order_flow_imbalance\"])).alias(\n            \"market_stress\"\n        )\n    )\n    df = df.with_columns((df[\"volume\"] / (df[\"bid_qty\"] + df[\"ask_qty\"] + 1e-10)).alias(\"depth_depletion\"))\n\n    # Directional Indicators\n    df = df.with_columns((df[\"net_order_flow\"] / (df[\"volume\"] + 1e-10)).alias(\"net_buying_ratio\"))\n    df = df.with_columns((df[\"net_order_flow\"] * np.log1p(df[\"volume\"])).alias(\"directional_volume\"))\n    df = df.with_columns((np.sign(df[\"net_order_flow\"]) * df[\"volume\"]).alias(\"signed_volume\"))\n\n    if test:\n        full_features_list = list(df.columns)\n    else:\n        full_features_list = list(df.drop([\"label\"]).columns)\n        \n    full_features_list = list(set(full_features_list))  # Ensure unique features\n\n    # Replace infinities and NaNs\n    n_nans = np.sum(df.select(pl.all().is_nan().sum()).to_numpy())\n    n_infs = np.sum(df.select(pl.all().is_infinite().sum()).to_numpy())\n    n_nulls = np.sum(df.select(pl.all().is_null().sum()).to_numpy())\n    if (n_nans > 0) or (n_infs > 0) or (n_nulls > 0):\n        raise ValueError(\n            f\"DataFrame contains NaNs: {n_nans}, Infs: {n_infs}, Nulls: {n_nulls}. Please clean the data.\"\n        )\n\n    assert \"label\" not in full_features_list, \"Label should not be in features list\"\n\n    return df, full_features_list","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-07-23T05:40:52.885933Z","iopub.execute_input":"2025-07-23T05:40:52.886280Z","iopub.status.idle":"2025-07-23T05:40:53.021997Z","shell.execute_reply.started":"2025-07-23T05:40:52.886257Z","shell.execute_reply":"2025-07-23T05:40:53.020742Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Define data loaders\n\nDefine a funtion that loads and preprocesses the training data according the configuration specified.","metadata":{}},{"cell_type":"code","source":"def load_train_data(config, final=False):\n    features = [\"bid_qty\", \"ask_qty\", \"buy_qty\", \"sell_qty\", \"volume\"]\n    features += [\"X\" + str(i) for i in range(1, 781)]\n\n    train_dtypes = {\n        **{feature: pl.Float32 for feature in features},\n        \"label\": pl.Float64,\n        # \"timestamp\": pl.Datetime(\"ns\", None),\n    }\n    train = pl.scan_parquet(\n        config.TRAIN_DATA_PATH,\n        low_memory=True,\n        schema=train_dtypes,\n        extra_columns=\"ignore\",\n        cast_options=pl.ScanCastOptions(float_cast=\"downcast\"),\n    ).collect()\n\n    ## Below is used for notebook fast testing\n    #!#!#!#!#!#!#!#!#!#!#!#!#!#!\n    # train = train[:50000]\n    #!#!#!#!#!#!#!#!#!#!#!#!#!#!\n\n    # Filter bad rows and columns\n    bad_cols = [\n        col\n        for col in train.columns\n        if train.select(\n            (pl.col(col).is_nan() | (pl.col(col).is_infinite()) | (pl.col(col).is_null())).any()\n        ).item()\n    ]\n    if len(bad_cols) > 0:\n        train = train.drop(bad_cols)\n        # drop any rows with NaN, Inf, or Null values\n    # Filter out rows containing null, NaN, or infinite\n    mask = train.select(\n        pl.any_horizontal([pl.all().is_null(), pl.all().is_nan(), pl.all().is_infinite()]).alias(\n            \"any_invalid\"\n        )\n    )[\"any_invalid\"]\n    train = train.filter(~mask)\n\n    # Add extra feature engineering features if specified\n    all_features = train.drop(\"label\").columns\n    if config.ADD_EXTRA_FEATURES:\n        initial_n_features = len(all_features)\n        train, all_features = add_features_polars(train)\n        print(\n            f\"→ Extra Features: Added {len(all_features) - initial_n_features} extra features, total: {len(all_features)} features\"\n        )\n        assert \"label\" not in all_features, \"Label should not be in features list\"\n\n    train_end_idx = int(len(train) * 0.6)\n    if final:\n        # X_train, y_train = train[all_features][-train_end_idx:], train[\"label\"][-train_end_idx:]\n        X_train, y_train = train[all_features][:], train[\"label\"][:]\n        X_val = None\n        y_val = None    \n    else:\n        X_train, y_train = train[all_features][:train_end_idx], train[\"label\"][:train_end_idx]\n        X_val, y_val = train[all_features][train_end_idx:], train[\"label\"][train_end_idx:]\n    train = None  # Free memory\n\n    # Filter outliers if specified\n    if config.FILTER_OUTLIERS is not None:\n        # get rows index in y_train that are above the 99 percentile and below the 1 percentile\n        y_train = y_train.to_numpy()\n        y_train_mask = (y_train > np.percentile(y_train, config.FILTER_OUTLIERS[0])) & (\n            y_train < np.percentile(y_train, config.FILTER_OUTLIERS[1])\n        )\n        print(f\"→ Filtering outliers: Deleted a total of {len(y_train) - np.sum(y_train_mask)} outliers\")\n        X_train = X_train.filter(y_train_mask)\n        y_train = y_train[y_train_mask]\n\n    # Apply scaling if specified\n    if config.SCALER is not None:\n        if config.SCALER == \"standard\":\n            from sklearn.preprocessing import StandardScaler\n\n            scaler = StandardScaler()\n        elif config.SCALER == \"minmax\":\n            from sklearn.preprocessing import MinMaxScaler\n\n            scaler = MinMaxScaler()\n        elif config.SCALER == \"robust\":\n            from sklearn.preprocessing import RobustScaler\n\n            scaler = RobustScaler()\n        columns = X_train.columns\n        X_train = scaler.fit_transform(X_train)\n        X_train = pl.DataFrame(X_train, schema=columns)\n        if not final:\n            X_val = scaler.transform(X_val)\n            X_val = pl.DataFrame(X_val, schema=columns)\n        print(f\"→ Scaler: Applied {scaler}\")\n\n    if final:\n        print(\n            f\"→ FINAL Data loaded: X_train patterns: {len(X_train)} patterns and {len(all_features)} features\"\n        )\n    else:\n        print(\n            f\"→ Data loaded: Final X_train|X_val patterns: {len(X_train)}|{len(X_val)} patterns and {len(all_features)} features\"\n        )\n\n    if final:\n        return X_train, y_train, X_val, y_val, all_features, scaler\n    else:\n        return X_train, y_train, X_val, y_val, all_features\n\ndef load_test_data(config, scaler):\n    features = [\"bid_qty\", \"ask_qty\", \"buy_qty\", \"sell_qty\", \"volume\"]\n    features += [\"X\" + str(i) for i in range(1, 781)]\n\n    test_dtypes = {\n        **{feature: pl.Float32 for feature in features},\n    }\n    test = pl.scan_parquet(\n        \"/kaggle/input/drw-crypto-market-prediction/test.parquet\",\n        low_memory=True,\n        schema=test_dtypes,\n        extra_columns=\"ignore\",\n        cast_options=pl.ScanCastOptions(float_cast=\"downcast\"),\n    ).collect()\n\n    # Filter bad rows and columns\n    bad_cols = [\n        col\n        for col in test.columns\n        if test.select(\n            (pl.col(col).is_nan() | (pl.col(col).is_infinite()) | (pl.col(col).is_null())).any()\n        ).item()\n    ]\n    if len(bad_cols) > 0:\n        test = test.drop(bad_cols)\n        # drop any rows with NaN, Inf, or Null values\n    # Filter out rows containing null, NaN, or infinite\n    mask = test.select(\n        pl.any_horizontal([pl.all().is_null(), pl.all().is_nan(), pl.all().is_infinite()]).alias(\n            \"any_invalid\"\n        )\n    )[\"any_invalid\"]\n    test = test.filter(~mask)\n\n    # Add extra feature engineering features if specified\n    all_features = test.columns\n    if config.ADD_EXTRA_FEATURES:\n        initial_n_features = len(all_features)\n        test, all_features = add_features_polars(test, test=True)\n        print(\n            f\"→ Extra Features: Added {len(all_features) - initial_n_features} extra features, total: {len(all_features)} features\"\n        )\n        assert \"label\" not in all_features, \"Label should not be in features list\"\n\n        columns = test.columns\n        test = scaler.fit_transform(test)\n        test = pl.DataFrame(test, schema=columns)\n\n    return test","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-07-23T06:03:52.386648Z","iopub.execute_input":"2025-07-23T06:03:52.387025Z","iopub.status.idle":"2025-07-23T06:03:52.414431Z","shell.execute_reply.started":"2025-07-23T06:03:52.387001Z","shell.execute_reply":"2025-07-23T06:03:52.413628Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Performance report function\n\nDefine an exhaustive performace report to deeply evaluate the regression performance of a model.","metadata":{}},{"cell_type":"code","source":"def performance_report_regression(*, y_train, y_train_pred, y_val, y_val_pred, n_decimals=4):\n\n    # convert arguments to numpy arrays if they are not already\n    if not isinstance(y_train, np.ndarray):\n        y_train = y_train.to_numpy()\n    if not isinstance(y_train_pred, np.ndarray):\n        y_train_pred = y_train_pred.to_numpy()\n    if not isinstance(y_val, np.ndarray):\n        y_val = y_val.to_numpy()\n    if not isinstance(y_val_pred, np.ndarray):\n        y_val_pred = y_val_pred.to_numpy()\n\n    from sklearn.metrics import (\n        r2_score,\n        mean_absolute_percentage_error,\n        mean_absolute_error,\n    )\n    from scipy.stats import pearsonr\n\n    epsilon = 1e-10\n\n    train_baseline_preds = np.repeat(y_train.mean(), len(y_train)) + np.random.normal(\n        0, 0.00000000001, len(y_train)\n    )\n    train_pearsonr = pearsonr(y_train, y_train_pred)[0]\n    train_r2 = r2_score(y_train, y_train_pred)\n    train_baseline_mae = mean_absolute_error(y_train, train_baseline_preds)\n    train_mae = mean_absolute_error(y_train, y_train_pred)\n    train_mae_imp = (train_mae - train_baseline_mae) / (train_baseline_mae + epsilon)\n    train_baseline_mape = mean_absolute_percentage_error(y_train, train_baseline_preds)\n    train_mape = mean_absolute_percentage_error(y_train, y_train_pred)\n    train_mape_imp = (train_mape - train_baseline_mape) / (train_baseline_mape + epsilon)\n    random_directional_accuracy_train = []\n    for _ in range(1000):\n        random_directional_accuracy_train.append(\n            np.mean(np.sign(y_train) == np.sign(np.random.normal(0, 1, len(y_train))))\n        )\n    random_directional_accuracy_train = np.mean(random_directional_accuracy_train)\n    train_directional_accuracy = np.mean(np.sign(y_train) == np.sign(y_train_pred))\n    train_directional_accuracy_imp = (train_directional_accuracy - random_directional_accuracy_train) / (\n        random_directional_accuracy_train + epsilon\n    )\n\n    val_baseline_preds = np.repeat(y_train.mean(), len(y_val)) + np.random.normal(\n        0, 0.00000000001, len(y_val)\n    )\n    val_pearsonr = pearsonr(y_val, y_val_pred)[0]\n    val_r2 = r2_score(y_val, y_val_pred)\n    val_baseline_mae = mean_absolute_error(y_val, val_baseline_preds)\n    val_mae = mean_absolute_error(y_val, y_val_pred)\n    val_mae_imp = (val_mae - val_baseline_mae) / (val_baseline_mae + epsilon)\n    val_baseline_mape = mean_absolute_percentage_error(y_val, val_baseline_preds)\n    val_mape = mean_absolute_percentage_error(y_val, y_val_pred)\n    val_mape_imp = (val_mape - val_baseline_mape) / (val_baseline_mape + epsilon)\n    random_directional_accuracy_val = []\n    for _ in range(1000):\n        random_directional_accuracy_val.append(\n            np.mean(np.sign(y_val) == np.sign(np.random.normal(0, 1, len(y_val))))\n        )\n    random_directional_accuracy_val = np.mean(random_directional_accuracy_val)\n    val_directional_accuracy = np.mean(np.sign(y_val) == np.sign(y_val_pred))\n    val_directional_accuracy_imp = (val_directional_accuracy - random_directional_accuracy_val) / (\n        random_directional_accuracy_val + epsilon\n    )\n\n    print(\n        \"\\n==========================================================================================================\"\n    )\n    print(\"Performance Report:\")\n    print(\"·····················\")\n    print(\n        f\"Train Pearson r: {train_pearsonr:.{n_decimals}f} →→ Validation Pearson r: {val_pearsonr:.{n_decimals}f}\"\n    )\n    print(f\"Train R2: {train_r2:.{n_decimals}f} →→ Validation R2: {val_r2:.{n_decimals}f}\")\n    print(\"···\")\n    print(\n        f\"Train Baseline MAE: \\t{train_baseline_mae:.{n_decimals}f} →→ Validation Baseline MAE: \\t{val_baseline_mae:.{n_decimals}f}\"\n    )\n    print(f\"Train MAE: \\t\\t{train_mae:.{n_decimals}f} →→ Validation MAE: \\t\\t{val_mae:.{n_decimals}f}\")\n    print(\n        f\"Train MAE improvement: {(train_mae_imp*100):.{n_decimals//2}f}% →→ Validation MAE improvement:\\t{(val_mae_imp*100):.{n_decimals//2}f}%\"\n    )\n    print(\"···\")\n    print(\n        f\"Train Baseline MAPE:  \\t{train_baseline_mape:.{n_decimals}f} →→ Validation Baseline MAPE: \\t{val_baseline_mape:.{n_decimals}f}\"\n    )\n    print(f\"Train MAPE:  \\t\\t{train_mape:.{n_decimals}f} →→ Validation MAPE: \\t\\t{val_mape:.{n_decimals}f}\")\n    print(\n        f\"Train MAPE improvement: {(train_mape_imp*100):.{n_decimals//2}f}% →→ Validation MAPE improvement:\\t{(val_mape_imp*100):.{n_decimals//2}f}%\"\n    )\n    print(\"···\")\n    print(\n        f\"Random Directional Accuracy Train: \\t{random_directional_accuracy_train:.{n_decimals}f} →→ Random Directional Accuracy Validation: \\t{random_directional_accuracy_val:.{n_decimals}f}\"\n    )\n    print(\n        f\"Train Directional Accuracy: \\t\\t{train_directional_accuracy:.{n_decimals}f} →→ Validation Directional Accuracy: \\t\\t{val_directional_accuracy:.{n_decimals}f}\"\n    )\n    print(\n        f\"Train Directional Accuracy improvement: {(train_directional_accuracy_imp*100):.{n_decimals//2}f}% →→ Validation Directional Accuracy improvement:\\t{val_directional_accuracy_imp:.{n_decimals}f}%\"\n    )\n    print(\"·····················\")\n    print(\n        \"==========================================================================================================\"\n    )","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-07-23T05:40:53.060669Z","iopub.execute_input":"2025-07-23T05:40:53.060964Z","iopub.status.idle":"2025-07-23T05:40:53.091576Z","shell.execute_reply.started":"2025-07-23T05:40:53.060941Z","shell.execute_reply":"2025-07-23T05:40:53.090691Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# The XGB Ensemble model\n\nDefine our XGB Ensemble model. The main characteristics of the model can be listed as follows:\n\n* Train *n_models* XGBoost regressors with different initial random seeds.\n* Each regressor uses the validation set to do early stopping in the optimal boosting round.\n* The optimal number of boosting rounds is saved for each model in the ensemble to perform a final training with no validation set.\n* The final prediction is simply the mean of the ensemble.","metadata":{}},{"cell_type":"code","source":"class EarlyStoppingPearsonCallback(xgb.callback.TrainingCallback):\n    def __init__(self, dvalid, y_val, patience=10, maximize=True):\n        self.dvalid = dvalid\n        self.y_val = y_val\n        self.patience = patience\n        self.maximize = maximize\n        self.best_score = None\n        self.best_iteration = 0\n        self.counter = 0\n        self.stopped_iteration = None\n\n    def before_training(self, model):\n        self.start_time = time.time()\n        return model\n\n    def after_iteration(self, model, epoch, evals_log):\n        y_pred = model.predict(self.dvalid, iteration_range=(0, epoch + 1))\n        if len(np.unique(y_pred)) == 1:\n            score = 0.0\n        else:\n            score, _ = pearsonr(self.y_val, y_pred)\n        improved = (\n            (score > self.best_score if self.best_score is not None else True)\n            if self.maximize\n            else (score < self.best_score if self.best_score is not None else True)\n        )\n\n        if improved:\n            self.best_score = score\n            self.best_iteration = epoch\n            self.counter = 0\n        else:\n            self.counter += 1\n            if self.counter >= self.patience:  # and (epoch > self.patience * 2):\n                print(\n                    f\"Early stopping at iteration {epoch + 1}. Best Pearson: {self.best_score:.4f} at iteration {self.best_iteration + 1}\"\n                )\n                self.stopped_iteration = epoch\n                model.set_attr(best_iteration=str(self.best_iteration))\n                model.set_attr(best_score=str(self.best_score))\n                print(f\"Total training time: {time.time() - self.start_time:.1f}s\")\n\n                return True  # signal to stop training\n\n        if (epoch + 1) % 50 == 0:\n            print(\n                f\"  [{epoch + 1}] Pearson Correlation: {score:.4f} (Best: {self.best_score:.4f}). Elapsed: {time.time() - self.start_time:.1f}s\"\n            )\n        return False\n\n    def after_training(self, model):\n        if self.stopped_iteration is not None:\n            model.set_attr(best_iteration=str(self.best_iteration))\n            model.set_attr(best_score=str(self.best_score))\n        return model\n\n\nclass XGBEnsemble:\n    def __init__(\n        self,\n        n_models,\n        base_params,\n        params,\n        model_n_rounds,\n        model_early_stopping_patience,\n        models_save_dir=None,\n    ):\n        self.n_models = n_models\n        self.base_params = base_params\n        self.params = params\n        self.model_n_rounds = model_n_rounds\n        self.model_early_stopping_patience = model_early_stopping_patience\n        self.models_save_dir = models_save_dir\n        self.fit_status = \"not_fitted\"\n\n        if not os.path.exists(self.models_save_dir):\n            os.makedirs(self.models_save_dir, exist_ok=True)\n\n    def fit(self, dtrain, dval, y_train, y_val):\n        self.models = []\n        self.best_n_epochs_per_model = []\n        for seed in range(self.n_models):\n            self.base_params[\"random_state\"] = seed\n            self.base_params[\"seed\"] = seed\n            print(f\"\\n\\nTraining model {seed}|{self.n_models - 1} with params:\\n {self.params}\")\n            model = xgb.train(\n                params={**self.base_params, **self.params},\n                dtrain=dtrain,\n                num_boost_round=self.model_n_rounds,\n                evals=[(dval, \"valid\")],\n                callbacks=[\n                    EarlyStoppingPearsonCallback(\n                        dval,\n                        y_val,\n                        patience=self.model_early_stopping_patience,\n                        maximize=True,\n                    )\n                ],\n            )\n            try:\n                best_iteration = model.best_iteration\n            except:\n                best_iteration = self.model_n_rounds - 1\n\n            self.best_n_epochs_per_model.append(best_iteration)\n            self.models.append(model)\n\n            # preds_train = model.predict(dtrain, iteration_range=(0, best_iteration + 1))\n            # preds_val = model.predict(dval, iteration_range=(0, best_iteration + 1))\n\n            # print(f\"\\n\\nTrain finished. Printing performance report...\")\n            # print(\"Ensemble single model pearsonr in validation set:\", pearsonr(y_val, preds_val)[0])\n\n            if self.models_save_dir:\n                model_path = os.path.join(self.models_save_dir, f\"xgb_model_seed_{seed}.pkl\")\n                with open(model_path, \"wb\") as f:\n                    pickle.dump(model, f)\n                print(f\"Model {seed} saved to {model_path}\")\n        self.fit_status = \"fitted\"\n        return self\n\n    def fit_final(self, dtrain):\n        if self.fit_status != \"fitted\":\n            raise RuntimeError(\"Model must be fitted before calling fit_final.\")\n\n        self.models = []\n        for seed in range(self.n_models):\n            self.base_params[\"random_state\"] = seed\n            self.base_params[\"seed\"] = seed\n            print(\n                f\"\\n Model {seed}: Training final model {seed}|{self.n_models} with {self.best_n_epochs_per_model[seed]} epochs and params:\\n {self.params}\"\n            )\n            model = xgb.train(\n                params={**self.base_params, **self.params},\n                dtrain=dtrain,\n                num_boost_round=self.best_n_epochs_per_model[seed],\n            )\n            print(f\"Model {seed}: Train finished.\")\n            self.models.append(model)\n\n            if self.models_save_dir:\n                model_path = os.path.join(self.models_save_dir, f\"xgb_model_seed_{seed}.pkl\")\n                with open(model_path, \"wb\") as f:\n                    pickle.dump(model, f)\n                print(f\"Model {seed}: saved to disk at {model_path}\")\n        self.fit_status = \"final_fitted\"\n        return self\n\n    def load_models(self):\n        self.models = []\n        for seed in range(self.n_models):\n            model_path = os.path.join(self.models_save_dir, f\"xgb_model_seed_{seed}.pkl\")\n            if os.path.exists(model_path):\n                with open(model_path, \"rb\") as f:\n                    model = pickle.load(f)\n                self.models.append(model)\n            else:\n                raise FileNotFoundError(f\"Model file {model_path} does not exist.\")\n        return self\n\n    def predict(self, dtest):\n        if self.fit_status not in [\"fitted\", \"final_fitted\"]:\n            raise RuntimeError(\"Model must be fitted before calling predict.\")\n        print(f\"→ Model: Performing predictions with fit_status: {self.fit_status}\")\n        preds = []\n        for seed, model in enumerate(self.models):\n            preds.append(model.predict(dtest, iteration_range=(0, self.best_n_epochs_per_model[seed])))\n        return sum(preds) / len(preds)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-07-23T05:40:53.093226Z","iopub.execute_input":"2025-07-23T05:40:53.093657Z","iopub.status.idle":"2025-07-23T05:40:53.123953Z","shell.execute_reply.started":"2025-07-23T05:40:53.093628Z","shell.execute_reply":"2025-07-23T05:40:53.123196Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Define the ConfigGrid","metadata":{}},{"cell_type":"code","source":"config_grid = ConfigGrid(\n    add_extra_features=True,\n    filter_outliers=[None],  # (1, 99)\n    filter_features=[25],  # [75, 50, 25, 5, None]\n    filter_features_model_n_rounds=[150],\n    scaler=[\"robust\"],  # [\"standard\", \"robust\"],\n    n_models=10, #[10, 20],\n    eastop_patience=250,\n    model_n_rounds=2500,\n    base_params={\n        \"tree_method\": \"hist\",\n        \"device\": \"cpu\",\n        \"verbosity\": 0,\n        \"n_jobs\": -1,\n        \"disable_default_eval_metric\": True,\n    },\n    params={\n        \"subsample\": 0.025,\n        \"colsample_bytree\": 0.5,\n        \"colsample_bylevel\": 0.4,\n        \"colsample_bynode\": 0.35,\n        \"gamma\": 2.0,\n        \"learning_rate\": 0.075,\n        \"max_leaves\": 8,\n        \"min_child_weight\": 16,\n        \"reg_alpha\": 60.0,\n        \"reg_lambda\": 50.0,\n    },\n    train_data_path=\"/kaggle/input/drw-crypto-market-prediction/train.parquet\",\n    models_save_dir=False,\n    # models_save_dir=\"/home/rayllon/_inception/Kaggle/DRW_Crypto/xgboost/ensemble_models\",\n)\n\n# class TrainedConfig:\n#     def __init__(self, model, features, scaler):\n#         self.MODEL = model\n#         self.FEATURES = features\n#         self.SCALER = scaler","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-07-23T05:40:53.124688Z","iopub.execute_input":"2025-07-23T05:40:53.124860Z","iopub.status.idle":"2025-07-23T05:40:53.154409Z","shell.execute_reply.started":"2025-07-23T05:40:53.124845Z","shell.execute_reply":"2025-07-23T05:40:53.153706Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Iterates the different configurations derived from the ConfigGrid","metadata":{}},{"cell_type":"code","source":"configs_with_val_pearsonr = {}\n\nbest_config = None\nbest_config_model = None\nbest_config_pearsonr = None\n\nfor config in config_grid.stream_configs():\n    config_id = config.get_config_id()\n    print(f\"\\nRunning config: {config_id}\")\n    \n    X_train, y_train, X_val, y_val, features = load_train_data(config)\n    dtrain = xgb.DMatrix(X_train, label=y_train, feature_names=features)\n\n    # Feature selection\n    if config.FILTER_FEATURES is not None:\n        print(\n            f\"→ Filtering features: Filter those with importance above {config.FILTER_FEATURES}% based on avg. importance of 3 xgb models with {config.FILTER_FEATURES_MODEL_N_ROUNDS} epochs each\"\n        )\n        feature_importances = pd.DataFrame(index=dtrain.feature_names)\n        for seed in range(50, 53, 1):  # ensure seeds are not 0, 1, 2, 3,... as in the ensemble\n            params_random_seed = {\n                \"random_state\": seed,\n                \"seed\": seed,\n            }\n            model_base = xgb.train(\n                params={**config.BASE_PARAMS, **config.PARAMS, **params_random_seed},\n                dtrain=dtrain,\n                num_boost_round=config.FILTER_FEATURES_MODEL_N_ROUNDS,\n            )\n            model_feature_importances = model_base.get_score(importance_type=\"total_gain\")\n            feature_importances = pd.concat(\n                [\n                    feature_importances,\n                    pd.DataFrame(\n                        index=model_feature_importances.keys(),\n                        data=model_feature_importances.values(),\n                        columns=[f\"importance_{seed}\"],\n                    ),\n                ],\n                axis=1,\n                ignore_index=False,\n            )\n        feature_importances = feature_importances.fillna(0.0)\n        feature_importances[\"importance_avg\"] = feature_importances.filter(like=\"importance_\").mean(axis=1)\n        print(\"Threshold percentile:\", np.percentile(feature_importances[\"importance_avg\"], config.FILTER_FEATURES))\n        feature_importances = feature_importances[\n            feature_importances[\"importance_avg\"]\n            > np.percentile(feature_importances[\"importance_avg\"], config.FILTER_FEATURES)\n        ]\n        features = feature_importances.index.tolist()\n        config._FEATURES = features\n        X_train = X_train[:, features]\n        X_val = X_val[:, features]\n        print(f\"→ Filtering features: Keeping {len(feature_importances)} features based on avg. importance\")\n        print(\n            f\"→ Filtering features: Best selected importance: {feature_importances['importance_avg'].max():.4f}\"\n        )\n        print(\n            f\"→ Filtering features: Worst selected importance: {feature_importances['importance_avg'].min():.4f}\"\n        )\n        print(f\"→ Filtering features: Final X_train shape: {X_train.shape}, X_val shape: {X_val.shape}\")\n\n    # cast all float64 columns to float32\n    X_train = X_train.with_columns(\n        [\n            pl.col(col).cast(pl.Float32)\n            for col, dtype in zip(X_train.columns, X_train.dtypes)\n            if dtype == pl.Float64\n        ]\n    )\n    X_val = X_val.with_columns(\n        [\n            pl.col(col).cast(pl.Float32)\n            for col, dtype in zip(X_val.columns, X_val.dtypes)\n            if dtype == pl.Float64\n        ]\n    )\n\n    # Create DMatrix for training and validation sets\n    dtrain = xgb.DMatrix(X_train, label=y_train, feature_names=features)\n    dval = xgb.DMatrix(X_val, label=y_val, feature_names=features)\n\n    # Initialize the ensemble model with the specified configuration\n    model = XGBEnsemble(\n        n_models=config.N_MODELS,\n        base_params=config.BASE_PARAMS,\n        params=config.PARAMS,\n        model_n_rounds=config.MODEL_N_ROUNDS,\n        model_early_stopping_patience=config.EASTOP_PATIENCE,\n        models_save_dir=config.MODELS_SAVE_DIR,\n    )\n\n    model = model.fit(dtrain=dtrain, dval=dval, y_train=y_train, y_val=y_val)\n\n    try:\n        best_iteration = model.best_iteration\n        print(f\"→ Model: Tran finished with early stopping, best iteration: {best_iteration}\")\n    except:\n        best_iteration = config.MODEL_N_ROUNDS - 1\n        print(\"→ Model: Tran finished with NO early stopping, using last iteration as best.\")\n\n    preds_train = model.predict(dtrain)\n    preds_val = model.predict(dval)\n\n    print(f\"\\n\\n→ Model: Ensemble train finished. Printing final XGBEnsemble performance report...\")\n    print(\n        f\"→ Model: Ensemble Train predictions mean | std: {preds_train.mean():.4f} | {preds_train.std():.4f}\"\n    )\n    for percentile in [5, 25, 50, 75, 95]:\n        print(\n            f\"→ Model: Ensemble Train predictions {percentile} percentile: {np.percentile(preds_train, percentile):.4f} | {percentile}%\"\n        )\n    print(f\"→ Model: Ensemble Val predictions mean | std: {preds_val.mean():.4f} | {preds_val.std():.4f}\")\n    for percentile in [5, 25, 50, 75, 95]:\n        print(\n            f\"→ Model: Ensemble Val predictions {percentile} percentile: {np.percentile(preds_val, percentile):.4f} | {percentile}%\"\n        )\n\n    performance_report_regression(\n        y_val=y_val,\n        y_val_pred=preds_val,\n        y_train=y_train,\n        y_train_pred=preds_train,\n        n_decimals=4,\n    )\n\n    config_pearsonr = pearsonr(y_val, preds_val)[0]\n\n    if (best_config_pearsonr is None) or (config_pearsonr < best_config_pearsonr):\n        best_config = config\n        best_config_model = model\n        best_config_pearsonr = config_pearsonr\n        \n    configs_with_val_pearsonr[config_id] = config_pearsonr\n\n    print(\"→ → → Configuration finished!\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-07-23T05:40:53.155229Z","iopub.execute_input":"2025-07-23T05:40:53.155487Z","iopub.status.idle":"2025-07-23T05:44:40.357677Z","shell.execute_reply.started":"2025-07-23T05:40:53.155464Z","shell.execute_reply":"2025-07-23T05:44:40.356892Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Best configuration","metadata":{}},{"cell_type":"code","source":"best_config_id = sorted(configs_with_val_pearsonr, reverse=True)[0]\nprint(\"Best configuration id is:\\n→\", best_config_id)\nprint(\".. with a pearsonr in the validation set equal to\", round(configs_with_val_pearsonr[best_config_id], 5))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T05:44:40.358641Z","iopub.execute_input":"2025-07-23T05:44:40.359089Z","iopub.status.idle":"2025-07-23T05:44:40.364617Z","shell.execute_reply.started":"2025-07-23T05:44:40.359053Z","shell.execute_reply":"2025-07-23T05:44:40.363802Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"config_id = best_config.get_config_id()\nprint(f\"\\nRunning best config: {config_id}\")\n\nX_train, y_train, X_val, y_val, features, scaler = load_train_data(best_config, final=True)\ndtrain = xgb.DMatrix(X_train, label=y_train, feature_names=features)\n\n# Feature selection\nif best_config.FILTER_FEATURES is not None:\n    X_train = X_train[:, best_config._FEATURES]\n    print(f\"→ Filtering features: Final X_train shape: {X_train.shape}\")\n\n# cast all float64 columns to float32\nX_train = X_train.with_columns(\n    [\n        pl.col(col).cast(pl.Float32)\n        for col, dtype in zip(X_train.columns, X_train.dtypes)\n        if dtype == pl.Float64\n    ]\n)\n\n# Create DMatrix for training and validation sets\ndtrain = xgb.DMatrix(X_train, label=y_train, feature_names=best_config._FEATURES)\n\nmodel = model.fit_final(dtrain=dtrain)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T05:44:40.366274Z","iopub.execute_input":"2025-07-23T05:44:40.366501Z","iopub.status.idle":"2025-07-23T05:45:54.503631Z","shell.execute_reply.started":"2025-07-23T05:44:40.366484Z","shell.execute_reply":"2025-07-23T05:45:54.501982Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_test = load_test_data(\n    config=best_config,\n    scaler=scaler\n)\nX_test = X_test[:, best_config._FEATURES]\ndtest = xgb.DMatrix(X_test, feature_names=best_config._FEATURES)\ntest_preds = model.predict(dtest)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T05:45:54.505976Z","iopub.execute_input":"2025-07-23T05:45:54.506350Z","iopub.status.idle":"2025-07-23T05:47:01.733387Z","shell.execute_reply.started":"2025-07-23T05:45:54.506322Z","shell.execute_reply":"2025-07-23T05:47:01.732457Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission = pd.DataFrame(test_preds, columns=[\"prediction\"])\nsubmission.index.name = \"ID\"\nsubmission = submission.reset_index()\nsubmission[\"ID\"] = submission[\"ID\"] + 1\ndisplay(submission)\nsubmission.to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T05:54:03.934818Z","iopub.execute_input":"2025-07-23T05:54:03.935119Z","iopub.status.idle":"2025-07-23T05:54:04.524742Z","shell.execute_reply.started":"2025-07-23T05:54:03.935100Z","shell.execute_reply":"2025-07-23T05:54:04.523427Z"}},"outputs":[],"execution_count":null}]}