{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import KFold, train_test_split\nfrom xgboost import XGBRegressor\nfrom scipy.stats import pearsonr\nimport numpy as np\nimport pandas as pd\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.preprocessing import StandardScaler\nfrom tqdm import tqdm\nimport random\nimport warnings\nwarnings.filterwarnings(\"ignore\", category=RuntimeWarning)\n\n# Deep Learning imports\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, TensorDataset\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(f\"Using device: {device}\")\n\n# =========================\n# Configuration\n# =========================\nclass Config:\n    TRAIN_PATH = \"/kaggle/input/drw-crypto-market-prediction/train.parquet\"\n    TEST_PATH = \"/kaggle/input/drw-crypto-market-prediction/test.parquet\"\n    SUBMISSION_PATH = \"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\"\n\n    FEATURES = [\n        \"X863\", \"X856\", \"X598\", \"X862\", \"X385\", \"X852\", \"X603\", \"X860\", \"X674\",\n        \"X415\", \"X345\", \"X855\", \"X174\", \"X302\", \"X178\", \"X168\", \"X612\", \"bid_qty\",\n        \"ask_qty\", \"buy_qty\", \"sell_qty\", \"volume\", \"X888\", \"X421\", \"X333\",\"X817\", \n        \"X586\",  \"X292\"\n    ]\n    \n    # Features for MLP (subset)\n    MLP_FEATURES = [\n        \"X863\", \"X856\", \"X344\", \"X598\", \"X862\", \"X385\", \"X852\", \"X603\", \"X860\", \"X674\",\n        \"X415\", \"X345\", \"X137\", \"X855\", \"X174\", \"X302\", \"X178\", \"X532\", \"X168\", \"X612\",\n        \"bid_qty\", \"ask_qty\", \"buy_qty\", \"sell_qty\", \"volume\"\n    ]\n\n    LABEL_COLUMN = \"label\"\n    N_FOLDS = 3\n    RANDOM_STATE = 42\n    OUTLIER_FRACTION = 0.001  # 0.1% of records\n\nXGB_PARAMS = {\n    \"tree_method\": \"hist\",\n    \"device\": \"gpu\",\n    \"colsample_bylevel\": 0.4778,\n    \"colsample_bynode\": 0.3628,\n    \"colsample_bytree\": 0.7107,\n    \"gamma\": 1.7095,\n    \"learning_rate\": 0.02213,\n    \"max_depth\": 20,\n    \"max_leaves\": 12,\n    \"min_child_weight\": 16,\n    \"n_estimators\": 1667,\n    \"subsample\": 0.06567,\n    \"reg_alpha\": 39.3524,\n    \"reg_lambda\": 75.4484,\n    \"verbosity\": 0,\n    \"random_state\": Config.RANDOM_STATE,\n    \"n_jobs\": -1\n}\n\nLEARNERS = [\n    {\"name\": \"xgb\", \"Estimator\": XGBRegressor, \"params\": XGB_PARAMS}\n]\n\n# =========================\n# Deep Learning Components\n# =========================\ndef set_seed(seed=42):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\ndef get_activation_function(name):\n    \"\"\"Return the activation function based on the name.\"\"\"\n    if name == None:\n        return None\n    name = name.lower()\n    if name == 'relu':\n        return nn.ReLU()\n    elif name == 'tanh':\n        return nn.Tanh()\n    elif name == 'sigmoid':\n        return nn.Sigmoid()\n    else:\n        raise ValueError(f\"Unsupported activation function: {name}\")\n\nclass MLP(nn.Module):\n    def __init__(self, dropout_rate=0.6, \n                 layers=[128, 64], activation='relu', last_activation=None):\n        super(MLP, self).__init__()\n        \n        self.linears = nn.ModuleList()\n        self.activation = get_activation_function(activation)\n        self.last_activation = get_activation_function(last_activation)\n\n        for i in range(len(layers) - 1):\n            self.linears.append(nn.Linear(layers[i], layers[i + 1]))\n\n        self.dropout = nn.Dropout(dropout_rate)\n\n    def forward(self, x):\n        for k in range(len(self.linears) - 1):\n            x = self.activation(self.linears[k](x))\n            x = self.dropout(x)\n        x = self.linears[-1](x)\n        if self.last_activation is not None:\n            x = self.last_activation(x)\n        return x\n\nclass Checkpointer:\n    def __init__(self, path=\"best_model.pt\"):\n        self.path = path\n        self.best_pearson = -np.inf\n\n    def load(self, model):\n        \"\"\"Load the best model weights.\"\"\"\n        model.load_state_dict(torch.load(self.path))\n        print(f\"Model loaded from {self.path} with best Pearson: {self.best_pearson:.4f}\")\n        return model\n\n    def __call__(self, pearson_coef, model):\n        \"\"\"Call method to save the model if the Pearson coefficient is better than the best one.\"\"\"\n        if pearson_coef > self.best_pearson:\n            self.best_pearson = pearson_coef\n            torch.save(model.state_dict(), self.path)\n            print(f\"✅ New best model saved with Pearson: {pearson_coef:.4f}\")\n\ndef get_dataloaders(X, Y, hparams, device, shuffle=True):\n    \"\"\"Create DataLoader for training and validation datasets.\"\"\"\n    X_tensor = torch.tensor(X, dtype=torch.float32, device=device)\n    if Y is not None:\n        Y_tensor = torch.tensor(Y.values if hasattr(Y, 'values') else Y, \n                                dtype=torch.float32, device=device).unsqueeze(1)\n        dataset = TensorDataset(X_tensor, Y_tensor)\n    else:\n        dataset = TensorDataset(X_tensor)\n    \n    dataloader = DataLoader(dataset, batch_size=hparams[\"batch_size\"], shuffle=shuffle, \n                            generator=torch.Generator().manual_seed(hparams[\"seed\"]))\n    return dataloader\n\n# =========================\n# Feature Engineering\n# =========================\ndef add_features(df):\n    # Original features\n    df['bid_ask_interaction'] = df['bid_qty'] * df['ask_qty']\n    df['bid_buy_interaction'] = df['bid_qty'] * df['buy_qty']\n    df['bid_sell_interaction'] = df['bid_qty'] * df['sell_qty']\n    df['ask_buy_interaction'] = df['ask_qty'] * df['buy_qty']\n    df['ask_sell_interaction'] = df['ask_qty'] * df['sell_qty']\n\n    df['volume_weighted_sell'] = df['sell_qty'] * df['volume']\n    df['buy_sell_ratio'] = df['buy_qty'] / (df['sell_qty'] + 1e-10)\n    df['selling_pressure'] = df['sell_qty'] / (df['volume'] + 1e-10)\n    df['log_volume'] = np.log1p(df['volume'])\n\n    df['effective_spread_proxy'] = np.abs(df['buy_qty'] - df['sell_qty']) / (df['volume'] + 1e-10)\n    df['bid_ask_imbalance'] = (df['bid_qty'] - df['ask_qty']) / (df['bid_qty'] + df['ask_qty'] + 1e-10)\n    df['order_flow_imbalance'] = (df['buy_qty'] - df['sell_qty']) / (df['buy_qty'] + df['sell_qty'] + 1e-10)\n    df['liquidity_ratio'] = (df['bid_qty'] + df['ask_qty']) / (df['volume'] + 1e-10)\n    \n    # === NEW MICROSTRUCTURE FEATURES ===\n    \n    # Price Pressure Indicators\n    df['net_order_flow'] = df['buy_qty'] - df['sell_qty']\n    df['normalized_net_flow'] = df['net_order_flow'] / (df['volume'] + 1e-10)\n    df['buying_pressure'] = df['buy_qty'] / (df['volume'] + 1e-10)\n    df['volume_weighted_buy'] = df['buy_qty'] * df['volume']\n    \n    # Liquidity Depth Measures\n    df['total_depth'] = df['bid_qty'] + df['ask_qty']\n    df['depth_imbalance'] = (df['bid_qty'] - df['ask_qty']) / (df['total_depth'] + 1e-10)\n    df['relative_spread'] = np.abs(df['bid_qty'] - df['ask_qty']) / (df['total_depth'] + 1e-10)\n    df['log_depth'] = np.log1p(df['total_depth'])\n    \n    # Order Flow Toxicity Proxies\n    df['kyle_lambda'] = np.abs(df['net_order_flow']) / (df['volume'] + 1e-10)\n    df['flow_toxicity'] = np.abs(df['order_flow_imbalance']) * df['volume']\n    df['aggressive_flow_ratio'] = (df['buy_qty'] + df['sell_qty']) / (df['total_depth'] + 1e-10)\n    \n    # Market Activity Indicators\n    df['volume_depth_ratio'] = df['volume'] / (df['total_depth'] + 1e-10)\n    df['activity_intensity'] = (df['buy_qty'] + df['sell_qty']) / (df['volume'] + 1e-10)\n    df['log_buy_qty'] = np.log1p(df['buy_qty'])\n    df['log_sell_qty'] = np.log1p(df['sell_qty'])\n    df['log_bid_qty'] = np.log1p(df['bid_qty'])\n    df['log_ask_qty'] = np.log1p(df['ask_qty'])\n    \n    # Microstructure Volatility Proxies\n    df['realized_spread_proxy'] = 2 * np.abs(df['net_order_flow']) / (df['volume'] + 1e-10)\n    df['price_impact_proxy'] = df['net_order_flow'] / (df['total_depth'] + 1e-10)\n    df['quote_volatility_proxy'] = np.abs(df['depth_imbalance'])\n    \n    # Complex Interaction Terms\n    df['flow_depth_interaction'] = df['net_order_flow'] * df['total_depth']\n    df['imbalance_volume_interaction'] = df['order_flow_imbalance'] * df['volume']\n    df['depth_volume_interaction'] = df['total_depth'] * df['volume']\n    df['buy_sell_spread'] = np.abs(df['buy_qty'] - df['sell_qty'])\n    df['bid_ask_spread'] = np.abs(df['bid_qty'] - df['ask_qty'])\n    \n    # Information Asymmetry Measures\n    df['trade_informativeness'] = df['net_order_flow'] / (df['bid_qty'] + df['ask_qty'] + 1e-10)\n    df['execution_shortfall_proxy'] = df['buy_sell_spread'] / (df['volume'] + 1e-10)\n    df['adverse_selection_proxy'] = df['net_order_flow'] / (df['total_depth'] + 1e-10) * df['volume']\n    \n    # Market Efficiency Indicators\n    df['fill_probability'] = df['volume'] / (df['buy_qty'] + df['sell_qty'] + 1e-10)\n    df['execution_rate'] = (df['buy_qty'] + df['sell_qty']) / (df['total_depth'] + 1e-10)\n    df['market_efficiency'] = df['volume'] / (df['bid_ask_spread'] + 1e-10)\n    \n    # Non-linear Transformations\n    df['sqrt_volume'] = np.sqrt(df['volume'])\n    df['sqrt_depth'] = np.sqrt(df['total_depth'])\n    df['volume_squared'] = df['volume'] ** 2\n    df['imbalance_squared'] = df['order_flow_imbalance'] ** 2\n    \n    # Relative Measures\n    df['bid_ratio'] = df['bid_qty'] / (df['total_depth'] + 1e-10)\n    df['ask_ratio'] = df['ask_qty'] / (df['total_depth'] + 1e-10)\n    df['buy_ratio'] = df['buy_qty'] / (df['buy_qty'] + df['sell_qty'] + 1e-10)\n    df['sell_ratio'] = df['sell_qty'] / (df['buy_qty'] + df['sell_qty'] + 1e-10)\n    \n    # Market Stress Indicators\n    df['liquidity_consumption'] = (df['buy_qty'] + df['sell_qty']) / (df['total_depth'] + 1e-10)\n    df['market_stress'] = df['volume'] / (df['total_depth'] + 1e-10) * np.abs(df['order_flow_imbalance'])\n    df['depth_depletion'] = df['volume'] / (df['bid_qty'] + df['ask_qty'] + 1e-10)\n    \n    # Directional Indicators\n    df['net_buying_ratio'] = df['net_order_flow'] / (df['volume'] + 1e-10)\n    df['directional_volume'] = df['net_order_flow'] * np.log1p(df['volume'])\n    df['signed_volume'] = np.sign(df['net_order_flow']) * df['volume']\n    \n    # Replace infinities and NaNs\n    df = df.replace([np.inf, -np.inf], 0).fillna(0)\n    \n    return df\n\ndef create_time_decay_weights(n: int, decay: float = 0.9) -> np.ndarray:\n    positions = np.arange(n)\n    normalized = positions / (n - 1)\n    weights = decay ** (1.0 - normalized)\n    return weights * n / weights.sum()\n\ndef detect_outliers_and_adjust_weights(X, y, sample_weights, outlier_fraction=0.001):\n    \"\"\"\n    Detect outliers based on prediction residuals and adjust their weights.\n    Only adjusts weights for the top outlier_fraction of records.\n    \"\"\"\n    # Train a simple model to get residuals\n    rf = RandomForestRegressor(n_estimators=50, max_depth=10, random_state=42, n_jobs=-1)\n    rf.fit(X, y, sample_weight=sample_weights)\n    \n    # Calculate residuals\n    predictions = rf.predict(X)\n    residuals = np.abs(y - predictions)\n    \n    # Find threshold for top outlier_fraction\n    n_outliers = max(1, int(len(residuals) * outlier_fraction))\n    threshold = np.sort(residuals)[-n_outliers]\n    \n    # Create outlier mask\n    outlier_mask = residuals >= threshold\n    \n    # Adjust weights for outliers\n    adjusted_weights = sample_weights.copy()\n    \n    if outlier_mask.any():\n        # Calculate weight reduction factor based on residual magnitude\n        outlier_residuals = residuals[outlier_mask]\n        \n        # Normalize residuals to [0, 1] range for outliers\n        min_outlier_res = outlier_residuals.min()\n        max_outlier_res = outlier_residuals.max()\n        \n        if max_outlier_res > min_outlier_res:\n            normalized_residuals = (outlier_residuals - min_outlier_res) / (max_outlier_res - min_outlier_res)\n        else:\n            normalized_residuals = np.ones_like(outlier_residuals)\n        \n        # Reduce weights proportionally (from 0.2 to 0.8 of original weight)\n        # The most extreme outliers get 0.2x weight, least extreme get 0.8x weight\n        weight_factors = 0.8 - 0.6 * normalized_residuals\n        \n        # Apply weight adjustments\n        adjusted_weights[outlier_mask] *= weight_factors\n        \n        print(f\"    Adjusted weights for {n_outliers} outliers ({outlier_fraction*100:.1f}% of data)\")\n    \n    return adjusted_weights\n\ndef load_data():\n    # Load data with all features available\n    all_features = list(set(Config.FEATURES + Config.MLP_FEATURES))\n    train_df = pd.read_parquet(Config.TRAIN_PATH, columns=all_features + [Config.LABEL_COLUMN])\n    test_df = pd.read_parquet(Config.TEST_PATH, columns=all_features)\n    submission_df = pd.read_csv(Config.SUBMISSION_PATH)\n    print(f\"Loaded data - Train: {train_df.shape}, Test: {test_df.shape}, Submission: {submission_df.shape}\")\n\n    # Add features\n    train_df = add_features(train_df)\n    test_df = add_features(test_df)\n\n    # Update Config.FEATURES with new features\n    Config.FEATURES += [\n        \"log_volume\", 'bid_ask_interaction', 'bid_buy_interaction', 'bid_sell_interaction', \n        'ask_buy_interaction', 'ask_sell_interaction', 'net_order_flow', 'normalized_net_flow',\n        'buying_pressure', 'volume_weighted_buy', 'total_depth', 'depth_imbalance',\n        'relative_spread', 'log_depth', 'kyle_lambda', 'flow_toxicity', 'aggressive_flow_ratio',\n        'volume_depth_ratio', 'activity_intensity', 'log_buy_qty', 'log_sell_qty',\n        'log_bid_qty', 'log_ask_qty', 'realized_spread_proxy', 'price_impact_proxy',\n        'quote_volatility_proxy', 'flow_depth_interaction', 'imbalance_volume_interaction',\n        'depth_volume_interaction', 'buy_sell_spread', 'bid_ask_spread', 'trade_informativeness',\n        'execution_shortfall_proxy', 'adverse_selection_proxy', 'fill_probability',\n        'execution_rate', 'market_efficiency', 'sqrt_volume', 'sqrt_depth', 'volume_squared',\n        'imbalance_squared', 'bid_ratio', 'ask_ratio', 'buy_ratio', 'sell_ratio',\n        'liquidity_consumption', 'market_stress', 'depth_depletion', 'net_buying_ratio',\n        'directional_volume', 'signed_volume'\n    ]\n\n    return train_df.reset_index(drop=True), test_df.reset_index(drop=True), submission_df\n\ndef get_model_slices(n_samples: int):\n    # Original 5 slices\n    base_slices = [\n        {\"name\": \"full_data\", \"cutoff\": 0, \"is_oldest\": False, \"outlier_adjusted\": False},\n        {\"name\": \"last_90pct\", \"cutoff\": int(0.10 * n_samples), \"is_oldest\": False, \"outlier_adjusted\": False},\n        {\"name\": \"last_85pct\", \"cutoff\": int(0.15 * n_samples), \"is_oldest\": False, \"outlier_adjusted\": False},\n        {\"name\": \"last_80pct\", \"cutoff\": int(0.20 * n_samples), \"is_oldest\": False, \"outlier_adjusted\": False},\n        {\"name\": \"oldest_25pct\", \"cutoff\": int(0.25 * n_samples), \"is_oldest\": True, \"outlier_adjusted\": False},\n    ]\n    \n    # Duplicate slices with outlier adjustment\n    outlier_adjusted_slices = []\n    for slice_info in base_slices:\n        adjusted_slice = slice_info.copy()\n        adjusted_slice[\"name\"] = f\"{slice_info['name']}_outlier_adj\"\n        adjusted_slice[\"outlier_adjusted\"] = True\n        outlier_adjusted_slices.append(adjusted_slice)\n    \n    return base_slices + outlier_adjusted_slices\n\n# =========================\n# XGBoost Training and Evaluation\n# =========================\ndef train_and_evaluate_xgboost(train_df, test_df):\n    n_samples = len(train_df)\n    model_slices = get_model_slices(n_samples)\n\n    oof_preds = {\n        learner[\"name\"]: {s[\"name\"]: np.zeros(n_samples) for s in model_slices}\n        for learner in LEARNERS\n    }\n    test_preds = {\n        learner[\"name\"]: {s[\"name\"]: np.zeros(len(test_df)) for s in model_slices}\n        for learner in LEARNERS\n    }\n\n    full_weights = create_time_decay_weights(n_samples)\n    kf = KFold(n_splits=Config.N_FOLDS, shuffle=False)\n\n    for fold, (train_idx, valid_idx) in enumerate(kf.split(train_df), start=1):\n        print(f\"\\n--- Fold {fold}/{Config.N_FOLDS} ---\")\n        X_valid = train_df.iloc[valid_idx][Config.FEATURES]\n        y_valid = train_df.iloc[valid_idx][Config.LABEL_COLUMN]\n\n        for s in model_slices:\n            cutoff = s[\"cutoff\"]\n            slice_name = s[\"name\"]\n            is_oldest = s[\"is_oldest\"]\n            outlier_adjusted = s[\"outlier_adjusted\"]\n            \n            if is_oldest:\n                # For oldest data, take only the first 25% of samples\n                subset = train_df.iloc[:cutoff].reset_index(drop=True)\n                rel_idx = train_idx[train_idx < cutoff]\n                # No time decay for oldest data - use uniform weights\n                sw = np.ones(len(rel_idx))\n            else:\n                # For recent data slices, take from cutoff to end\n                subset = train_df.iloc[cutoff:].reset_index(drop=True)\n                rel_idx = train_idx[train_idx >= cutoff] - cutoff\n                sw = create_time_decay_weights(len(subset))[rel_idx] if cutoff > 0 else full_weights[train_idx]\n\n            X_train = subset.iloc[rel_idx][Config.FEATURES]\n            y_train = subset.iloc[rel_idx][Config.LABEL_COLUMN]\n            \n            # Apply outlier detection and weight adjustment if needed\n            if outlier_adjusted and len(X_train) > 100:  # Only apply if we have enough samples\n                sw = detect_outliers_and_adjust_weights(\n                    X_train.values, \n                    y_train.values, \n                    sw, \n                    outlier_fraction=Config.OUTLIER_FRACTION\n                )\n\n            print(f\"  Training slice: {slice_name}, samples: {len(X_train)}\")\n\n            for learner in LEARNERS:\n                model = learner[\"Estimator\"](**learner[\"params\"])\n                model.fit(X_train, y_train, sample_weight=sw, eval_set=[(X_valid, y_valid)], verbose=False)\n\n                if is_oldest:\n                    # For oldest data slice, predict on all validation indices\n                    oof_preds[learner[\"name\"]][slice_name][valid_idx] = model.predict(\n                        train_df.iloc[valid_idx][Config.FEATURES]\n                    )\n                else:\n                    # For recent data slices, handle cutoff logic\n                    mask = valid_idx >= cutoff\n                    if mask.any():\n                        idxs = valid_idx[mask]\n                        oof_preds[learner[\"name\"]][slice_name][idxs] = model.predict(\n                            train_df.iloc[idxs][Config.FEATURES]\n                        )\n                    if cutoff > 0 and (~mask).any():\n                        base_slice_name = slice_name.replace(\"_outlier_adj\", \"\")\n                        if base_slice_name == slice_name:\n                            fallback_slice = \"full_data\"\n                        else:\n                            fallback_slice = \"full_data_outlier_adj\"\n                        oof_preds[learner[\"name\"]][slice_name][valid_idx[~mask]] = oof_preds[learner[\"name\"]][fallback_slice][\n                            valid_idx[~mask]\n                        ]\n\n                test_preds[learner[\"name\"]][slice_name] += model.predict(test_df[Config.FEATURES])\n\n    # Normalize test predictions\n    for learner_name in test_preds:\n        for slice_name in test_preds[learner_name]:\n            test_preds[learner_name][slice_name] /= (Config.N_FOLDS - 1)\n\n    return oof_preds, test_preds, model_slices\n\n# =========================\n# MLP Training\n# =========================\ndef train_mlp(train_df, test_df):\n    print(\"\\n=== Training MLP Model ===\")\n    \n    # Hyperparameters\n    hparams = {\n        \"seed\": 42,\n        \"num_epochs\": 10,\n        \"batch_size\": 1024 * 8 * 4,\n        \"learning_rate\": 0.001,\n        \"weight_decay\": 1e-3,\n        \"dropout_rate\": 0.6,\n        \"layers\": [len(Config.MLP_FEATURES), 256, 64, 1],\n        \"hidden_activation\": None,\n        \"activation\": \"relu\",\n        \"delta\": 5,\n        \"noise_factor\": 0.005\n    }\n    \n    set_seed(hparams[\"seed\"])\n    \n    # Prepare data for MLP\n    X_train_full = train_df[Config.MLP_FEATURES].values\n    y_train_full = train_df[Config.LABEL_COLUMN].values\n    \n    # Split for validation\n    X_train, X_val, y_train, y_val = train_test_split(\n        X_train_full, y_train_full, test_size=0.2, shuffle=False, random_state=42\n    )\n    \n    # Scale data\n    scaler = StandardScaler()\n    X_train = scaler.fit_transform(X_train)\n    X_val = scaler.transform(X_val)\n    X_test = scaler.transform(test_df[Config.MLP_FEATURES].values)\n    \n    # Create dataloaders\n    train_loader = get_dataloaders(X_train, y_train, hparams, device, shuffle=True)\n    val_loader = get_dataloaders(X_val, y_val, hparams, device, shuffle=False)\n    test_loader = get_dataloaders(X_test, None, hparams, device, shuffle=False)\n    \n    # Initialize model\n    model = MLP(\n        layers=hparams[\"layers\"],\n        dropout_rate=hparams[\"dropout_rate\"],\n        activation=hparams[\"activation\"],\n        last_activation=hparams[\"hidden_activation\"],\n    ).to(device)\n    \n    criterion = nn.HuberLoss(delta=hparams[\"delta\"], reduction='sum')\n    optimizer = optim.Adam(model.parameters(), lr=hparams[\"learning_rate\"], \n                          weight_decay=hparams[\"weight_decay\"])\n    \n    checkpointer = Checkpointer(path=\"best_mlp_model.pt\")\n    \n    # Training loop\n    num_epochs = hparams[\"num_epochs\"]\n    for epoch in range(num_epochs):\n        model.train()\n        running_loss = 0.0\n\n        for inputs, targets in tqdm(train_loader, desc=f\"Epoch {epoch+1}/{num_epochs}\"):\n            inputs, targets = inputs.to(device), targets.to(device)\n            \n            # Add noise for robustness\n            inputs = inputs + torch.randn_like(inputs) * hparams[\"noise_factor\"]\n            \n            optimizer.zero_grad()\n            outputs = model(inputs)\n            loss = criterion(outputs, targets)\n            \n            loss.backward()\n            optimizer.step()\n            \n            running_loss += loss.item() * inputs.size(0)\n            \n        running_loss = running_loss / len(train_loader.dataset)\n        print(f\"Training Loss: {running_loss:.4f}\")\n\n        # Validation phase\n        model.eval()\n        val_loss = 0.0\n        preds = []\n        trues = []\n        with torch.no_grad():\n            for inputs, targets in tqdm(val_loader, desc=\"Validation\"):\n                inputs, targets = inputs.to(device), targets.to(device)\n                outputs = model(inputs)\n                loss = criterion(outputs, targets)\n                val_loss += loss.item() * inputs.size(0)\n                preds.append(outputs.cpu().numpy())\n                trues.append(targets.cpu().numpy())\n\n        val_loss /= len(val_loader.dataset)\n        preds = np.concatenate(preds).flatten()\n        trues = np.concatenate(trues).flatten()\n        pearson_coef = pearsonr(preds, trues)[0]\n        print(f\"Validation Pearson Coef: {pearson_coef:.4f} | Loss: {val_loss:.4f}\")\n\n        checkpointer(pearson_coef, model)\n    \n    # Load best model and make predictions\n    model = checkpointer.load(model)\n    model.eval()\n    predictions = []\n    with torch.no_grad():\n        for inputs in tqdm(test_loader, desc=\"Predicting\"):\n            inputs = inputs[0].to(device)\n            outputs = model(inputs)\n            predictions.append(outputs.cpu().numpy())\n\n    predictions = np.concatenate(predictions).flatten()\n    \n    return predictions\n\n# =========================\n# Ensemble & Submission Functions\n# =========================\ndef create_xgboost_submission(train_df, oof_preds, test_preds, submission_df):\n    learner_name = 'xgb'\n    \n    # Weights for 10 slices\n    weights = np.array([\n        1.0,   # full_data\n        1.0,   # last_90pct\n        1.0,   # last_85pct\n        1.0,   # last_80pct\n        0.25,  # oldest_25pct\n        0.9,   # full_data_outlier_adj\n        0.9,   # last_90pct_outlier_adj\n        0.9,   # last_85pct_outlier_adj\n        0.9,   # last_80pct_outlier_adj\n        0.2    # oldest_25pct_outlier_adj\n    ])\n    \n    # Normalize weights\n    weights = weights / weights.sum()\n\n    oof_weighted = pd.DataFrame(oof_preds[learner_name]).values @ weights\n    test_weighted = pd.DataFrame(test_preds[learner_name]).values @ weights\n    score_weighted = pearsonr(train_df[Config.LABEL_COLUMN], oof_weighted)[0]\n    print(f\"\\n{learner_name.upper()} Weighted Ensemble Pearson: {score_weighted:.4f}\")\n\n    # Print individual slice scores and weights for analysis\n    print(\"\\nIndividual slice OOF scores and weights:\")\n    slice_names = list(oof_preds[learner_name].keys())\n    for i, slice_name in enumerate(slice_names):\n        score = pearsonr(train_df[Config.LABEL_COLUMN], oof_preds[learner_name][slice_name])[0]\n        print(f\"  {slice_name}: {score:.4f} (weight: {weights[i]:.3f})\")\n\n    # Save XGBoost submission\n    xgb_submission = submission_df.copy()\n    xgb_submission[\"prediction\"] = test_weighted\n    xgb_submission.to_csv(\"submission_xgboost.csv\", index=False)\n    print(\"\\nSaved: submission_xgboost.csv\")\n    \n    return test_weighted\n\ndef create_ensemble_submission(xgb_predictions, mlp_predictions, submission_df, \n                             xgb_weight=0.9, mlp_weight=0.1):\n    # Ensemble predictions\n    ensemble_predictions = (xgb_weight * xgb_predictions + \n                          mlp_weight * mlp_predictions)\n    \n    # Save ensemble submission\n    ensemble_submission = submission_df.copy()\n    ensemble_submission[\"prediction\"] = ensemble_predictions\n    ensemble_submission.to_csv(\"submission_ensemble.csv\", index=False)\n    print(f\"\\nSaved: submission_ensemble.csv (XGBoost: {xgb_weight*100}%, MLP: {mlp_weight*100}%)\")\n    \n    return ensemble_predictions\n\n# =========================\n# Main Execution\n# =========================\nif __name__ == \"__main__\":\n    # Load data\n    train_df, test_df, submission_df = load_data()\n    \n    # Train XGBoost models\n    print(\"=== Training XGBoost Models ===\")\n    oof_preds, test_preds, model_slices = train_and_evaluate_xgboost(train_df, test_df)\n    \n    # Create XGBoost submission\n    xgb_predictions = create_xgboost_submission(train_df, oof_preds, test_preds, submission_df)\n    \n    # Train MLP model\n    mlp_predictions = train_mlp(train_df, test_df)\n    \n    # Save MLP submission\n    mlp_submission = submission_df.copy()\n    mlp_submission[\"prediction\"] = mlp_predictions\n    mlp_submission.to_csv(\"submission_mlp.csv\", index=False)\n    print(\"\\nSaved: submission_mlp.csv\")\n    \n    # Create ensemble submission\n    ensemble_predictions = create_ensemble_submission(\n        xgb_predictions, mlp_predictions, submission_df,\n        xgb_weight=0.78, mlp_weight=0.22\n    )\n    \n    # Print summary\n    print(\"\\n=== Summary ===\")\n    print(\"Created 3 submission files:\")\n    print(\"1. submission_xgboost.csv - XGBoost only\")\n    print(\"2. submission_mlp.csv - MLP only\")\n    print(\"3. submission_ensemble.csv - 90% XGBoost + 10% MLP\")\n    \n    # Show sample predictions\n    print(\"\\nSample predictions (first 10 rows):\")\n    comparison_df = pd.DataFrame({\n        'ID': submission_df['ID'][:10],\n        'XGBoost': xgb_predictions[:10],\n        'MLP': mlp_predictions[:10],\n        'Ensemble': ensemble_predictions[:10]\n    })\n    print(comparison_df)","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}