{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-07-07T16:16:08.002755Z","iopub.execute_input":"2025-07-07T16:16:08.002975Z","iopub.status.idle":"2025-07-07T16:16:08.350247Z","shell.execute_reply.started":"2025-07-07T16:16:08.002947Z","shell.execute_reply":"2025-07-07T16:16:08.349420Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Complete Enhanced Crypto Market Prediction - MLP with Target Transformation\n# Full working code with all components included\n\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, TensorDataset\nimport gc\nfrom sklearn.model_selection import KFold\nfrom sklearn.preprocessing import StandardScaler, RobustScaler, QuantileTransformer, PowerTransformer\nfrom sklearn.metrics import mean_squared_error, mean_absolute_error\nfrom scipy.stats import pearsonr, boxcox\nfrom scipy.special import inv_boxcox\nimport warnings\nimport time\nimport os\nfrom collections import defaultdict\nimport copy\nfrom typing import Tuple, Dict, List\nimport joblib\n\nwarnings.filterwarnings('ignore')\n\n# Set random seeds\nnp.random.seed(42)\ntorch.manual_seed(42)\nif torch.cuda.is_available():\n    torch.cuda.manual_seed(42)\n\nprint(\"=\"*80)\nprint(\"ENHANCED CRYPTO MARKET PREDICTION - COMPLETE SOLUTION\")\nprint(\"=\"*80)\n\nclass Config:\n    \"\"\"Configuration for enhanced MLP framework\"\"\"\n    TRAIN_PATH = \"/kaggle/input/drw-crypto-market-prediction/train.parquet\"\n    TEST_PATH = \"/kaggle/input/drw-crypto-market-prediction/test.parquet\"\n    SUBMISSION_PATH = \"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\"\n    \n    # Core features\n    CORE_FEATURES = [\"bid_qty\", \"ask_qty\", \"buy_qty\", \"sell_qty\", \"volume\"]\n    LABEL_COLUMN = \"label\"\n    \n    # Enhanced MLP parameters\n    HIDDEN_DIMS = [2048, 1024, 512, 256, 128, 64]\n    DROPOUT_RATES = [0.5, 0.4, 0.3, 0.3, 0.2, 0.2]\n    NOISE_STD = 0.05\n    USE_BATCH_NORM = True\n    USE_LAYER_NORM = False\n    ACTIVATION = 'gelu'\n    \n    # Training parameters\n    BATCH_SIZE = 2048\n    LEARNING_RATE = 0.001\n    WEIGHT_DECAY = 1e-4\n    MAX_EPOCHS = 50  # Reduced for faster execution\n    EARLY_STOPPING_PATIENCE = 10\n    GRADIENT_CLIP = 1.0\n    \n    # Target transformation\n    USE_TARGET_TRANSFORM = True\n    TARGET_TRANSFORM_METHOD = 'quantile'\n    \n    # Ensemble\n    USE_ENSEMBLE = True\n    N_ENSEMBLE_MODELS = 3\n    \n    # Cross-validation\n    N_FOLDS = 3  # Reduced for faster execution\n    \n    # Data cleaning\n    OUTLIER_STD_THRESHOLD = 4.0\n    INF_REPLACEMENT = 'median'\n\nconfig = Config()\n\nclass DataTypeOptimizer:\n    \"\"\"Optimize data types for memory efficiency\"\"\"\n    \n    @staticmethod\n    def optimize_dtypes(df, verbose=True):\n        \"\"\"Convert columns to smallest appropriate dtype\"\"\"\n        start_mem = df.memory_usage().sum() / 1024**2\n        \n        for col in df.columns:\n            col_type = df[col].dtype\n            \n            if col_type != 'object':\n                c_min = df[col].min()\n                c_max = df[col].max()\n                \n                if str(col_type)[:3] == 'int':\n                    if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                        df[col] = df[col].astype(np.int8)\n                    elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                        df[col] = df[col].astype(np.int16)\n                    elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                        df[col] = df[col].astype(np.int32)\n                else:\n                    if c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                        df[col] = df[col].astype(np.float32)\n        \n        end_mem = df.memory_usage().sum() / 1024**2\n        \n        if verbose:\n            print(f'    Memory usage decreased from {start_mem:.2f} MB to {end_mem:.2f} MB ({100 * (start_mem - end_mem) / start_mem:.1f}% reduction)')\n        \n        return df\n\nclass DataCleaner:\n    \"\"\"Clean and preprocess data\"\"\"\n    \n    def __init__(self, config):\n        self.config = config\n        self.columns_to_remove = set()\n        self.column_stats = {}\n        \n    def identify_zero_variance_columns(self, df):\n        \"\"\"Identify columns with zero variance\"\"\"\n        zero_var_cols = []\n        \n        for col in df.columns:\n            if col != self.config.LABEL_COLUMN:\n                unique_vals = df[col].nunique()\n                if unique_vals <= 1:\n                    zero_var_cols.append(col)\n        \n        return zero_var_cols\n    \n    def handle_infinities(self, df):\n        \"\"\"Handle infinite values in dataframe\"\"\"\n        numeric_cols = df.select_dtypes(include=[np.number]).columns\n        \n        for col in numeric_cols:\n            if col != self.config.LABEL_COLUMN:\n                inf_mask = np.isinf(df[col])\n                n_inf = inf_mask.sum()\n                \n                if n_inf > 0:\n                    if self.config.INF_REPLACEMENT == 'median':\n                        median_val = df[col][~inf_mask].median()\n                        df.loc[inf_mask, col] = median_val\n                    elif self.config.INF_REPLACEMENT == 'clip':\n                        finite_vals = df[col][~inf_mask]\n                        min_val = finite_vals.min()\n                        max_val = finite_vals.max()\n                        df.loc[df[col] == np.inf, col] = max_val\n                        df.loc[df[col] == -np.inf, col] = min_val\n        \n        return df\n    \n    def handle_outliers(self, df):\n        \"\"\"Handle outliers using robust statistics\"\"\"\n        numeric_cols = df.select_dtypes(include=[np.number]).columns\n        \n        for col in numeric_cols:\n            if col != self.config.LABEL_COLUMN:\n                Q1 = df[col].quantile(0.25)\n                Q3 = df[col].quantile(0.75)\n                IQR = Q3 - Q1\n                \n                lower_bound = Q1 - self.config.OUTLIER_STD_THRESHOLD * IQR\n                upper_bound = Q3 + self.config.OUTLIER_STD_THRESHOLD * IQR\n                \n                df[col] = df[col].clip(lower=lower_bound, upper=upper_bound)\n                \n                self.column_stats[col] = {\n                    'lower_bound': lower_bound,\n                    'upper_bound': upper_bound,\n                    'median': df[col].median(),\n                    'Q1': Q1,\n                    'Q3': Q3\n                }\n        \n        return df\n    \n    def clean_data(self, train_df, test_df):\n        \"\"\"Complete data cleaning pipeline\"\"\"\n        print(\"\\nCleaning data...\")\n        \n        # 1. Identify zero variance columns\n        print(\"  Identifying zero variance columns...\")\n        train_zero_var = set(self.identify_zero_variance_columns(train_df))\n        test_zero_var = set(self.identify_zero_variance_columns(test_df))\n        zero_var_cols = train_zero_var.union(test_zero_var)\n        \n        if zero_var_cols:\n            print(f\"    Removing {len(zero_var_cols)} zero variance columns\")\n            train_df = train_df.drop(columns=list(zero_var_cols), errors='ignore')\n            test_df = test_df.drop(columns=list(zero_var_cols), errors='ignore')\n            self.columns_to_remove.update(zero_var_cols)\n        \n        # 2. Handle infinities\n        print(\"  Handling infinite values...\")\n        train_df = self.handle_infinities(train_df)\n        test_df = self.handle_infinities(test_df)\n        \n        # 3. Handle outliers\n        print(\"  Handling outliers...\")\n        train_df = self.handle_outliers(train_df)\n        \n        # Apply same bounds to test data\n        for col in test_df.select_dtypes(include=[np.number]).columns:\n            if col in self.column_stats:\n                stats = self.column_stats[col]\n                test_df[col] = test_df[col].clip(\n                    lower=stats['lower_bound'],\n                    upper=stats['upper_bound']\n                )\n        \n        # 4. Handle missing values\n        print(\"  Handling missing values...\")\n        train_df = train_df.fillna(0)\n        test_df = test_df.fillna(0)\n        \n        return train_df, test_df\n\nclass TargetTransformer:\n    \"\"\"Handle target variable transformations\"\"\"\n    \n    def __init__(self, method='quantile'):\n        self.method = method\n        self.transformer = None\n        self.lambda_param = None\n        self.shift = None\n        \n    def fit_transform(self, y):\n        \"\"\"Fit and transform target variable\"\"\"\n        y = y.flatten()\n        \n        if self.method == 'boxcox':\n            # Ensure all values are positive for Box-Cox\n            self.shift = np.abs(y.min()) + 1e-6 if y.min() <= 0 else 0\n            y_shifted = y + self.shift\n            y_transformed, self.lambda_param = boxcox(y_shifted)\n            return y_transformed\n            \n        elif self.method == 'yeo-johnson':\n            self.transformer = PowerTransformer(method='yeo-johnson')\n            return self.transformer.fit_transform(y.reshape(-1, 1)).flatten()\n            \n        elif self.method == 'quantile':\n            self.transformer = QuantileTransformer(\n                n_quantiles=min(len(y), 1000),\n                output_distribution='normal',\n                subsample=100000\n            )\n            return self.transformer.fit_transform(y.reshape(-1, 1)).flatten()\n            \n        elif self.method == 'log':\n            self.shift = np.abs(y.min()) + 1 if y.min() <= 0 else 0\n            return np.log1p(y + self.shift)\n            \n        else:\n            return y\n    \n    def transform(self, y):\n        \"\"\"Transform target variable using fitted parameters\"\"\"\n        y = y.flatten()\n        \n        if self.method == 'boxcox':\n            y_shifted = y + self.shift\n            return boxcox(y_shifted, lmbda=self.lambda_param)\n            \n        elif self.method in ['yeo-johnson', 'quantile']:\n            return self.transformer.transform(y.reshape(-1, 1)).flatten()\n            \n        elif self.method == 'log':\n            return np.log1p(y + self.shift)\n            \n        else:\n            return y\n    \n    def inverse_transform(self, y_transformed):\n        \"\"\"Inverse transform predictions\"\"\"\n        y_transformed = y_transformed.flatten()\n        \n        if self.method == 'boxcox':\n            y_original = inv_boxcox(y_transformed, self.lambda_param)\n            return y_original - self.shift\n            \n        elif self.method in ['yeo-johnson', 'quantile']:\n            return self.transformer.inverse_transform(\n                y_transformed.reshape(-1, 1)\n            ).flatten()\n            \n        elif self.method == 'log':\n            return np.expm1(y_transformed) - self.shift\n            \n        else:\n            return y_transformed\n\nclass AdvancedMLPWithNoise(nn.Module):\n    \"\"\"Enhanced MLP with advanced features\"\"\"\n    \n    def __init__(self, input_dim, hidden_dims, dropout_rates=None, \n                 noise_std=0.05, use_batch_norm=True, use_layer_norm=False,\n                 activation='gelu'):\n        super(AdvancedMLPWithNoise, self).__init__()\n        \n        self.noise_std = noise_std\n        self.use_batch_norm = use_batch_norm\n        self.use_layer_norm = use_layer_norm\n        \n        if dropout_rates is None:\n            dropout_rates = [0.5] * len(hidden_dims)\n        \n        # Select activation function\n        if activation == 'relu':\n            self.activation = nn.ReLU()\n        elif activation == 'gelu':\n            self.activation = nn.GELU()\n        elif activation == 'silu':\n            self.activation = nn.SiLU()\n        else:\n            self.activation = nn.ReLU()\n        \n        # Build network layers\n        layers = []\n        prev_dim = input_dim\n        \n        for i, (hidden_dim, dropout_rate) in enumerate(zip(hidden_dims, dropout_rates)):\n            layers.append(nn.Linear(prev_dim, hidden_dim))\n            \n            if use_batch_norm:\n                layers.append(nn.BatchNorm1d(hidden_dim))\n            elif use_layer_norm:\n                layers.append(nn.LayerNorm(hidden_dim))\n            \n            layers.append(self.activation)\n            layers.append(nn.Dropout(dropout_rate))\n            \n            prev_dim = hidden_dim\n        \n        layers.append(nn.Linear(prev_dim, 1))\n        \n        self.network = nn.Sequential(*layers)\n        self._init_weights()\n    \n    def _init_weights(self):\n        \"\"\"Initialize weights using Xavier/He initialization\"\"\"\n        for m in self.modules():\n            if isinstance(m, nn.Linear):\n                if isinstance(self.activation, nn.ReLU):\n                    nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')\n                else:\n                    nn.init.xavier_uniform_(m.weight)\n                if m.bias is not None:\n                    nn.init.constant_(m.bias, 0)\n            elif isinstance(m, nn.BatchNorm1d):\n                nn.init.constant_(m.weight, 1)\n                nn.init.constant_(m.bias, 0)\n    \n    def forward(self, x, add_noise=True):\n        \"\"\"Forward pass with optional noise injection\"\"\"\n        if self.training and add_noise and self.noise_std > 0:\n            noise = torch.randn_like(x) * self.noise_std\n            x = x + noise\n        \n        return self.network(x).squeeze()\n\ndef create_features(df, config):\n    \"\"\"Create comprehensive feature set\"\"\"\n    print(\"  Engineering features...\")\n    \n    eps = 1e-8\n    \n    # Basic ratios\n    df['buy_sell_ratio'] = df['buy_qty'] / (df['sell_qty'] + eps)\n    df['bid_ask_ratio'] = df['bid_qty'] / (df['ask_qty'] + eps)\n    \n    # Imbalances\n    df['order_imbalance'] = (df['buy_qty'] - df['sell_qty']) / (df['buy_qty'] + df['sell_qty'] + eps)\n    df['quote_imbalance'] = (df['bid_qty'] - df['ask_qty']) / (df['bid_qty'] + df['ask_qty'] + eps)\n    \n    # Volume features\n    df['volume_ratio'] = df['volume'] / (df['bid_qty'] + df['ask_qty'] + eps)\n    df['trade_intensity'] = (df['buy_qty'] + df['sell_qty']) / (df['volume'] + eps)\n    \n    # Liquidity\n    df['liquidity'] = df['bid_qty'] + df['ask_qty']\n    df['liquidity_ratio'] = df['liquidity'] / (df['volume'] + eps)\n    \n    # Market pressure\n    df['buy_pressure'] = df['buy_qty'] / (df['volume'] + eps)\n    df['sell_pressure'] = df['sell_qty'] / (df['volume'] + eps)\n    \n    # Advanced features\n    df['spread_proxy'] = np.abs(df['buy_qty'] - df['sell_qty']) / (df['volume'] + eps)\n    df['trade_direction'] = np.sign(df['buy_qty'] - df['sell_qty'])\n    df['quote_depth'] = df['bid_qty'] + df['ask_qty']\n    df['depth_imbalance'] = (df['bid_qty'] - df['ask_qty']) / (df['quote_depth'] + eps)\n    \n    # Volume-weighted features\n    df['vwap_proxy'] = (df['buy_qty'] + df['sell_qty']) / 2\n    df['volume_concentration'] = (df['buy_qty']**2 + df['sell_qty']**2) / (df['volume']**2 + eps)\n    \n    # Market activity\n    df['total_activity'] = df['buy_qty'] + df['sell_qty'] + df['bid_qty'] + df['ask_qty']\n    df['activity_ratio'] = df['total_activity'] / (df['volume'] + eps)\n    \n    # Liquidity consumption\n    df['liquidity_consumption'] = (df['buy_qty'] + df['sell_qty']) / (df['liquidity'] + eps)\n    \n    # Log transforms\n    for col in ['volume', 'buy_qty', 'sell_qty', 'bid_qty', 'ask_qty', 'liquidity', 'total_activity']:\n        if col in df.columns:\n            df[f'{col}_log'] = np.log1p(df[col])\n    \n    # Polynomial features\n    df['order_imbalance_sq'] = df['order_imbalance'] ** 2\n    df['quote_imbalance_sq'] = df['quote_imbalance'] ** 2\n    df['liquidity_ratio_sq'] = df['liquidity_ratio'] ** 2\n    \n    # Interaction features\n    df['imbalance_interaction'] = df['order_imbalance'] * df['quote_imbalance']\n    df['pressure_interaction'] = df['buy_pressure'] * df['sell_pressure']\n    df['volume_liquidity_interaction'] = df['volume_log'] * df['liquidity_log']\n    \n    return df\n\nclass EnsembleMLPTrainer:\n    \"\"\"Enhanced trainer with ensemble capabilities\"\"\"\n    \n    def __init__(self, config):\n        self.config = config\n        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n        self.cv_results = defaultdict(list)\n        \n    def train_single_model(self, X_train, y_train, X_val, y_val, \n                          target_transformer=None, model_seed=42):\n        \"\"\"Train a single model with target transformation\"\"\"\n        \n        # Set seed for this model\n        torch.manual_seed(model_seed)\n        np.random.seed(model_seed)\n        \n        # Transform targets if specified\n        if target_transformer is not None:\n            y_train_transformed = target_transformer.fit_transform(y_train)\n            y_val_transformed = target_transformer.transform(y_val)\n        else:\n            y_train_transformed = y_train\n            y_val_transformed = y_val\n        \n        # Create model\n        input_dim = X_train.shape[1]\n        model = AdvancedMLPWithNoise(\n            input_dim=input_dim,\n            hidden_dims=self.config.HIDDEN_DIMS,\n            dropout_rates=self.config.DROPOUT_RATES,\n            noise_std=self.config.NOISE_STD,\n            use_batch_norm=self.config.USE_BATCH_NORM,\n            use_layer_norm=self.config.USE_LAYER_NORM,\n            activation=self.config.ACTIVATION\n        ).to(self.device)\n        \n        # Create dataloader\n        dataset = TensorDataset(\n            torch.FloatTensor(X_train),\n            torch.FloatTensor(y_train_transformed)\n        )\n        dataloader = DataLoader(\n            dataset,\n            batch_size=self.config.BATCH_SIZE,\n            shuffle=True,\n            num_workers=0,\n            pin_memory=True if torch.cuda.is_available() else False\n        )\n        \n        # Optimizer and scheduler\n        optimizer = optim.AdamW(\n            model.parameters(),\n            lr=self.config.LEARNING_RATE,\n            weight_decay=self.config.WEIGHT_DECAY\n        )\n        \n        scheduler = optim.lr_scheduler.CosineAnnealingWarmRestarts(\n            optimizer, T_0=10, T_mult=2, eta_min=1e-6\n        )\n        \n        # Training loop\n        best_val_loss = float('inf')\n        best_model_state = None\n        patience_counter = 0\n        \n        for epoch in range(self.config.MAX_EPOCHS):\n            # Training\n            model.train()\n            train_loss = 0\n            \n            for batch_x, batch_y in dataloader:\n                batch_x = batch_x.to(self.device)\n                batch_y = batch_y.to(self.device)\n                \n                optimizer.zero_grad()\n                output = model(batch_x)\n                loss = F.mse_loss(output, batch_y)\n                \n                loss.backward()\n                torch.nn.utils.clip_grad_norm_(model.parameters(), self.config.GRADIENT_CLIP)\n                optimizer.step()\n                \n                train_loss += loss.item()\n            \n            avg_train_loss = train_loss / len(dataloader)\n            \n            # Validation\n            model.eval()\n            with torch.no_grad():\n                X_val_tensor = torch.FloatTensor(X_val).to(self.device)\n                y_val_tensor = torch.FloatTensor(y_val_transformed).to(self.device)\n                val_output = model(X_val_tensor, add_noise=False)\n                val_loss = F.mse_loss(val_output, y_val_tensor).item()\n            \n            scheduler.step()\n            \n            # Early stopping\n            if val_loss < best_val_loss:\n                best_val_loss = val_loss\n                best_model_state = copy.deepcopy(model.state_dict())\n                patience_counter = 0\n            else:\n                patience_counter += 1\n                if patience_counter >= self.config.EARLY_STOPPING_PATIENCE:\n                    break\n        \n        # Load best model\n        model.load_state_dict(best_model_state)\n        \n        return model, target_transformer\n    \n    def predict_with_model(self, model, X, target_transformer=None):\n        \"\"\"Make predictions with a single model\"\"\"\n        model.eval()\n        \n        predictions = []\n        batch_size = 10000\n        \n        with torch.no_grad():\n            for i in range(0, len(X), batch_size):\n                batch_X = X[i:i+batch_size]\n                X_tensor = torch.FloatTensor(batch_X).to(self.device)\n                batch_pred = model(X_tensor, add_noise=False).cpu().numpy()\n                predictions.extend(batch_pred)\n        \n        predictions = np.array(predictions)\n        \n        # Inverse transform if needed\n        if target_transformer is not None:\n            predictions = target_transformer.inverse_transform(predictions)\n        \n        return predictions\n    \n    def train_ensemble(self, X_train, y_train, X_val, y_val):\n        \"\"\"Train ensemble of models\"\"\"\n        ensemble_models = []\n        ensemble_predictions = []\n        \n        for i in range(self.config.N_ENSEMBLE_MODELS):\n            print(f\"    Training ensemble model {i+1}/{self.config.N_ENSEMBLE_MODELS}\")\n            \n            # Create target transformer for each model\n            if self.config.USE_TARGET_TRANSFORM:\n                target_transformer = TargetTransformer(\n                    method=self.config.TARGET_TRANSFORM_METHOD\n                )\n            else:\n                target_transformer = None\n            \n            # Train model with different seed\n            model, transformer = self.train_single_model(\n                X_train, y_train, X_val, y_val,\n                target_transformer=target_transformer,\n                model_seed=42 + i\n            )\n            \n            # Store model and transformer\n            ensemble_models.append((model, transformer))\n            \n            # Get predictions\n            val_predictions = self.predict_with_model(model, X_val, transformer)\n            ensemble_predictions.append(val_predictions)\n        \n        # Ensemble predictions (simple average)\n        ensemble_pred = np.mean(ensemble_predictions, axis=0)\n        \n        return ensemble_models, ensemble_pred\n    \n    def cross_validate(self, X, y, scaler_class):\n        \"\"\"Enhanced cross-validation with ensemble\"\"\"\n        print(\"\\nPerforming cross-validation...\")\n        \n        kf = KFold(n_splits=self.config.N_FOLDS, shuffle=True, random_state=42)\n        fold_results = []\n        \n        for fold, (train_idx, val_idx) in enumerate(kf.split(X)):\n            print(f\"\\n  Fold {fold + 1}/{self.config.N_FOLDS}\")\n            \n            # Split data\n            X_train, X_val = X[train_idx], X[val_idx]\n            y_train, y_val = y[train_idx], y[val_idx]\n            \n            # Scale features\n            scaler = scaler_class()\n            X_train_scaled = scaler.fit_transform(X_train)\n            X_val_scaled = scaler.transform(X_val)\n            \n            if self.config.USE_ENSEMBLE:\n                # Train ensemble\n                ensemble_models, predictions = self.train_ensemble(\n                    X_train_scaled, y_train, X_val_scaled, y_val\n                )\n            else:\n                # Train single model\n                target_transformer = TargetTransformer(\n                    method=self.config.TARGET_TRANSFORM_METHOD\n                ) if self.config.USE_TARGET_TRANSFORM else None\n                \n                model, transformer = self.train_single_model(\n                    X_train_scaled, y_train, X_val_scaled, y_val,\n                    target_transformer=target_transformer\n                )\n                predictions = self.predict_with_model(\n                    model, X_val_scaled, transformer\n                )\n            \n            # Calculate metrics\n            mse = mean_squared_error(y_val, predictions)\n            mae = mean_absolute_error(y_val, predictions)\n            correlation = pearsonr(y_val, predictions)[0]\n            \n            fold_results.append({\n                'mse': mse,\n                'mae': mae,\n                'correlation': correlation,\n                'predictions': predictions\n            })\n            \n            print(f\"    Fold {fold + 1} Results:\")\n            print(f\"      MSE: {mse:.6f}\")\n            print(f\"      MAE: {mae:.6f}\")\n            print(f\"      Correlation: {correlation:.6f}\")\n            \n            # Store results\n            self.cv_results['mse'].append(mse)\n            self.cv_results['mae'].append(mae)\n            self.cv_results['correlation'].append(correlation)\n            \n            # Clean up\n            gc.collect()\n            if torch.cuda.is_available():\n                torch.cuda.empty_cache()\n        \n        return fold_results\n    \n    def train_final_ensemble(self, X, y, scaler):\n        \"\"\"Train final ensemble on all data\"\"\"\n        print(\"\\nTraining final ensemble model...\")\n        \n        # Scale data\n        X_scaled = scaler.fit_transform(X)\n        \n        ensemble_models = []\n        \n        for i in range(self.config.N_ENSEMBLE_MODELS):\n            print(f\"  Training final model {i+1}/{self.config.N_ENSEMBLE_MODELS}\")\n            \n            # Create target transformer\n            if self.config.USE_TARGET_TRANSFORM:\n                target_transformer = TargetTransformer(\n                    method=self.config.TARGET_TRANSFORM_METHOD\n                )\n            else:\n                target_transformer = None\n            \n            # For final training, use all data as both train and validation\n            # (validation is just for early stopping)\n            model, transformer = self.train_single_model(\n                X_scaled, y, X_scaled[:10000], y[:10000],  # Small validation set\n                target_transformer=target_transformer,\n                model_seed=42 + i\n            )\n            \n            ensemble_models.append((model, transformer))\n        \n        return ensemble_models, scaler\n\ndef main():\n    \"\"\"Main execution pipeline\"\"\"\n    start_time = time.time()\n    \n    print(\"\\nLoading data...\")\n    \n    # Load data\n    train_df = pd.read_parquet(config.TRAIN_PATH)\n    test_df = pd.read_parquet(config.TEST_PATH)\n    submission_df = pd.read_csv(config.SUBMISSION_PATH)\n    \n    print(f\"  Train shape: {train_df.shape}\")\n    print(f\"  Test shape: {test_df.shape}\")\n    \n    # Optimize data types\n    print(\"\\nOptimizing data types...\")\n    train_df = DataTypeOptimizer.optimize_dtypes(train_df)\n    test_df = DataTypeOptimizer.optimize_dtypes(test_df)\n    \n    # Clean data\n    cleaner = DataCleaner(config)\n    train_df, test_df = cleaner.clean_data(train_df, test_df)\n    \n    # Create features\n    print(\"\\nFeature engineering...\")\n    train_df = create_features(train_df, config)\n    test_df = create_features(test_df, config)\n    \n    # Get common features\n    feature_cols = [col for col in train_df.columns if col != config.LABEL_COLUMN]\n    test_cols = [col for col in test_df.columns]\n    common_features = list(set(feature_cols).intersection(set(test_cols)))\n    \n    print(f\"\\nUsing {len(common_features)} features\")\n    \n    # Optimize data types again after feature engineering\n    print(\"\\nOptimizing data types (final pass)...\")\n    train_df = DataTypeOptimizer.optimize_dtypes(train_df)\n    test_df = DataTypeOptimizer.optimize_dtypes(test_df)\n    \n    # Prepare data\n    X = train_df[common_features].values.astype(np.float32)\n    y = train_df[config.LABEL_COLUMN].values.astype(np.float32)\n    X_test = test_df[common_features].values.astype(np.float32)\n    \n    print(f\"\\nFinal data shapes:\")\n    print(f\"  X_train: {X.shape}\")\n    print(f\"  X_test: {X_test.shape}\")\n    \n    # Initialize trainer\n    trainer = EnsembleMLPTrainer(config)\n    \n    # Cross-validation\n    cv_results = trainer.cross_validate(X, y, RobustScaler)\n    \n    # Print CV results\n    print(\"\\n\" + \"=\"*80)\n    print(\"CROSS-VALIDATION RESULTS\")\n    print(\"=\"*80)\n    print(f\"Average MSE: {np.mean(trainer.cv_results['mse']):.6f} ± {np.std(trainer.cv_results['mse']):.6f}\")\n    print(f\"Average MAE: {np.mean(trainer.cv_results['mae']):.6f} ± {np.std(trainer.cv_results['mae']):.6f}\")\n    print(f\"Average Correlation: {np.mean(trainer.cv_results['correlation']):.6f} ± {np.std(trainer.cv_results['correlation']):.6f}\")\n    \n    # Train final ensemble\n    final_models, final_scaler = trainer.train_final_ensemble(X, y, RobustScaler())\n    \n    # Make predictions\n    print(\"\\nGenerating ensemble predictions...\")\n    X_test_scaled = final_scaler.transform(X_test)\n    \n    all_predictions = []\n    for i, (model, transformer) in enumerate(final_models):\n        print(f\"  Predicting with model {i+1}/{len(final_models)}\")\n        predictions = trainer.predict_with_model(model, X_test_scaled, transformer)\n        all_predictions.append(predictions)\n    \n    # Ensemble predictions\n    final_predictions = np.mean(all_predictions, axis=0)\n    \n    # Create submission\n    print(\"\\nCreating submission...\")\n    submission_df['prediction'] = final_predictions\n    submission_df.to_csv('submission_enhanced_mlp.csv', index=False)\n    \n    # Save models (optional)\n    print(\"\\nSaving models...\")\n    model_data = {\n        'models': final_models,\n        'scaler': final_scaler,\n        'config': config,\n        'cv_results': dict(trainer.cv_results)\n    }\n    joblib.dump(model_data, 'enhanced_mlp_models.pkl')\n    \n    # Final summary\n    elapsed_time = time.time() - start_time\n    \n    print(\"\\n\" + \"=\"*80)\n    print(\"TRAINING COMPLETE\")\n    print(\"=\"*80)\n    print(f\"Total execution time: {elapsed_time:.1f} seconds\")\n    print(f\"Features used: {len(common_features)}\")\n    print(f\"Zero variance columns removed: {len(cleaner.columns_to_remove)}\")\n    print(f\"Target transformation: {config.TARGET_TRANSFORM_METHOD if config.USE_TARGET_TRANSFORM else 'None'}\")\n    print(f\"Ensemble size: {config.N_ENSEMBLE_MODELS if config.USE_ENSEMBLE else 1}\")\n    print(f\"Activation function: {config.ACTIVATION}\")\n    print(\"\\nSubmission saved as: submission_enhanced_mlp.csv\")\n    print(\"=\"*80)\n\nif __name__ == \"__main__\":\n    main()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-07T16:16:08.352061Z","iopub.execute_input":"2025-07-07T16:16:08.352396Z","iopub.status.idle":"2025-07-07T17:15:29.571903Z","shell.execute_reply.started":"2025-07-07T16:16:08.352377Z","shell.execute_reply":"2025-07-07T17:15:29.571160Z"}},"outputs":[],"execution_count":null}]}