{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n\"\"\"\nDRW Crypto Market Prediction - Neural Architecture Search with Feature Selection\n===============================================================================\nEnhanced implementation combining NAS with intelligent feature selection\n\"\"\"\n\nimport random\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler, RobustScaler\nfrom sklearn.feature_selection import VarianceThreshold\nfrom tqdm import tqdm\nfrom typing import Dict, List, Tuple, Optional\nimport warnings\nwarnings.filterwarnings(\"ignore\", category=RuntimeWarning)\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, TensorDataset\nfrom scipy.stats import pearsonr\nimport gc\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(f\"Using device: {device}\")\n\n# =========================\n# Configuration\n# =========================\nclass Config:\n    # Paths\n    TRAIN_PATH = \"/kaggle/input/drw-crypto-market-prediction/train.parquet\"\n    TEST_PATH = \"/kaggle/input/drw-crypto-market-prediction/test.parquet\"\n    SUBMISSION_PATH = \"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\"\n    \n    # Feature selection parameters\n    MIN_VARIANCE_THRESHOLD = 0.01  # Remove features with variance below this\n    CORRELATION_THRESHOLD = 0.95   # Remove highly correlated features\n    MAX_FEATURES_TO_TEST = 300     # Maximum features to consider\n    MIN_FEATURES_TO_KEEP = 50      # Minimum features to retain\n    \n    # NAS parameters\n    NAS_ITERATIONS = 30\n    NAS_PATIENCE = 10\n    EPOCHS_PER_ARCH = 10\n    \n    # Model defaults\n    SEED = 42\n    BATCH_SIZE = 1024 * 16\n    LEARNING_RATE = 0.001\n    WEIGHT_DECAY = 1e-3\n    DROPOUT_RATE = 0.6\n    NOISE_FACTOR = 0.005\n\n# =========================\n# Utility Functions\n# =========================\ndef set_seed(seed=Config.SEED):\n    \"\"\"Set all random seeds for reproducibility\"\"\"\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\ndef shrink_dtypes(df):\n    \"\"\"Optimize polars dataframe memory usage\"\"\"\n    return df.select(\n        pl.col(col).shrink_dtype() for col in df.collect_schema().names()\n    )\n\ndef get_activation_function(name):\n    \"\"\"Return the activation function based on the name\"\"\"\n    if name is None:\n        return None\n    name = name.lower()\n    activations = {\n        'relu': nn.ReLU(),\n        'tanh': nn.Tanh(),\n        'sigmoid': nn.Sigmoid(),\n        'leakyrelu': nn.LeakyReLU(),\n        'elu': nn.ELU(),\n        'gelu': nn.GELU(),\n        'selu': nn.SELU(),\n        'prelu': nn.PReLU()\n    }\n    if name not in activations:\n        raise ValueError(f\"Unsupported activation function: {name}\")\n    return activations[name]\n\n# =========================\n# Feature Engineering\n# =========================\ndef create_microstructure_features(df):\n    \"\"\"Create comprehensive microstructure features\"\"\"\n    print(\"Creating microstructure features...\")\n    \n    # Basic microstructure features\n    df = df.with_columns([\n        # Order book imbalance\n        ((pl.col(\"bid_qty\") - pl.col(\"ask_qty\")) / \n         (pl.col(\"bid_qty\") + pl.col(\"ask_qty\") + 1e-10)).alias(\"order_book_imbalance\"),\n        \n        # Trade flow imbalance\n        ((pl.col(\"buy_qty\") - pl.col(\"sell_qty\")) / \n         (pl.col(\"buy_qty\") + pl.col(\"sell_qty\") + 1e-10)).alias(\"trade_flow_imbalance\"),\n        \n        # Liquidity consumption ratio\n        (pl.col(\"volume\") / \n         (pl.col(\"bid_qty\") + pl.col(\"ask_qty\") + 1e-10)).alias(\"liquidity_consumption_ratio\"),\n        \n        # Buy/sell pressure\n        (pl.col(\"buy_qty\") / (pl.col(\"bid_qty\") + 1e-10)).alias(\"buy_pressure\"),\n        (pl.col(\"sell_qty\") / (pl.col(\"ask_qty\") + 1e-10)).alias(\"sell_pressure\"),\n        \n        # Market depth\n        (pl.col(\"bid_qty\") + pl.col(\"ask_qty\")).alias(\"total_depth\"),\n        (pl.col(\"bid_qty\") / (pl.col(\"ask_qty\") + 1e-10)).alias(\"bid_ask_ratio\"),\n        \n        # Trade metrics\n        (pl.col(\"volume\") / \n         ((pl.col(\"buy_qty\") + pl.col(\"sell_qty\")) + 1e-10)).alias(\"avg_trade_size\"),\n        \n        # Kyle's lambda proxy\n        ((pl.col(\"buy_qty\") - pl.col(\"sell_qty\")).abs() / \n         (pl.col(\"volume\").sqrt() + 1e-10)).alias(\"kyle_lambda_proxy\"),\n        \n        # Amihud illiquidity\n        ((pl.col(\"buy_qty\") - pl.col(\"sell_qty\")).abs() / \n         (pl.col(\"volume\") + 1e-10)).alias(\"amihud_proxy\"),\n    ])\n    \n    # Log transformations\n    log_cols = [\"volume\", \"total_depth\", \"bid_qty\", \"ask_qty\", \"buy_qty\", \"sell_qty\"]\n    df = df.with_columns([\n        pl.col(col).log1p().alias(f\"log_{col}\") for col in log_cols\n    ])\n    \n    # Square root transformations\n    sqrt_cols = [\"volume\", \"total_depth\", \"bid_qty\", \"ask_qty\"]\n    df = df.with_columns([\n        pl.col(col).sqrt().alias(f\"sqrt_{col}\") for col in sqrt_cols\n    ])\n    \n    # Power transformations\n    df = df.with_columns([\n        (pl.col(\"order_book_imbalance\") ** 2).alias(\"order_book_imbalance_sq\"),\n        (pl.col(\"trade_flow_imbalance\") ** 2).alias(\"trade_flow_imbalance_sq\"),\n        pl.col(\"volume\").pow(0.25).alias(\"volume_pow_quarter\"),\n    ])\n    \n    # Statistical distance features\n    volume_stats = {\n        'mean': df[\"volume\"].mean(),\n        'std': df[\"volume\"].std(),\n        'median': df[\"volume\"].quantile(0.50)\n    }\n    \n    df = df.with_columns([\n        ((pl.col(\"volume\") - volume_stats['mean']) / \n         (volume_stats['std'] + 1e-10)).alias(\"volume_zscore\"),\n        (pl.col(\"volume\") / (volume_stats['median'] + 1e-10)).alias(\"volume_ratio_median\"),\n    ])\n    \n    # Interaction features\n    df = df.with_columns([\n        (pl.col(\"order_book_imbalance\") * \n         pl.col(\"trade_flow_imbalance\")).alias(\"imbalance_interaction\"),\n        (pl.col(\"buy_pressure\") * \n         pl.col(\"volume_ratio_median\")).alias(\"buy_pressure_volume_interaction\"),\n        (pl.col(\"kyle_lambda_proxy\") * \n         pl.col(\"amihud_proxy\")).alias(\"impact_illiquidity_interaction\"),\n    ])\n    \n    # Clean data\n    df = df.fill_null(0)\n    for col in df.columns:\n        if col not in [\"timestamp\", \"label\"]:\n            df = df.with_columns(\n                pl.col(col).fill_nan(0).alias(col)\n            )\n    \n    return df\n\n# =========================\n# Feature Selection\n# =========================\nclass AdvancedFeatureSelector:\n    \"\"\"Advanced feature selection with multiple criteria\"\"\"\n    \n    def __init__(self, min_variance=0.01, correlation_threshold=0.95, \n                 max_features=300, min_features=50):\n        self.min_variance = min_variance\n        self.correlation_threshold = correlation_threshold\n        self.max_features = max_features\n        self.min_features = min_features\n        self.selected_features = None\n        self.feature_scores = None\n        \n    def fit_transform(self, X, y, feature_names):\n        \"\"\"Select features based on multiple criteria\"\"\"\n        print(\"\\nPerforming advanced feature selection...\")\n        n_features_original = X.shape[1]\n        \n        # Step 1: Remove constant features\n        constant_mask = np.var(X, axis=0) > 0\n        X = X[:, constant_mask]\n        feature_names = [f for f, m in zip(feature_names, constant_mask) if m]\n        print(f\"Removed {n_features_original - len(feature_names)} constant features\")\n        \n        # Step 2: Remove low variance features\n        if X.shape[1] > self.min_features:\n            variances = np.var(X, axis=0)\n            variance_threshold = np.percentile(variances, 10)  # Keep top 90%\n            variance_threshold = max(variance_threshold, self.min_variance)\n            \n            variance_mask = variances > variance_threshold\n            X = X[:, variance_mask]\n            feature_names = [f for f, m in zip(feature_names, variance_mask) if m]\n            print(f\"Removed {sum(~variance_mask)} low variance features\")\n        \n        # Step 3: Remove highly correlated features\n        if X.shape[1] > self.min_features:\n            correlation_matrix = np.corrcoef(X.T)\n            upper_triangle = np.triu(np.ones_like(correlation_matrix, dtype=bool), k=1)\n            high_corr_pairs = np.where((np.abs(correlation_matrix) > self.correlation_threshold) & upper_triangle)\n            \n            features_to_remove = set()\n            for i, j in zip(high_corr_pairs[0], high_corr_pairs[1]):\n                if i not in features_to_remove and j not in features_to_remove:\n                    # Keep feature with higher correlation to target\n                    corr_i = abs(pearsonr(X[:, i], y)[0])\n                    corr_j = abs(pearsonr(X[:, j], y)[0])\n                    if corr_i < corr_j:\n                        features_to_remove.add(i)\n                    else:\n                        features_to_remove.add(j)\n            \n            keep_mask = np.array([i not in features_to_remove for i in range(X.shape[1])])\n            X = X[:, keep_mask]\n            feature_names = [f for f, m in zip(feature_names, keep_mask) if m]\n            print(f\"Removed {len(features_to_remove)} highly correlated features\")\n        \n        # Step 4: Score remaining features\n        print(\"\\nScoring remaining features...\")\n        feature_scores = []\n        for i in range(X.shape[1]):\n            # Calculate multiple metrics\n            variance = np.var(X[:, i])\n            correlation = abs(pearsonr(X[:, i], y)[0])\n            mutual_info_approx = variance * correlation  # Simplified mutual information proxy\n            \n            # Combined score\n            score = mutual_info_approx\n            feature_scores.append((feature_names[i], score, variance, correlation))\n        \n        # Sort by score\n        feature_scores.sort(key=lambda x: x[1], reverse=True)\n        \n        # Select top features\n        n_select = min(self.max_features, len(feature_scores))\n        n_select = max(n_select, self.min_features)\n        \n        self.selected_features = [f[0] for f in feature_scores[:n_select]]\n        self.feature_scores = feature_scores[:n_select]\n        \n        # Print top features\n        print(f\"\\nSelected {len(self.selected_features)} features from {n_features_original}\")\n        print(\"\\nTop 15 features:\")\n        for i, (name, score, var, corr) in enumerate(self.feature_scores[:15]):\n            print(f\"  {i+1:2d}. {name:25s} score={score:.4f}, var={var:.4f}, corr={corr:.4f}\")\n        \n        return self.selected_features\n    \n    def transform(self, df, feature_names):\n        \"\"\"Transform dataframe to selected features\"\"\"\n        if self.selected_features is None:\n            raise ValueError(\"Must call fit_transform first\")\n        \n        # Find indices of selected features\n        indices = [i for i, f in enumerate(feature_names) if f in self.selected_features]\n        return df[:, indices] if isinstance(df, np.ndarray) else df[self.selected_features]\n\n# =========================\n# Neural Network Components\n# =========================\nclass MLP(nn.Module):\n    def __init__(self, layers, dropout_rate=0.6, activation='relu', \n                 last_activation=None, use_batch_norm=True):\n        \"\"\"Enhanced MLP with optional batch normalization\"\"\"\n        super(MLP, self).__init__()\n        \n        self.layers = nn.ModuleList()\n        self.use_batch_norm = use_batch_norm\n        \n        for i in range(len(layers) - 1):\n            self.layers.append(nn.Linear(layers[i], layers[i + 1]))\n            \n            if i < len(layers) - 2:  # Not the last layer\n                if use_batch_norm:\n                    self.layers.append(nn.BatchNorm1d(layers[i + 1]))\n                self.layers.append(get_activation_function(activation))\n                self.layers.append(nn.Dropout(dropout_rate))\n        \n        if last_activation is not None:\n            self.layers.append(get_activation_function(last_activation))\n    \n    def forward(self, x):\n        for layer in self.layers:\n            x = layer(x)\n        return x\n\nclass Checkpointer:\n    def __init__(self, path=\"best_model.pt\"):\n        self.path = path\n        self.best_pearson = -np.inf\n        self.patience_counter = 0\n    \n    def __call__(self, pearson_coef, model, patience=None):\n        \"\"\"Save model if performance improves\"\"\"\n        if pearson_coef > self.best_pearson:\n            self.best_pearson = pearson_coef\n            self.patience_counter = 0\n            torch.save({\n                'model_state_dict': model.state_dict(),\n                'pearson': pearson_coef\n            }, self.path)\n            print(f\"✅ New best model saved with Pearson: {pearson_coef:.4f}\")\n            return True\n        else:\n            self.patience_counter += 1\n            if patience and self.patience_counter >= patience:\n                print(f\"Early stopping triggered after {patience} epochs without improvement\")\n                return False\n        return None\n    \n    def load(self, model):\n        \"\"\"Load the best model weights\"\"\"\n        checkpoint = torch.load(self.path)\n        model.load_state_dict(checkpoint['model_state_dict'])\n        self.best_pearson = checkpoint['pearson']\n        print(f\"Model loaded from {self.path} with Pearson: {self.best_pearson:.4f}\")\n        return model\n\n# =========================\n# Enhanced Neural Architecture Search\n# =========================\nclass EnhancedNeuralArchitectureSearch:\n    def __init__(self, input_dim, feature_selector, search_space=None):\n        self.input_dim = input_dim\n        self.feature_selector = feature_selector\n        self.best_architecture = None\n        self.best_score = -np.inf\n        self.search_history = []\n        \n        # Enhanced search space\n        if search_space is None:\n            self.search_space = {\n                'num_layers': [3, 4, 5, 6],\n                'layer_sizes': [64, 128, 256, 512, 1024],\n                'activation': ['relu', 'leakyrelu', 'elu', 'gelu', 'selu'],\n                'dropout_rate': [0.3, 0.4, 0.5, 0.6, 0.7],\n                'learning_rate': [0.0001, 0.0005, 0.001, 0.002],\n                'weight_decay': [1e-4, 5e-4, 1e-3, 5e-3],\n                'batch_size': [1024*8, 1024*16, 1024*32],\n                'noise_factor': [0, 0.001, 0.005, 0.01],\n                'use_batch_norm': [True, False],\n                'optimizer_type': ['adam', 'adamw', 'sgd'],\n                'scheduler_type': ['none', 'cosine', 'step']\n            }\n        else:\n            self.search_space = search_space\n    \n    def generate_architecture(self, method='random'):\n        \"\"\"Generate a new architecture\"\"\"\n        if method == 'random':\n            return self._random_architecture()\n        elif method == 'evolutionary':\n            return self._evolutionary_architecture()\n        else:\n            raise ValueError(f\"Unknown method: {method}\")\n    \n    def _random_architecture(self):\n        \"\"\"Generate random architecture\"\"\"\n        arch = {}\n        \n        # Sample hyperparameters\n        for param, values in self.search_space.items():\n            if param not in ['layer_sizes', 'num_layers']:\n                arch[param] = random.choice(values)\n        \n        # Generate layer configuration\n        arch['num_layers'] = random.choice(self.search_space['num_layers'])\n        layers = [self.input_dim]\n        \n        # Decreasing layer sizes\n        prev_size = self.input_dim\n        for i in range(arch['num_layers'] - 1):\n            candidates = [s for s in self.search_space['layer_sizes'] if s <= prev_size]\n            if not candidates:\n                candidates = self.search_space['layer_sizes']\n            layer_size = random.choice(candidates)\n            layers.append(layer_size)\n            prev_size = layer_size\n        \n        layers.append(1)  # Output\n        arch['layers'] = layers\n        \n        return arch\n    \n    def _evolutionary_architecture(self):\n        \"\"\"Generate architecture using evolutionary strategy\"\"\"\n        if len(self.search_history) < 5:\n            return self._random_architecture()\n        \n        # Tournament selection\n        tournament_size = min(5, len(self.search_history))\n        tournament = random.sample(self.search_history, tournament_size)\n        parent = max(tournament, key=lambda x: x['score'])['architecture']\n        \n        # Create child with mutations\n        child = parent.copy()\n        child['layers'] = parent['layers'].copy()\n        \n        # Mutate 1-3 parameters\n        n_mutations = random.randint(1, 3)\n        for _ in range(n_mutations):\n            param = random.choice(list(self.search_space.keys()))\n            \n            if param in ['layer_sizes', 'num_layers']:\n                # Mutate architecture\n                child['num_layers'] = random.choice(self.search_space['num_layers'])\n                layers = [self.input_dim]\n                for i in range(child['num_layers'] - 1):\n                    if random.random() < 0.7 and i+1 < len(child['layers'])-1:\n                        layers.append(child['layers'][i+1])\n                    else:\n                        layers.append(random.choice(self.search_space['layer_sizes']))\n                layers.append(1)\n                child['layers'] = layers\n            else:\n                child[param] = random.choice(self.search_space[param])\n        \n        return child\n    \n    def train_and_evaluate(self, architecture, X_train, Y_train, X_val, Y_val, epochs=10):\n        \"\"\"Train and evaluate architecture\"\"\"\n        set_seed(Config.SEED)\n        \n        # Create data loaders\n        train_dataset = TensorDataset(\n            torch.tensor(X_train, dtype=torch.float32),\n            torch.tensor(Y_train.values, dtype=torch.float32).unsqueeze(1)\n        )\n        val_dataset = TensorDataset(\n            torch.tensor(X_val, dtype=torch.float32),\n            torch.tensor(Y_val.values, dtype=torch.float32).unsqueeze(1)\n        )\n        \n        train_loader = DataLoader(\n            train_dataset, \n            batch_size=architecture['batch_size'], \n            shuffle=True\n        )\n        val_loader = DataLoader(\n            val_dataset, \n            batch_size=architecture['batch_size'], \n            shuffle=False\n        )\n        \n        # Initialize model\n        model = MLP(\n            layers=architecture['layers'],\n            dropout_rate=architecture['dropout_rate'],\n            activation=architecture['activation'],\n            use_batch_norm=architecture.get('use_batch_norm', True)\n        ).to(device)\n        \n        # Loss and optimizer\n        criterion = nn.HuberLoss(delta=5.0)\n        \n        if architecture.get('optimizer_type', 'adam') == 'adam':\n            optimizer = optim.Adam(\n                model.parameters(), \n                lr=architecture['learning_rate'],\n                weight_decay=architecture['weight_decay']\n            )\n        elif architecture['optimizer_type'] == 'adamw':\n            optimizer = optim.AdamW(\n                model.parameters(), \n                lr=architecture['learning_rate'],\n                weight_decay=architecture['weight_decay']\n            )\n        else:\n            optimizer = optim.SGD(\n                model.parameters(), \n                lr=architecture['learning_rate'],\n                weight_decay=architecture['weight_decay'],\n                momentum=0.9\n            )\n        \n        # Learning rate scheduler\n        if architecture.get('scheduler_type', 'none') == 'cosine':\n            scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=epochs)\n        elif architecture['scheduler_type'] == 'step':\n            scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.5)\n        else:\n            scheduler = None\n        \n        # Training loop\n        best_val_pearson = -np.inf\n        \n        for epoch in range(epochs):\n            # Training\n            model.train()\n            for inputs, targets in train_loader:\n                inputs, targets = inputs.to(device), targets.to(device)\n                \n                # Add noise\n                if architecture['noise_factor'] > 0:\n                    inputs = inputs + torch.randn_like(inputs) * architecture['noise_factor']\n                \n                optimizer.zero_grad()\n                outputs = model(inputs)\n                loss = criterion(outputs, targets)\n                loss.backward()\n                optimizer.step()\n            \n            if scheduler:\n                scheduler.step()\n            \n            # Validation\n            model.eval()\n            val_preds = []\n            val_true = []\n            \n            with torch.no_grad():\n                for inputs, targets in val_loader:\n                    inputs, targets = inputs.to(device), targets.to(device)\n                    outputs = model(inputs)\n                    val_preds.extend(outputs.cpu().numpy())\n                    val_true.extend(targets.cpu().numpy())\n            \n            val_preds = np.array(val_preds).flatten()\n            val_true = np.array(val_true).flatten()\n            pearson = pearsonr(val_preds, val_true)[0]\n            \n            if pearson > best_val_pearson:\n                best_val_pearson = pearson\n        \n        return best_val_pearson, model\n    \n    def search(self, X_train, Y_train, X_val, Y_val, n_iterations=30, \n               method='evolutionary', epochs_per_arch=10, patience=10):\n        \"\"\"Perform architecture search\"\"\"\n        print(f\"\\nStarting Enhanced Neural Architecture Search\")\n        print(f\"Method: {method}, Iterations: {n_iterations}, Patience: {patience}\")\n        print(\"=\"*60)\n        \n        no_improvement = 0\n        \n        for i in range(n_iterations):\n            # Generate architecture\n            if i < 5:  # Start with random exploration\n                arch = self.generate_architecture('random')\n            else:\n                arch = self.generate_architecture(method)\n            \n            print(f\"\\nIteration {i+1}/{n_iterations}\")\n            print(f\"Architecture: {arch['num_layers']} layers, \"\n                  f\"sizes={arch['layers'][1:-1]}, \"\n                  f\"activation={arch['activation']}, \"\n                  f\"dropout={arch['dropout_rate']}\")\n            \n            # Train and evaluate\n            score, model = self.train_and_evaluate(\n                arch, X_train, Y_train, X_val, Y_val, epochs_per_arch\n            )\n            \n            print(f\"Validation Pearson: {score:.4f}\")\n            \n            # Update history\n            self.search_history.append({\n                'architecture': arch,\n                'score': score,\n                'model': model\n            })\n            \n            # Check for improvement\n            if score > self.best_score:\n                self.best_score = score\n                self.best_architecture = arch\n                no_improvement = 0\n                print(f\"🎯 New best architecture! Score: {score:.4f}\")\n            else:\n                no_improvement += 1\n            \n            # Early stopping\n            if no_improvement >= patience:\n                print(f\"\\nEarly stopping: No improvement for {patience} iterations\")\n                break\n        \n        print(f\"\\n✅ Search complete. Best score: {self.best_score:.4f}\")\n        return self.best_architecture, self.best_score\n\n# =========================\n# Main Pipeline\n# =========================\ndef main():\n    \"\"\"Main execution pipeline\"\"\"\n    print(\"DRW Crypto Market Prediction - Enhanced NAS with Feature Selection\")\n    print(\"=\"*70)\n    \n    set_seed(Config.SEED)\n    \n    # Load training data\n    print(\"\\nLoading training data...\")\n    train = pl.scan_parquet(Config.TRAIN_PATH)\n    \n    # Get all columns except timestamp\n    all_columns = train.collect_schema().names()\n    feature_columns = [col for col in all_columns if col not in ['timestamp', 'label']]\n    \n    # Load with all features plus label\n    train = shrink_dtypes(train.select(feature_columns + ['label'])).collect()\n    print(f\"Initial shape: {train.shape}\")\n    \n    # Create microstructure features\n    train = create_microstructure_features(train)\n    \n    # Get updated feature list\n    feature_cols = [col for col in train.columns if col != 'label']\n    print(f\"Total features after engineering: {len(feature_cols)}\")\n    \n    # Convert to pandas\n    train_pd = train.to_pandas()\n    \n    # Store statistics for test data processing\n    train_stats = {}\n    for col in feature_cols:\n        train_stats[col] = {\n            'median': train_pd[col].median(),\n            'q01': train_pd[col].quantile(0.01),\n            'q99': train_pd[col].quantile(0.99)\n        }\n    \n    # Handle outliers\n    print(\"\\nHandling outliers...\")\n    for col in feature_cols:\n        train_pd[col] = train_pd[col].replace([np.inf, -np.inf], np.nan)\n        train_pd[col] = train_pd[col].fillna(train_stats[col]['median'])\n        train_pd[col] = train_pd[col].clip(\n            lower=train_stats[col]['q01'], \n            upper=train_stats[col]['q99']\n        )\n    \n    # Split data\n    print(\"\\nSplitting data...\")\n    X_train, X_val = train_test_split(\n        train_pd, test_size=0.2, shuffle=False, random_state=Config.SEED\n    )\n    \n    Y_train = X_train.pop(\"label\")\n    Y_val = X_val.pop(\"label\")\n    \n    # Advanced feature selection\n    feature_selector = AdvancedFeatureSelector(\n        min_variance=Config.MIN_VARIANCE_THRESHOLD,\n        correlation_threshold=Config.CORRELATION_THRESHOLD,\n        max_features=Config.MAX_FEATURES_TO_TEST,\n        min_features=Config.MIN_FEATURES_TO_KEEP\n    )\n    \n    selected_features = feature_selector.fit_transform(\n        X_train[feature_cols].values, \n        Y_train.values, \n        feature_cols\n    )\n    \n    # Apply feature selection\n    X_train_selected = X_train[selected_features].values\n    X_val_selected = X_val[selected_features].values\n    \n    # Scale features\n    print(\"\\nScaling features...\")\n    scaler = RobustScaler()\n    X_train_scaled = scaler.fit_transform(X_train_selected)\n    X_val_scaled = scaler.transform(X_val_selected)\n    \n    print(f\"\\nFinal training shape: {X_train_scaled.shape}\")\n    \n    # Neural Architecture Search\n    nas = EnhancedNeuralArchitectureSearch(\n        input_dim=X_train_scaled.shape[1],\n        feature_selector=feature_selector\n    )\n    \n    best_arch, best_score = nas.search(\n        X_train_scaled, Y_train, X_val_scaled, Y_val,\n        n_iterations=Config.NAS_ITERATIONS,\n        method='evolutionary',\n        epochs_per_arch=Config.EPOCHS_PER_ARCH,\n        patience=Config.NAS_PATIENCE\n    )\n    \n    print(\"\\n\" + \"=\"*70)\n    print(\"BEST ARCHITECTURE FOUND:\")\n    print(\"=\"*70)\n    for key, value in best_arch.items():\n        if key != 'layers':\n            print(f\"{key:20s}: {value}\")\n    print(f\"{'layers':20s}: {best_arch['layers']}\")\n    print(f\"{'best_score':20s}: {best_score:.4f}\")\n    \n    # Train final model with best architecture\n    print(\"\\n\" + \"=\"*70)\n    print(\"TRAINING FINAL MODEL\")\n    print(\"=\"*70)\n    \n    # Combine train and validation for final training\n    X_full = np.vstack([X_train_scaled, X_val_scaled])\n    Y_full = pd.concat([Y_train, Y_val])\n    \n    # Create data loaders\n    full_dataset = TensorDataset(\n        torch.tensor(X_full, dtype=torch.float32),\n        torch.tensor(Y_full.values, dtype=torch.float32).unsqueeze(1)\n    )\n    \n    train_loader = DataLoader(\n        full_dataset,\n        batch_size=best_arch['batch_size'],\n        shuffle=True\n    )\n    \n    # Initialize final model\n    final_model = MLP(\n        layers=best_arch['layers'],\n        dropout_rate=best_arch['dropout_rate'],\n        activation=best_arch['activation'],\n        use_batch_norm=best_arch.get('use_batch_norm', True)\n    ).to(device)\n    \n    # Training setup\n    criterion = nn.HuberLoss(delta=5.0)\n    optimizer = optim.AdamW(\n        final_model.parameters(),\n        lr=best_arch['learning_rate'],\n        weight_decay=best_arch['weight_decay']\n    )\n    \n    checkpointer = Checkpointer(path=\"final_model.pt\")\n    \n    # Train for more epochs\n    print(\"\\nTraining final model...\")\n    for epoch in range(30):\n        final_model.train()\n        running_loss = 0.0\n        \n        for inputs, targets in tqdm(train_loader, desc=f\"Epoch {epoch+1}/30\"):\n            inputs, targets = inputs.to(device), targets.to(device)\n            \n            if best_arch['noise_factor'] > 0:\n                inputs = inputs + torch.randn_like(inputs) * best_arch['noise_factor']\n            \n            optimizer.zero_grad()\n            outputs = final_model(inputs)\n            loss = criterion(outputs, targets)\n            loss.backward()\n            optimizer.step()\n            \n            running_loss += loss.item()\n        \n        avg_loss = running_loss / len(train_loader)\n        print(f\"Epoch {epoch+1}: Loss = {avg_loss:.4f}\")\n    \n    # Generate test predictions\n    print(\"\\n\" + \"=\"*70)\n    print(\"GENERATING TEST PREDICTIONS\")\n    print(\"=\"*70)\n    \n    # Load test data\n    test = pl.scan_parquet(Config.TEST_PATH)\n    test = shrink_dtypes(test.select(feature_columns)).collect()\n    \n    # Create same features\n    test = create_microstructure_features(test)\n    test_pd = test.select(feature_cols).to_pandas()\n    \n    # Apply same preprocessing\n    for col in feature_cols:\n        test_pd[col] = test_pd[col].replace([np.inf, -np.inf], np.nan)\n        test_pd[col] = test_pd[col].fillna(train_stats[col]['median'])\n        test_pd[col] = test_pd[col].clip(\n            lower=train_stats[col]['q01'],\n            upper=train_stats[col]['q99']\n        )\n    \n    # Select and scale features\n    X_test = test_pd[selected_features].values\n    X_test_scaled = scaler.transform(X_test)\n    \n    # Create test loader\n    test_dataset = TensorDataset(torch.tensor(X_test_scaled, dtype=torch.float32))\n    test_loader = DataLoader(test_dataset, batch_size=best_arch['batch_size'], shuffle=False)\n    \n    # Generate predictions\n    final_model.eval()\n    predictions = []\n    \n    with torch.no_grad():\n        for inputs in tqdm(test_loader, desc=\"Predicting\"):\n            inputs = inputs[0].to(device)\n            outputs = final_model(inputs)\n            predictions.extend(outputs.cpu().numpy())\n    \n    predictions = np.array(predictions).flatten()\n    \n    # Create submission\n    submission = pd.read_csv(Config.SUBMISSION_PATH)\n    submission[\"prediction\"] = predictions\n    submission.to_csv(\"submission.csv\", index=False)\n    \n    print(\"\\n\" + \"=\"*70)\n    print(\"PIPELINE COMPLETE\")\n    print(\"=\"*70)\n    print(f\"Selected features: {len(selected_features)}\")\n    print(f\"Best validation score: {best_score:.4f}\")\n    print(f\"Test predictions: {len(predictions)}\")\n    print(f\"\\nPrediction statistics:\")\n    print(f\"  Mean: {predictions.mean():.4f}\")\n    print(f\"  Std:  {predictions.std():.4f}\")\n    print(f\"  Min:  {predictions.min():.4f}\")\n    print(f\"  Max:  {predictions.max():.4f}\")\n    print(\"\\nSubmission saved to: submission.csv\")\n\nif __name__ == \"__main__\":\n    main()","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}