{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Complete Working DRW Crypto Market Prediction Framework\n# Fully debugged and tested implementation\n\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.preprocessing import StandardScaler, QuantileTransformer, RobustScaler\nfrom sklearn.model_selection import KFold, TimeSeriesSplit\nfrom sklearn.feature_selection import mutual_info_regression, f_regression\nfrom sklearn.decomposition import PCA\nfrom sklearn.metrics import mean_squared_error\nfrom scipy.stats import pearsonr, spearmanr\nimport xgboost as xgb\nimport lightgbm as lgb\nimport catboost as cb\nimport warnings\nimport random\nimport os\nimport pickle\nfrom pathlib import Path\nimport gc\nfrom tqdm import tqdm\n\nwarnings.filterwarnings('ignore')\n\n# Set all random seeds for reproducibility\ndef set_random_seeds(seed=42):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    os.environ['PYTHONHASHSEED'] = str(seed)\n\n# Robust data handling\ndef handle_infinite_values(data, method='clip'):\n    \"\"\"Handle infinite and extreme values in data\"\"\"\n    data = np.array(data, dtype=np.float32)\n    \n    # Handle infinities\n    if method == 'clip':\n        if len(data.shape) == 2:\n            for i in range(data.shape[1]):\n                col = data[:, i]\n                finite_mask = np.isfinite(col)\n                if np.sum(finite_mask) > 0:\n                    finite_vals = col[finite_mask]\n                    p1, p99 = np.percentile(finite_vals, [1, 99])\n                    col[np.isposinf(col)] = p99\n                    col[np.isneginf(col)] = p1\n                    data[:, i] = np.clip(col, p1, p99)\n                else:\n                    data[:, i] = 0.0\n        else:\n            finite_mask = np.isfinite(data)\n            if np.sum(finite_mask) > 0:\n                finite_vals = data[finite_mask]\n                p1, p99 = np.percentile(finite_vals, [1, 99])\n                data[np.isposinf(data)] = p99\n                data[np.isneginf(data)] = p1\n                data = np.clip(data, p1, p99)\n            else:\n                data[:] = 0.0\n    \n    # Handle NaN values\n    if len(data.shape) == 2:\n        for i in range(data.shape[1]):\n            col_nan_mask = np.isnan(data[:, i])\n            if np.any(col_nan_mask):\n                finite_vals = data[np.isfinite(data[:, i]), i]\n                if len(finite_vals) > 0:\n                    data[col_nan_mask, i] = np.median(finite_vals)\n                else:\n                    data[col_nan_mask, i] = 0.0\n    else:\n        nan_mask = np.isnan(data)\n        if np.any(nan_mask):\n            finite_vals = data[np.isfinite(data)]\n            if len(finite_vals) > 0:\n                data[nan_mask] = np.median(finite_vals)\n            else:\n                data[nan_mask] = 0.0\n    \n    return data\n\n# Autoencoder for feature extraction\nclass Autoencoder(nn.Module):\n    def __init__(self, input_dim, encoding_dim=32, dropout=0.2):\n        super(Autoencoder, self).__init__()\n        \n        # Encoder\n        self.encoder = nn.Sequential(\n            nn.Linear(input_dim, 128),\n            nn.BatchNorm1d(128),\n            nn.ReLU(),\n            nn.Dropout(dropout),\n            nn.Linear(128, 64),\n            nn.BatchNorm1d(64),\n            nn.ReLU(),\n            nn.Dropout(dropout),\n            nn.Linear(64, encoding_dim),\n            nn.BatchNorm1d(encoding_dim),\n            nn.ReLU()\n        )\n        \n        # Decoder\n        self.decoder = nn.Sequential(\n            nn.Linear(encoding_dim, 64),\n            nn.BatchNorm1d(64),\n            nn.ReLU(),\n            nn.Dropout(dropout),\n            nn.Linear(64, 128),\n            nn.BatchNorm1d(128),\n            nn.ReLU(),\n            nn.Dropout(dropout),\n            nn.Linear(128, input_dim)\n        )\n    \n    def forward(self, x):\n        encoded = self.encoder(x)\n        decoded = self.decoder(encoded)\n        return decoded, encoded\n    \n    def encode(self, x):\n        return self.encoder(x)\n\n# Feature engineering\ndef create_enhanced_features(df, feature_cols, autoencoder_model=None, scaler=None):\n    \"\"\"Create comprehensive enhanced features\"\"\"\n    print(f\"   Creating features from {len(feature_cols)} base features...\")\n    \n    # Base features\n    X_base = df[feature_cols].values.astype(np.float32)\n    X_base = handle_infinite_values(X_base)\n    \n    enhanced_features = []\n    feature_names = []\n    \n    # 1. Base features\n    enhanced_features.append(X_base)\n    feature_names.extend(feature_cols)\n    \n    # 2. Microstructure features\n    if all(col in df.columns for col in ['bid_qty', 'ask_qty', 'buy_qty', 'sell_qty', 'volume']):\n        eps = 1e-8\n        \n        # Order flow features\n        order_imbalance = (df['bid_qty'] - df['ask_qty']) / (df['bid_qty'] + df['ask_qty'] + eps)\n        buy_sell_imbalance = (df['buy_qty'] - df['sell_qty']) / (df['buy_qty'] + df['sell_qty'] + eps)\n        \n        # Pressure indicators\n        bid_pressure = df['bid_qty'] / (df['volume'] + eps)\n        ask_pressure = df['ask_qty'] / (df['volume'] + eps)\n        buy_pressure = df['buy_qty'] / (df['volume'] + eps)\n        sell_pressure = df['sell_qty'] / (df['volume'] + eps)\n        \n        # Volume features\n        volume_log = np.log1p(df['volume'])\n        volume_sqrt = np.sqrt(df['volume'])\n        \n        # Liquidity indicators\n        spread_proxy = (df['ask_qty'] + df['bid_qty']) / (df['volume'] + eps)\n        depth_imbalance = (df['bid_qty'] + df['ask_qty']) / (df['buy_qty'] + df['sell_qty'] + eps)\n        \n        microstructure = np.column_stack([\n            order_imbalance, buy_sell_imbalance,\n            bid_pressure, ask_pressure, buy_pressure, sell_pressure,\n            volume_log, volume_sqrt,\n            spread_proxy, depth_imbalance\n        ])\n        \n        microstructure = handle_infinite_values(microstructure)\n        enhanced_features.append(microstructure)\n        \n        feature_names.extend([\n            'order_imbalance', 'buy_sell_imbalance',\n            'bid_pressure', 'ask_pressure', 'buy_pressure', 'sell_pressure',\n            'volume_log', 'volume_sqrt',\n            'spread_proxy', 'depth_imbalance'\n        ])\n    \n    # 3. Statistical transformations (for top features only)\n    top_n = min(10, X_base.shape[1])\n    \n    stat_features = []\n    for i in range(top_n):\n        feature = X_base[:, i]\n        \n        # Log transform\n        log_feat = np.sign(feature) * np.log1p(np.abs(feature))\n        stat_features.append(log_feat)\n        feature_names.append(f'{feature_cols[i]}_log')\n        \n        # Power transform\n        power_feat = np.sign(feature) * np.power(np.abs(feature) + eps, 0.5)\n        stat_features.append(power_feat)\n        feature_names.append(f'{feature_cols[i]}_sqrt')\n    \n    if stat_features:\n        stat_matrix = np.column_stack(stat_features)\n        stat_matrix = handle_infinite_values(stat_matrix)\n        enhanced_features.append(stat_matrix)\n    \n    # 4. Interaction features (limited to top features)\n    interaction_features = []\n    \n    top_indices = list(range(min(5, X_base.shape[1])))\n    for i in range(len(top_indices)):\n        for j in range(i+1, len(top_indices)):\n            # Multiplication\n            inter = X_base[:, top_indices[i]] * X_base[:, top_indices[j]]\n            interaction_features.append(inter)\n            feature_names.append(f'{feature_cols[top_indices[i]]}_x_{feature_cols[top_indices[j]]}')\n    \n    if interaction_features:\n        inter_matrix = np.column_stack(interaction_features)\n        inter_matrix = handle_infinite_values(inter_matrix)\n        enhanced_features.append(inter_matrix)\n    \n    # Combine all features\n    X_enhanced = np.hstack(enhanced_features)\n    X_enhanced = handle_infinite_values(X_enhanced)\n    \n    # 5. Autoencoder features (if model provided)\n    if autoencoder_model is not None and scaler is not None:\n        device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n        autoencoder_model.eval()\n        \n        # Scale features\n        X_scaled = scaler.transform(X_enhanced)\n        X_scaled = handle_infinite_values(X_scaled)\n        \n        # Get encoded features\n        with torch.no_grad():\n            X_tensor = torch.FloatTensor(X_scaled).to(device)\n            # Process in batches to avoid memory issues\n            batch_size = 4096\n            encoded_features = []\n            \n            for i in range(0, len(X_tensor), batch_size):\n                batch = X_tensor[i:i+batch_size]\n                encoded = autoencoder_model.encode(batch)\n                encoded_features.append(encoded.cpu().numpy())\n            \n            encoded_features = np.vstack(encoded_features)\n        \n        # Add encoded features\n        X_enhanced = np.hstack([X_enhanced, encoded_features])\n        feature_names.extend([f'ae_feature_{i}' for i in range(encoded_features.shape[1])])\n    \n    return X_enhanced, feature_names\n\n# Feature selection\ndef advanced_feature_selection(X, y, max_features=100):\n    \"\"\"Select top features using multiple methods\"\"\"\n    print(f\"🔍 Selecting top {max_features} features from {X.shape[1]}...\")\n    \n    X = handle_infinite_values(X)\n    y = handle_infinite_values(y.reshape(-1, 1)).flatten()\n    \n    # Calculate feature importance scores\n    scores = []\n    \n    # 1. Correlation\n    corr_scores = []\n    for i in range(X.shape[1]):\n        try:\n            corr = abs(pearsonr(X[:, i], y)[0])\n            corr_scores.append(corr if not np.isnan(corr) else 0)\n        except:\n            corr_scores.append(0)\n    \n    # 2. Mutual information\n    try:\n        mi_scores = mutual_info_regression(X, y, random_state=42)\n        mi_scores = np.nan_to_num(mi_scores, 0)\n    except:\n        mi_scores = np.zeros(X.shape[1])\n    \n    # 3. Random Forest importance\n    try:\n        from sklearn.ensemble import RandomForestRegressor\n        rf = RandomForestRegressor(n_estimators=50, max_depth=10, random_state=42, n_jobs=-1)\n        rf.fit(X, y)\n        rf_scores = rf.feature_importances_\n    except:\n        rf_scores = np.zeros(X.shape[1])\n    \n    # Normalize and combine scores\n    def normalize_scores(s):\n        if s.max() == s.min():\n            return np.ones_like(s) / len(s)\n        return (s - s.min()) / (s.max() - s.min() + 1e-8)\n    \n    corr_norm = normalize_scores(np.array(corr_scores))\n    mi_norm = normalize_scores(mi_scores)\n    rf_norm = normalize_scores(rf_scores)\n    \n    # Weighted ensemble of scores\n    ensemble_scores = 0.4 * corr_norm + 0.3 * mi_norm + 0.3 * rf_norm\n    \n    # Select top features\n    n_features = min(max_features, X.shape[1])\n    top_features = np.argsort(ensemble_scores)[-n_features:]\n    \n    print(f\"   ✅ Selected {len(top_features)} features\")\n    return top_features, ensemble_scores\n\n# Data transformations\ndef create_rank_transform(X):\n    \"\"\"Create rank transformation\"\"\"\n    X_rank = np.zeros_like(X)\n    for i in range(X.shape[1]):\n        X_rank[:, i] = pd.Series(X[:, i]).rank(pct=True, method='average').fillna(0.5).values\n    return handle_infinite_values(X_rank)\n\n# Neural Network Components\nclass SimpleNN(nn.Module):\n    \"\"\"Simple neural network for regression\"\"\"\n    def __init__(self, input_dim, hidden_dim=128, dropout=0.5):\n        super(SimpleNN, self).__init__()\n        \n        self.net = nn.Sequential(\n            nn.Linear(input_dim, hidden_dim),\n            nn.BatchNorm1d(hidden_dim),\n            nn.ReLU(),\n            nn.Dropout(dropout),\n            \n            nn.Linear(hidden_dim, hidden_dim // 2),\n            nn.BatchNorm1d(hidden_dim // 2),\n            nn.ReLU(),\n            nn.Dropout(dropout),\n            \n            nn.Linear(hidden_dim // 2, hidden_dim // 4),\n            nn.BatchNorm1d(hidden_dim // 4),\n            nn.ReLU(),\n            nn.Dropout(dropout * 0.5),\n            \n            nn.Linear(hidden_dim // 4, 1)\n        )\n        \n        # Initialize weights\n        for m in self.modules():\n            if isinstance(m, nn.Linear):\n                nn.init.xavier_uniform_(m.weight, gain=0.5)\n                if m.bias is not None:\n                    nn.init.constant_(m.bias, 0)\n    \n    def forward(self, x):\n        return self.net(x)\n\nclass CryptoDataset(Dataset):\n    \"\"\"PyTorch dataset for crypto data\"\"\"\n    def __init__(self, X, y=None):\n        self.X = torch.FloatTensor(X)\n        self.y = torch.FloatTensor(y) if y is not None else None\n    \n    def __len__(self):\n        return len(self.X)\n    \n    def __getitem__(self, idx):\n        if self.y is not None:\n            return self.X[idx], self.y[idx]\n        return self.X[idx]\n\ndef train_neural_network(X_train, y_train, X_val, y_val, input_dim, epochs=20):\n    \"\"\"Train neural network model\"\"\"\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    \n    # Create model\n    model = SimpleNN(input_dim, hidden_dim=128, dropout=0.5).to(device)\n    \n    # Create datasets\n    train_dataset = CryptoDataset(X_train, y_train)\n    val_dataset = CryptoDataset(X_val, y_val)\n    \n    train_loader = DataLoader(train_dataset, batch_size=2048, shuffle=True)\n    val_loader = DataLoader(val_dataset, batch_size=4096, shuffle=False)\n    \n    # Training setup\n    criterion = nn.MSELoss()\n    optimizer = torch.optim.AdamW(model.parameters(), lr=1e-4, weight_decay=1e-3)\n    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience=5, factor=0.5)\n    \n    best_val_loss = float('inf')\n    best_val_corr = -float('inf')\n    patience = 0\n    \n    # Training loop\n    for epoch in range(epochs):\n        # Train\n        model.train()\n        train_loss = 0\n        for X_batch, y_batch in train_loader:\n            X_batch, y_batch = X_batch.to(device), y_batch.to(device)\n            \n            optimizer.zero_grad()\n            outputs = model(X_batch).squeeze()\n            loss = criterion(outputs, y_batch)\n            loss.backward()\n            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n            optimizer.step()\n            \n            train_loss += loss.item()\n        \n        # Validate\n        model.eval()\n        val_preds = []\n        val_targets = []\n        val_loss = 0\n        \n        with torch.no_grad():\n            for X_batch, y_batch in val_loader:\n                X_batch, y_batch = X_batch.to(device), y_batch.to(device)\n                outputs = model(X_batch).squeeze()\n                loss = criterion(outputs, y_batch)\n                val_loss += loss.item()\n                \n                val_preds.extend(outputs.cpu().numpy())\n                val_targets.extend(y_batch.cpu().numpy())\n        \n        # Calculate correlation\n        val_corr = pearsonr(val_preds, val_targets)[0]\n        if np.isnan(val_corr):\n            val_corr = 0\n        \n        scheduler.step(val_loss)\n        \n        # Early stopping\n        if val_corr > best_val_corr:\n            best_val_corr = val_corr\n            patience = 0\n        else:\n            patience += 1\n            if patience >= 10:\n                break\n    \n    return model, best_val_corr\n\n# Gradient Boosting Models\ndef train_xgboost(X_train, y_train, X_val, y_val):\n    \"\"\"Train XGBoost model\"\"\"\n    params = {\n        'objective': 'reg:squarederror',\n        'max_depth': 5,\n        'learning_rate': 0.02,\n        'subsample': 0.8,\n        'colsample_bytree': 0.8,\n        'reg_alpha': 1,\n        'reg_lambda': 1,\n        'random_state': 42,\n        'n_estimators': 1000,\n        'early_stopping_rounds': 50\n    }\n    \n    model = xgb.XGBRegressor(**params)\n    model.fit(\n        X_train, y_train,\n        eval_set=[(X_val, y_val)],\n        verbose=False\n    )\n    \n    return model\n\ndef train_lightgbm(X_train, y_train, X_val, y_val):\n    \"\"\"Train LightGBM model\"\"\"\n    params = {\n        'objective': 'regression',\n        'metric': 'rmse',\n        'max_depth': 5,\n        'learning_rate': 0.02,\n        'subsample': 0.8,\n        'colsample_bytree': 0.8,\n        'reg_alpha': 1,\n        'reg_lambda': 1,\n        'random_state': 42,\n        'n_estimators': 1000,\n        'verbose': -1,\n        'force_col_wise': True\n    }\n    \n    model = lgb.LGBMRegressor(**params)\n    \n    # Use callbacks for early stopping\n    callbacks = [\n        lgb.early_stopping(stopping_rounds=50),\n        lgb.log_evaluation(0)\n    ]\n    \n    model.fit(\n        X_train, y_train,\n        eval_set=[(X_val, y_val)],\n        callbacks=callbacks\n    )\n    \n    return model\n\ndef train_catboost(X_train, y_train, X_val, y_val):\n    \"\"\"Train CatBoost model\"\"\"\n    params = {\n        'iterations': 1000,\n        'depth': 5,\n        'learning_rate': 0.02,\n        'l2_leaf_reg': 3,\n        'random_seed': 42,\n        'verbose': False,\n        'early_stopping_rounds': 50\n    }\n    \n    model = cb.CatBoostRegressor(**params)\n    model.fit(\n        X_train, y_train,\n        eval_set=(X_val, y_val),\n        verbose=False\n    )\n    \n    return model\n\n# Main pipeline\ndef main_pipeline():\n    \"\"\"Main prediction pipeline\"\"\"\n    print(\"🚀 Complete DRW Crypto Prediction Pipeline\")\n    print(\"=\" * 80)\n    \n    set_random_seeds(42)\n    \n    # Create output directory\n    output_dir = Path(\"/kaggle/working/drw_predictions\")\n    output_dir.mkdir(exist_ok=True)\n    \n    # Load data\n    print(\"\\n📊 Loading data...\")\n    train_df = pd.read_parquet(\"/kaggle/input/drw-crypto-market-prediction/train.parquet\")\n    test_df = pd.read_parquet(\"/kaggle/input/drw-crypto-market-prediction/test.parquet\")\n    \n    print(f\"   Train shape: {train_df.shape}\")\n    print(f\"   Test shape: {test_df.shape}\")\n    \n    # Use recent data (last 70% of training data)\n    train_size = int(0.7 * len(train_df))\n    train_df = train_df.iloc[-train_size:].reset_index(drop=True)\n    print(f\"   Using recent {len(train_df)} samples for training\")\n    \n    # Top features from analysis\n    top_features = [\n        \"X612\", \"X860\", \"X168\", \"X174\", \"X333\", \"X345\", \"X385\", \"X598\", \n        \"X421\", \"X852\", \"X863\", \"X856\", \"X344\", \"X862\", \"X603\", \"X674\", \n        \"X415\", \"X137\", \"X855\", \"X302\", \"bid_qty\", \"ask_qty\", \"buy_qty\", \n        \"sell_qty\", \"volume\"\n    ]\n    \n    # Filter available features\n    feature_cols = [col for col in top_features if col in train_df.columns]\n    print(f\"   Using {len(feature_cols)} features\")\n    \n    # Extract target\n    y_train = train_df['label'].values\n    y_train = handle_infinite_values(y_train.reshape(-1, 1)).flatten()\n    \n    # STEP 1: Create base features (without autoencoder)\n    print(\"\\n🔧 Feature Engineering - Step 1: Base features...\")\n    X_train_base, train_feature_names = create_enhanced_features(\n        train_df, feature_cols, autoencoder_model=None, scaler=None\n    )\n    X_test_base, test_feature_names = create_enhanced_features(\n        test_df, [col for col in feature_cols if col != 'label'],\n        autoencoder_model=None, scaler=None\n    )\n    \n    # STEP 2: Train autoencoder on base features\n    print(\"\\n🤖 Training autoencoder...\")\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    \n    # Scale features for autoencoder\n    scaler = StandardScaler()\n    X_train_scaled = scaler.fit_transform(X_train_base)\n    X_train_scaled = handle_infinite_values(X_train_scaled)\n    \n    # Train autoencoder\n    encoding_dim = 20\n    autoencoder = Autoencoder(X_train_base.shape[1], encoding_dim).to(device)\n    optimizer = torch.optim.Adam(autoencoder.parameters(), lr=1e-3)\n    criterion = nn.MSELoss()\n    \n    # Create dataset for autoencoder training\n    subset_size = min(50000, len(X_train_scaled))\n    subset_idx = np.random.choice(len(X_train_scaled), subset_size, replace=False)\n    \n    dataset = torch.utils.data.TensorDataset(torch.FloatTensor(X_train_scaled[subset_idx]))\n    loader = DataLoader(dataset, batch_size=2048, shuffle=True)\n    \n    # Train autoencoder\n    autoencoder.train()\n    for epoch in range(30):\n        total_loss = 0\n        for batch in loader:\n            data = batch[0].to(device)\n            \n            optimizer.zero_grad()\n            reconstructed, _ = autoencoder(data)\n            loss = criterion(reconstructed, data)\n            loss.backward()\n            optimizer.step()\n            \n            total_loss += loss.item()\n        \n        if epoch % 10 == 0:\n            print(f\"   Epoch {epoch}: Loss = {total_loss/len(loader):.4f}\")\n    \n    # STEP 3: Create features with autoencoder\n    print(\"\\n🔧 Feature Engineering - Step 2: Adding autoencoder features...\")\n    X_train_enhanced, train_feature_names = create_enhanced_features(\n        train_df, feature_cols, autoencoder_model=autoencoder, scaler=scaler\n    )\n    X_test_enhanced, test_feature_names = create_enhanced_features(\n        test_df, [col for col in feature_cols if col != 'label'],\n        autoencoder_model=autoencoder, scaler=scaler\n    )\n    \n    print(f\"   Train features: {X_train_enhanced.shape}\")\n    print(f\"   Test features: {X_test_enhanced.shape}\")\n    \n    # Feature selection\n    top_indices, feature_scores = advanced_feature_selection(\n        X_train_enhanced, y_train, max_features=100\n    )\n    \n    X_train_selected = X_train_enhanced[:, top_indices]\n    X_test_selected = X_test_enhanced[:, top_indices]\n    \n    # Apply rank transformation\n    print(\"\\n🔄 Applying rank transformation...\")\n    X_train_rank = create_rank_transform(X_train_selected)\n    X_test_rank = create_rank_transform(X_test_selected)\n    \n    # Time series cross-validation\n    print(\"\\n🎯 Training models with time series cross-validation...\")\n    tscv = TimeSeriesSplit(n_splits=3)\n    \n    all_test_predictions = []\n    cv_scores = []\n    \n    for fold, (train_idx, val_idx) in enumerate(tscv.split(X_train_rank)):\n        print(f\"\\n📊 Fold {fold + 1}/3\")\n        \n        X_fold_train = X_train_rank[train_idx]\n        X_fold_val = X_train_rank[val_idx]\n        y_fold_train = y_train[train_idx]\n        y_fold_val = y_train[val_idx]\n        \n        fold_predictions = []\n        fold_scores = {}\n        \n        # Train Neural Network\n        print(\"   🧠 Training Neural Network...\")\n        try:\n            nn_model, nn_score = train_neural_network(\n                X_fold_train, y_fold_train, \n                X_fold_val, y_fold_val,\n                input_dim=X_fold_train.shape[1],\n                epochs=20\n            )\n            \n            # Make predictions\n            nn_model.eval()\n            test_dataset = CryptoDataset(X_test_rank)\n            test_loader = DataLoader(test_dataset, batch_size=4096, shuffle=False)\n            \n            nn_preds = []\n            with torch.no_grad():\n                for batch in test_loader:\n                    batch = batch.to(device)\n                    preds = nn_model(batch).squeeze().cpu().numpy()\n                    nn_preds.extend(preds)\n            \n            fold_predictions.append(np.array(nn_preds))\n            fold_scores['neural'] = nn_score\n            print(f\"      Validation correlation: {nn_score:.4f}\")\n        except Exception as e:\n            print(f\"      Failed: {e}\")\n            fold_scores['neural'] = 0\n        \n        # Train XGBoost\n        print(\"   🌳 Training XGBoost...\")\n        try:\n            xgb_model = train_xgboost(X_fold_train, y_fold_train, X_fold_val, y_fold_val)\n            xgb_preds = xgb_model.predict(X_test_rank)\n            xgb_val_preds = xgb_model.predict(X_fold_val)\n            xgb_score = pearsonr(xgb_val_preds, y_fold_val)[0]\n            \n            fold_predictions.append(xgb_preds)\n            fold_scores['xgboost'] = xgb_score\n            print(f\"      Validation correlation: {xgb_score:.4f}\")\n        except Exception as e:\n            print(f\"      Failed: {e}\")\n            fold_scores['xgboost'] = 0\n        \n        # Train LightGBM\n        print(\"   🌿 Training LightGBM...\")\n        try:\n            lgb_model = train_lightgbm(X_fold_train, y_fold_train, X_fold_val, y_fold_val)\n            lgb_preds = lgb_model.predict(X_test_rank)\n            lgb_val_preds = lgb_model.predict(X_fold_val)\n            lgb_score = pearsonr(lgb_val_preds, y_fold_val)[0]\n            \n            fold_predictions.append(lgb_preds)\n            fold_scores['lightgbm'] = lgb_score\n            print(f\"      Validation correlation: {lgb_score:.4f}\")\n        except Exception as e:\n            print(f\"      Failed: {e}\")\n            fold_scores['lightgbm'] = 0\n        \n        # Train CatBoost\n        print(\"   🐱 Training CatBoost...\")\n        try:\n            cb_model = train_catboost(X_fold_train, y_fold_train, X_fold_val, y_fold_val)\n            cb_preds = cb_model.predict(X_test_rank)\n            cb_val_preds = cb_model.predict(X_fold_val)\n            cb_score = pearsonr(cb_val_preds, y_fold_val)[0]\n            \n            fold_predictions.append(cb_preds)\n            fold_scores['catboost'] = cb_score\n            print(f\"      Validation correlation: {cb_score:.4f}\")\n        except Exception as e:\n            print(f\"      Failed: {e}\")\n            fold_scores['catboost'] = 0\n        \n        # Ensemble fold predictions\n        if fold_predictions:\n            # Weight by squared correlation (emphasize better models)\n            weights = []\n            for model_name in ['neural', 'xgboost', 'lightgbm', 'catboost']:\n                score = fold_scores.get(model_name, 0)\n                if score > 0:\n                    weights.append(score ** 2)\n                else:\n                    weights.append(0)\n            \n            weights = np.array(weights[:len(fold_predictions)])\n            \n            if weights.sum() > 0:\n                weights = weights / weights.sum()\n                fold_ensemble = np.average(fold_predictions, axis=0, weights=weights)\n            else:\n                fold_ensemble = np.mean(fold_predictions, axis=0)\n            \n            all_test_predictions.append(fold_ensemble)\n            valid_scores = [s for s in fold_scores.values() if s > 0]\n            cv_scores.append(np.mean(valid_scores) if valid_scores else 0)\n            \n            print(f\"   📊 Fold average correlation: {cv_scores[-1]:.4f}\")\n    \n    # Final ensemble across folds\n    print(\"\\n🔮 Creating final predictions...\")\n    if all_test_predictions:\n        # Weight folds by their average score\n        if cv_scores and np.array(cv_scores).sum() > 0:\n            fold_weights = np.array(cv_scores)\n            fold_weights = fold_weights / fold_weights.sum()\n            final_predictions = np.average(all_test_predictions, axis=0, weights=fold_weights)\n        else:\n            final_predictions = np.mean(all_test_predictions, axis=0)\n    else:\n        print(\"⚠️ No successful predictions, using baseline\")\n        final_predictions = np.full(len(test_df), y_train.mean())\n    \n    # Post-processing: clip extreme values\n    p5, p95 = np.percentile(y_train, [5, 95])\n    final_predictions = np.clip(final_predictions, p5, p95)\n    \n    # Create submission\n    print(\"\\n💾 Creating submission...\")\n    submission = pd.read_csv(\"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\")\n    submission['label'] = final_predictions\n    submission.to_csv(output_dir / 'submission_final.csv', index=False)\n    \n    print(\"\\n✅ Pipeline completed successfully!\")\n    if cv_scores:\n        print(f\"   Average CV score: {np.mean(cv_scores):.4f}\")\n    print(f\"   Output saved to: {output_dir / 'submission_final.csv'}\")\n    \n    # Clean up\n    gc.collect()\n    if torch.cuda.is_available():\n        torch.cuda.empty_cache()\n    \n    return submission\n\n# Run the pipeline\nif __name__ == \"__main__\":\n    submission = main_pipeline()","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}