{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Memory-Efficient Crypto Market Prediction Pipeline\nimport numpy as np\nimport pandas as pd\nimport os\nimport json\nimport datetime\nimport math\nimport gc  # For garbage collection\nfrom pathlib import Path\nfrom typing import Dict, List, Tuple, Optional\n\n# Machine Learning imports\nfrom sklearn.model_selection import KFold, train_test_split\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.preprocessing import StandardScaler, QuantileTransformer, RobustScaler\nfrom sklearn.feature_selection import mutual_info_regression, SelectKBest, f_regression\nfrom xgboost import XGBRegressor\nfrom scipy.stats import pearsonr, spearmanr, skew, kurtosis, rankdata\nfrom scipy.optimize import minimize\n\n# Deep Learning imports\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, TensorDataset\nfrom torch.cuda.amp import autocast, GradScaler\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau, CosineAnnealingWarmRestarts\n\n# Utilities\nfrom tqdm import tqdm\nimport random\nimport warnings\nwarnings.filterwarnings(\"ignore\", category=RuntimeWarning)\n\n# =========================\n# Directory Setup\n# =========================\nBASE_DIR = Path(\"/kaggle/working\")\nMODEL_DIR = BASE_DIR / \"models\"\nPREDICTIONS_DIR = BASE_DIR / \"predictions\"\nCHECKPOINTS_DIR = BASE_DIR / \"checkpoints\"\nCONFIGS_DIR = BASE_DIR / \"configs\"\nSUBMISSIONS_DIR = BASE_DIR / \"submissions\"\n\n# Create all directories\nfor directory in [MODEL_DIR, PREDICTIONS_DIR, CHECKPOINTS_DIR, CONFIGS_DIR, SUBMISSIONS_DIR]:\n    directory.mkdir(parents=True, exist_ok=True)\n    print(f\"Created directory: {directory}\")\n\n# =========================\n# Model Configuration\n# =========================\nclass Config:\n    TRAIN_PATH = \"/kaggle/input/drw-crypto-market-prediction/train.parquet\"\n    TEST_PATH = \"/kaggle/input/drw-crypto-market-prediction/test.parquet\"\n    SUBMISSION_PATH = \"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\"\n\n    # Core XGBoost features (before feature engineering)\n    CORE_FEATURES = [\n        \"X863\", \"X856\", \"X598\", \"X862\", \"X385\", \"X852\", \"X603\", \"X860\", \"X674\",\n        \"X415\", \"X345\", \"X855\", \"X174\", \"X302\", \"X178\", \"X168\", \"X612\", \"bid_qty\",\n        \"ask_qty\", \"buy_qty\", \"sell_qty\", \"volume\", \"X888\", \"X421\", \"X333\",\"X817\", \n        \"X586\",  \"X292\"\n    ]\n    \n    # Additional feature pools for creating variations\n    FEATURE_POOL_1 = [  # ~20 additional features\n        \"X344\", \"X137\", \"X532\", \"X425\", \"X132\", \"X691\", \"X593\", \"X377\", \n        \"X285\", \"X126\", \"X419\", \"X604\", \"X84\", \"X138\", \"X413\", \"X291\", \n        \"X40\", \"X123\", \"X81\", \"X853\"\n    ]\n    \n    FEATURE_POOL_2 = [  # ~20 more features\n        \"X854\", \"X777\", \"X219\", \"X776\", \"X180\", \"X781\", \"X445\", \"X444\", \n        \"X384\", \"X466\", \"X95\", \"X583\", \"X272\", \"X533\", \"X758\", \"X279\", \n        \"X297\", \"X21\", \"X20\", \"X28\"\n    ]\n    \n    FEATURE_POOL_3 = [  # ~20 more features\n        \"X29\", \"X19\", \"X27\", \"X22\", \"X198\", \"X89\", \"X90\", \"X98\", \"X96\", \n        \"X97\", \"X383\", \"X427\", \"X451\", \"X283\", \"X753\", \"X497\", \"X748\", \n        \"X820\", \"X566\", \"X535\"\n    ]\n    \n    FEATURE_POOL_4 = [  # ~20 more features\n        \"X394\", \"X618\", \"X429\", \"X381\", \"X387\", \"X890\", \"X752\", \"X375\", \n        \"X68\", \"X152\", \"X110\", \"X850\", \"X851\", \"X481\", \"X321\", \"X363\", \n        \"X405\", \"X492\", \"X750\", \"X751\"\n    ]\n    \n    # For memory efficiency, we'll select top features dynamically\n    ALL_X_FEATURES = None  # Will be populated as needed\n    \n    # XGBoost features remain unchanged\n    FEATURES = CORE_FEATURES.copy()\n    \n    # Original features for comparison\n    MLP_FEATURES_ORIGINAL = [\n        \"X863\", \"X856\", \"X344\", \"X598\", \"X862\", \"X385\", \"X852\", \"X603\", \"X860\", \"X674\",\n        \"X415\", \"X345\", \"X137\", \"X855\", \"X174\", \"X302\", \"X178\", \"X532\", \"X168\", \"X612\",\n        \"bid_qty\", \"ask_qty\", \"buy_qty\", \"sell_qty\", \"volume\"\n    ]\n    \n    GANDALF_FEATURES_ORIGINAL = [\n        \"X863\", \"X856\", \"X598\", \"X862\", \"X385\", \"X852\", \"X603\", \"X860\", \"X674\",\n        \"X415\", \"X345\", \"X855\", \"X174\", \"X302\", \"X178\", \"X168\", \"X612\",\n        \"X425\", \"X132\", \"X691\", \"X593\", \"X377\", \"X285\", \"X126\", \"X419\", \"X604\",\n        \"X84\", \"X138\", \"X413\", \"X291\", \"X40\", \"X123\", \"X81\", \"X853\", \"X854\",\n        \"X777\", \"X219\", \"X776\", \"X180\", \"X781\", \"X445\", \"X444\", \"X384\", \"X466\",\n        \"X95\", \"X583\", \"X272\", \"X137\", \"X533\", \"X758\", \"X279\", \"X297\",\n        \"X21\", \"X20\", \"X28\", \"X29\", \"X19\", \"X27\", \"X22\", \"X198\", \"X89\", \"X90\",\n        \"X98\", \"X96\", \"X97\", \"X383\", \"X427\", \"X451\", \"X283\",\n        \"X753\", \"X497\", \"X748\", \"X820\", \"X566\", \"X535\", \"X394\", \"X618\",\n        \"X429\", \"X381\", \"X387\", \"X890\", \"X752\", \"X375\", \"X68\", \"X152\",\n        \"X110\", \"X850\", \"X851\", \"X481\", \"X321\", \"X363\", \"X405\", \"X492\",\n        \"X888\", \"X421\", \"X333\", \"X817\", \"X586\", \"X292\", \"X344\", \"X532\",\n        \"bid_qty\", \"ask_qty\", \"buy_qty\", \"sell_qty\", \"volume\"\n    ]\n\n    LABEL_COLUMN = \"label\"\n    N_FOLDS = 3\n    RANDOM_STATE = 42\n    OUTLIER_FRACTION = 0.001\n    \n    # Memory optimization parameters\n    MAX_FEATURES_TO_LOAD = 300  # Maximum features to load at once\n    FEATURE_SELECTION_SAMPLE_SIZE = 50000  # Sample size for feature selection\n\ndef create_feature_variations():\n    \"\"\"Create different feature sets for model variations including large sets\"\"\"\n    variations = {\n        \"core\": Config.CORE_FEATURES.copy(),\n        \"core_plus_20\": Config.CORE_FEATURES + Config.FEATURE_POOL_1[:20],\n        \"core_plus_40\": Config.CORE_FEATURES + Config.FEATURE_POOL_1 + Config.FEATURE_POOL_2[:20],\n        \"core_plus_60\": Config.CORE_FEATURES + Config.FEATURE_POOL_1 + Config.FEATURE_POOL_2 + Config.FEATURE_POOL_3[:20],\n        \"core_plus_80\": Config.CORE_FEATURES + Config.FEATURE_POOL_1 + Config.FEATURE_POOL_2 + Config.FEATURE_POOL_3 + Config.FEATURE_POOL_4[:20],\n        \"top_150\": None,  # Will be selected based on feature importance\n        \"top_200\": None,  # Will be selected based on feature importance\n        \"top_300\": None,  # Will be selected based on feature importance\n        \"original\": None  # Will use model-specific original features\n    }\n    \n    # Remove duplicates\n    for key, features in variations.items():\n        if features is not None:\n            variations[key] = list(dict.fromkeys(features))  # Preserves order while removing duplicates\n    \n    return variations\n\n# XGBoost parameters (unchanged)\nXGB_PARAMS = {\n    \"tree_method\": \"hist\",\n    \"device\": \"gpu\",\n    \"colsample_bylevel\": 0.4778,\n    \"colsample_bynode\": 0.3628,\n    \"colsample_bytree\": 0.7107,\n    \"gamma\": 1.7095,\n    \"learning_rate\": 0.02213,\n    \"max_depth\": 20,\n    \"max_leaves\": 12,\n    \"min_child_weight\": 16,\n    \"n_estimators\": 1667,\n    \"subsample\": 0.06567,\n    \"reg_alpha\": 39.3524,\n    \"reg_lambda\": 75.4484,\n    \"verbosity\": 0,\n    \"random_state\": Config.RANDOM_STATE,\n    \"n_jobs\": -1\n}\n\n# =========================\n# Memory-Efficient Feature Engineering\n# =========================\ndef add_essential_features(df):\n    \"\"\"Add only the most essential engineered features to save memory\"\"\"\n    \n    # Core features\n    df['net_order_flow'] = df['buy_qty'] - df['sell_qty']\n    df['total_depth'] = df['bid_qty'] + df['ask_qty']\n    df['log_volume'] = np.log1p(df['volume'])\n    \n    # Key ratios\n    df['buy_sell_ratio'] = df['buy_qty'] / (df['sell_qty'] + 1e-10)\n    df['bid_ask_imbalance'] = (df['bid_qty'] - df['ask_qty']) / (df['total_depth'] + 1e-10)\n    df['order_flow_imbalance'] = df['net_order_flow'] / (df['buy_qty'] + df['sell_qty'] + 1e-10)\n    \n    # Liquidity measures\n    df['liquidity_ratio'] = df['total_depth'] / (df['volume'] + 1e-10)\n    df['kyle_lambda'] = np.abs(df['net_order_flow']) / (df['volume'] + 1e-10)\n    df['vpin_proxy'] = np.abs(df['net_order_flow']) / (df['buy_qty'] + df['sell_qty'] + 1e-10)\n    \n    # Market stress\n    df['market_stress'] = df['volume'] / (df['total_depth'] + 1e-10) * np.abs(df['order_flow_imbalance'])\n    df['execution_rate'] = (df['buy_qty'] + df['sell_qty']) / (df['total_depth'] + 1e-10)\n    \n    # Price impact\n    df['price_impact_proxy'] = df['net_order_flow'] / (df['total_depth'] + 1e-10)\n    df['amihud_illiquidity'] = np.abs(df['net_order_flow']) / (df['volume'] ** 2 + 1e-10)\n    \n    # Log transformations\n    df['log_buy_qty'] = np.log1p(df['buy_qty'])\n    df['log_sell_qty'] = np.log1p(df['sell_qty'])\n    df['log_total_depth'] = np.log1p(df['total_depth'])\n    \n    # Squares\n    df['order_flow_squared'] = df['net_order_flow'] ** 2\n    df['imbalance_squared'] = df['order_flow_imbalance'] ** 2\n    \n    # Replace infinities and NaNs\n    df = df.replace([np.inf, -np.inf], 0).fillna(0)\n    \n    return df\n\ndef add_advanced_features_selective(df, level='medium'):\n    \"\"\"Add features based on memory constraints\"\"\"\n    \n    if level == 'minimal':\n        # Only most essential features\n        return add_essential_features(df)\n    \n    # Start with essential features\n    df = add_essential_features(df)\n    \n    if level == 'medium':\n        # Add medium complexity features\n        df['depth_imbalance'] = (df['bid_qty'] - df['ask_qty']) / (df['total_depth'] + 1e-10)\n        df['normalized_net_flow'] = df['net_order_flow'] / (df['volume'] + 1e-10)\n        df['buying_pressure'] = df['buy_qty'] / (df['volume'] + 1e-10)\n        df['selling_pressure'] = df['sell_qty'] / (df['volume'] + 1e-10)\n        \n        # Interaction features\n        df['volume_depth_interaction'] = df['volume'] * df['total_depth']\n        df['flow_volume_interaction'] = df['net_order_flow'] * df['volume']\n        \n        # Additional measures\n        df['trade_informativeness'] = df['net_order_flow'] / (df['bid_qty'] + df['ask_qty'] + 1e-10)\n        df['liquidity_consumption'] = (df['buy_qty'] + df['sell_qty']) / (df['total_depth'] + 1e-10)\n        \n    elif level == 'full':\n        # This should be used sparingly due to memory constraints\n        # Add all the original advanced features\n        # [Previous full feature engineering code would go here]\n        pass\n    \n    # Replace infinities and NaNs\n    df = df.replace([np.inf, -np.inf], 0).fillna(0)\n    \n    return df\n\n# =========================\n# Memory-Efficient Feature Selection\n# =========================\ndef select_top_features_memory_efficient(train_path, n_features, sample_size=50000):\n    \"\"\"Select top features using a sample to save memory\"\"\"\n    \n    print(f\"Selecting top {n_features} features using sample size {sample_size}...\")\n    \n    # First, get all X feature names\n    all_x_features = [f\"X{i}\" for i in range(1, 891)]\n    base_features = [\"bid_qty\", \"ask_qty\", \"buy_qty\", \"sell_qty\", \"volume\"]\n    \n    # Load a sample to evaluate features\n    sample_df = pd.read_parquet(train_path, \n                               columns=all_x_features + base_features + [Config.LABEL_COLUMN])\n    \n    # Take a random sample\n    if len(sample_df) > sample_size:\n        sample_df = sample_df.sample(n=sample_size, random_state=Config.RANDOM_STATE)\n    \n    # Add engineered features\n    sample_df = add_essential_features(sample_df)\n    \n    # Get all feature columns\n    feature_cols = [col for col in sample_df.columns if col != Config.LABEL_COLUMN]\n    X = sample_df[feature_cols]\n    y = sample_df[Config.LABEL_COLUMN]\n    \n    # Use mutual information for feature selection\n    selector = SelectKBest(score_func=mutual_info_regression, k=min(n_features, len(feature_cols)))\n    selector.fit(X, y)\n    \n    # Get selected features\n    feature_scores = pd.DataFrame({\n        'feature': feature_cols,\n        'score': selector.scores_\n    }).sort_values('score', ascending=False)\n    \n    selected_features = feature_scores['feature'].head(n_features).tolist()\n    \n    # Clean up memory\n    del sample_df, X, y\n    gc.collect()\n    \n    return selected_features\n\n# =========================\n# Utility Functions\n# =========================\ndef set_seed(seed=42):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\ndef create_time_decay_weights(n: int, decay: float = 0.9) -> np.ndarray:\n    positions = np.arange(n)\n    normalized = positions / (n - 1)\n    weights = decay ** (1.0 - normalized)\n    return weights * n / weights.sum()\n\ndef detect_outliers_and_adjust_weights(X, y, sample_weights, outlier_fraction=0.001):\n    \"\"\"Detect outliers and adjust weights\"\"\"\n    rf = RandomForestRegressor(n_estimators=30, max_depth=10, random_state=42, n_jobs=-1)\n    rf.fit(X, y, sample_weight=sample_weights)\n    \n    predictions = rf.predict(X)\n    residuals = np.abs(y - predictions)\n    \n    n_outliers = max(1, int(len(residuals) * outlier_fraction))\n    threshold = np.sort(residuals)[-n_outliers]\n    \n    outlier_mask = residuals >= threshold\n    adjusted_weights = sample_weights.copy()\n    \n    if outlier_mask.any():\n        outlier_residuals = residuals[outlier_mask]\n        min_outlier_res = outlier_residuals.min()\n        max_outlier_res = outlier_residuals.max()\n        \n        if max_outlier_res > min_outlier_res:\n            normalized_residuals = (outlier_residuals - min_outlier_res) / (max_outlier_res - min_outlier_res)\n        else:\n            normalized_residuals = np.ones_like(outlier_residuals)\n        \n        weight_factors = 0.8 - 0.6 * normalized_residuals\n        adjusted_weights[outlier_mask] *= weight_factors\n        \n        print(f\"    Adjusted weights for {n_outliers} outliers ({outlier_fraction*100:.1f}% of data)\")\n    \n    return adjusted_weights\n\ndef load_data_memory_efficient(feature_list=None):\n    \"\"\"Load data with only necessary features to save memory\"\"\"\n    \n    if feature_list is None:\n        # Load only the predefined features\n        feature_list = list(set(\n            Config.CORE_FEATURES + \n            Config.FEATURE_POOL_1 + \n            Config.FEATURE_POOL_2 + \n            Config.FEATURE_POOL_3 + \n            Config.FEATURE_POOL_4\n        ))\n    \n    # Always include base features\n    base_features = [\"bid_qty\", \"ask_qty\", \"buy_qty\", \"sell_qty\", \"volume\"]\n    feature_list = list(set(feature_list + base_features))\n    \n    print(f\"Loading {len(feature_list)} features...\")\n    \n    train_df = pd.read_parquet(Config.TRAIN_PATH, columns=feature_list + [Config.LABEL_COLUMN])\n    test_df = pd.read_parquet(Config.TEST_PATH, columns=feature_list)\n    submission_df = pd.read_csv(Config.SUBMISSION_PATH)\n    \n    print(f\"Loaded data - Train: {train_df.shape}, Test: {test_df.shape}\")\n    \n    # Add engineered features\n    train_df = add_essential_features(train_df)\n    test_df = add_essential_features(test_df)\n    \n    # Update Config.FEATURES with engineered features\n    engineered_features = [\n        col for col in train_df.columns \n        if col not in feature_list and col != Config.LABEL_COLUMN\n    ]\n    \n    Config.FEATURES = feature_list + engineered_features\n    print(f\"Total features after engineering: {len(Config.FEATURES)}\")\n    \n    return train_df.reset_index(drop=True), test_df.reset_index(drop=True), submission_df\n\ndef get_model_slices(n_samples: int):\n    \"\"\"Get model training slices\"\"\"\n    base_slices = [\n        {\"name\": \"full_data\", \"cutoff\": 0, \"is_oldest\": False, \"outlier_adjusted\": False},\n        {\"name\": \"last_90pct\", \"cutoff\": int(0.10 * n_samples), \"is_oldest\": False, \"outlier_adjusted\": False},\n        {\"name\": \"last_85pct\", \"cutoff\": int(0.15 * n_samples), \"is_oldest\": False, \"outlier_adjusted\": False},\n        {\"name\": \"last_80pct\", \"cutoff\": int(0.20 * n_samples), \"is_oldest\": False, \"outlier_adjusted\": False},\n        {\"name\": \"oldest_25pct\", \"cutoff\": int(0.25 * n_samples), \"is_oldest\": True, \"outlier_adjusted\": False},\n    ]\n    \n    outlier_adjusted_slices = []\n    for slice_info in base_slices:\n        adjusted_slice = slice_info.copy()\n        adjusted_slice[\"name\"] = f\"{slice_info['name']}_outlier_adj\"\n        adjusted_slice[\"outlier_adjusted\"] = True\n        outlier_adjusted_slices.append(adjusted_slice)\n    \n    return base_slices + outlier_adjusted_slices\n\ndef save_predictions(model_name: str, predictions: np.ndarray, metadata: dict = None):\n    \"\"\"Save model predictions with metadata\"\"\"\n    pred_path = PREDICTIONS_DIR / f\"{model_name}_predictions.npy\"\n    np.save(pred_path, predictions)\n    \n    # Save metadata\n    meta = {\n        \"model_name\": model_name,\n        \"shape\": list(predictions.shape),\n        \"stats\": {\n            \"mean\": float(np.mean(predictions)),\n            \"std\": float(np.std(predictions)),\n            \"min\": float(np.min(predictions)),\n            \"max\": float(np.max(predictions)),\n            \"skew\": float(skew(predictions)),\n            \"kurtosis\": float(kurtosis(predictions))\n        },\n        \"saved_at\": datetime.datetime.now().isoformat()\n    }\n    \n    if metadata:\n        # Convert all numpy types to Python types in metadata\n        for key, value in metadata.items():\n            if isinstance(value, np.ndarray):\n                metadata[key] = value.tolist()\n            elif isinstance(value, (np.float32, np.float64)):\n                metadata[key] = float(value)\n            elif isinstance(value, (np.int32, np.int64)):\n                metadata[key] = int(value)\n        meta.update(metadata)\n        \n    meta_path = pred_path.with_suffix('.json')\n    with open(meta_path, 'w') as f:\n        json.dump(meta, f, indent=4)\n        \n    print(f\"💾 Saved {model_name} predictions to {pred_path}\")\n    print(f\"   Mean: {meta['stats']['mean']:.6f}, Std: {meta['stats']['std']:.6f}\")\n\n# =========================\n# Deep Learning Components\n# =========================\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(f\"Using device: {device}\")\n\ndef get_activation_function(name):\n    \"\"\"Return the activation function based on the name.\"\"\"\n    if name == None:\n        return None\n    name = name.lower()\n    if name == 'relu':\n        return nn.ReLU()\n    elif name == 'tanh':\n        return nn.Tanh()\n    elif name == 'sigmoid':\n        return nn.Sigmoid()\n    elif name == 'gelu':\n        return nn.GELU()\n    elif name == 'selu':\n        return nn.SELU()\n    elif name == 'elu':\n        return nn.ELU()\n    else:\n        raise ValueError(f\"Unsupported activation function: {name}\")\n\ndef get_dataloaders(X, Y, hparams, device, shuffle=True):\n    \"\"\"Create DataLoader for training and validation datasets.\"\"\"\n    X_tensor = torch.tensor(X, dtype=torch.float32, device=device)\n    if Y is not None:\n        Y_tensor = torch.tensor(Y.values if hasattr(Y, 'values') else Y, \n                                dtype=torch.float32, device=device).unsqueeze(1)\n        dataset = TensorDataset(X_tensor, Y_tensor)\n    else:\n        dataset = TensorDataset(X_tensor)\n    \n    dataloader = DataLoader(dataset, batch_size=hparams[\"batch_size\"], shuffle=shuffle, \n                            generator=torch.Generator().manual_seed(hparams[\"seed\"]))\n    return dataloader\n\n# Enhanced MLP with multiple regularization strategies\nclass RegularizedMLP(nn.Module):\n    def __init__(self, dropout_rate=0.6, dropout_strategy='standard',\n                 layers=[128, 64], activation='relu', last_activation=None,\n                 use_batch_norm=True, use_layer_norm=False, \n                 use_weight_norm=False, use_spectral_norm=False):\n        super(RegularizedMLP, self).__init__()\n        \n        self.linears = nn.ModuleList()\n        self.norms = nn.ModuleList()\n        self.activation = get_activation_function(activation)\n        self.last_activation = get_activation_function(last_activation)\n        self.dropout_strategy = dropout_strategy\n        self.use_batch_norm = use_batch_norm\n        self.use_layer_norm = use_layer_norm\n\n        for i in range(len(layers) - 1):\n            linear = nn.Linear(layers[i], layers[i + 1])\n            \n            # Apply weight normalization strategies\n            if use_weight_norm:\n                linear = nn.utils.weight_norm(linear)\n            elif use_spectral_norm:\n                linear = nn.utils.spectral_norm(linear)\n                \n            self.linears.append(linear)\n            \n            # Add normalization layers\n            if i < len(layers) - 2:  # Not on the last layer\n                if use_batch_norm:\n                    self.norms.append(nn.BatchNorm1d(layers[i + 1]))\n                elif use_layer_norm:\n                    self.norms.append(nn.LayerNorm(layers[i + 1]))\n                else:\n                    self.norms.append(None)\n\n        # Different dropout strategies\n        if dropout_strategy == 'standard':\n            self.dropout = nn.Dropout(dropout_rate)\n        elif dropout_strategy == 'alpha':\n            self.dropout = nn.AlphaDropout(dropout_rate)\n        elif dropout_strategy == 'gaussian':\n            self.dropout = lambda x: x + torch.randn_like(x) * dropout_rate\n        else:\n            self.dropout = nn.Dropout(dropout_rate)\n\n    def forward(self, x):\n        for k in range(len(self.linears) - 1):\n            x = self.linears[k](x)\n            if k < len(self.norms) and self.norms[k] is not None:\n                x = self.norms[k](x)\n            x = self.activation(x)\n            x = self.dropout(x)\n            \n        x = self.linears[-1](x)\n        if self.last_activation is not None:\n            x = self.last_activation(x)\n        return x\n\n# =========================\n# XGBoost Training\n# =========================\ndef train_xgboost(train_df, test_df):\n    \"\"\"Train XGBoost with multiple slices\"\"\"\n    print(\"\\n=== Training XGBoost Model ===\")\n    \n    n_samples = len(train_df)\n    model_slices = get_model_slices(n_samples)\n\n    oof_preds = {s[\"name\"]: np.zeros(n_samples) for s in model_slices}\n    test_preds = {s[\"name\"]: np.zeros(len(test_df)) for s in model_slices}\n\n    full_weights = create_time_decay_weights(n_samples)\n    kf = KFold(n_splits=Config.N_FOLDS, shuffle=False)\n\n    for fold, (train_idx, valid_idx) in enumerate(kf.split(train_df), start=1):\n        print(f\"\\n--- Fold {fold}/{Config.N_FOLDS} ---\")\n        X_valid = train_df.iloc[valid_idx][Config.FEATURES]\n        y_valid = train_df.iloc[valid_idx][Config.LABEL_COLUMN]\n\n        for s in model_slices:\n            cutoff = s[\"cutoff\"]\n            slice_name = s[\"name\"]\n            is_oldest = s[\"is_oldest\"]\n            outlier_adjusted = s[\"outlier_adjusted\"]\n            \n            if is_oldest:\n                subset = train_df.iloc[:cutoff].reset_index(drop=True)\n                rel_idx = train_idx[train_idx < cutoff]\n                sw = np.ones(len(rel_idx))\n            else:\n                subset = train_df.iloc[cutoff:].reset_index(drop=True)\n                rel_idx = train_idx[train_idx >= cutoff] - cutoff\n                sw = create_time_decay_weights(len(subset))[rel_idx] if cutoff > 0 else full_weights[train_idx]\n\n            X_train = subset.iloc[rel_idx][Config.FEATURES]\n            y_train = subset.iloc[rel_idx][Config.LABEL_COLUMN]\n            \n            if outlier_adjusted and len(X_train) > 100:\n                sw = detect_outliers_and_adjust_weights(\n                    X_train.values, \n                    y_train.values, \n                    sw, \n                    outlier_fraction=Config.OUTLIER_FRACTION\n                )\n\n            print(f\"  Training slice: {slice_name}, samples: {len(X_train)}\")\n\n            # Use fewer trees for memory efficiency if needed\n            xgb_params = XGB_PARAMS.copy()\n            if len(Config.FEATURES) > 200:\n                xgb_params['n_estimators'] = 1000  # Reduce from 1667\n                \n            model = XGBRegressor(**xgb_params)\n            model.fit(X_train, y_train, sample_weight=sw, eval_set=[(X_valid, y_valid)], verbose=False)\n\n            if is_oldest:\n                oof_preds[slice_name][valid_idx] = model.predict(\n                    train_df.iloc[valid_idx][Config.FEATURES]\n                )\n            else:\n                mask = valid_idx >= cutoff\n                if mask.any():\n                    idxs = valid_idx[mask]\n                    oof_preds[slice_name][idxs] = model.predict(\n                        train_df.iloc[idxs][Config.FEATURES]\n                    )\n                if cutoff > 0 and (~mask).any():\n                    base_slice_name = slice_name.replace(\"_outlier_adj\", \"\")\n                    if base_slice_name == slice_name:\n                        fallback_slice = \"full_data\"\n                    else:\n                        fallback_slice = \"full_data_outlier_adj\"\n                    oof_preds[slice_name][valid_idx[~mask]] = oof_preds[fallback_slice][valid_idx[~mask]]\n\n            test_preds[slice_name] += model.predict(test_df[Config.FEATURES])\n\n    # Normalize test predictions\n    for slice_name in test_preds:\n        test_preds[slice_name] /= (Config.N_FOLDS - 1)\n\n    # Create XGBoost ensemble\n    weights = np.array([\n        1.0,   # full_data\n        1.0,   # last_90pct\n        1.0,   # last_85pct\n        1.0,   # last_80pct\n        0.25,  # oldest_25pct\n        0.9,   # full_data_outlier_adj\n        0.9,   # last_90pct_outlier_adj\n        0.9,   # last_85pct_outlier_adj\n        0.9,   # last_80pct_outlier_adj\n        0.2    # oldest_25pct_outlier_adj\n    ])\n    \n    weights = weights / weights.sum()\n\n    oof_weighted = pd.DataFrame(oof_preds).values @ weights\n    test_weighted = pd.DataFrame(test_preds).values @ weights\n    score_weighted = pearsonr(train_df[Config.LABEL_COLUMN], oof_weighted)[0]\n    print(f\"\\nXGBoost Weighted Ensemble Pearson: {score_weighted:.4f}\")\n\n    # Print individual slice scores\n    print(\"\\nIndividual slice OOF scores and weights:\")\n    slice_names = list(oof_preds.keys())\n    slice_scores = {}\n    for i, slice_name in enumerate(slice_names):\n        score = pearsonr(train_df[Config.LABEL_COLUMN], oof_preds[slice_name])[0]\n        slice_scores[slice_name] = score\n        print(f\"  {slice_name}: {score:.4f} (weight: {weights[i]:.3f})\")\n\n    # Save predictions and metadata\n    save_predictions(\"xgboost\", test_weighted, {\n        \"slice_scores\": slice_scores,\n        \"ensemble_weights\": weights.tolist(),\n        \"oof_score\": score_weighted\n    })\n    \n    return test_weighted\n\n# =========================\n# MLP Training\n# =========================\ndef train_mlp_single(train_df, test_df, features, model_suffix, hparams, regularization_config):\n    \"\"\"Train a single MLP model with given features and regularization\"\"\"\n    print(f\"\\n--- Training MLP {model_suffix} with {len(features)} features ---\")\n    \n    set_seed(hparams[\"seed\"])\n    \n    # Filter features to only those that exist\n    features = [f for f in features if f in train_df.columns]\n    \n    X_train_full = train_df[features].values\n    y_train_full = train_df[Config.LABEL_COLUMN].values\n    \n    X_train, X_val, y_train, y_val = train_test_split(\n        X_train_full, y_train_full, test_size=0.2, shuffle=False, random_state=42\n    )\n    \n    # Try different scalers\n    if regularization_config.get('scaler_type') == 'quantile':\n        scaler = QuantileTransformer(output_distribution='normal')\n    elif regularization_config.get('scaler_type') == 'robust':\n        scaler = RobustScaler()\n    else:\n        scaler = StandardScaler()\n        \n    X_train = scaler.fit_transform(X_train)\n    X_val = scaler.transform(X_val)\n    X_test = scaler.transform(test_df[features].values)\n    \n    # Update layers for different feature sizes\n    hidden_dims = regularization_config.get('hidden_dims', [256, 128, 64])\n    layers = [len(features)] + hidden_dims + [1]\n    \n    # Create dataloaders\n    train_loader = get_dataloaders(X_train, y_train, hparams, device, shuffle=True)\n    val_loader = get_dataloaders(X_val, y_val, hparams, device, shuffle=False)\n    test_loader = get_dataloaders(X_test, None, hparams, device, shuffle=False)\n    \n    model = RegularizedMLP(\n        layers=layers,\n        dropout_rate=regularization_config.get('dropout_rate', 0.5),\n        dropout_strategy=regularization_config.get('dropout_strategy', 'standard'),\n        activation=hparams[\"activation\"],\n        last_activation=hparams[\"hidden_activation\"],\n        use_batch_norm=regularization_config.get('use_batch_norm', True),\n        use_layer_norm=regularization_config.get('use_layer_norm', False),\n        use_weight_norm=regularization_config.get('use_weight_norm', False),\n        use_spectral_norm=regularization_config.get('use_spectral_norm', False),\n    ).to(device)\n    \n    criterion = nn.HuberLoss(delta=hparams[\"delta\"], reduction='mean')\n    \n    # Different optimizers\n    if regularization_config.get('optimizer') == 'sgd':\n        optimizer = optim.SGD(model.parameters(), lr=hparams[\"learning_rate\"], \n                             momentum=0.9, weight_decay=hparams[\"weight_decay\"])\n    elif regularization_config.get('optimizer') == 'adamw':\n        optimizer = optim.AdamW(model.parameters(), lr=hparams[\"learning_rate\"], \n                               weight_decay=hparams[\"weight_decay\"])\n    else:\n        optimizer = optim.Adam(model.parameters(), lr=hparams[\"learning_rate\"], \n                              weight_decay=hparams[\"weight_decay\"])\n    \n    # Learning rate scheduler\n    scheduler = ReduceLROnPlateau(optimizer, mode='max', factor=0.5, patience=3)\n    \n    checkpoint_path = CHECKPOINTS_DIR / f\"best_mlp_{model_suffix}_model.pt\"\n    best_pearson = -np.inf\n    \n    # Training loop\n    num_epochs = hparams[\"num_epochs\"]\n    \n    for epoch in range(num_epochs):\n        model.train()\n        running_loss = 0.0\n\n        for inputs, targets in tqdm(train_loader, desc=f\"Epoch {epoch+1}/{num_epochs}\"):\n            inputs, targets = inputs.to(device), targets.to(device)\n            \n            # Dynamic noise injection\n            noise_factor = hparams[\"noise_factor\"] * (1 - epoch / num_epochs)\n            inputs = inputs + torch.randn_like(inputs) * noise_factor\n            \n            optimizer.zero_grad()\n            outputs = model(inputs)\n            loss = criterion(outputs, targets)\n            \n            # Add L1 regularization if specified\n            if regularization_config.get('l1_lambda', 0) > 0:\n                l1_loss = sum(p.abs().sum() for p in model.parameters())\n                loss = loss + regularization_config['l1_lambda'] * l1_loss\n            \n            loss.backward()\n            \n            # Gradient clipping\n            if regularization_config.get('grad_clip', 0) > 0:\n                torch.nn.utils.clip_grad_norm_(model.parameters(), regularization_config['grad_clip'])\n            \n            optimizer.step()\n            \n            running_loss += loss.item() * inputs.size(0)\n            \n        running_loss = running_loss / len(train_loader.dataset)\n        print(f\"Training Loss: {running_loss:.4f}\")\n\n        # Validation\n        model.eval()\n        val_loss = 0.0\n        preds = []\n        trues = []\n        with torch.no_grad():\n            for inputs, targets in val_loader:\n                inputs, targets = inputs.to(device), targets.to(device)\n                outputs = model(inputs)\n                loss = criterion(outputs, targets)\n                val_loss += loss.item() * inputs.size(0)\n                preds.append(outputs.cpu().numpy())\n                trues.append(targets.cpu().numpy())\n\n        val_loss /= len(val_loader.dataset)\n        preds = np.concatenate(preds).flatten()\n        trues = np.concatenate(trues).flatten()\n        pearson_coef = pearsonr(preds, trues)[0]\n        print(f\"Validation Pearson Coef: {pearson_coef:.4f} | Loss: {val_loss:.4f}\")\n        \n        scheduler.step(pearson_coef)\n\n        if pearson_coef > best_pearson:\n            best_pearson = pearson_coef\n            torch.save(model.state_dict(), checkpoint_path)\n            print(f\"✅ New best model saved with Pearson: {best_pearson:.4f}\")\n    \n    # Load best model and make predictions\n    model.load_state_dict(torch.load(checkpoint_path))\n    \n    model.eval()\n    predictions = []\n    with torch.no_grad():\n        for inputs in test_loader:\n            inputs = inputs[0].to(device)\n            outputs = model(inputs)\n            predictions.append(outputs.cpu().numpy())\n\n    predictions = np.concatenate(predictions).flatten()\n    \n    # Save predictions\n    save_predictions(f\"mlp_{model_suffix}\", predictions, {\n        \"best_val_pearson\": best_pearson,\n        \"n_epochs_trained\": num_epochs,\n        \"n_features\": len(features),\n        \"regularization\": regularization_config\n    })\n    \n    return predictions\n\ndef train_mlp(train_df, test_df):\n    \"\"\"Train MLP models with multiple feature sets and regularization strategies\"\"\"\n    print(\"\\n=== Training MLP Models with Multiple Feature Sets and Regularization ===\")\n    \n    # Base hyperparameters\n    base_hparams = {\n        \"seed\": 42,\n        \"num_epochs\": 10,\n        \"batch_size\": 2048,\n        \"learning_rate\": 0.001,\n        \"weight_decay\": 1e-4,\n        \"dropout_rate\": 0.5,\n        \"hidden_activation\": None,\n        \"activation\": \"relu\",\n        \"delta\": 5,\n        \"noise_factor\": 0.005\n    }\n    \n    # Different regularization configurations\n    regularization_configs = {\n        \"standard\": {\n            \"dropout_rate\": 0.5,\n            \"dropout_strategy\": \"standard\",\n            \"use_batch_norm\": True,\n            \"hidden_dims\": [256, 128, 64],\n            \"optimizer\": \"adam\",\n            \"scaler_type\": \"standard\",\n            \"grad_clip\": 1.0\n        },\n        \"heavy_reg\": {\n            \"dropout_rate\": 0.7,\n            \"dropout_strategy\": \"standard\",\n            \"use_batch_norm\": True,\n            \"hidden_dims\": [128, 64, 32],\n            \"optimizer\": \"adamw\",\n            \"scaler_type\": \"robust\",\n            \"l1_lambda\": 1e-5,\n            \"grad_clip\": 0.5\n        }\n    }\n    \n    feature_variations = create_feature_variations()\n    \n    # For large feature sets, we need to be more careful\n    if feature_variations[\"top_150\"] is None:\n        # Select top features using the memory-efficient method\n        feature_variations[\"top_150\"] = select_top_features_memory_efficient(\n            Config.TRAIN_PATH, 150, Config.FEATURE_SELECTION_SAMPLE_SIZE\n        )\n        feature_variations[\"top_200\"] = select_top_features_memory_efficient(\n            Config.TRAIN_PATH, 200, Config.FEATURE_SELECTION_SAMPLE_SIZE\n        )\n        feature_variations[\"top_300\"] = select_top_features_memory_efficient(\n            Config.TRAIN_PATH, 300, Config.FEATURE_SELECTION_SAMPLE_SIZE\n        )\n    \n    all_predictions = {}\n    \n    # Train models with different feature sets\n    for variant_name, features in feature_variations.items():\n        if variant_name == \"original\":\n            features = Config.MLP_FEATURES_ORIGINAL\n        \n        # Skip if features not available in current dataframe\n        if not all(f in train_df.columns for f in features):\n            print(f\"Skipping {variant_name} - features not available in current data\")\n            continue\n        \n        # For larger feature sets, use heavier regularization\n        if len(features) > 150:\n            reg_config = regularization_configs[\"heavy_reg\"]\n        else:\n            reg_config = regularization_configs[\"standard\"]\n            \n        predictions = train_mlp_single(\n            train_df, test_df, features, \n            f\"{variant_name}_reg\", \n            base_hparams, reg_config\n        )\n        all_predictions[variant_name] = predictions\n    \n    return all_predictions\n\n# =========================\n# Advanced Ensemble Creation\n# =========================\ndef create_advanced_ensembles(submission_df, train_df):\n    \"\"\"Create multiple ensemble submissions with different strategies\"\"\"\n    print(\"\\n=== Creating Advanced Ensemble Submissions ===\")\n    \n    # Collect available model predictions\n    predictions = {}\n    model_scores = {}\n    \n    pred_files = list(PREDICTIONS_DIR.glob(\"*_predictions.npy\"))\n    \n    for pred_file in pred_files:\n        model_name = pred_file.stem.replace(\"_predictions\", \"\")\n        predictions[model_name] = np.load(pred_file)\n        \n        # Load metadata\n        meta_file = pred_file.with_suffix('.json')\n        if meta_file.exists():\n            with open(meta_file, 'r') as f:\n                meta = json.load(f)\n                if 'best_val_pearson' in meta:\n                    model_scores[model_name] = meta['best_val_pearson']\n                elif 'oof_score' in meta:\n                    model_scores[model_name] = meta['oof_score']\n                else:\n                    model_scores[model_name] = 0.05\n        \n        print(f\"✓ Loaded {model_name} predictions (score: {model_scores.get(model_name, 'N/A'):.4f})\")\n    \n    # Create different ensemble strategies\n    ensemble_configs = {\n        \"xgb_dominant_80\": {\n            \"description\": \"XGBoost 80% weight, others split equally\",\n            \"weights\": lambda models: {\n                m: 0.8 if m == \"xgboost\" else 0.2 / (len(models) - 1)\n                for m in models\n            }\n        },\n        \n        \"xgb_dominant_85\": {\n            \"description\": \"XGBoost 85% weight, others split equally\",\n            \"weights\": lambda models: {\n                m: 0.85 if m == \"xgboost\" else 0.15 / (len(models) - 1)\n                for m in models\n            }\n        },\n        \n        \"xgb_dominant_90\": {\n            \"description\": \"XGBoost 90% weight, others split equally\",\n            \"weights\": lambda models: {\n                m: 0.9 if m == \"xgboost\" else 0.1 / (len(models) - 1)\n                for m in models\n            }\n        },\n        \n        \"performance_weighted\": {\n            \"description\": \"Weights based on validation performance\",\n            \"weights\": lambda models: {\n                m: max(0.01, model_scores.get(m, 0.05))\n                for m in models\n            }\n        },\n        \n        \"top_models_only\": {\n            \"description\": \"Only use models with score > 0.08\",\n            \"weights\": lambda models: {\n                m: 1.0 if model_scores.get(m, 0) > 0.08 else 0.0\n                for m in models\n            }\n        },\n        \n        \"stable_models\": {\n            \"description\": \"Favor models with lower std deviation\",\n            \"weights\": lambda models: {\n                m: 1.0 / (np.std(predictions[m]) + 0.1)\n                for m in models\n            }\n        }\n    }\n    \n    submissions = {}\n    \n    for config_name, config in ensemble_configs.items():\n        print(f\"\\n📊 Creating ensemble: {config['description']}\")\n        \n        # Get weights\n        weights = config['weights'](list(predictions.keys()))\n        \n        # Normalize weights\n        total_weight = sum(weights.values())\n        if total_weight > 0:\n            weights = {k: v/total_weight for k, v in weights.items() if v > 0}\n        else:\n            # Fallback to equal weights\n            weights = {k: 1.0/len(predictions) for k in predictions.keys()}\n        \n        # Create ensemble\n        ensemble_pred = np.zeros_like(list(predictions.values())[0])\n        \n        for model_name, weight in weights.items():\n            if weight > 0 and model_name in predictions:\n                ensemble_pred += weight * predictions[model_name]\n        \n        # Post-processing\n        if config_name.startswith(\"xgb_dominant\"):\n            # Minimal post-processing for XGB-heavy ensembles\n            ensemble_pred = np.clip(ensemble_pred, \n                                   np.percentile(ensemble_pred, 0.1),\n                                   np.percentile(ensemble_pred, 99.9))\n        else:\n            # Light post-processing for other ensembles\n            mean_pred = ensemble_pred.mean()\n            std_pred = ensemble_pred.std()\n            ensemble_pred = np.clip(ensemble_pred, mean_pred - 4*std_pred, mean_pred + 4*std_pred)\n        \n        # Create submission\n        submission = submission_df.copy()\n        submission[\"prediction\"] = ensemble_pred\n        \n        # Save submission\n        submission_path = SUBMISSIONS_DIR / f\"ensemble_{config_name}.csv\"\n        submission.to_csv(submission_path, index=False)\n        submissions[config_name] = submission_path\n        \n        # Print statistics\n        print(f\"   Mean: {ensemble_pred.mean():.6f}, Std: {ensemble_pred.std():.6f}\")\n        print(f\"   Min: {ensemble_pred.min():.6f}, Max: {ensemble_pred.max():.6f}\")\n        \n        # Show top weighted models\n        sorted_weights = sorted(weights.items(), key=lambda x: x[1], reverse=True)[:5]\n        print(\"   Top weights:\")\n        for model_name, weight in sorted_weights:\n            print(f\"     {model_name}: {weight:.1%}\")\n    \n    return submissions\n\n# =========================\n# Main Execution\n# =========================\ndef main():\n    \"\"\"Main execution function\"\"\"\n    \n    # Load data with memory-efficient approach\n    print(\"\\n=== Loading Data (Memory Efficient) ===\")\n    train_df, test_df, submission_df = load_data_memory_efficient()\n    \n    # Train XGBoost\n    xgb_predictions = train_xgboost(train_df, test_df)\n    \n    # Clean up memory before training neural networks\n    gc.collect()\n    \n    # Train MLP models\n    mlp_predictions = train_mlp(train_df, test_df)\n    \n    # Create multiple ensemble submissions\n    ensemble_paths = create_advanced_ensembles(submission_df, train_df)\n    \n    # Print summary\n    print(\"\\n\" + \"=\"*60)\n    print(\"EXECUTION SUMMARY\")\n    print(\"=\"*60)\n    \n    print(\"\\n✅ Pipeline execution completed successfully!\")\n    print(f\"📊 Models trained:\")\n    print(f\"   - XGBoost: 1 model\")\n    print(f\"   - MLP: {len(mlp_predictions)} variations\")\n    print(f\"\\n📈 Ensemble submissions created:\")\n    for name, path in ensemble_paths.items():\n        print(f\"   - {name}: {path}\")\n\nif __name__ == \"__main__\":\n    main()","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}