{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":96164,"databundleVersionId":12993472,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n\nimport numpy as np\nimport pandas as pd\nimport os\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# List input files\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Install required packages\nprint(\"Installing required packages...\")\nos.system('pip install koolbox scikit-learn==1.5.2 prophet catboost --quiet')\n\n# Import all required libraries\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom prophet import Prophet\nfrom sklearn.model_selection import KFold, TimeSeriesSplit\nfrom sklearn.preprocessing import RobustScaler, StandardScaler, QuantileTransformer\nfrom sklearn.isotonic import IsotonicRegression\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom scipy.stats import pearsonr, rankdata\nfrom statsmodels.robust.scale import mad\nfrom sklearn.base import BaseEstimator, RegressorMixin, TransformerMixin\nfrom koolbox import Trainer\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, TensorDataset\nimport tensorflow as tf\nimport joblib\nimport gc\nimport logging\nlogging.getLogger('prophet').setLevel(logging.WARNING)\n\nprint(\"Starting Comprehensive DRW Crypto Prediction Pipeline...\")\n\n# Configuration\nclass CFG:\n    train_path = \"/kaggle/input/drw-crypto-market-prediction/train.parquet\"\n    test_path = \"/kaggle/input/drw-crypto-market-prediction/test.parquet\"\n    sample_sub_path = \"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\"\n    \n    target = \"label\"\n    n_folds = 5\n    seed = 42\n    random_state = 42\n\n# Define features\nX_FEATURES = ['X363', 'X321', 'X405', 'X730', 'X523', 'X756', 'X589', 'X462', 'X779',\n              'X25', 'X532', 'X520', 'X329', 'X383', 'X751', 'X535', 'X639', 'X596', 'X761',\n              'X752', 'X287', 'X298', 'X759', 'X302', 'X55', 'X56', 'X52', 'X303', 'X51',\n              'X598', 'X385', 'X603', 'X674', 'X415', 'X345', 'X174', 'X178', 'X168', 'X612',\n              'bid_qty', 'ask_qty', 'buy_qty', 'sell_qty']\n\nSELECTED_COLUMNS = X_FEATURES + ['volume']\n\n# Memory reduction function\ndef reduce_mem_usage(dataframe, dataset):\n    print(f'Reducing memory usage for: {dataset}')\n    initial_mem_usage = dataframe.memory_usage().sum() / 1024**2\n    \n    for col in dataframe.columns:\n        col_type = dataframe[col].dtype\n        \n        if col_type != 'object':\n            c_min = dataframe[col].min()\n            c_max = dataframe[col].max()\n            \n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    dataframe[col] = dataframe[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    dataframe[col] = dataframe[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    dataframe[col] = dataframe[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    dataframe[col] = dataframe[col].astype(np.int64)\n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    dataframe[col] = dataframe[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    dataframe[col] = dataframe[col].astype(np.float32)\n                else:\n                    dataframe[col] = dataframe[col].astype(np.float64)\n\n    final_mem_usage = dataframe.memory_usage().sum() / 1024**2\n    print(f'--- Memory usage before: {initial_mem_usage:.2f} MB')\n    print(f'--- Memory usage after: {final_mem_usage:.2f} MB')\n    print(f'--- Decreased memory usage by {100 * (initial_mem_usage - final_mem_usage) / initial_mem_usage:.1f}%\\n')\n    \n    return dataframe\n\n# Prophet-based outlier detection\ndef simple_prophet_outlier_detection(df, feature_col, timestamp_col='__index_level_0__'):\n    \"\"\"Use Prophet to detect outliers in a single feature\"\"\"\n    print(f\"Detecting outliers in {feature_col} using Prophet...\")\n    \n    # Prepare data for Prophet\n    prophet_df = pd.DataFrame({\n        'ds': df[timestamp_col],\n        'y': df[feature_col]\n    })\n    \n    # Remove obvious bad values\n    prophet_df = prophet_df[np.isfinite(prophet_df['y'])]\n    \n    # Resample to hourly for efficiency\n    prophet_hourly = prophet_df.set_index('ds').resample('1H').mean().reset_index()\n    prophet_hourly = prophet_hourly.dropna()\n    \n    if len(prophet_hourly) < 100:\n        print(f\"    Insufficient data for {feature_col}\")\n        return None\n    \n    try:\n        # Fit Prophet model\n        model = Prophet(\n            changepoint_prior_scale=0.05,\n            interval_width=0.95,\n            yearly_seasonality=False,\n            weekly_seasonality=True,\n            daily_seasonality=True\n        )\n        model.fit(prophet_hourly)\n        \n        # Generate predictions\n        forecast = model.predict(prophet_hourly)\n        \n        # Identify outliers\n        outliers_ci = (\n            (prophet_hourly['y'] < forecast['yhat_lower']) | \n            (prophet_hourly['y'] > forecast['yhat_upper'])\n        )\n        \n        residuals = prophet_hourly['y'] - forecast['yhat']\n        residual_std = residuals.std()\n        outliers_residual = np.abs(residuals) > 3 * residual_std\n        \n        # Combine both methods\n        outliers = outliers_ci & outliers_residual\n        \n        outlier_info = {\n            'n_outliers': outliers.sum(),\n            'outlier_pct': outliers.sum() / len(prophet_hourly) * 100,\n            'outlier_timestamps': prophet_hourly[outliers]['ds'].tolist(),\n            'outlier_values': prophet_hourly[outliers]['y'].tolist(),\n            'residual_std': residual_std\n        }\n        \n        print(f\"Found {outlier_info['n_outliers']} outliers ({outlier_info['outlier_pct']:.2f}%)\")\n        \n        return outlier_info\n        \n    except Exception as e:\n        print(f\"    Prophet failed: {str(e)}\")\n        return None\n\ndef detect_outliers_multiple_features(df, features_to_check=None):\n    \"\"\"Run outlier detection on multiple features\"\"\"\n    if features_to_check is None:\n        features_to_check = ['volume', 'bid_qty', 'ask_qty', 'buy_qty', 'sell_qty']\n    \n    # Ensure timestamp\n    if '__index_level_0__' not in df.columns:\n        df['__index_level_0__'] = pd.date_range('2023-03-01', periods=len(df), freq='T')\n    \n    outlier_summary = {}\n    \n    for feature in features_to_check:\n        if feature in df.columns:\n            try:\n                print(f\"\\n{'='*60}\")\n                outlier_info = simple_prophet_outlier_detection(df, feature)\n                if outlier_info:\n                    outlier_summary[feature] = outlier_info\n                else:\n                    # Create dummy outlier info if Prophet fails\n                    outlier_summary[feature] = {\n                        'n_outliers': 0,\n                        'outlier_pct': 0,\n                        'outlier_timestamps': [],\n                        'outlier_values': [],\n                        'residual_std': 1.0\n                    }\n            except Exception as e:\n                print(f\"Error processing {feature}: {str(e)}\")\n                outlier_summary[feature] = {\n                    'n_outliers': 0,\n                    'outlier_pct': 0,\n                    'outlier_timestamps': [],\n                    'outlier_values': [],\n                    'residual_std': 1.0\n                }\n    \n    return outlier_summary\n\ndef clean_feature_outliers(df, feature_col, outlier_info, method='cap', lower_percentile=1, upper_percentile=99):\n    \"\"\"Clean outliers from a feature using different methods\"\"\"\n    outlier_timestamps = outlier_info['outlier_timestamps']\n    df_clean = df.copy()\n    \n    if method == 'cap':\n        # Cap outliers at specified percentiles\n        lower_cap = df_clean[feature_col].quantile(lower_percentile / 100)\n        upper_cap = df_clean[feature_col].quantile(upper_percentile / 100)\n        \n        df_clean[feature_col] = df_clean[feature_col].clip(lower=lower_cap, upper=upper_cap)\n        print(f\"Capped {feature_col} to range [{lower_cap:.2f}, {upper_cap:.2f}] (percentiles: {lower_percentile}th-{upper_percentile}th)\")\n    \n    return df_clean\n\n# Advanced Label Compressor (optimized version based on results)\nclass OptimizedLabelCompressor(BaseEstimator, TransformerMixin):\n    \"\"\"IQR-based label compression with optimal parameters\"\"\"\n    \n    def __init__(self, strategy='iqr', params=None):\n        self.strategy = strategy\n        self.params = params or {}\n        self.fitted = False\n        \n    def fit(self, y):\n        \"\"\"Fit the compression parameters\"\"\"\n        if self.strategy == 'iqr':\n            self.q1 = np.percentile(y, 25)\n            self.q3 = np.percentile(y, 75)\n            self.iqr = self.q3 - self.q1\n            self.whisker_width = self.params.get('whisker_width', 2.98)\n        elif self.strategy == 'percentile':\n            self.lower_percentile = self.params.get('lower_percentile', 5)\n            self.upper_percentile = self.params.get('upper_percentile', 95)\n            self.lower_bound = np.percentile(y, self.lower_percentile)\n            self.upper_bound = np.percentile(y, self.upper_percentile)\n        elif self.strategy == 'mad':\n            self.median = np.median(y)\n            self.mad = mad(y)\n            self.n_mads = self.params.get('n_mads', 3)\n        \n        self.fitted = True\n        return self\n    \n    def transform(self, y):\n        \"\"\"Apply compression to labels\"\"\"\n        if not self.fitted:\n            raise ValueError(\"Must fit before transform\")\n        \n        if self.strategy == 'iqr':\n            lower_bound = self.q1 - self.whisker_width * self.iqr\n            upper_bound = self.q3 + self.whisker_width * self.iqr\n            \n            if self.params.get('soft_clip', False):\n                # Soft clipping\n                y_compressed = y.copy()\n                \n                # Lower outliers\n                mask_lower = y < lower_bound\n                if np.any(mask_lower):\n                    dist = lower_bound - y[mask_lower]\n                    weight = np.exp(-dist / self.iqr)\n                    y_compressed[mask_lower] = weight * y[mask_lower] + (1 - weight) * lower_bound\n                \n                # Upper outliers\n                mask_upper = y > upper_bound\n                if np.any(mask_upper):\n                    dist = y[mask_upper] - upper_bound\n                    weight = np.exp(-dist / self.iqr)\n                    y_compressed[mask_upper] = weight * y[mask_upper] + (1 - weight) * upper_bound\n                \n                return y_compressed\n            else:\n                # Hard clipping\n                return np.clip(y, lower_bound, upper_bound)\n        \n        elif self.strategy == 'percentile':\n            return np.clip(y, self.lower_bound, self.upper_bound)\n        \n        elif self.strategy == 'mad':\n            z_scores = (y - self.median) / (self.mad + 1e-8)\n            compressed_z = self.n_mads * np.tanh(z_scores / self.n_mads)\n            return self.median + compressed_z * self.mad\n    \n    def fit_transform(self, y):\n        \"\"\"Fit and transform in one step\"\"\"\n        return self.fit(y).transform(y)\n\n# Pearson correlation metric\ndef _pearsonr(y_true, y_pred):\n    return pearsonr(y_true, y_pred)[0]\n\n# Model parameters (from best results)\nlgbm_params = {\n    \"boosting_type\": \"gbdt\",\n    \"colsample_bytree\": 0.5625888953382505,\n    \"learning_rate\": 0.029312951475451557,\n    \"min_child_samples\": 63,\n    \"min_child_weight\": 0.11456572852335424,\n    \"n_estimators\": 126,\n    \"n_jobs\": -1,\n    \"num_leaves\": 37,\n    \"random_state\": 42,\n    \"reg_alpha\": 85.2476527854083,\n    \"reg_lambda\": 99.38305361388907,\n    \"subsample\": 0.450669817684892,\n    \"verbose\": -1\n}\n\nlgbm_goss_params = {\n    \"boosting_type\": \"goss\",\n    \"colsample_bytree\": 0.34695458228489784,\n    \"learning_rate\": 0.031023014900595287,\n    \"min_child_samples\": 30,\n    \"min_child_weight\": 0.4727729225033618,\n    \"n_estimators\": 220,\n    \"n_jobs\": -1,\n    \"num_leaves\": 58,\n    \"random_state\": 42,\n    \"reg_alpha\": 38.665994901468224,\n    \"reg_lambda\": 92.76991677464294,\n    \"subsample\": 0.4810891284493255,\n    \"verbose\": -1\n}\n\nxgb_params = {\n    \"colsample_bylevel\": 0.4778015829774066,\n    \"colsample_bynode\": 0.362764358742407,\n    \"colsample_bytree\": 0.7107423488010493,\n    \"gamma\": 1.7094857725240398,\n    \"learning_rate\": 0.02213323588455387,\n    \"max_depth\": 20,\n    \"max_leaves\": 12,\n    \"min_child_weight\": 16,\n    \"n_estimators\": 1667,\n    \"n_jobs\": -1,\n    \"random_state\": 42,\n    \"reg_alpha\": 39.352415706891264,\n    \"reg_lambda\": 75.44843704068275,\n    \"subsample\": 0.06566669853471274,\n    \"verbosity\": 0\n}\n\n# GANDALF Model Implementation\nclass GANDALF(BaseEstimator, RegressorMixin):\n    def __init__(self, n_estimators=100, learning_rate=0.01, max_depth=5, \n                 feature_fraction=0.8, bagging_fraction=0.8, lambda_reg=1.0,\n                 min_data_in_leaf=20, num_iterations=100, random_state=42):\n        self.n_estimators = n_estimators\n        self.learning_rate = learning_rate\n        self.max_depth = max_depth\n        self.feature_fraction = feature_fraction\n        self.bagging_fraction = bagging_fraction\n        self.lambda_reg = lambda_reg\n        self.min_data_in_leaf = min_data_in_leaf\n        self.num_iterations = num_iterations\n        self.random_state = random_state\n        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n        \n    def _build_network(self, input_dim):\n        \"\"\"Build the neural network architecture for GANDALF\"\"\"\n        class GANDALFNet(nn.Module):\n            def __init__(self, input_dim, hidden_dims=[256, 128, 64]):\n                super(GANDALFNet, self).__init__()\n                \n                layers = []\n                prev_dim = input_dim\n                \n                for hidden_dim in hidden_dims:\n                    layers.extend([\n                        nn.Linear(prev_dim, hidden_dim),\n                        nn.BatchNorm1d(hidden_dim),\n                        nn.ReLU(),\n                        nn.Dropout(0.3)\n                    ])\n                    prev_dim = hidden_dim\n                \n                layers.append(nn.Linear(prev_dim, 1))\n                \n                self.network = nn.Sequential(*layers)\n                \n            def forward(self, x):\n                return self.network(x)\n        \n        return GANDALFNet(input_dim).to(self.device)\n    \n    def fit(self, X, y):\n        # Convert to tensors\n        X_tensor = torch.FloatTensor(X.values if hasattr(X, 'values') else X).to(self.device)\n        y_tensor = torch.FloatTensor(y.values if hasattr(y, 'values') else y).reshape(-1, 1).to(self.device)\n        \n        # Build network\n        self.model = self._build_network(X_tensor.shape[1])\n        \n        # Create dataset and dataloader\n        dataset = TensorDataset(X_tensor, y_tensor)\n        dataloader = DataLoader(dataset, batch_size=1024, shuffle=True)\n        \n        # Optimizer\n        optimizer = optim.AdamW(self.model.parameters(), lr=self.learning_rate, weight_decay=self.lambda_reg)\n        criterion = nn.MSELoss()\n        \n        # Training loop\n        self.model.train()\n        for epoch in range(self.num_iterations):\n            total_loss = 0\n            for batch_X, batch_y in dataloader:\n                optimizer.zero_grad()\n                outputs = self.model(batch_X)\n                loss = criterion(outputs, batch_y)\n                loss.backward()\n                optimizer.step()\n                total_loss += loss.item()\n            \n            if (epoch + 1) % 20 == 0:\n                print(f'Epoch [{epoch+1}/{self.num_iterations}], Loss: {total_loss/len(dataloader):.4f}')\n        \n        return self\n    \n    def predict(self, X):\n        self.model.eval()\n        X_tensor = torch.FloatTensor(X.values if hasattr(X, 'values') else X).to(self.device)\n        \n        with torch.no_grad():\n            predictions = self.model(X_tensor).cpu().numpy().flatten()\n        \n        return predictions\n\n# AutoEncoder MLP\nclass AutoEncoderMLP(BaseEstimator, RegressorMixin):\n    def __init__(self, num_columns, hidden_units, dropout_rates, lr=1e-3):\n        self.num_columns = num_columns\n        self.hidden_units = hidden_units\n        self.dropout_rates = dropout_rates\n        self.lr = lr\n        self.model = self._build_model()\n    \n    def _build_model(self):\n        inp = tf.keras.layers.Input(shape=(self.num_columns,))\n        x0 = tf.keras.layers.BatchNormalization()(inp)\n\n        encoder = tf.keras.layers.GaussianNoise(self.dropout_rates[0])(x0)\n        encoder = tf.keras.layers.Dense(self.hidden_units[0])(encoder)\n        encoder = tf.keras.layers.BatchNormalization()(encoder)\n        encoder = tf.keras.layers.Activation('swish')(encoder)\n\n        decoder = tf.keras.layers.Dropout(self.dropout_rates[1])(encoder)\n        decoder = tf.keras.layers.Dense(self.num_columns, name='decoder')(decoder)\n\n        x_reg = tf.keras.layers.Dense(self.hidden_units[1])(encoder)\n        x_reg = tf.keras.layers.BatchNormalization()(x_reg)\n        x_reg = tf.keras.layers.Activation('swish')(x_reg)\n        x_reg = tf.keras.layers.Dropout(self.dropout_rates[2])(x_reg)\n\n        out_reg = tf.keras.layers.Dense(1, activation='linear', name='target')(x_reg)\n\n        model = tf.keras.models.Model(inputs=inp, outputs=[decoder, out_reg])\n        model.compile(\n            optimizer=tf.keras.optimizers.Adam(learning_rate=self.lr),\n            loss={\"decoder\": tf.keras.losses.MeanSquaredError(),\n                  \"target\": tf.keras.losses.MeanSquaredError()},\n            loss_weights={\"decoder\": 0.3, \"target\": 1.0}\n        )\n        return model\n\n    def fit(self, X, y):\n        self.model.fit(\n            X, {\"decoder\": X, \"target\": y},\n            epochs=50,\n            batch_size=8192,\n            validation_split=0.2,\n            callbacks=[\n                tf.keras.callbacks.EarlyStopping(patience=10, restore_best_weights=True),\n                tf.keras.callbacks.ReduceLROnPlateau(patience=5)\n            ],\n            verbose=0\n        )\n        return self\n\n    def predict(self, X):\n        _, y_pred = self.model.predict(X, verbose=0)\n        return y_pred.flatten()\n\n# Feature engineering\ndef create_features(df):\n    \"\"\"Create additional features\"\"\"\n    # Spread features\n    df['bid_ask_spread'] = df['ask_qty'] - df['bid_qty']\n    df['buy_sell_imbalance'] = df['buy_qty'] - df['sell_qty']\n    df['bid_ask_ratio'] = df['bid_qty'] / (df['ask_qty'] + 1e-8)\n    df['buy_sell_ratio'] = df['buy_qty'] / (df['sell_qty'] + 1e-8)\n    \n    # Volume features\n    df['volume_per_trade'] = df['volume'] / (df['buy_qty'] + df['sell_qty'] + 1e-8)\n    df['buy_volume_ratio'] = df['buy_qty'] / (df['volume'] + 1e-8)\n    df['sell_volume_ratio'] = df['sell_qty'] / (df['volume'] + 1e-8)\n    \n    # Log transforms for skewed features\n    for col in ['volume', 'bid_qty', 'ask_qty', 'buy_qty', 'sell_qty']:\n        if col in df.columns:\n            df[f'{col}_log'] = np.log1p(df[col])\n    \n    return df\n\ndef run_pipeline_with_outlier_strategy(lower_percentile, upper_percentile, strategy_name):\n    \"\"\"Run the entire pipeline with a specific outlier capping strategy\"\"\"\n    print(f\"\\n{'='*80}\")\n    print(f\"Running pipeline with outlier capping strategy: {strategy_name}\")\n    print(f\"Percentiles: {lower_percentile}th - {upper_percentile}th\")\n    print(f\"{'='*80}\\n\")\n    \n    # Load data\n    train = pd.read_parquet(CFG.train_path).reset_index(drop=True)\n    test = pd.read_parquet(CFG.test_path).reset_index(drop=True)\n    \n    # Select columns\n    train = train[SELECTED_COLUMNS + [CFG.target]]\n    test = test[SELECTED_COLUMNS]\n    \n    # Add timestamp if needed\n    if '__index_level_0__' not in train.columns:\n        train['__index_level_0__'] = pd.date_range('2023-03-01', periods=len(train), freq='T')\n    \n    # Detect outliers\n    print(\"\\nDetecting outliers...\")\n    outlier_summary = detect_outliers_multiple_features(train, X_FEATURES)\n    \n    # Clean data with specified percentiles\n    df_clean = train.copy()\n    print(f\"\\nApplying outlier capping with {lower_percentile}th-{upper_percentile}th percentiles...\")\n    for feat in X_FEATURES:\n        if feat in outlier_summary:\n            df_clean = clean_feature_outliers(\n                df_clean, feat, outlier_summary[feat], \n                method='cap', \n                lower_percentile=lower_percentile, \n                upper_percentile=upper_percentile\n            )\n    \n    # Remove timestamp column if it was added\n    if '__index_level_0__' in df_clean.columns:\n        df_clean = df_clean.drop('__index_level_0__', axis=1)\n    \n    # Add feature engineering\n    df_clean = create_features(df_clean)\n    test = create_features(test)\n    \n    # Reduce memory\n    df_clean = reduce_mem_usage(df_clean, \"train\")\n    test = reduce_mem_usage(test, \"test\")\n    \n    # Prepare data\n    X = df_clean.drop(CFG.target, axis=1)\n    y = df_clean[CFG.target]\n    X_test = test\n    \n    # Apply label compression (optimal IQR-based)\n    label_compressor = OptimizedLabelCompressor(\n        strategy='iqr', \n        params={'whisker_width': 2.98, 'soft_clip': False}\n    )\n    y_compressed = label_compressor.fit_transform(y)\n    \n    # Initialize storage\n    scores = {}\n    oof_preds = {}\n    test_preds = {}\n    \n    # Train LightGBM (gbdt)\n    print(f\"\\nTraining LightGBM (gbdt) for {strategy_name}...\")\n    lgbm_trainer = Trainer(\n        LGBMRegressor(**lgbm_params),\n        cv=KFold(n_splits=5, shuffle=False),\n        metric=_pearsonr,\n        task=\"regression\",\n        metric_precision=6\n    )\n    lgbm_trainer.fit(X, y_compressed)\n    scores[\"LightGBM (gbdt)\"] = lgbm_trainer.fold_scores\n    oof_preds[\"LightGBM (gbdt)\"] = lgbm_trainer.oof_preds\n    test_preds[\"LightGBM (gbdt)\"] = lgbm_trainer.predict(X_test)\n    \n    # Train LightGBM (goss)\n    print(f\"\\nTraining LightGBM (goss) for {strategy_name}...\")\n    lgbm_goss_trainer = Trainer(\n        LGBMRegressor(**lgbm_goss_params),\n        cv=KFold(n_splits=5, shuffle=False),\n        metric=_pearsonr,\n        task=\"regression\",\n        metric_precision=6\n    )\n    lgbm_goss_trainer.fit(X, y_compressed)\n    scores[\"LightGBM (goss)\"] = lgbm_goss_trainer.fold_scores\n    oof_preds[\"LightGBM (goss)\"] = lgbm_goss_trainer.oof_preds\n    test_preds[\"LightGBM (goss)\"] = lgbm_goss_trainer.predict(X_test)\n    \n    # Train XGBoost\n    print(f\"\\nTraining XGBoost for {strategy_name}...\")\n    xgb_trainer = Trainer(\n        XGBRegressor(**xgb_params),\n        cv=KFold(n_splits=5, shuffle=False),\n        metric=_pearsonr,\n        task=\"regression\",\n        metric_precision=6\n    )\n    xgb_trainer.fit(X, y_compressed)\n    scores[\"XGBoost\"] = xgb_trainer.fold_scores\n    oof_preds[\"XGBoost\"] = xgb_trainer.oof_preds\n    test_preds[\"XGBoost\"] = xgb_trainer.predict(X_test)\n    \n    # Train CatBoost\n    print(f\"\\nTraining CatBoost for {strategy_name}...\")\n    cat_model = CatBoostRegressor(\n        iterations=800,\n        depth=10,\n        learning_rate=0.02,\n        l2_leaf_reg=3,\n        subsample=0.85,\n        random_state=42,\n        verbose=False\n    )\n    cat_trainer = Trainer(\n        cat_model,\n        cv=KFold(n_splits=5, shuffle=False),\n        metric=_pearsonr,\n        task=\"regression\",\n        metric_precision=6\n    )\n    cat_trainer.fit(X, y_compressed)\n    scores[\"CatBoost\"] = cat_trainer.fold_scores\n    oof_preds[\"CatBoost\"] = cat_trainer.oof_preds\n    test_preds[\"CatBoost\"] = cat_trainer.predict(X_test)\n    \n    # Train GANDALF\n    print(f\"\\nTraining GANDALF for {strategy_name}...\")\n    gandalf_model = GANDALF(\n        n_estimators=100,\n        learning_rate=0.001,\n        max_depth=5,\n        feature_fraction=0.8,\n        bagging_fraction=0.8,\n        lambda_reg=0.1,\n        min_data_in_leaf=20,\n        num_iterations=50,\n        random_state=42\n    )\n    gandalf_trainer = Trainer(\n        gandalf_model,\n        cv=KFold(n_splits=5, shuffle=False),\n        metric=_pearsonr,\n        task=\"regression\",\n        metric_precision=6\n    )\n    gandalf_trainer.fit(X, y_compressed)\n    scores[\"GANDALF\"] = gandalf_trainer.fold_scores\n    oof_preds[\"GANDALF\"] = gandalf_trainer.oof_preds\n    test_preds[\"GANDALF\"] = gandalf_trainer.predict(X_test)\n    \n    # Create ensemble predictions\n    X_ensemble = pd.DataFrame(oof_preds)\n    X_test_ensemble = pd.DataFrame(test_preds)\n    \n    # Train AutoEncoder\n    print(f\"\\nTraining AutoEncoder for {strategy_name}...\")\n    ae_model = AutoEncoderMLP(\n        num_columns=X_ensemble.shape[1],\n        hidden_units=[128, 128],\n        dropout_rates=[0.05, 0.1, 0.2],\n        lr=1e-3\n    )\n    ae_trainer = Trainer(\n        ae_model,\n        cv=KFold(n_splits=5, shuffle=False),\n        metric=_pearsonr,\n        task=\"regression\",\n        metric_precision=6\n    )\n    ae_trainer.fit(X_ensemble, y)  # Note: use original y, not compressed\n    scores[\"AutoEncoder\"] = ae_trainer.fold_scores\n    oof_preds[\"AutoEncoder\"] = ae_trainer.oof_preds\n    ae_test_preds = ae_trainer.predict(X_test_ensemble)\n    \n    # Create weighted ensemble\n    ensemble_weights = {\n        \"LightGBM (gbdt)\": 0.25,\n        \"LightGBM (goss)\": 0.25,\n        \"XGBoost\": 0.25,\n        \"CatBoost\": 0.15,\n        \"GANDALF\": 0.05,\n        \"AutoEncoder\": 0.05\n    }\n    \n    # Calculate weighted predictions\n    weighted_test_pred = np.zeros(len(X_test))\n    for model_name, weight in ensemble_weights.items():\n        if model_name == \"AutoEncoder\":\n            weighted_test_pred += weight * ae_test_preds\n        else:\n            weighted_test_pred += weight * test_preds[model_name]\n    \n    # Print results\n    print(f\"\\nResults for {strategy_name}:\")\n    scores_df = pd.DataFrame(scores)\n    mean_scores = scores_df.mean()\n    print(mean_scores.sort_values(ascending=False))\n    \n    # Clean up memory\n    del X, y, X_test, train, test, df_clean\n    gc.collect()\n    \n    return weighted_test_pred, scores_df, label_compressor\n\n# Main execution\ndef main():\n    print(\"\\n\" + \"=\"*80)\n    print(\"DRW CRYPTO PREDICTION - COMPREHENSIVE SOLUTION\")\n    print(\"=\"*80)\n    \n    # Define outlier capping strategies\n    strategies = [\n        (1, 99, \"1st-99th_percentile\"),\n        (5, 95, \"5th-95th_percentile\"),\n        (10, 90, \"10th-90th_percentile\"),\n        (25, 75, \"25th-75th_percentile\")\n    ]\n    \n    # Store predictions for each strategy\n    all_predictions = {}\n    all_scores = {}\n    \n    # Run each strategy\n    for lower, upper, name in strategies:\n        try:\n            predictions, scores, label_compressor = run_pipeline_with_outlier_strategy(lower, upper, name)\n            all_predictions[name] = predictions\n            all_scores[name] = scores\n            \n            # Save individual submission\n            sub = pd.read_csv(CFG.sample_sub_path)\n            sub[\"prediction\"] = predictions\n            sub.to_csv(f\"submission_{name}.csv\", index=False)\n            print(f\"\\nSaved submission_{name}.csv\")\n            \n        except Exception as e:\n            print(f\"\\nError in strategy {name}: {str(e)}\")\n            continue\n    \n    # Create ensemble of all strategies\n    print(\"\\n\" + \"=\"*80)\n    print(\"Creating ensemble of all strategies...\")\n    print(\"=\"*80)\n    \n    if len(all_predictions) > 0:\n        # Simple average ensemble\n        ensemble_predictions = np.zeros(len(next(iter(all_predictions.values()))))\n        for strategy_name in all_predictions:\n            ensemble_predictions += all_predictions[strategy_name]\n        ensemble_predictions /= len(all_predictions)\n        \n        # Save ensemble submission\n        sub = pd.read_csv(CFG.sample_sub_path)\n        sub[\"prediction\"] = ensemble_predictions\n        sub.to_csv(\"submission_ensemble_all_strategies.csv\", index=False)\n        print(\"\\nSaved submission_ensemble_all_strategies.csv\")\n        \n        # Create weighted ensemble (favor middle strategies)\n        if len(all_predictions) == 4:\n            weights = [0.2, 0.3, 0.3, 0.2]  # Bell curve weights\n            weighted_ensemble = np.zeros(len(ensemble_predictions))\n            for i, (strategy_name, weight) in enumerate(zip(sorted(all_predictions.keys()), weights)):\n                weighted_ensemble += weight * all_predictions[strategy_name]\n            \n            sub[\"prediction\"] = weighted_ensemble\n            sub.to_csv(\"submission_weighted_ensemble.csv\", index=False)\n            print(\"Saved submission_weighted_ensemble.csv\")\n        \n        # Create median ensemble (most robust)\n        predictions_matrix = np.array(list(all_predictions.values()))\n        median_ensemble = np.median(predictions_matrix, axis=0)\n        \n        sub[\"prediction\"] = median_ensemble\n        sub.to_csv(\"submission_median_ensemble.csv\", index=False)\n        print(\"Saved submission_median_ensemble.csv\")\n        \n        # Create summary visualization\n        fig, axes = plt.subplots(2, 2, figsize=(15, 12))\n        axes = axes.ravel()\n        \n        for idx, (strategy_name, scores_df) in enumerate(all_scores.items()):\n            if idx < 4:\n                mean_scores = scores_df.mean().sort_values(ascending=False)\n                \n                ax = axes[idx]\n                colors = ['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd', '#8c564b']\n                bars = ax.barh(range(len(mean_scores)), mean_scores.values, color=colors[:len(mean_scores)])\n                ax.set_yticks(range(len(mean_scores)))\n                ax.set_yticklabels(mean_scores.index)\n                ax.set_title(f\"Mean Scores - {strategy_name}\", fontsize=12)\n                ax.set_xlabel(\"Pearson Correlation\")\n                \n                # Add value labels\n                for i, (value, bar) in enumerate(zip(mean_scores.values, bars)):\n                    ax.text(value + 0.001, bar.get_y() + bar.get_height()/2, \n                           f'{value:.4f}', va='center', fontsize=9)\n        \n        plt.tight_layout()\n        plt.savefig(\"outlier_strategies_comparison.png\", dpi=300, bbox_inches='tight')\n        plt.show()\n        \n        # Create summary table\n        print(\"\\n\" + \"=\"*80)\n        print(\"SUMMARY OF ALL STRATEGIES\")\n        print(\"=\"*80)\n        \n        summary_data = []\n        for strategy_name in all_scores:\n            scores_df = all_scores[strategy_name]\n            mean_scores = scores_df.mean()\n            summary_data.append({\n                'Strategy': strategy_name,\n                'LightGBM (gbdt)': mean_scores.get('LightGBM (gbdt)', 0),\n                'LightGBM (goss)': mean_scores.get('LightGBM (goss)', 0),\n                'XGBoost': mean_scores.get('XGBoost', 0),\n                'CatBoost': mean_scores.get('CatBoost', 0),\n                'GANDALF': mean_scores.get('GANDALF', 0),\n                'AutoEncoder': mean_scores.get('AutoEncoder', 0),\n                'Average': mean_scores.mean()\n            })\n        \n        summary_df = pd.DataFrame(summary_data)\n        print(summary_df.to_string(index=False))\n        \n        # Correlation analysis\n        print(\"\\n\" + \"=\"*80)\n        print(\"PREDICTION CORRELATION ANALYSIS\")\n        print(\"=\"*80)\n        \n        pred_df = pd.DataFrame(all_predictions)\n        corr_matrix = pred_df.corr()\n        \n        plt.figure(figsize=(8, 6))\n        sns.heatmap(corr_matrix, annot=True, fmt='.3f', cmap='coolwarm', center=0.9,\n                    square=True, linewidths=1, cbar_kws={\"shrink\": .8})\n        plt.title('Correlation Between Different Outlier Strategies')\n        plt.tight_layout()\n        plt.savefig('strategy_correlation_matrix.png', dpi=300, bbox_inches='tight')\n        plt.show()\n        \n        print(\"\\n\" + \"=\"*80)\n        print(\"PIPELINE COMPLETED!\")\n        print(\"=\"*80)\n        print(\"Generated files:\")\n        for i, (_, _, name) in enumerate(strategies, 1):\n            if name in all_predictions:\n                print(f\"{i}. submission_{name}.csv\")\n        print(f\"{len(strategies)+1}. submission_ensemble_all_strategies.csv\")\n        print(f\"{len(strategies)+2}. submission_weighted_ensemble.csv\")\n        print(f\"{len(strategies)+3}. submission_median_ensemble.csv\")\n        print(f\"{len(strategies)+4}. outlier_strategies_comparison.png\")\n        print(f\"{len(strategies)+5}. strategy_correlation_matrix.png\")\n        print(\"\\n\" + \"=\"*80)\n        print(\"RECOMMENDATIONS:\")\n        print(\"1. Try 'submission_weighted_ensemble.csv' first (bell-curve weighted)\")\n        print(\"2. If unstable, use 'submission_median_ensemble.csv' (most robust)\")\n        print(\"3. Individual strategy files available for experimentation\")\n        print(\"=\"*80)\n        \n    else:\n        print(\"No successful strategies completed. Please check the errors above.\")\n\nif __name__ == \"__main__\":\n    main()","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}