{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import sys\nimport pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import KFold\nfrom xgboost import XGBRegressor\nfrom lightgbm import LGBMRegressor\nfrom sklearn.linear_model import HuberRegressor, RANSACRegressor, Lasso, ElasticNet\nfrom sklearn.preprocessing import RobustScaler, StandardScaler\nfrom sklearn.neural_network import MLPRegressor\nfrom sklearn.feature_selection import SelectKBest, f_regression\nfrom scipy.stats import pearsonr, rankdata\nimport warnings\nwarnings.filterwarnings('ignore')\nimport gc\n\n# ===== Feature Engineering =====\ndef feature_engineering(df):\n    \"\"\"Enhanced feature engineering with market microstructure focus\"\"\"\n    # Original features\n    df['volume_weighted_sell'] = df['sell_qty'] * df['volume']\n    df['buy_sell_ratio'] = df['buy_qty'] / (df['sell_qty'] + 1e-8)\n    df['selling_pressure'] = df['sell_qty'] / (df['volume'] + 1e-8)\n    df['effective_spread_proxy'] = np.abs(df['buy_qty'] - df['sell_qty']) / (df['volume'] + 1e-8)\n    \n    # Enhanced market microstructure features\n    df['log_volume'] = np.log1p(df['volume'])\n    df['bid_ask_imbalance'] = (df['bid_qty'] - df['ask_qty']) / (df['bid_qty'] + df['ask_qty'] + 1e-8)\n    df['order_flow_imbalance'] = (df['buy_qty'] - df['sell_qty']) / (df['buy_qty'] + df['sell_qty'] + 1e-8)\n    df['liquidity_ratio'] = (df['bid_qty'] + df['ask_qty']) / (df['volume'] + 1e-8)\n    \n    # Additional time-aware features\n    df['volume_intensity'] = df['volume'] / (df['bid_qty'] + df['ask_qty'] + 1e-8)\n    df['trade_aggressiveness'] = (df['buy_qty'] + df['sell_qty']) / (df['volume'] + 1e-8)\n    \n    # Handle infinities and NaN\n    df = df.replace([np.inf, -np.inf], np.nan)\n    for col in df.columns:\n        if df[col].isna().any():\n            median_val = df[col].median()\n            df[col] = df[col].fillna(median_val if not pd.isna(median_val) else 0)\n    \n    return df\n\n# ===== Configuration =====\nclass Config:\n    TRAIN_PATH = \"/kaggle/input/drw-crypto-market-prediction/train.parquet\"\n    TEST_PATH = \"/kaggle/input/drw-crypto-market-prediction/test.parquet\"\n    SUBMISSION_PATH = \"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\"\n    \n    # Baseline features that should NOT be rank transformed (they already work well)\n    BASELINE_FEATURES = [\n        \"X863\", \"X856\", \"X598\", \"X862\", \"X385\", \"X852\", \"X603\", \"X860\", \"X674\",\n        \"X415\", \"X345\", \"X855\", \"X174\", \"X302\", \"X178\", \"X168\", \"X612\",\n        \"buy_qty\", \"sell_qty\", \"volume\", \"X888\", \"X421\", \"X333\",\n        \"bid_qty\", \"ask_qty\"\n    ]\n    \n    LABEL_COLUMN = \"label\"\n    N_FOLDS = 3\n    ENHANCED_FOLDS = 5\n    RANDOM_STATE = 42\n\n# ===== Selective Rank Transformation =====\nclass SelectiveRankTransformer:\n    \"\"\"\n    Revolutionary approach: Apply rank transformation ONLY to non-baseline features\n    while preserving the strong baseline features unchanged.\n    \"\"\"\n    \n    def __init__(self, baseline_features):\n        self.baseline_features = set(baseline_features)\n        self.rank_mappings = {}\n        self.fitted = False\n    \n    def _rank_transform_feature(self, values, feature_name, fit=False):\n        \"\"\"Apply rank transformation to a single feature\"\"\"\n        finite_mask = np.isfinite(values)\n        \n        if not finite_mask.any():\n            return np.full_like(values, 0.5)\n        \n        finite_vals = values[finite_mask]\n        \n        if fit:\n            # Store unique sorted values for consistent transformation\n            unique_vals = np.sort(np.unique(finite_vals))\n            self.rank_mappings[feature_name] = unique_vals\n        \n        if feature_name in self.rank_mappings:\n            # Transform using stored mapping\n            mapping_vals = self.rank_mappings[feature_name]\n            ranks = np.searchsorted(mapping_vals, finite_vals, side='left')\n            percentiles = ranks / len(mapping_vals)\n            percentiles = np.clip(percentiles, 0, 1)\n        else:\n            # Fallback to direct ranking\n            ranks = rankdata(finite_vals, method='average')\n            percentiles = (ranks - 1) / (len(ranks) - 1) if len(ranks) > 1 else np.array([0.5])\n        \n        result = np.full_like(values, 0.5)\n        result[finite_mask] = percentiles\n        \n        return result\n    \n    def fit_transform(self, df, feature_cols):\n        \"\"\"Fit and transform: rank transform only non-baseline features\"\"\"\n        print(f\"   🎯 Selective Rank Transform:\")\n        \n        baseline_count = sum(1 for f in feature_cols if f in self.baseline_features)\n        rank_count = len(feature_cols) - baseline_count\n        \n        print(f\"      • Preserving {baseline_count} baseline features (no transform)\")\n        print(f\"      • Rank transforming {rank_count} additional features\")\n        \n        result_df = df.copy()\n        \n        for col in feature_cols:\n            if col not in self.baseline_features and col in df.columns:\n                # Apply rank transformation\n                transformed_vals = self._rank_transform_feature(\n                    df[col].values, col, fit=True\n                )\n                result_df[col] = transformed_vals\n        \n        self.fitted = True\n        return result_df\n    \n    def transform(self, df, feature_cols):\n        \"\"\"Transform new data using fitted parameters\"\"\"\n        if not self.fitted:\n            raise ValueError(\"Must fit before transform\")\n        \n        result_df = df.copy()\n        \n        for col in feature_cols:\n            if col not in self.baseline_features and col in df.columns:\n                # Apply rank transformation using fitted mapping\n                transformed_vals = self._rank_transform_feature(\n                    df[col].values, col, fit=False\n                )\n                result_df[col] = transformed_vals\n        \n        return result_df\n\n# ===== Smart Feature Selector =====\nclass SmartFeatureSelector:\n    \"\"\"Select top features for analysis beyond baseline\"\"\"\n    \n    def __init__(self, baseline_features, max_additional=50):\n        self.baseline_features = set(baseline_features)\n        self.max_additional = max_additional\n        self.selected_additional = []\n        self.all_features = []\n    \n    def fit(self, df, target):\n        \"\"\"Select best additional features beyond baseline\"\"\"\n        print(f\"   🔍 Smart Feature Selection:\")\n        \n        # Get all possible features (engineered features)\n        all_features = [col for col in df.columns \n                       if col not in ['label', 'timestamp'] \n                       and col not in self.baseline_features]\n        \n        print(f\"      • Baseline features: {len(self.baseline_features)}\")\n        print(f\"      • Additional candidates: {len(all_features)}\")\n        \n        # Calculate correlations for additional features\n        correlations = []\n        for col in all_features:\n            if col in df.columns:\n                corr = abs(df[col].corr(target))\n                if not np.isnan(corr):\n                    correlations.append((col, corr))\n        \n        # Sort and select top features\n        correlations.sort(key=lambda x: x[1], reverse=True)\n        self.selected_additional = [col for col, _ in correlations[:self.max_additional]]\n        \n        # Combine baseline + selected additional\n        self.all_features = list(self.baseline_features) + self.selected_additional\n        \n        print(f\"      • Selected additional: {len(self.selected_additional)}\")\n        print(f\"      • Total features: {len(self.all_features)}\")\n        \n        return self.all_features\n    \n    def get_features(self):\n        \"\"\"Get selected features\"\"\"\n        return self.all_features\n\n# ===== Intelligent Ensemble Manager =====\nclass IntelligentEnsembleManager:\n    \"\"\"Improved ensemble manager with better model selection\"\"\"\n    \n    def __init__(self):\n        self.models = {}\n        self.baseline_score = None\n    \n    def add_model(self, name, oof_preds, test_preds, score, model_type=\"standard\"):\n        \"\"\"Add a model with intelligent tracking\"\"\"\n        self.models[name] = {\n            'oof': oof_preds,\n            'test': test_preds,\n            'score': score,\n            'type': model_type\n        }\n        \n        if self.baseline_score is None:\n            self.baseline_score = score\n            print(f\"   📊 Baseline set: {name} (score: {score:.4f})\")\n        else:\n            improvement = (score - self.baseline_score) / self.baseline_score * 100\n            status = \"✅\" if score > self.baseline_score else \"📈\" if score > self.baseline_score * 0.9 else \"⚠️\"\n            print(f\"   📊 {status} {name}: {score:.4f} ({improvement:+.1f}%)\")\n    \n    def get_tree_ensemble(self, train_labels):\n        \"\"\"Get the strong tree ensemble\"\"\"\n        tree_models = {k: v for k, v in self.models.items() \n                      if any(x in k.lower() for x in ['xgb', 'lgbm', 'tree'])}\n        \n        if len(tree_models) >= 2:\n            # Equal weight ensemble of tree models\n            tree_oof = np.mean([model['oof'] for model in tree_models.values()], axis=0)\n            tree_test = np.mean([model['test'] for model in tree_models.values()], axis=0)\n            tree_score = pearsonr(train_labels, tree_oof)[0]\n            return tree_oof, tree_test, tree_score\n        elif tree_models:\n            # Single tree model\n            model = list(tree_models.values())[0]\n            return model['oof'], model['test'], model['score']\n        else:\n            return None, None, 0.0\n    \n    def get_weighted_ensemble(self, train_labels, min_score_threshold=0.01):\n        \"\"\"Get performance-weighted ensemble\"\"\"\n        valid_models = {k: v for k, v in self.models.items() \n                       if v['score'] > min_score_threshold and not np.isnan(v['score'])}\n        \n        if not valid_models:\n            return None, None, 0.0, {}\n        \n        # Calculate weights based on performance\n        total_score = sum(model['score'] for model in valid_models.values())\n        weights = {k: v['score'] / total_score for k, v in valid_models.items()}\n        \n        # Create weighted ensemble\n        weighted_oof = sum(weights[k] * valid_models[k]['oof'] for k in weights)\n        weighted_test = sum(weights[k] * valid_models[k]['test'] for k in weights)\n        weighted_score = pearsonr(train_labels, weighted_oof)[0]\n        \n        return weighted_oof, weighted_test, weighted_score, weights\n    \n    def get_rank_ensemble(self, train_labels):\n        \"\"\"Get ensemble of rank-transformed models only\"\"\"\n        rank_models = {k: v for k, v in self.models.items() if 'rank' in k.lower()}\n        \n        if not rank_models:\n            return None, None, 0.0\n        \n        # Simple average of rank models\n        rank_oof = np.mean([model['oof'] for model in rank_models.values()], axis=0)\n        rank_test = np.mean([model['test'] for model in rank_models.values()], axis=0)\n        rank_score = pearsonr(train_labels, rank_oof)[0]\n        \n        return rank_oof, rank_test, rank_score\n\n# ===== Model Parameters (CPU Only) =====\ndef get_baseline_xgb_params():\n    \"\"\"Baseline XGBoost parameters (CPU)\"\"\"\n    return {\n        \"tree_method\": \"hist\",\n        \"device\": \"cpu\",\n        \"colsample_bylevel\": 0.4778,\n        \"colsample_bynode\": 0.3628,\n        \"colsample_bytree\": 0.7107,\n        \"gamma\": 1.7095,\n        \"learning_rate\": 0.02213,\n        \"max_depth\": 20,\n        \"max_leaves\": 12,\n        \"min_child_weight\": 16,\n        \"n_estimators\": 1667,\n        \"subsample\": 0.06567,\n        \"reg_alpha\": 39.3524,\n        \"reg_lambda\": 75.4484,\n        \"verbosity\": 0,\n        \"random_state\": Config.RANDOM_STATE,\n        \"n_jobs\": -1\n    }\n\ndef get_baseline_lgbm_params():\n    \"\"\"Baseline LightGBM parameters (CPU)\"\"\"\n    return {\n        \"n_estimators\": 500,\n        \"learning_rate\": 0.03,\n        \"num_leaves\": 31,\n        \"min_child_samples\": 50,\n        \"subsample\": 0.8,\n        \"colsample_bytree\": 0.8,\n        \"reg_alpha\": 10,\n        \"reg_lambda\": 10,\n        \"random_state\": Config.RANDOM_STATE,\n        \"device\": \"cpu\",\n        \"verbosity\": -1,\n        \"n_jobs\": -1\n    }\n\ndef get_conservative_xgb_params():\n    \"\"\"Conservative XGBoost for rank features\"\"\"\n    base = get_baseline_xgb_params()\n    return {\n        **base,\n        \"learning_rate\": 0.015,\n        \"max_depth\": 15,\n        \"min_child_weight\": 25,\n        \"subsample\": 0.08,\n        \"colsample_bytree\": 0.6,\n        \"reg_alpha\": 50,\n        \"reg_lambda\": 100,\n        \"n_estimators\": 1200,\n    }\n\n# ===== Utility Functions =====\ndef create_time_decay_weights(n: int, decay: float = 0.9) -> np.ndarray:\n    \"\"\"Create time decay weights for more recent data importance\"\"\"\n    positions = np.arange(n)\n    normalized = positions / (n - 1)\n    weights = decay ** (1.0 - normalized)\n    return weights * n / weights.sum()\n\ndef get_model_slices(n_samples: int):\n    \"\"\"Define different data slices for training\"\"\"\n    return [\n        {\"name\": \"full_data\", \"cutoff\": 0},\n        {\"name\": \"last_75pct\", \"cutoff\": int(0.25 * n_samples)},\n        {\"name\": \"last_50pct\", \"cutoff\": int(0.50 * n_samples)},\n    ]\n\ndef train_single_model(X_train, y_train, X_valid, y_valid, X_test, model_type, params, sample_weights=None):\n    \"\"\"Train a single model\"\"\"\n    if model_type == \"xgb\":\n        model = XGBRegressor(**params)\n        model.fit(X_train, y_train, \n                 sample_weight=sample_weights,\n                 eval_set=[(X_valid, y_valid)], \n                 verbose=False)\n    elif model_type == \"lgbm\":\n        model = LGBMRegressor(**params)\n        model.fit(X_train, y_train,\n                 sample_weight=sample_weights,\n                 eval_set=[(X_valid, y_valid)],\n                 callbacks=[])\n    elif model_type == \"huber\":\n        scaler = RobustScaler()\n        X_train_scaled = scaler.fit_transform(X_train)\n        X_valid_scaled = scaler.transform(X_valid)\n        X_test_scaled = scaler.transform(X_test)\n        \n        model = HuberRegressor(**params)\n        model.fit(X_train_scaled, y_train, sample_weight=sample_weights)\n        \n        valid_pred = model.predict(X_valid_scaled)\n        test_pred = model.predict(X_test_scaled)\n        return valid_pred, test_pred\n    elif model_type == \"lasso\":\n        scaler = RobustScaler()\n        X_train_scaled = scaler.fit_transform(X_train)\n        X_valid_scaled = scaler.transform(X_valid)\n        X_test_scaled = scaler.transform(X_test)\n        \n        model = Lasso(**params)\n        model.fit(X_train_scaled, y_train, sample_weight=sample_weights)\n        \n        valid_pred = model.predict(X_valid_scaled)\n        test_pred = model.predict(X_test_scaled)\n        return valid_pred, test_pred\n    elif model_type == \"sklearn_mlp\":\n        # Feature selection for MLP\n        selector = SelectKBest(score_func=f_regression, k=min(30, X_train.shape[1]))\n        X_train_selected = selector.fit_transform(X_train, y_train)\n        X_valid_selected = selector.transform(X_valid)\n        X_test_selected = selector.transform(X_test)\n        \n        # Scale features\n        scaler = StandardScaler()\n        X_train_scaled = scaler.fit_transform(X_train_selected)\n        X_valid_scaled = scaler.transform(X_valid_selected)\n        X_test_scaled = scaler.transform(X_test_selected)\n        \n        # Train MLP\n        mlp = MLPRegressor(\n            hidden_layer_sizes=(100, 50),\n            activation='relu',\n            solver='adam',\n            alpha=0.01,\n            learning_rate='adaptive',\n            learning_rate_init=0.001,\n            max_iter=300,\n            early_stopping=True,\n            validation_fraction=0.1,\n            n_iter_no_change=20,\n            random_state=42\n        )\n        \n        try:\n            mlp.fit(X_train_scaled, y_train)\n            valid_pred = mlp.predict(X_valid_scaled)\n            test_pred = mlp.predict(X_test_scaled)\n            return valid_pred, test_pred\n        except:\n            return np.zeros(len(y_valid)), np.zeros(X_test.shape[0])\n    \n    valid_pred = model.predict(X_valid)\n    test_pred = model.predict(X_test)\n    return valid_pred, test_pred\n\n# ===== Data Loading =====\ndef load_data():\n    \"\"\"Load and preprocess data\"\"\"\n    print(\"Loading data...\")\n    \n    # Load with baseline features first\n    train_df = pd.read_parquet(Config.TRAIN_PATH, columns=Config.BASELINE_FEATURES + [Config.LABEL_COLUMN])\n    test_df = pd.read_parquet(Config.TEST_PATH, columns=Config.BASELINE_FEATURES)\n    submission_df = pd.read_csv(Config.SUBMISSION_PATH)\n    \n    print(f\"Raw data - Train: {train_df.shape}, Test: {test_df.shape}\")\n    \n    # Apply feature engineering\n    train_df = feature_engineering(train_df)\n    test_df = feature_engineering(test_df)\n    \n    # Smart feature selection for additional features\n    feature_selector = SmartFeatureSelector(Config.BASELINE_FEATURES, max_additional=50)\n    selected_features = feature_selector.fit(train_df, train_df[Config.LABEL_COLUMN])\n    \n    print(f\"Enhanced data - Train: {train_df.shape}, Test: {test_df.shape}\")\n    print(f\"Selected features: {len(selected_features)}\")\n    \n    return train_df.reset_index(drop=True), test_df.reset_index(drop=True), submission_df, selected_features\n\n# ===== Model Training Functions =====\ndef train_baseline_models(train_df, test_df, features, ensemble_manager):\n    \"\"\"Train baseline models (no rank transformation)\"\"\"\n    print(\"\\n🚀 Training Baseline Models...\")\n    \n    n_samples = len(train_df)\n    model_slices = get_model_slices(n_samples)\n    \n    # Working models configuration\n    models_config = [\n        {\"name\": \"xgb_baseline\", \"type\": \"xgb\", \"params\": get_baseline_xgb_params()},\n        {\"name\": \"lgbm_baseline\", \"type\": \"lgbm\", \"params\": get_baseline_lgbm_params()},\n        {\"name\": \"huber_baseline\", \"type\": \"huber\", \"params\": {\"epsilon\": 1.5, \"alpha\": 0.01, \"max_iter\": 500}},\n        {\"name\": \"lasso_baseline\", \"type\": \"lasso\", \"params\": {\"alpha\": 0.001, \"max_iter\": 1000}},\n    ]\n    \n    for model_config in models_config:\n        print(f\"\\n   Training {model_config['name']}...\")\n        \n        # Initialize prediction storage\n        oof_preds = {s[\"name\"]: np.zeros(n_samples) for s in model_slices}\n        test_preds = {s[\"name\"]: np.zeros(len(test_df)) for s in model_slices}\n        \n        full_weights = create_time_decay_weights(n_samples)\n        kf = KFold(n_splits=Config.N_FOLDS, shuffle=False)\n        \n        for fold, (train_idx, valid_idx) in enumerate(kf.split(train_df), start=1):\n            X_valid = train_df.iloc[valid_idx][features].values\n            y_valid = train_df.iloc[valid_idx][Config.LABEL_COLUMN].values\n            X_test = test_df[features].values\n            \n            for s in model_slices:\n                cutoff = s[\"cutoff\"]\n                slice_name = s[\"name\"]\n                subset = train_df.iloc[cutoff:].reset_index(drop=True)\n                rel_idx = train_idx[train_idx >= cutoff] - cutoff\n                \n                if len(rel_idx) == 0:\n                    continue\n                    \n                X_train = subset.iloc[rel_idx][features].values\n                y_train = subset.iloc[rel_idx][Config.LABEL_COLUMN].values\n                sw = create_time_decay_weights(len(subset))[rel_idx] if cutoff > 0 else full_weights[train_idx]\n                \n                try:\n                    valid_pred, test_pred = train_single_model(\n                        X_train, y_train, X_valid, y_valid, X_test, \n                        model_config['type'], model_config['params'], sw\n                    )\n                    \n                    # Store OOF predictions\n                    mask = valid_idx >= cutoff\n                    if mask.any():\n                        oof_preds[slice_name][valid_idx[mask]] = valid_pred[mask]\n                    \n                    if cutoff > 0 and (~mask).any():\n                        oof_preds[slice_name][valid_idx[~mask]] = oof_preds[\"full_data\"][valid_idx[~mask]]\n                    \n                    test_preds[slice_name] += test_pred\n                    \n                except Exception as e:\n                    print(f\"      Error in {slice_name}: {str(e)}\")\n                    continue\n        \n        # Normalize test predictions\n        for slice_name in test_preds:\n            test_preds[slice_name] /= Config.N_FOLDS\n        \n        # Calculate ensemble of slices\n        final_oof = np.mean(list(oof_preds.values()), axis=0)\n        final_test = np.mean(list(test_preds.values()), axis=0)\n        final_score = pearsonr(train_df[Config.LABEL_COLUMN], final_oof)[0]\n        \n        # Add to ensemble manager\n        ensemble_manager.add_model(model_config['name'], final_oof, final_test, final_score, \"baseline\")\n\ndef train_rank_models(train_df, test_df, features, ensemble_manager):\n    \"\"\"Train models with selective rank transformation\"\"\"\n    print(\"\\n🎯 Training Rank-Transformed Models...\")\n    \n    # Apply selective rank transformation\n    rank_transformer = SelectiveRankTransformer(Config.BASELINE_FEATURES)\n    train_rank = rank_transformer.fit_transform(train_df, features)\n    test_rank = rank_transformer.transform(test_df, features)\n    \n    # Rank-specific models (more conservative)\n    rank_models_config = [\n        {\"name\": \"xgb_rank\", \"type\": \"xgb\", \"params\": get_conservative_xgb_params()},\n        {\"name\": \"lgbm_rank\", \"type\": \"lgbm\", \"params\": get_baseline_lgbm_params()},\n        {\"name\": \"mlp_rank\", \"type\": \"sklearn_mlp\", \"params\": {}},\n    ]\n    \n    kf = KFold(n_splits=Config.ENHANCED_FOLDS, shuffle=False)\n    n_samples = len(train_df)\n    \n    for model_config in rank_models_config:\n        print(f\"\\n   Training {model_config['name']} with rank features...\")\n        \n        model_oof = np.zeros(n_samples)\n        model_test = np.zeros(len(test_df))\n        fold_scores = []\n        \n        for fold, (train_idx, valid_idx) in enumerate(kf.split(train_df), start=1):\n            X_train = train_rank.iloc[train_idx][features].values\n            y_train = train_rank.iloc[train_idx][Config.LABEL_COLUMN].values\n            X_valid = train_rank.iloc[valid_idx][features].values\n            y_valid = train_rank.iloc[valid_idx][Config.LABEL_COLUMN].values\n            X_test = test_rank[features].values\n            \n            # Time decay weights\n            sw = create_time_decay_weights(len(train_idx))\n            \n            try:\n                valid_pred, test_pred = train_single_model(\n                    X_train, y_train, X_valid, y_valid, X_test, \n                    model_config['type'], model_config['params'], sw\n                )\n                \n                model_oof[valid_idx] = valid_pred\n                model_test += test_pred\n                \n                fold_score = pearsonr(y_valid, valid_pred)[0] if len(np.unique(valid_pred)) > 1 else 0\n                fold_scores.append(fold_score)\n                \n            except Exception as e:\n                print(f\"      Error in fold {fold}: {str(e)}\")\n                fold_scores.append(0.0)\n        \n        # Normalize test predictions\n        model_test /= Config.ENHANCED_FOLDS\n        \n        # Calculate final score\n        final_score = pearsonr(train_df[Config.LABEL_COLUMN], model_oof)[0]\n        \n        print(f\"      CV scores: {fold_scores}\")\n        print(f\"      Final score: {final_score:.4f}\")\n        \n        # Add to ensemble manager\n        ensemble_manager.add_model(model_config['name'], model_oof, model_test, final_score, \"rank\")\n\ndef train_combined_model(train_df, test_df, features, ensemble_manager):\n    \"\"\"Train combined model: Baseline + Top rank-transformed features\"\"\"\n    print(\"\\n🚀 Training Combined Model (Baseline + Top Rank Features)...\")\n    \n    # Apply rank transformation\n    rank_transformer = SelectiveRankTransformer(Config.BASELINE_FEATURES)\n    train_rank = rank_transformer.fit_transform(train_df, features)\n    test_rank = rank_transformer.transform(test_df, features)\n    \n    # Select top additional features (beyond baseline)\n    baseline_features = [f for f in Config.BASELINE_FEATURES if f in train_df.columns]\n    additional_features = [f for f in features if f not in Config.BASELINE_FEATURES]\n    \n    # Calculate correlations for additional features and select top 20\n    correlations = []\n    for col in additional_features:\n        if col in train_rank.columns:\n            corr = abs(train_rank[col].corr(train_df[Config.LABEL_COLUMN]))\n            if not np.isnan(corr):\n                correlations.append((col, corr))\n    \n    correlations.sort(key=lambda x: x[1], reverse=True)\n    top_additional = [col for col, _ in correlations[:20]]\n    \n    print(f\"   Combined features: {len(baseline_features)} baseline + {len(top_additional)} rank-transformed\")\n    \n    # Train combined XGBoost model\n    kf = KFold(n_splits=Config.ENHANCED_FOLDS, shuffle=False)\n    n_samples = len(train_df)\n    \n    model_oof = np.zeros(n_samples)\n    model_test = np.zeros(len(test_df))\n    fold_scores = []\n    \n    for fold, (train_idx, valid_idx) in enumerate(kf.split(train_df), start=1):\n        # Combine baseline (original) + rank-transformed additional features\n        X_train_baseline = train_df.iloc[train_idx][baseline_features].values\n        X_train_rank = train_rank.iloc[train_idx][top_additional].values\n        X_train = np.hstack([X_train_baseline, X_train_rank]) if top_additional else X_train_baseline\n        \n        X_valid_baseline = train_df.iloc[valid_idx][baseline_features].values\n        X_valid_rank = train_rank.iloc[valid_idx][top_additional].values\n        X_valid = np.hstack([X_valid_baseline, X_valid_rank]) if top_additional else X_valid_baseline\n        \n        X_test_baseline = test_df[baseline_features].values\n        X_test_rank = test_rank[top_additional].values\n        X_test = np.hstack([X_test_baseline, X_test_rank]) if top_additional else X_test_baseline\n        \n        y_train = train_df.iloc[train_idx][Config.LABEL_COLUMN].values\n        y_valid = train_df.iloc[valid_idx][Config.LABEL_COLUMN].values\n        \n        # Time decay weights\n        sw = create_time_decay_weights(len(train_idx))\n        \n        try:\n            valid_pred, test_pred = train_single_model(\n                X_train, y_train, X_valid, y_valid, X_test,\n                \"xgb\", get_conservative_xgb_params(), sw\n            )\n            \n            model_oof[valid_idx] = valid_pred\n            model_test += test_pred\n            \n            fold_score = pearsonr(y_valid, valid_pred)[0] if len(np.unique(valid_pred)) > 1 else 0\n            fold_scores.append(fold_score)\n            \n            print(f\"      Fold {fold} score: {fold_score:.4f}\")\n            \n        except Exception as e:\n            print(f\"      Error in fold {fold}: {str(e)}\")\n            fold_scores.append(0.0)\n    \n    # Normalize test predictions\n    model_test /= Config.ENHANCED_FOLDS\n    \n    # Calculate final score\n    final_score = pearsonr(train_df[Config.LABEL_COLUMN], model_oof)[0]\n    \n    print(f\"   📊 Combined model CV scores: {fold_scores}\")\n    print(f\"   📊 Combined model final score: {final_score:.4f}\")\n    \n    # Add to ensemble manager\n    ensemble_manager.add_model(\"combined_rank\", model_oof, model_test, final_score, \"combined\")\n    \n    return final_score\n\n# ===== Main Execution =====\ndef main():\n    \"\"\"Main execution pipeline\"\"\"\n    print(\"🎯 Clean Ensemble with Selective Rank Transformation\")\n    print(\"=\" * 60)\n    print(\"📋 Approach:\")\n    print(\"   • Fixed all device and component issues\")\n    print(\"   • Added selective rank transformation\")\n    print(\"   • Intelligent ensemble management\")\n    print(\"   • Multiple submission strategies\")\n    print(\"=\" * 60)\n    \n    # Load data\n    train_df, test_df, submission_df, features = load_data()\n    \n    # Target analysis\n    target = train_df[Config.LABEL_COLUMN]\n    print(f\"\\n📊 Target Analysis:\")\n    print(f\"   Mean: {target.mean():.4f}\")\n    print(f\"   Std:  {target.std():.4f}\")\n    print(f\"   Range: [{target.min():.2f}, {target.max():.2f}]\")\n    \n    # Initialize ensemble manager\n    ensemble_manager = IntelligentEnsembleManager()\n    \n    # Stage 1: Train baseline models\n    train_baseline_models(train_df, test_df, features, ensemble_manager)\n    \n    # Stage 2: Train rank-transformed models\n    train_rank_models(train_df, test_df, features, ensemble_manager)\n    \n    # Stage 3: Train combined model\n    combined_score = train_combined_model(train_df, test_df, features, ensemble_manager)\n    \n    # Create multiple submissions\n    print(\"\\n📁 Creating Multiple Submission Strategies...\")\n    \n    train_labels = train_df[Config.LABEL_COLUMN]\n    \n    # 1. Tree Ensemble (XGBoost + LightGBM)\n    tree_oof, tree_test, tree_score = ensemble_manager.get_tree_ensemble(train_labels)\n    if tree_test is not None:\n        submission_tree = submission_df.copy()\n        submission_tree[\"prediction\"] = tree_test\n        submission_tree.to_csv(\"submission_tree_ensemble.csv\", index=False)\n        print(f\"   📊 Tree Ensemble: {tree_score:.4f}\")\n    \n    # 2. Weighted Performance Ensemble\n    weighted_result = ensemble_manager.get_weighted_ensemble(train_labels)\n    if weighted_result[0] is not None:\n        weighted_oof, weighted_test, weighted_score, weights = weighted_result\n        submission_weighted = submission_df.copy()\n        submission_weighted[\"prediction\"] = weighted_test\n        submission_weighted.to_csv(\"submission_weighted_ensemble.csv\", index=False)\n        print(f\"   📊 Weighted Ensemble: {weighted_score:.4f}\")\n        print(f\"      Top weights: {', '.join([f'{k}: {v:.2f}' for k, v in sorted(weights.items(), key=lambda x: x[1], reverse=True)[:3]])}\")\n    \n    # 3. Rank Models Only\n    rank_oof, rank_test, rank_score = ensemble_manager.get_rank_ensemble(train_labels)\n    if rank_test is not None:\n        submission_rank = submission_df.copy()\n        submission_rank[\"prediction\"] = rank_test\n        submission_rank.to_csv(\"submission_rank_only.csv\", index=False)\n        print(f\"   📊 Rank Ensemble: {rank_score:.4f}\")\n    \n    # 4. Combined Model Only (Expected Best)\n    if \"combined_rank\" in ensemble_manager.models:\n        submission_combined = submission_df.copy()\n        submission_combined[\"prediction\"] = ensemble_manager.models[\"combined_rank\"][\"test\"]\n        submission_combined.to_csv(\"submission_combined_rank.csv\", index=False)\n        print(f\"   📊 Combined Model: {combined_score:.4f}\")\n    \n    # 5. XGBoost Baseline Only\n    if \"xgb_baseline\" in ensemble_manager.models:\n        submission_baseline = submission_df.copy()\n        submission_baseline[\"prediction\"] = ensemble_manager.models[\"xgb_baseline\"][\"test\"]\n        submission_baseline.to_csv(\"submission_xgb_baseline.csv\", index=False)\n        baseline_score = ensemble_manager.models[\"xgb_baseline\"][\"score\"]\n        print(f\"   📊 XGBoost Baseline: {baseline_score:.4f}\")\n    \n    # Summary and Analysis\n    print(\"\\n🏆 Final Results Summary:\")\n    print(\"=\" * 50)\n    \n    print(\"\\nModel Performance:\")\n    for name, model in sorted(ensemble_manager.models.items(), key=lambda x: x[1]['score'], reverse=True):\n        print(f\"   {name:20s}: {model['score']:.4f}\")\n    \n    print(f\"\\nFiles Created:\")\n    print(f\"   • submission_tree_ensemble.csv\")\n    print(f\"   • submission_weighted_ensemble.csv\") \n    print(f\"   • submission_rank_only.csv\")\n    print(f\"   • submission_combined_rank.csv (Expected Best)\")\n    print(f\"   • submission_xgb_baseline.csv\")\n    \n    # Expected improvement analysis\n    if \"combined_rank\" in ensemble_manager.models and \"xgb_baseline\" in ensemble_manager.models:\n        baseline = ensemble_manager.models[\"xgb_baseline\"][\"score\"]\n        combined = ensemble_manager.models[\"combined_rank\"][\"score\"]\n        improvement = (combined - baseline) / baseline * 100\n        \n        print(f\"\\n📈 Selective Rank Transformation Impact:\")\n        print(f\"   Baseline Score:    {baseline:.4f}\")\n        print(f\"   Combined Score:    {combined:.4f}\")\n        print(f\"   Improvement:       {improvement:+.1f}%\")\n        print(f\"   Expected (research): +6.2%\")\n        \n        if improvement >= 5.0:\n            print(f\"   🎉 SUCCESS! Achieved target improvement!\")\n        elif improvement >= 2.0:\n            print(f\"   ✅ GOOD! Meaningful improvement achieved!\")\n        elif improvement > 0:\n            print(f\"   📈 PROGRESS! Some improvement achieved!\")\n        else:\n            print(f\"   ⚠️  Room for optimization...\")\n    \n    print(f\"\\n🔬 Recommendation: Try submission_combined_rank.csv first!\")\n    \n    # Clean up memory\n    gc.collect()\n\nif __name__ == \"__main__\":\n    main()","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}