{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":96164,"databundleVersionId":12993472,"sourceType":"competition"},{"sourceId":258969650,"sourceType":"kernelVersion"}],"dockerImageVersionId":31090,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-08-30T00:43:29.788755Z","iopub.execute_input":"2025-08-30T00:43:29.78896Z","iopub.status.idle":"2025-08-30T00:43:32.337227Z","shell.execute_reply.started":"2025-08-30T00:43:29.788934Z","shell.execute_reply":"2025-08-30T00:43:32.336455Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.preprocessing import StandardScaler, RobustScaler, QuantileTransformer\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.linear_model import Ridge, Lasso, ElasticNet\nfrom sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor\nimport xgboost as xgb\nimport lightgbm as lgb\nimport warnings\nwarnings.filterwarnings('ignore')\n\nclass DRWPredictor:\n    def __init__(self):\n        self.train_data = None\n        self.test_data = None\n        self.feature_sets = {}\n        self.cv_analysis = {}\n        self.models = {}\n        \n    def load_data(self):\n        \"\"\"Load DRW competition data\"\"\"\n        print(\"Loading DRW Data...\")\n        \n        # Load the preprocessed data from your previous work\n        self.train_data = pd.read_parquet('/kaggle/input/drw-remix-of-winning-strategies-part-1/train_final.parquet')\n        self.test_data = pd.read_parquet('/kaggle/input/drw-remix-of-winning-strategies-part-1/test_final.parquet')\n        \n        print(f\"Train shape: {self.train_data.shape}\")\n        print(f\"Test shape: {self.test_data.shape}\")\n        \n        return self.train_data, self.test_data\n    \n    def create_preprocessing_strategies(self):\n        \"\"\"Create different preprocessing strategies\"\"\"\n        print(\"\\nCreating Preprocessing Strategies...\")\n        \n        # Get feature columns\n        feature_cols = [col for col in self.train_data.columns if col not in ['label', 'timestamp', 'id']]\n        \n        # Extract features and target\n        X = self.train_data[feature_cols].values\n        y = self.train_data['label'].values\n        X_test = self.test_data[feature_cols].values\n        \n        # Clean data\n        X = np.nan_to_num(X, nan=0.0, posinf=0.0, neginf=0.0)\n        X_test = np.nan_to_num(X_test, nan=0.0, posinf=0.0, neginf=0.0)\n        \n        # Strategy 1: StandardScaler\n        scaler1 = StandardScaler()\n        X1_train = scaler1.fit_transform(X)\n        X1_test = scaler1.transform(X_test)\n        \n        # Strategy 2: RobustScaler (handles outliers better)\n        scaler2 = RobustScaler()\n        X2_train = scaler2.fit_transform(X)\n        X2_test = scaler2.transform(X_test)\n        \n        # Strategy 3: QuantileTransformer (makes features more Gaussian)\n        scaler3 = QuantileTransformer(n_quantiles=100, output_distribution='normal', random_state=42)\n        X3_train = scaler3.fit_transform(X)\n        X3_test = scaler3.transform(X_test)\n        \n        # Strategy 4: Log transform + StandardScaler\n        X_log = np.sign(X) * np.log1p(np.abs(X))\n        X_test_log = np.sign(X_test) * np.log1p(np.abs(X_test))\n        scaler4 = StandardScaler()\n        X4_train = scaler4.fit_transform(X_log)\n        X4_test = scaler4.transform(X_test_log)\n        \n        self.preprocessing_strategies = {\n            'StandardScaler': (X1_train, X1_test, y),\n            'RobustScaler': (X2_train, X2_test, y),\n            'QuantileTransformer': (X3_train, X3_test, y),\n            'LogTransform+Standard': (X4_train, X4_test, y)\n        }\n        \n        return self.preprocessing_strategies\n    \n    def analyze_train_cv_gaps(self):\n        \"\"\"Analyze train-CV gaps across preprocessing methods and models\"\"\"\n        print(\"\\nAnalyzing Train-CV Gaps...\")\n        \n        # Define models to test\n        test_models = {\n            'Ridge': Ridge(alpha=1.0, random_state=42),\n            'Lasso': Lasso(alpha=0.1, random_state=42),\n            'ElasticNet': ElasticNet(alpha=0.1, random_state=42),\n            'RandomForest': RandomForestRegressor(n_estimators=100, max_depth=10, random_state=42),\n            'GradientBoosting': GradientBoostingRegressor(n_estimators=100, max_depth=5, random_state=42),\n            'XGBoost': xgb.XGBRegressor(n_estimators=100, max_depth=5, random_state=42, verbosity=0),\n            'LightGBM': lgb.LGBMRegressor(n_estimators=100, max_depth=5, random_state=42, verbose=-1)\n        }\n        \n        gap_analysis = {}\n        \n        for prep_name, (X_train, X_test, y_train) in self.preprocessing_strategies.items():\n            print(f\"\\n{prep_name}:\")\n            prep_gaps = {}\n            \n            for model_name, model in test_models.items():\n                try:\n                    # Fit model for training score\n                    model_copy = model.__class__(**model.get_params())\n                    model_copy.fit(X_train, y_train)\n                    \n                    # Training score (using Pearson correlation like DRW)\n                    train_pred = model_copy.predict(X_train)\n                    train_corr = np.corrcoef(y_train, train_pred)[0, 1]\n                    \n                    # Cross-validation score\n                    cv_scores = cross_val_score(model, X_train, y_train, cv=5, \n                                               scoring='r2')  # R² is related to correlation\n                    cv_score = cv_scores.mean()\n                    cv_std = cv_scores.std()\n                    \n                    # Gap analysis\n                    gap = train_corr - cv_score\n                    gap_percentage = (gap / train_corr) * 100 if train_corr > 0 else 0\n                    \n                    prep_gaps[model_name] = {\n                        'train_score': train_corr,\n                        'cv_score': cv_score,\n                        'cv_std': cv_std,\n                        'gap': gap,\n                        'gap_percentage': gap_percentage\n                    }\n                    \n                    print(f\"  {model_name}: Train={train_corr:.4f}, CV={cv_score:.4f}, Gap={gap:.4f} ({gap_percentage:.1f}%)\")\n                    \n                except Exception as e:\n                    print(f\"  {model_name}: Failed - {str(e)}\")\n                    \n            gap_analysis[prep_name] = prep_gaps\n        \n        self.cv_analysis = gap_analysis\n        return gap_analysis\n    \n    def visualize_gaps(self):\n        \"\"\"Create visualization of train-CV gaps\"\"\"\n        print(\"\\nCreating Gap Visualization...\")\n        \n        # Prepare data for heatmap\n        prep_methods = list(self.cv_analysis.keys())\n        model_names = set()\n        for method_data in self.cv_analysis.values():\n            model_names.update(method_data.keys())\n        model_names = sorted(list(model_names))\n        \n        # Create gap percentage matrix\n        gap_matrix = []\n        for prep in prep_methods:\n            row = []\n            for model in model_names:\n                if model in self.cv_analysis[prep]:\n                    gap_pct = self.cv_analysis[prep][model]['gap_percentage']\n                    row.append(gap_pct)\n                else:\n                    row.append(0)\n            gap_matrix.append(row)\n        \n        gap_df = pd.DataFrame(gap_matrix, index=prep_methods, columns=model_names)\n        \n        # Create visualization\n        fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(16, 6))\n        \n        # Gap percentage heatmap\n        sns.heatmap(gap_df, annot=True, fmt='.1f', cmap='RdYlBu_r', \n                   ax=ax1, cbar_kws={'label': 'Gap %'})\n        ax1.set_title('Train-CV Gap % by Preprocessing & Model')\n        ax1.set_xlabel('Model')\n        ax1.set_ylabel('Preprocessing Method')\n        \n        # CV scores heatmap\n        cv_matrix = []\n        for prep in prep_methods:\n            row = []\n            for model in model_names:\n                if model in self.cv_analysis[prep]:\n                    cv_score = self.cv_analysis[prep][model]['cv_score']\n                    row.append(cv_score)\n                else:\n                    row.append(0)\n            cv_matrix.append(row)\n        \n        cv_df = pd.DataFrame(cv_matrix, index=prep_methods, columns=model_names)\n        \n        sns.heatmap(cv_df, annot=True, fmt='.3f', cmap='YlGn', \n                   ax=ax2, cbar_kws={'label': 'CV Score'})\n        ax2.set_title('CV Scores by Preprocessing & Model')\n        ax2.set_xlabel('Model')\n        ax2.set_ylabel('Preprocessing Method')\n        \n        plt.tight_layout()\n        plt.savefig('drw_preprocessing_analysis.png', dpi=300, bbox_inches='tight')\n        plt.show()\n        \n        return gap_df, cv_df\n    \n    def train_best_models(self):\n        \"\"\"Train the best model combinations based on CV analysis\"\"\"\n        print(\"\\nTraining Best Models...\")\n        \n        # Find best preprocessing-model combinations\n        best_combinations = []\n        for prep_name, prep_data in self.cv_analysis.items():\n            for model_name, scores in prep_data.items():\n                # Look for low gap and high CV score\n                if scores['gap_percentage'] < 50 and scores['cv_score'] > 0.3:\n                    best_combinations.append({\n                        'prep': prep_name,\n                        'model': model_name,\n                        'cv_score': scores['cv_score'],\n                        'gap': scores['gap_percentage']\n                    })\n        \n        # Sort by CV score\n        best_combinations = sorted(best_combinations, key=lambda x: x['cv_score'], reverse=True)[:10]\n        \n        print(f\"\\nTop {len(best_combinations)} model combinations:\")\n        for combo in best_combinations:\n            print(f\"  {combo['prep']} + {combo['model']}: CV={combo['cv_score']:.4f}, Gap={combo['gap']:.1f}%\")\n        \n        # Train these models\n        trained_models = []\n        predictions = []\n        \n        model_classes = {\n            'Ridge': Ridge(alpha=1.0, random_state=42),\n            'Lasso': Lasso(alpha=0.1, random_state=42),\n            'ElasticNet': ElasticNet(alpha=0.1, random_state=42),\n            'RandomForest': RandomForestRegressor(n_estimators=200, max_depth=10, random_state=42),\n            'GradientBoosting': GradientBoostingRegressor(n_estimators=200, max_depth=5, random_state=42),\n            'XGBoost': xgb.XGBRegressor(n_estimators=200, max_depth=5, random_state=42, verbosity=0),\n            'LightGBM': lgb.LGBMRegressor(n_estimators=200, max_depth=5, random_state=42, verbose=-1)\n        }\n        \n        for combo in best_combinations:\n            X_train, X_test, y_train = self.preprocessing_strategies[combo['prep']]\n            model = model_classes[combo['model']]\n            \n            # Train model\n            model.fit(X_train, y_train)\n            \n            # Generate predictions\n            pred = model.predict(X_test)\n            \n            trained_models.append((combo, model))\n            predictions.append(pred)\n        \n        # Create ensemble\n        if predictions:\n            # Weight by CV score\n            weights = np.array([combo['cv_score'] for combo in best_combinations[:len(predictions)]])\n            weights = weights / weights.sum()\n            \n            ensemble_pred = np.average(predictions, axis=0, weights=weights)\n            \n            print(f\"\\nEnsemble created with {len(predictions)} models\")\n            print(f\"Prediction range: {ensemble_pred.min():.4f} to {ensemble_pred.max():.4f}\")\n            \n            return ensemble_pred, trained_models\n        \n        return None, []\n    \n    def generate_submission(self, predictions):\n        \"\"\"Generate submission file\"\"\"\n        submission = pd.DataFrame({\n            'id': range(1, len(predictions) + 1),\n            'prediction': predictions\n        })\n        \n        submission.to_csv('drw_submission.csv', index=False)\n        print(f\"\\nSubmission saved: drw_submission.csv\")\n        print(f\"Shape: {submission.shape}\")\n        print(f\"\\nPrediction statistics:\")\n        print(submission['prediction'].describe())\n        \n        return submission\n\n# Run the pipeline\ndef main():\n    predictor = DRWPredictor()\n    \n    # Load data\n    predictor.load_data()\n    \n    # Create preprocessing strategies\n    predictor.create_preprocessing_strategies()\n    \n    # Analyze train-CV gaps\n    predictor.analyze_train_cv_gaps()\n    \n    # Visualize gaps\n    gap_df, cv_df = predictor.visualize_gaps()\n    \n    # Train best models and create ensemble\n    ensemble_predictions, models = predictor.train_best_models()\n    \n    if ensemble_predictions is not None:\n        # Generate submission\n        submission = predictor.generate_submission(ensemble_predictions)\n        print(\"\\nPipeline completed successfully!\")\n    else:\n        print(\"\\nPipeline failed - no valid models trained\")\n\nif __name__ == \"__main__\":\n    main()","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}