{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-06-06T12:31:52.750144Z","iopub.execute_input":"2025-06-06T12:31:52.750502Z","iopub.status.idle":"2025-06-06T12:31:54.762437Z","shell.execute_reply.started":"2025-06-06T12:31:52.750470Z","shell.execute_reply":"2025-06-06T12:31:54.761285Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Memory-Efficient Baseline for DRW Crypto Market Prediction\nimport numpy as np\nimport pandas as pd\nimport xgboost as xgb\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_squared_error\nfrom scipy.stats import pearsonr\nimport gc\nimport warnings\nwarnings.filterwarnings('ignore')\n\nprint(\"🚀 Starting Memory-Efficient Baseline Model\")\n\n# =============================================================================\n# Memory optimization function\n# =============================================================================\ndef reduce_mem_usage(df, name=\"DataFrame\"):\n    \"\"\"Reduce memory usage of dataframe by downcasting numeric types\"\"\"\n    print(f\"Optimizing memory for {name}...\")\n    start_mem = df.memory_usage().sum() / 1024**2\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            \n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                else:\n                    df[col] = df[col].astype(np.int64)\n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float32)  # float16 can be unstable\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n    \n    end_mem = df.memory_usage().sum() / 1024**2\n    reduction = 100 * (start_mem - end_mem) / start_mem\n    print(f\"Memory usage: {start_mem:.1f}MB → {end_mem:.1f}MB ({reduction:.1f}% reduction)\")\n    return df\n\n# =============================================================================\n# Step 1: Load and Optimize Data\n# =============================================================================\nprint(\"\\n📂 Loading data...\")\ntrain = pd.read_parquet('/kaggle/input/drw-crypto-market-prediction/train.parquet')\ntest = pd.read_parquet('/kaggle/input/drw-crypto-market-prediction/test.parquet')\n\nprint(f\"Initial shapes - Train: {train.shape}, Test: {test.shape}\")\n\n# Optimize memory immediately\ntrain = reduce_mem_usage(train, \"train\")\ntest = reduce_mem_usage(test, \"test\")\n\n# =============================================================================\n# Step 2: Feature Selection (Keep only most important to save memory)\n# =============================================================================\nprint(\"\\n🎯 Feature selection for memory efficiency...\")\n\ntarget = 'label'\ny_train = train[target].astype(np.float32)\n\n# Get all feature columns\nall_features = [col for col in train.columns if col not in [target, 'ID', 'timestamp']]\nprint(f\"Total features available: {len(all_features)}\")\n\n# Quick feature selection based on correlation with target\nfeature_correlations = []\nsample_size = min(50000, len(train))  # Use sample for quick correlation calculation\ntrain_sample = train.sample(sample_size, random_state=42)\n\nprint(\"Calculating feature correlations (using sample)...\")\nfor col in all_features:\n    try:\n        corr = abs(train_sample[col].corr(train_sample[target]))\n        if not np.isnan(corr):\n            feature_correlations.append((col, corr))\n    except:\n        continue\n\n# Sort by correlation and take top features\nfeature_correlations.sort(key=lambda x: x[1], reverse=True)\ntop_n_features = min(200, len(feature_correlations))  # Limit to 200 features\nselected_features = [feat[0] for feat in feature_correlations[:top_n_features]]\n\nprint(f\"Selected top {len(selected_features)} features\")\nprint(f\"Top 5 features: {[f'{feat}({corr:.4f})' for feat, corr in feature_correlations[:5]]}\")\n\n# =============================================================================\n# Step 3: Prepare Final Datasets\n# =============================================================================\nprint(\"\\n🔧 Preparing final datasets...\")\n\nX_train = train[selected_features].astype(np.float32)\nX_test = test[selected_features].astype(np.float32)\n\n# Clean up original dataframes\ndel train, test\ngc.collect()\n\nprint(f\"Final shapes - X_train: {X_train.shape}, X_test: {X_test.shape}\")\n\n# =============================================================================\n# Step 4: Data Cleaning (Memory Efficient)\n# =============================================================================\nprint(\"\\n🧹 Data cleaning...\")\n\n# Handle missing values\nmissing_cols = X_train.columns[X_train.isnull().any()].tolist()\nif missing_cols:\n    print(f\"Filling {len(missing_cols)} columns with missing values...\")\n    for col in missing_cols:\n        median_val = X_train[col].median()\n        X_train[col].fillna(median_val, inplace=True)\n        X_test[col].fillna(median_val, inplace=True)\n\n# Handle infinite values\nprint(\"Checking for infinite values...\")\ninf_mask_train = np.isinf(X_train.values).any(axis=1)\ninf_mask_test = np.isinf(X_test.values).any(axis=1)\n\nif inf_mask_train.any() or inf_mask_test.any():\n    print(\"Replacing infinite values...\")\n    X_train.replace([np.inf, -np.inf], np.nan, inplace=True)\n    X_test.replace([np.inf, -np.inf], np.nan, inplace=True)\n    \n    # Fill with median again\n    for col in X_train.columns:\n        median_val = X_train[col].median()\n        X_train[col].fillna(median_val, inplace=True)\n        X_test[col].fillna(median_val, inplace=True)\n\nprint(\"✅ Data cleaning completed\")\n\n# =============================================================================\n# Step 5: Simple XGBoost Model\n# =============================================================================\nprint(\"\\n🎯 Training lightweight XGBoost model...\")\n\n# Memory-efficient XGBoost parameters\nxgb_params = {\n    'objective': 'reg:squarederror',\n    'eval_metric': 'rmse',\n    'tree_method': 'hist',  # Memory efficient\n    'max_depth': 4,         # Reduced depth\n    'learning_rate': 0.1,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'n_estimators': 100,    # Reduced trees\n    'random_state': 42,\n    'verbosity': 0,\n    'n_jobs': 1             # Single thread to save memory\n}\n\n# Simple train/validation split instead of full CV to save memory\nfrom sklearn.model_selection import train_test_split\n\nX_tr, X_val, y_tr, y_val = train_test_split(\n    X_train, y_train, test_size=0.2, random_state=42\n)\n\nprint(f\"Training set: {X_tr.shape}, Validation set: {X_val.shape}\")\n\n# Train model\nprint(\"Training XGBoost...\")\nmodel = xgb.XGBRegressor(**xgb_params)\nmodel.fit(\n    X_tr, y_tr,\n    eval_set=[(X_val, y_val)],\n    early_stopping_rounds=10,\n    verbose=False\n)\n\n# =============================================================================\n# Step 6: Evaluation\n# =============================================================================\nprint(\"\\n📊 Model evaluation...\")\n\n# Validation predictions\nval_pred = model.predict(X_val)\n\n# Calculate metrics\nval_rmse = np.sqrt(mean_squared_error(y_val, val_pred))\nval_corr = pearsonr(y_val, val_pred)[0]\n\nprint(f\"Validation RMSE: {val_rmse:.6f}\")\nprint(f\"Validation Correlation: {val_corr:.6f}\")\n\n# =============================================================================\n# Step 7: Make Predictions and Create Submission\n# =============================================================================\nprint(\"\\n🔮 Making test predictions...\")\n\ntest_predictions = model.predict(X_test)\n\n# Load sample submission\nsample_sub = pd.read_csv('/kaggle/input/drw-crypto-market-prediction/sample_submission.csv')\n\n# Create submission\nsubmission = pd.DataFrame({\n    sample_sub.columns[0]: sample_sub.iloc[:, 0],\n    'prediction': test_predictions.astype(np.float32)\n})\n\n# Save submission\nsubmission.to_csv('memory_efficient_baseline.csv', index=False)\n\nprint(\"✅ Submission saved as 'memory_efficient_baseline.csv'\")\n\n# =============================================================================\n# Step 8: Summary\n# =============================================================================\nprint(f\"\\n🎉 Memory-Efficient Baseline Summary:\")\nprint(f\"  📊 Validation Correlation: {val_corr:.6f}\")\nprint(f\"  📈 Validation RMSE: {val_rmse:.6f}\")\nprint(f\"  🎯 Features Used: {len(selected_features)}\")\nprint(f\"  🧠 Model: Lightweight XGBoost\")\nprint(f\"  💾 Memory: Optimized for Kaggle limits\")\n\nprint(f\"\\n💡 This baseline should run within memory limits!\")\nprint(f\"   Next steps: Gradually add more features/complexity\")\n\n# Clean up\ndel X_train, X_test, X_tr, X_val, y_tr, y_val\ngc.collect()\n\nprint(\"🎯 Memory cleanup completed\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-06T12:40:45.560446Z","iopub.execute_input":"2025-06-06T12:40:45.560846Z","iopub.status.idle":"2025-06-06T12:43:11.991764Z","shell.execute_reply.started":"2025-06-06T12:40:45.560736Z","shell.execute_reply":"2025-06-06T12:43:11.990214Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Memory-Efficient Baseline for DRW Crypto Market Prediction\nimport numpy as np\nimport pandas as pd\nimport xgboost as xgb\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_squared_error\nfrom scipy.stats import pearsonr\nimport gc\nimport warnings\nwarnings.filterwarnings('ignore')\n\nprint(\"🚀 Starting Memory-Efficient Baseline Model\")\n\n# =============================================================================\n# Memory optimization function\n# =============================================================================\ndef reduce_mem_usage(df, name=\"DataFrame\"):\n    \"\"\"Reduce memory usage of dataframe by downcasting numeric types\"\"\"\n    print(f\"Optimizing memory for {name}...\")\n    start_mem = df.memory_usage().sum() / 1024**2\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            \n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                else:\n                    df[col] = df[col].astype(np.int64)\n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float32)  # float16 can be unstable\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n    \n    end_mem = df.memory_usage().sum() / 1024**2\n    reduction = 100 * (start_mem - end_mem) / start_mem\n    print(f\"Memory usage: {start_mem:.1f}MB → {end_mem:.1f}MB ({reduction:.1f}% reduction)\")\n    return df\n\n# =============================================================================\n# Step 1: Load and Optimize Data\n# =============================================================================\nprint(\"\\n📂 Loading data...\")\ntrain = pd.read_parquet('/kaggle/input/drw-crypto-market-prediction/train.parquet')\ntest = pd.read_parquet('/kaggle/input/drw-crypto-market-prediction/test.parquet')\n\nprint(f\"Initial shapes - Train: {train.shape}, Test: {test.shape}\")\n\n# Optimize memory immediately\ntrain = reduce_mem_usage(train, \"train\")\ntest = reduce_mem_usage(test, \"test\")\n\n# =============================================================================\n# Step 2: Feature Selection (Keep only most important to save memory)\n# =============================================================================\nprint(\"\\n🎯 Feature selection for memory efficiency...\")\n\ntarget = 'label'\ny_train = train[target].astype(np.float32)\n\n# Get all feature columns\nall_features = [col for col in train.columns if col not in [target, 'ID', 'timestamp']]\nprint(f\"Total features available: {len(all_features)}\")\n\n# Quick feature selection based on correlation with target\nfeature_correlations = []\nsample_size = min(50000, len(train))  # Use sample for quick correlation calculation\ntrain_sample = train.sample(sample_size, random_state=42)\n\nprint(\"Calculating feature correlations (using sample)...\")\nfor col in all_features:\n    try:\n        corr = abs(train_sample[col].corr(train_sample[target]))\n        if not np.isnan(corr):\n            feature_correlations.append((col, corr))\n    except:\n        continue\n\n# Sort by correlation and take top features\nfeature_correlations.sort(key=lambda x: x[1], reverse=True)\ntop_n_features = min(50, len(feature_correlations))  # Limit to 50 features\nselected_features = [feat[0] for feat in feature_correlations[:top_n_features]]\n\nprint(f\"Selected top {len(selected_features)} features\")\nprint(f\"Top 5 features: {[f'{feat}({corr:.4f})' for feat, corr in feature_correlations[:5]]}\")\n\n# =============================================================================\n# Step 3: Prepare Final Datasets\n# =============================================================================\nprint(\"\\n🔧 Preparing final datasets...\")\n\nX_train = train[selected_features].astype(np.float32)\nX_test = test[selected_features].astype(np.float32)\n\n# Clean up original dataframes\ndel train, test\ngc.collect()\n\nprint(f\"Final shapes - X_train: {X_train.shape}, X_test: {X_test.shape}\")\n\n# =============================================================================\n# Step 4: Data Cleaning (Memory Efficient)\n# =============================================================================\nprint(\"\\n🧹 Data cleaning...\")\n\n# Handle missing values\nmissing_cols = X_train.columns[X_train.isnull().any()].tolist()\nif missing_cols:\n    print(f\"Filling {len(missing_cols)} columns with missing values...\")\n    for col in missing_cols:\n        median_val = X_train[col].median()\n        X_train[col].fillna(median_val, inplace=True)\n        X_test[col].fillna(median_val, inplace=True)\n\n# Handle infinite values\nprint(\"Checking for infinite values...\")\ninf_mask_train = np.isinf(X_train.values).any(axis=1)\ninf_mask_test = np.isinf(X_test.values).any(axis=1)\n\nif inf_mask_train.any() or inf_mask_test.any():\n    print(\"Replacing infinite values...\")\n    X_train.replace([np.inf, -np.inf], np.nan, inplace=True)\n    X_test.replace([np.inf, -np.inf], np.nan, inplace=True)\n    \n    # Fill with median again\n    for col in X_train.columns:\n        median_val = X_train[col].median()\n        X_train[col].fillna(median_val, inplace=True)\n        X_test[col].fillna(median_val, inplace=True)\n\nprint(\"✅ Data cleaning completed\")\n\n# =============================================================================\n# Step 5: Simple XGBoost Model\n# =============================================================================\nprint(\"\\n🎯 Training lightweight XGBoost model...\")\n\n# Memory-efficient XGBoost parameters\nxgb_params = {\n    'objective': 'reg:squarederror',\n    'eval_metric': 'rmse',\n    'tree_method': 'hist',  # Memory efficient\n    'max_depth': 3,         # Reduced depth\n    'learning_rate': 0.05,\n    'subsample': 0.7,\n    'colsample_bytree': 0.7,\n    'n_estimators': 50,    # Reduced trees\n    'random_state': 42,\n    'verbosity': 0,\n    'n_jobs': 1             # Single thread to save memory\n}\n\n# Simple train/validation split instead of full CV to save memory\nfrom sklearn.model_selection import train_test_split\n\nX_tr, X_val, y_tr, y_val = train_test_split(\n    X_train, y_train, test_size=0.2, random_state=42\n)\n\nprint(f\"Training set: {X_tr.shape}, Validation set: {X_val.shape}\")\n\n# Train model\nprint(\"Training XGBoost...\")\nmodel = xgb.XGBRegressor(**xgb_params)\nmodel.fit(\n    X_tr, y_tr,\n    eval_set=[(X_val, y_val)],\n    early_stopping_rounds=10,\n    verbose=False\n)\n\n# =============================================================================\n# Step 6: Evaluation\n# =============================================================================\nprint(\"\\n📊 Model evaluation...\")\n\n# Validation predictions\nval_pred = model.predict(X_val)\n\n# Calculate metrics\nval_rmse = np.sqrt(mean_squared_error(y_val, val_pred))\nval_corr = pearsonr(y_val, val_pred)[0]\n\nprint(f\"Validation RMSE: {val_rmse:.6f}\")\nprint(f\"Validation Correlation: {val_corr:.6f}\")\n\n# =============================================================================\n# Step 7: Make Predictions and Create Submission\n# =============================================================================\nprint(\"\\n🔮 Making test predictions...\")\n\ntest_predictions = model.predict(X_test)\n\n# Load sample submission\nsample_sub = pd.read_csv('/kaggle/input/drw-crypto-market-prediction/sample_submission.csv')\n\n# Create submission\nsubmission = pd.DataFrame({\n    sample_sub.columns[0]: sample_sub.iloc[:, 0],\n    'prediction': test_predictions.astype(np.float32)\n})\n\n# Save submission\nsubmission.to_csv('memory_efficient_baseline.csv', index=False)\n\nprint(\"✅ Submission saved as 'memory_efficient_baseline.csv'\")\n\n# =============================================================================\n# Step 8: Summary\n# =============================================================================\nprint(f\"\\n🎉 Memory-Efficient Baseline Summary:\")\nprint(f\"  📊 Validation Correlation: {val_corr:.6f}\")\nprint(f\"  📈 Validation RMSE: {val_rmse:.6f}\")\nprint(f\"  🎯 Features Used: {len(selected_features)}\")\nprint(f\"  🧠 Model: Lightweight XGBoost\")\nprint(f\"  💾 Memory: Optimized for Kaggle limits\")\n\nprint(f\"\\n💡 This baseline should run within memory limits!\")\nprint(f\"   Next steps: Gradually add more features/complexity\")\n\n# Clean up\ndel X_train, X_test, X_tr, X_val, y_tr, y_val\ngc.collect()\n\nprint(\"🎯 Memory cleanup completed\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-06T13:06:29.212272Z","iopub.execute_input":"2025-06-06T13:06:29.213752Z","iopub.status.idle":"2025-06-06T13:07:26.089124Z","shell.execute_reply.started":"2025-06-06T13:06:29.213700Z","shell.execute_reply":"2025-06-06T13:07:26.087700Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Improved Baseline with 4 Key Enhancements\nimport numpy as np\nimport pandas as pd\nimport xgboost as xgb\nimport lightgbm as lgb\nfrom sklearn.metrics import mean_squared_error\nfrom scipy.stats import pearsonr\nimport gc\nimport warnings\nwarnings.filterwarnings('ignore')\n\nprint(\"🚀 Starting Improved Baseline with 4 Key Enhancements\")\nprint(\"   1. 300 features (was 200)\")\nprint(\"   2. XGBoost + LightGBM ensemble\")\nprint(\"   3. Conservative hyperparameters\")\nprint(\"   4. Time-series validation\")\n\n# =============================================================================\n# Memory optimization function\n# =============================================================================\ndef reduce_mem_usage(df, name=\"DataFrame\"):\n    \"\"\"Reduce memory usage of dataframe by downcasting numeric types\"\"\"\n    print(f\"Optimizing memory for {name}...\")\n    start_mem = df.memory_usage().sum() / 1024**2\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            \n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                else:\n                    df[col] = df[col].astype(np.int64)\n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float32)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n    \n    end_mem = df.memory_usage().sum() / 1024**2\n    reduction = 100 * (start_mem - end_mem) / start_mem\n    print(f\"Memory usage: {start_mem:.1f}MB → {end_mem:.1f}MB ({reduction:.1f}% reduction)\")\n    return df\n\n# =============================================================================\n# Step 1: Load and Optimize Data\n# =============================================================================\nprint(\"\\n📂 Loading data...\")\ntrain = pd.read_parquet('/kaggle/input/drw-crypto-market-prediction/train.parquet')\ntest = pd.read_parquet('/kaggle/input/drw-crypto-market-prediction/test.parquet')\n\nprint(f\"Initial shapes - Train: {train.shape}, Test: {test.shape}\")\n\n# Optimize memory immediately\ntrain = reduce_mem_usage(train, \"train\")\ntest = reduce_mem_usage(test, \"test\")\n\n# =============================================================================\n# Step 2: Enhanced Feature Selection (Improvement #1: 300 features)\n# =============================================================================\nprint(\"\\n🎯 Enhanced feature selection (300 features)...\")\n\ntarget = 'label'\ny_train = train[target].astype(np.float32)\n\n# Get all feature columns\nall_features = [col for col in train.columns if col not in [target, 'ID', 'timestamp']]\nprint(f\"Total features available: {len(all_features)}\")\n\n# Quick feature selection based on correlation with target\nfeature_correlations = []\nsample_size = min(50000, len(train))\ntrain_sample = train.sample(sample_size, random_state=42)\n\nprint(\"Calculating feature correlations (using sample)...\")\nfor col in all_features:\n    try:\n        corr = abs(train_sample[col].corr(train_sample[target]))\n        if not np.isnan(corr):\n            feature_correlations.append((col, corr))\n    except:\n        continue\n\n# Sort by correlation and take top features\nfeature_correlations.sort(key=lambda x: x[1], reverse=True)\ntop_n_features = min(300, len(feature_correlations))  # Increased from 200 to 300\nselected_features = [feat[0] for feat in feature_correlations[:top_n_features]]\n\nprint(f\"Selected top {len(selected_features)} features (improved from 200)\")\nprint(f\"Top 5 features: {[f'{feat}({corr:.4f})' for feat, corr in feature_correlations[:5]]}\")\n\n# =============================================================================\n# Step 3: Prepare Final Datasets\n# =============================================================================\nprint(\"\\n🔧 Preparing final datasets...\")\n\nX_train = train[selected_features].astype(np.float32)\nX_test = test[selected_features].astype(np.float32)\n\n# Clean up original dataframes\ndel train, test\ngc.collect()\n\nprint(f\"Final shapes - X_train: {X_train.shape}, X_test: {X_test.shape}\")\n\n# =============================================================================\n# Step 4: Enhanced Data Cleaning\n# =============================================================================\nprint(\"\\n🧹 Enhanced data cleaning...\")\n\n# Handle missing values more carefully\nmissing_cols = X_train.columns[X_train.isnull().any()].tolist()\nif missing_cols:\n    print(f\"Filling {len(missing_cols)} columns with missing values...\")\n    for col in missing_cols:\n        # Use median for robustness\n        median_val = X_train[col].median()\n        if pd.isna(median_val):\n            median_val = 0.0\n        X_train[col].fillna(median_val, inplace=True)\n        X_test[col].fillna(median_val, inplace=True)\n\n# Handle infinite values more robustly\nprint(\"Handling infinite values...\")\nX_train.replace([np.inf, -np.inf], np.nan, inplace=True)\nX_test.replace([np.inf, -np.inf], np.nan, inplace=True)\n\n# Fill any remaining NaN with column median\nfor col in X_train.columns:\n    if X_train[col].isnull().any() or X_test[col].isnull().any():\n        median_val = X_train[col].median()\n        if pd.isna(median_val):\n            median_val = 0.0\n        X_train[col].fillna(median_val, inplace=True)\n        X_test[col].fillna(median_val, inplace=True)\n\nprint(\"✅ Enhanced data cleaning completed\")\n\n# =============================================================================\n# Step 5: Time-Series Validation (Improvement #4)\n# =============================================================================\nprint(\"\\n📊 Time-series validation (chronological split)...\")\n\n# Split data chronologically instead of random split\n# Use first 80% for training, last 20% for validation\nsplit_point = int(0.8 * len(X_train))\n\nX_tr = X_train.iloc[:split_point].copy()\nX_val = X_train.iloc[split_point:].copy()\ny_tr = y_train.iloc[:split_point].copy()\ny_val = y_train.iloc[split_point:].copy()\n\nprint(f\"Time-series split:\")\nprint(f\"  Training set: {X_tr.shape} (first 80% chronologically)\")\nprint(f\"  Validation set: {X_val.shape} (last 20% chronologically)\")\n\n# =============================================================================\n# Step 6: Conservative Model Parameters (Improvement #3)\n# =============================================================================\nprint(\"\\n🎯 Training models with conservative parameters...\")\n\n# Conservative XGBoost parameters to prevent overfitting\nxgb_params = {\n    'objective': 'reg:squarederror',\n    'eval_metric': 'rmse',\n    'tree_method': 'hist',\n    'max_depth': 3,            # Reduced from 4 to 3\n    'learning_rate': 0.05,     # Reduced from 0.1 to 0.05\n    'subsample': 0.7,          # Reduced from 0.8 to 0.7\n    'colsample_bytree': 0.7,   # Reduced from 0.8 to 0.7\n    'min_child_weight': 5,     # Increased from 1 to 5\n    'reg_alpha': 1,            # Increased from 0 to 1\n    'reg_lambda': 2,           # Increased from 1 to 2\n    'n_estimators': 150,       # Increased for slower learning\n    'random_state': 42,\n    'verbosity': 0,\n    'n_jobs': 1\n}\n\n# Conservative LightGBM parameters\nlgb_params = {\n    'objective': 'regression',\n    'metric': 'rmse',\n    'boosting_type': 'gbdt',\n    'max_depth': 3,            # Conservative depth\n    'learning_rate': 0.05,     # Slow learning\n    'subsample': 0.7,          # Conservative sampling\n    'colsample_bytree': 0.7,   # Conservative feature sampling\n    'min_child_samples': 20,   # Increased for regularization\n    'reg_alpha': 1,            # L1 regularization\n    'reg_lambda': 2,           # L2 regularization\n    'n_estimators': 150,       # More trees with slower learning\n    'random_state': 42,\n    'verbosity': -1,\n    'n_jobs': 1\n}\n\nprint(\"Conservative parameters applied:\")\nprint(\"  - Reduced max_depth (3)\")\nprint(\"  - Slower learning_rate (0.05)\")\nprint(\"  - Stronger regularization\")\nprint(\"  - Conservative sampling rates\")\n\n# =============================================================================\n# Step 7: Ensemble Training (Improvement #2: XGBoost + LightGBM)\n# =============================================================================\nprint(\"\\n🔄 Training ensemble models...\")\n\n# Train XGBoost\nprint(\"Training XGBoost...\")\nxgb_model = xgb.XGBRegressor(**xgb_params)\nxgb_model.fit(\n    X_tr, y_tr,\n    eval_set=[(X_val, y_val)],\n    early_stopping_rounds=20,\n    verbose=False\n)\n\n# Get XGBoost predictions\nxgb_val_pred = xgb_model.predict(X_val)\nxgb_test_pred = xgb_model.predict(X_test)\n\n# Calculate XGBoost scores\nxgb_val_rmse = np.sqrt(mean_squared_error(y_val, xgb_val_pred))\nxgb_val_corr = pearsonr(y_val, xgb_val_pred)[0]\n\nprint(f\"XGBoost - RMSE: {xgb_val_rmse:.6f}, Correlation: {xgb_val_corr:.6f}\")\n\n# Train LightGBM\nprint(\"Training LightGBM...\")\nlgb_model = lgb.LGBMRegressor(**lgb_params)\nlgb_model.fit(\n    X_tr, y_tr,\n    eval_set=[(X_val, y_val)],\n    callbacks=[lgb.early_stopping(20), lgb.log_evaluation(0)]\n)\n\n# Get LightGBM predictions\nlgb_val_pred = lgb_model.predict(X_val)\nlgb_test_pred = lgb_model.predict(X_test)\n\n# Calculate LightGBM scores\nlgb_val_rmse = np.sqrt(mean_squared_error(y_val, lgb_val_pred))\nlgb_val_corr = pearsonr(y_val, lgb_val_pred)[0]\n\nprint(f\"LightGBM - RMSE: {lgb_val_rmse:.6f}, Correlation: {lgb_val_corr:.6f}\")\n\n# =============================================================================\n# Step 8: Ensemble Combination\n# =============================================================================\nprint(\"\\n🔗 Creating ensemble...\")\n\n# Simple average ensemble (you could optimize weights here)\nensemble_val_pred = (xgb_val_pred + lgb_val_pred) / 2\nensemble_test_pred = (xgb_test_pred + lgb_test_pred) / 2\n\n# Calculate ensemble scores\nensemble_val_rmse = np.sqrt(mean_squared_error(y_val, ensemble_val_pred))\nensemble_val_corr = pearsonr(y_val, ensemble_val_pred)[0]\n\nprint(f\"Ensemble - RMSE: {ensemble_val_rmse:.6f}, Correlation: {ensemble_val_corr:.6f}\")\n\n# =============================================================================\n# Step 9: Model Selection (Best performing model)\n# =============================================================================\nprint(\"\\n🏆 Model selection...\")\n\nmodels_performance = {\n    'XGBoost': xgb_val_corr,\n    'LightGBM': lgb_val_corr,\n    'Ensemble': ensemble_val_corr\n}\n\nbest_model = max(models_performance, key=models_performance.get)\nprint(f\"Best model: {best_model} (Correlation: {models_performance[best_model]:.6f})\")\n\n# Select predictions from best model\nif best_model == 'XGBoost':\n    final_predictions = xgb_test_pred\nelif best_model == 'LightGBM':\n    final_predictions = lgb_test_pred\nelse:\n    final_predictions = ensemble_test_pred\n\n# =============================================================================\n# Step 10: Create Enhanced Submission\n# =============================================================================\nprint(\"\\n💾 Creating enhanced submission...\")\n\n# Load sample submission\nsample_sub = pd.read_csv('/kaggle/input/drw-crypto-market-prediction/sample_submission.csv')\n\n# Create submission with best model\nsubmission = pd.DataFrame({\n    sample_sub.columns[0]: sample_sub.iloc[:, 0],\n    'prediction': final_predictions.astype(np.float32)\n})\n\n# Save submission\nsubmission.to_csv('improved_baseline_v2.csv', index=False)\n\nprint(\"✅ Submission saved as 'improved_baseline_v2.csv'\")\n\n# =============================================================================\n# Step 11: Enhanced Summary\n# =============================================================================\nprint(f\"\\n🎉 Enhanced Baseline Summary:\")\nprint(f\"  🔧 Improvements Applied:\")\nprint(f\"     1. Features: 200 → {len(selected_features)}\")\nprint(f\"     2. Models: XGBoost + LightGBM ensemble\")\nprint(f\"     3. Conservative hyperparameters\")\nprint(f\"     4. Time-series validation\")\nprint(f\"\")\nprint(f\"  📊 Individual Model Performance:\")\nprint(f\"     XGBoost:  {xgb_val_corr:.6f}\")\nprint(f\"     LightGBM: {lgb_val_corr:.6f}\")\nprint(f\"     Ensemble: {ensemble_val_corr:.6f}\")\nprint(f\"\")\nprint(f\"  🏆 Best Model: {best_model}\")\nprint(f\"  📈 Best Correlation: {models_performance[best_model]:.6f}\")\nprint(f\"  💾 Memory: Optimized for Kaggle limits\")\n\nprint(f\"\\n🎯 Expected Improvements:\")\nprint(f\"  - More stable predictions (time-series validation)\")\nprint(f\"  - Better generalization (conservative parameters)\")\nprint(f\"  - Higher accuracy (more features + ensemble)\")\nprint(f\"  - Reduced overfitting (regularization)\")\n\n# Feature importance analysis\nprint(f\"\\n🔍 Top 10 Most Important Features:\")\nif best_model == 'XGBoost':\n    importance = xgb_model.feature_importances_\nelif best_model == 'LightGBM':\n    importance = lgb_model.feature_importances_\nelse:\n    # Average importance for ensemble\n    importance = (xgb_model.feature_importances_ + lgb_model.feature_importances_) / 2\n\nimportance_df = pd.DataFrame({\n    'feature': selected_features,\n    'importance': importance\n}).sort_values('importance', ascending=False)\n\nprint(importance_df.head(10).to_string(index=False))\n\n# Clean up\ndel X_train, X_test, X_tr, X_val, y_tr, y_val\ngc.collect()\n\nprint(\"\\n🎯 Enhanced baseline completed!\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-06T13:16:09.462068Z","iopub.execute_input":"2025-06-06T13:16:09.462402Z","iopub.status.idle":"2025-06-06T13:18:41.020263Z","shell.execute_reply.started":"2025-06-06T13:16:09.462376Z","shell.execute_reply":"2025-06-06T13:18:41.018304Z"}},"outputs":[],"execution_count":null}]}