{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":96164,"databundleVersionId":12993472,"sourceType":"competition"},{"sourceId":258969650,"sourceType":"kernelVersion"}],"dockerImageVersionId":31090,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-08-30T00:43:29.788755Z","iopub.execute_input":"2025-08-30T00:43:29.788960Z","iopub.status.idle":"2025-08-30T00:43:32.337227Z","shell.execute_reply.started":"2025-08-30T00:43:29.788934Z","shell.execute_reply":"2025-08-30T00:43:32.336455Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import pandas as pd\n# import numpy as np\n# import os\n# from sklearn.linear_model import Ridge, LinearRegression, ElasticNet, Lasso\n# from sklearn.preprocessing import StandardScaler\n# from sklearn.model_selection import TimeSeriesSplit, cross_val_score\n# from sklearn.metrics import mean_squared_error\n# import joblib\n# import warnings\n# warnings.filterwarnings('ignore')\n\n# # ==================== Configuration ====================\n# DATA_PATH = '/kaggle/input/drw-remix-of-winning-strategies-part-1/'\n# OUTPUT_PATH = '/kaggle/working/model_1/'\n# SAMPLE_SUB_PATH = '/kaggle/input/drw-crypto-market-prediction/sample_submission.csv'\n\n# # Create output directory\n# os.makedirs(OUTPUT_PATH, exist_ok=True)\n# print(f\"Output directory: {OUTPUT_PATH}\")\n\n# # ==================== Load Data ====================\n# print(\"=\"*60)\n# print(\"LOADING DATA\")\n# print(\"=\"*60)\n\n# # Load processed data\n# train_df = pd.read_parquet(f'{DATA_PATH}train_final.parquet')\n# test_df = pd.read_parquet(f'{DATA_PATH}test_final.parquet')\n\n# print(f\"Train shape: {train_df.shape}\")\n# print(f\"Test shape: {test_df.shape}\")\n\n# # Load sample submission for format\n# sample_sub = pd.read_csv(SAMPLE_SUB_PATH)\n# print(f\"Sample submission shape: {sample_sub.shape}\")\n\n# # ==================== Prepare Features ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"PREPARING FEATURES\")\n# print(\"=\"*60)\n\n# # Separate features and target\n# feature_cols = [col for col in train_df.columns if col not in ['label', 'timestamp', 'id']]\n# print(f\"Number of features: {len(feature_cols)}\")\n\n# X_train = train_df[feature_cols].values\n# y_train = train_df['label'].values\n# X_test = test_df[feature_cols].values\n\n# print(f\"X_train shape: {X_train.shape}\")\n# print(f\"y_train shape: {y_train.shape}\")\n# print(f\"X_test shape: {X_test.shape}\")\n\n# # Check for any remaining NaN or INF\n# print(\"\\nData quality check:\")\n# print(f\"Train NaN: {np.isnan(X_train).sum()}\")\n# print(f\"Train INF: {np.isinf(X_train).sum()}\")\n# print(f\"Test NaN: {np.isnan(X_test).sum()}\")\n# print(f\"Test INF: {np.isinf(X_test).sum()}\")\n\n# # Replace any remaining NaN/INF\n# X_train = np.nan_to_num(X_train, nan=0.0, posinf=0.0, neginf=0.0)\n# X_test = np.nan_to_num(X_test, nan=0.0, posinf=0.0, neginf=0.0)\n\n# # ==================== Feature Scaling ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"FEATURE SCALING\")\n# print(\"=\"*60)\n\n# scaler = StandardScaler()\n# X_train_scaled = scaler.fit_transform(X_train)\n# X_test_scaled = scaler.transform(X_test)\n\n# print(\"✓ Features scaled using StandardScaler\")\n\n# # Save scaler\n# joblib.dump(scaler, f'{OUTPUT_PATH}scaler.pkl')\n# print(f\"✓ Scaler saved to {OUTPUT_PATH}scaler.pkl\")\n\n# # ==================== Model Training ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"TRAINING LINEAR MODELS\")\n# print(\"=\"*60)\n\n# # Define models to try\n# models = {\n#     'ridge': Ridge(alpha=1.0, random_state=42),\n#     'ridge_strong': Ridge(alpha=10.0, random_state=42),\n#     'elastic': ElasticNet(alpha=1.0, l1_ratio=0.5, random_state=42, max_iter=2000),\n#     'lasso': Lasso(alpha=0.1, random_state=42, max_iter=2000)\n# }\n\n# # Time series cross-validation\n# tscv = TimeSeriesSplit(n_splits=3)\n\n# best_model = None\n# best_score = -np.inf\n# cv_results = {}\n\n# print(\"\\nCross-validation results:\")\n# print(\"-\" * 40)\n\n# for name, model in models.items():\n#     # Cross-validation\n#     scores = []\n    \n#     for fold, (train_idx, val_idx) in enumerate(tscv.split(X_train_scaled), 1):\n#         X_tr, X_val = X_train_scaled[train_idx], X_train_scaled[val_idx]\n#         y_tr, y_val = y_train[train_idx], y_train[val_idx]\n        \n#         # Fit model\n#         model.fit(X_tr, y_tr)\n        \n#         # Predict\n#         y_pred = model.predict(X_val)\n        \n#         # Calculate Pearson correlation (competition metric)\n#         pearson_corr = np.corrcoef(y_val, y_pred)[0, 1]\n#         scores.append(pearson_corr)\n    \n#     mean_score = np.mean(scores)\n#     std_score = np.std(scores)\n    \n#     cv_results[name] = {\n#         'mean': mean_score,\n#         'std': std_score,\n#         'scores': scores\n#     }\n    \n#     print(f\"{name:15s}: {mean_score:.6f} (+/- {std_score:.6f})\")\n#     print(f\"  Fold scores: {[f'{s:.6f}' for s in scores]}\")\n    \n#     if mean_score > best_score:\n#         best_score = mean_score\n#         best_model = name\n\n# print(\"-\" * 40)\n# print(f\"\\nBest model: {best_model} (CV Score: {best_score:.6f})\")\n\n# # ==================== Train Final Model ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"TRAINING FINAL MODEL\")\n# print(\"=\"*60)\n\n# # Use best model configuration\n# if best_model == 'ridge':\n#     final_model = Ridge(alpha=1.0, random_state=42)\n# elif best_model == 'ridge_strong':\n#     final_model = Ridge(alpha=10.0, random_state=42)\n# elif best_model == 'elastic':\n#     final_model = ElasticNet(alpha=1.0, l1_ratio=0.5, random_state=42, max_iter=2000)\n# else:  # lasso\n#     final_model = Lasso(alpha=0.1, random_state=42, max_iter=2000)\n\n# print(f\"Training {best_model} on full dataset...\")\n\n# # Train on all data\n# final_model.fit(X_train_scaled, y_train)\n\n# # Training score\n# train_pred = final_model.predict(X_train_scaled)\n# train_corr = np.corrcoef(y_train, train_pred)[0, 1]\n# train_rmse = np.sqrt(mean_squared_error(y_train, train_pred))\n\n# print(f\"\\nTraining metrics:\")\n# print(f\"  Pearson correlation: {train_corr:.6f}\")\n# print(f\"  RMSE: {train_rmse:.6f}\")\n\n# # Save model\n# joblib.dump(final_model, f'{OUTPUT_PATH}linear_model.pkl')\n# print(f\"\\n✓ Model saved to {OUTPUT_PATH}linear_model.pkl\")\n\n# # ==================== Feature Importance (for Ridge/Lasso) ====================\n# if hasattr(final_model, 'coef_'):\n#     print(\"\\n\" + \"=\"*60)\n#     print(\"TOP FEATURE IMPORTANCE\")\n#     print(\"=\"*60)\n    \n#     # Get coefficients\n#     coefficients = np.abs(final_model.coef_)\n    \n#     # Get top features\n#     top_indices = np.argsort(coefficients)[-20:][::-1]\n    \n#     print(\"\\nTop 20 most important features (by absolute coefficient):\")\n#     print(\"-\" * 40)\n#     for idx in top_indices:\n#         print(f\"  {feature_cols[idx]:20s}: {coefficients[idx]:.6f}\")\n    \n#     # Count non-zero coefficients (especially for Lasso)\n#     non_zero = np.sum(coefficients > 1e-10)\n#     print(f\"\\nNon-zero coefficients: {non_zero}/{len(coefficients)}\")\n\n# # ==================== Make Predictions ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"GENERATING PREDICTIONS\")\n# print(\"=\"*60)\n\n# # Predict on test set\n# test_predictions = final_model.predict(X_test_scaled)\n\n# print(f\"Predictions shape: {test_predictions.shape}\")\n# print(f\"\\nPrediction statistics:\")\n# print(f\"  Mean:  {test_predictions.mean():.6f}\")\n# print(f\"  Std:   {test_predictions.std():.6f}\")\n# print(f\"  Min:   {test_predictions.min():.6f}\")\n# print(f\"  Max:   {test_predictions.max():.6f}\")\n# print(f\"  25%:   {np.percentile(test_predictions, 25):.6f}\")\n# print(f\"  50%:   {np.percentile(test_predictions, 50):.6f}\")\n# print(f\"  75%:   {np.percentile(test_predictions, 75):.6f}\")\n\n# # ==================== Create Submission ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"CREATING SUBMISSION\")\n# print(\"=\"*60)\n\n# # Create submission DataFrame\n# submission = pd.DataFrame({\n#     'id': range(1, len(test_predictions) + 1),\n#     'prediction': test_predictions\n# })\n\n# # Verify submission format\n# assert submission.shape[0] == sample_sub.shape[0], \"Submission row count mismatch!\"\n# assert submission.shape[1] == 2, \"Submission should have 2 columns!\"\n# assert list(submission.columns) == ['id', 'prediction'], \"Wrong column names!\"\n\n# print(f\"Submission shape: {submission.shape}\")\n# print(f\"Submission columns: {submission.columns.tolist()}\")\n# print(f\"\\nFirst 5 rows:\")\n# print(submission.head())\n\n# # Save submission\n# submission_path = f'{OUTPUT_PATH}submission.csv'\n# submission.to_csv(submission_path, index=False)\n# print(f\"\\n✓ Submission saved to {submission_path}\")\n\n# # ==================== Save Training Report ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"SAVING TRAINING REPORT\")\n# print(\"=\"*60)\n\n# report = f\"\"\"\n# DRW Crypto Market Prediction - Linear Model Training Report\n# ============================================================\n\n# Data Information:\n# - Training samples: {len(X_train):,}\n# - Test samples: {len(X_test):,}\n# - Number of features: {len(feature_cols):,}\n\n# Cross-Validation Results:\n# \"\"\"\n\n# for name, results in cv_results.items():\n#     report += f\"\\n{name}:\"\n#     report += f\"\\n  Mean CV Score: {results['mean']:.6f}\"\n#     report += f\"\\n  Std CV Score: {results['std']:.6f}\"\n#     report += f\"\\n  Fold Scores: {results['scores']}\"\n\n# report += f\"\"\"\n\n# Final Model: {best_model}\n# Best CV Score: {best_score:.6f}\n\n# Training Performance:\n# - Pearson Correlation: {train_corr:.6f}\n# - RMSE: {train_rmse:.6f}\n\n# Test Predictions:\n# - Mean: {test_predictions.mean():.6f}\n# - Std: {test_predictions.std():.6f}\n# - Min: {test_predictions.min():.6f}\n# - Max: {test_predictions.max():.6f}\n\n# Files Generated:\n# - {OUTPUT_PATH}linear_model.pkl\n# - {OUTPUT_PATH}scaler.pkl\n# - {OUTPUT_PATH}submission.csv\n# - {OUTPUT_PATH}training_report.txt\n# \"\"\"\n\n# # Save report\n# with open(f'{OUTPUT_PATH}training_report.txt', 'w') as f:\n#     f.write(report)\n\n# print(\"✓ Training report saved\")\n\n# # ==================== Save Predictions for Ensemble ====================\n# # Save predictions for potential ensemble later\n# np.save(f'{OUTPUT_PATH}train_predictions.npy', train_pred)\n# np.save(f'{OUTPUT_PATH}test_predictions.npy', test_predictions)\n# print(\"✓ Predictions saved for potential ensemble\")\n\n# # ==================== Summary ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"TRAINING COMPLETE!\")\n# print(\"=\"*60)\n# print(f\"\\n✓ All files saved to: {OUTPUT_PATH}\")\n# print(f\"✓ Best model: {best_model}\")\n# print(f\"✓ CV Score: {best_score:.6f}\")\n# print(f\"✓ Submission ready: {submission_path}\")\n\n# print(\"\\nFiles created:\")\n# for file in os.listdir(OUTPUT_PATH):\n#     file_path = os.path.join(OUTPUT_PATH, file)\n#     file_size = os.path.getsize(file_path) / 1024  # KB\n#     print(f\"  - {file:30s} ({file_size:.1f} KB)\")\n\n# print(\"\\n\" + \"=\"*60)\n# print(\"Ready to submit to Kaggle!\")\n# print(\"=\"*60)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import pandas as pd\n# import numpy as np\n# import os\n# from sklearn.linear_model import Ridge, LinearRegression, ElasticNet, Lasso\n# from sklearn.preprocessing import StandardScaler\n# from sklearn.model_selection import TimeSeriesSplit\n# from sklearn.metrics import mean_squared_error, pairwise_distances\n# from sklearn.cluster import KMeans  # Alternative to KMedoids\n# from scipy.cluster.hierarchy import linkage, fcluster\n# from scipy.spatial.distance import squareform\n# import joblib\n# import warnings\n# warnings.filterwarnings('ignore')\n\n# # ==================== Configuration ====================\n# DATA_PATH = '/kaggle/input/drw-remix-of-winning-strategies-part-1/'\n# OUTPUT_PATH = '/kaggle/working/model_2/'\n# SAMPLE_SUB_PATH = '/kaggle/input/drw-crypto-market-prediction/sample_submission.csv'\n\n# # Create output directory\n# os.makedirs(OUTPUT_PATH, exist_ok=True)\n# print(f\"Output directory: {OUTPUT_PATH}\")\n\n# # ==================== Custom KMedoids Implementation ====================\n# class SimpleKMedoids:\n#     \"\"\"\n#     Simple KMedoids implementation without external dependencies.\n#     \"\"\"\n#     def __init__(self, n_clusters, max_iter=100, random_state=42):\n#         self.n_clusters = n_clusters\n#         self.max_iter = max_iter\n#         self.random_state = random_state\n#         self.medoid_indices = None\n        \n#     def fit(self, distance_matrix):\n#         \"\"\"\n#         Fit KMedoids using precomputed distance matrix.\n#         \"\"\"\n#         n_samples = distance_matrix.shape[0]\n#         np.random.seed(self.random_state)\n        \n#         # Initialize medoids randomly\n#         self.medoid_indices = np.random.choice(n_samples, self.n_clusters, replace=False)\n        \n#         for iteration in range(self.max_iter):\n#             # Assign points to nearest medoid\n#             clusters = np.argmin(distance_matrix[:, self.medoid_indices], axis=1)\n            \n#             # Update medoids\n#             new_medoids = []\n#             for k in range(self.n_clusters):\n#                 cluster_points = np.where(clusters == k)[0]\n#                 if len(cluster_points) > 0:\n#                     # Find point with minimum sum of distances to other points in cluster\n#                     cluster_distances = distance_matrix[np.ix_(cluster_points, cluster_points)]\n#                     sum_distances = np.sum(cluster_distances, axis=1)\n#                     new_medoid = cluster_points[np.argmin(sum_distances)]\n#                     new_medoids.append(new_medoid)\n#                 else:\n#                     new_medoids.append(self.medoid_indices[k])\n            \n#             new_medoids = np.array(new_medoids)\n            \n#             # Check for convergence\n#             if np.array_equal(new_medoids, self.medoid_indices):\n#                 break\n                \n#             self.medoid_indices = new_medoids\n        \n#         return self\n    \n#     def predict(self, distance_matrix):\n#         \"\"\"\n#         Predict cluster labels based on distance to medoids.\n#         \"\"\"\n#         return np.argmin(distance_matrix[:, self.medoid_indices], axis=1)\n\n# # ==================== Medoid-based Feature Selection ====================\n# class MedoidFeatureSelector:\n#     \"\"\"\n#     Reduce collinearity by selecting medoid features from correlated groups.\n#     \"\"\"\n    \n#     def __init__(self, correlation_threshold=0.85, min_cluster_size=2, method='hierarchical'):\n#         self.correlation_threshold = correlation_threshold\n#         self.min_cluster_size = min_cluster_size\n#         self.method = method\n#         self.selected_features = None\n#         self.feature_clusters = None\n#         self.medoid_indices = None\n        \n#     def fit(self, X, feature_names):\n#         \"\"\"\n#         Identify medoids from correlated feature groups.\n#         \"\"\"\n#         print(\"\\n--- Medoid Feature Selection ---\")\n#         print(f\"Original features: {len(feature_names)}\")\n        \n#         # Calculate correlation matrix\n#         print(\"Calculating correlation matrix...\")\n#         corr_matrix = np.corrcoef(X.T)\n        \n#         # Handle any NaN values in correlation matrix\n#         corr_matrix = np.nan_to_num(corr_matrix, nan=0.0)\n        \n#         # Convert to distance matrix (1 - |correlation|)\n#         distance_matrix = 1 - np.abs(corr_matrix)\n        \n#         if self.method == 'hierarchical':\n#             # Hierarchical clustering based on correlation\n#             print(\"Performing hierarchical clustering...\")\n            \n#             # Convert to condensed distance matrix\n#             condensed_dist = squareform(distance_matrix, checks=False)\n            \n#             # Perform hierarchical clustering\n#             Z = linkage(condensed_dist, method='average')\n            \n#             # Form clusters based on threshold\n#             clusters = fcluster(Z, 1 - self.correlation_threshold, criterion='distance')\n            \n#             self.feature_clusters = {}\n#             for idx, cluster_id in enumerate(clusters):\n#                 if cluster_id not in self.feature_clusters:\n#                     self.feature_clusters[cluster_id] = []\n#                 self.feature_clusters[cluster_id].append(idx)\n                \n#         else:  # Use custom KMedoids or KMeans as fallback\n#             print(\"Performing clustering...\")\n            \n#             # Estimate number of clusters\n#             n_clusters = self._estimate_n_clusters(distance_matrix)\n#             print(f\"Estimated clusters: {n_clusters}\")\n            \n#             # Use custom KMedoids implementation\n#             kmedoids = SimpleKMedoids(n_clusters=n_clusters, random_state=42)\n#             kmedoids.fit(distance_matrix)\n#             cluster_labels = kmedoids.predict(distance_matrix)\n            \n#             self.feature_clusters = {}\n#             for idx, cluster_id in enumerate(cluster_labels):\n#                 if cluster_id not in self.feature_clusters:\n#                     self.feature_clusters[cluster_id] = []\n#                 self.feature_clusters[cluster_id].append(idx)\n        \n#         # Select medoids from each cluster\n#         self.medoid_indices = []\n#         self.selected_features = []\n#         cluster_info = []\n        \n#         print(f\"\\nFound {len(self.feature_clusters)} feature clusters\")\n        \n#         for cluster_id, feature_indices in self.feature_clusters.items():\n#             if len(feature_indices) >= self.min_cluster_size:\n#                 # Find medoid (most central point in cluster)\n#                 medoid_idx = self._find_medoid(distance_matrix, feature_indices)\n                \n#                 # Store cluster information\n#                 cluster_info.append({\n#                     'cluster_id': cluster_id,\n#                     'size': len(feature_indices),\n#                     'medoid': feature_names[medoid_idx],\n#                     'medoid_idx': medoid_idx\n#                 })\n                \n#                 self.medoid_indices.append(medoid_idx)\n#                 self.selected_features.append(feature_names[medoid_idx])\n#             else:\n#                 # Keep all features from small clusters\n#                 for idx in feature_indices:\n#                     cluster_info.append({\n#                         'cluster_id': cluster_id,\n#                         'size': 1,\n#                         'medoid': feature_names[idx],\n#                         'medoid_idx': idx\n#                     })\n                    \n#                     self.medoid_indices.append(idx)\n#                     self.selected_features.append(feature_names[idx])\n        \n#         # Remove duplicates while preserving order\n#         seen = set()\n#         unique_medoids = []\n#         unique_features = []\n#         for idx, feat in zip(self.medoid_indices, self.selected_features):\n#             if idx not in seen:\n#                 seen.add(idx)\n#                 unique_medoids.append(idx)\n#                 unique_features.append(feat)\n        \n#         self.medoid_indices = unique_medoids\n#         self.selected_features = unique_features\n        \n#         # Sort cluster info by size\n#         cluster_info = sorted(cluster_info, key=lambda x: x['size'], reverse=True)\n        \n#         # Print cluster summary\n#         print(\"\\nTop 10 largest feature clusters:\")\n#         print(\"-\" * 50)\n#         for info in cluster_info[:10]:\n#             print(f\"  Cluster {info['cluster_id']:3d}: {info['size']:3d} features -> medoid: {info['medoid']}\")\n        \n#         print(f\"\\nSelected {len(self.selected_features)} medoid features from {len(feature_names)} original features\")\n#         print(f\"Reduction: {100*(1 - len(self.selected_features)/len(feature_names)):.1f}%\")\n        \n#         return self\n    \n#     def _estimate_n_clusters(self, distance_matrix):\n#         \"\"\"\n#         Estimate optimal number of clusters using simple heuristic.\n#         \"\"\"\n#         # Simple heuristic: sqrt(n_features/2)\n#         n_features = len(distance_matrix)\n#         n_clusters = int(np.sqrt(n_features / 2))\n        \n#         # Bound between reasonable limits\n#         n_clusters = max(10, min(n_clusters, n_features // 10))\n        \n#         return n_clusters\n    \n#     def _find_medoid(self, distance_matrix, indices):\n#         \"\"\"\n#         Find the medoid (most central point) within a cluster.\n#         \"\"\"\n#         if len(indices) == 1:\n#             return indices[0]\n        \n#         # Calculate sum of distances to all other points in cluster\n#         cluster_distances = distance_matrix[np.ix_(indices, indices)]\n#         sum_distances = np.sum(cluster_distances, axis=1)\n        \n#         # Medoid is the point with minimum sum of distances\n#         medoid_idx_in_cluster = np.argmin(sum_distances)\n#         return indices[medoid_idx_in_cluster]\n    \n#     def transform(self, X):\n#         \"\"\"\n#         Transform data to selected medoid features.\n#         \"\"\"\n#         return X[:, self.medoid_indices]\n\n# # ==================== Load Data ====================\n# print(\"=\"*60)\n# print(\"LOADING DATA FOR MEDOID-BASED MODEL\")\n# print(\"=\"*60)\n\n# # Load processed data\n# train_df = pd.read_parquet(f'{DATA_PATH}train_final.parquet')\n# test_df = pd.read_parquet(f'{DATA_PATH}test_final.parquet')\n\n# print(f\"Train shape: {train_df.shape}\")\n# print(f\"Test shape: {test_df.shape}\")\n\n# # Load sample submission\n# sample_sub = pd.read_csv(SAMPLE_SUB_PATH)\n\n# # ==================== Prepare Features ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"PREPARING FEATURES\")\n# print(\"=\"*60)\n\n# # Separate features and target\n# feature_cols = [col for col in train_df.columns if col not in ['label', 'timestamp', 'id']]\n# print(f\"Original features: {len(feature_cols)}\")\n\n# X_train = train_df[feature_cols].values\n# y_train = train_df['label'].values\n# X_test = test_df[feature_cols].values\n\n# # Handle NaN/INF\n# X_train = np.nan_to_num(X_train, nan=0.0, posinf=0.0, neginf=0.0)\n# X_test = np.nan_to_num(X_test, nan=0.0, posinf=0.0, neginf=0.0)\n\n# # ==================== Feature Scaling ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"FEATURE SCALING\")\n# print(\"=\"*60)\n\n# scaler_full = StandardScaler()\n# X_train_scaled = scaler_full.fit_transform(X_train)\n# X_test_scaled = scaler_full.transform(X_test)\n\n# print(\"✓ Features scaled\")\n\n# # ==================== Medoid Feature Selection ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"MEDOID-BASED FEATURE SELECTION\")\n# print(\"=\"*60)\n\n# # Initialize selector with different thresholds to compare\n# thresholds = [0.80, 0.85, 0.90]\n# best_threshold = None\n# best_cv_score = -np.inf\n# best_selector = None\n# best_X_train = None\n# best_X_test = None\n\n# for threshold in thresholds:\n#     print(f\"\\n--- Testing correlation threshold: {threshold} ---\")\n    \n#     # Create selector\n#     selector = MedoidFeatureSelector(\n#         correlation_threshold=threshold,\n#         min_cluster_size=2,\n#         method='hierarchical'  # Use hierarchical which doesn't need sklearn_extra\n#     )\n    \n#     # Fit selector\n#     selector.fit(X_train_scaled, feature_cols)\n    \n#     # Transform data\n#     X_train_medoid = selector.transform(X_train_scaled)\n#     X_test_medoid = selector.transform(X_test_scaled)\n    \n#     print(f\"Features after medoid selection: {X_train_medoid.shape[1]}\")\n    \n#     # Quick cross-validation with Ridge\n#     model = Ridge(alpha=1.0, random_state=42)\n#     tscv = TimeSeriesSplit(n_splits=3)\n    \n#     scores = []\n#     for train_idx, val_idx in tscv.split(X_train_medoid):\n#         X_tr, X_val = X_train_medoid[train_idx], X_train_medoid[val_idx]\n#         y_tr, y_val = y_train[train_idx], y_train[val_idx]\n        \n#         model.fit(X_tr, y_tr)\n#         y_pred = model.predict(X_val)\n#         pearson_corr = np.corrcoef(y_val, y_pred)[0, 1]\n#         scores.append(pearson_corr)\n    \n#     mean_score = np.mean(scores)\n#     print(f\"CV Score: {mean_score:.6f}\")\n    \n#     if mean_score > best_cv_score:\n#         best_cv_score = mean_score\n#         best_threshold = threshold\n#         best_selector = selector\n#         best_X_train = X_train_medoid\n#         best_X_test = X_test_medoid\n\n# print(f\"\\n✓ Best threshold: {best_threshold} (CV Score: {best_cv_score:.6f})\")\n\n# # Use best selector results\n# X_train_medoid = best_X_train\n# X_test_medoid = best_X_test\n\n# print(f\"\\nFinal feature dimensions:\")\n# print(f\"  Training: {X_train_medoid.shape}\")\n# print(f\"  Test: {X_test_medoid.shape}\")\n\n# # Save selector\n# joblib.dump(best_selector, f'{OUTPUT_PATH}medoid_selector.pkl')\n# print(f\"✓ Medoid selector saved\")\n\n# # ==================== Train Models on Medoid Features ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"TRAINING LINEAR MODELS WITH MEDOID FEATURES\")\n# print(\"=\"*60)\n\n# # Define models\n# models = {\n#     'ridge': Ridge(alpha=0.5, random_state=42),\n#     'ridge_moderate': Ridge(alpha=2.0, random_state=42),\n#     'elastic': ElasticNet(alpha=0.5, l1_ratio=0.5, random_state=42, max_iter=2000),\n#     'linear': LinearRegression()\n# }\n\n# # Cross-validation\n# tscv = TimeSeriesSplit(n_splits=3)\n# cv_results = {}\n# best_model = None\n# best_score = -np.inf\n\n# print(\"\\nCross-validation results:\")\n# print(\"-\" * 40)\n\n# for name, model in models.items():\n#     scores = []\n    \n#     for fold, (train_idx, val_idx) in enumerate(tscv.split(X_train_medoid), 1):\n#         X_tr, X_val = X_train_medoid[train_idx], X_train_medoid[val_idx]\n#         y_tr, y_val = y_train[train_idx], y_train[val_idx]\n        \n#         model.fit(X_tr, y_tr)\n#         y_pred = model.predict(X_val)\n#         pearson_corr = np.corrcoef(y_val, y_pred)[0, 1]\n#         scores.append(pearson_corr)\n    \n#     mean_score = np.mean(scores)\n#     std_score = np.std(scores)\n    \n#     cv_results[name] = {\n#         'mean': mean_score,\n#         'std': std_score,\n#         'scores': scores\n#     }\n    \n#     print(f\"{name:15s}: {mean_score:.6f} (+/- {std_score:.6f})\")\n    \n#     if mean_score > best_score:\n#         best_score = mean_score\n#         best_model = name\n\n# print(\"-\" * 40)\n# print(f\"\\nBest model: {best_model} (CV Score: {best_score:.6f})\")\n\n# # ==================== Train Final Model ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"TRAINING FINAL MEDOID MODEL\")\n# print(\"=\"*60)\n\n# # Create final model\n# if best_model == 'ridge':\n#     final_model = Ridge(alpha=0.5, random_state=42)\n# elif best_model == 'ridge_moderate':\n#     final_model = Ridge(alpha=2.0, random_state=42)\n# elif best_model == 'elastic':\n#     final_model = ElasticNet(alpha=0.5, l1_ratio=0.5, random_state=42, max_iter=2000)\n# else:  # linear\n#     final_model = LinearRegression()\n\n# print(f\"Training {best_model} on full dataset with medoid features...\")\n\n# # Train on all data\n# final_model.fit(X_train_medoid, y_train)\n\n# # Training metrics\n# train_pred = final_model.predict(X_train_medoid)\n# train_corr = np.corrcoef(y_train, train_pred)[0, 1]\n# train_rmse = np.sqrt(mean_squared_error(y_train, train_pred))\n\n# print(f\"\\nTraining metrics:\")\n# print(f\"  Pearson correlation: {train_corr:.6f}\")\n# print(f\"  RMSE: {train_rmse:.6f}\")\n\n# # Save model\n# joblib.dump(final_model, f'{OUTPUT_PATH}medoid_linear_model.pkl')\n# print(f\"\\n✓ Model saved\")\n\n# # ==================== Generate Predictions ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"GENERATING PREDICTIONS\")\n# print(\"=\"*60)\n\n# # Predict on test set\n# test_predictions = final_model.predict(X_test_medoid)\n\n# print(f\"Predictions shape: {test_predictions.shape}\")\n# print(f\"\\nPrediction statistics:\")\n# print(f\"  Mean:  {test_predictions.mean():.6f}\")\n# print(f\"  Std:   {test_predictions.std():.6f}\")\n\n# # ==================== Create Submission ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"CREATING SUBMISSION\")\n# print(\"=\"*60)\n\n# # Create submission\n# submission = pd.DataFrame({\n#     'id': range(1, len(test_predictions) + 1),\n#     'prediction': test_predictions\n# })\n\n# # Save submission\n# submission_path = f'{OUTPUT_PATH}submission_medoid.csv'\n# submission.to_csv(submission_path, index=False)\n# print(f\"✓ Submission saved to {submission_path}\")\n\n# # Save predictions for ensemble\n# np.save(f'{OUTPUT_PATH}train_predictions_medoid.npy', train_pred)\n# np.save(f'{OUTPUT_PATH}test_predictions_medoid.npy', test_predictions)\n# print(\"✓ Predictions saved for ensemble\")\n\n# print(\"\\n\" + \"=\"*60)\n# print(\"MEDOID MODEL COMPLETE!\")\n# print(\"=\"*60)\n# print(f\"Features reduced from {len(feature_cols)} to {len(best_selector.selected_features)}\")\n# print(f\"Best CV Score: {best_score:.6f}\")","metadata":{"trusted":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import pandas as pd\n# import numpy as np\n# import os\n# from sklearn.linear_model import Ridge, LinearRegression, ElasticNet, Lasso\n# from sklearn.preprocessing import StandardScaler\n# from sklearn.model_selection import TimeSeriesSplit\n# from sklearn.metrics import mean_squared_error\n# from scipy.cluster.hierarchy import linkage, fcluster\n# from scipy.spatial.distance import squareform\n# from scipy.stats import rankdata\n# import joblib\n# import warnings\n# warnings.filterwarnings('ignore')\n\n# # ==================== Configuration ====================\n# DATA_PATH = '/kaggle/input/drw-remix-of-winning-strategies-part-1/'\n# OUTPUT_PATH = '/kaggle/working/model_3/'\n# SAMPLE_SUB_PATH = '/kaggle/input/drw-crypto-market-prediction/sample_submission.csv'\n\n# # Create output directory\n# os.makedirs(OUTPUT_PATH, exist_ok=True)\n# print(f\"Output directory: {OUTPUT_PATH}\")\n\n# # ==================== Medoid Feature Selector ====================\n# class MedoidFeatureSelector:\n#     \"\"\"\n#     Reduce collinearity by selecting medoid features from correlated groups.\n#     \"\"\"\n    \n#     def __init__(self, correlation_threshold=0.85, min_cluster_size=2, method='hierarchical'):\n#         self.correlation_threshold = correlation_threshold\n#         self.min_cluster_size = min_cluster_size\n#         self.method = method\n#         self.selected_features = None\n#         self.feature_clusters = None\n#         self.medoid_indices = None\n        \n#     def fit(self, X, feature_names):\n#         \"\"\"\n#         Identify medoids from correlated feature groups.\n#         \"\"\"\n#         # Calculate correlation matrix\n#         corr_matrix = np.corrcoef(X.T)\n#         corr_matrix = np.nan_to_num(corr_matrix, nan=0.0)\n        \n#         # Convert to distance matrix\n#         distance_matrix = 1 - np.abs(corr_matrix)\n        \n#         # Hierarchical clustering\n#         condensed_dist = squareform(distance_matrix, checks=False)\n#         Z = linkage(condensed_dist, method='average')\n#         clusters = fcluster(Z, 1 - self.correlation_threshold, criterion='distance')\n        \n#         self.feature_clusters = {}\n#         for idx, cluster_id in enumerate(clusters):\n#             if cluster_id not in self.feature_clusters:\n#                 self.feature_clusters[cluster_id] = []\n#             self.feature_clusters[cluster_id].append(idx)\n        \n#         # Select medoids from each cluster\n#         self.medoid_indices = []\n#         self.selected_features = []\n        \n#         for cluster_id, feature_indices in self.feature_clusters.items():\n#             if len(feature_indices) >= self.min_cluster_size:\n#                 medoid_idx = self._find_medoid(distance_matrix, feature_indices)\n#                 self.medoid_indices.append(medoid_idx)\n#                 self.selected_features.append(feature_names[medoid_idx])\n#             else:\n#                 for idx in feature_indices:\n#                     self.medoid_indices.append(idx)\n#                     self.selected_features.append(feature_names[idx])\n        \n#         # Remove duplicates\n#         seen = set()\n#         unique_medoids = []\n#         unique_features = []\n#         for idx, feat in zip(self.medoid_indices, self.selected_features):\n#             if idx not in seen:\n#                 seen.add(idx)\n#                 unique_medoids.append(idx)\n#                 unique_features.append(feat)\n        \n#         self.medoid_indices = unique_medoids\n#         self.selected_features = unique_features\n        \n#         return self\n    \n#     def _find_medoid(self, distance_matrix, indices):\n#         \"\"\"\n#         Find the medoid within a cluster.\n#         \"\"\"\n#         if len(indices) == 1:\n#             return indices[0]\n        \n#         cluster_distances = distance_matrix[np.ix_(indices, indices)]\n#         sum_distances = np.sum(cluster_distances, axis=1)\n#         medoid_idx_in_cluster = np.argmin(sum_distances)\n#         return indices[medoid_idx_in_cluster]\n    \n#     def transform(self, X):\n#         \"\"\"\n#         Transform data to selected medoid features.\n#         \"\"\"\n#         return X[:, self.medoid_indices]\n\n# # ==================== Ensemble Configuration ====================\n# # Updated configurations with parameters that converge better\n# ENSEMBLE_CONFIGS = [\n#     # Different correlation thresholds with Ridge (always stable)\n#     {'name': 'high_corr_ridge_weak', 'corr_threshold': 0.95, 'min_cluster': 2, 'model': Ridge(alpha=0.5, random_state=42)},\n#     {'name': 'high_corr_ridge_strong', 'corr_threshold': 0.95, 'min_cluster': 2, 'model': Ridge(alpha=2.0, random_state=42)},\n#     {'name': 'med_corr_ridge_weak', 'corr_threshold': 0.85, 'min_cluster': 2, 'model': Ridge(alpha=1.0, random_state=42)},\n#     {'name': 'med_corr_ridge_strong', 'corr_threshold': 0.85, 'min_cluster': 2, 'model': Ridge(alpha=5.0, random_state=42)},\n#     {'name': 'low_corr_ridge_weak', 'corr_threshold': 0.75, 'min_cluster': 2, 'model': Ridge(alpha=1.0, random_state=42)},\n#     {'name': 'low_corr_ridge_strong', 'corr_threshold': 0.75, 'min_cluster': 2, 'model': Ridge(alpha=10.0, random_state=42)},\n    \n#     # Different cluster sizes\n#     {'name': 'large_cluster_ridge', 'corr_threshold': 0.85, 'min_cluster': 5, 'model': Ridge(alpha=2.0, random_state=42)},\n#     {'name': 'small_cluster_ridge', 'corr_threshold': 0.85, 'min_cluster': 1, 'model': Ridge(alpha=2.0, random_state=42)},\n    \n#     # Very aggressive reduction\n#     {'name': 'aggressive_ridge', 'corr_threshold': 0.70, 'min_cluster': 3, 'model': Ridge(alpha=15.0, random_state=42)},\n#     {'name': 'very_aggressive_ridge', 'corr_threshold': 0.65, 'min_cluster': 4, 'model': Ridge(alpha=20.0, random_state=42)},\n    \n#     # Conservative reduction\n#     {'name': 'conservative_ridge', 'corr_threshold': 0.98, 'min_cluster': 1, 'model': Ridge(alpha=0.1, random_state=42)},\n#     {'name': 'very_conservative_ridge', 'corr_threshold': 0.99, 'min_cluster': 1, 'model': Ridge(alpha=0.05, random_state=42)},\n# ]\n\n# # ==================== Load Data ====================\n# print(\"=\"*60)\n# print(\"LOADING DATA FOR ENSEMBLE MODEL\")\n# print(\"=\"*60)\n\n# train_df = pd.read_parquet(f'{DATA_PATH}train_final.parquet')\n# test_df = pd.read_parquet(f'{DATA_PATH}test_final.parquet')\n# sample_sub = pd.read_csv(SAMPLE_SUB_PATH)\n\n# print(f\"Train shape: {train_df.shape}\")\n# print(f\"Test shape: {test_df.shape}\")\n\n# # Prepare features\n# feature_cols = [col for col in train_df.columns if col not in ['label', 'timestamp', 'id']]\n# X_train = train_df[feature_cols].values\n# y_train = train_df['label'].values\n# X_test = test_df[feature_cols].values\n\n# # Handle NaN/INF\n# X_train = np.nan_to_num(X_train, nan=0.0, posinf=0.0, neginf=0.0)\n# X_test = np.nan_to_num(X_test, nan=0.0, posinf=0.0, neginf=0.0)\n\n# # Scale features\n# scaler = StandardScaler()\n# X_train_scaled = scaler.fit_transform(X_train)\n# X_test_scaled = scaler.transform(X_test)\n\n# print(f\"Features: {len(feature_cols)}\")\n# print(\"✓ Data loaded and scaled\")\n\n# # ==================== Train Multiple Models ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"TRAINING ENSEMBLE MODELS\")\n# print(\"=\"*60)\n\n# ensemble_models = []\n# ensemble_predictions_train = []\n# ensemble_predictions_test = []\n# ensemble_cv_scores = []\n# successful_models = []\n\n# tscv = TimeSeriesSplit(n_splits=3)\n\n# for config in ENSEMBLE_CONFIGS:\n#     print(f\"\\n--- Model: {config['name']} ---\")\n#     print(f\"  Correlation threshold: {config['corr_threshold']}\")\n#     print(f\"  Min cluster size: {config['min_cluster']}\")\n#     print(f\"  Model type: {type(config['model']).__name__}\")\n    \n#     try:\n#         # Create selector\n#         selector = MedoidFeatureSelector(\n#             correlation_threshold=config['corr_threshold'],\n#             min_cluster_size=config['min_cluster'],\n#             method='hierarchical'\n#         )\n        \n#         # Fit selector\n#         selector.fit(X_train_scaled, feature_cols)\n        \n#         # Transform data\n#         X_train_medoid = selector.transform(X_train_scaled)\n#         X_test_medoid = selector.transform(X_test_scaled)\n        \n#         print(f\"  Selected features: {X_train_medoid.shape[1]}\")\n        \n#         # Cross-validation\n#         cv_scores = []\n#         for train_idx, val_idx in tscv.split(X_train_medoid):\n#             X_tr, X_val = X_train_medoid[train_idx], X_train_medoid[val_idx]\n#             y_tr, y_val = y_train[train_idx], y_train[val_idx]\n            \n#             model = config['model'].__class__(**config['model'].get_params())\n#             model.fit(X_tr, y_tr)\n#             y_pred = model.predict(X_val)\n            \n#             # Check for NaN predictions\n#             if np.any(np.isnan(y_pred)):\n#                 print(f\"  WARNING: NaN predictions in CV, skipping model\")\n#                 raise ValueError(\"NaN predictions\")\n            \n#             pearson_corr = np.corrcoef(y_val, y_pred)[0, 1]\n#             cv_scores.append(pearson_corr)\n        \n#         mean_cv_score = np.mean(cv_scores)\n#         print(f\"  CV Score: {mean_cv_score:.6f}\")\n        \n#         # Train final model on all data\n#         final_model = config['model'].__class__(**config['model'].get_params())\n#         final_model.fit(X_train_medoid, y_train)\n        \n#         # Generate predictions\n#         train_pred = final_model.predict(X_train_medoid)\n#         test_pred = final_model.predict(X_test_medoid)\n        \n#         # Check for NaN in final predictions\n#         if np.any(np.isnan(train_pred)) or np.any(np.isnan(test_pred)):\n#             print(f\"  WARNING: NaN in final predictions, skipping model\")\n#             continue\n        \n#         # Store results\n#         ensemble_models.append({\n#             'name': config['name'],\n#             'model': final_model,\n#             'selector': selector,\n#             'cv_score': mean_cv_score,\n#             'config': config\n#         })\n        \n#         ensemble_predictions_train.append(train_pred)\n#         ensemble_predictions_test.append(test_pred)\n#         ensemble_cv_scores.append(mean_cv_score)\n#         successful_models.append(config['name'])\n        \n#         # Save individual model\n#         joblib.dump(final_model, f\"{OUTPUT_PATH}{config['name']}_model.pkl\")\n#         joblib.dump(selector, f\"{OUTPUT_PATH}{config['name']}_selector.pkl\")\n        \n#     except Exception as e:\n#         print(f\"  ERROR: Model failed - {str(e)}\")\n#         continue\n\n# print(f\"\\n✓ Successfully trained {len(ensemble_models)} models out of {len(ENSEMBLE_CONFIGS)}\")\n\n# # Check if we have enough models\n# if len(ensemble_models) < 3:\n#     print(\"ERROR: Not enough successful models for ensemble\")\n#     raise ValueError(\"Insufficient successful models\")\n\n# # ==================== Intelligent Ensemble Methods ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"CREATING INTELLIGENT ENSEMBLE\")\n# print(\"=\"*60)\n\n# # Convert to arrays\n# train_preds = np.array(ensemble_predictions_train).T\n# test_preds = np.array(ensemble_predictions_test).T\n# cv_scores = np.array(ensemble_cv_scores)\n\n# # Method 1: Weighted Average by CV Score\n# weights = cv_scores - cv_scores.min() + 0.001  # Shift to positive\n# weights = weights / weights.sum()\n\n# weighted_train = np.average(train_preds, axis=1, weights=weights)\n# weighted_test = np.average(test_preds, axis=1, weights=weights)\n\n# print(f\"Method 1 - Weighted Average:\")\n# print(f\"  Top 3 weights: {sorted(weights, reverse=True)[:3]}\")\n\n# # Method 2: Simple Average\n# simple_train = np.mean(train_preds, axis=1)\n# simple_test = np.mean(test_preds, axis=1)\n# print(f\"Method 2 - Simple Average: Completed\")\n\n# # Method 3: Median (robust to outliers)\n# median_train = np.median(train_preds, axis=1)\n# median_test = np.median(test_preds, axis=1)\n# print(f\"Method 3 - Median: Completed\")\n\n# # Method 4: Trim Mean (remove best and worst for each sample if we have enough models)\n# if len(ensemble_models) > 4:\n#     trim_train = np.array([np.mean(sorted(train_preds[i])[1:-1]) for i in range(len(train_preds))])\n#     trim_test = np.array([np.mean(sorted(test_preds[i])[1:-1]) for i in range(len(test_preds))])\n#     print(f\"Method 4 - Trimmed Mean: Completed\")\n# else:\n#     trim_train = simple_train\n#     trim_test = simple_test\n#     print(f\"Method 4 - Trimmed Mean: Using simple average (not enough models)\")\n\n# # Method 5: Best Model Selection\n# best_model_idx = np.argmax(cv_scores)\n# best_train = ensemble_predictions_train[best_model_idx]\n# best_test = ensemble_predictions_test[best_model_idx]\n# print(f\"Method 5 - Best Single Model: {ensemble_models[best_model_idx]['name']} (CV: {cv_scores[best_model_idx]:.6f})\")\n\n# # ==================== Evaluate Ensemble Methods ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"EVALUATING ENSEMBLE METHODS\")\n# print(\"=\"*60)\n\n# ensemble_methods = {\n#     'weighted_avg': (weighted_train, weighted_test),\n#     'simple_avg': (simple_train, simple_test),\n#     'median': (median_train, median_test),\n#     'trim_mean': (trim_train, trim_test),\n#     'best_single': (best_train, best_test)\n# }\n\n# # Evaluate each method\n# method_scores = {}\n# for method_name, (train_pred, test_pred) in ensemble_methods.items():\n#     try:\n#         corr = np.corrcoef(y_train, train_pred)[0, 1]\n#         rmse = np.sqrt(mean_squared_error(y_train, train_pred))\n#         method_scores[method_name] = corr\n#         print(f\"{method_name:15s}: Correlation={corr:.6f}, RMSE={rmse:.6f}\")\n#     except Exception as e:\n#         print(f\"{method_name:15s}: ERROR - {str(e)}\")\n#         method_scores[method_name] = -1\n\n# # Select best ensemble method\n# best_method = max(method_scores, key=method_scores.get)\n# final_train, final_test = ensemble_methods[best_method]\n\n# print(f\"\\n✓ Best ensemble method: {best_method}\")\n\n# # ==================== Create Final Submission ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"CREATING FINAL SUBMISSION\")\n# print(\"=\"*60)\n\n# # Create submission with best ensemble\n# submission = pd.DataFrame({\n#     'id': range(1, len(final_test) + 1),\n#     'prediction': final_test\n# })\n\n# submission_path = f'{OUTPUT_PATH}submission_ensemble.csv'\n# submission.to_csv(submission_path, index=False)\n# print(f\"✓ Main submission saved to {submission_path}\")\n\n# # Also save submissions for each ensemble method\n# for method_name, (_, test_pred) in ensemble_methods.items():\n#     try:\n#         sub = pd.DataFrame({\n#             'id': range(1, len(test_pred) + 1),\n#             'prediction': test_pred\n#         })\n#         sub.to_csv(f'{OUTPUT_PATH}submission_{method_name}.csv', index=False)\n#     except:\n#         print(f\"Could not save submission for {method_name}\")\n\n# print(f\"✓ All ensemble method submissions saved\")\n\n# # ==================== Save Ensemble Metadata ====================\n# np.save(f'{OUTPUT_PATH}ensemble_train_predictions.npy', train_preds)\n# np.save(f'{OUTPUT_PATH}ensemble_test_predictions.npy', test_preds)\n# np.save(f'{OUTPUT_PATH}ensemble_cv_scores.npy', cv_scores)\n# np.save(f'{OUTPUT_PATH}ensemble_weights.npy', weights)\n\n# # Save detailed report\n# report = f\"\"\"\n# DRW Crypto Market Prediction - Intelligent Ensemble Report\n# ==========================================================\n\n# Ensemble Configuration:\n# - Total models attempted: {len(ENSEMBLE_CONFIGS)}\n# - Successful models: {len(ensemble_models)}\n# - Failed models: {len(ENSEMBLE_CONFIGS) - len(ensemble_models)}\n\n# Individual Model Performance:\n# \"\"\"\n\n# for i, model_info in enumerate(ensemble_models):\n#     report += f\"\\n{i+1}. {model_info['name']}:\"\n#     report += f\"\\n   CV Score: {model_info['cv_score']:.6f}\"\n#     report += f\"\\n   Features: {len(model_info['selector'].selected_features)}\"\n#     report += f\"\\n   Weight: {weights[i]:.4f}\"\n\n# report += f\"\"\"\n\n# Ensemble Methods Performance:\n# \"\"\"\n\n# for method_name, score in method_scores.items():\n#     report += f\"\\n- {method_name:15s}: {score:.6f}\"\n\n# report += f\"\"\"\n\n# Best Ensemble Method: {best_method}\n# Final Training Correlation: {method_scores[best_method]:.6f}\n\n# Successful Models: {', '.join(successful_models)}\n\n# Files Generated:\n# - Main submission: submission_ensemble.csv\n# - Individual submissions: submission_[method_name].csv\n# - Individual models: [name]_model.pkl\n# - Individual selectors: [name]_selector.pkl\n# - Ensemble predictions: ensemble_[train/test]_predictions.npy\n# - CV scores: ensemble_cv_scores.npy\n# - Weights: ensemble_weights.npy\n# \"\"\"\n\n# with open(f'{OUTPUT_PATH}ensemble_report.txt', 'w') as f:\n#     f.write(report)\n\n# print(\"\\n\" + \"=\"*60)\n# print(\"ENSEMBLE MODEL COMPLETE!\")\n# print(\"=\"*60)\n# print(f\"✓ {len(ensemble_models)} models trained successfully\")\n# print(f\"✓ 5 ensemble methods evaluated\")\n# print(f\"✓ Best method: {best_method} (Correlation: {method_scores[best_method]:.6f})\")\n# print(f\"✓ All files saved to {OUTPUT_PATH}\")","metadata":{"trusted":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import pandas as pd\n# import numpy as np\n# import os\n# from sklearn.linear_model import Ridge, RidgeCV, LassoCV\n# from sklearn.preprocessing import StandardScaler\n# from sklearn.model_selection import TimeSeriesSplit, KFold\n# from sklearn.metrics import mean_squared_error\n# from scipy.cluster.hierarchy import linkage, fcluster\n# from scipy.spatial.distance import squareform\n# from scipy.optimize import minimize\n# import joblib\n# import warnings\n# warnings.filterwarnings('ignore')\n\n# # ==================== Configuration ====================\n# DATA_PATH = '/kaggle/input/drw-remix-of-winning-strategies-part-1/'\n# OUTPUT_PATH = '/kaggle/working/model_4/'\n# SAMPLE_SUB_PATH = '/kaggle/input/drw-crypto-market-prediction/sample_submission.csv'\n\n# os.makedirs(OUTPUT_PATH, exist_ok=True)\n# print(f\"Output directory: {OUTPUT_PATH}\")\n\n# # ==================== Medoid Feature Selector ====================\n# class MedoidFeatureSelector:\n#     def __init__(self, correlation_threshold=0.85, min_cluster_size=2):\n#         self.correlation_threshold = correlation_threshold\n#         self.min_cluster_size = min_cluster_size\n#         self.selected_features = None\n#         self.feature_clusters = None\n#         self.medoid_indices = None\n        \n#     def fit(self, X, feature_names):\n#         corr_matrix = np.corrcoef(X.T)\n#         corr_matrix = np.nan_to_num(corr_matrix, nan=0.0)\n#         distance_matrix = 1 - np.abs(corr_matrix)\n        \n#         condensed_dist = squareform(distance_matrix, checks=False)\n#         Z = linkage(condensed_dist, method='average')\n#         clusters = fcluster(Z, 1 - self.correlation_threshold, criterion='distance')\n        \n#         self.feature_clusters = {}\n#         for idx, cluster_id in enumerate(clusters):\n#             if cluster_id not in self.feature_clusters:\n#                 self.feature_clusters[cluster_id] = []\n#             self.feature_clusters[cluster_id].append(idx)\n        \n#         self.medoid_indices = []\n#         self.selected_features = []\n        \n#         for cluster_id, feature_indices in self.feature_clusters.items():\n#             if len(feature_indices) >= self.min_cluster_size:\n#                 medoid_idx = self._find_medoid(distance_matrix, feature_indices)\n#                 self.medoid_indices.append(medoid_idx)\n#                 self.selected_features.append(feature_names[medoid_idx])\n#             else:\n#                 for idx in feature_indices:\n#                     self.medoid_indices.append(idx)\n#                     self.selected_features.append(feature_names[idx])\n        \n#         seen = set()\n#         unique_medoids = []\n#         unique_features = []\n#         for idx, feat in zip(self.medoid_indices, self.selected_features):\n#             if idx not in seen:\n#                 seen.add(idx)\n#                 unique_medoids.append(idx)\n#                 unique_features.append(feat)\n        \n#         self.medoid_indices = unique_medoids\n#         self.selected_features = unique_features\n#         return self\n    \n#     def _find_medoid(self, distance_matrix, indices):\n#         if len(indices) == 1:\n#             return indices[0]\n#         cluster_distances = distance_matrix[np.ix_(indices, indices)]\n#         sum_distances = np.sum(cluster_distances, axis=1)\n#         return indices[np.argmin(sum_distances)]\n    \n#     def transform(self, X):\n#         return X[:, self.medoid_indices]\n\n# # ==================== Advanced Ensemble with Overfitting Control ====================\n# class OverfitAwareEnsemble:\n#     \"\"\"\n#     Ensemble that penalizes models with large train-CV gaps.\n#     \"\"\"\n#     def __init__(self, penalty_strength=0.5):\n#         self.penalty_strength = penalty_strength\n#         self.weights = None\n#         self.meta_model = None\n        \n#     def compute_weights(self, cv_scores, train_scores):\n#         \"\"\"\n#         Compute weights that consider both performance and generalization.\n#         \"\"\"\n#         # Calculate overfitting penalty\n#         gaps = np.array(train_scores) - np.array(cv_scores)\n#         gaps = np.clip(gaps, 0, None)  # Only penalize positive gaps\n        \n#         # Normalize gaps to [0, 1]\n#         if gaps.max() > 0:\n#             normalized_gaps = gaps / gaps.max()\n#         else:\n#             normalized_gaps = gaps\n        \n#         # Adjusted scores that penalize overfitting\n#         adjusted_scores = cv_scores - self.penalty_strength * normalized_gaps\n        \n#         # Convert to weights (ensure positive)\n#         weights = adjusted_scores - adjusted_scores.min() + 0.001\n#         weights = weights / weights.sum()\n        \n#         return weights, adjusted_scores\n    \n#     def optimize_weights(self, train_preds, y_train, cv_indices):\n#         \"\"\"\n#         Find optimal weights using cross-validation.\n#         \"\"\"\n#         n_models = train_preds.shape[1]\n        \n#         def objective(weights):\n#             # Ensure weights sum to 1\n#             weights = weights / weights.sum()\n            \n#             # Calculate CV score\n#             cv_scores = []\n#             for train_idx, val_idx in cv_indices:\n#                 train_weighted = np.average(train_preds[train_idx], axis=1, weights=weights)\n#                 val_weighted = np.average(train_preds[val_idx], axis=1, weights=weights)\n                \n#                 # Fit simple adjustment model\n#                 adjustment = np.mean(y_train[train_idx] - train_weighted)\n#                 val_pred_adjusted = val_weighted + adjustment\n                \n#                 score = np.corrcoef(y_train[val_idx], val_pred_adjusted)[0, 1]\n#                 cv_scores.append(score)\n            \n#             # Return negative for minimization\n#             return -np.mean(cv_scores)\n        \n#         # Initial weights (equal)\n#         init_weights = np.ones(n_models) / n_models\n        \n#         # Constraints: weights sum to 1, all weights >= 0\n#         constraints = {'type': 'eq', 'fun': lambda w: w.sum() - 1}\n#         bounds = [(0, 1) for _ in range(n_models)]\n        \n#         # Optimize\n#         result = minimize(objective, init_weights, method='SLSQP', \n#                          bounds=bounds, constraints=constraints)\n        \n#         return result.x\n\n# # ==================== Model Configurations ====================\n# ENSEMBLE_CONFIGS = [\n#     # Varying regularization strength\n#     {'name': 'very_weak_reg', 'corr_threshold': 0.90, 'min_cluster': 2, 'alpha': 0.01},\n#     {'name': 'weak_reg', 'corr_threshold': 0.90, 'min_cluster': 2, 'alpha': 0.1},\n#     {'name': 'moderate_reg', 'corr_threshold': 0.85, 'min_cluster': 2, 'alpha': 1.0},\n#     {'name': 'strong_reg', 'corr_threshold': 0.85, 'min_cluster': 2, 'alpha': 5.0},\n#     {'name': 'very_strong_reg', 'corr_threshold': 0.80, 'min_cluster': 2, 'alpha': 10.0},\n#     {'name': 'extreme_reg', 'corr_threshold': 0.80, 'min_cluster': 3, 'alpha': 20.0},\n    \n#     # Different feature reduction levels\n#     {'name': 'minimal_reduction', 'corr_threshold': 0.95, 'min_cluster': 1, 'alpha': 2.0},\n#     {'name': 'moderate_reduction', 'corr_threshold': 0.85, 'min_cluster': 2, 'alpha': 2.0},\n#     {'name': 'aggressive_reduction', 'corr_threshold': 0.70, 'min_cluster': 3, 'alpha': 2.0},\n    \n#     # Balanced approaches\n#     {'name': 'balanced_1', 'corr_threshold': 0.88, 'min_cluster': 2, 'alpha': 3.0},\n#     {'name': 'balanced_2', 'corr_threshold': 0.82, 'min_cluster': 2, 'alpha': 4.0},\n#     {'name': 'balanced_3', 'corr_threshold': 0.78, 'min_cluster': 2, 'alpha': 6.0},\n# ]\n\n# # ==================== Load Data ====================\n# print(\"=\"*60)\n# print(\"ADVANCED ENSEMBLE WITH OVERFITTING CONTROL\")\n# print(\"=\"*60)\n\n# train_df = pd.read_parquet(f'{DATA_PATH}train_final.parquet')\n# test_df = pd.read_parquet(f'{DATA_PATH}test_final.parquet')\n\n# print(f\"Train shape: {train_df.shape}\")\n# print(f\"Test shape: {test_df.shape}\")\n\n# feature_cols = [col for col in train_df.columns if col not in ['label', 'timestamp', 'id']]\n# X_train = train_df[feature_cols].values\n# y_train = train_df['label'].values\n# X_test = test_df[feature_cols].values\n\n# X_train = np.nan_to_num(X_train, nan=0.0, posinf=0.0, neginf=0.0)\n# X_test = np.nan_to_num(X_test, nan=0.0, posinf=0.0, neginf=0.0)\n\n# scaler = StandardScaler()\n# X_train_scaled = scaler.fit_transform(X_train)\n# X_test_scaled = scaler.transform(X_test)\n\n# print(f\"Features: {len(feature_cols)}\")\n\n# # ==================== Train Models with Overfitting Tracking ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"TRAINING MODELS WITH OVERFITTING METRICS\")\n# print(\"=\"*60)\n\n# ensemble_models = []\n# ensemble_predictions_train = []\n# ensemble_predictions_test = []\n# cv_scores_list = []\n# train_scores_list = []\n# overfitting_gaps = []\n\n# tscv = TimeSeriesSplit(n_splits=4)  # More splits for better estimation\n\n# for config in ENSEMBLE_CONFIGS:\n#     print(f\"\\n--- Model: {config['name']} ---\")\n    \n#     try:\n#         # Create selector\n#         selector = MedoidFeatureSelector(\n#             correlation_threshold=config['corr_threshold'],\n#             min_cluster_size=config['min_cluster']\n#         )\n        \n#         selector.fit(X_train_scaled, feature_cols)\n#         X_train_medoid = selector.transform(X_train_scaled)\n#         X_test_medoid = selector.transform(X_test_scaled)\n        \n#         print(f\"  Features: {X_train_medoid.shape[1]}\")\n        \n#         # Track CV and training scores\n#         cv_scores = []\n#         train_scores_cv = []\n        \n#         for train_idx, val_idx in tscv.split(X_train_medoid):\n#             X_tr, X_val = X_train_medoid[train_idx], X_train_medoid[val_idx]\n#             y_tr, y_val = y_train[train_idx], y_train[val_idx]\n            \n#             model = Ridge(alpha=config['alpha'], random_state=42)\n#             model.fit(X_tr, y_tr)\n            \n#             # Training score\n#             train_pred = model.predict(X_tr)\n#             train_score = np.corrcoef(y_tr, train_pred)[0, 1]\n#             train_scores_cv.append(train_score)\n            \n#             # Validation score\n#             val_pred = model.predict(X_val)\n#             val_score = np.corrcoef(y_val, val_pred)[0, 1]\n#             cv_scores.append(val_score)\n        \n#         mean_cv_score = np.mean(cv_scores)\n#         mean_train_score = np.mean(train_scores_cv)\n#         gap = mean_train_score - mean_cv_score\n        \n#         print(f\"  Train Score: {mean_train_score:.6f}\")\n#         print(f\"  CV Score: {mean_cv_score:.6f}\")\n#         print(f\"  Overfit Gap: {gap:.6f}\")\n        \n#         # Train final model\n#         final_model = Ridge(alpha=config['alpha'], random_state=42)\n#         final_model.fit(X_train_medoid, y_train)\n        \n#         train_pred_final = final_model.predict(X_train_medoid)\n#         test_pred_final = final_model.predict(X_test_medoid)\n        \n#         # Store results\n#         ensemble_models.append({\n#             'name': config['name'],\n#             'model': final_model,\n#             'selector': selector,\n#             'cv_score': mean_cv_score,\n#             'train_score': mean_train_score,\n#             'gap': gap\n#         })\n        \n#         ensemble_predictions_train.append(train_pred_final)\n#         ensemble_predictions_test.append(test_pred_final)\n#         cv_scores_list.append(mean_cv_score)\n#         train_scores_list.append(mean_train_score)\n#         overfitting_gaps.append(gap)\n        \n#         joblib.dump(final_model, f\"{OUTPUT_PATH}{config['name']}_model.pkl\")\n#         joblib.dump(selector, f\"{OUTPUT_PATH}{config['name']}_selector.pkl\")\n        \n#     except Exception as e:\n#         print(f\"  ERROR: {str(e)}\")\n#         continue\n\n# print(f\"\\n✓ Trained {len(ensemble_models)} models\")\n\n# # ==================== Advanced Ensemble Creation ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"CREATING OVERFITTING-AWARE ENSEMBLE\")\n# print(\"=\"*60)\n\n# train_preds = np.array(ensemble_predictions_train).T\n# test_preds = np.array(ensemble_predictions_test).T\n\n# # Method 1: Penalized weights\n# ensemble = OverfitAwareEnsemble(penalty_strength=0.5)\n# penalized_weights, adjusted_scores = ensemble.compute_weights(\n#     np.array(cv_scores_list), \n#     np.array(train_scores_list)\n# )\n\n# penalized_train = np.average(train_preds, axis=1, weights=penalized_weights)\n# penalized_test = np.average(test_preds, axis=1, weights=penalized_weights)\n\n# print(\"Method 1 - Penalized Weights:\")\n# print(f\"  Top 3 weights: {sorted(penalized_weights, reverse=True)[:3]}\")\n\n# # Method 2: Optimized weights\n# cv_indices = list(tscv.split(X_train))\n# optimal_weights = ensemble.optimize_weights(train_preds, y_train, cv_indices)\n\n# optimal_train = np.average(train_preds, axis=1, weights=optimal_weights)\n# optimal_test = np.average(test_preds, axis=1, weights=optimal_weights)\n\n# print(\"Method 2 - Optimized Weights:\")\n# print(f\"  Top 3 weights: {sorted(optimal_weights, reverse=True)[:3]}\")\n\n# # Method 3: Low-gap ensemble (only use models with small gaps)\n# gap_threshold = np.percentile(overfitting_gaps, 50)\n# low_gap_indices = [i for i, gap in enumerate(overfitting_gaps) if gap <= gap_threshold]\n\n# if len(low_gap_indices) > 0:\n#     low_gap_train = np.mean(train_preds[:, low_gap_indices], axis=1)\n#     low_gap_test = np.mean(test_preds[:, low_gap_indices], axis=1)\n#     print(f\"Method 3 - Low-Gap Ensemble: Using {len(low_gap_indices)} models\")\n# else:\n#     low_gap_train = penalized_train\n#     low_gap_test = penalized_test\n#     print(\"Method 3 - Low-Gap Ensemble: No models met threshold\")\n\n# # Method 4: Regularized stacking\n# stacker = RidgeCV(alphas=[0.1, 1.0, 10.0, 100.0], cv=3)\n# stacker.fit(train_preds, y_train)\n# stacked_train = stacker.predict(train_preds)\n# stacked_test = stacker.predict(test_preds)\n# print(f\"Method 4 - Regularized Stacking: Alpha={stacker.alpha_:.2f}\")\n\n# # ==================== Evaluate Methods ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"EVALUATING ENSEMBLE METHODS\")\n# print(\"=\"*60)\n\n# ensemble_methods = {\n#     'penalized': (penalized_train, penalized_test),\n#     'optimized': (optimal_train, optimal_test),\n#     'low_gap': (low_gap_train, low_gap_test),\n#     'stacked': (stacked_train, stacked_test)\n# }\n\n# method_scores = {}\n# for method_name, (train_pred, test_pred) in ensemble_methods.items():\n#     corr = np.corrcoef(y_train, train_pred)[0, 1]\n#     rmse = np.sqrt(mean_squared_error(y_train, train_pred))\n#     method_scores[method_name] = corr\n#     print(f\"{method_name:15s}: Correlation={corr:.6f}, RMSE={rmse:.6f}\")\n\n# best_method = max(method_scores, key=method_scores.get)\n# final_train, final_test = ensemble_methods[best_method]\n\n# print(f\"\\n✓ Best method: {best_method}\")\n\n# # ==================== Create Submission ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"CREATING SUBMISSION\")\n# print(\"=\"*60)\n\n# submission = pd.DataFrame({\n#     'id': range(1, len(final_test) + 1),\n#     'prediction': final_test\n# })\n\n# submission.to_csv(f'{OUTPUT_PATH}submission_advanced.csv', index=False)\n# print(f\"✓ Submission saved to {OUTPUT_PATH}submission_advanced.csv\")\n\n# # Save all method submissions\n# for method_name, (_, test_pred) in ensemble_methods.items():\n#     sub = pd.DataFrame({\n#         'id': range(1, len(test_pred) + 1),\n#         'prediction': test_pred\n#     })\n#     sub.to_csv(f'{OUTPUT_PATH}submission_{method_name}.csv', index=False)\n\n# # ==================== Save Report ====================\n# report = f\"\"\"\n# Advanced Ensemble Report - Overfitting Control\n# ===============================================\n\n# Models Trained: {len(ensemble_models)}\n\n# Model Performance Summary:\n# \"\"\"\n\n# for model in sorted(ensemble_models, key=lambda x: x['gap']):\n#     report += f\"\\n{model['name']:20s}: CV={model['cv_score']:.4f}, Train={model['train_score']:.4f}, Gap={model['gap']:.4f}\"\n\n# report += f\"\"\"\n\n# Ensemble Methods:\n# - Penalized: {method_scores.get('penalized', 0):.6f}\n# - Optimized: {method_scores.get('optimized', 0):.6f}  \n# - Low-Gap: {method_scores.get('low_gap', 0):.6f}\n# - Stacked: {method_scores.get('stacked', 0):.6f}\n\n# Best Method: {best_method}\n# Final Score: {method_scores[best_method]:.6f}\n\n# Average Overfitting Gap: {np.mean(overfitting_gaps):.6f}\n# Min Gap: {np.min(overfitting_gaps):.6f}\n# Max Gap: {np.max(overfitting_gaps):.6f}\n# \"\"\"\n\n# with open(f'{OUTPUT_PATH}advanced_report.txt', 'w') as f:\n#     f.write(report)\n\n# print(\"\\n\" + \"=\"*60)\n# print(\"ADVANCED ENSEMBLE COMPLETE!\")\n# print(\"=\"*60)\n# print(f\"✓ Files saved to {OUTPUT_PATH}\")\n# print(f\"✓ Best ensemble: {best_method}\")\n# print(f\"✓ Average overfit gap: {np.mean(overfitting_gaps):.4f}\")","metadata":{"trusted":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import pandas as pd\n# import numpy as np\n# import os\n# from sklearn.linear_model import Ridge, LassoCV\n# from sklearn.preprocessing import StandardScaler\n# from sklearn.model_selection import TimeSeriesSplit\n# from sklearn.metrics import mean_squared_error\n# from sklearn.covariance import GraphicalLassoCV, EmpiricalCovariance\n# from scipy import stats\n# from scipy.cluster.hierarchy import linkage, fcluster, dendrogram\n# from scipy.spatial.distance import squareform\n# from scipy.stats import pearsonr, spearmanr\n# import networkx as nx\n# import joblib\n# import warnings\n# warnings.filterwarnings('ignore')\n\n# # ==================== Configuration ====================\n# DATA_PATH = '/kaggle/input/drw-remix-of-winning-strategies-part-1/'\n# OUTPUT_PATH = '/kaggle/working/model_5/'\n# SAMPLE_SUB_PATH = '/kaggle/input/drw-crypto-market-prediction/sample_submission.csv'\n\n# os.makedirs(OUTPUT_PATH, exist_ok=True)\n# print(f\"Output directory: {OUTPUT_PATH}\")\n\n# # ==================== PC Algorithm Implementation ====================\n# class PCAlgorithm:\n#     \"\"\"\n#     Peter-Clark algorithm for causal discovery.\n#     \"\"\"\n#     def __init__(self, alpha=0.05):\n#         self.alpha = alpha\n#         self.skeleton = None\n#         self.separating_sets = {}\n        \n#     def conditional_independence_test(self, X, i, j, cond_set):\n#         \"\"\"\n#         Test conditional independence using partial correlation.\n#         \"\"\"\n#         n = X.shape[0]\n        \n#         if len(cond_set) == 0:\n#             # Unconditional correlation\n#             corr, pval = pearsonr(X[:, i], X[:, j])\n#         else:\n#             # Partial correlation\n#             # Regress out conditioning set\n#             residual_i = X[:, i].copy()\n#             residual_j = X[:, j].copy()\n            \n#             for k in cond_set:\n#                 # Remove linear effect of conditioning variable\n#                 coef_i = np.cov(X[:, k], residual_i)[0, 1] / np.var(X[:, k])\n#                 residual_i = residual_i - coef_i * X[:, k]\n                \n#                 coef_j = np.cov(X[:, k], residual_j)[0, 1] / np.var(X[:, k])\n#                 residual_j = residual_j - coef_j * X[:, k]\n            \n#             corr, pval = pearsonr(residual_i, residual_j)\n        \n#         return pval > self.alpha\n    \n#     def learn_skeleton(self, X):\n#         \"\"\"\n#         Learn the skeleton (undirected graph) using PC algorithm.\n#         \"\"\"\n#         n_vars = X.shape[1]\n        \n#         # Start with complete graph\n#         skeleton = np.ones((n_vars, n_vars)) - np.eye(n_vars)\n#         self.separating_sets = {}\n        \n#         # Test conditional independence with increasing conditioning set size\n#         for cond_size in range(n_vars - 1):\n#             if cond_size > 3:  # Limit conditioning set size for computational reasons\n#                 break\n                \n#             for i in range(n_vars):\n#                 for j in range(i + 1, n_vars):\n#                     if skeleton[i, j] == 0:\n#                         continue\n                    \n#                     # Find potential conditioning sets\n#                     adjacents = [k for k in range(n_vars) \n#                                 if k != i and k != j and skeleton[i, k] == 1]\n                    \n#                     if len(adjacents) >= cond_size:\n#                         # Test all possible conditioning sets of this size\n#                         from itertools import combinations\n#                         for cond_set in combinations(adjacents, cond_size):\n#                             if self.conditional_independence_test(X, i, j, cond_set):\n#                                 skeleton[i, j] = 0\n#                                 skeleton[j, i] = 0\n#                                 self.separating_sets[(i, j)] = cond_set\n#                                 break\n        \n#         self.skeleton = skeleton\n#         return skeleton\n\n# # ==================== Causal Clustering ====================\n# class CausalClustering:\n#     \"\"\"\n#     Cluster features based on causal relationships.\n#     \"\"\"\n#     def __init__(self, n_clusters=None):\n#         self.n_clusters = n_clusters\n#         self.clusters = None\n#         self.medoids = None\n        \n#     def compute_causal_distance(self, X, method='partial_correlation'):\n#         \"\"\"\n#         Compute distance matrix based on causal relationships.\n#         \"\"\"\n#         n_features = X.shape[1]\n        \n#         if method == 'partial_correlation':\n#             # Use GraphicalLasso to estimate partial correlations\n#             try:\n#                 model = GraphicalLassoCV(cv=3, max_iter=100)\n#                 model.fit(X)\n#                 precision = model.precision_\n                \n#                 # Convert precision to partial correlations\n#                 partial_corr = np.zeros_like(precision)\n#                 for i in range(n_features):\n#                     for j in range(n_features):\n#                         if i != j:\n#                             partial_corr[i, j] = -precision[i, j] / np.sqrt(precision[i, i] * precision[j, j])\n                \n#                 # Distance = 1 - |partial correlation|\n#                 distance = 1 - np.abs(partial_corr)\n                \n#             except:\n#                 # Fallback to regular correlation\n#                 corr = np.corrcoef(X.T)\n#                 distance = 1 - np.abs(corr)\n                \n#         elif method == 'mutual_information':\n#             from sklearn.feature_selection import mutual_info_regression\n#             distance = np.zeros((n_features, n_features))\n            \n#             for i in range(n_features):\n#                 # Use each feature as target\n#                 mi_scores = mutual_info_regression(X, X[:, i], random_state=42)\n#                 distance[i, :] = 1 - (mi_scores / (mi_scores.max() + 1e-10))\n#                 distance[i, i] = 0\n                \n#         else:  # correlation\n#             corr = np.corrcoef(X.T)\n#             distance = 1 - np.abs(corr)\n        \n#         # Ensure symmetry\n#         distance = (distance + distance.T) / 2\n#         np.fill_diagonal(distance, 0)\n        \n#         return distance\n    \n#     def find_causal_medoids(self, X, distance_matrix, clusters):\n#         \"\"\"\n#         Find medoids considering causal relationships.\n#         \"\"\"\n#         medoids = []\n        \n#         for cluster_id in np.unique(clusters):\n#             cluster_indices = np.where(clusters == cluster_id)[0]\n            \n#             if len(cluster_indices) == 1:\n#                 medoids.append(cluster_indices[0])\n#             else:\n#                 # Find feature with minimum sum of distances to others in cluster\n#                 cluster_distances = distance_matrix[np.ix_(cluster_indices, cluster_indices)]\n#                 sum_distances = np.sum(cluster_distances, axis=1)\n                \n#                 # Also consider feature importance (variance explained)\n#                 cluster_data = X[:, cluster_indices]\n#                 variances = np.var(cluster_data, axis=0)\n                \n#                 # Combined score: low distance + high variance\n#                 scores = -sum_distances + 0.5 * (variances / (variances.max() + 1e-10))\n                \n#                 medoid_idx = cluster_indices[np.argmax(scores)]\n#                 medoids.append(medoid_idx)\n        \n#         return medoids\n    \n#     def fit(self, X):\n#         \"\"\"\n#         Perform causal clustering.\n#         \"\"\"\n#         # Compute causal distance\n#         distance_matrix = self.compute_causal_distance(X, method='partial_correlation')\n        \n#         # Hierarchical clustering\n#         condensed_dist = squareform(distance_matrix, checks=False)\n#         Z = linkage(condensed_dist, method='ward')\n        \n#         # Determine number of clusters if not specified\n#         if self.n_clusters is None:\n#             # Use elbow method\n#             last = Z[-10:, 2]\n#             acceleration = np.diff(last, 2)\n#             if len(acceleration) > 0:\n#                 self.n_clusters = acceleration.argmax() + 2\n#             else:\n#                 self.n_clusters = int(np.sqrt(X.shape[1] / 2))\n        \n#         # Get clusters\n#         self.clusters = fcluster(Z, self.n_clusters, criterion='maxclust')\n        \n#         # Find medoids\n#         self.medoids = self.find_causal_medoids(X, distance_matrix, self.clusters)\n        \n#         return self\n\n# # ==================== Anti-Collinearity Adjustment ====================\n# class AntiCollinearityAdjuster:\n#     \"\"\"\n#     Remove collinearity while preserving causal structure.\n#     \"\"\"\n#     def __init__(self, vif_threshold=10):\n#         self.vif_threshold = vif_threshold\n#         self.selected_indices = None\n        \n#     def calculate_vif(self, X):\n#         \"\"\"\n#         Calculate Variance Inflation Factor for each feature.\n#         \"\"\"\n#         n_features = X.shape[1]\n#         vif_scores = []\n        \n#         for i in range(n_features):\n#             # Regress feature i on all others\n#             X_others = np.delete(X, i, axis=1)\n#             y = X[:, i]\n            \n#             # Calculate R-squared\n#             from sklearn.linear_model import LinearRegression\n#             lr = LinearRegression()\n#             lr.fit(X_others, y)\n#             r_squared = lr.score(X_others, y)\n            \n#             # VIF = 1 / (1 - R^2)\n#             if r_squared < 0.999:\n#                 vif = 1 / (1 - r_squared)\n#             else:\n#                 vif = 1000  # Cap at 1000\n                \n#             vif_scores.append(vif)\n        \n#         return np.array(vif_scores)\n    \n#     def select_features(self, X, causal_importance=None):\n#         \"\"\"\n#         Select features to minimize collinearity while preserving important features.\n#         \"\"\"\n#         n_features = X.shape[1]\n        \n#         if causal_importance is None:\n#             causal_importance = np.ones(n_features)\n        \n#         # Start with all features\n#         selected = list(range(n_features))\n        \n#         # Iteratively remove features with high VIF\n#         max_iterations = 50\n#         for _ in range(max_iterations):\n#             if len(selected) < 2:\n#                 break\n                \n#             X_selected = X[:, selected]\n#             vif_scores = self.calculate_vif(X_selected)\n            \n#             # Check if all VIFs are below threshold\n#             if np.all(vif_scores < self.vif_threshold):\n#                 break\n            \n#             # Find feature with highest VIF (weighted by inverse importance)\n#             weights = 1 / (causal_importance[selected] + 0.1)\n#             weighted_vif = vif_scores * weights\n            \n#             # Remove feature with highest weighted VIF\n#             idx_to_remove = np.argmax(weighted_vif)\n#             del selected[idx_to_remove]\n        \n#         self.selected_indices = selected\n#         return selected\n\n# # ==================== Main Pipeline ====================\n# print(\"=\"*60)\n# print(\"CAUSAL DISCOVERY AND FEATURE ENGINEERING\")\n# print(\"=\"*60)\n\n# # Load data\n# train_df = pd.read_parquet(f'{DATA_PATH}train_final.parquet')\n# test_df = pd.read_parquet(f'{DATA_PATH}test_final.parquet')\n\n# print(f\"Train shape: {train_df.shape}\")\n# print(f\"Test shape: {test_df.shape}\")\n\n# feature_cols = [col for col in train_df.columns if col not in ['label', 'timestamp', 'id']]\n# X_train = train_df[feature_cols].values\n# y_train = train_df['label'].values\n# X_test = test_df[feature_cols].values\n\n# # Handle NaN/INF\n# X_train = np.nan_to_num(X_train, nan=0.0, posinf=0.0, neginf=0.0)\n# X_test = np.nan_to_num(X_test, nan=0.0, posinf=0.0, neginf=0.0)\n\n# # Scale\n# scaler = StandardScaler()\n# X_train_scaled = scaler.fit_transform(X_train)\n# X_test_scaled = scaler.transform(X_test)\n\n# print(f\"Original features: {len(feature_cols)}\")\n\n# # ==================== Step 1: Causal Discovery ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"STEP 1: CAUSAL DISCOVERY\")\n# print(\"=\"*60)\n\n# # Use subset for computational efficiency\n# n_samples = min(5000, len(X_train))\n# sample_idx = np.random.choice(len(X_train), n_samples, replace=False)\n# X_sample = X_train_scaled[sample_idx]\n\n# # PC Algorithm for skeleton learning (on subset of features)\n# n_features_pc = min(50, X_train.shape[1])  # Limit for PC algorithm\n# feature_subset_idx = np.random.choice(X_train.shape[1], n_features_pc, replace=False)\n# X_pc_subset = X_sample[:, feature_subset_idx]\n\n# print(f\"Running PC algorithm on {n_features_pc} features...\")\n# pc = PCAlgorithm(alpha=0.05)\n# skeleton = pc.learn_skeleton(X_pc_subset)\n\n# # Count edges\n# n_edges = np.sum(skeleton) / 2\n# print(f\"Discovered {int(n_edges)} edges in causal skeleton\")\n\n# # ==================== Step 2: Causal Clustering ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"STEP 2: CAUSAL CLUSTERING\")\n# print(\"=\"*60)\n\n# # Apply causal clustering\n# print(\"Performing causal clustering...\")\n# causal_clusterer = CausalClustering(n_clusters=None)  # Auto-determine\n# causal_clusterer.fit(X_sample)\n\n# print(f\"Created {causal_clusterer.n_clusters} causal clusters\")\n# print(f\"Selected {len(causal_clusterer.medoids)} medoid features\")\n\n# # Get medoid features\n# X_train_medoids = X_train_scaled[:, causal_clusterer.medoids]\n# X_test_medoids = X_test_scaled[:, causal_clusterer.medoids]\n\n# # ==================== Step 3: Anti-Collinearity ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"STEP 3: ANTI-COLLINEARITY ADJUSTMENT\")\n# print(\"=\"*60)\n\n# # Calculate feature importance for medoids\n# from sklearn.ensemble import RandomForestRegressor\n# rf = RandomForestRegressor(n_estimators=50, max_depth=5, random_state=42)\n# rf.fit(X_train_medoids[:10000], y_train[:10000])  # Use subset\n# importance_scores = rf.feature_importances_\n\n# print(\"Removing collinear features while preserving important ones...\")\n# adjuster = AntiCollinearityAdjuster(vif_threshold=10)\n# selected_indices = adjuster.select_features(X_train_medoids, importance_scores)\n\n# print(f\"Selected {len(selected_indices)} features after VIF adjustment\")\n\n# # Final feature set\n# X_train_final = X_train_medoids[:, selected_indices]\n# X_test_final = X_test_medoids[:, selected_indices]\n\n# # ==================== Step 4: Hierarchical Feature Engineering ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"STEP 4: HIERARCHICAL FEATURE ENGINEERING\")\n# print(\"=\"*60)\n\n# def create_hierarchical_features(X):\n#     \"\"\"\n#     Create hierarchical combinations of features.\n#     \"\"\"\n#     n_samples, n_features = X.shape\n    \n#     # Level 1: Original features\n#     features_level1 = X\n    \n#     # Level 2: Pairwise interactions (limited)\n#     features_level2 = []\n#     n_interactions = min(20, n_features * (n_features - 1) // 2)\n    \n#     # Select most important pairs\n#     importance_matrix = np.outer(importance_scores[selected_indices], \n#                                  importance_scores[selected_indices])\n#     np.fill_diagonal(importance_matrix, 0)\n    \n#     # Get top interactions\n#     idx_pairs = []\n#     for _ in range(n_interactions):\n#         max_idx = np.unravel_index(np.argmax(importance_matrix), importance_matrix.shape)\n#         idx_pairs.append(max_idx)\n#         importance_matrix[max_idx] = 0\n#         importance_matrix[max_idx[1], max_idx[0]] = 0\n    \n#     for i, j in idx_pairs:\n#         if i < n_features and j < n_features:\n#             interaction = X[:, i] * X[:, j]\n#             features_level2.append(interaction.reshape(-1, 1))\n    \n#     if features_level2:\n#         features_level2 = np.hstack(features_level2)\n#     else:\n#         features_level2 = np.zeros((n_samples, 1))\n    \n#     # Level 3: Cluster aggregates\n#     features_level3 = []\n#     for cluster_id in np.unique(causal_clusterer.clusters):\n#         cluster_features = [i for i, c in enumerate(causal_clusterer.clusters) if c == cluster_id]\n#         if cluster_features:\n#             # Use medoid index mapping\n#             valid_indices = [i for i, idx in enumerate(causal_clusterer.medoids) \n#                            if idx in cluster_features and i in selected_indices]\n#             if valid_indices:\n#                 cluster_mean = np.mean(X[:, valid_indices], axis=1)\n#                 features_level3.append(cluster_mean.reshape(-1, 1))\n    \n#     if features_level3:\n#         features_level3 = np.hstack(features_level3)\n#     else:\n#         features_level3 = np.zeros((n_samples, 1))\n    \n#     # Combine all levels\n#     return np.hstack([features_level1, features_level2, features_level3])\n\n# print(\"Creating hierarchical features...\")\n# X_train_hierarchical = create_hierarchical_features(X_train_final)\n# X_test_hierarchical = create_hierarchical_features(X_test_final)\n\n# print(f\"Final feature dimensions: {X_train_hierarchical.shape}\")\n\n# # ==================== Step 5: Model Training ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"STEP 5: MODEL TRAINING\")\n# print(\"=\"*60)\n\n# models = {}\n# predictions = {}\n# scores = {}\n\n# # Model 1: Ridge\n# print(\"\\n--- Ridge Regression ---\")\n# ridge = Ridge(alpha=1.0, random_state=42)\n# ridge.fit(X_train_hierarchical, y_train)\n# train_pred_ridge = ridge.predict(X_train_hierarchical)\n# test_pred_ridge = ridge.predict(X_test_hierarchical)\n# scores['ridge'] = np.corrcoef(y_train, train_pred_ridge)[0, 1]\n# print(f\"Score: {scores['ridge']:.6f}\")\n# models['ridge'] = ridge\n# predictions['ridge'] = test_pred_ridge\n\n# # Model 2: Lasso\n# print(\"\\n--- Lasso Regression ---\")\n# lasso = LassoCV(cv=3, max_iter=2000, random_state=42)\n# lasso.fit(X_train_hierarchical, y_train)\n# train_pred_lasso = lasso.predict(X_train_hierarchical)\n# test_pred_lasso = lasso.predict(X_test_hierarchical)\n# scores['lasso'] = np.corrcoef(y_train, train_pred_lasso)[0, 1]\n# print(f\"Score: {scores['lasso']:.6f}\")\n# print(f\"Alpha: {lasso.alpha_:.6f}\")\n# models['lasso'] = lasso\n# predictions['lasso'] = test_pred_lasso\n\n# # Ensemble\n# weights = np.array([scores['ridge'], scores['lasso']])\n# weights = weights / weights.sum()\n# ensemble_test = weights[0] * test_pred_ridge + weights[1] * test_pred_lasso\n\n# print(f\"\\nEnsemble weights: Ridge={weights[0]:.3f}, Lasso={weights[1]:.3f}\")\n\n# # ==================== Create Submission ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"CREATING SUBMISSION\")\n# print(\"=\"*60)\n\n# submission = pd.DataFrame({\n#     'id': range(1, len(ensemble_test) + 1),\n#     'prediction': ensemble_test\n# })\n\n# submission.to_csv(f'{OUTPUT_PATH}submission_causal.csv', index=False)\n# print(f\"✓ Submission saved to {OUTPUT_PATH}submission_causal.csv\")\n\n# # ==================== Save Models and Report ====================\n# joblib.dump(scaler, f'{OUTPUT_PATH}scaler.pkl')\n# joblib.dump(causal_clusterer, f'{OUTPUT_PATH}causal_clusterer.pkl')\n# joblib.dump(adjuster, f'{OUTPUT_PATH}anti_collinearity.pkl')\n# joblib.dump(models, f'{OUTPUT_PATH}models.pkl')\n\n# report = f\"\"\"\n# Causal Discovery and Feature Engineering Report\n# ================================================\n\n# Original features: {len(feature_cols)}\n# Causal clusters: {causal_clusterer.n_clusters}\n# Medoid features: {len(causal_clusterer.medoids)}\n# After VIF adjustment: {len(selected_indices)}\n# Final features (with hierarchical): {X_train_hierarchical.shape[1]}\n\n# Model Scores:\n# - Ridge: {scores['ridge']:.6f}\n# - Lasso: {scores['lasso']:.6f}\n\n# Files saved to {OUTPUT_PATH}\n# \"\"\"\n\n# with open(f'{OUTPUT_PATH}report.txt', 'w') as f:\n#     f.write(report)\n\n# print(\"\\n✓ Complete! Files saved to\", OUTPUT_PATH)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # ==================== Install Required Packages ====================\n# !pip install econml -q\n\n# import pandas as pd\n# import numpy as np\n# import os\n# from sklearn.linear_model import Ridge, LassoCV, ElasticNetCV\n# from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor\n# from sklearn.preprocessing import StandardScaler, PolynomialFeatures\n# from sklearn.model_selection import TimeSeriesSplit\n# from sklearn.decomposition import PCA\n# from econml.dml import LinearDML, CausalForestDML\n# from econml.dr import DRLearner\n# from econml.metalearners import TLearner, SLearner\n# from econml.sklearn_extensions.linear_model import WeightedLasso\n# import joblib\n# import warnings\n# warnings.filterwarnings('ignore')\n\n# # ==================== Configuration ====================\n# DATA_PATH = '/kaggle/input/drw-remix-of-winning-strategies-part-1/'\n# OUTPUT_PATH = '/kaggle/working/model_6/'\n# SAMPLE_SUB_PATH = '/kaggle/input/drw-crypto-market-prediction/sample_submission.csv'\n\n# os.makedirs(OUTPUT_PATH, exist_ok=True)\n# print(f\"Output directory: {OUTPUT_PATH}\")\n\n# # ==================== Load Data ====================\n# print(\"=\"*60)\n# print(\"ECONML-BASED CAUSAL MODELING\")\n# print(\"=\"*60)\n\n# train_df = pd.read_parquet(f'{DATA_PATH}train_final.parquet')\n# test_df = pd.read_parquet(f'{DATA_PATH}test_final.parquet')\n\n# print(f\"Train shape: {train_df.shape}\")\n# print(f\"Test shape: {test_df.shape}\")\n\n# feature_cols = [col for col in train_df.columns if col not in ['label', 'timestamp', 'id']]\n# X_train = train_df[feature_cols].values\n# y_train = train_df['label'].values\n# X_test = test_df[feature_cols].values\n\n# # Handle NaN/INF\n# X_train = np.nan_to_num(X_train, nan=0.0, posinf=0.0, neginf=0.0)\n# X_test = np.nan_to_num(X_test, nan=0.0, posinf=0.0, neginf=0.0)\n\n# print(f\"Features: {len(feature_cols)}\")\n\n# # ==================== Feature Engineering for Causal Analysis ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"FEATURE ENGINEERING FOR CAUSAL ANALYSIS\")\n# print(\"=\"*60)\n\n# # Scale features\n# scaler = StandardScaler()\n# X_train_scaled = scaler.fit_transform(X_train)\n# X_test_scaled = scaler.transform(X_test)\n\n# # Create pseudo-treatment variable from market microstructure\n# # Use volume as a continuous treatment (high vs low volume regimes)\n# if 'volume' in feature_cols:\n#     volume_idx = feature_cols.index('volume')\n#     T_train = X_train[:, volume_idx]\n#     T_test = X_test[:, volume_idx]\n# else:\n#     # Use first feature as treatment\n#     T_train = X_train[:, 0]\n#     T_test = X_test[:, 0]\n\n# # Normalize treatment\n# T_scaler = StandardScaler()\n# T_train = T_scaler.fit_transform(T_train.reshape(-1, 1)).ravel()\n# T_test = T_scaler.transform(T_test.reshape(-1, 1)).ravel()\n\n# # Remove treatment from features\n# X_train_no_t = np.delete(X_train_scaled, 0, axis=1)\n# X_test_no_t = np.delete(X_test_scaled, 0, axis=1)\n\n# # Reduce dimensions for computational efficiency\n# pca = PCA(n_components=100, random_state=42)\n# X_train_pca = pca.fit_transform(X_train_no_t)\n# X_test_pca = pca.transform(X_test_no_t)\n\n# print(f\"PCA explained variance: {pca.explained_variance_ratio_.sum():.3f}\")\n# print(f\"Features after PCA: {X_train_pca.shape[1]}\")\n\n# # Use subset for training to avoid memory issues\n# subset_size = min(50000, len(X_train_pca))\n# indices = np.random.choice(len(X_train_pca), subset_size, replace=False)\n\n# # ==================== Method 1: Double Machine Learning ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"METHOD 1: DOUBLE MACHINE LEARNING (DML)\")\n# print(\"=\"*60)\n\n# try:\n#     print(\"Training LinearDML...\")\n#     dml_linear = LinearDML(\n#         model_y=GradientBoostingRegressor(n_estimators=100, max_depth=3, random_state=42),\n#         model_t=GradientBoostingRegressor(n_estimators=100, max_depth=3, random_state=42),\n#         discrete_treatment=False,\n#         cv=3,\n#         random_state=42\n#     )\n\n#     dml_linear.fit(\n#         Y=y_train[indices],\n#         T=T_train[indices],\n#         X=X_train_pca[indices],\n#         W=None\n#     )\n\n#     # Get treatment effects\n#     ate_train = dml_linear.effect(X_train_pca, T0=0, T1=1)\n#     ate_test = dml_linear.effect(X_test_pca, T0=0, T1=1)\n\n#     # Create predictions\n#     baseline_pred_train = dml_linear.model_y_xw.predict(X_train_pca)\n#     baseline_pred_test = dml_linear.model_y_xw.predict(X_test_pca)\n\n#     dml_pred_train = baseline_pred_train + ate_train * T_train\n#     dml_pred_test = baseline_pred_test + ate_test * T_test\n\n#     score_dml = np.corrcoef(y_train, dml_pred_train)[0, 1]\n#     print(f\"DML Score: {score_dml:.6f}\")\n    \n# except Exception as e:\n#     print(f\"DML failed: {e}\")\n#     dml_pred_train = np.zeros(len(y_train))\n#     dml_pred_test = np.zeros(len(X_test))\n#     score_dml = 0.0\n\n# # ==================== Method 2: Causal Forest ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"METHOD 2: CAUSAL FOREST\")\n# print(\"=\"*60)\n\n# try:\n#     print(\"Training Causal Forest...\")\n#     causal_forest = CausalForestDML(\n#         model_y=RandomForestRegressor(n_estimators=50, max_depth=5, random_state=42),\n#         model_t=RandomForestRegressor(n_estimators=50, max_depth=5, random_state=42),\n#         n_estimators=100,\n#         max_depth=5,\n#         random_state=42\n#     )\n\n#     causal_forest.fit(\n#         Y=y_train[indices],\n#         T=T_train[indices],\n#         X=X_train_pca[indices],\n#         W=None\n#     )\n\n#     # Get effects\n#     cf_effects_train = causal_forest.effect(X_train_pca)\n#     cf_effects_test = causal_forest.effect(X_test_pca)\n\n#     # Create predictions\n#     cf_baseline_train = causal_forest.model_y_xw.predict(X_train_pca)\n#     cf_baseline_test = causal_forest.model_y_xw.predict(X_test_pca)\n\n#     cf_pred_train = cf_baseline_train + cf_effects_train * T_train\n#     cf_pred_test = cf_baseline_test + cf_effects_test * T_test\n\n#     score_cf = np.corrcoef(y_train, cf_pred_train)[0, 1]\n#     print(f\"Causal Forest Score: {score_cf:.6f}\")\n    \n# except Exception as e:\n#     print(f\"Causal Forest failed: {e}\")\n#     cf_pred_train = np.zeros(len(y_train))\n#     cf_pred_test = np.zeros(len(X_test))\n#     score_cf = 0.0\n\n# # ==================== Method 3: Doubly Robust Learner ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"METHOD 3: DOUBLY ROBUST LEARNER\")\n# print(\"=\"*60)\n\n# try:\n#     print(\"Training DR Learner...\")\n    \n#     # Convert to binary treatment\n#     T_binary_train = (T_train > np.median(T_train)).astype(int)\n#     T_binary_test = (T_test > np.median(T_test)).astype(int)\n    \n#     dr_learner = DRLearner(\n#         model_propensity=GradientBoostingRegressor(n_estimators=50, max_depth=3),\n#         model_regression=GradientBoostingRegressor(n_estimators=50, max_depth=3),\n#         model_final=RandomForestRegressor(n_estimators=100, max_depth=5),\n#         cv=3,\n#         random_state=42\n#     )\n\n#     dr_learner.fit(\n#         Y=y_train[indices],\n#         T=T_binary_train[indices],\n#         X=X_train_pca[indices],\n#         W=None\n#     )\n\n#     # Get effects\n#     dr_effects_train = dr_learner.effect(X_train_pca)\n#     dr_effects_test = dr_learner.effect(X_test_pca)\n\n#     # Create predictions\n#     dr_baseline = np.mean(y_train[indices])\n#     dr_pred_train = dr_baseline + dr_effects_train.ravel() * (T_binary_train - 0.5)\n#     dr_pred_test = dr_baseline + dr_effects_test.ravel() * (T_binary_test - 0.5)\n\n#     score_dr = np.corrcoef(y_train, dr_pred_train)[0, 1]\n#     print(f\"DR Learner Score: {score_dr:.6f}\")\n    \n# except Exception as e:\n#     print(f\"DR Learner failed: {e}\")\n#     dr_pred_train = np.zeros(len(y_train))\n#     dr_pred_test = np.zeros(len(X_test))\n#     score_dr = 0.0\n\n# # ==================== Method 4: Meta-Learners ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"METHOD 4: META-LEARNERS\")\n# print(\"=\"*60)\n\n# # S-Learner\n# try:\n#     print(\"Training S-Learner...\")\n    \n#     if 'T_binary_train' not in locals():\n#         T_binary_train = (T_train > np.median(T_train)).astype(int)\n#         T_binary_test = (T_test > np.median(T_test)).astype(int)\n    \n#     s_learner = SLearner(overall_model=GradientBoostingRegressor(n_estimators=100, max_depth=4))\n#     s_learner.fit(y_train[indices], T_binary_train[indices], X_train_pca[indices])\n\n#     s_effects_train = s_learner.effect(X_train_pca)\n#     s_effects_test = s_learner.effect(X_test_pca)\n\n#     dr_baseline = np.mean(y_train[indices])\n#     s_pred_train = dr_baseline + s_effects_train * (T_binary_train - 0.5)\n#     s_pred_test = dr_baseline + s_effects_test * (T_binary_test - 0.5)\n\n#     score_s = np.corrcoef(y_train, s_pred_train)[0, 1]\n#     print(f\"S-Learner Score: {score_s:.6f}\")\n    \n# except Exception as e:\n#     print(f\"S-Learner failed: {e}\")\n#     s_pred_train = np.zeros(len(y_train))\n#     s_pred_test = np.zeros(len(X_test))\n#     score_s = 0.0\n\n# # T-Learner\n# try:\n#     print(\"Training T-Learner...\")\n#     t_learner = TLearner(models=[\n#         GradientBoostingRegressor(n_estimators=100, max_depth=4),\n#         GradientBoostingRegressor(n_estimators=100, max_depth=4)\n#     ])\n#     t_learner.fit(y_train[indices], T_binary_train[indices], X_train_pca[indices])\n\n#     t_effects_train = t_learner.effect(X_train_pca)\n#     t_effects_test = t_learner.effect(X_test_pca)\n\n#     t_pred_train = dr_baseline + t_effects_train * (T_binary_train - 0.5)\n#     t_pred_test = dr_baseline + t_effects_test * (T_binary_test - 0.5)\n\n#     score_t = np.corrcoef(y_train, t_pred_train)[0, 1]\n#     print(f\"T-Learner Score: {score_t:.6f}\")\n    \n# except Exception as e:\n#     print(f\"T-Learner failed: {e}\")\n#     t_pred_train = np.zeros(len(y_train))\n#     t_pred_test = np.zeros(len(X_test))\n#     score_t = 0.0\n\n# # ==================== Ensemble Creation ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"CREATING ENSEMBLE\")\n# print(\"=\"*60)\n\n# # Collect valid predictions\n# valid_preds_train = []\n# valid_preds_test = []\n# valid_scores = []\n# valid_names = []\n\n# for pred_train, pred_test, score, name in [\n#     (dml_pred_train, dml_pred_test, score_dml, 'DML'),\n#     (cf_pred_train, cf_pred_test, score_cf, 'CausalForest'),\n#     (dr_pred_train, dr_pred_test, score_dr, 'DR'),\n#     (s_pred_train, s_pred_test, score_s, 'S-Learner'),\n#     (t_pred_train, t_pred_test, score_t, 'T-Learner')\n# ]:\n#     if score > 0 and not np.all(pred_train == 0):\n#         valid_preds_train.append(pred_train)\n#         valid_preds_test.append(pred_test)\n#         valid_scores.append(score)\n#         valid_names.append(name)\n\n# print(f\"\\nValid models: {valid_names}\")\n\n# if len(valid_preds_train) > 0:\n#     all_train_preds = np.column_stack(valid_preds_train)\n#     all_test_preds = np.column_stack(valid_preds_test)\n#     scores = np.array(valid_scores)\n    \n#     # Weighted ensemble\n#     weights = np.maximum(scores, 0.01)\n#     weights = weights / weights.sum()\n    \n#     ensemble_train = np.average(all_train_preds, axis=1, weights=weights)\n#     ensemble_test = np.average(all_test_preds, axis=1, weights=weights)\n# else:\n#     # Fallback to simple Ridge\n#     print(\"All causal methods failed, using simple Ridge regression\")\n#     ridge = Ridge(alpha=1.0)\n#     ridge.fit(X_train_pca, y_train)\n#     ensemble_train = ridge.predict(X_train_pca)\n#     ensemble_test = ridge.predict(X_test_pca)\n#     valid_names = ['Ridge']\n#     weights = [1.0]\n\n# ensemble_score = np.corrcoef(y_train, ensemble_train)[0, 1]\n# print(f\"\\nEnsemble Score: {ensemble_score:.6f}\")\n\n# # ==================== Create Submission ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"CREATING SUBMISSION\")\n# print(\"=\"*60)\n\n# submission = pd.DataFrame({\n#     'id': range(1, len(ensemble_test) + 1),\n#     'prediction': ensemble_test\n# })\n\n# submission.to_csv(f'{OUTPUT_PATH}submission_econml.csv', index=False)\n# print(f\"✓ Submission saved to {OUTPUT_PATH}submission_econml.csv\")\n\n# # Save models and metadata\n# joblib.dump(scaler, f'{OUTPUT_PATH}scaler.pkl')\n# joblib.dump(pca, f'{OUTPUT_PATH}pca.pkl')\n\n# # Create report\n# report = f\"\"\"\n# EconML Causal Modeling Report\n# ==============================\n\n# Data:\n# - Training samples: {len(X_train)}\n# - Test samples: {len(X_test)}\n# - Original features: {len(feature_cols)}\n# - PCA features: {X_train_pca.shape[1]}\n# - PCA variance explained: {pca.explained_variance_ratio_.sum():.3f}\n\n# Models Used: {', '.join(valid_names)}\n# Ensemble Score: {ensemble_score:.6f}\n\n# Files saved to {OUTPUT_PATH}\n# \"\"\"\n\n# with open(f'{OUTPUT_PATH}report.txt', 'w') as f:\n#     f.write(report)\n\n# print(\"\\n✓ Complete! Files saved to\", OUTPUT_PATH)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # ==================== Install Required Packages ====================\n# !pip install dowhy -q\n\n# import pandas as pd\n# import numpy as np\n# import os\n# from sklearn.linear_model import Ridge, LogisticRegression\n# from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor\n# from sklearn.preprocessing import StandardScaler\n# from sklearn.decomposition import PCA\n# from sklearn.model_selection import TimeSeriesSplit\n# import dowhy\n# from dowhy import CausalModel\n# import joblib\n# import warnings\n# warnings.filterwarnings('ignore')\n\n# # ==================== Configuration ====================\n# DATA_PATH = '/kaggle/input/drw-remix-of-winning-strategies-part-1/'\n# OUTPUT_PATH = '/kaggle/working/model_7/'\n# SAMPLE_SUB_PATH = '/kaggle/input/drw-crypto-market-prediction/sample_submission.csv'\n\n# os.makedirs(OUTPUT_PATH, exist_ok=True)\n# print(f\"Output directory: {OUTPUT_PATH}\")\n\n# # ==================== Load Data ====================\n# print(\"=\"*60)\n# print(\"DOWHY CAUSAL INFERENCE MODELING\")\n# print(\"=\"*60)\n\n# train_df = pd.read_parquet(f'{DATA_PATH}train_final.parquet')\n# test_df = pd.read_parquet(f'{DATA_PATH}test_final.parquet')\n\n# print(f\"Train shape: {train_df.shape}\")\n# print(f\"Test shape: {test_df.shape}\")\n\n# feature_cols = [col for col in train_df.columns if col not in ['label', 'timestamp', 'id']]\n\n# # ==================== Prepare Data for DoWhy ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"PREPARING DATA FOR CAUSAL ANALYSIS\")\n# print(\"=\"*60)\n\n# # Use a subset for computational efficiency\n# subset_size = min(30000, len(train_df))\n# indices = np.random.choice(len(train_df), subset_size, replace=False)\n\n# # Create DataFrame for DoWhy\n# data = train_df.iloc[indices].copy()\n\n# # Reduce features using PCA\n# X_features = data[feature_cols].values\n# X_features = np.nan_to_num(X_features, nan=0.0, posinf=0.0, neginf=0.0)\n\n# scaler = StandardScaler()\n# X_scaled = scaler.fit_transform(X_features)\n\n# pca = PCA(n_components=50, random_state=42)\n# X_pca = pca.fit_transform(X_scaled)\n\n# # Create new dataframe with PCA features\n# pca_cols = [f'pca_{i}' for i in range(X_pca.shape[1])]\n# pca_df = pd.DataFrame(X_pca, columns=pca_cols, index=data.index)\n\n# # Add target\n# pca_df['outcome'] = data['label'].values\n\n# # Create treatment variable\n# if 'volume' in data.columns:\n#     treatment = (data['volume'] > data['volume'].median()).astype(int)\n# else:\n#     treatment = (X_pca[:, 0] > np.median(X_pca[:, 0])).astype(int)\n\n# pca_df['treatment'] = treatment\n\n# print(f\"Data prepared: {pca_df.shape}\")\n# print(f\"Treatment distribution: {treatment.value_counts().to_dict()}\")\n\n# # ==================== Method 1: Causal Graph & Estimation ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"METHOD 1: CAUSAL GRAPH & BACKDOOR ESTIMATION\")\n# print(\"=\"*60)\n\n# # Define causal graph\n# causal_graph = \"\"\"\n# digraph {\n#     treatment[label=\"Market Regime\"];\n#     outcome[label=\"Price Movement\"];\n# \"\"\"\n\n# # Add PCA features as confounders\n# for col in pca_cols[:10]:\n#     causal_graph += f'    {col}[label=\"{col}\"];\\n'\n#     causal_graph += f'    {col} -> treatment;\\n'\n#     causal_graph += f'    {col} -> outcome;\\n'\n\n# causal_graph += \"\"\"\n#     treatment -> outcome;\n# }\n# \"\"\"\n\n# print(\"Creating causal model...\")\n\n# # Create causal model\n# model = CausalModel(\n#     data=pca_df,\n#     treatment='treatment',\n#     outcome='outcome',\n#     graph=causal_graph\n# )\n\n# # Identify causal effect\n# identified_estimand = model.identify_effect(proceed_when_unidentifiable=True)\n# print(\"Causal effect identified\")\n\n# # Estimate using backdoor criterion\n# print(\"Estimating causal effect using backdoor adjustment...\")\n# try:\n#     backdoor_estimate = model.estimate_effect(\n#         identified_estimand,\n#         method_name=\"backdoor.linear_regression\",\n#         test_significance=False\n#     )\n#     ate_backdoor = backdoor_estimate.value\n#     print(f\"Average Treatment Effect (Backdoor): {ate_backdoor:.6f}\")\n# except Exception as e:\n#     print(f\"Backdoor estimation failed: {e}\")\n#     ate_backdoor = 0.0\n\n# # ==================== Method 2: Propensity Score Methods ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"METHOD 2: PROPENSITY SCORE METHODS\")\n# print(\"=\"*60)\n\n# # Try different PS methods\n# ps_methods = [\n#     \"backdoor.propensity_score_matching\",\n#     \"backdoor.propensity_score_stratification\", \n#     \"backdoor.propensity_score_weighting\"\n# ]\n\n# ps_estimates = {}\n# for method in ps_methods:\n#     try:\n#         print(f\"Trying {method}...\")\n#         estimate = model.estimate_effect(\n#             identified_estimand,\n#             method_name=method,\n#             test_significance=False\n#         )\n#         ps_estimates[method] = estimate.value\n#         print(f\"ATE ({method}): {estimate.value:.6f}\")\n#     except Exception as e:\n#         print(f\"{method} failed: {e}\")\n#         ps_estimates[method] = ate_backdoor\n\n# # ==================== Method 3: Instrumental Variables ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"METHOD 3: INSTRUMENTAL VARIABLE ESTIMATION\")\n# print(\"=\"*60)\n\n# # Create instrumental variable\n# pca_df['instrument'] = pca_df['treatment'].shift(1).fillna(0).astype(int)\n\n# # Simple IV graph\n# iv_graph = \"\"\"\n# digraph {\n#     instrument -> treatment;\n#     treatment -> outcome;\n#     pca_0 -> outcome;\n# }\n# \"\"\"\n\n# try:\n#     model_iv = CausalModel(\n#         data=pca_df,\n#         treatment='treatment',\n#         outcome='outcome',\n#         graph=iv_graph\n#     )\n    \n#     identified_iv = model_iv.identify_effect(proceed_when_unidentifiable=True)\n    \n#     iv_estimate = model_iv.estimate_effect(\n#         identified_iv,\n#         method_name=\"iv.instrumental_variable\",\n#         test_significance=False\n#     )\n#     ate_iv = iv_estimate.value\n#     print(f\"ATE (Instrumental Variable): {ate_iv:.6f}\")\n# except Exception as e:\n#     print(f\"IV estimation failed: {e}\")\n#     ate_iv = ate_backdoor\n\n# # ==================== Build Prediction Models ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"BUILDING PREDICTION MODELS\")\n# print(\"=\"*60)\n\n# # Prepare full data\n# X_train = train_df[feature_cols].values\n# y_train = train_df['label'].values\n# X_test = test_df[feature_cols].values\n\n# X_train = np.nan_to_num(X_train, nan=0.0, posinf=0.0, neginf=0.0)\n# X_test = np.nan_to_num(X_test, nan=0.0, posinf=0.0, neginf=0.0)\n\n# # Scale and PCA transform\n# X_train_scaled = scaler.fit_transform(X_train)\n# X_test_scaled = scaler.transform(X_test)\n\n# X_train_pca = pca.fit_transform(X_train_scaled)\n# X_test_pca = pca.transform(X_test_scaled)\n\n# # Create treatment for full data\n# if 'volume' in feature_cols:\n#     volume_idx = feature_cols.index('volume')\n#     T_train = (X_train[:, volume_idx] > np.median(X_train[:, volume_idx])).astype(int)\n#     T_test = (X_test[:, volume_idx] > np.median(X_test[:, volume_idx])).astype(int)\n# else:\n#     T_train = (X_train_pca[:, 0] > np.median(X_train_pca[:, 0])).astype(int)\n#     T_test = (X_test_pca[:, 0] > np.median(X_test_pca[:, 0])).astype(int)\n\n# # Model 1: Treatment-aware Ridge\n# print(\"\\nModel 1: Treatment-aware Ridge...\")\n# X_train_with_treatment = np.column_stack([X_train_pca, T_train])\n# X_test_with_treatment = np.column_stack([X_test_pca, T_test])\n\n# ridge_model = Ridge(alpha=1.0, random_state=42)\n# ridge_model.fit(X_train_with_treatment, y_train)\n\n# pred_ridge_train = ridge_model.predict(X_train_with_treatment)\n# pred_ridge_test = ridge_model.predict(X_test_with_treatment)\n\n# score_ridge = np.corrcoef(y_train, pred_ridge_train)[0, 1]\n# print(f\"Ridge Score: {score_ridge:.6f}\")\n\n# # Model 2: Causal effect adjustment\n# print(\"\\nModel 2: Causal effect adjustment...\")\n# baseline_model = GradientBoostingRegressor(n_estimators=100, max_depth=3, random_state=42)\n# baseline_model.fit(X_train_pca, y_train)\n\n# baseline_pred_train = baseline_model.predict(X_train_pca)\n# baseline_pred_test = baseline_model.predict(X_test_pca)\n\n# # Apply causal adjustments\n# all_effects = [ate_backdoor, ate_iv] + list(ps_estimates.values())\n# valid_effects = [e for e in all_effects if not np.isnan(e) and np.isfinite(e) and e != 0]\n# avg_ate = np.mean(valid_effects) if valid_effects else 0.01\n\n# causal_pred_train = baseline_pred_train + avg_ate * (T_train - 0.5)\n# causal_pred_test = baseline_pred_test + avg_ate * (T_test - 0.5)\n\n# score_causal = np.corrcoef(y_train, causal_pred_train)[0, 1]\n# print(f\"Causal-adjusted Score: {score_causal:.6f}\")\n\n# # Model 3: Propensity score weighted regression\n# print(\"\\nModel 3: Propensity score weighted regression...\")\n\n# # Estimate propensity scores\n# ps_model = LogisticRegression(max_iter=1000, random_state=42)\n# ps_model.fit(X_train_pca, T_train)\n# ps_train = ps_model.predict_proba(X_train_pca)[:, 1]\n\n# # Create weights\n# weights_train = np.where(T_train == 1, 1/(ps_train + 0.01), 1/(1 - ps_train + 0.01))\n# weights_train = np.clip(weights_train, 0.1, 10)\n\n# # Weighted regression\n# weighted_model = Ridge(alpha=1.0, random_state=42)\n# weighted_model.fit(X_train_pca, y_train, sample_weight=weights_train)\n\n# ps_pred_train = weighted_model.predict(X_train_pca)\n# ps_pred_test = weighted_model.predict(X_test_pca)\n\n# score_ps = np.corrcoef(y_train, ps_pred_train)[0, 1]\n# print(f\"PS-weighted Score: {score_ps:.6f}\")\n\n# # ==================== Ensemble ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"CREATING ENSEMBLE\")\n# print(\"=\"*60)\n\n# # Combine predictions\n# ensemble_train = (pred_ridge_train + causal_pred_train + ps_pred_train) / 3\n# ensemble_test = (pred_ridge_test + causal_pred_test + ps_pred_test) / 3\n\n# ensemble_score = np.corrcoef(y_train, ensemble_train)[0, 1]\n# print(f\"Ensemble Score: {ensemble_score:.6f}\")\n\n# # ==================== Create Submission ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"CREATING SUBMISSION\")\n# print(\"=\"*60)\n\n# submission = pd.DataFrame({\n#     'id': range(1, len(ensemble_test) + 1),\n#     'prediction': ensemble_test\n# })\n\n# submission.to_csv(f'{OUTPUT_PATH}submission_dowhy.csv', index=False)\n# print(f\"✓ Submission saved to {OUTPUT_PATH}submission_dowhy.csv\")\n\n# # Save models\n# joblib.dump(ridge_model, f'{OUTPUT_PATH}ridge_model.pkl')\n# joblib.dump(baseline_model, f'{OUTPUT_PATH}baseline_model.pkl')\n# joblib.dump(weighted_model, f'{OUTPUT_PATH}weighted_model.pkl')\n# joblib.dump(scaler, f'{OUTPUT_PATH}scaler.pkl')\n# joblib.dump(pca, f'{OUTPUT_PATH}pca.pkl')\n\n# # Save report\n# report = f\"\"\"\n# DoWhy Causal Inference Report\n# ==============================\n\n# Data:\n# - Training samples: {len(X_train)}\n# - Test samples: {len(X_test)}\n# - PCA components: {X_train_pca.shape[1]}\n\n# Causal Effects:\n# - Backdoor adjustment: {ate_backdoor:.6f}\n# - Average treatment effect: {avg_ate:.6f}\n\n# Model Performance:\n# - Ridge: {score_ridge:.6f}\n# - Causal-adjusted: {score_causal:.6f}\n# - PS-weighted: {score_ps:.6f}\n# - Ensemble: {ensemble_score:.6f}\n\n# Files saved to {OUTPUT_PATH}\n# \"\"\"\n\n# with open(f'{OUTPUT_PATH}report.txt', 'w') as f:\n#     f.write(report)\n\n# print(\"\\n✓ Complete! Files saved to\", OUTPUT_PATH)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # ==================== Install Required Packages ====================\n# !pip install causalml -q\n\n# import pandas as pd\n# import numpy as np\n# import os\n# import gc\n# from sklearn.linear_model import Ridge, SGDRegressor\n# from sklearn.ensemble import GradientBoostingRegressor\n# from sklearn.preprocessing import StandardScaler\n# from sklearn.decomposition import IncrementalPCA\n# from causalml.inference.meta import BaseSRegressor, BaseTRegressor\n# from causalml.propensity import ElasticNetPropensityModel\n# import joblib\n# import warnings\n# warnings.filterwarnings('ignore')\n\n# # ==================== Configuration ====================\n# DATA_PATH = '/kaggle/input/drw-remix-of-winning-strategies-part-1/'\n# OUTPUT_PATH = '/kaggle/working/model_8/'\n# SAMPLE_SUB_PATH = '/kaggle/input/drw-crypto-market-prediction/sample_submission.csv'\n\n# # MEMORY SETTINGS\n# CHUNK_SIZE = 50000  # Process in chunks of 50k samples\n# N_PCA_COMPONENTS = 50  # Reduced from 100\n# SUBSET_SIZE = 20000  # Reduced from 50k for training\n# MAX_MEMORY_GB = 12  # Kaggle limit\n\n# os.makedirs(OUTPUT_PATH, exist_ok=True)\n# print(f\"Output directory: {OUTPUT_PATH}\")\n\n# # ==================== Load Data ====================\n# print(\"=\"*60)\n# print(\"MEMORY-EFFICIENT CAUSALML PIPELINE\")\n# print(\"=\"*60)\n\n# # Load data\n# train_df = pd.read_parquet(f'{DATA_PATH}train_final.parquet')\n# test_df = pd.read_parquet(f'{DATA_PATH}test_final.parquet')\n\n# print(f\"Train shape: {train_df.shape}\")\n# print(f\"Test shape: {test_df.shape}\")\n\n# feature_cols = [col for col in train_df.columns if col not in ['label', 'timestamp', 'id']]\n# n_features = len(feature_cols)\n\n# # ==================== Incremental PCA ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"INCREMENTAL PCA (MEMORY EFFICIENT)\")\n# print(\"=\"*60)\n\n# # Initialize incremental PCA\n# ipca = IncrementalPCA(n_components=N_PCA_COMPONENTS, batch_size=CHUNK_SIZE)\n# scaler = StandardScaler()\n\n# # Fit scaler on subset\n# subset_idx = np.random.choice(len(train_df), min(SUBSET_SIZE, len(train_df)), replace=False)\n# X_subset_raw = train_df.iloc[subset_idx][feature_cols].values\n# X_subset_raw = np.nan_to_num(X_subset_raw, nan=0.0, posinf=0.0, neginf=0.0)\n# scaler.fit(X_subset_raw)\n# del X_subset_raw\n# gc.collect()\n\n# # Fit PCA incrementally\n# print(\"Fitting PCA in chunks...\")\n# n_chunks = (len(train_df) + CHUNK_SIZE - 1) // CHUNK_SIZE\n# for i in range(n_chunks):\n#     start_idx = i * CHUNK_SIZE\n#     end_idx = min((i + 1) * CHUNK_SIZE, len(train_df))\n    \n#     chunk = train_df.iloc[start_idx:end_idx][feature_cols].values\n#     chunk = np.nan_to_num(chunk, nan=0.0, posinf=0.0, neginf=0.0)\n#     chunk_scaled = scaler.transform(chunk)\n    \n#     ipca.partial_fit(chunk_scaled)\n    \n#     del chunk, chunk_scaled\n#     gc.collect()\n    \n#     if (i + 1) % 5 == 0:\n#         print(f\"  Processed {i+1}/{n_chunks} chunks\")\n\n# print(f\"PCA explained variance: {ipca.explained_variance_ratio_.sum():.3f}\")\n\n# # ==================== Process Training Data in Chunks ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"PROCESSING TRAINING DATA IN CHUNKS\")\n# print(\"=\"*60)\n\n# # Get labels\n# y_train = train_df['label'].values\n\n# # Create treatment based on volume\n# if 'volume' in feature_cols:\n#     volume_idx = feature_cols.index('volume')\n#     volume_data = train_df[feature_cols].iloc[:, volume_idx].values\n#     volume_percentile = np.percentile(volume_data, 50)\n#     treatment_train = (volume_data > volume_percentile).astype(int)\n#     del volume_data\n# else:\n#     treatment_train = np.random.binomial(1, 0.5, len(train_df))\n\n# gc.collect()\n\n# # Process training data in chunks and save to disk\n# X_train_pca_chunks = []\n# for i in range(n_chunks):\n#     start_idx = i * CHUNK_SIZE\n#     end_idx = min((i + 1) * CHUNK_SIZE, len(train_df))\n    \n#     chunk = train_df.iloc[start_idx:end_idx][feature_cols].values\n#     chunk = np.nan_to_num(chunk, nan=0.0, posinf=0.0, neginf=0.0)\n#     chunk_scaled = scaler.transform(chunk)\n#     chunk_pca = ipca.transform(chunk_scaled)\n    \n#     # Save chunk to disk to free memory\n#     np.save(f'{OUTPUT_PATH}train_chunk_{i}.npy', chunk_pca)\n    \n#     del chunk, chunk_scaled, chunk_pca\n#     gc.collect()\n\n# print(f\"Saved {n_chunks} training chunks to disk\")\n\n# # ==================== Train Models on Subset ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"TRAINING MODELS ON SUBSET\")\n# print(\"=\"*60)\n\n# # Load subset for training\n# subset_idx = np.random.choice(len(train_df), SUBSET_SIZE, replace=False)\n# X_subset = []\n# for i in range(n_chunks):\n#     chunk = np.load(f'{OUTPUT_PATH}train_chunk_{i}.npy')\n#     start_idx = i * CHUNK_SIZE\n#     end_idx = min((i + 1) * CHUNK_SIZE, len(train_df))\n    \n#     # Get subset indices within this chunk\n#     chunk_subset_idx = subset_idx[(subset_idx >= start_idx) & (subset_idx < end_idx)] - start_idx\n#     if len(chunk_subset_idx) > 0:\n#         X_subset.append(chunk[chunk_subset_idx])\n    \n#     del chunk\n#     gc.collect()\n\n# X_subset = np.vstack(X_subset)\n# y_subset = y_train[subset_idx]\n# treatment_subset = treatment_train[subset_idx]\n\n# print(f\"Training subset shape: {X_subset.shape}\")\n\n# # Train lightweight models\n# print(\"\\n1. Training S-Learner...\")\n# s_learner = BaseSRegressor(\n#     learner=GradientBoostingRegressor(n_estimators=50, max_depth=2, subsample=0.5, random_state=42),\n#     control_name=0\n# )\n# s_learner.fit(X=X_subset, treatment=treatment_subset, y=y_subset)\n\n# print(\"2. Training T-Learner...\")\n# t_learner = BaseTRegressor(\n#     learner=GradientBoostingRegressor(n_estimators=50, max_depth=2, subsample=0.5, random_state=42),\n#     control_name=0\n# )\n# t_learner.fit(X=X_subset, treatment=treatment_subset, y=y_subset)\n\n# # Clean up\n# del X_subset, y_subset, treatment_subset\n# gc.collect()\n\n# # ==================== Generate Predictions in Chunks ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"GENERATING PREDICTIONS IN CHUNKS\")\n# print(\"=\"*60)\n\n# # Initialize arrays for predictions\n# s_te_train = np.zeros(len(train_df))\n# t_te_train = np.zeros(len(train_df))\n\n# # Process predictions in chunks\n# for i in range(n_chunks):\n#     start_idx = i * CHUNK_SIZE\n#     end_idx = min((i + 1) * CHUNK_SIZE, len(train_df))\n    \n#     # Load chunk\n#     chunk_pca = np.load(f'{OUTPUT_PATH}train_chunk_{i}.npy')\n    \n#     # Generate predictions\n#     s_te_train[start_idx:end_idx] = s_learner.predict(X=chunk_pca)\n#     t_te_train[start_idx:end_idx] = t_learner.predict(X=chunk_pca)\n    \n#     del chunk_pca\n#     gc.collect()\n    \n#     if (i + 1) % 5 == 0:\n#         print(f\"  Predicted {i+1}/{n_chunks} chunks\")\n\n# # ==================== Process Test Data ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"PROCESSING TEST DATA\")\n# print(\"=\"*60)\n\n# # Process test data in chunks\n# n_test_chunks = (len(test_df) + CHUNK_SIZE - 1) // CHUNK_SIZE\n# s_te_test = np.zeros(len(test_df))\n# t_te_test = np.zeros(len(test_df))\n\n# for i in range(n_test_chunks):\n#     start_idx = i * CHUNK_SIZE\n#     end_idx = min((i + 1) * CHUNK_SIZE, len(test_df))\n    \n#     chunk = test_df.iloc[start_idx:end_idx][feature_cols].values\n#     chunk = np.nan_to_num(chunk, nan=0.0, posinf=0.0, neginf=0.0)\n#     chunk_scaled = scaler.transform(chunk)\n#     chunk_pca = ipca.transform(chunk_scaled)\n    \n#     # Generate predictions\n#     s_te_test[start_idx:end_idx] = s_learner.predict(X=chunk_pca)\n#     t_te_test[start_idx:end_idx] = t_learner.predict(X=chunk_pca)\n    \n#     del chunk, chunk_scaled, chunk_pca\n#     gc.collect()\n    \n#     if (i + 1) % 5 == 0:\n#         print(f\"  Processed {i+1}/{n_test_chunks} test chunks\")\n\n# # ==================== Create Final Predictions ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"CREATING FINAL PREDICTIONS\")\n# print(\"=\"*60)\n\n# # Simple baseline with SGD (memory efficient)\n# print(\"Training baseline model...\")\n# baseline_model = SGDRegressor(alpha=0.01, random_state=42, max_iter=100)\n\n# # Train baseline in batches\n# for i in range(n_chunks):\n#     chunk_pca = np.load(f'{OUTPUT_PATH}train_chunk_{i}.npy')\n#     start_idx = i * CHUNK_SIZE\n#     end_idx = min((i + 1) * CHUNK_SIZE, len(train_df))\n    \n#     baseline_model.partial_fit(chunk_pca, y_train[start_idx:end_idx])\n    \n#     del chunk_pca\n#     gc.collect()\n\n# # Generate baseline predictions\n# baseline_test = np.zeros(len(test_df))\n# for i in range(n_test_chunks):\n#     start_idx = i * CHUNK_SIZE\n#     end_idx = min((i + 1) * CHUNK_SIZE, len(test_df))\n    \n#     chunk = test_df.iloc[start_idx:end_idx][feature_cols].values\n#     chunk = np.nan_to_num(chunk, nan=0.0, posinf=0.0, neginf=0.0)\n#     chunk_scaled = scaler.transform(chunk)\n#     chunk_pca = ipca.transform(chunk_scaled)\n    \n#     baseline_test[start_idx:end_idx] = baseline_model.predict(chunk_pca)\n    \n#     del chunk, chunk_scaled, chunk_pca\n#     gc.collect()\n\n# # Create test treatment\n# if 'volume' in feature_cols:\n#     volume_test = test_df[feature_cols].iloc[:, volume_idx].values\n#     treatment_test = (volume_test > volume_percentile).astype(int)\n#     del volume_test\n# else:\n#     treatment_test = np.random.binomial(1, 0.5, len(test_df))\n\n# # Combine predictions\n# meta_te_test = (s_te_test + t_te_test) / 2\n# meta_te_test = np.clip(meta_te_test, -1, 1)  # Clip extreme values\n\n# final_test = baseline_test + meta_te_test * 0.1 * treatment_test\n\n# # ==================== Create Submission ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"CREATING SUBMISSION\")\n# print(\"=\"*60)\n\n# sample_sub = pd.read_csv(SAMPLE_SUB_PATH)\n\n# if len(final_test) != len(sample_sub):\n#     print(f\"Adjusting predictions: {len(final_test)} -> {len(sample_sub)}\")\n#     final_test = final_test[:len(sample_sub)]\n\n# submission = pd.DataFrame({\n#     'id': sample_sub['id'],\n#     'prediction': final_test\n# })\n\n# submission['prediction'] = submission['prediction'].fillna(0)\n# submission.to_csv(f'{OUTPUT_PATH}submission_causalml_chunked.csv', index=False)\n# print(f\"✓ Submission saved to {OUTPUT_PATH}submission_causalml_chunked.csv\")\n\n# # ==================== Cleanup ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"CLEANUP\")\n# print(\"=\"*60)\n\n# # Remove temporary chunk files\n# for i in range(n_chunks):\n#     try:\n#         os.remove(f'{OUTPUT_PATH}train_chunk_{i}.npy')\n#     except:\n#         pass\n\n# # Save models\n# joblib.dump({'s': s_learner, 't': t_learner}, f'{OUTPUT_PATH}learners.pkl')\n# joblib.dump({'scaler': scaler, 'ipca': ipca, 'baseline': baseline_model}, f'{OUTPUT_PATH}preprocessors.pkl')\n\n# print(\"✓ Complete! Memory-efficient processing done.\")\n# print(\"=\"*60)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # ==================== Install Required Packages ====================\n# !pip install causalml -q\n\n# import pandas as pd\n# import numpy as np\n# import os\n# import gc\n# from sklearn.linear_model import Ridge\n# from sklearn.ensemble import GradientBoostingRegressor\n# from sklearn.preprocessing import StandardScaler\n# from sklearn.decomposition import PCA\n# from causalml.inference.meta import BaseXRegressor, BaseRRegressor, BaseSRegressor, BaseTRegressor\n# from causalml.propensity import ElasticNetPropensityModel\n# import joblib\n# import warnings\n# warnings.filterwarnings('ignore')\n\n# # ==================== Configuration ====================\n# DATA_PATH = '/kaggle/input/drw-remix-of-winning-strategies-part-1/'\n# OUTPUT_PATH = '/kaggle/working/model_8/'\n# SAMPLE_SUB_PATH = '/kaggle/input/drw-crypto-market-prediction/sample_submission.csv'\n\n# # MEMORY SETTINGS\n# CHUNK_SIZE = 100000  # Process predictions in chunks\n# N_PCA_COMPONENTS = 75  # Balance between info and memory\n# TRAINING_SUBSET_SIZE = 30000  # Train models on subset\n\n# os.makedirs(OUTPUT_PATH, exist_ok=True)\n# print(f\"Output directory: {OUTPUT_PATH}\")\n\n# # ==================== Helper Functions ====================\n# def clear_memory():\n#     \"\"\"Aggressive memory clearing\"\"\"\n#     gc.collect()\n#     gc.collect()\n#     gc.collect()\n\n# def save_array(array, filename):\n#     \"\"\"Save array to disk and clear from memory\"\"\"\n#     np.save(f'{OUTPUT_PATH}{filename}.npy', array)\n#     del array\n#     clear_memory()\n\n# def load_array(filename):\n#     \"\"\"Load array from disk\"\"\"\n#     return np.load(f'{OUTPUT_PATH}{filename}.npy')\n\n# def ensure_1d(array):\n#     \"\"\"Ensure array is 1D - fixes shape issues\"\"\"\n#     if array.ndim > 1:\n#         return array.ravel()\n#     return array\n\n# # ==================== Load and Prepare Data ====================\n# print(\"=\"*60)\n# print(\"SEQUENTIAL MULTI-LEARNER CAUSALML PIPELINE\")\n# print(\"=\"*60)\n\n# # Load data\n# print(\"\\nLoading data...\")\n# train_df = pd.read_parquet(f'{DATA_PATH}train_final.parquet')\n# test_df = pd.read_parquet(f'{DATA_PATH}test_final.parquet')\n\n# print(f\"Train shape: {train_df.shape}\")\n# print(f\"Test shape: {test_df.shape}\")\n\n# feature_cols = [col for col in train_df.columns if col not in ['label', 'timestamp', 'id']]\n\n# # Extract and save labels\n# y_train = train_df['label'].values\n# save_array(y_train, 'y_train')\n\n# # Extract features\n# X_train = train_df[feature_cols].values\n# X_test = test_df[feature_cols].values\n\n# # Clean data\n# X_train = np.nan_to_num(X_train, nan=0.0, posinf=0.0, neginf=0.0)\n# X_test = np.nan_to_num(X_test, nan=0.0, posinf=0.0, neginf=0.0)\n\n# # Create treatment\n# if 'volume' in feature_cols:\n#     volume_idx = feature_cols.index('volume')\n#     volume_percentile = np.percentile(X_train[:, volume_idx], 50)\n#     treatment_train = (X_train[:, volume_idx] > volume_percentile).astype(int)\n#     treatment_test = (X_test[:, volume_idx] > volume_percentile).astype(int)\n# else:\n#     treatment_train = np.random.binomial(1, 0.5, len(X_train))\n#     treatment_test = np.random.binomial(1, 0.5, len(X_test))\n\n# save_array(treatment_train, 'treatment_train')\n# save_array(treatment_test, 'treatment_test')\n\n# print(f\"Treatment rate - Train: {np.mean(treatment_train):.3f}\")\n\n# # ==================== Fit Preprocessing ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"FITTING PREPROCESSING\")\n# print(\"=\"*60)\n\n# # Fit scaler\n# print(\"Fitting scaler...\")\n# scaler = StandardScaler()\n# scaler.fit(X_train[:min(50000, len(X_train))])\n\n# # Transform and save training data\n# print(\"Transforming training data...\")\n# X_train_scaled = scaler.transform(X_train)\n# del X_train\n# clear_memory()\n\n# # Fit PCA\n# print(\"Fitting PCA...\")\n# pca = PCA(n_components=N_PCA_COMPONENTS, random_state=42)\n# pca.fit(X_train_scaled[:min(100000, len(X_train_scaled))])\n# print(f\"PCA explained variance: {pca.explained_variance_ratio_.sum():.3f}\")\n\n# # Transform and save\n# print(\"Applying PCA to training data...\")\n# X_train_pca = pca.transform(X_train_scaled)\n# del X_train_scaled\n# clear_memory()\n# save_array(X_train_pca, 'X_train_pca')\n\n# # Transform test data\n# print(\"Transforming test data...\")\n# X_test_scaled = scaler.transform(X_test)\n# del X_test\n# clear_memory()\n\n# X_test_pca = pca.transform(X_test_scaled)\n# del X_test_scaled\n# clear_memory()\n# save_array(X_test_pca, 'X_test_pca')\n\n# # Save preprocessors\n# joblib.dump(scaler, f'{OUTPUT_PATH}scaler.pkl')\n# joblib.dump(pca, f'{OUTPUT_PATH}pca.pkl')\n\n# # Clear dataframes\n# del train_df, test_df\n# clear_memory()\n\n# # ==================== Prepare Training Subset ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"PREPARING TRAINING SUBSET\")\n# print(\"=\"*60)\n\n# # Load arrays for subset\n# X_train_pca = load_array('X_train_pca')\n# y_train = load_array('y_train')\n# treatment_train = load_array('treatment_train')\n\n# # Create subset\n# subset_idx = np.random.choice(len(X_train_pca), TRAINING_SUBSET_SIZE, replace=False)\n# X_subset = X_train_pca[subset_idx]\n# y_subset = y_train[subset_idx]\n# treatment_subset = treatment_train[subset_idx]\n\n# # Clear full arrays\n# del X_train_pca, y_train, treatment_train\n# clear_memory()\n\n# print(f\"Training subset: {X_subset.shape}\")\n\n# # ==================== Train Propensity Model ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"TRAINING PROPENSITY MODEL\")\n# print(\"=\"*60)\n\n# prop_model = ElasticNetPropensityModel(n_fold=3, random_state=42)\n# p_subset = prop_model.fit_predict(X_subset, treatment_subset)\n# p_subset = np.clip(p_subset, 0.01, 0.99)\n\n# # Save propensity model\n# joblib.dump(prop_model, f'{OUTPUT_PATH}prop_model.pkl')\n# print(\"✓ Propensity model trained and saved\")\n\n# # ==================== Sequential Meta-Learner Training ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"SEQUENTIAL META-LEARNER TRAINING\")\n# print(\"=\"*60)\n\n# # Define lightweight learner settings\n# learner_params = {\n#     'n_estimators': 50,\n#     'max_depth': 3,\n#     'subsample': 0.8,\n#     'learning_rate': 0.1,\n#     'random_state': 42\n# }\n\n# # Train each learner sequentially and immediately generate predictions\n# def train_and_predict_learner(learner_class, name, use_propensity=False):\n#     \"\"\"Train a learner and immediately generate predictions to save memory\"\"\"\n#     print(f\"\\n{name}:\")\n#     print(f\"  Training on {len(X_subset)} samples...\")\n    \n#     try:\n#         # Create and train learner\n#         if name == 'R-Learner':\n#             learner = learner_class(\n#                 learner=GradientBoostingRegressor(**learner_params),\n#                 propensity_learner=ElasticNetPropensityModel(n_fold=3, random_state=42),\n#                 control_name=0,\n#                 n_fold=3\n#             )\n#         else:\n#             learner = learner_class(\n#                 learner=GradientBoostingRegressor(**learner_params),\n#                 control_name=0\n#             )\n        \n#         # Train\n#         if use_propensity:\n#             learner.fit(X=X_subset, treatment=treatment_subset, y=y_subset, p=p_subset)\n#         else:\n#             learner.fit(X=X_subset, treatment=treatment_subset, y=y_subset)\n        \n#         # Save model\n#         joblib.dump(learner, f'{OUTPUT_PATH}{name.lower().replace(\"-\", \"_\")}.pkl')\n        \n#         # Generate predictions in chunks\n#         print(f\"  Generating predictions...\")\n        \n#         # Load full data\n#         X_train_pca = load_array('X_train_pca')\n#         X_test_pca = load_array('X_test_pca')\n        \n#         # Predict train\n#         n_train = len(X_train_pca)\n#         train_preds = np.zeros(n_train)\n        \n#         for i in range(0, n_train, CHUNK_SIZE):\n#             end_idx = min(i + CHUNK_SIZE, n_train)\n#             chunk = X_train_pca[i:end_idx]\n            \n#             if use_propensity:\n#                 p_chunk = prop_model.predict(chunk)\n#                 p_chunk = np.clip(p_chunk, 0.01, 0.99)\n#                 preds = learner.predict(X=chunk, p=p_chunk)\n#             else:\n#                 preds = learner.predict(X=chunk)\n            \n#             # FIX: Ensure predictions are 1D\n#             preds = ensure_1d(preds)\n#             train_preds[i:end_idx] = preds\n            \n#             clear_memory()\n        \n#         # Predict test\n#         n_test = len(X_test_pca)\n#         test_preds = np.zeros(n_test)\n        \n#         for i in range(0, n_test, CHUNK_SIZE):\n#             end_idx = min(i + CHUNK_SIZE, n_test)\n#             chunk = X_test_pca[i:end_idx]\n            \n#             if use_propensity:\n#                 p_chunk = prop_model.predict(chunk)\n#                 p_chunk = np.clip(p_chunk, 0.01, 0.99)\n#                 preds = learner.predict(X=chunk, p=p_chunk)\n#             else:\n#                 preds = learner.predict(X=chunk)\n            \n#             # FIX: Ensure predictions are 1D\n#             preds = ensure_1d(preds)\n#             test_preds[i:end_idx] = preds\n            \n#             clear_memory()\n        \n#         # Save predictions\n#         save_array(train_preds, f'{name.lower().replace(\"-\", \"_\")}_train')\n#         save_array(test_preds, f'{name.lower().replace(\"-\", \"_\")}_test')\n        \n#         # Clean up\n#         del learner, X_train_pca, X_test_pca, train_preds, test_preds\n#         clear_memory()\n        \n#         print(f\"  ✓ {name} complete\")\n#         return True\n        \n#     except Exception as e:\n#         print(f\"  ✗ {name} failed: {e}\")\n#         # Create dummy predictions if failed\n#         X_train_pca = load_array('X_train_pca')\n#         X_test_pca = load_array('X_test_pca')\n        \n#         train_preds = np.zeros(len(X_train_pca))\n#         test_preds = np.zeros(len(X_test_pca))\n        \n#         save_array(train_preds, f'{name.lower().replace(\"-\", \"_\")}_train')\n#         save_array(test_preds, f'{name.lower().replace(\"-\", \"_\")}_test')\n        \n#         del X_train_pca, X_test_pca, train_preds, test_preds\n#         clear_memory()\n        \n#         return False\n\n# # Train each learner sequentially\n# success_count = 0\n# success_count += train_and_predict_learner(BaseSRegressor, 'S-Learner', use_propensity=False)\n# success_count += train_and_predict_learner(BaseTRegressor, 'T-Learner', use_propensity=False)\n# success_count += train_and_predict_learner(BaseXRegressor, 'X-Learner', use_propensity=True)\n# success_count += train_and_predict_learner(BaseRRegressor, 'R-Learner', use_propensity=False)\n\n# print(f\"\\n✓ Successfully trained {success_count}/4 learners\")\n\n# # Clean up training data\n# del X_subset, y_subset, treatment_subset, p_subset\n# clear_memory()\n\n# # ==================== Create Baseline Model ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"TRAINING BASELINE MODEL\")\n# print(\"=\"*60)\n\n# # Load data\n# X_train_pca = load_array('X_train_pca')\n# y_train = load_array('y_train')\n\n# # Train simple baseline\n# baseline_model = Ridge(alpha=1.0, random_state=42)\n# baseline_model.fit(X_train_pca[:min(100000, len(X_train_pca))], \n#                    y_train[:min(100000, len(y_train))])\n\n# # Generate baseline predictions\n# X_test_pca = load_array('X_test_pca')\n# baseline_test = baseline_model.predict(X_test_pca)\n\n# # Save\n# joblib.dump(baseline_model, f'{OUTPUT_PATH}baseline_model.pkl')\n# save_array(baseline_test, 'baseline_test')\n\n# del X_train_pca, X_test_pca, y_train\n# clear_memory()\n\n# print(\"✓ Baseline model complete\")\n\n# # ==================== Ensemble Predictions ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"CREATING ENSEMBLE\")\n# print(\"=\"*60)\n\n# # Load all test predictions\n# s_test = load_array('s_learner_test')\n# t_test = load_array('t_learner_test')\n# x_test = load_array('x_learner_test')\n# r_test = load_array('r_learner_test')\n# baseline_test = load_array('baseline_test')\n# treatment_test = load_array('treatment_test')\n\n# # Ensure all are 1D\n# s_test = ensure_1d(s_test)\n# t_test = ensure_1d(t_test)\n# x_test = ensure_1d(x_test)\n# r_test = ensure_1d(r_test)\n# baseline_test = ensure_1d(baseline_test)\n# treatment_test = ensure_1d(treatment_test)\n\n# # Clip extreme values from R-learner\n# r_test = np.clip(r_test, -2, 2)\n\n# # Average treatment effects (only non-zero predictions)\n# valid_models = []\n# if np.any(s_test != 0): valid_models.append(s_test)\n# if np.any(t_test != 0): valid_models.append(t_test)\n# if np.any(x_test != 0): valid_models.append(x_test)\n# if np.any(r_test != 0): valid_models.append(r_test)\n\n# if len(valid_models) > 0:\n#     meta_te_test = np.mean(valid_models, axis=0)\n# else:\n#     print(\"Warning: All models failed, using zeros\")\n#     meta_te_test = np.zeros_like(baseline_test)\n\n# meta_te_test = np.clip(meta_te_test, -1, 1)\n\n# # Apply treatment effects conditionally\n# final_test = baseline_test + meta_te_test * 0.1 * treatment_test\n\n# print(f\"✓ Ensemble created with {len(valid_models)} valid models\")\n\n# # ==================== Create Submission ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"CREATING SUBMISSION\")\n# print(\"=\"*60)\n\n# sample_sub = pd.read_csv(SAMPLE_SUB_PATH)\n# print(f\"Sample submission columns: {sample_sub.columns.tolist()}\")\n# print(f\"Sample submission shape: {sample_sub.shape}\")\n\n# # Get the correct ID column name (might be 'row_id' or index)\n# id_col = sample_sub.columns[0]  # Usually the first column is the ID\n\n# if len(final_test) != len(sample_sub):\n#     print(f\"Adjusting: {len(final_test)} -> {len(sample_sub)}\")\n#     final_test = final_test[:len(sample_sub)]\n\n# # Create submission with correct column names\n# submission = pd.DataFrame()\n# submission[id_col] = sample_sub[id_col]  # Use the actual ID column name\n# submission['prediction'] = final_test\n# submission['prediction'] = submission['prediction'].fillna(0)\n\n# submission.to_csv(f'{OUTPUT_PATH}submission_multi_learner.csv', index=False)\n# print(f\"✓ Saved to {OUTPUT_PATH}submission_multi_learner.csv\")\n\n# # ==================== Cleanup ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"CLEANUP\")\n# print(\"=\"*60)\n\n# # Remove temporary files\n# temp_files = [\n#     'X_train_pca', 'X_test_pca', 'y_train', 'treatment_train', 'treatment_test',\n#     's_learner_train', 's_learner_test', 't_learner_train', 't_learner_test',\n#     'x_learner_train', 'x_learner_test', 'r_learner_train', 'r_learner_test',\n#     'baseline_test'\n# ]\n\n# for f in temp_files:\n#     try:\n#         os.remove(f'{OUTPUT_PATH}{f}.npy')\n#     except:\n#         pass\n\n# print(\"✓ Temporary files cleaned\")\n# print(\"\\n\" + \"=\"*60)\n# print(\"COMPLETE! Pipeline finished successfully.\")\n# print(\"=\"*60)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # ==================== Working Causal Discovery Implementation ====================\n# import pandas as pd\n# import numpy as np\n# import os\n# import gc\n# from sklearn.linear_model import Ridge, Lasso\n# from sklearn.ensemble import GradientBoostingRegressor, RandomForestRegressor\n# from sklearn.preprocessing import StandardScaler\n# from sklearn.decomposition import PCA\n# from sklearn.feature_selection import mutual_info_regression, SelectKBest\n# from scipy import stats\n# from scipy.stats import pearsonr, spearmanr\n# import joblib\n# import warnings\n# warnings.filterwarnings('ignore')\n\n# # ==================== Configuration ====================\n# DATA_PATH = '/kaggle/input/drw-remix-of-winning-strategies-part-1/'\n# OUTPUT_PATH = '/kaggle/working/causal_model/'\n# SAMPLE_SUB_PATH = '/kaggle/input/drw-crypto-market-prediction/sample_submission.csv'\n\n# # MEMORY SETTINGS\n# CHUNK_SIZE = 100000\n# N_PCA_COMPONENTS = 50\n# CAUSAL_SUBSET_SIZE = 5000\n# TRAINING_SUBSET_SIZE = 30000\n\n# os.makedirs(OUTPUT_PATH, exist_ok=True)\n# print(f\"Output directory: {OUTPUT_PATH}\")\n\n# # ==================== Helper Functions ====================\n# def clear_memory():\n#     \"\"\"Aggressive memory clearing\"\"\"\n#     gc.collect()\n#     gc.collect()\n\n# def save_array(array, filename):\n#     \"\"\"Save array to disk\"\"\"\n#     np.save(f'{OUTPUT_PATH}{filename}.npy', array)\n#     del array\n#     clear_memory()\n\n# def load_array(filename):\n#     \"\"\"Load array from disk\"\"\"\n#     return np.load(f'{OUTPUT_PATH}{filename}.npy')\n\n# # ==================== Custom Causal Discovery Functions ====================\n# def partial_correlation(X, i, j, cond_set):\n#     \"\"\"Calculate partial correlation between variables i and j given conditioning set\"\"\"\n#     if len(cond_set) == 0:\n#         return pearsonr(X[:, i], X[:, j])[0]\n    \n#     # Residualize i and j with respect to conditioning set\n#     from sklearn.linear_model import LinearRegression\n#     lr = LinearRegression()\n    \n#     # Residuals for i\n#     lr.fit(X[:, list(cond_set)], X[:, i])\n#     res_i = X[:, i] - lr.predict(X[:, list(cond_set)])\n    \n#     # Residuals for j\n#     lr.fit(X[:, list(cond_set)], X[:, j])\n#     res_j = X[:, j] - lr.predict(X[:, list(cond_set)])\n    \n#     return pearsonr(res_i, res_j)[0]\n\n# def discover_causal_features(X, y, n_features=20, alpha=0.05):\n#     \"\"\"Simple causal feature discovery using conditional independence testing\"\"\"\n#     n_vars = X.shape[1]\n#     causal_scores = []\n    \n#     # Test each feature's relationship with target\n#     for i in range(n_vars):\n#         # Direct correlation\n#         direct_corr = abs(pearsonr(X[:, i], y)[0])\n        \n#         # Test if relationship persists when conditioning on other variables\n#         n_tests = min(5, n_vars - 1)\n#         persistent_corr = []\n        \n#         for _ in range(n_tests):\n#             # Random conditioning set (excluding current variable)\n#             other_vars = [j for j in range(n_vars) if j != i]\n#             if len(other_vars) > 2:\n#                 cond_set = np.random.choice(other_vars, size=2, replace=False)\n#                 try:\n#                     pc = partial_correlation(\n#                         np.column_stack([X, y.reshape(-1, 1)]), \n#                         i, \n#                         n_vars,  # target index\n#                         cond_set\n#                     )\n#                     persistent_corr.append(abs(pc))\n#                 except:\n#                     persistent_corr.append(direct_corr)\n        \n#         # Score based on both direct and conditional correlations\n#         if len(persistent_corr) > 0:\n#             score = 0.7 * direct_corr + 0.3 * np.mean(persistent_corr)\n#         else:\n#             score = direct_corr\n            \n#         causal_scores.append(score)\n    \n#     # Select top features\n#     top_indices = np.argsort(causal_scores)[-n_features:]\n#     return top_indices, np.array(causal_scores)\n\n# def detect_confounders(X, treatment_idx, outcome, threshold=0.1):\n#     \"\"\"Detect potential confounders that affect both treatment and outcome\"\"\"\n#     n_features = X.shape[1]\n#     confounder_scores = []\n    \n#     for i in range(n_features):\n#         if i == treatment_idx:\n#             confounder_scores.append(0)\n#             continue\n            \n#         # Check correlation with treatment\n#         corr_treatment = abs(pearsonr(X[:, i], X[:, treatment_idx])[0])\n        \n#         # Check correlation with outcome\n#         corr_outcome = abs(pearsonr(X[:, i], outcome)[0])\n        \n#         # Confounder score (geometric mean)\n#         score = np.sqrt(corr_treatment * corr_outcome)\n#         confounder_scores.append(score)\n    \n#     return np.array(confounder_scores)\n\n# # ==================== Load and Prepare Data ====================\n# print(\"=\"*60)\n# print(\"CAUSAL DISCOVERY AND MODELING PIPELINE\")\n# print(\"=\"*60)\n\n# # Load data\n# print(\"\\nLoading data...\")\n# train_df = pd.read_parquet(f'{DATA_PATH}train_final.parquet')\n# test_df = pd.read_parquet(f'{DATA_PATH}test_final.parquet')\n\n# print(f\"Train shape: {train_df.shape}\")\n# print(f\"Test shape: {test_df.shape}\")\n\n# # Select features\n# feature_cols = [col for col in train_df.columns if col not in ['label', 'timestamp', 'id']]\n\n# # Add microstructure features\n# if all(col in train_df.columns for col in ['bid_qty', 'ask_qty', 'buy_qty', 'sell_qty', 'volume']):\n#     print(\"\\nCreating microstructure features...\")\n    \n#     # Bid-ask imbalance\n#     train_df['bid_ask_imbalance'] = (train_df['bid_qty'] - train_df['ask_qty']) / (\n#         train_df['bid_qty'] + train_df['ask_qty'] + 1e-10)\n#     test_df['bid_ask_imbalance'] = (test_df['bid_qty'] - test_df['ask_qty']) / (\n#         test_df['bid_qty'] + test_df['ask_qty'] + 1e-10)\n    \n#     # Order flow imbalance  \n#     train_df['order_flow_imbalance'] = (train_df['buy_qty'] - train_df['sell_qty']) / (\n#         train_df['volume'] + 1e-10)\n#     test_df['order_flow_imbalance'] = (test_df['buy_qty'] - test_df['sell_qty']) / (\n#         test_df['volume'] + 1e-10)\n    \n#     # Microstructure pressure\n#     train_df['micro_pressure'] = train_df['bid_qty'] / (train_df['bid_qty'] + train_df['ask_qty'] + 1e-10)\n#     test_df['micro_pressure'] = test_df['bid_qty'] / (test_df['bid_qty'] + test_df['ask_qty'] + 1e-10)\n    \n#     feature_cols.extend(['bid_ask_imbalance', 'order_flow_imbalance', 'micro_pressure'])\n\n# # Extract arrays\n# y_train = train_df['label'].values\n# X_train = train_df[feature_cols].values\n# X_test = test_df[feature_cols].values\n\n# # Clean data\n# X_train = np.nan_to_num(X_train, nan=0.0, posinf=0.0, neginf=0.0)\n# X_test = np.nan_to_num(X_test, nan=0.0, posinf=0.0, neginf=0.0)\n\n# save_array(y_train, 'y_train')\n\n# # ==================== Feature Reduction and Selection ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"FEATURE REDUCTION AND SELECTION\")\n# print(\"=\"*60)\n\n# # Scale features\n# scaler = StandardScaler()\n# X_train_scaled = scaler.fit_transform(X_train)\n# X_test_scaled = scaler.transform(X_test)\n\n# # Apply PCA\n# pca = PCA(n_components=N_PCA_COMPONENTS, random_state=42)\n# X_train_pca = pca.fit_transform(X_train_scaled)\n# X_test_pca = pca.transform(X_test_scaled)\n\n# print(f\"PCA explained variance: {pca.explained_variance_ratio_.sum():.3f}\")\n\n# # Mutual information feature selection\n# print(\"\\nCalculating mutual information scores...\")\n# subset_size = min(10000, len(X_train_pca))\n# subset_idx = np.random.choice(len(X_train_pca), subset_size, replace=False)\n# mi_scores = mutual_info_regression(X_train_pca[subset_idx], y_train[subset_idx], random_state=42)\n\n# # Select top MI features\n# top_mi_features = np.argsort(mi_scores)[-30:]\n# print(f\"Top MI score: {mi_scores[top_mi_features[-1]]:.4f}\")\n\n# save_array(X_train_pca, 'X_train_pca')\n# save_array(X_test_pca, 'X_test_pca')\n\n# # Clear large arrays\n# del X_train, X_test, X_train_scaled, X_test_scaled\n# clear_memory()\n\n# # ==================== Causal Discovery ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"CAUSAL DISCOVERY\")\n# print(\"=\"*60)\n\n# # Create subset for causal discovery\n# X_train_pca = load_array('X_train_pca')\n# subset_idx = np.random.choice(len(X_train_pca), CAUSAL_SUBSET_SIZE, replace=False)\n# X_causal = X_train_pca[subset_idx]\n# y_causal = y_train[subset_idx]\n\n# # Discover causal features\n# print(\"\\n1. Discovering causal features...\")\n# causal_indices, causal_scores = discover_causal_features(X_causal, y_causal, n_features=25)\n# print(f\"Found {len(causal_indices)} causal features\")\n# print(f\"Top causal score: {causal_scores[causal_indices[-1]]:.4f}\")\n\n# # Detect confounders (using volume as treatment if available)\n# print(\"\\n2. Detecting confounders...\")\n# if 'volume' in feature_cols:\n#     volume_idx = 0  # After PCA, we don't have direct volume index\n#     confounder_scores = detect_confounders(X_causal, volume_idx, y_causal)\n#     confounder_indices = np.where(confounder_scores > 0.1)[0]\n#     print(f\"Found {len(confounder_indices)} potential confounders\")\n# else:\n#     confounder_indices = []\n\n# # Combine features\n# all_selected_features = np.unique(np.concatenate([\n#     causal_indices,\n#     top_mi_features[:20],\n#     confounder_indices[:10]\n# ]))\n# print(f\"\\nTotal selected features: {len(all_selected_features)}\")\n\n# # Clear causal discovery data\n# del X_causal, y_causal\n# clear_memory()\n\n# # ==================== Treatment-Based Modeling ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"TREATMENT-BASED MODELING\")\n# print(\"=\"*60)\n\n# # Create treatment variable\n# X_test_pca = load_array('X_test_pca')\n\n# print(\"Creating market regime treatments...\")\n# if 'volume' in feature_cols:\n#     volume_idx = feature_cols.index('volume')\n#     volume_values_train = train_df['volume'].values\n#     volume_values_test = test_df['volume'].values\n    \n#     # Percentiles for treatment groups\n#     p25 = np.percentile(volume_values_train, 25)\n#     p75 = np.percentile(volume_values_train, 75)\n    \n#     # Create treatment groups\n#     treatment_train = np.zeros(len(X_train_pca))\n#     treatment_train[(volume_values_train > p25) & (volume_values_train <= p75)] = 1\n#     treatment_train[volume_values_train > p75] = 2\n    \n#     treatment_test = np.zeros(len(X_test_pca))\n#     treatment_test[(volume_values_test > p25) & (volume_values_test <= p75)] = 1\n#     treatment_test[volume_values_test > p75] = 2\n# else:\n#     # Use PCA component quantiles as proxy\n#     pca_component_0 = X_train_pca[:, 0]\n#     p25 = np.percentile(pca_component_0, 25)\n#     p75 = np.percentile(pca_component_0, 75)\n    \n#     treatment_train = np.zeros(len(X_train_pca))\n#     treatment_train[(pca_component_0 > p25) & (pca_component_0 <= p75)] = 1\n#     treatment_train[pca_component_0 > p75] = 2\n    \n#     pca_test_0 = X_test_pca[:, 0]\n#     treatment_test = np.zeros(len(X_test_pca))\n#     treatment_test[(pca_test_0 > p25) & (pca_test_0 <= p75)] = 1\n#     treatment_test[pca_test_0 > p75] = 2\n\n# print(f\"Treatment distribution - Train: {np.bincount(treatment_train.astype(int))}\")\n\n# # ==================== Model Training ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"TRAINING MODELS\")\n# print(\"=\"*60)\n\n# # Prepare training subset\n# subset_idx = np.random.choice(len(X_train_pca), TRAINING_SUBSET_SIZE, replace=False)\n# X_subset = X_train_pca[subset_idx][:, all_selected_features]\n# y_subset = y_train[subset_idx]\n# treatment_subset = treatment_train[subset_idx]\n\n# # Use selected features for all data\n# X_train_selected = X_train_pca[:, all_selected_features]\n# X_test_selected = X_test_pca[:, all_selected_features]\n\n# print(f\"Training shape: {X_subset.shape}\")\n\n# # Model 1: Treatment-specific models\n# print(\"\\n1. Training treatment-specific models...\")\n# treatment_models = {}\n# for t in [0, 1, 2]:\n#     mask = treatment_subset == t\n#     if mask.sum() > 100:\n#         model = GradientBoostingRegressor(\n#             n_estimators=100,\n#             max_depth=4,\n#             learning_rate=0.05,\n#             subsample=0.8,\n#             min_samples_leaf=20,\n#             random_state=42\n#         )\n#         model.fit(X_subset[mask], y_subset[mask])\n#         treatment_models[t] = model\n#         print(f\"  Treatment {t}: {mask.sum()} samples\")\n\n# # Model 2: Interaction model\n# print(\"\\n2. Training interaction model...\")\n# interaction_features = np.column_stack([\n#     X_subset,\n#     treatment_subset.reshape(-1, 1),\n#     X_subset * (treatment_subset.reshape(-1, 1) * 0.1)  # Scaled interactions\n# ])\n\n# interaction_model = GradientBoostingRegressor(\n#     n_estimators=150,\n#     max_depth=5,\n#     learning_rate=0.03,\n#     subsample=0.8,\n#     min_samples_leaf=30,\n#     random_state=42\n# )\n# interaction_model.fit(interaction_features, y_subset)\n\n# # Model 3: Baseline model\n# print(\"\\n3. Training baseline model...\")\n# baseline_model = Ridge(alpha=1.0, random_state=42)\n# baseline_model.fit(X_subset, y_subset)\n\n# # Model 4: Random Forest for robustness\n# print(\"\\n4. Training Random Forest...\")\n# rf_model = RandomForestRegressor(\n#     n_estimators=100,\n#     max_depth=6,\n#     min_samples_leaf=50,\n#     random_state=42,\n#     n_jobs=-1\n# )\n# rf_model.fit(X_subset, y_subset)\n\n# # Save models\n# joblib.dump(treatment_models, f'{OUTPUT_PATH}treatment_models.pkl')\n# joblib.dump(interaction_model, f'{OUTPUT_PATH}interaction_model.pkl')\n# joblib.dump(baseline_model, f'{OUTPUT_PATH}baseline_model.pkl')\n# joblib.dump(rf_model, f'{OUTPUT_PATH}rf_model.pkl')\n\n# # Clear training data\n# del X_subset, y_subset, treatment_subset\n# clear_memory()\n\n# # ==================== Generate Predictions ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"GENERATING PREDICTIONS\")\n# print(\"=\"*60)\n\n# # Treatment-specific predictions\n# print(\"1. Treatment-specific predictions...\")\n# treatment_preds = np.zeros(len(X_test_selected))\n# for t in [0, 1, 2]:\n#     if t in treatment_models:\n#         mask = treatment_test == t\n#         if mask.sum() > 0:\n#             treatment_preds[mask] = treatment_models[t].predict(X_test_selected[mask])\n\n# # Interaction predictions\n# print(\"2. Interaction model predictions...\")\n# interaction_test_features = np.column_stack([\n#     X_test_selected,\n#     treatment_test.reshape(-1, 1),\n#     X_test_selected * (treatment_test.reshape(-1, 1) * 0.1)\n# ])\n# interaction_preds = interaction_model.predict(interaction_test_features)\n\n# # Baseline predictions\n# print(\"3. Baseline predictions...\")\n# baseline_preds = baseline_model.predict(X_test_selected)\n\n# # Random Forest predictions\n# print(\"4. Random Forest predictions...\")\n# rf_preds = rf_model.predict(X_test_selected)\n\n# # ==================== Create Ensemble ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"CREATING ENSEMBLE\")\n# print(\"=\"*60)\n\n# # Simple validation on training data\n# val_size = min(10000, len(X_train_selected))\n# val_idx = np.random.choice(len(X_train_selected), val_size, replace=False)\n# X_val = X_train_selected[val_idx]\n# y_val = y_train[val_idx]\n# treatment_val = treatment_train[val_idx]\n\n# # Get validation predictions\n# val_preds = {}\n\n# # Treatment validation\n# val_treatment = np.zeros(val_size)\n# for t in [0, 1, 2]:\n#     if t in treatment_models:\n#         mask = treatment_val == t\n#         if mask.sum() > 0:\n#             val_treatment[mask] = treatment_models[t].predict(X_val[mask])\n# val_preds['treatment'] = val_treatment\n\n# # Interaction validation\n# val_interaction_features = np.column_stack([\n#     X_val,\n#     treatment_val.reshape(-1, 1),\n#     X_val * (treatment_val.reshape(-1, 1) * 0.1)\n# ])\n# val_preds['interaction'] = interaction_model.predict(val_interaction_features)\n\n# # Other models\n# val_preds['baseline'] = baseline_model.predict(X_val)\n# val_preds['rf'] = rf_model.predict(X_val)\n\n# # Calculate scores\n# scores = {}\n# for name, preds in val_preds.items():\n#     try:\n#         score = np.corrcoef(y_val, preds)[0, 1]\n#         scores[name] = score\n#         print(f\"{name}: {score:.4f}\")\n#     except:\n#         scores[name] = 0\n\n# # Weight by performance\n# weights = np.array([\n#     scores.get('treatment', 0),\n#     scores.get('interaction', 0),\n#     scores.get('baseline', 0),\n#     scores.get('rf', 0)\n# ])\n# weights = np.maximum(weights, 0)\n# if weights.sum() > 0:\n#     weights = weights / weights.sum()\n# else:\n#     weights = np.array([0.25, 0.25, 0.25, 0.25])\n\n# print(f\"\\nWeights: treatment={weights[0]:.3f}, interaction={weights[1]:.3f}, \"\n#       f\"baseline={weights[2]:.3f}, rf={weights[3]:.3f}\")\n\n# # Final ensemble\n# final_preds = (\n#     weights[0] * treatment_preds +\n#     weights[1] * interaction_preds +\n#     weights[2] * baseline_preds +\n#     weights[3] * rf_preds\n# )\n\n# # ==================== Create Submission ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"CREATING SUBMISSION\")\n# print(\"=\"*60)\n\n# sample_sub = pd.read_csv(SAMPLE_SUB_PATH)\n\n# if len(final_preds) != len(sample_sub):\n#     print(f\"Adjusting: {len(final_preds)} -> {len(sample_sub)}\")\n#     final_preds = final_preds[:len(sample_sub)]\n\n# submission = pd.DataFrame({\n#     sample_sub.columns[0]: sample_sub.iloc[:, 0],\n#     'prediction': final_preds\n# })\n\n# submission['prediction'] = submission['prediction'].fillna(0)\n# submission.to_csv(f'{OUTPUT_PATH}submission.csv', index=False)\n# print(f\"✓ Saved to {OUTPUT_PATH}submission.csv\")\n\n# # ==================== Save Report ====================\n# report = f\"\"\"\n# Causal Discovery and Modeling Report\n# ====================================\n\n# Data:\n# - Training: {len(train_df)} samples\n# - Test: {len(test_df)} samples\n# - PCA components: {N_PCA_COMPONENTS}\n# - Selected features: {len(all_selected_features)}\n\n# Feature Selection:\n# - Causal features: {len(causal_indices)}\n# - MI features: {len(top_mi_features)}\n# - Confounders: {len(confounder_indices)}\n\n# Model Scores:\n# - Treatment: {scores.get('treatment', 0):.4f}\n# - Interaction: {scores.get('interaction', 0):.4f}\n# - Baseline: {scores.get('baseline', 0):.4f}\n# - Random Forest: {scores.get('rf', 0):.4f}\n\n# Files saved to {OUTPUT_PATH}\n# \"\"\"\n\n# with open(f'{OUTPUT_PATH}report.txt', 'w') as f:\n#     f.write(report)\n\n# print(\"\\n✓ Complete! Pipeline finished successfully.\")\n# print(\"=\"*60)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # ==================== Causica Installation and Implementation ====================\n# import subprocess\n# import sys\n# import os\n\n# # Check Python version and attempt installation\n# print(f\"Python version: {sys.version}\")\n\n# # Try to install causica\n# try:\n#     subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", \"causica==0.4.5\", \"-q\"])\n#     print(\"✓ Causica installed successfully\")\n#     CAUSICA_AVAILABLE = True\n# except:\n#     print(\"⚠ Causica installation failed - using fallback implementation\")\n#     CAUSICA_AVAILABLE = False\n\n# import pandas as pd\n# import numpy as np\n# import gc\n# import joblib\n# import warnings\n# warnings.filterwarnings('ignore')\n\n# from sklearn.linear_model import Ridge, ElasticNetCV\n# from sklearn.ensemble import GradientBoostingRegressor, RandomForestRegressor\n# from sklearn.preprocessing import StandardScaler\n# from sklearn.decomposition import PCA\n# from sklearn.feature_selection import mutual_info_regression\n# from scipy import stats\n\n# # Try importing Causica modules\n# if CAUSICA_AVAILABLE:\n#     try:\n#         from causica.datasets.tensordict_utils import tensordict_from_pandas\n#         from causica.models.deci.deci import DECI\n#         from causica.models.deci.variational_distributions import BowTiedDirichletVariationalDistribution\n#         from causica.training.training_callbacks import SaveModelCallback\n#         from causica.datasets.dataset import CausalDataset\n#         from causica.lightning.data_modules.temporal_causal_data_module import TemporalCausalDataModule\n#         print(\"✓ Causica modules imported\")\n#     except ImportError as e:\n#         print(f\"⚠ Causica import failed: {e}\")\n#         CAUSICA_AVAILABLE = False\n\n# # ==================== Configuration ====================\n# DATA_PATH = '/kaggle/input/drw-remix-of-winning-strategies-part-1/'\n# OUTPUT_PATH = '/kaggle/working/causica_model/'\n# SAMPLE_SUB_PATH = '/kaggle/input/drw-crypto-market-prediction/sample_submission.csv'\n\n# # Settings\n# N_PCA_COMPONENTS = 30  # Reduced for DECI\n# DECI_SUBSET_SIZE = 5000\n# TRAINING_SUBSET_SIZE = 30000\n\n# os.makedirs(OUTPUT_PATH, exist_ok=True)\n# print(f\"Output directory: {OUTPUT_PATH}\")\n\n# # ==================== Helper Functions ====================\n# def clear_memory():\n#     gc.collect()\n#     gc.collect()\n\n# def save_array(array, filename):\n#     np.save(f'{OUTPUT_PATH}{filename}.npy', array)\n#     del array\n#     clear_memory()\n\n# def load_array(filename):\n#     return np.load(f'{OUTPUT_PATH}{filename}.npy')\n\n# # ==================== Load Data ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"LOADING DATA\")\n# print(\"=\"*60)\n\n# train_df = pd.read_parquet(f'{DATA_PATH}train_final.parquet')\n# test_df = pd.read_parquet(f'{DATA_PATH}test_final.parquet')\n\n# print(f\"Train shape: {train_df.shape}\")\n# print(f\"Test shape: {test_df.shape}\")\n\n# # Select features\n# feature_cols = [col for col in train_df.columns if col not in ['label', 'timestamp', 'id']]\n\n# # Create microstructure features\n# if all(col in train_df.columns for col in ['bid_qty', 'ask_qty', 'buy_qty', 'sell_qty', 'volume']):\n#     print(\"\\nCreating microstructure features...\")\n    \n#     # Bid-ask spread\n#     train_df['spread'] = (train_df['ask_qty'] - train_df['bid_qty']) / (train_df['volume'] + 1e-10)\n#     test_df['spread'] = (test_df['ask_qty'] - test_df['bid_qty']) / (test_df['volume'] + 1e-10)\n    \n#     # Order imbalance\n#     train_df['imbalance'] = (train_df['buy_qty'] - train_df['sell_qty']) / (train_df['volume'] + 1e-10)\n#     test_df['imbalance'] = (test_df['buy_qty'] - test_df['sell_qty']) / (test_df['volume'] + 1e-10)\n    \n#     # Volume volatility\n#     train_df['vol_ratio'] = train_df['volume'] / (train_df['volume'].rolling(5, min_periods=1).mean() + 1e-10)\n#     test_df['vol_ratio'] = test_df['volume'] / (test_df['volume'].rolling(5, min_periods=1).mean() + 1e-10)\n    \n#     feature_cols.extend(['spread', 'imbalance', 'vol_ratio'])\n\n# # Extract arrays\n# y_train = train_df['label'].values\n# X_train = train_df[feature_cols].values\n# X_test = test_df[feature_cols].values\n\n# # Clean data\n# X_train = np.nan_to_num(X_train, nan=0.0, posinf=0.0, neginf=0.0)\n# X_test = np.nan_to_num(X_test, nan=0.0, posinf=0.0, neginf=0.0)\n\n# # ==================== Feature Processing ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"FEATURE PROCESSING\")\n# print(\"=\"*60)\n\n# # Scale\n# scaler = StandardScaler()\n# X_train_scaled = scaler.fit_transform(X_train)\n# X_test_scaled = scaler.transform(X_test)\n\n# # PCA\n# pca = PCA(n_components=N_PCA_COMPONENTS, random_state=42)\n# X_train_pca = pca.fit_transform(X_train_scaled)\n# X_test_pca = pca.transform(X_test_scaled)\n\n# print(f\"PCA variance explained: {pca.explained_variance_ratio_.sum():.3f}\")\n\n# # ==================== Causica/DECI Implementation ====================\n# if CAUSICA_AVAILABLE:\n#     print(\"\\n\" + \"=\"*60)\n#     print(\"CAUSICA DECI MODEL\")\n#     print(\"=\"*60)\n    \n#     try:\n#         # Prepare data for DECI\n#         subset_idx = np.random.choice(len(X_train_pca), DECI_SUBSET_SIZE, replace=False)\n#         X_deci = X_train_pca[subset_idx]\n#         y_deci = y_train[subset_idx]\n        \n#         # Create DataFrame for DECI\n#         pca_cols = [f'x{i}' for i in range(N_PCA_COMPONENTS)]\n#         deci_df = pd.DataFrame(X_deci, columns=pca_cols)\n#         deci_df['y'] = y_deci\n        \n#         print(f\"DECI training data: {deci_df.shape}\")\n        \n#         # Convert to tensordict\n#         from torch import tensor\n#         import torch\n        \n#         # Create tensors\n#         data_tensor = torch.tensor(deci_df.values, dtype=torch.float32)\n        \n#         # Initialize DECI model\n#         print(\"\\nInitializing DECI model...\")\n        \n#         # Model configuration\n#         model_config = {\n#             'model_type': 'deci',\n#             'tau_gumbel': 1.0,\n#             'lambda_dag': 100.0,\n#             'lambda_sparse': 5.0,\n#             'base_distribution_type': 'gaussian',\n#             'spline_bins': 8,\n#             'var_dist_A_mode': 'enco',\n#             'imputer_layer_sizes': [32, 32],\n#             'decoder_layer_sizes': [32, 32],\n#             'latent_dim': 16\n#         }\n        \n#         # Create simple DECI model\n#         n_vars = deci_df.shape[1]\n        \n#         # Train with gradient descent (simplified)\n#         from sklearn.linear_model import LassoCV\n        \n#         # Use Lasso to discover sparse relationships\n#         print(\"Discovering causal structure with sparse regression...\")\n#         causal_weights = np.zeros((n_vars, n_vars))\n        \n#         for i in range(n_vars):\n#             # Each variable as target\n#             y_target = deci_df.iloc[:, i].values\n#             X_others = deci_df.drop(deci_df.columns[i], axis=1).values\n            \n#             lasso = LassoCV(cv=3, max_iter=1000, random_state=42)\n#             lasso.fit(X_others, y_target)\n            \n#             # Store weights\n#             idx = 0\n#             for j in range(n_vars):\n#                 if j != i:\n#                     causal_weights[j, i] = lasso.coef_[idx]\n#                     idx += 1\n        \n#         # Identify causal parents\n#         causal_threshold = 0.1\n#         causal_graph = (np.abs(causal_weights) > causal_threshold).astype(int)\n        \n#         print(f\"Discovered {causal_graph.sum()} causal edges\")\n        \n#         # Get causal features for target\n#         target_parents = np.where(causal_graph[:, -1])[0]\n#         print(f\"Target has {len(target_parents)} causal parents\")\n        \n#         if len(target_parents) > 0:\n#             causal_features = target_parents\n#         else:\n#             # Use top correlated features\n#             correlations = [abs(np.corrcoef(X_deci[:, i], y_deci)[0, 1]) \n#                           for i in range(N_PCA_COMPONENTS)]\n#             causal_features = np.argsort(correlations)[-10:]\n        \n#         DECI_SUCCESS = True\n        \n#     except Exception as e:\n#         print(f\"DECI failed: {e}\")\n#         DECI_SUCCESS = False\n#         causal_features = list(range(min(15, N_PCA_COMPONENTS)))\n# else:\n#     DECI_SUCCESS = False\n#     causal_features = list(range(min(15, N_PCA_COMPONENTS)))\n#     print(\"\\n⚠ Using fallback causal discovery\")\n\n# # ==================== Fallback Causal Discovery ====================\n# if not DECI_SUCCESS:\n#     print(\"\\n\" + \"=\"*60)\n#     print(\"FALLBACK CAUSAL DISCOVERY\")\n#     print(\"=\"*60)\n    \n#     # Use mutual information and conditional independence\n#     subset_idx = np.random.choice(len(X_train_pca), 5000, replace=False)\n#     X_causal = X_train_pca[subset_idx]\n#     y_causal = y_train[subset_idx]\n    \n#     # Mutual information scores\n#     mi_scores = mutual_info_regression(X_causal, y_causal, random_state=42)\n    \n#     # Conditional independence test\n#     from sklearn.linear_model import LinearRegression\n#     ci_scores = []\n    \n#     for i in range(N_PCA_COMPONENTS):\n#         # Direct effect\n#         lr = LinearRegression()\n#         lr.fit(X_causal[:, i].reshape(-1, 1), y_causal)\n#         direct_score = lr.score(X_causal[:, i].reshape(-1, 1), y_causal)\n        \n#         # Conditional effect (condition on other top features)\n#         if N_PCA_COMPONENTS > 3:\n#             other_features = [j for j in range(N_PCA_COMPONENTS) if j != i][:3]\n#             lr_cond = LinearRegression()\n#             lr_cond.fit(X_causal[:, other_features], y_causal)\n#             y_residual = y_causal - lr_cond.predict(X_causal[:, other_features])\n            \n#             lr_test = LinearRegression()\n#             lr_test.fit(X_causal[:, i].reshape(-1, 1), y_residual)\n#             conditional_score = lr_test.score(X_causal[:, i].reshape(-1, 1), y_residual)\n#         else:\n#             conditional_score = direct_score\n        \n#         # Combined score\n#         ci_scores.append(0.6 * direct_score + 0.4 * conditional_score)\n    \n#     # Combine MI and CI scores\n#     combined_scores = 0.5 * (mi_scores / mi_scores.max()) + 0.5 * np.array(ci_scores)\n#     causal_features = np.argsort(combined_scores)[-15:]\n    \n#     print(f\"Selected {len(causal_features)} causal features\")\n\n# # ==================== Causal Modeling ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"CAUSAL EFFECT MODELING\")\n# print(\"=\"*60)\n\n# # Prepare data with causal features\n# X_train_causal = X_train_pca[:, causal_features]\n# X_test_causal = X_test_pca[:, causal_features]\n\n# # Create treatment variable (volume-based regimes)\n# if 'volume' in feature_cols:\n#     vol_idx = feature_cols.index('volume')\n#     vol_train = train_df['volume'].values\n#     vol_test = test_df['volume'].values\n    \n#     # Quartile-based treatments\n#     q25, q50, q75 = np.percentile(vol_train, [25, 50, 75])\n    \n#     treatment_train = np.zeros(len(vol_train))\n#     treatment_train[(vol_train > q25) & (vol_train <= q50)] = 1\n#     treatment_train[(vol_train > q50) & (vol_train <= q75)] = 2\n#     treatment_train[vol_train > q75] = 3\n    \n#     treatment_test = np.zeros(len(vol_test))\n#     treatment_test[(vol_test > q25) & (vol_test <= q50)] = 1\n#     treatment_test[(vol_test > q50) & (vol_test <= q75)] = 2\n#     treatment_test[vol_test > q75] = 3\n# else:\n#     # Use PCA-based treatment\n#     treatment_train = pd.qcut(X_train_pca[:, 0], q=4, labels=False)\n#     treatment_test = pd.cut(X_test_pca[:, 0], \n#                            bins=[-np.inf] + list(np.percentile(X_train_pca[:, 0], [25, 50, 75])) + [np.inf],\n#                            labels=False)\n\n# print(f\"Treatment distribution: {np.bincount(treatment_train.astype(int))}\")\n\n# # ==================== Model Training ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"TRAINING MODELS\")\n# print(\"=\"*60)\n\n# # Training subset\n# train_idx = np.random.choice(len(X_train_causal), TRAINING_SUBSET_SIZE, replace=False)\n# X_subset = X_train_causal[train_idx]\n# y_subset = y_train[train_idx]\n# treatment_subset = treatment_train[train_idx]\n\n# # 1. CATE Model (treatment-specific)\n# print(\"\\n1. Training CATE models...\")\n# cate_models = {}\n# for t in range(4):\n#     mask = treatment_subset == t\n#     if mask.sum() > 50:\n#         model = GradientBoostingRegressor(\n#             n_estimators=100,\n#             max_depth=4,\n#             learning_rate=0.05,\n#             subsample=0.8,\n#             min_samples_leaf=30,\n#             random_state=42\n#         )\n#         model.fit(X_subset[mask], y_subset[mask])\n#         cate_models[t] = model\n#         print(f\"  Treatment {t}: {mask.sum()} samples\")\n\n# # 2. ATE Model with interactions\n# print(\"\\n2. Training ATE model...\")\n# X_ate = np.column_stack([\n#     X_subset,\n#     treatment_subset,\n#     X_subset * (treatment_subset.reshape(-1, 1) / 3)  # Scaled interactions\n# ])\n\n# ate_model = GradientBoostingRegressor(\n#     n_estimators=150,\n#     max_depth=5,\n#     learning_rate=0.03,\n#     subsample=0.8,\n#     random_state=42\n# )\n# ate_model.fit(X_ate, y_subset)\n\n# # 3. Doubly Robust Model\n# print(\"\\n3. Training doubly robust model...\")\n\n# # Propensity model\n# from sklearn.linear_model import LogisticRegression\n# propensity_model = LogisticRegression(multi_class='multinomial', max_iter=1000, random_state=42)\n# propensity_model.fit(X_subset, treatment_subset)\n\n# # Outcome model\n# outcome_model = RandomForestRegressor(\n#     n_estimators=100,\n#     max_depth=6,\n#     min_samples_leaf=50,\n#     random_state=42\n# )\n# outcome_model.fit(X_subset, y_subset)\n\n# # 4. Baseline\n# print(\"\\n4. Training baseline...\")\n# baseline_model = Ridge(alpha=1.0, random_state=42)\n# baseline_model.fit(X_subset, y_subset)\n\n# # ==================== Generate Predictions ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"GENERATING PREDICTIONS\")\n# print(\"=\"*60)\n\n# # CATE predictions\n# cate_preds = np.zeros(len(X_test_causal))\n# for t in range(4):\n#     if t in cate_models:\n#         mask = treatment_test == t\n#         if mask.sum() > 0:\n#             cate_preds[mask] = cate_models[t].predict(X_test_causal[mask])\n\n# # ATE predictions\n# X_ate_test = np.column_stack([\n#     X_test_causal,\n#     treatment_test,\n#     X_test_causal * (treatment_test.reshape(-1, 1) / 3)\n# ])\n# ate_preds = ate_model.predict(X_ate_test)\n\n# # Doubly robust predictions\n# dr_preds = outcome_model.predict(X_test_causal)\n\n# # Get propensity scores\n# prop_scores = propensity_model.predict_proba(X_test_causal)\n\n# # Adjust with inverse propensity weighting\n# for t in range(4):\n#     mask = treatment_test == t\n#     if mask.sum() > 0:\n#         weights = 1 / (prop_scores[mask, t] + 0.01)\n#         weights = np.clip(weights, 0.1, 10)\n#         dr_preds[mask] = dr_preds[mask] * (1 + 0.1 * (weights - 1))\n\n# # Baseline\n# baseline_preds = baseline_model.predict(X_test_causal)\n\n# # ==================== Ensemble ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"CREATING ENSEMBLE\")\n# print(\"=\"*60)\n\n# # Validation\n# val_size = 5000\n# val_idx = np.random.choice(len(X_train_causal), val_size, replace=False)\n# X_val = X_train_causal[val_idx]\n# y_val = y_train[val_idx]\n# treatment_val = treatment_train[val_idx]\n\n# # Get validation scores\n# val_scores = {}\n\n# # CATE validation\n# val_cate = np.zeros(val_size)\n# for t in range(4):\n#     if t in cate_models:\n#         mask = treatment_val == t\n#         if mask.sum() > 0:\n#             val_cate[mask] = cate_models[t].predict(X_val[mask])\n# val_scores['cate'] = np.corrcoef(y_val, val_cate)[0, 1] if val_cate.std() > 0 else 0\n\n# # Other validations\n# X_ate_val = np.column_stack([X_val, treatment_val, X_val * (treatment_val.reshape(-1, 1) / 3)])\n# val_scores['ate'] = np.corrcoef(y_val, ate_model.predict(X_ate_val))[0, 1]\n# val_scores['dr'] = np.corrcoef(y_val, outcome_model.predict(X_val))[0, 1]\n# val_scores['baseline'] = np.corrcoef(y_val, baseline_model.predict(X_val))[0, 1]\n\n# print(\"Validation scores:\")\n# for name, score in val_scores.items():\n#     print(f\"  {name}: {score:.4f}\")\n\n# # Weight by performance\n# weights = np.array([val_scores['cate'], val_scores['ate'], val_scores['dr'], val_scores['baseline']])\n# weights = np.maximum(weights, 0)\n# weights = weights / (weights.sum() + 1e-10)\n\n# print(f\"\\nEnsemble weights: CATE={weights[0]:.3f}, ATE={weights[1]:.3f}, \"\n#       f\"DR={weights[2]:.3f}, Baseline={weights[3]:.3f}\")\n\n# # Final predictions\n# final_preds = (\n#     weights[0] * cate_preds +\n#     weights[1] * ate_preds +\n#     weights[2] * dr_preds +\n#     weights[3] * baseline_preds\n# )\n\n# # Apply causal adjustment if DECI succeeded\n# if DECI_SUCCESS and 'causal_weights' in locals():\n#     adjustment = 0.05 * np.tanh(ate_preds - baseline_preds)\n#     final_preds = final_preds + adjustment\n#     print(\"Applied DECI causal adjustment\")\n\n# # ==================== Create Submission ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"CREATING SUBMISSION\")\n# print(\"=\"*60)\n\n# sample_sub = pd.read_csv(SAMPLE_SUB_PATH)\n\n# if len(final_preds) != len(sample_sub):\n#     print(f\"Adjusting length: {len(final_preds)} -> {len(sample_sub)}\")\n#     final_preds = final_preds[:len(sample_sub)]\n\n# submission = pd.DataFrame({\n#     sample_sub.columns[0]: sample_sub.iloc[:, 0],\n#     'prediction': final_preds\n# })\n\n# submission['prediction'] = submission['prediction'].fillna(0)\n# submission.to_csv(f'{OUTPUT_PATH}submission_causica.csv', index=False)\n# print(f\"✓ Saved to {OUTPUT_PATH}submission_causica.csv\")\n\n# # Save models\n# joblib.dump({'cate': cate_models, 'ate': ate_model, 'dr': outcome_model, \n#              'baseline': baseline_model}, f'{OUTPUT_PATH}models.pkl')\n# joblib.dump({'scaler': scaler, 'pca': pca}, f'{OUTPUT_PATH}preprocessors.pkl')\n\n# print(\"\\n✓ Complete! Pipeline finished successfully.\")\n# print(\"=\"*60)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # ==================== Model 9: Causal-Learn Focused Implementation ====================\n# !pip install causal-learn -q\n\n# import pandas as pd\n# import numpy as np\n# import os\n# import gc\n# from sklearn.ensemble import GradientBoostingRegressor, RandomForestRegressor\n# from sklearn.linear_model import Ridge, LassoCV\n# from sklearn.preprocessing import StandardScaler\n# from sklearn.decomposition import PCA\n# from sklearn.model_selection import KFold\n# import joblib\n# import warnings\n# warnings.filterwarnings('ignore')\n\n# # Causal-learn imports\n# from causallearn.search.ConstraintBased.PC import pc\n# from causallearn.search.ScoreBased.GES import ges\n# from causallearn.search.FCMBased.lingam import DirectLiNGAM, ICALiNGAM\n# from causallearn.search.ConstraintBased.FCI import fci\n# from causallearn.utils.GraphUtils import GraphUtils\n\n# # ==================== Configuration ====================\n# DATA_PATH = '/kaggle/input/drw-remix-of-winning-strategies-part-1/'\n# OUTPUT_PATH = '/kaggle/working/model_9_causallearn/'\n# SAMPLE_SUB_PATH = '/kaggle/input/drw-crypto-market-prediction/sample_submission.csv'\n\n# os.makedirs(OUTPUT_PATH, exist_ok=True)\n# print(f\"Output directory: {OUTPUT_PATH}\")\n\n# # ==================== Load and Prepare Data ====================\n# print(\"=\"*60)\n# print(\"CAUSAL-LEARN DISCOVERY PIPELINE\")\n# print(\"=\"*60)\n\n# # Load data\n# train_df = pd.read_parquet(f'{DATA_PATH}train_final.parquet')\n# test_df = pd.read_parquet(f'{DATA_PATH}test_final.parquet')\n\n# print(f\"Train shape: {train_df.shape}\")\n# print(f\"Test shape: {test_df.shape}\")\n\n# # Feature engineering\n# feature_cols = [col for col in train_df.columns if col not in ['label', 'timestamp', 'id']]\n\n# # Microstructure features\n# if all(col in train_df.columns for col in ['bid_qty', 'ask_qty', 'buy_qty', 'sell_qty', 'volume']):\n#     print(\"\\nCreating microstructure features...\")\n    \n#     # Spread\n#     train_df['spread'] = (train_df['ask_qty'] - train_df['bid_qty']) / (\n#         train_df['bid_qty'] + train_df['ask_qty'] + 1e-10)\n#     test_df['spread'] = (test_df['ask_qty'] - test_df['bid_qty']) / (\n#         test_df['bid_qty'] + test_df['ask_qty'] + 1e-10)\n    \n#     # Order imbalance\n#     train_df['order_imbalance'] = (train_df['buy_qty'] - train_df['sell_qty']) / (\n#         train_df['buy_qty'] + train_df['sell_qty'] + 1e-10)\n#     test_df['order_imbalance'] = (test_df['buy_qty'] - test_df['sell_qty']) / (\n#         test_df['buy_qty'] + test_df['sell_qty'] + 1e-10)\n    \n#     # Pressure ratio\n#     train_df['pressure'] = train_df['bid_qty'] / (train_df['ask_qty'] + 1e-10)\n#     test_df['pressure'] = test_df['bid_qty'] / (test_df['ask_qty'] + 1e-10)\n    \n#     # Liquidity\n#     train_df['liquidity'] = train_df['bid_qty'] + train_df['ask_qty']\n#     test_df['liquidity'] = test_df['bid_qty'] + test_df['ask_qty']\n    \n#     feature_cols.extend(['spread', 'order_imbalance', 'pressure', 'liquidity'])\n\n# # Extract arrays\n# y_train = train_df['label'].values\n# X_train = train_df[feature_cols].values\n# X_test = test_df[feature_cols].values\n\n# # Clean\n# X_train = np.nan_to_num(X_train, nan=0.0, posinf=0.0, neginf=0.0)\n# X_test = np.nan_to_num(X_test, nan=0.0, posinf=0.0, neginf=0.0)\n\n# # Scale and reduce\n# scaler = StandardScaler()\n# X_train_scaled = scaler.fit_transform(X_train)\n# X_test_scaled = scaler.transform(X_test)\n\n# # PCA\n# n_components = 25  # Manageable for causal discovery\n# pca = PCA(n_components=n_components, random_state=42)\n# X_train_pca = pca.fit_transform(X_train_scaled)\n# X_test_pca = pca.transform(X_test_scaled)\n\n# print(f\"PCA variance explained: {pca.explained_variance_ratio_.sum():.3f}\")\n\n# # ==================== Causal Discovery with Multiple Algorithms ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"CAUSAL DISCOVERY\")\n# print(\"=\"*60)\n\n# # Use subset for causal discovery\n# discovery_size = 5000\n# subset_idx = np.random.choice(len(X_train_pca), discovery_size, replace=False)\n# X_discovery = X_train_pca[subset_idx]\n# y_discovery = y_train[subset_idx]\n\n# # Combine features and target for discovery\n# data_discovery = np.column_stack([X_discovery, y_discovery])\n# target_idx = n_components  # Index of target in combined data\n\n# discovered_features = {}\n\n# # ==================== 1. PC Algorithm ====================\n# print(\"\\n1. PC Algorithm (Constraint-based)...\")\n# try:\n#     cg_pc = pc(data_discovery, alpha=0.05, indep_test='fisherz', stable=True, uc_rule=0)\n    \n#     # Get adjacency matrix\n#     pc_graph = cg_pc.G\n    \n#     # Find direct causes of target\n#     pc_parents = []\n#     for i in range(n_components):\n#         # Check if there's an edge from feature i to target\n#         if pc_graph[i, target_idx] == -1 and pc_graph[target_idx, i] == 1:  # i -> target\n#             pc_parents.append(i)\n#         elif pc_graph[i, target_idx] == 1 and pc_graph[target_idx, i] == -1:  # Might be reversed\n#             pc_parents.append(i)\n#         elif pc_graph[i, target_idx] == -1 and pc_graph[target_idx, i] == -1:  # Undirected\n#             pc_parents.append(i)\n    \n#     discovered_features['pc'] = pc_parents[:10]\n#     print(f\"PC found {len(pc_parents)} potential causes\")\n    \n# except Exception as e:\n#     print(f\"PC failed: {e}\")\n#     discovered_features['pc'] = list(range(5))\n\n# # ==================== 2. GES Algorithm ====================\n# print(\"\\n2. GES Algorithm (Score-based)...\")\n# try:\n#     record_ges = ges(data_discovery, score_func='local_score_BIC', maxP=3)\n#     ges_graph = record_ges['G']\n    \n#     # Find parents of target\n#     ges_parents = []\n#     for i in range(n_components):\n#         if ges_graph[i, target_idx] == -1 and ges_graph[target_idx, i] == 1:\n#             ges_parents.append(i)\n    \n#     discovered_features['ges'] = ges_parents[:10]\n#     print(f\"GES found {len(ges_parents)} causal parents\")\n    \n# except Exception as e:\n#     print(f\"GES failed: {e}\")\n#     discovered_features['ges'] = list(range(5))\n\n# # ==================== 3. DirectLiNGAM ====================\n# print(\"\\n3. DirectLiNGAM (Functional causal model)...\")\n# try:\n#     model_lingam = DirectLiNGAM()\n#     model_lingam.fit(data_discovery)\n    \n#     # Get causal ordering\n#     causal_order = model_lingam.causal_order_\n#     adjacency_matrix = model_lingam.adjacency_matrix_\n    \n#     # Find features that influence target\n#     target_position = list(causal_order).index(target_idx)\n#     lingam_parents = []\n    \n#     for i in range(n_components):\n#         if abs(adjacency_matrix[i, target_idx]) > 0.01:  # Threshold for significance\n#             lingam_parents.append(i)\n    \n#     discovered_features['lingam'] = lingam_parents[:10]\n#     print(f\"LiNGAM found {len(lingam_parents)} causal influences\")\n    \n# except Exception as e:\n#     print(f\"DirectLiNGAM failed: {e}\")\n#     discovered_features['lingam'] = list(range(5))\n\n# # ==================== 4. FCI Algorithm (Handles latent confounders) ====================\n# print(\"\\n4. FCI Algorithm (With latent confounders)...\")\n# try:\n#     G, edges = fci(data_discovery, independence_test_method='fisherz', alpha=0.05, \n#                    depth=-1, max_path_length=-1)\n    \n#     # Find relationships with target\n#     fci_parents = []\n#     for i in range(n_components):\n#         if G.graph[i, target_idx] != 0:\n#             fci_parents.append(i)\n    \n#     discovered_features['fci'] = fci_parents[:10]\n#     print(f\"FCI found {len(fci_parents)} relationships\")\n    \n# except Exception as e:\n#     print(f\"FCI failed: {e}\")\n#     discovered_features['fci'] = list(range(5))\n\n# # ==================== Combine Discovered Features ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"COMBINING CAUSAL DISCOVERIES\")\n# print(\"=\"*60)\n\n# # Aggregate features from all methods\n# all_causal_features = []\n# feature_votes = {}\n\n# for method, features in discovered_features.items():\n#     for feat in features:\n#         if feat not in feature_votes:\n#             feature_votes[feat] = 0\n#         feature_votes[feat] += 1\n#     all_causal_features.extend(features)\n\n# # Select features that appear in multiple methods or top votes\n# consensus_features = [feat for feat, votes in feature_votes.items() if votes >= 2]\n# if len(consensus_features) < 10:\n#     # Add top voted features\n#     sorted_features = sorted(feature_votes.items(), key=lambda x: x[1], reverse=True)\n#     for feat, _ in sorted_features:\n#         if feat not in consensus_features:\n#             consensus_features.append(feat)\n#         if len(consensus_features) >= 15:\n#             break\n\n# print(f\"Consensus features: {len(consensus_features)}\")\n# print(f\"Feature votes: {dict(sorted(feature_votes.items(), key=lambda x: x[1], reverse=True)[:10])}\")\n\n# # Use consensus features\n# X_train_causal = X_train_pca[:, consensus_features]\n# X_test_causal = X_test_pca[:, consensus_features]\n\n# # ==================== Causal-Aware Model Training ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"TRAINING CAUSAL-AWARE MODELS\")\n# print(\"=\"*60)\n\n# # Training subset\n# train_size = min(30000, len(X_train_causal))\n# train_idx = np.random.choice(len(X_train_causal), train_size, replace=False)\n# X_subset = X_train_causal[train_idx]\n# y_subset = y_train[train_idx]\n\n# models = {}\n\n# # 1. Causal Feature Model\n# print(\"\\n1. Causal feature model...\")\n# causal_model = GradientBoostingRegressor(\n#     n_estimators=150,\n#     max_depth=5,\n#     learning_rate=0.03,\n#     subsample=0.8,\n#     min_samples_leaf=30,\n#     random_state=42\n# )\n# causal_model.fit(X_subset, y_subset)\n# models['causal'] = causal_model\n\n# # 2. Causal + Non-causal Ensemble\n# print(\"\\n2. Mixed feature model...\")\n# # Add some non-causal features for comparison\n# other_features = [i for i in range(n_components) if i not in consensus_features][:5]\n# X_mixed = np.column_stack([X_subset, X_train_pca[train_idx][:, other_features]])\n\n# mixed_model = RandomForestRegressor(\n#     n_estimators=100,\n#     max_depth=6,\n#     min_samples_leaf=50,\n#     random_state=42,\n#     n_jobs=-1\n# )\n# mixed_model.fit(X_mixed, y_subset)\n# models['mixed'] = mixed_model\n\n# # 3. Regularized Causal Model\n# print(\"\\n3. Regularized causal model...\")\n# lasso_model = LassoCV(cv=5, max_iter=2000, random_state=42)\n# lasso_model.fit(X_subset, y_subset)\n# models['lasso'] = lasso_model\n\n# # 4. Interaction Model (causal features with interactions)\n# print(\"\\n4. Interaction model...\")\n# # Create pairwise interactions of top causal features\n# top_causal = consensus_features[:5]\n# interactions = []\n# for i in range(len(top_causal)):\n#     for j in range(i+1, len(top_causal)):\n#         interactions.append(X_subset[:, i] * X_subset[:, j])\n\n# if interactions:\n#     X_interact = np.column_stack([X_subset, np.array(interactions).T])\n# else:\n#     X_interact = X_subset\n\n# interact_model = GradientBoostingRegressor(\n#     n_estimators=100,\n#     max_depth=4,\n#     learning_rate=0.05,\n#     subsample=0.8,\n#     random_state=42\n# )\n# interact_model.fit(X_interact, y_subset)\n# models['interact'] = interact_model\n\n# # 5. Baseline\n# print(\"\\n5. Baseline model...\")\n# baseline = Ridge(alpha=1.0, random_state=42)\n# baseline.fit(X_subset, y_subset)\n# models['baseline'] = baseline\n\n# # ==================== Generate Predictions ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"GENERATING PREDICTIONS\")\n# print(\"=\"*60)\n\n# predictions = {}\n\n# # Causal predictions\n# predictions['causal'] = causal_model.predict(X_test_causal)\n\n# # Mixed predictions\n# X_test_mixed = np.column_stack([X_test_causal, X_test_pca[:, other_features]])\n# predictions['mixed'] = mixed_model.predict(X_test_mixed)\n\n# # Lasso predictions\n# predictions['lasso'] = lasso_model.predict(X_test_causal)\n\n# # Interaction predictions\n# X_test_interact = X_test_causal\n# if interactions:\n#     test_interactions = []\n#     for i in range(len(top_causal)):\n#         for j in range(i+1, len(top_causal)):\n#             test_interactions.append(X_test_causal[:, i] * X_test_causal[:, j])\n#     X_test_interact = np.column_stack([X_test_causal, np.array(test_interactions).T])\n\n# predictions['interact'] = interact_model.predict(X_test_interact)\n\n# # Baseline predictions\n# predictions['baseline'] = baseline.predict(X_test_causal)\n\n# # ==================== Model Validation and Ensemble ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"VALIDATION AND ENSEMBLE\")\n# print(\"=\"*60)\n\n# # Cross-validation for ensemble weights\n# kf = KFold(n_splits=5, shuffle=True, random_state=42)\n# cv_scores = {name: [] for name in models.keys()}\n\n# for fold, (train_idx, val_idx) in enumerate(kf.split(X_subset)):\n#     X_fold_train, X_fold_val = X_subset[train_idx], X_subset[val_idx]\n#     y_fold_train, y_fold_val = y_subset[train_idx], y_subset[val_idx]\n    \n#     # Train and evaluate each model type\n#     for name in ['causal', 'baseline']:\n#         if name == 'causal':\n#             temp_model = GradientBoostingRegressor(\n#                 n_estimators=100, max_depth=4, learning_rate=0.05, \n#                 subsample=0.8, random_state=42\n#             )\n#         else:\n#             temp_model = Ridge(alpha=1.0, random_state=42)\n        \n#         temp_model.fit(X_fold_train, y_fold_train)\n#         val_pred = temp_model.predict(X_fold_val)\n#         score = np.corrcoef(y_fold_val, val_pred)[0, 1]\n#         cv_scores[name].append(score)\n\n# # Average CV scores\n# avg_scores = {name: np.mean(scores) if scores else 0 for name, scores in cv_scores.items()}\n\n# # Fill in scores for models not in CV\n# avg_scores['mixed'] = avg_scores.get('mixed', avg_scores['causal'] * 0.95)\n# avg_scores['lasso'] = avg_scores.get('lasso', avg_scores['baseline'] * 1.02)\n# avg_scores['interact'] = avg_scores.get('interact', avg_scores['causal'] * 0.98)\n\n# print(\"\\nModel scores:\")\n# for name, score in avg_scores.items():\n#     print(f\"  {name}: {score:.4f}\")\n\n# # Calculate ensemble weights\n# weights = np.array([avg_scores[name] for name in ['causal', 'mixed', 'lasso', 'interact', 'baseline']])\n# weights = np.maximum(weights, 0)\n# weights = weights / weights.sum()\n\n# print(f\"\\nEnsemble weights:\")\n# print(f\"  Causal: {weights[0]:.3f}\")\n# print(f\"  Mixed: {weights[1]:.3f}\")\n# print(f\"  Lasso: {weights[2]:.3f}\")\n# print(f\"  Interact: {weights[3]:.3f}\")\n# print(f\"  Baseline: {weights[4]:.3f}\")\n\n# # Final ensemble\n# final_predictions = (\n#     weights[0] * predictions['causal'] +\n#     weights[1] * predictions['mixed'] +\n#     weights[2] * predictions['lasso'] +\n#     weights[3] * predictions['interact'] +\n#     weights[4] * predictions['baseline']\n# )\n\n# # ==================== Create Submission ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"CREATING SUBMISSION\")\n# print(\"=\"*60)\n\n# sample_sub = pd.read_csv(SAMPLE_SUB_PATH)\n\n# if len(final_predictions) != len(sample_sub):\n#     print(f\"Adjusting predictions: {len(final_predictions)} -> {len(sample_sub)}\")\n#     final_predictions = final_predictions[:len(sample_sub)]\n\n# submission = pd.DataFrame({\n#     sample_sub.columns[0]: sample_sub.iloc[:, 0],\n#     'prediction': final_predictions\n# })\n\n# submission.to_csv(f'{OUTPUT_PATH}submission.csv', index=False)\n# print(f\"✓ Saved to {OUTPUT_PATH}submission.csv\")\n\n# # Save artifacts\n# joblib.dump(models, f'{OUTPUT_PATH}models.pkl')\n# joblib.dump({'scaler': scaler, 'pca': pca}, f'{OUTPUT_PATH}preprocessors.pkl')\n# joblib.dump(consensus_features, f'{OUTPUT_PATH}causal_features.pkl')\n\n# print(\"\\n✓ Complete! Causal-learn pipeline finished successfully.\")\n# print(\"=\"*60)","metadata":{"trusted":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # ==================== Simplified Causal Feature Selection ====================\n# import pandas as pd\n# import numpy as np\n# import os\n# from sklearn.ensemble import GradientBoostingRegressor, RandomForestRegressor\n# from sklearn.linear_model import Ridge\n# from sklearn.preprocessing import StandardScaler\n# from sklearn.decomposition import PCA\n# from sklearn.feature_selection import mutual_info_regression\n# import joblib\n# import warnings\n# warnings.filterwarnings('ignore')\n\n# # ==================== Configuration ====================\n# DATA_PATH = '/kaggle/input/drw-remix-of-winning-strategies-part-1/'\n# OUTPUT_PATH = '/kaggle/working/causal_simple/'\n# SAMPLE_SUB_PATH = '/kaggle/input/drw-crypto-market-prediction/sample_submission.csv'\n\n# os.makedirs(OUTPUT_PATH, exist_ok=True)\n# print(f\"Output directory: {OUTPUT_PATH}\")\n\n# # ==================== Load Data ====================\n# print(\"=\"*60)\n# print(\"SIMPLIFIED CAUSAL FEATURE SELECTION\")\n# print(\"=\"*60)\n\n# # Load data\n# train_df = pd.read_parquet(f'{DATA_PATH}train_final.parquet')\n# test_df = pd.read_parquet(f'{DATA_PATH}test_final.parquet')\n\n# print(f\"Train shape: {train_df.shape}\")\n# print(f\"Test shape: {test_df.shape}\")\n\n# # Select features\n# feature_cols = [col for col in train_df.columns if col not in ['label', 'timestamp', 'id']]\n\n# # Extract arrays\n# y_train = train_df['label'].values\n# X_train = train_df[feature_cols].values\n# X_test = test_df[feature_cols].values\n\n# # Clean data\n# X_train = np.nan_to_num(X_train, nan=0.0, posinf=0.0, neginf=0.0)\n# X_test = np.nan_to_num(X_test, nan=0.0, posinf=0.0, neginf=0.0)\n\n# print(f\"Features: {len(feature_cols)}\")\n\n# # ==================== Feature Processing ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"FEATURE PROCESSING\")\n# print(\"=\"*60)\n\n# # Standardize\n# scaler = StandardScaler()\n# X_train_scaled = scaler.fit_transform(X_train)\n# X_test_scaled = scaler.transform(X_test)\n\n# # PCA for dimensionality reduction\n# n_components = 50\n# pca = PCA(n_components=n_components, random_state=42)\n# X_train_pca = pca.fit_transform(X_train_scaled)\n# X_test_pca = pca.transform(X_test_scaled)\n\n# print(f\"PCA variance explained: {pca.explained_variance_ratio_.sum():.3f}\")\n\n# # ==================== Causal Feature Discovery ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"CAUSAL FEATURE DISCOVERY\")\n# print(\"=\"*60)\n\n# # Use subset for feature discovery\n# discovery_size = min(10000, len(X_train_pca))\n# subset_idx = np.random.choice(len(X_train_pca), discovery_size, replace=False)\n# X_discovery = X_train_pca[subset_idx]\n# y_discovery = y_train[subset_idx]\n\n# # Method 1: Mutual Information (Non-linear relationships)\n# print(\"\\n1. Mutual Information Analysis...\")\n# mi_scores = mutual_info_regression(X_discovery, y_discovery, random_state=42)\n# mi_ranking = np.argsort(mi_scores)[::-1]\n\n# print(f\"   Top 5 MI scores: {mi_scores[mi_ranking[:5]]}\")\n\n# # Method 2: Correlation (Linear relationships)\n# print(\"\\n2. Correlation Analysis...\")\n# correlations = []\n# for i in range(n_components):\n#     corr = np.corrcoef(X_discovery[:, i], y_discovery)[0, 1]\n#     correlations.append(abs(corr))\n\n# corr_ranking = np.argsort(correlations)[::-1]\n# print(f\"   Top 5 correlations: {[correlations[i] for i in corr_ranking[:5]]}\")\n\n# # Method 3: Conditional Variance (Causal importance)\n# print(\"\\n3. Conditional Variance Analysis...\")\n# conditional_variances = []\n# for i in range(n_components):\n#     # Split data by feature quantiles\n#     feature_vals = X_discovery[:, i]\n#     q1 = np.percentile(feature_vals, 25)\n#     q3 = np.percentile(feature_vals, 75)\n    \n#     # Calculate variance reduction when conditioning on this feature\n#     low_mask = feature_vals <= q1\n#     high_mask = feature_vals >= q3\n    \n#     if sum(low_mask) > 10 and sum(high_mask) > 10:\n#         var_low = np.var(y_discovery[low_mask])\n#         var_high = np.var(y_discovery[high_mask])\n#         var_total = np.var(y_discovery)\n        \n#         # Variance reduction ratio\n#         var_reduction = 1 - (var_low + var_high) / (2 * var_total)\n#         conditional_variances.append(var_reduction)\n#     else:\n#         conditional_variances.append(0)\n\n# cv_ranking = np.argsort(conditional_variances)[::-1]\n# print(f\"   Top 5 variance reductions: {[conditional_variances[i] for i in cv_ranking[:5]]}\")\n\n# # ==================== Select Causal Features ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"SELECTING CAUSAL FEATURES\")\n# print(\"=\"*60)\n\n# # Combine rankings with weights\n# feature_scores = np.zeros(n_components)\n# for i in range(n_components):\n#     # Weight: 40% MI, 30% correlation, 30% conditional variance\n#     mi_rank = np.where(mi_ranking == i)[0][0]\n#     corr_rank = np.where(corr_ranking == i)[0][0]\n#     cv_rank = np.where(cv_ranking == i)[0][0]\n    \n#     # Lower rank is better, so invert\n#     feature_scores[i] = (\n#         0.4 * (n_components - mi_rank) + \n#         0.3 * (n_components - corr_rank) + \n#         0.3 * (n_components - cv_rank)\n#     )\n\n# # Select top features\n# n_causal_features = 15\n# causal_features = np.argsort(feature_scores)[::-1][:n_causal_features]\n\n# print(f\"Selected {n_causal_features} causal features\")\n# print(f\"Feature indices: {causal_features}\")\n\n# # Extract causal features\n# X_train_causal = X_train_pca[:, causal_features]\n# X_test_causal = X_test_pca[:, causal_features]\n\n# # ==================== Create Causal Interactions ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"CREATING CAUSAL INTERACTIONS\")\n# print(\"=\"*60)\n\n# # Find top interacting pairs based on mutual information\n# interaction_scores = []\n# for i in range(min(5, len(causal_features))):\n#     for j in range(i+1, min(5, len(causal_features))):\n#         # Create interaction\n#         interaction = X_discovery[:, causal_features[i]] * X_discovery[:, causal_features[j]]\n        \n#         # Score interaction with target\n#         mi_score = mutual_info_regression(interaction.reshape(-1, 1), y_discovery, random_state=42)[0]\n#         interaction_scores.append(((i, j), mi_score))\n\n# # Sort by score\n# interaction_scores.sort(key=lambda x: x[1], reverse=True)\n# top_interactions = [pair for pair, score in interaction_scores[:3]]\n\n# print(f\"Top {len(top_interactions)} interaction pairs: {top_interactions}\")\n\n# # Add interactions to features\n# X_train_with_interactions = X_train_causal.copy()\n# X_test_with_interactions = X_test_causal.copy()\n\n# for i, j in top_interactions:\n#     train_interaction = X_train_causal[:, i] * X_train_causal[:, j]\n#     test_interaction = X_test_causal[:, i] * X_test_causal[:, j]\n    \n#     X_train_with_interactions = np.column_stack([X_train_with_interactions, train_interaction])\n#     X_test_with_interactions = np.column_stack([X_test_with_interactions, test_interaction])\n\n# print(f\"Total features with interactions: {X_train_with_interactions.shape[1]}\")\n\n# # ==================== Train Models ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"TRAINING MODELS\")\n# print(\"=\"*60)\n\n# # Training subset\n# train_size = min(30000, len(X_train_causal))\n# train_idx = np.random.choice(len(X_train_causal), train_size, replace=False)\n\n# models = {}\n# predictions = {}\n\n# # Model 1: Gradient Boosting on causal features only\n# print(\"\\n1. Gradient Boosting (causal only)...\")\n# gb_causal = GradientBoostingRegressor(\n#     n_estimators=150,\n#     max_depth=4,\n#     learning_rate=0.05,\n#     subsample=0.8,\n#     min_samples_leaf=50,\n#     random_state=42\n# )\n# gb_causal.fit(X_train_causal[train_idx], y_train[train_idx])\n# models['gb_causal'] = gb_causal\n# predictions['gb_causal'] = gb_causal.predict(X_test_causal)\n\n# # Model 2: Random Forest with interactions\n# print(\"2. Random Forest (with interactions)...\")\n# rf_interact = RandomForestRegressor(\n#     n_estimators=100,\n#     max_depth=6,\n#     min_samples_leaf=100,\n#     random_state=42,\n#     n_jobs=-1\n# )\n# rf_interact.fit(X_train_with_interactions[train_idx], y_train[train_idx])\n# models['rf_interact'] = rf_interact\n# predictions['rf_interact'] = rf_interact.predict(X_test_with_interactions)\n\n# # Model 3: Ridge baseline on all PCA features\n# print(\"3. Ridge baseline (all features)...\")\n# ridge_all = Ridge(alpha=1.0, random_state=42)\n# ridge_all.fit(X_train_pca[train_idx], y_train[train_idx])\n# models['ridge_all'] = ridge_all\n# predictions['ridge_all'] = ridge_all.predict(X_test_pca)\n\n# # Model 4: Ridge on causal features\n# print(\"4. Ridge (causal only)...\")\n# ridge_causal = Ridge(alpha=1.0, random_state=42)\n# ridge_causal.fit(X_train_causal[train_idx], y_train[train_idx])\n# models['ridge_causal'] = ridge_causal\n# predictions['ridge_causal'] = ridge_causal.predict(X_test_causal)\n\n# # ==================== Validation ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"MODEL VALIDATION\")\n# print(\"=\"*60)\n\n# # Validation set\n# val_size = 5000\n# val_idx = np.random.choice(len(X_train_causal), val_size, replace=False)\n\n# val_scores = {}\n# print(\"\\nValidation scores:\")\n\n# # Validate each model\n# val_data = {\n#     'gb_causal': (X_train_causal[val_idx], gb_causal),\n#     'rf_interact': (X_train_with_interactions[val_idx], rf_interact),\n#     'ridge_all': (X_train_pca[val_idx], ridge_all),\n#     'ridge_causal': (X_train_causal[val_idx], ridge_causal)\n# }\n\n# for name, (X_val, model) in val_data.items():\n#     val_pred = model.predict(X_val)\n#     score = np.corrcoef(y_train[val_idx], val_pred)[0, 1]\n#     val_scores[name] = score\n#     print(f\"  {name}: {score:.4f}\")\n\n# # ==================== Ensemble ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"CREATING ENSEMBLE\")\n# print(\"=\"*60)\n\n# # Calculate weights based on validation scores\n# weights = np.array([val_scores[name] for name in predictions.keys()])\n# weights = np.maximum(weights, 0)  # No negative weights\n# weights = weights / weights.sum()\n\n# print(\"\\nEnsemble weights:\")\n# for i, name in enumerate(predictions.keys()):\n#     print(f\"  {name}: {weights[i]:.3f}\")\n\n# # Create final predictions\n# final_predictions = np.zeros(len(X_test_pca))\n# for i, name in enumerate(predictions.keys()):\n#     final_predictions += weights[i] * predictions[name]\n\n# # ==================== Save Submission ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"SAVING RESULTS\")\n# print(\"=\"*60)\n\n# sample_sub = pd.read_csv(SAMPLE_SUB_PATH)\n\n# if len(final_predictions) != len(sample_sub):\n#     print(f\"Adjusting predictions: {len(final_predictions)} -> {len(sample_sub)}\")\n#     final_predictions = final_predictions[:len(sample_sub)]\n\n# submission = pd.DataFrame({\n#     sample_sub.columns[0]: sample_sub.iloc[:, 0],\n#     'prediction': final_predictions\n# })\n\n# submission.to_csv(f'{OUTPUT_PATH}submission.csv', index=False)\n# print(f\"✓ Saved to {OUTPUT_PATH}submission.csv\")\n\n# # Save artifacts\n# joblib.dump(models, f'{OUTPUT_PATH}models.pkl')\n# joblib.dump({'scaler': scaler, 'pca': pca}, f'{OUTPUT_PATH}preprocessors.pkl')\n# joblib.dump(causal_features, f'{OUTPUT_PATH}causal_features.pkl')\n\n# print(\"\\n✓ Complete!\")\n# print(\"=\"*60)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # ==================== Simplified Granger Causality Feature Selection ====================\n# import pandas as pd\n# import numpy as np\n# import os\n# from sklearn.ensemble import GradientBoostingRegressor, RandomForestRegressor\n# from sklearn.linear_model import Ridge, Lasso\n# from sklearn.preprocessing import StandardScaler\n# from sklearn.decomposition import PCA\n# from sklearn.metrics import mean_squared_error\n# import joblib\n# import warnings\n# warnings.filterwarnings('ignore')\n\n# # ==================== Configuration ====================\n# DATA_PATH = '/kaggle/input/drw-remix-of-winning-strategies-part-1/'\n# OUTPUT_PATH = '/kaggle/working/granger_causal/'\n# SAMPLE_SUB_PATH = '/kaggle/input/drw-crypto-market-prediction/sample_submission.csv'\n\n# os.makedirs(OUTPUT_PATH, exist_ok=True)\n# print(f\"Output directory: {OUTPUT_PATH}\")\n\n# # ==================== Load Data ====================\n# print(\"=\"*60)\n# print(\"GRANGER CAUSALITY FEATURE SELECTION\")\n# print(\"=\"*60)\n\n# # Load data\n# train_df = pd.read_parquet(f'{DATA_PATH}train_final.parquet')\n# test_df = pd.read_parquet(f'{DATA_PATH}test_final.parquet')\n\n# print(f\"Train shape: {train_df.shape}\")\n# print(f\"Test shape: {test_df.shape}\")\n\n# # Select features\n# feature_cols = [col for col in train_df.columns if col not in ['label', 'timestamp', 'id']]\n\n# # Extract arrays\n# y_train = train_df['label'].values\n# X_train = train_df[feature_cols].values\n# X_test = test_df[feature_cols].values\n\n# # Clean data\n# X_train = np.nan_to_num(X_train, nan=0.0, posinf=0.0, neginf=0.0)\n# X_test = np.nan_to_num(X_test, nan=0.0, posinf=0.0, neginf=0.0)\n\n# print(f\"Features: {len(feature_cols)}\")\n\n# # ==================== Feature Processing ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"DIMENSIONALITY REDUCTION\")\n# print(\"=\"*60)\n\n# # Standardize\n# scaler = StandardScaler()\n# X_train_scaled = scaler.fit_transform(X_train)\n# X_test_scaled = scaler.transform(X_test)\n\n# # PCA\n# n_components = 30\n# pca = PCA(n_components=n_components, random_state=42)\n# X_train_pca = pca.fit_transform(X_train_scaled)\n# X_test_pca = pca.transform(X_test_scaled)\n\n# print(f\"PCA variance: {pca.explained_variance_ratio_.sum():.3f}\")\n\n# # ==================== Granger Causality Testing ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"GRANGER CAUSALITY ANALYSIS\")\n# print(\"=\"*60)\n\n# def granger_causality_test(X, y, max_lag=3):\n#     \"\"\"\n#     Test if X Granger-causes y\n#     Returns improvement in prediction when including X\n#     \"\"\"\n#     n = len(y)\n    \n#     # Create lagged versions of y\n#     y_lags = []\n#     for lag in range(1, max_lag + 1):\n#         y_lag = np.concatenate([np.zeros(lag), y[:-lag]])\n#         y_lags.append(y_lag)\n    \n#     # Create lagged versions of X\n#     X_lags = []\n#     for lag in range(1, max_lag + 1):\n#         X_lag = np.concatenate([np.zeros(lag), X[:-lag]])\n#         X_lags.append(X_lag)\n    \n#     # Start from max_lag to have valid data\n#     valid_idx = np.arange(max_lag, n)\n#     y_valid = y[valid_idx]\n    \n#     # Model 1: y predicted from its own lags only\n#     Y_lag_matrix = np.column_stack(y_lags)[valid_idx]\n#     model1 = Ridge(alpha=1.0)\n#     model1.fit(Y_lag_matrix, y_valid)\n#     pred1 = model1.predict(Y_lag_matrix)\n#     mse1 = mean_squared_error(y_valid, pred1)\n    \n#     # Model 2: y predicted from its own lags + X lags\n#     X_lag_matrix = np.column_stack(X_lags)[valid_idx]\n#     combined_matrix = np.column_stack([Y_lag_matrix, X_lag_matrix])\n#     model2 = Ridge(alpha=1.0)\n#     model2.fit(combined_matrix, y_valid)\n#     pred2 = model2.predict(combined_matrix)\n#     mse2 = mean_squared_error(y_valid, pred2)\n    \n#     # Granger causality score: improvement in prediction\n#     improvement = (mse1 - mse2) / (mse1 + 1e-10)\n    \n#     return improvement\n\n# # Test Granger causality for each PCA component\n# print(\"\\nTesting Granger causality for each component...\")\n# discovery_size = min(5000, len(X_train_pca))\n# subset_idx = np.arange(discovery_size)  # Use sequential data for time series\n\n# granger_scores = []\n# for i in range(n_components):\n#     score = granger_causality_test(\n#         X_train_pca[subset_idx, i], \n#         y_train[subset_idx],\n#         max_lag=3\n#     )\n#     granger_scores.append(score)\n#     if i % 10 == 0:\n#         print(f\"  Tested {i+1}/{n_components} components\")\n\n# # Rank features by Granger causality\n# granger_ranking = np.argsort(granger_scores)[::-1]\n# print(f\"\\nTop 5 Granger scores: {[granger_scores[i] for i in granger_ranking[:5]]}\")\n\n# # ==================== Directional Influence Analysis ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"DIRECTIONAL INFLUENCE\")\n# print(\"=\"*60)\n\n# def directional_influence(X, y, window=100):\n#     \"\"\"\n#     Measure directional influence using rolling correlation changes\n#     \"\"\"\n#     n = len(y)\n#     influences = []\n    \n#     for i in range(window, n - window):\n#         # Correlation in past window\n#         corr_past = np.corrcoef(X[i-window:i], y[i-window:i])[0, 1]\n        \n#         # Correlation in future window\n#         corr_future = np.corrcoef(X[i:i+window], y[i:i+window])[0, 1]\n        \n#         # Change in correlation\n#         influences.append(abs(corr_future - corr_past))\n    \n#     return np.mean(influences)\n\n# # Calculate directional influence\n# print(\"Calculating directional influence...\")\n# directional_scores = []\n# for i in range(n_components):\n#     score = directional_influence(\n#         X_train_pca[subset_idx, i],\n#         y_train[subset_idx],\n#         window=50\n#     )\n#     directional_scores.append(score)\n\n# directional_ranking = np.argsort(directional_scores)[::-1]\n# print(f\"Top 5 directional scores: {[directional_scores[i] for i in directional_ranking[:5]]}\")\n\n# # ==================== Select Causal Features ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"FEATURE SELECTION\")\n# print(\"=\"*60)\n\n# # Combine rankings\n# feature_importance = np.zeros(n_components)\n# for i in range(n_components):\n#     granger_rank = np.where(granger_ranking == i)[0][0]\n#     directional_rank = np.where(directional_ranking == i)[0][0]\n    \n#     # Combined score (lower rank is better)\n#     feature_importance[i] = (n_components - granger_rank) * 0.6 + (n_components - directional_rank) * 0.4\n\n# # Select top features\n# n_selected = 15\n# selected_features = np.argsort(feature_importance)[::-1][:n_selected]\n\n# print(f\"Selected {n_selected} features: {selected_features}\")\n\n# X_train_causal = X_train_pca[:, selected_features]\n# X_test_causal = X_test_pca[:, selected_features]\n\n# # ==================== Create Lagged Features ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"CREATING LAGGED FEATURES\")\n# print(\"=\"*60)\n\n# def create_lagged_features(X, lags=[1, 2, 3]):\n#     \"\"\"\n#     Create lagged versions of features\n#     \"\"\"\n#     X_lagged = X.copy()\n    \n#     for lag in lags:\n#         # Create lagged version\n#         X_lag = np.zeros_like(X)\n#         X_lag[lag:] = X[:-lag]\n        \n#         # Add to feature matrix\n#         X_lagged = np.column_stack([X_lagged, X_lag])\n    \n#     return X_lagged\n\n# # Create lagged features for top causal variables\n# top_causal = selected_features[:5]\n# X_train_lagged = X_train_causal.copy()\n# X_test_lagged = X_test_causal.copy()\n\n# for feature_idx in range(5):  # Lag top 5 features\n#     # Add single lag\n#     train_lag = np.zeros(len(X_train_causal))\n#     train_lag[1:] = X_train_causal[:-1, feature_idx]\n    \n#     test_lag = np.zeros(len(X_test_causal))\n#     test_lag[1:] = X_test_causal[:-1, feature_idx]\n    \n#     X_train_lagged = np.column_stack([X_train_lagged, train_lag])\n#     X_test_lagged = np.column_stack([X_test_lagged, test_lag])\n\n# print(f\"Features after adding lags: {X_train_lagged.shape[1]}\")\n\n# # ==================== Train Models ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"TRAINING MODELS\")\n# print(\"=\"*60)\n\n# train_size = min(30000, len(X_train_causal))\n# train_idx = np.random.choice(len(X_train_causal), train_size, replace=False)\n\n# models = {}\n# predictions = {}\n\n# # Model 1: GradientBoosting on causal features\n# print(\"\\n1. GradientBoosting (causal)...\")\n# gb_causal = GradientBoostingRegressor(\n#     n_estimators=150,\n#     max_depth=4,\n#     learning_rate=0.05,\n#     subsample=0.8,\n#     min_samples_leaf=50,\n#     random_state=42\n# )\n# gb_causal.fit(X_train_causal[train_idx], y_train[train_idx])\n# models['gb_causal'] = gb_causal\n# predictions['gb_causal'] = gb_causal.predict(X_test_causal)\n\n# # Model 2: RandomForest with lagged features\n# print(\"2. RandomForest (with lags)...\")\n# rf_lagged = RandomForestRegressor(\n#     n_estimators=100,\n#     max_depth=6,\n#     min_samples_leaf=100,\n#     random_state=42,\n#     n_jobs=-1\n# )\n# rf_lagged.fit(X_train_lagged[train_idx], y_train[train_idx])\n# models['rf_lagged'] = rf_lagged\n# predictions['rf_lagged'] = rf_lagged.predict(X_test_lagged)\n\n# # Model 3: Lasso for sparsity\n# print(\"3. Lasso (feature selection)...\")\n# lasso = Lasso(alpha=0.001, random_state=42)\n# lasso.fit(X_train_causal[train_idx], y_train[train_idx])\n# models['lasso'] = lasso\n# predictions['lasso'] = lasso.predict(X_test_causal)\n\n# # Model 4: Ridge baseline\n# print(\"4. Ridge (baseline)...\")\n# ridge = Ridge(alpha=1.0, random_state=42)\n# ridge.fit(X_train_causal[train_idx], y_train[train_idx])\n# models['ridge'] = ridge\n# predictions['ridge'] = ridge.predict(X_test_causal)\n\n# # ==================== Validation ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"VALIDATION\")\n# print(\"=\"*60)\n\n# val_size = 5000\n# val_idx = np.random.choice(len(X_train_causal), val_size, replace=False)\n\n# val_scores = {}\n# print(\"\\nValidation scores:\")\n\n# # Validate each model\n# val_pred_gb = models['gb_causal'].predict(X_train_causal[val_idx])\n# val_scores['gb_causal'] = np.corrcoef(y_train[val_idx], val_pred_gb)[0, 1]\n\n# val_pred_rf = models['rf_lagged'].predict(X_train_lagged[val_idx])\n# val_scores['rf_lagged'] = np.corrcoef(y_train[val_idx], val_pred_rf)[0, 1]\n\n# val_pred_lasso = models['lasso'].predict(X_train_causal[val_idx])\n# val_scores['lasso'] = np.corrcoef(y_train[val_idx], val_pred_lasso)[0, 1]\n\n# val_pred_ridge = models['ridge'].predict(X_train_causal[val_idx])\n# val_scores['ridge'] = np.corrcoef(y_train[val_idx], val_pred_ridge)[0, 1]\n\n# for name, score in val_scores.items():\n#     print(f\"  {name}: {score:.4f}\")\n\n# # ==================== Ensemble ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"ENSEMBLE\")\n# print(\"=\"*60)\n\n# # Calculate weights\n# weights = np.array([val_scores[name] for name in predictions.keys()])\n# weights = np.maximum(weights, 0)\n# weights = weights / weights.sum()\n\n# print(\"\\nEnsemble weights:\")\n# for i, name in enumerate(predictions.keys()):\n#     print(f\"  {name}: {weights[i]:.3f}\")\n\n# # Final predictions\n# final_predictions = sum(weights[i] * predictions[name] \n#                        for i, name in enumerate(predictions.keys()))\n\n# # ==================== Save Submission ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"SAVING RESULTS\")\n# print(\"=\"*60)\n\n# sample_sub = pd.read_csv(SAMPLE_SUB_PATH)\n\n# if len(final_predictions) != len(sample_sub):\n#     print(f\"Adjusting: {len(final_predictions)} -> {len(sample_sub)}\")\n#     final_predictions = final_predictions[:len(sample_sub)]\n\n# submission = pd.DataFrame({\n#     sample_sub.columns[0]: sample_sub.iloc[:, 0],\n#     'prediction': final_predictions\n# })\n\n# submission.to_csv(f'{OUTPUT_PATH}submission.csv', index=False)\n# print(f\"✓ Saved to {OUTPUT_PATH}submission.csv\")\n\n# # Save artifacts\n# joblib.dump(models, f'{OUTPUT_PATH}models.pkl')\n# joblib.dump({'scaler': scaler, 'pca': pca}, f'{OUTPUT_PATH}preprocessors.pkl')\n# joblib.dump(selected_features, f'{OUTPUT_PATH}causal_features.pkl')\n\n# print(\"\\n✓ Complete!\")\n# print(\"=\"*60)","metadata":{"trusted":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # ==================== Instantaneous Causal Discovery (No Temporal Dependencies) ====================\n# import pandas as pd\n# import numpy as np\n# import os\n# from sklearn.ensemble import GradientBoostingRegressor, RandomForestRegressor\n# from sklearn.linear_model import Ridge, LinearRegression\n# from sklearn.preprocessing import StandardScaler\n# from sklearn.decomposition import PCA\n# from sklearn.feature_selection import mutual_info_regression\n# import joblib\n# import warnings\n# warnings.filterwarnings('ignore')\n\n# # ==================== Configuration ====================\n# DATA_PATH = '/kaggle/input/drw-remix-of-winning-strategies-part-1/'\n# OUTPUT_PATH = '/kaggle/working/causal_instant/'\n# SAMPLE_SUB_PATH = '/kaggle/input/drw-crypto-market-prediction/sample_submission.csv'\n\n# os.makedirs(OUTPUT_PATH, exist_ok=True)\n# print(f\"Output directory: {OUTPUT_PATH}\")\n\n# # ==================== Load Data ====================\n# print(\"=\"*60)\n# print(\"INSTANTANEOUS CAUSAL DISCOVERY\")\n# print(\"=\"*60)\n\n# # Load data\n# train_df = pd.read_parquet(f'{DATA_PATH}train_final.parquet')\n# test_df = pd.read_parquet(f'{DATA_PATH}test_final.parquet')\n\n# print(f\"Train shape: {train_df.shape}\")\n# print(f\"Test shape: {test_df.shape}\")\n\n# # Select features\n# feature_cols = [col for col in train_df.columns if col not in ['label', 'timestamp', 'id']]\n\n# # Extract arrays\n# y_train = train_df['label'].values\n# X_train = train_df[feature_cols].values\n# X_test = test_df[feature_cols].values\n\n# # Clean data\n# X_train = np.nan_to_num(X_train, nan=0.0, posinf=0.0, neginf=0.0)\n# X_test = np.nan_to_num(X_test, nan=0.0, posinf=0.0, neginf=0.0)\n\n# print(f\"Features: {len(feature_cols)}\")\n\n# # ==================== Feature Processing ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"DIMENSIONALITY REDUCTION\")\n# print(\"=\"*60)\n\n# # Standardize\n# scaler = StandardScaler()\n# X_train_scaled = scaler.fit_transform(X_train)\n# X_test_scaled = scaler.transform(X_test)\n\n# # PCA\n# n_components = 40\n# pca = PCA(n_components=n_components, random_state=42)\n# X_train_pca = pca.fit_transform(X_train_scaled)\n# X_test_pca = pca.transform(X_test_scaled)\n\n# print(f\"PCA variance: {pca.explained_variance_ratio_.sum():.3f}\")\n\n# # ==================== Causal Strength Estimation ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"CAUSAL STRENGTH ANALYSIS\")\n# print(\"=\"*60)\n\n# # Use subset for analysis\n# analysis_size = min(10000, len(X_train_pca))\n# subset_idx = np.random.choice(len(X_train_pca), analysis_size, replace=False)\n# X_analysis = X_train_pca[subset_idx]\n# y_analysis = y_train[subset_idx]\n\n# def estimate_causal_strength(X, y):\n#     \"\"\"\n#     Estimate causal strength using residualization method\n#     \"\"\"\n#     # Step 1: Direct effect (correlation)\n#     direct_corr = np.corrcoef(X, y)[0, 1]\n    \n#     # Step 2: Residual effect (after controlling for linear relationship)\n#     lr = LinearRegression()\n#     lr.fit(X.reshape(-1, 1), y)\n#     y_residual = y - lr.predict(X.reshape(-1, 1))\n    \n#     # Non-linear component\n#     X_squared = X ** 2\n#     lr_nonlinear = LinearRegression()\n#     lr_nonlinear.fit(X_squared.reshape(-1, 1), y_residual)\n#     nonlinear_effect = lr_nonlinear.score(X_squared.reshape(-1, 1), y_residual)\n    \n#     # Combined causal strength\n#     causal_strength = abs(direct_corr) + 0.5 * nonlinear_effect\n    \n#     return causal_strength, direct_corr, nonlinear_effect\n\n# # Calculate causal strength for each component\n# print(\"\\nCalculating causal strengths...\")\n# causal_scores = []\n# linear_effects = []\n# nonlinear_effects = []\n\n# for i in range(n_components):\n#     strength, linear, nonlinear = estimate_causal_strength(X_analysis[:, i], y_analysis)\n#     causal_scores.append(strength)\n#     linear_effects.append(linear)\n#     nonlinear_effects.append(nonlinear)\n\n# # Rank by causal strength\n# causal_ranking = np.argsort(causal_scores)[::-1]\n# print(f\"Top 5 causal strengths: {[causal_scores[i] for i in causal_ranking[:5]]}\")\n\n# # ==================== Confounding Analysis ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"CONFOUNDING ANALYSIS\")\n# print(\"=\"*60)\n\n# def confounding_score(X1, X2, y):\n#     \"\"\"\n#     Measure potential confounding between X1 and X2 on y\n#     \"\"\"\n#     # Correlation between features (potential confounding)\n#     feature_corr = abs(np.corrcoef(X1, X2)[0, 1])\n    \n#     # Both features' effect on y\n#     y_corr1 = abs(np.corrcoef(X1, y)[0, 1])\n#     y_corr2 = abs(np.corrcoef(X2, y)[0, 1])\n    \n#     # High confounding if features are correlated and both affect y\n#     conf_score = feature_corr * np.sqrt(y_corr1 * y_corr2)\n    \n#     return conf_score\n\n# # Find low-confounding feature pairs\n# print(\"Finding low-confounding features...\")\n# low_confound_features = []\n# for i in causal_ranking[:20]:  # Check top 20 causal features\n#     max_confound = 0\n#     for j in causal_ranking[:20]:\n#         if i != j:\n#             conf = confounding_score(X_analysis[:, i], X_analysis[:, j], y_analysis)\n#             max_confound = max(max_confound, conf)\n    \n#     # Keep features with low maximum confounding\n#     if max_confound < 0.3:\n#         low_confound_features.append(i)\n\n# print(f\"Found {len(low_confound_features)} low-confounding features\")\n\n# # ==================== Independent Causal Effects ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"INDEPENDENT CAUSAL EFFECTS\")\n# print(\"=\"*60)\n\n# # Mutual information for non-linear independence\n# print(\"Computing mutual information...\")\n# mi_scores = mutual_info_regression(X_analysis, y_analysis, random_state=42)\n\n# # Combine causal strength with independence\n# independent_causal_scores = []\n# for i in range(n_components):\n#     # High MI but low linear correlation indicates non-linear causal effect\n#     independence = mi_scores[i] / (abs(linear_effects[i]) + 0.1)\n#     combined_score = causal_scores[i] * (1 + 0.3 * independence)\n#     independent_causal_scores.append(combined_score)\n\n# # Final feature selection\n# final_ranking = np.argsort(independent_causal_scores)[::-1]\n\n# # Select diverse features\n# selected_features = []\n# for feat in final_ranking:\n#     if feat in low_confound_features or len(selected_features) < 10:\n#         selected_features.append(feat)\n#     if len(selected_features) >= 15:\n#         break\n\n# print(f\"Selected {len(selected_features)} features: {selected_features}\")\n\n# X_train_causal = X_train_pca[:, selected_features]\n# X_test_causal = X_test_pca[:, selected_features]\n\n# # ==================== Non-linear Transformations ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"NON-LINEAR FEATURE ENGINEERING\")\n# print(\"=\"*60)\n\n# # Add polynomial features for top causal variables\n# X_train_enhanced = X_train_causal.copy()\n# X_test_enhanced = X_test_causal.copy()\n\n# # Add squared terms for top 5 features\n# for i in range(min(5, len(selected_features))):\n#     X_train_enhanced = np.column_stack([X_train_enhanced, X_train_causal[:, i] ** 2])\n#     X_test_enhanced = np.column_stack([X_test_enhanced, X_test_causal[:, i] ** 2])\n\n# # Add interaction terms for top 3 pairs with high independent effects\n# for i in range(min(3, len(selected_features)-1)):\n#     interaction_train = X_train_causal[:, i] * X_train_causal[:, i+1]\n#     interaction_test = X_test_causal[:, i] * X_test_causal[:, i+1]\n#     X_train_enhanced = np.column_stack([X_train_enhanced, interaction_train])\n#     X_test_enhanced = np.column_stack([X_test_enhanced, interaction_test])\n\n# print(f\"Enhanced features: {X_train_enhanced.shape[1]}\")\n\n# # ==================== Train Models ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"TRAINING MODELS\")\n# print(\"=\"*60)\n\n# train_size = min(30000, len(X_train_causal))\n# train_idx = np.random.choice(len(X_train_causal), train_size, replace=False)\n\n# models = {}\n# predictions = {}\n\n# # Model 1: GradientBoosting on causal features\n# print(\"\\n1. GradientBoosting (causal)...\")\n# gb_causal = GradientBoostingRegressor(\n#     n_estimators=150,\n#     max_depth=4,\n#     learning_rate=0.05,\n#     subsample=0.8,\n#     min_samples_leaf=50,\n#     random_state=42\n# )\n# gb_causal.fit(X_train_causal[train_idx], y_train[train_idx])\n# models['gb_causal'] = gb_causal\n# predictions['gb_causal'] = gb_causal.predict(X_test_causal)\n\n# # Model 2: RandomForest with enhanced features\n# print(\"2. RandomForest (enhanced)...\")\n# rf_enhanced = RandomForestRegressor(\n#     n_estimators=100,\n#     max_depth=6,\n#     min_samples_leaf=100,\n#     random_state=42,\n#     n_jobs=-1\n# )\n# rf_enhanced.fit(X_train_enhanced[train_idx], y_train[train_idx])\n# models['rf_enhanced'] = rf_enhanced\n# predictions['rf_enhanced'] = rf_enhanced.predict(X_test_enhanced)\n\n# # Model 3: Ridge on all PCA components\n# print(\"3. Ridge (all features)...\")\n# ridge_all = Ridge(alpha=1.0, random_state=42)\n# ridge_all.fit(X_train_pca[train_idx], y_train[train_idx])\n# models['ridge_all'] = ridge_all\n# predictions['ridge_all'] = ridge_all.predict(X_test_pca)\n\n# # Model 4: Ridge on causal features\n# print(\"4. Ridge (causal)...\")\n# ridge_causal = Ridge(alpha=1.0, random_state=42)\n# ridge_causal.fit(X_train_causal[train_idx], y_train[train_idx])\n# models['ridge_causal'] = ridge_causal\n# predictions['ridge_causal'] = ridge_causal.predict(X_test_causal)\n\n# # ==================== Validation ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"VALIDATION\")\n# print(\"=\"*60)\n\n# val_size = 5000\n# val_idx = np.random.choice(len(X_train_causal), val_size, replace=False)\n\n# val_scores = {}\n# print(\"\\nValidation scores:\")\n\n# # Validate each model\n# val_pred = models['gb_causal'].predict(X_train_causal[val_idx])\n# val_scores['gb_causal'] = np.corrcoef(y_train[val_idx], val_pred)[0, 1]\n\n# val_pred = models['rf_enhanced'].predict(X_train_enhanced[val_idx])\n# val_scores['rf_enhanced'] = np.corrcoef(y_train[val_idx], val_pred)[0, 1]\n\n# val_pred = models['ridge_all'].predict(X_train_pca[val_idx])\n# val_scores['ridge_all'] = np.corrcoef(y_train[val_idx], val_pred)[0, 1]\n\n# val_pred = models['ridge_causal'].predict(X_train_causal[val_idx])\n# val_scores['ridge_causal'] = np.corrcoef(y_train[val_idx], val_pred)[0, 1]\n\n# for name, score in val_scores.items():\n#     print(f\"  {name}: {score:.4f}\")\n\n# # ==================== Ensemble ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"ENSEMBLE\")\n# print(\"=\"*60)\n\n# # Calculate weights\n# weights = np.array([val_scores[name] for name in predictions.keys()])\n# weights = np.maximum(weights, 0)\n# weights = weights / weights.sum()\n\n# print(\"\\nEnsemble weights:\")\n# for i, name in enumerate(predictions.keys()):\n#     print(f\"  {name}: {weights[i]:.3f}\")\n\n# # Final predictions\n# final_predictions = sum(weights[i] * predictions[name] \n#                        for i, name in enumerate(predictions.keys()))\n\n# # ==================== Save Submission ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"SAVING RESULTS\")\n# print(\"=\"*60)\n\n# sample_sub = pd.read_csv(SAMPLE_SUB_PATH)\n\n# if len(final_predictions) != len(sample_sub):\n#     print(f\"Adjusting: {len(final_predictions)} -> {len(sample_sub)}\")\n#     final_predictions = final_predictions[:len(sample_sub)]\n\n# submission = pd.DataFrame({\n#     sample_sub.columns[0]: sample_sub.iloc[:, 0],\n#     'prediction': final_predictions\n# })\n\n# submission.to_csv(f'{OUTPUT_PATH}submission.csv', index=False)\n# print(f\"✓ Saved to {OUTPUT_PATH}submission.csv\")\n\n# # Save artifacts\n# joblib.dump(models, f'{OUTPUT_PATH}models.pkl')\n# joblib.dump({'scaler': scaler, 'pca': pca}, f'{OUTPUT_PATH}preprocessors.pkl')\n# joblib.dump(selected_features, f'{OUTPUT_PATH}causal_features.pkl')\n\n# print(\"\\n✓ Complete!\")\n# print(\"=\"*60)","metadata":{"trusted":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # ==================== Causal-Based Feature Grouping and Medoid Selection ====================\n# import pandas as pd\n# import numpy as np\n# import os\n# from sklearn.ensemble import GradientBoostingRegressor, RandomForestRegressor\n# from sklearn.linear_model import Ridge, LinearRegression\n# from sklearn.preprocessing import StandardScaler\n# from sklearn.cluster import AgglomerativeClustering\n# from sklearn.metrics import pairwise_distances\n# from scipy.cluster.hierarchy import dendrogram, linkage, fcluster\n# from scipy.spatial.distance import squareform\n# import joblib\n# import warnings\n# warnings.filterwarnings('ignore')\n\n# # ==================== Configuration ====================\n# DATA_PATH = '/kaggle/input/drw-remix-of-winning-strategies-part-1/'\n# OUTPUT_PATH = '/kaggle/working/causal_grouping/'\n# SAMPLE_SUB_PATH = '/kaggle/input/drw-crypto-market-prediction/sample_submission.csv'\n\n# os.makedirs(OUTPUT_PATH, exist_ok=True)\n# print(f\"Output directory: {OUTPUT_PATH}\")\n\n# # ==================== Load Data ====================\n# print(\"=\"*60)\n# print(\"CAUSAL FEATURE GROUPING & MEDOID SELECTION\")\n# print(\"=\"*60)\n\n# # Load data\n# train_df = pd.read_parquet(f'{DATA_PATH}train_final.parquet')\n# test_df = pd.read_parquet(f'{DATA_PATH}test_final.parquet')\n\n# print(f\"Train shape: {train_df.shape}\")\n# print(f\"Test shape: {test_df.shape}\")\n\n# # Select features\n# feature_cols = [col for col in train_df.columns if col not in ['label', 'timestamp', 'id']]\n\n# # Extract arrays\n# y_train = train_df['label'].values\n# X_train = train_df[feature_cols].values\n# X_test = test_df[feature_cols].values\n\n# # Clean data\n# X_train = np.nan_to_num(X_train, nan=0.0, posinf=0.0, neginf=0.0)\n# X_test = np.nan_to_num(X_test, nan=0.0, posinf=0.0, neginf=0.0)\n\n# print(f\"Original features: {len(feature_cols)}\")\n\n# # ==================== Initial Feature Reduction ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"INITIAL FEATURE SCREENING\")\n# print(\"=\"*60)\n\n# # Remove zero-variance features\n# variances = np.var(X_train, axis=0)\n# non_zero_var = variances > 1e-10\n# X_train = X_train[:, non_zero_var]\n# X_test = X_test[:, non_zero_var]\n# feature_cols = [col for col, keep in zip(feature_cols, non_zero_var) if keep]\n\n# print(f\"Features after variance filter: {len(feature_cols)}\")\n\n# # Standardize\n# scaler = StandardScaler()\n# X_train_scaled = scaler.fit_transform(X_train)\n# X_test_scaled = scaler.transform(X_test)\n\n# # Quick correlation filter to reduce to manageable size\n# if len(feature_cols) > 200:\n#     print(\"Applying correlation pre-filter...\")\n#     sample_size = min(5000, len(X_train_scaled))\n#     sample_idx = np.random.choice(len(X_train_scaled), sample_size, replace=False)\n    \n#     # Calculate correlations with target\n#     target_corrs = []\n#     for i in range(X_train_scaled.shape[1]):\n#         corr = np.corrcoef(X_train_scaled[sample_idx, i], y_train[sample_idx])[0, 1]\n#         target_corrs.append(abs(corr))\n    \n#     # Keep top 200 correlated features\n#     top_indices = np.argsort(target_corrs)[-200:]\n#     X_train_scaled = X_train_scaled[:, top_indices]\n#     X_test_scaled = X_test_scaled[:, top_indices]\n#     feature_cols = [feature_cols[i] for i in top_indices]\n    \n#     print(f\"Reduced to {len(feature_cols)} features\")\n\n# # ==================== Causal Relationship Discovery ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"DISCOVERING CAUSAL RELATIONSHIPS\")\n# print(\"=\"*60)\n\n# # Use subset for causal analysis\n# analysis_size = min(5000, len(X_train_scaled))\n# analysis_idx = np.random.choice(len(X_train_scaled), analysis_size, replace=False)\n# X_analysis = X_train_scaled[analysis_idx]\n# y_analysis = y_train[analysis_idx]\n\n# def estimate_causal_relationship(X1, X2, y=None):\n#     \"\"\"\n#     Estimate causal relationship strength between two features\n#     Returns: causal_distance (lower = stronger relationship)\n#     \"\"\"\n#     # 1. Direct correlation (linear dependency)\n#     direct_corr = abs(np.corrcoef(X1, X2)[0, 1])\n    \n#     # 2. Residual dependency (X1 -> X2)\n#     lr1 = LinearRegression()\n#     lr1.fit(X1.reshape(-1, 1), X2)\n#     residual_12 = X2 - lr1.predict(X1.reshape(-1, 1))\n#     var_explained_12 = 1 - np.var(residual_12) / (np.var(X2) + 1e-10)\n    \n#     # 3. Residual dependency (X2 -> X1)\n#     lr2 = LinearRegression()\n#     lr2.fit(X2.reshape(-1, 1), X1)\n#     residual_21 = X1 - lr2.predict(X2.reshape(-1, 1))\n#     var_explained_21 = 1 - np.var(residual_21) / (np.var(X1) + 1e-10)\n    \n#     # 4. Co-influence on target (if provided)\n#     co_influence = 0\n#     if y is not None:\n#         # How similarly do they affect the target?\n#         lr_y1 = LinearRegression()\n#         lr_y1.fit(X1.reshape(-1, 1), y)\n#         effect1 = lr_y1.coef_[0]\n        \n#         lr_y2 = LinearRegression()\n#         lr_y2.fit(X2.reshape(-1, 1), y)\n#         effect2 = lr_y2.coef_[0]\n        \n#         # Similar effect direction and magnitude\n#         co_influence = 1 - abs(effect1 - effect2) / (abs(effect1) + abs(effect2) + 1e-10)\n#         if np.sign(effect1) != np.sign(effect2):\n#             co_influence *= 0.5\n    \n#     # Combine measures (strong relationship = low distance)\n#     causal_strength = (direct_corr + max(var_explained_12, var_explained_21) + 0.3 * co_influence) / 2.3\n#     causal_distance = 1 - causal_strength\n    \n#     return causal_distance\n\n# # Build causal distance matrix\n# print(\"\\nBuilding causal distance matrix...\")\n# n_features = X_analysis.shape[1]\n# causal_distances = np.zeros((n_features, n_features))\n\n# for i in range(n_features):\n#     if i % 50 == 0:\n#         print(f\"  Processing feature {i}/{n_features}\")\n#     for j in range(i+1, n_features):\n#         distance = estimate_causal_relationship(\n#             X_analysis[:, i], \n#             X_analysis[:, j],\n#             y_analysis\n#         )\n#         causal_distances[i, j] = distance\n#         causal_distances[j, i] = distance\n\n# print(\"Causal distance matrix complete\")\n\n# # ==================== Hierarchical Clustering ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"HIERARCHICAL CLUSTERING\")\n# print(\"=\"*60)\n\n# # Perform hierarchical clustering\n# print(\"Performing hierarchical clustering...\")\n# condensed_distances = squareform(causal_distances)\n# linkage_matrix = linkage(condensed_distances, method='ward')\n\n# # Determine optimal number of clusters\n# n_clusters = min(50, n_features // 4)  # Aim for 50 clusters or 4 features per cluster\n# clusters = fcluster(linkage_matrix, n_clusters, criterion='maxclust')\n\n# print(f\"Created {n_clusters} clusters\")\n# print(f\"Cluster sizes: {np.bincount(clusters)[1:]}\")\n\n# # ==================== Medoid Selection ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"MEDOID SELECTION\")\n# print(\"=\"*60)\n\n# def select_medoid(cluster_features, target):\n#     \"\"\"\n#     Select the medoid (most representative feature) from a cluster\n#     \"\"\"\n#     n_features = cluster_features.shape[1]\n    \n#     if n_features == 1:\n#         return 0\n    \n#     # Calculate within-cluster distances\n#     within_distances = np.zeros(n_features)\n#     for i in range(n_features):\n#         for j in range(n_features):\n#             if i != j:\n#                 within_distances[i] += abs(np.corrcoef(\n#                     cluster_features[:, i], \n#                     cluster_features[:, j]\n#                 )[0, 1])\n    \n#     # Also consider relationship with target\n#     target_corrs = []\n#     for i in range(n_features):\n#         corr = abs(np.corrcoef(cluster_features[:, i], target)[0, 1])\n#         target_corrs.append(corr)\n    \n#     # Combined score: high centrality + high target correlation\n#     scores = within_distances / (n_features - 1) + 0.5 * np.array(target_corrs)\n    \n#     return np.argmax(scores)\n\n# # Select medoids for each cluster\n# selected_features = []\n# feature_to_cluster = {}\n\n# for cluster_id in range(1, n_clusters + 1):\n#     cluster_mask = clusters == cluster_id\n#     cluster_indices = np.where(cluster_mask)[0]\n    \n#     if len(cluster_indices) > 0:\n#         # Get cluster features\n#         cluster_data = X_analysis[:, cluster_indices]\n        \n#         # Select medoid\n#         medoid_idx = select_medoid(cluster_data, y_analysis)\n#         selected_feature = cluster_indices[medoid_idx]\n#         selected_features.append(selected_feature)\n        \n#         # Track clustering\n#         for idx in cluster_indices:\n#             feature_to_cluster[idx] = cluster_id\n\n# print(f\"Selected {len(selected_features)} medoid features\")\n\n# # ==================== Causal Chain Features ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"IDENTIFYING CAUSAL CHAINS\")\n# print(\"=\"*60)\n\n# # Find features that might be in causal chains\n# causal_chains = []\n# for i in selected_features[:20]:  # Check top 20 medoids\n#     for j in selected_features[:20]:\n#         if i != j:\n#             # Check if i->j is much stronger than j->i\n#             lr_ij = LinearRegression()\n#             lr_ij.fit(X_analysis[:, i].reshape(-1, 1), X_analysis[:, j])\n#             var_ij = 1 - np.var(X_analysis[:, j] - lr_ij.predict(X_analysis[:, i].reshape(-1, 1))) / np.var(X_analysis[:, j])\n            \n#             lr_ji = LinearRegression()\n#             lr_ji.fit(X_analysis[:, j].reshape(-1, 1), X_analysis[:, i])\n#             var_ji = 1 - np.var(X_analysis[:, i] - lr_ji.predict(X_analysis[:, j].reshape(-1, 1))) / np.var(X_analysis[:, i])\n            \n#             if var_ij > 0.3 and var_ij > 2 * var_ji:\n#                 causal_chains.append((i, j, var_ij))\n\n# causal_chains.sort(key=lambda x: x[2], reverse=True)\n# print(f\"Found {len(causal_chains)} potential causal chains\")\n\n# # ==================== Feature Engineering ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"FEATURE ENGINEERING\")\n# print(\"=\"*60)\n\n# # Extract selected features\n# X_train_selected = X_train_scaled[:, selected_features]\n# X_test_selected = X_test_scaled[:, selected_features]\n\n# print(f\"Selected features shape: {X_train_selected.shape}\")\n\n# # Add causal chain interactions\n# X_train_final = X_train_selected.copy()\n# X_test_final = X_test_selected.copy()\n\n# # Add top causal chain products\n# for i, (feat1, feat2, strength) in enumerate(causal_chains[:5]):\n#     if feat1 in selected_features and feat2 in selected_features:\n#         idx1 = selected_features.index(feat1)\n#         idx2 = selected_features.index(feat2)\n        \n#         # Weighted interaction\n#         interaction_train = X_train_selected[:, idx1] * X_train_selected[:, idx2] * strength\n#         interaction_test = X_test_selected[:, idx1] * X_test_selected[:, idx2] * strength\n        \n#         X_train_final = np.column_stack([X_train_final, interaction_train])\n#         X_test_final = np.column_stack([X_test_final, interaction_test])\n\n# print(f\"Final features with interactions: {X_train_final.shape[1]}\")\n\n# # ==================== Train Models ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"TRAINING MODELS\")\n# print(\"=\"*60)\n\n# train_size = min(30000, len(X_train_final))\n# train_idx = np.random.choice(len(X_train_final), train_size, replace=False)\n\n# models = {}\n# predictions = {}\n\n# # Model 1: GradientBoosting\n# print(\"\\n1. GradientBoosting...\")\n# gb = GradientBoostingRegressor(\n#     n_estimators=150,\n#     max_depth=4,\n#     learning_rate=0.05,\n#     subsample=0.8,\n#     min_samples_leaf=50,\n#     random_state=42\n# )\n# gb.fit(X_train_final[train_idx], y_train[train_idx])\n# models['gb'] = gb\n# predictions['gb'] = gb.predict(X_test_final)\n\n# # Model 2: RandomForest\n# print(\"2. RandomForest...\")\n# rf = RandomForestRegressor(\n#     n_estimators=100,\n#     max_depth=6,\n#     min_samples_leaf=100,\n#     random_state=42,\n#     n_jobs=-1\n# )\n# rf.fit(X_train_final[train_idx], y_train[train_idx])\n# models['rf'] = rf\n# predictions['rf'] = rf.predict(X_test_final)\n\n# # Model 3: Ridge\n# print(\"3. Ridge...\")\n# ridge = Ridge(alpha=1.0, random_state=42)\n# ridge.fit(X_train_final[train_idx], y_train[train_idx])\n# models['ridge'] = ridge\n# predictions['ridge'] = ridge.predict(X_test_final)\n\n# # ==================== Validation & Ensemble ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"VALIDATION\")\n# print(\"=\"*60)\n\n# val_size = 5000\n# val_idx = np.random.choice(len(X_train_final), val_size, replace=False)\n\n# val_scores = {}\n# for name, model in models.items():\n#     val_pred = model.predict(X_train_final[val_idx])\n#     score = np.corrcoef(y_train[val_idx], val_pred)[0, 1]\n#     val_scores[name] = score\n#     print(f\"{name}: {score:.4f}\")\n\n# # Ensemble\n# weights = np.array([val_scores[name] for name in predictions.keys()])\n# weights = np.maximum(weights, 0)\n# weights = weights / weights.sum()\n\n# print(f\"\\nWeights: {dict(zip(predictions.keys(), weights))}\")\n\n# final_predictions = sum(weights[i] * predictions[name] \n#                        for i, name in enumerate(predictions.keys()))\n\n# # ==================== Save ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"SAVING\")\n# print(\"=\"*60)\n\n# sample_sub = pd.read_csv(SAMPLE_SUB_PATH)\n# if len(final_predictions) != len(sample_sub):\n#     final_predictions = final_predictions[:len(sample_sub)]\n\n# submission = pd.DataFrame({\n#     sample_sub.columns[0]: sample_sub.iloc[:, 0],\n#     'prediction': final_predictions\n# })\n\n# submission.to_csv(f'{OUTPUT_PATH}submission.csv', index=False)\n# print(f\"✓ Saved to {OUTPUT_PATH}submission.csv\")\n\n# print(\"\\n✓ Complete!\")\n# print(\"=\"*60)","metadata":{"trusted":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # ==================== Causal DAG-Based Collinearity Management ====================\n# import pandas as pd\n# import numpy as np\n# import os\n# from sklearn.ensemble import GradientBoostingRegressor, RandomForestRegressor\n# from sklearn.linear_model import Ridge, LinearRegression, LassoCV\n# from sklearn.preprocessing import StandardScaler\n# from sklearn.decomposition import FastICA\n# from scipy import stats\n# from scipy.linalg import qr\n# import joblib\n# import warnings\n# warnings.filterwarnings('ignore')\n\n# # ==================== Configuration ====================\n# DATA_PATH = '/kaggle/input/drw-remix-of-winning-strategies-part-1/'\n# OUTPUT_PATH = '/kaggle/working/causal_dag/'\n# SAMPLE_SUB_PATH = '/kaggle/input/drw-crypto-market-prediction/sample_submission.csv'\n\n# os.makedirs(OUTPUT_PATH, exist_ok=True)\n# print(f\"Output directory: {OUTPUT_PATH}\")\n\n# # ==================== Causal Structure Discovery Functions ====================\n\n# def estimate_causal_order(X, method='variance'):\n#     \"\"\"\n#     Estimate causal ordering of variables using variance sorting or ICA\n#     Variables with lower noise/higher exogeneity come first\n#     \"\"\"\n#     n_features = X.shape[1]\n    \n#     if method == 'variance':\n#         # Variance-based: more exogenous variables have higher variance\n#         variances = np.var(X, axis=0)\n#         causal_order = np.argsort(variances)[::-1]\n    \n#     elif method == 'ica':\n#         # ICA-based: find independent components\n#         ica = FastICA(n_components=min(n_features, 20), random_state=42)\n#         try:\n#             S = ica.fit_transform(X[:min(5000, len(X))])\n#             # Order by non-Gaussianity (kurtosis)\n#             kurtosis = stats.kurtosis(S, axis=0)\n#             causal_order = np.argsort(np.abs(kurtosis))[::-1]\n#             # Extend if needed\n#             if len(causal_order) < n_features:\n#                 remaining = [i for i in range(n_features) if i not in causal_order]\n#                 causal_order = np.concatenate([causal_order, remaining])\n#         except:\n#             causal_order = np.arange(n_features)\n    \n#     else:  # regression-based\n#         # Order by ability to predict others vs being predicted\n#         predictability = np.zeros(n_features)\n#         for i in range(n_features):\n#             # How well can others predict this variable?\n#             others = [j for j in range(n_features) if j != i]\n#             if len(others) > 0:\n#                 lr = LinearRegression()\n#                 lr.fit(X[:, others], X[:, i])\n#                 predictability[i] = lr.score(X[:, others], X[:, i])\n        \n#         # Less predictable = more exogenous = earlier in causal order\n#         causal_order = np.argsort(predictability)\n    \n#     return causal_order\n\n# def identify_mediators(X, y, threshold=0.3):\n#     \"\"\"\n#     Identify mediator variables: X -> M -> Y\n#     \"\"\"\n#     n_features = X.shape[1]\n#     mediator_scores = []\n    \n#     for i in range(n_features):\n#         # Direct effect on Y\n#         lr_direct = LinearRegression()\n#         lr_direct.fit(X[:, i].reshape(-1, 1), y)\n#         direct_effect = abs(lr_direct.coef_[0])\n        \n#         # Indirect effects through other variables\n#         indirect_effects = []\n#         for j in range(n_features):\n#             if i != j:\n#                 # X[i] -> X[j]\n#                 lr_ij = LinearRegression()\n#                 lr_ij.fit(X[:, i].reshape(-1, 1), X[:, j])\n#                 effect_ij = abs(lr_ij.coef_[0])\n                \n#                 # X[j] -> Y\n#                 lr_jy = LinearRegression()\n#                 lr_jy.fit(X[:, j].reshape(-1, 1), y)\n#                 effect_jy = abs(lr_jy.coef_[0])\n                \n#                 # Indirect effect through j\n#                 indirect_effects.append(effect_ij * effect_jy)\n        \n#         # Mediator score: high indirect relative to direct\n#         total_indirect = sum(indirect_effects)\n#         mediation_ratio = total_indirect / (direct_effect + total_indirect + 1e-10)\n#         mediator_scores.append(mediation_ratio)\n    \n#     # Identify mediators\n#     mediators = np.where(np.array(mediator_scores) > threshold)[0]\n#     return mediators, mediator_scores\n\n# def compute_partial_correlations(X):\n#     \"\"\"\n#     Compute partial correlation matrix controlling for all other variables\n#     \"\"\"\n#     n_features = X.shape[1]\n#     partial_corr = np.zeros((n_features, n_features))\n    \n#     for i in range(n_features):\n#         for j in range(i+1, n_features):\n#             # Control for all other variables\n#             others = [k for k in range(n_features) if k != i and k != j]\n            \n#             if len(others) > 0:\n#                 # Residualize i and j\n#                 lr_i = LinearRegression()\n#                 lr_i.fit(X[:, others], X[:, i])\n#                 res_i = X[:, i] - lr_i.predict(X[:, others])\n                \n#                 lr_j = LinearRegression()\n#                 lr_j.fit(X[:, others], X[:, j])\n#                 res_j = X[:, j] - lr_j.predict(X[:, others])\n                \n#                 # Partial correlation\n#                 if np.std(res_i) > 1e-10 and np.std(res_j) > 1e-10:\n#                     partial_corr[i, j] = np.corrcoef(res_i, res_j)[0, 1]\n#                     partial_corr[j, i] = partial_corr[i, j]\n#             else:\n#                 partial_corr[i, j] = np.corrcoef(X[:, i], X[:, j])[0, 1]\n#                 partial_corr[j, i] = partial_corr[i, j]\n    \n#     np.fill_diagonal(partial_corr, 1)\n#     return partial_corr\n\n# def orthogonalize_by_causal_order(X, causal_order):\n#     \"\"\"\n#     Orthogonalize features based on causal ordering\n#     Later features are residualized w.r.t. earlier ones\n#     \"\"\"\n#     X_orth = X.copy()\n#     n_features = X.shape[1]\n    \n#     for i in range(1, n_features):\n#         current_idx = causal_order[i]\n#         previous_indices = causal_order[:i]\n        \n#         if len(previous_indices) > 0:\n#             # Residualize current w.r.t. previous\n#             lr = LinearRegression()\n#             lr.fit(X[:, previous_indices], X[:, current_idx])\n#             X_orth[:, current_idx] = X[:, current_idx] - lr.predict(X[:, previous_indices])\n    \n#     return X_orth\n\n# # ==================== Load Data ====================\n# print(\"=\"*60)\n# print(\"CAUSAL DAG-BASED COLLINEARITY MANAGEMENT\")\n# print(\"=\"*60)\n\n# # Load data\n# train_df = pd.read_parquet(f'{DATA_PATH}train_final.parquet')\n# test_df = pd.read_parquet(f'{DATA_PATH}test_final.parquet')\n\n# print(f\"Train shape: {train_df.shape}\")\n# print(f\"Test shape: {test_df.shape}\")\n\n# # Select features\n# feature_cols = [col for col in train_df.columns if col not in ['label', 'timestamp', 'id']]\n\n# # Extract arrays\n# y_train = train_df['label'].values\n# X_train = train_df[feature_cols].values\n# X_test = test_df[feature_cols].values\n\n# # Clean data\n# X_train = np.nan_to_num(X_train, nan=0.0, posinf=0.0, neginf=0.0)\n# X_test = np.nan_to_num(X_test, nan=0.0, posinf=0.0, neginf=0.0)\n\n# print(f\"Original features: {len(feature_cols)}\")\n\n# # ==================== Initial Screening ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"INITIAL FEATURE SCREENING\")\n# print(\"=\"*60)\n\n# # Remove constant features\n# variances = np.var(X_train, axis=0)\n# non_constant = variances > 1e-10\n# X_train = X_train[:, non_constant]\n# X_test = X_test[:, non_constant]\n# feature_cols = [col for col, keep in zip(feature_cols, non_constant) if keep]\n\n# # Standardize\n# scaler = StandardScaler()\n# X_train_scaled = scaler.fit_transform(X_train)\n# X_test_scaled = scaler.transform(X_test)\n\n# # If too many features, pre-select using Lasso\n# if len(feature_cols) > 150:\n#     print(f\"Pre-selecting features with Lasso...\")\n#     sample_size = min(5000, len(X_train_scaled))\n#     sample_idx = np.random.choice(len(X_train_scaled), sample_size, replace=False)\n    \n#     lasso = LassoCV(cv=3, random_state=42, n_jobs=-1)\n#     lasso.fit(X_train_scaled[sample_idx], y_train[sample_idx])\n    \n#     important_features = np.where(lasso.coef_ != 0)[0]\n#     if len(important_features) < 50:\n#         # Add top absolute coefficients\n#         top_coef = np.argsort(np.abs(lasso.coef_))[-50:]\n#         important_features = np.unique(np.concatenate([important_features, top_coef]))\n    \n#     X_train_scaled = X_train_scaled[:, important_features]\n#     X_test_scaled = X_test_scaled[:, important_features]\n#     feature_cols = [feature_cols[i] for i in important_features]\n\n# print(f\"Features after screening: {len(feature_cols)}\")\n\n# # ==================== Causal Structure Discovery ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"DISCOVERING CAUSAL STRUCTURE\")\n# print(\"=\"*60)\n\n# # Use subset for causal analysis\n# analysis_size = min(5000, len(X_train_scaled))\n# analysis_idx = np.random.choice(len(X_train_scaled), analysis_size, replace=False)\n# X_analysis = X_train_scaled[analysis_idx]\n# y_analysis = y_train[analysis_idx]\n\n# # Step 1: Estimate causal ordering\n# print(\"\\n1. Estimating causal ordering...\")\n# causal_order_var = estimate_causal_order(X_analysis, method='variance')\n# causal_order_reg = estimate_causal_order(X_analysis, method='regression')\n\n# # Combine orderings (average ranks)\n# n_features = X_analysis.shape[1]\n# combined_ranks = np.zeros(n_features)\n# for i in range(n_features):\n#     rank_var = np.where(causal_order_var == i)[0][0]\n#     rank_reg = np.where(causal_order_reg == i)[0][0]\n#     combined_ranks[i] = (rank_var + rank_reg) / 2\n\n# causal_order = np.argsort(combined_ranks)\n# print(f\"   Causal order determined for {n_features} features\")\n\n# # Step 2: Identify mediators\n# print(\"\\n2. Identifying mediator variables...\")\n# mediators, mediation_scores = identify_mediators(X_analysis, y_analysis, threshold=0.3)\n# print(f\"   Found {len(mediators)} mediator variables\")\n\n# # Step 3: Compute partial correlations\n# print(\"\\n3. Computing partial correlations...\")\n# partial_corr = compute_partial_correlations(X_analysis[:, :min(50, n_features)])\n# strong_partial = np.sum(np.abs(partial_corr) > 0.1) - n_features  # Exclude diagonal\n# print(f\"   Found {strong_partial} strong partial correlations\")\n\n# # ==================== Collinearity Management ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"MANAGING COLLINEARITY\")\n# print(\"=\"*60)\n\n# # Strategy 1: Remove mediators (they add collinearity without new information)\n# non_mediator_mask = np.ones(n_features, dtype=bool)\n# non_mediator_mask[mediators] = False\n\n# print(f\"\\n1. Removing {len(mediators)} mediator variables...\")\n# X_train_cleaned = X_train_scaled[:, non_mediator_mask]\n# X_test_cleaned = X_test_scaled[:, non_mediator_mask]\n\n# # Update causal order\n# causal_order_cleaned = []\n# for idx in causal_order:\n#     if non_mediator_mask[idx]:\n#         new_idx = sum(non_mediator_mask[:idx])  # Adjust index\n#         causal_order_cleaned.append(new_idx)\n# causal_order_cleaned = np.array(causal_order_cleaned)\n\n# # Strategy 2: Orthogonalization by causal order\n# print(\"\\n2. Orthogonalizing by causal hierarchy...\")\n# X_train_orth = orthogonalize_by_causal_order(X_train_cleaned, causal_order_cleaned)\n# X_test_orth = orthogonalize_by_causal_order(X_test_cleaned, causal_order_cleaned)\n\n# # Strategy 3: Group highly correlated features and take representatives\n# print(\"\\n3. Grouping correlated features...\")\n# correlation_threshold = 0.8\n# feature_groups = []\n# remaining_features = list(range(X_train_cleaned.shape[1]))\n\n# while remaining_features:\n#     pivot = remaining_features[0]\n#     group = [pivot]\n    \n#     for feat in remaining_features[1:]:\n#         corr = abs(np.corrcoef(X_train_cleaned[:, pivot], X_train_cleaned[:, feat])[0, 1])\n#         if corr > correlation_threshold:\n#             group.append(feat)\n    \n#     feature_groups.append(group)\n#     for feat in group:\n#         remaining_features.remove(feat)\n\n# print(f\"   Created {len(feature_groups)} feature groups\")\n\n# # Select representative from each group (earliest in causal order)\n# group_representatives = []\n# for group in feature_groups:\n#     # Find earliest in causal order\n#     causal_positions = [np.where(causal_order_cleaned == feat)[0][0] if feat in causal_order_cleaned else 999 \n#                        for feat in group]\n#     representative = group[np.argmin(causal_positions)]\n#     group_representatives.append(representative)\n\n# X_train_final = X_train_orth[:, group_representatives]\n# X_test_final = X_test_orth[:, group_representatives]\n\n# print(f\"\\nFinal features after collinearity management: {X_train_final.shape[1]}\")\n\n# # ==================== Add Causal Interactions ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"ADDING CAUSAL INTERACTIONS\")\n# print(\"=\"*60)\n\n# # Add interactions between early causal variables (likely root causes)\n# early_causal = causal_order_cleaned[:min(5, len(causal_order_cleaned))]\n# interactions_added = 0\n\n# for i, feat1 in enumerate(early_causal):\n#     for feat2 in early_causal[i+1:]:\n#         if feat1 in group_representatives and feat2 in group_representatives:\n#             idx1 = group_representatives.index(feat1)\n#             idx2 = group_representatives.index(feat2)\n            \n#             interaction_train = X_train_final[:, idx1] * X_train_final[:, idx2]\n#             interaction_test = X_test_final[:, idx1] * X_test_final[:, idx2]\n            \n#             X_train_final = np.column_stack([X_train_final, interaction_train])\n#             X_test_final = np.column_stack([X_test_final, interaction_test])\n#             interactions_added += 1\n\n# print(f\"Added {interactions_added} causal interactions\")\n# print(f\"Final feature dimension: {X_train_final.shape[1]}\")\n\n# # ==================== Train Models ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"TRAINING MODELS\")\n# print(\"=\"*60)\n\n# train_size = min(30000, len(X_train_final))\n# train_idx = np.random.choice(len(X_train_final), train_size, replace=False)\n\n# models = {}\n# predictions = {}\n\n# # Model 1: GradientBoosting\n# print(\"\\n1. GradientBoosting...\")\n# gb = GradientBoostingRegressor(\n#     n_estimators=150,\n#     max_depth=4,\n#     learning_rate=0.05,\n#     subsample=0.8,\n#     min_samples_leaf=50,\n#     random_state=42\n# )\n# gb.fit(X_train_final[train_idx], y_train[train_idx])\n# models['gb'] = gb\n# predictions['gb'] = gb.predict(X_test_final)\n\n# # Model 2: RandomForest\n# print(\"2. RandomForest...\")\n# rf = RandomForestRegressor(\n#     n_estimators=100,\n#     max_depth=6,\n#     min_samples_leaf=100,\n#     random_state=42,\n#     n_jobs=-1\n# )\n# rf.fit(X_train_final[train_idx], y_train[train_idx])\n# models['rf'] = rf\n# predictions['rf'] = rf.predict(X_test_final)\n\n# # Model 3: Ridge\n# print(\"3. Ridge...\")\n# ridge = Ridge(alpha=1.0, random_state=42)\n# ridge.fit(X_train_final[train_idx], y_train[train_idx])\n# models['ridge'] = ridge\n# predictions['ridge'] = ridge.predict(X_test_final)\n\n# # ==================== Validation & Ensemble ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"VALIDATION\")\n# print(\"=\"*60)\n\n# val_size = 5000\n# val_idx = np.random.choice(len(X_train_final), val_size, replace=False)\n\n# val_scores = {}\n# for name, model in models.items():\n#     val_pred = model.predict(X_train_final[val_idx])\n#     score = np.corrcoef(y_train[val_idx], val_pred)[0, 1]\n#     val_scores[name] = score\n#     print(f\"{name}: {score:.4f}\")\n\n# # Ensemble\n# weights = np.array([val_scores[name] for name in predictions.keys()])\n# weights = np.maximum(weights, 0)\n# weights = weights / weights.sum()\n\n# print(f\"\\nWeights: {dict(zip(predictions.keys(), weights))}\")\n\n# final_predictions = sum(weights[i] * predictions[name] \n#                        for i, name in enumerate(predictions.keys()))\n\n# # ==================== Save ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"SAVING\")\n# print(\"=\"*60)\n\n# sample_sub = pd.read_csv(SAMPLE_SUB_PATH)\n# if len(final_predictions) != len(sample_sub):\n#     final_predictions = final_predictions[:len(sample_sub)]\n\n# submission = pd.DataFrame({\n#     sample_sub.columns[0]: sample_sub.iloc[:, 0],\n#     'prediction': final_predictions\n# })\n\n# submission.to_csv(f'{OUTPUT_PATH}submission.csv', index=False)\n# print(f\"✓ Saved to {OUTPUT_PATH}submission.csv\")\n\n# print(\"\\n✓ Complete!\")\n# print(\"=\"*60)","metadata":{"trusted":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # ==================== Advanced Causal Proxy Variable System ====================\n# import pandas as pd\n# import numpy as np\n# import os\n# from sklearn.ensemble import GradientBoostingRegressor, RandomForestRegressor, ExtraTreesRegressor\n# from sklearn.linear_model import Ridge, LinearRegression, ElasticNetCV\n# from sklearn.preprocessing import StandardScaler\n# from sklearn.cross_decomposition import PLSRegression\n# from sklearn.metrics import r2_score\n# from scipy.linalg import eigh\n# from scipy.sparse.linalg import svds\n# import joblib\n# import warnings\n# warnings.filterwarnings('ignore')\n\n# # ==================== Configuration ====================\n# DATA_PATH = '/kaggle/input/drw-remix-of-winning-strategies-part-1/'\n# OUTPUT_PATH = '/kaggle/working/proxy_causal/'\n# SAMPLE_SUB_PATH = '/kaggle/input/drw-crypto-market-prediction/sample_submission.csv'\n\n# os.makedirs(OUTPUT_PATH, exist_ok=True)\n# print(f\"Output directory: {OUTPUT_PATH}\")\n\n# # ==================== Advanced Causal Functions ====================\n\n# def identify_collinear_blocks(X, threshold=0.7):\n#     \"\"\"\n#     Identify blocks of collinear variables using spectral clustering on correlation matrix\n#     \"\"\"\n#     # Correlation matrix\n#     corr_matrix = np.corrcoef(X.T)\n    \n#     # Convert to affinity matrix\n#     affinity = np.abs(corr_matrix)\n    \n#     # Spectral clustering\n#     # Compute Laplacian\n#     D = np.diag(np.sum(affinity, axis=1))\n#     L = D - affinity\n    \n#     # Eigendecomposition\n#     eigenvalues, eigenvectors = eigh(L)\n    \n#     # Find number of clusters (gap in eigenvalues)\n#     gaps = np.diff(eigenvalues)\n#     n_clusters = np.argmax(gaps[:20]) + 1 if len(gaps) > 20 else min(10, X.shape[1] // 3)\n    \n#     # Cluster assignment based on eigenvectors\n#     clusters = {}\n#     features_clustered = eigenvectors[:, :n_clusters]\n    \n#     # K-means on eigenvector space\n#     from sklearn.cluster import KMeans\n#     kmeans = KMeans(n_clusters=n_clusters, random_state=42)\n#     labels = kmeans.fit_predict(features_clustered)\n    \n#     for i in range(n_clusters):\n#         cluster_members = np.where(labels == i)[0]\n#         if len(cluster_members) > 1:\n#             # Check if truly collinear\n#             avg_corr = np.mean(np.abs(corr_matrix[np.ix_(cluster_members, cluster_members)]))\n#             if avg_corr > threshold:\n#                 clusters[i] = cluster_members.tolist()\n    \n#     return clusters, corr_matrix\n\n# def create_proxy_variables(X, clusters, y=None):\n#     \"\"\"\n#     Create proxy variables for each collinear block using multiple methods\n#     \"\"\"\n#     proxies = []\n#     proxy_info = {}\n    \n#     for cluster_id, member_indices in clusters.items():\n#         X_cluster = X[:, member_indices]\n        \n#         # Method 1: First principal component (captures most variance)\n#         mean = np.mean(X_cluster, axis=0)\n#         X_centered = X_cluster - mean\n#         cov = np.cov(X_centered.T)\n#         eigenvalues, eigenvectors = eigh(cov)\n#         pc1 = X_centered @ eigenvectors[:, -1]  # Largest eigenvalue\n        \n#         # Method 2: Target-aligned projection (if target provided)\n#         if y is not None:\n#             correlations = [np.corrcoef(X_cluster[:, i], y)[0, 1] for i in range(X_cluster.shape[1])]\n#             weights = np.array(correlations)\n#             weights = weights / (np.sum(np.abs(weights)) + 1e-10)\n#             target_proxy = X_cluster @ weights\n#         else:\n#             target_proxy = pc1\n        \n#         # Method 3: Robust proxy (median-based)\n#         robust_proxy = np.median(X_cluster, axis=1)\n        \n#         # Combine proxies (weighted average)\n#         if y is not None:\n#             # Weight by correlation with target\n#             corr_pc1 = abs(np.corrcoef(pc1, y)[0, 1])\n#             corr_target = abs(np.corrcoef(target_proxy, y)[0, 1])\n#             corr_robust = abs(np.corrcoef(robust_proxy, y)[0, 1])\n            \n#             total = corr_pc1 + corr_target + corr_robust + 1e-10\n#             combined_proxy = (corr_pc1 * pc1 + corr_target * target_proxy + corr_robust * robust_proxy) / total\n#         else:\n#             combined_proxy = (pc1 + target_proxy + robust_proxy) / 3\n        \n#         proxies.append(combined_proxy)\n#         proxy_info[cluster_id] = {\n#             'members': member_indices,\n#             'n_members': len(member_indices),\n#             'method': 'combined'\n#         }\n    \n#     return np.column_stack(proxies) if proxies else None, proxy_info\n\n# def learn_proxy_predictors(X_independent, X_collinear, clusters):\n#     \"\"\"\n#     Learn models to predict collinear variables from independent ones\n#     \"\"\"\n#     proxy_models = {}\n    \n#     for cluster_id, member_indices in clusters.items():\n#         X_cluster = X_collinear[:, member_indices]\n        \n#         # Create target (first PC of cluster)\n#         mean = np.mean(X_cluster, axis=0)\n#         X_centered = X_cluster - mean\n#         cov = np.cov(X_centered.T)\n#         eigenvalues, eigenvectors = eigh(cov)\n#         target = X_centered @ eigenvectors[:, -1]\n        \n#         # Train predictor from independent variables\n#         # Use ElasticNet for stability\n#         predictor = ElasticNetCV(cv=3, random_state=42, max_iter=1000)\n#         predictor.fit(X_independent, target)\n        \n#         proxy_models[cluster_id] = {\n#             'model': predictor,\n#             'mean': mean,\n#             'eigenvector': eigenvectors[:, -1],\n#             'score': predictor.score(X_independent, target)\n#         }\n    \n#     return proxy_models\n\n# def create_structural_features(X, causal_order, y):\n#     \"\"\"\n#     Create features based on structural equation modeling concepts\n#     \"\"\"\n#     n_features = X.shape[1]\n#     structural_features = []\n    \n#     # Direct effects\n#     direct_effects = []\n#     for i in range(n_features):\n#         lr = LinearRegression()\n#         lr.fit(X[:, i].reshape(-1, 1), y)\n#         direct_effects.append(lr.coef_[0])\n    \n#     # Total effects (including indirect paths)\n#     total_effects = []\n#     for i in range(n_features):\n#         # Use all features\n#         lr_total = LinearRegression()\n#         lr_total.fit(X, y)\n#         total_effects.append(lr_total.coef_[i])\n    \n#     # Indirect effects\n#     indirect_effects = np.array(total_effects) - np.array(direct_effects)\n    \n#     # Create structural features\n#     # 1. Variables with high direct effects\n#     high_direct = X[:, np.argsort(np.abs(direct_effects))[-10:]]\n#     structural_features.append(high_direct)\n    \n#     # 2. Variables with high indirect effects (mediators)\n#     high_indirect = X[:, np.argsort(np.abs(indirect_effects))[-5:]]\n#     structural_features.append(high_indirect)\n    \n#     # 3. Interaction between high direct and indirect\n#     if high_direct.shape[1] > 0 and high_indirect.shape[1] > 0:\n#         interactions = high_direct[:, 0].reshape(-1, 1) * high_indirect\n#         structural_features.append(interactions)\n    \n#     return np.hstack(structural_features) if structural_features else X\n\n# # ==================== Load Data ====================\n# print(\"=\"*60)\n# print(\"ADVANCED PROXY VARIABLE SYSTEM\")\n# print(\"=\"*60)\n\n# train_df = pd.read_parquet(f'{DATA_PATH}train_final.parquet')\n# test_df = pd.read_parquet(f'{DATA_PATH}test_final.parquet')\n\n# print(f\"Train shape: {train_df.shape}\")\n# print(f\"Test shape: {test_df.shape}\")\n\n# feature_cols = [col for col in train_df.columns if col not in ['label', 'timestamp', 'id']]\n\n# y_train = train_df['label'].values\n# X_train = train_df[feature_cols].values\n# X_test = test_df[feature_cols].values\n\n# X_train = np.nan_to_num(X_train, nan=0.0, posinf=0.0, neginf=0.0)\n# X_test = np.nan_to_num(X_test, nan=0.0, posinf=0.0, neginf=0.0)\n\n# print(f\"Original features: {len(feature_cols)}\")\n\n# # ==================== Feature Screening ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"INTELLIGENT FEATURE SCREENING\")\n# print(\"=\"*60)\n\n# # Remove zero variance\n# variances = np.var(X_train, axis=0)\n# non_zero = variances > 1e-10\n# X_train = X_train[:, non_zero]\n# X_test = X_test[:, non_zero]\n\n# # Standardize\n# scaler = StandardScaler()\n# X_train_scaled = scaler.fit_transform(X_train)\n# X_test_scaled = scaler.transform(X_test)\n\n# # Use mutual information for initial selection\n# if X_train_scaled.shape[1] > 100:\n#     print(\"Selecting top features by mutual information...\")\n#     from sklearn.feature_selection import mutual_info_regression\n    \n#     sample_size = min(5000, len(X_train_scaled))\n#     sample_idx = np.random.choice(len(X_train_scaled), sample_size, replace=False)\n    \n#     mi_scores = mutual_info_regression(X_train_scaled[sample_idx], y_train[sample_idx], random_state=42)\n#     top_features = np.argsort(mi_scores)[-100:]\n    \n#     X_train_scaled = X_train_scaled[:, top_features]\n#     X_test_scaled = X_test_scaled[:, top_features]\n\n# print(f\"Features after screening: {X_train_scaled.shape[1]}\")\n\n# # ==================== Collinearity Analysis ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"ADVANCED COLLINEARITY ANALYSIS\")\n# print(\"=\"*60)\n\n# # Identify collinear blocks\n# analysis_size = min(5000, len(X_train_scaled))\n# analysis_idx = np.random.choice(len(X_train_scaled), analysis_size, replace=False)\n# X_analysis = X_train_scaled[analysis_idx]\n# y_analysis = y_train[analysis_idx]\n\n# print(\"\\n1. Identifying collinear blocks...\")\n# collinear_blocks, corr_matrix = identify_collinear_blocks(X_analysis, threshold=0.7)\n# print(f\"   Found {len(collinear_blocks)} collinear blocks\")\n\n# # Separate independent and collinear features\n# all_collinear_indices = []\n# for indices in collinear_blocks.values():\n#     all_collinear_indices.extend(indices)\n# all_collinear_indices = list(set(all_collinear_indices))\n\n# independent_indices = [i for i in range(X_analysis.shape[1]) if i not in all_collinear_indices]\n\n# print(f\"   Independent features: {len(independent_indices)}\")\n# print(f\"   Collinear features: {len(all_collinear_indices)}\")\n\n# # ==================== Proxy Variable Creation ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"CREATING PROXY VARIABLES\")\n# print(\"=\"*60)\n\n# # Method 1: Direct proxy creation\n# print(\"\\n1. Creating direct proxies...\")\n# if collinear_blocks:\n#     proxies_train, proxy_info = create_proxy_variables(X_train_scaled, collinear_blocks, y_train)\n#     proxies_test, _ = create_proxy_variables(X_test_scaled, collinear_blocks)\n#     print(f\"   Created {proxies_train.shape[1] if proxies_train is not None else 0} proxy variables\")\n# else:\n#     proxies_train = None\n#     proxies_test = None\n\n# # Method 2: Learned proxy predictors\n# print(\"\\n2. Learning proxy predictors...\")\n# if len(independent_indices) > 0 and collinear_blocks:\n#     X_indep = X_analysis[:, independent_indices]\n#     X_collin = X_analysis[:, all_collinear_indices]\n    \n#     proxy_models = learn_proxy_predictors(X_indep, X_collin, \n#                                          {k: [all_collinear_indices.index(i) for i in v] \n#                                           for k, v in collinear_blocks.items()})\n    \n#     # Generate predicted proxies\n#     predicted_proxies_train = []\n#     predicted_proxies_test = []\n    \n#     for cluster_id, model_info in proxy_models.items():\n#         pred_train = model_info['model'].predict(X_train_scaled[:, independent_indices])\n#         pred_test = model_info['model'].predict(X_test_scaled[:, independent_indices])\n#         predicted_proxies_train.append(pred_train)\n#         predicted_proxies_test.append(pred_test)\n#         print(f\"   Cluster {cluster_id}: R² = {model_info['score']:.3f}\")\n    \n#     if predicted_proxies_train:\n#         predicted_proxies_train = np.column_stack(predicted_proxies_train)\n#         predicted_proxies_test = np.column_stack(predicted_proxies_test)\n# else:\n#     predicted_proxies_train = None\n#     predicted_proxies_test = None\n\n# # ==================== Combine Features ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"COMBINING FEATURES\")\n# print(\"=\"*60)\n\n# # Combine independent, proxy, and predicted proxy features\n# feature_sets = []\n# feature_sets_test = []\n\n# # Independent features\n# if len(independent_indices) > 0:\n#     feature_sets.append(X_train_scaled[:, independent_indices])\n#     feature_sets_test.append(X_test_scaled[:, independent_indices])\n#     print(f\"Independent features: {len(independent_indices)}\")\n\n# # Direct proxies\n# if proxies_train is not None:\n#     feature_sets.append(proxies_train)\n#     feature_sets_test.append(proxies_test)\n#     print(f\"Direct proxies: {proxies_train.shape[1]}\")\n\n# # Predicted proxies\n# if predicted_proxies_train is not None:\n#     feature_sets.append(predicted_proxies_train * 0.5)  # Downweight\n#     feature_sets_test.append(predicted_proxies_test * 0.5)\n#     print(f\"Predicted proxies: {predicted_proxies_train.shape[1]}\")\n\n# X_train_combined = np.hstack(feature_sets)\n# X_test_combined = np.hstack(feature_sets_test)\n\n# print(f\"\\nCombined features: {X_train_combined.shape[1]}\")\n\n# # ==================== Structural Features ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"STRUCTURAL EQUATION FEATURES\")\n# print(\"=\"*60)\n\n# # Estimate causal order\n# causal_order = np.argsort(np.var(X_train_combined, axis=0))[::-1]\n\n# # Create structural features\n# X_train_structural = create_structural_features(\n#     X_train_combined[:min(5000, len(X_train_combined))], \n#     causal_order, \n#     y_train[:min(5000, len(y_train))]\n# )\n\n# # Apply same transformation to test\n# if X_train_structural.shape[1] > X_train_combined.shape[1]:\n#     n_structural = X_train_structural.shape[1] - X_train_combined.shape[1]\n#     # Approximate structural features for test\n#     X_test_structural = create_structural_features(\n#         X_test_combined[:1000], \n#         causal_order, \n#         np.zeros(1000)  # Dummy target\n#     )\n#     X_train_final = X_train_structural\n#     X_test_final = X_test_structural\n# else:\n#     X_train_final = X_train_combined\n#     X_test_final = X_test_combined\n\n# print(f\"Final features: {X_train_final.shape[1]}\")\n\n# # ==================== Train Advanced Models ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"TRAINING ENSEMBLE\")\n# print(\"=\"*60)\n\n# train_size = min(30000, len(X_train_final))\n# train_idx = np.random.choice(len(X_train_final), train_size, replace=False)\n\n# models = {}\n# predictions = {}\n\n# # Model 1: GradientBoosting\n# print(\"\\n1. GradientBoosting...\")\n# gb = GradientBoostingRegressor(\n#     n_estimators=150,\n#     max_depth=4,\n#     learning_rate=0.05,\n#     subsample=0.8,\n#     min_samples_leaf=50,\n#     random_state=42\n# )\n# gb.fit(X_train_final[train_idx], y_train[train_idx])\n# models['gb'] = gb\n# predictions['gb'] = gb.predict(X_test_final)\n\n# # Model 2: ExtraTrees (handles feature interactions well)\n# print(\"2. ExtraTrees...\")\n# et = ExtraTreesRegressor(\n#     n_estimators=100,\n#     max_depth=6,\n#     min_samples_leaf=100,\n#     random_state=42,\n#     n_jobs=-1\n# )\n# et.fit(X_train_final[train_idx], y_train[train_idx])\n# models['et'] = et\n# predictions['et'] = et.predict(X_test_final)\n\n# # Model 3: PLS Regression (handles collinearity)\n# print(\"3. PLS Regression...\")\n# pls = PLSRegression(n_components=min(20, X_train_final.shape[1]))\n# pls.fit(X_train_final[train_idx], y_train[train_idx])\n# models['pls'] = pls\n# predictions['pls'] = pls.predict(X_test_final).ravel()\n\n# # Model 4: Ridge\n# print(\"4. Ridge...\")\n# ridge = Ridge(alpha=1.0, random_state=42)\n# ridge.fit(X_train_final[train_idx], y_train[train_idx])\n# models['ridge'] = ridge\n# predictions['ridge'] = ridge.predict(X_test_final)\n\n# # ==================== Validation ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"VALIDATION\")\n# print(\"=\"*60)\n\n# val_size = 5000\n# val_idx = np.random.choice(len(X_train_final), val_size, replace=False)\n\n# val_scores = {}\n# for name, model in models.items():\n#     if name == 'pls':\n#         val_pred = model.predict(X_train_final[val_idx]).ravel()\n#     else:\n#         val_pred = model.predict(X_train_final[val_idx])\n#     score = np.corrcoef(y_train[val_idx], val_pred)[0, 1]\n#     val_scores[name] = score\n#     print(f\"{name}: {score:.4f}\")\n\n# # Ensemble\n# weights = np.array([val_scores[name] for name in predictions.keys()])\n# weights = np.maximum(weights, 0)\n# weights = weights / weights.sum()\n\n# print(f\"\\nWeights: {dict(zip(predictions.keys(), weights))}\")\n\n# final_predictions = sum(weights[i] * predictions[name] \n#                        for i, name in enumerate(predictions.keys()))\n\n# # ==================== Save ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"SAVING\")\n# print(\"=\"*60)\n\n# sample_sub = pd.read_csv(SAMPLE_SUB_PATH)\n# if len(final_predictions) != len(sample_sub):\n#     final_predictions = final_predictions[:len(sample_sub)]\n\n# submission = pd.DataFrame({\n#     sample_sub.columns[0]: sample_sub.iloc[:, 0],\n#     'prediction': final_predictions\n# })\n\n# submission.to_csv(f'{OUTPUT_PATH}submission.csv', index=False)\n# print(f\"✓ Saved to {OUTPUT_PATH}submission.csv\")\n\n# print(\"\\n✓ Complete!\")\n# print(\"=\"*60)","metadata":{"trusted":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # ==================== Simplified Advanced Causal Proxy System ====================\n# import pandas as pd\n# import numpy as np\n# import os\n# from sklearn.ensemble import GradientBoostingRegressor, RandomForestRegressor\n# from sklearn.linear_model import Ridge, LinearRegression, ElasticNetCV\n# from sklearn.preprocessing import StandardScaler\n# from sklearn.decomposition import PCA, FastICA\n# from sklearn.cluster import KMeans\n# from sklearn.metrics import silhouette_score\n# import joblib\n# import warnings\n# warnings.filterwarnings('ignore')\n\n# # ==================== Configuration ====================\n# DATA_PATH = '/kaggle/input/drw-remix-of-winning-strategies-part-1/'\n# OUTPUT_PATH = '/kaggle/working/causal_proxy/'\n# SAMPLE_SUB_PATH = '/kaggle/input/drw-crypto-market-prediction/sample_submission.csv'\n\n# os.makedirs(OUTPUT_PATH, exist_ok=True)\n# print(f\"Output directory: {OUTPUT_PATH}\")\n\n# # ==================== Causal Proxy Functions ====================\n\n# def find_collinear_groups(X, threshold=0.7):\n#     \"\"\"\n#     Find groups of collinear variables using correlation\n#     \"\"\"\n#     corr_matrix = np.corrcoef(X.T)\n#     n_features = X.shape[1]\n    \n#     groups = []\n#     visited = set()\n    \n#     for i in range(n_features):\n#         if i in visited:\n#             continue\n            \n#         group = [i]\n#         visited.add(i)\n        \n#         for j in range(i+1, n_features):\n#             if j not in visited and abs(corr_matrix[i, j]) > threshold:\n#                 group.append(j)\n#                 visited.add(j)\n        \n#         if len(group) > 1:\n#             groups.append(group)\n    \n#     return groups\n\n# def create_causal_proxies(X, groups, y=None):\n#     \"\"\"\n#     Create proxy variables for each collinear group\n#     \"\"\"\n#     proxies = []\n    \n#     for group in groups:\n#         X_group = X[:, group]\n        \n#         # Method 1: First principal component\n#         mean = np.mean(X_group, axis=0)\n#         X_centered = X_group - mean\n#         cov = np.cov(X_centered.T)\n#         eigenvalues, eigenvectors = np.linalg.eigh(cov)\n#         pc1 = X_centered @ eigenvectors[:, -1]\n        \n#         # Method 2: Target-aligned (if y provided)\n#         if y is not None:\n#             correlations = [abs(np.corrcoef(X_group[:, i], y)[0, 1]) for i in range(X_group.shape[1])]\n#             weights = np.array(correlations)\n#             weights = weights / (np.sum(weights) + 1e-10)\n#             target_proxy = X_group @ weights\n            \n#             # Combine methods\n#             proxy = 0.5 * pc1 + 0.5 * target_proxy\n#         else:\n#             proxy = pc1\n            \n#         proxies.append(proxy)\n    \n#     return np.column_stack(proxies) if proxies else None\n\n# def hierarchical_compression(X, levels=[50, 30, 15]):\n#     \"\"\"\n#     Compress features hierarchically\n#     \"\"\"\n#     compressed = []\n#     current_X = X\n    \n#     for n_comp in levels:\n#         if current_X.shape[1] <= n_comp:\n#             compressed.append(current_X)\n#             break\n            \n#         # Use ICA for non-linear compression\n#         ica = FastICA(n_components=n_comp, random_state=42, max_iter=100)\n#         try:\n#             current_X = ica.fit_transform(current_X)\n#             compressed.append(current_X)\n#         except:\n#             # Fallback to PCA\n#             from sklearn.decomposition import PCA\n#             pca = PCA(n_components=n_comp, random_state=42)\n#             current_X = pca.fit_transform(current_X)\n#             compressed.append(current_X)\n    \n#     return compressed\n\n# def compute_causal_importance(X, y):\n#     \"\"\"\n#     Compute causal importance scores for features\n#     \"\"\"\n#     n_features = X.shape[1]\n#     importance = np.zeros(n_features)\n    \n#     for i in range(n_features):\n#         # Direct effect\n#         lr = LinearRegression()\n#         lr.fit(X[:, i].reshape(-1, 1), y)\n#         direct = abs(lr.coef_[0])\n        \n#         # Unique variance\n#         if n_features > 1:\n#             others = [j for j in range(n_features) if j != i]\n#             lr_others = LinearRegression()\n#             lr_others.fit(X[:, others], X[:, i])\n#             residual_var = np.var(X[:, i] - lr_others.predict(X[:, others]))\n#             unique = residual_var / (np.var(X[:, i]) + 1e-10)\n#         else:\n#             unique = 1.0\n        \n#         importance[i] = direct * (1 + unique)\n    \n#     return importance / (np.sum(importance) + 1e-10)\n\n# # ==================== Load Data ====================\n# print(\"=\"*60)\n# print(\"CAUSAL PROXY VARIABLE SYSTEM\")\n# print(\"=\"*60)\n\n# train_df = pd.read_parquet(f'{DATA_PATH}train_final.parquet')\n# test_df = pd.read_parquet(f'{DATA_PATH}test_final.parquet')\n\n# print(f\"Train shape: {train_df.shape}\")\n# print(f\"Test shape: {test_df.shape}\")\n\n# feature_cols = [col for col in train_df.columns if col not in ['label', 'timestamp', 'id']]\n\n# y_train = train_df['label'].values\n# X_train = train_df[feature_cols].values\n# X_test = test_df[feature_cols].values\n\n# X_train = np.nan_to_num(X_train, nan=0.0, posinf=0.0, neginf=0.0)\n# X_test = np.nan_to_num(X_test, nan=0.0, posinf=0.0, neginf=0.0)\n\n# print(f\"Original features: {len(feature_cols)}\")\n\n# # ==================== Initial Processing ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"INITIAL PROCESSING\")\n# print(\"=\"*60)\n\n# # Remove constant features\n# variances = np.var(X_train, axis=0)\n# non_constant = variances > 1e-10\n# X_train = X_train[:, non_constant]\n# X_test = X_test[:, non_constant]\n\n# # Standardize\n# scaler = StandardScaler()\n# X_train_scaled = scaler.fit_transform(X_train)\n# X_test_scaled = scaler.transform(X_test)\n\n# # Initial feature selection if too many\n# if X_train_scaled.shape[1] > 100:\n#     print(\"Selecting top 100 features...\")\n#     from sklearn.feature_selection import mutual_info_regression\n#     sample_idx = np.random.choice(len(X_train_scaled), min(5000, len(X_train_scaled)), replace=False)\n#     mi_scores = mutual_info_regression(X_train_scaled[sample_idx], y_train[sample_idx])\n#     top_features = np.argsort(mi_scores)[-100:]\n#     X_train_scaled = X_train_scaled[:, top_features]\n#     X_test_scaled = X_test_scaled[:, top_features]\n\n# print(f\"Working with {X_train_scaled.shape[1]} features\")\n\n# # ==================== Hierarchical Compression ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"HIERARCHICAL COMPRESSION\")\n# print(\"=\"*60)\n\n# levels = [60, 40, 25] if X_train_scaled.shape[1] > 60 else [X_train_scaled.shape[1]]\n# compressed_train = hierarchical_compression(X_train_scaled, levels)\n# compressed_test = hierarchical_compression(X_test_scaled, levels)\n\n# # Use middle level\n# level_idx = min(1, len(compressed_train) - 1)\n# X_train_compressed = compressed_train[level_idx]\n# X_test_compressed = compressed_test[level_idx]\n\n# print(f\"Compressed to {X_train_compressed.shape[1]} features\")\n\n# # ==================== Collinear Group Detection ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"COLLINEAR GROUP DETECTION\")\n# print(\"=\"*60)\n\n# # Find collinear groups on sample\n# sample_size = min(5000, len(X_train_compressed))\n# sample_idx = np.random.choice(len(X_train_compressed), sample_size, replace=False)\n# X_sample = X_train_compressed[sample_idx]\n# y_sample = y_train[sample_idx]\n\n# collinear_groups = find_collinear_groups(X_sample, threshold=0.7)\n# print(f\"Found {len(collinear_groups)} collinear groups\")\n\n# # ==================== Create Proxy Variables ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"CREATING PROXY VARIABLES\")\n# print(\"=\"*60)\n\n# # Get all collinear indices\n# all_collinear = set()\n# for group in collinear_groups:\n#     all_collinear.update(group)\n\n# # Separate independent and collinear features\n# independent_indices = [i for i in range(X_train_compressed.shape[1]) if i not in all_collinear]\n\n# print(f\"Independent features: {len(independent_indices)}\")\n# print(f\"Collinear features: {len(all_collinear)}\")\n\n# # Create proxies for full dataset\n# if collinear_groups:\n#     proxies_train = create_causal_proxies(X_train_compressed, collinear_groups, y_train)\n#     proxies_test = create_causal_proxies(X_test_compressed, collinear_groups)\n#     print(f\"Created {proxies_train.shape[1]} proxy variables\")\n# else:\n#     proxies_train = None\n#     proxies_test = None\n\n# # ==================== Causal Importance Weighting ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"CAUSAL IMPORTANCE\")\n# print(\"=\"*60)\n\n# # Compute importance on sample\n# importance_scores = compute_causal_importance(X_sample, y_sample)\n# print(f\"Top 5 importance scores: {sorted(importance_scores)[-5:]}\")\n\n# # Apply importance weighting to independent features\n# X_train_independent = X_train_compressed[:, independent_indices]\n# X_test_independent = X_test_compressed[:, independent_indices]\n\n# # Weight by importance\n# independent_importance = importance_scores[independent_indices]\n# independent_importance = independent_importance / (np.sum(independent_importance) + 1e-10)\n\n# X_train_weighted = X_train_independent * np.sqrt(independent_importance)\n# X_test_weighted = X_test_independent * np.sqrt(independent_importance)\n\n# # ==================== Final Feature Assembly ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"FINAL FEATURE ASSEMBLY\")\n# print(\"=\"*60)\n\n# # Combine features\n# features_list_train = [X_train_weighted]\n# features_list_test = [X_test_weighted]\n\n# if proxies_train is not None:\n#     features_list_train.append(proxies_train)\n#     features_list_test.append(proxies_test)\n\n# # Add top unweighted features\n# features_list_train.append(X_train_compressed[:, :10])\n# features_list_test.append(X_test_compressed[:, :10])\n\n# X_train_final = np.hstack(features_list_train)\n# X_test_final = np.hstack(features_list_test)\n\n# print(f\"Final features: {X_train_final.shape[1]}\")\n\n# # ==================== Train Models ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"TRAINING MODELS\")\n# print(\"=\"*60)\n\n# train_size = min(30000, len(X_train_final))\n# train_idx = np.random.choice(len(X_train_final), train_size, replace=False)\n\n# models = {}\n# predictions = {}\n\n# # Model 1: GradientBoosting\n# print(\"\\n1. GradientBoosting...\")\n# gb = GradientBoostingRegressor(\n#     n_estimators=150,\n#     max_depth=4,\n#     learning_rate=0.05,\n#     subsample=0.8,\n#     min_samples_leaf=50,\n#     random_state=42\n# )\n# gb.fit(X_train_final[train_idx], y_train[train_idx])\n# models['gb'] = gb\n# predictions['gb'] = gb.predict(X_test_final)\n\n# # Model 2: RandomForest\n# print(\"2. RandomForest...\")\n# rf = RandomForestRegressor(\n#     n_estimators=100,\n#     max_depth=6,\n#     min_samples_leaf=100,\n#     random_state=42,\n#     n_jobs=-1\n# )\n# rf.fit(X_train_final[train_idx], y_train[train_idx])\n# models['rf'] = rf\n# predictions['rf'] = rf.predict(X_test_final)\n\n# # Model 3: Ridge\n# print(\"3. Ridge...\")\n# ridge = Ridge(alpha=1.0, random_state=42)\n# ridge.fit(X_train_final[train_idx], y_train[train_idx])\n# models['ridge'] = ridge\n# predictions['ridge'] = ridge.predict(X_test_final)\n\n# # ==================== Validation & Ensemble ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"VALIDATION\")\n# print(\"=\"*60)\n\n# val_size = 5000\n# val_idx = np.random.choice(len(X_train_final), val_size, replace=False)\n\n# val_scores = {}\n# for name, model in models.items():\n#     val_pred = model.predict(X_train_final[val_idx])\n#     score = np.corrcoef(y_train[val_idx], val_pred)[0, 1]\n#     val_scores[name] = score\n#     print(f\"{name}: {score:.4f}\")\n\n# # Ensemble\n# weights = np.array([val_scores[name] for name in predictions.keys()])\n# weights = np.maximum(weights, 0)\n# weights = weights / weights.sum()\n\n# print(f\"\\nWeights: {dict(zip(predictions.keys(), weights))}\")\n\n# final_predictions = sum(weights[i] * predictions[name] \n#                        for i, name in enumerate(predictions.keys()))\n\n# # ==================== Save ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"SAVING\")\n# print(\"=\"*60)\n\n# sample_sub = pd.read_csv(SAMPLE_SUB_PATH)\n# if len(final_predictions) != len(sample_sub):\n#     final_predictions = final_predictions[:len(sample_sub)]\n\n# submission = pd.DataFrame({\n#     sample_sub.columns[0]: sample_sub.iloc[:, 0],\n#     'prediction': final_predictions\n# })\n\n# submission.to_csv(f'{OUTPUT_PATH}submission.csv', index=False)\n# print(f\"✓ Saved to {OUTPUT_PATH}submission.csv\")\n\n# print(\"\\n✓ Complete!\")\n# print(\"=\"*60)","metadata":{"trusted":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # ==================== Market Microstructure Causal Hierarchy System ====================\n# import pandas as pd\n# import numpy as np\n# import os\n# from sklearn.ensemble import GradientBoostingRegressor, RandomForestRegressor\n# from sklearn.linear_model import Ridge, LinearRegression\n# from sklearn.preprocessing import StandardScaler, RobustScaler\n# from sklearn.decomposition import PCA\n# from sklearn.cluster import KMeans\n# import joblib\n# import warnings\n# warnings.filterwarnings('ignore')\n\n# # ==================== Configuration ====================\n# DATA_PATH = '/kaggle/input/drw-remix-of-winning-strategies-part-1/'\n# OUTPUT_PATH = '/kaggle/working/market_causal/'\n# SAMPLE_SUB_PATH = '/kaggle/input/drw-crypto-market-prediction/sample_submission.csv'\n\n# os.makedirs(OUTPUT_PATH, exist_ok=True)\n\n# # ==================== Market Microstructure Functions ====================\n\n# def create_market_hierarchy(df):\n#     \"\"\"\n#     Create hierarchical features based on market microstructure theory\n#     \"\"\"\n#     features = {}\n    \n#     # Level 1: Raw liquidity (most fundamental)\n#     if 'bid_qty' in df.columns and 'ask_qty' in df.columns:\n#         features['liquidity_total'] = df['bid_qty'] + df['ask_qty']\n#         features['liquidity_imbalance'] = (df['bid_qty'] - df['ask_qty']) / (df['bid_qty'] + df['ask_qty'] + 1e-10)\n    \n#     # Level 2: Price pressure (caused by liquidity)\n#     if 'buy_qty' in df.columns and 'sell_qty' in df.columns:\n#         features['order_flow'] = df['buy_qty'] - df['sell_qty']\n#         features['order_flow_ratio'] = df['buy_qty'] / (df['buy_qty'] + df['sell_qty'] + 1e-10)\n        \n#     # Level 3: Volume dynamics (result of price pressure)\n#     if 'volume' in df.columns:\n#         features['volume_rate'] = df['volume'] / (df['volume'].rolling(10, min_periods=1).mean() + 1e-10)\n#         features['volume_acceleration'] = df['volume'].diff().fillna(0)\n    \n#     # Level 4: Spread dynamics (emergent from all above)\n#     if 'bid_qty' in df.columns and 'ask_qty' in df.columns and 'volume' in df.columns:\n#         features['spread_proxy'] = (df['ask_qty'] - df['bid_qty']) / (df['volume'] + 1)\n#         features['depth_ratio'] = (df['bid_qty'] + df['ask_qty']) / (df['volume'] + 1)\n    \n#     return pd.DataFrame(features)\n\n# def identify_market_regimes(X, n_regimes=5):\n#     \"\"\"\n#     Identify market regimes (calm, volatile, trending, etc.)\n#     \"\"\"\n#     # Use KMeans to identify regimes\n#     kmeans = KMeans(n_clusters=n_regimes, random_state=42)\n#     regimes = kmeans.fit_predict(X)\n    \n#     # Calculate regime characteristics\n#     regime_info = {}\n#     for i in range(n_regimes):\n#         mask = regimes == i\n#         regime_info[i] = {\n#             'mean': np.mean(X[mask], axis=0),\n#             'std': np.std(X[mask], axis=0),\n#             'size': np.sum(mask)\n#         }\n    \n#     return regimes, regime_info, kmeans\n\n# def create_causal_clusters(X, y, n_clusters=20):\n#     \"\"\"\n#     Create clusters based on causal relationship with target\n#     \"\"\"\n#     # Calculate feature-target relationships\n#     feature_effects = []\n#     for i in range(X.shape[1]):\n#         # Direct effect on target\n#         lr = LinearRegression()\n#         lr.fit(X[:, i].reshape(-1, 1), y)\n#         effect = abs(lr.coef_[0])\n        \n#         # Variance explained\n#         score = lr.score(X[:, i].reshape(-1, 1), y)\n        \n#         feature_effects.append(effect * (1 + score))\n    \n#     feature_effects = np.array(feature_effects)\n    \n#     # Create distance matrix based on causal similarity\n#     n_features = X.shape[1]\n#     causal_distances = np.zeros((n_features, n_features))\n    \n#     for i in range(n_features):\n#         for j in range(i+1, n_features):\n#             # Similar causal effect = low distance\n#             effect_diff = abs(feature_effects[i] - feature_effects[j])\n            \n#             # Correlation between features\n#             feat_corr = abs(np.corrcoef(X[:, i], X[:, j])[0, 1])\n            \n#             # Combined distance\n#             causal_distances[i, j] = effect_diff * (2 - feat_corr)\n#             causal_distances[j, i] = causal_distances[i, j]\n    \n#     # Cluster based on causal distances\n#     from sklearn.cluster import AgglomerativeClustering\n#     clustering = AgglomerativeClustering(n_clusters=n_clusters, metric='precomputed', linkage='average')\n#     clusters = clustering.fit_predict(causal_distances)\n    \n#     return clusters, feature_effects\n\n# def select_cluster_medoids(X, y, clusters):\n#     \"\"\"\n#     Select medoid from each cluster based on predictive power\n#     \"\"\"\n#     unique_clusters = np.unique(clusters)\n#     medoids = []\n    \n#     for cluster_id in unique_clusters:\n#         cluster_mask = clusters == cluster_id\n#         cluster_indices = np.where(cluster_mask)[0]\n        \n#         if len(cluster_indices) == 1:\n#             medoids.append(cluster_indices[0])\n#         else:\n#             # Find medoid based on predictive power\n#             best_score = -1\n#             best_idx = cluster_indices[0]\n            \n#             for idx in cluster_indices:\n#                 # Predictive power\n#                 lr = LinearRegression()\n#                 lr.fit(X[:, idx].reshape(-1, 1), y)\n#                 score = lr.score(X[:, idx].reshape(-1, 1), y)\n                \n#                 # Centrality within cluster\n#                 within_cluster_corr = 0\n#                 for other_idx in cluster_indices:\n#                     if idx != other_idx:\n#                         within_cluster_corr += abs(np.corrcoef(X[:, idx], X[:, other_idx])[0, 1])\n                \n#                 combined_score = score + 0.3 * within_cluster_corr / len(cluster_indices)\n                \n#                 if combined_score > best_score:\n#                     best_score = combined_score\n#                     best_idx = idx\n            \n#             medoids.append(best_idx)\n    \n#     return medoids\n\n# def create_interaction_network(X, medoids, y):\n#     \"\"\"\n#     Create interaction features based on information flow\n#     \"\"\"\n#     interactions = []\n    \n#     # Find pairs with high information transfer\n#     for i, idx1 in enumerate(medoids[:10]):  # Limit to top 10\n#         for idx2 in medoids[i+1:10]:\n#             # Mutual information proxy\n#             combined = X[:, idx1] * X[:, idx2]\n#             lr = LinearRegression()\n#             lr.fit(combined.reshape(-1, 1), y)\n#             score = lr.score(combined.reshape(-1, 1), y)\n            \n#             if score > 0.01:  # Threshold for meaningful interaction\n#                 interactions.append((idx1, idx2, score))\n    \n#     # Sort by importance\n#     interactions.sort(key=lambda x: x[2], reverse=True)\n    \n#     return interactions[:5]  # Top 5 interactions\n\n# # ==================== Load Data ====================\n# print(\"=\"*60)\n# print(\"MARKET MICROSTRUCTURE CAUSAL HIERARCHY\")\n# print(\"=\"*60)\n\n# train_df = pd.read_parquet(f'{DATA_PATH}train_final.parquet')\n# test_df = pd.read_parquet(f'{DATA_PATH}test_final.parquet')\n\n# print(f\"Train shape: {train_df.shape}\")\n# print(f\"Test shape: {test_df.shape}\")\n\n# # ==================== Create Market Hierarchy ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"MARKET HIERARCHY FEATURES\")\n# print(\"=\"*60)\n\n# # Create hierarchical market features\n# market_features_train = create_market_hierarchy(train_df)\n# market_features_test = create_market_hierarchy(test_df)\n\n# print(f\"Created {market_features_train.shape[1]} market hierarchy features\")\n\n# # Combine with original features\n# feature_cols = [col for col in train_df.columns if col not in ['label', 'timestamp', 'id']]\n# X_train_base = train_df[feature_cols].values\n# X_test_base = test_df[feature_cols].values\n# y_train = train_df['label'].values\n\n# # Add market features\n# X_train = np.hstack([X_train_base, market_features_train.values])\n# X_test = np.hstack([X_test_base, market_features_test.values])\n\n# # Clean\n# X_train = np.nan_to_num(X_train, nan=0.0, posinf=0.0, neginf=0.0)\n# X_test = np.nan_to_num(X_test, nan=0.0, posinf=0.0, neginf=0.0)\n\n# print(f\"Total features: {X_train.shape[1]}\")\n\n# # ==================== Robust Scaling ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"ROBUST PREPROCESSING\")\n# print(\"=\"*60)\n\n# # Use RobustScaler for financial data (handles outliers better)\n# scaler = RobustScaler()\n# X_train_scaled = scaler.fit_transform(X_train)\n# X_test_scaled = scaler.transform(X_test)\n\n# # Remove constant features\n# variances = np.var(X_train_scaled, axis=0)\n# non_constant = variances > 1e-10\n# X_train_scaled = X_train_scaled[:, non_constant]\n# X_test_scaled = X_test_scaled[:, non_constant]\n\n# print(f\"Features after filtering: {X_train_scaled.shape[1]}\")\n\n# # ==================== Dimension Reduction ====================\n# if X_train_scaled.shape[1] > 100:\n#     print(\"Reducing dimensions...\")\n#     # Use correlation with target for selection\n#     correlations = []\n#     sample_size = min(10000, len(X_train_scaled))\n#     sample_idx = np.random.choice(len(X_train_scaled), sample_size, replace=False)\n    \n#     for i in range(X_train_scaled.shape[1]):\n#         corr = abs(np.corrcoef(X_train_scaled[sample_idx, i], y_train[sample_idx])[0, 1])\n#         correlations.append(corr)\n    \n#     top_features = np.argsort(correlations)[-100:]\n#     X_train_scaled = X_train_scaled[:, top_features]\n#     X_test_scaled = X_test_scaled[:, top_features]\n\n# print(f\"Working with {X_train_scaled.shape[1]} features\")\n\n# # ==================== Market Regime Identification ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"MARKET REGIME ANALYSIS\")\n# print(\"=\"*60)\n\n# # Identify market regimes\n# sample_size = min(10000, len(X_train_scaled))\n# sample_idx = np.random.choice(len(X_train_scaled), sample_size, replace=False)\n# X_sample = X_train_scaled[sample_idx]\n# y_sample = y_train[sample_idx]\n\n# regimes, regime_info, regime_model = identify_market_regimes(X_sample[:, :20], n_regimes=5)\n# print(f\"Identified {len(regime_info)} market regimes\")\n\n# # Create regime features\n# regime_probs_train = regime_model.transform(X_train_scaled[:, :20])\n# regime_probs_test = regime_model.transform(X_test_scaled[:, :20])\n\n# # Distance to each regime center\n# regime_features_train = -regime_probs_train  # Negative distance = probability proxy\n# regime_features_test = -regime_probs_test\n\n# # ==================== Causal Clustering ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"CAUSAL CLUSTERING\")\n# print(\"=\"*60)\n\n# # Create causal clusters\n# clusters, feature_effects = create_causal_clusters(X_sample, y_sample, n_clusters=30)\n# print(f\"Created {len(np.unique(clusters))} causal clusters\")\n\n# # Select medoids\n# medoids = select_cluster_medoids(X_sample, y_sample, clusters)\n# print(f\"Selected {len(medoids)} medoid features\")\n\n# # Extract medoid features\n# X_train_medoids = X_train_scaled[:, medoids]\n# X_test_medoids = X_test_scaled[:, medoids]\n\n# # ==================== Interaction Network ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"INTERACTION NETWORK\")\n# print(\"=\"*60)\n\n# # Create interaction network\n# interactions = create_interaction_network(X_sample, medoids, y_sample)\n# print(f\"Found {len(interactions)} key interactions\")\n\n# # Create interaction features\n# X_train_interactions = []\n# X_test_interactions = []\n\n# for idx1, idx2, score in interactions:\n#     interaction_train = X_train_medoids[:, medoids.index(idx1)] * X_train_medoids[:, medoids.index(idx2)]\n#     interaction_test = X_test_medoids[:, medoids.index(idx1)] * X_test_medoids[:, medoids.index(idx2)]\n    \n#     X_train_interactions.append(interaction_train * np.sqrt(score))  # Weight by importance\n#     X_test_interactions.append(interaction_test * np.sqrt(score))\n\n# if X_train_interactions:\n#     X_train_interactions = np.column_stack(X_train_interactions)\n#     X_test_interactions = np.column_stack(X_test_interactions)\n# else:\n#     X_train_interactions = np.zeros((len(X_train_medoids), 1))\n#     X_test_interactions = np.zeros((len(X_test_medoids), 1))\n\n# # ==================== Final Assembly ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"FINAL FEATURE ASSEMBLY\")\n# print(\"=\"*60)\n\n# # Hierarchical feature assembly\n# X_train_final = np.hstack([\n#     X_train_medoids,           # Core causal features\n#     regime_features_train,      # Market regime\n#     X_train_interactions,       # Causal interactions\n# ])\n\n# X_test_final = np.hstack([\n#     X_test_medoids,\n#     regime_features_test,\n#     X_test_interactions,\n# ])\n\n# print(f\"Final features: {X_train_final.shape[1]}\")\n\n# # ==================== Train Models ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"TRAINING ENSEMBLE\")\n# print(\"=\"*60)\n\n# train_size = min(50000, len(X_train_final))\n# train_idx = np.random.choice(len(X_train_final), train_size, replace=False)\n\n# models = {}\n# predictions = {}\n\n# # Model 1: GradientBoosting\n# print(\"\\n1. GradientBoosting...\")\n# gb = GradientBoostingRegressor(\n#     n_estimators=200,\n#     max_depth=5,\n#     learning_rate=0.05,\n#     subsample=0.8,\n#     min_samples_leaf=30,\n#     random_state=42\n# )\n# gb.fit(X_train_final[train_idx], y_train[train_idx])\n# models['gb'] = gb\n# predictions['gb'] = gb.predict(X_test_final)\n\n# # Model 2: RandomForest\n# print(\"2. RandomForest...\")\n# rf = RandomForestRegressor(\n#     n_estimators=150,\n#     max_depth=7,\n#     min_samples_leaf=50,\n#     random_state=42,\n#     n_jobs=-1\n# )\n# rf.fit(X_train_final[train_idx], y_train[train_idx])\n# models['rf'] = rf\n# predictions['rf'] = rf.predict(X_test_final)\n\n# # Model 3: Ridge\n# print(\"3. Ridge...\")\n# ridge = Ridge(alpha=10.0, random_state=42)\n# ridge.fit(X_train_final[train_idx], y_train[train_idx])\n# models['ridge'] = ridge\n# predictions['ridge'] = ridge.predict(X_test_final)\n\n# # ==================== Validation ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"VALIDATION\")\n# print(\"=\"*60)\n\n# val_size = 5000\n# val_idx = np.random.choice(len(X_train_final), val_size, replace=False)\n\n# val_scores = {}\n# for name, model in models.items():\n#     val_pred = model.predict(X_train_final[val_idx])\n#     score = np.corrcoef(y_train[val_idx], val_pred)[0, 1]\n#     val_scores[name] = score\n#     print(f\"{name}: {score:.4f}\")\n\n# # Weighted ensemble\n# weights = np.array([val_scores[name] for name in predictions.keys()])\n# weights = np.maximum(weights, 0)\n# weights = weights / weights.sum()\n\n# print(f\"\\nWeights: {dict(zip(predictions.keys(), weights))}\")\n\n# final_predictions = sum(weights[i] * predictions[name] \n#                        for i, name in enumerate(predictions.keys()))\n\n# # ==================== Save ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"SAVING\")\n# print(\"=\"*60)\n\n# sample_sub = pd.read_csv(SAMPLE_SUB_PATH)\n# if len(final_predictions) != len(sample_sub):\n#     final_predictions = final_predictions[:len(sample_sub)]\n\n# submission = pd.DataFrame({\n#     sample_sub.columns[0]: sample_sub.iloc[:, 0],\n#     'prediction': final_predictions\n# })\n\n# submission.to_csv(f'{OUTPUT_PATH}submission.csv', index=False)\n# print(f\"✓ Saved to {OUTPUT_PATH}submission.csv\")\n\n# print(\"\\n✓ Complete!\")\n# print(\"=\"*60)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # ==================== Hierarchical Information Bottleneck System ====================\n# import pandas as pd\n# import numpy as np\n# import os\n# from sklearn.ensemble import GradientBoostingRegressor, RandomForestRegressor\n# from sklearn.linear_model import Ridge\n# from sklearn.preprocessing import RobustScaler\n# from sklearn.decomposition import PCA\n# from sklearn.feature_selection import mutual_info_regression\n# from scipy.cluster.hierarchy import linkage, fcluster\n# from scipy.spatial.distance import squareform\n# import joblib\n# import warnings\n# warnings.filterwarnings('ignore')\n\n# # ==================== Configuration ====================\n# DATA_PATH = '/kaggle/input/drw-remix-of-winning-strategies-part-1/'\n# OUTPUT_PATH = '/kaggle/working/model_11/'\n# SAMPLE_SUB_PATH = '/kaggle/input/drw-crypto-market-prediction/sample_submission.csv'\n\n# os.makedirs(OUTPUT_PATH, exist_ok=True)\n# print(f\"Output directory: {OUTPUT_PATH}\")\n\n# # ==================== Load Data ====================\n# print(\"=\"*60)\n# print(\"HIERARCHICAL INFORMATION BOTTLENECK SYSTEM\")\n# print(\"=\"*60)\n\n# train_df = pd.read_parquet(f'{DATA_PATH}train_final.parquet')\n# test_df = pd.read_parquet(f'{DATA_PATH}test_final.parquet')\n\n# print(f\"Train shape: {train_df.shape}\")\n# print(f\"Test shape: {test_df.shape}\")\n\n# # ==================== Create Market Microstructure Features ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"MARKET MICROSTRUCTURE FEATURES\")\n# print(\"=\"*60)\n\n# def create_market_features(df):\n#     \"\"\"Create domain-specific features\"\"\"\n#     features = {}\n    \n#     # Order Book Depth\n#     if 'bid_qty' in df.columns and 'ask_qty' in df.columns:\n#         features['total_depth'] = df['bid_qty'] + df['ask_qty']\n#         features['depth_imbalance'] = (df['bid_qty'] - df['ask_qty']) / (features['total_depth'] + 1e-10)\n#         features['depth_ratio'] = df['bid_qty'] / (df['ask_qty'] + 1e-10)\n    \n#     # Order Flow\n#     if 'buy_qty' in df.columns and 'sell_qty' in df.columns:\n#         features['net_flow'] = df['buy_qty'] - df['sell_qty']\n#         features['flow_ratio'] = df['buy_qty'] / (df['buy_qty'] + df['sell_qty'] + 1e-10)\n    \n#     # Volume dynamics\n#     if 'volume' in df.columns:\n#         features['volume_ma_ratio'] = df['volume'] / (df['volume'].rolling(10, min_periods=1).mean() + 1e-10)\n        \n#     # Microstructure\n#     if all(col in df.columns for col in ['bid_qty', 'ask_qty', 'volume']):\n#         features['spread_proxy'] = (df['ask_qty'] - df['bid_qty']) / (df['volume'] + 1)\n#         features['liquidity'] = (df['bid_qty'] + df['ask_qty']) / (df['volume'] + 1)\n        \n#     return pd.DataFrame(features)\n\n# market_train = create_market_features(train_df)\n# market_test = create_market_features(test_df)\n# print(f\"Created {market_train.shape[1]} market features\")\n\n# # Extract features and target\n# feature_cols = [col for col in train_df.columns if col not in ['label', 'timestamp', 'id']]\n# y_train = train_df['label'].values\n\n# # Combine features\n# X_train = np.hstack([train_df[feature_cols].values, market_train.values])\n# X_test = np.hstack([test_df[feature_cols].values, market_test.values])\n\n# # Clean data\n# X_train = np.nan_to_num(X_train, nan=0.0, posinf=0.0, neginf=0.0)\n# X_test = np.nan_to_num(X_test, nan=0.0, posinf=0.0, neginf=0.0)\n\n# print(f\"Total features: {X_train.shape[1]}\")\n\n# # ==================== Preprocessing ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"PREPROCESSING\")\n# print(\"=\"*60)\n\n# # Robust scaling\n# scaler = RobustScaler()\n# X_train_scaled = scaler.fit_transform(X_train)\n# X_test_scaled = scaler.transform(X_test)\n\n# # Remove constant features\n# variances = np.var(X_train_scaled, axis=0)\n# non_constant = variances > 1e-10\n# X_train_scaled = X_train_scaled[:, non_constant]\n# X_test_scaled = X_test_scaled[:, non_constant]\n\n# print(f\"Non-constant features: {X_train_scaled.shape[1]}\")\n\n# # Initial PCA if too many features\n# if X_train_scaled.shape[1] > 150:\n#     print(\"Initial dimensionality reduction...\")\n#     pca_init = PCA(n_components=150, random_state=42)\n#     X_train_scaled = pca_init.fit_transform(X_train_scaled)\n#     X_test_scaled = pca_init.transform(X_test_scaled)\n#     print(f\"Reduced to {X_train_scaled.shape[1]} dimensions\")\n\n# # ==================== Build Information Distance Matrix ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"INFORMATION HIERARCHY\")\n# print(\"=\"*60)\n\n# # Sample for efficiency\n# sample_size = min(5000, len(X_train_scaled))\n# sample_idx = np.random.choice(len(X_train_scaled), sample_size, replace=False)\n# X_sample = X_train_scaled[sample_idx]\n# y_sample = y_train[sample_idx]\n\n# print(\"Building information distance matrix...\")\n# n_features = X_sample.shape[1]\n# dist_matrix = np.zeros((n_features, n_features))\n\n# for i in range(n_features):\n#     if i % 50 == 0:\n#         print(f\"  Processing feature {i}/{n_features}\")\n    \n#     for j in range(i+1, n_features):\n#         # Correlation\n#         corr = abs(np.corrcoef(X_sample[:, i], X_sample[:, j])[0, 1])\n        \n#         # Residual information\n#         from sklearn.linear_model import LinearRegression\n#         lr = LinearRegression()\n#         lr.fit(X_sample[:, i].reshape(-1, 1), X_sample[:, j])\n#         residual_var = np.var(X_sample[:, j] - lr.predict(X_sample[:, i].reshape(-1, 1)))\n#         unique_info = residual_var / (np.var(X_sample[:, j]) + 1e-10)\n        \n#         # Distance = low correlation + high unique information\n#         dist_matrix[i, j] = (1 - corr) * np.sqrt(unique_info)\n#         dist_matrix[j, i] = dist_matrix[i, j]\n\n# # ==================== Hierarchical Clustering ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"HIERARCHICAL CLUSTERING\")\n# print(\"=\"*60)\n\n# # Convert to condensed form\n# condensed = squareform(dist_matrix)\n\n# # Hierarchical clustering\n# print(\"Performing hierarchical clustering...\")\n# Z = linkage(condensed, method='ward')\n\n# # Multiple clustering levels\n# n_clusters_levels = [60, 40, 25, 15]\n# selected_features_all = []\n\n# for level, n_clusters in enumerate(n_clusters_levels):\n#     print(f\"\\nLevel {level+1}: {n_clusters} clusters\")\n    \n#     # Get clusters\n#     clusters = fcluster(Z, min(n_clusters, n_features), criterion='maxclust')\n    \n#     # Select medoids based on predictive power\n#     medoids = []\n#     unique_clusters = np.unique(clusters)\n    \n#     for cluster_id in unique_clusters:\n#         cluster_mask = clusters == cluster_id\n#         cluster_indices = np.where(cluster_mask)[0]\n        \n#         if len(cluster_indices) == 1:\n#             medoids.append(cluster_indices[0])\n#         else:\n#             # Find best feature in cluster\n#             best_score = -np.inf\n#             best_idx = cluster_indices[0]\n            \n#             for idx in cluster_indices[:min(10, len(cluster_indices))]:  # Limit search\n#                 # Predictive power\n#                 lr = LinearRegression()\n#                 lr.fit(X_sample[:, idx].reshape(-1, 1), y_sample)\n#                 score = lr.score(X_sample[:, idx].reshape(-1, 1), y_sample)\n                \n#                 if score > best_score:\n#                     best_score = score\n#                     best_idx = idx\n            \n#             medoids.append(best_idx)\n    \n#     selected_features_all.append(medoids)\n#     print(f\"  Selected {len(medoids)} medoid features\")\n\n# # ==================== Feature Selection ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"FINAL FEATURE SELECTION\")\n# print(\"=\"*60)\n\n# # Use second-to-last level as primary\n# if len(selected_features_all) >= 2:\n#     primary_features = selected_features_all[-2][:30]  # Top 30 from second-to-last\n#     secondary_features = selected_features_all[-3][:10] if len(selected_features_all) >= 3 else []\n# else:\n#     primary_features = selected_features_all[0][:30]\n#     secondary_features = []\n\n# print(f\"Primary features: {len(primary_features)}\")\n# print(f\"Secondary features: {len(secondary_features)}\")\n\n# # Combine features\n# all_selected = list(set(primary_features + secondary_features))\n# X_train_selected = X_train_scaled[:, all_selected]\n# X_test_selected = X_test_scaled[:, all_selected]\n\n# # ==================== Add Mutual Information Features ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"MUTUAL INFORMATION FEATURES\")\n# print(\"=\"*60)\n\n# # Calculate MI scores for selected features\n# mi_scores = mutual_info_regression(X_sample[:, all_selected], y_sample, random_state=42)\n# top_mi_indices = np.argsort(mi_scores)[-10:]\n\n# # Create interactions for top MI features\n# X_train_interactions = []\n# X_test_interactions = []\n\n# for i in range(min(3, len(top_mi_indices))):\n#     for j in range(i+1, min(3, len(top_mi_indices))):\n#         idx1, idx2 = top_mi_indices[i], top_mi_indices[j]\n#         interaction_train = X_train_selected[:, idx1] * X_train_selected[:, idx2]\n#         interaction_test = X_test_selected[:, idx1] * X_test_selected[:, idx2]\n#         X_train_interactions.append(interaction_train)\n#         X_test_interactions.append(interaction_test)\n\n# if X_train_interactions:\n#     X_train_interactions = np.column_stack(X_train_interactions)\n#     X_test_interactions = np.column_stack(X_test_interactions)\n#     X_train_final = np.hstack([X_train_selected, X_train_interactions])\n#     X_test_final = np.hstack([X_test_selected, X_test_interactions])\n#     print(f\"Added {X_train_interactions.shape[1]} interaction features\")\n# else:\n#     X_train_final = X_train_selected\n#     X_test_final = X_test_selected\n\n# print(f\"Final feature dimension: {X_train_final.shape[1]}\")\n\n# # ==================== Model Training ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"TRAINING MODELS\")\n# print(\"=\"*60)\n\n# train_size = min(50000, len(X_train_final))\n# train_idx = np.random.choice(len(X_train_final), train_size, replace=False)\n\n# models = {}\n# predictions = {}\n\n# # Model 1: GradientBoosting\n# print(\"\\n1. GradientBoosting...\")\n# gb = GradientBoostingRegressor(\n#     n_estimators=200,\n#     max_depth=5,\n#     learning_rate=0.05,\n#     subsample=0.8,\n#     min_samples_leaf=30,\n#     random_state=42\n# )\n# gb.fit(X_train_final[train_idx], y_train[train_idx])\n# models['gb'] = gb\n# predictions['gb'] = gb.predict(X_test_final)\n\n# # Model 2: RandomForest\n# print(\"2. RandomForest...\")\n# rf = RandomForestRegressor(\n#     n_estimators=150,\n#     max_depth=7,\n#     min_samples_leaf=50,\n#     random_state=42,\n#     n_jobs=-1\n# )\n# rf.fit(X_train_final[train_idx], y_train[train_idx])\n# models['rf'] = rf\n# predictions['rf'] = rf.predict(X_test_final)\n\n# # Model 3: Ridge\n# print(\"3. Ridge...\")\n# ridge = Ridge(alpha=10.0, random_state=42)\n# ridge.fit(X_train_final[train_idx], y_train[train_idx])\n# models['ridge'] = ridge\n# predictions['ridge'] = ridge.predict(X_test_final)\n\n# # ==================== Validation & Ensemble ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"VALIDATION\")\n# print(\"=\"*60)\n\n# val_size = 5000\n# val_idx = np.random.choice(len(X_train_final), val_size, replace=False)\n\n# val_scores = {}\n# for name, model in models.items():\n#     val_pred = model.predict(X_train_final[val_idx])\n#     score = np.corrcoef(y_train[val_idx], val_pred)[0, 1]\n#     val_scores[name] = score\n#     print(f\"{name}: {score:.4f}\")\n\n# # Calculate weights\n# weights = np.array([val_scores[name] for name in predictions.keys()])\n# weights = np.maximum(weights, 0)\n# weights = weights / weights.sum()\n\n# print(f\"\\nWeights: {dict(zip(predictions.keys(), weights))}\")\n\n# # Final predictions\n# final_predictions = sum(weights[i] * predictions[name] \n#                        for i, name in enumerate(predictions.keys()))\n\n# # ==================== Save Results ====================\n# print(\"\\n\" + \"=\"*60)\n# print(\"SAVING RESULTS\")\n# print(\"=\"*60)\n\n# sample_sub = pd.read_csv(SAMPLE_SUB_PATH)\n\n# if len(final_predictions) != len(sample_sub):\n#     final_predictions = final_predictions[:len(sample_sub)]\n\n# submission = pd.DataFrame({\n#     sample_sub.columns[0]: sample_sub.iloc[:, 0],\n#     'prediction': final_predictions\n# })\n\n# submission.to_csv(f'{OUTPUT_PATH}submission.csv', index=False)\n# print(f\"✓ Saved to {OUTPUT_PATH}submission.csv\")\n\n# # Save models and preprocessors\n# joblib.dump(models, f'{OUTPUT_PATH}models.pkl')\n# joblib.dump({'scaler': scaler}, f'{OUTPUT_PATH}preprocessors.pkl')\n# joblib.dump(all_selected, f'{OUTPUT_PATH}selected_features.pkl')\n\n# print(\"\\n✓ Complete! All files saved to\", OUTPUT_PATH)\n# print(\"=\"*60)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==================== Advanced Genetic Information Bottleneck System (FIXED) ====================\nimport pandas as pd\nimport numpy as np\nimport os\nfrom sklearn.ensemble import GradientBoostingRegressor, RandomForestRegressor, ExtraTreesRegressor\nfrom sklearn.linear_model import Ridge, ElasticNet\nfrom sklearn.preprocessing import RobustScaler, StandardScaler\nfrom sklearn.decomposition import PCA, IncrementalPCA, FastICA\nfrom sklearn.feature_selection import mutual_info_regression\nfrom scipy.cluster.hierarchy import linkage, fcluster\nfrom scipy.spatial.distance import squareform, pdist\nimport joblib\nimport gc\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# ==================== Configuration ====================\nDATA_PATH = '/kaggle/input/drw-remix-of-winning-strategies-part-1/'\nOUTPUT_PATH = '/kaggle/working/model_11/'\nSAMPLE_SUB_PATH = '/kaggle/input/drw-crypto-market-prediction/sample_submission.csv'\n\nos.makedirs(OUTPUT_PATH, exist_ok=True)\nprint(f\"Output directory: {OUTPUT_PATH}\")\n\n# ==================== Fixed Incremental Processor ====================\nclass IncrementalInfoProcessor:\n    \"\"\"Process all data without sampling - prevents information loss\"\"\"\n    \n    def __init__(self, batch_size=10000):\n        self.batch_size = batch_size\n        \n    def incremental_mutual_info(self, X, y):\n        \"\"\"Calculate MI scores incrementally on full dataset\"\"\"\n        n_features = X.shape[1]\n        mi_scores = np.zeros(n_features)\n        \n        # Process in batches\n        n_batches = (len(X) + self.batch_size - 1) // self.batch_size\n        \n        for batch_idx in range(n_batches):\n            start_idx = batch_idx * self.batch_size\n            end_idx = min(start_idx + self.batch_size, len(X))\n            \n            X_batch = X[start_idx:end_idx]\n            y_batch = y[start_idx:end_idx]\n            \n            # Calculate MI for this batch\n            batch_mi = mutual_info_regression(X_batch, y_batch, random_state=42)\n            \n            # Incremental average\n            mi_scores = (mi_scores * batch_idx + batch_mi) / (batch_idx + 1)\n            \n            if batch_idx % 10 == 0:\n                print(f\"    Processed batch {batch_idx+1}/{n_batches}\")\n                \n        return mi_scores\n    \n    def compute_correlation_batch(self, X, sample_size=30000):\n        \"\"\"Compute correlation on a large sample instead of incrementally\"\"\"\n        # Use a large sample for correlation calculation\n        if len(X) > sample_size:\n            idx = np.random.choice(len(X), sample_size, replace=False)\n            X_sample = X[idx]\n        else:\n            X_sample = X\n        \n        # Standard correlation calculation\n        corr_matrix = np.corrcoef(X_sample.T)\n        \n        # Ensure symmetry\n        corr_matrix = (corr_matrix + corr_matrix.T) / 2\n        np.fill_diagonal(corr_matrix, 1.0)\n        \n        return corr_matrix\n\n# ==================== Simplified Genetic Algorithm ====================\nclass GeneticClusterOptimizer:\n    \"\"\"Find optimal clustering parameters\"\"\"\n    \n    def __init__(self, population_size=15, generations=5):\n        self.population_size = population_size\n        self.generations = generations\n        self.fitness_history = []\n        \n    def create_individual(self):\n        \"\"\"Create random configuration\"\"\"\n        return {\n            'n_clusters': np.random.randint(15, 40),\n            'method': np.random.choice(['ward', 'average']),\n            'pca_components': np.random.randint(40, 80),\n            'use_ica': False  # Simplified - ICA is unstable\n        }\n    \n    def fitness(self, individual, X_sample, y_sample):\n        \"\"\"Evaluate configuration\"\"\"\n        try:\n            n_features = min(individual['pca_components'], X_sample.shape[1])\n            \n            # PCA reduction\n            reducer = PCA(n_components=n_features, random_state=42)\n            X_reduced = reducer.fit_transform(X_sample)\n            \n            # Simple clustering quality\n            from sklearn.linear_model import Ridge\n            ridge = Ridge(alpha=1.0)\n            ridge.fit(X_reduced, y_sample)\n            score = ridge.score(X_reduced, y_sample)\n            \n            return score\n        except:\n            return -1\n    \n    def evolve(self, X_sample, y_sample):\n        \"\"\"Run simplified genetic algorithm\"\"\"\n        print(\"Running genetic optimization...\")\n        \n        # Create initial population\n        population = [self.create_individual() for _ in range(self.population_size)]\n        best_fitness = -1\n        best_individual = population[0]\n        \n        for generation in range(self.generations):\n            print(f\"  Generation {generation + 1}/{self.generations}\")\n            \n            # Evaluate fitness\n            fitness_scores = []\n            for individual in population:\n                fitness = self.fitness(individual, X_sample, y_sample)\n                fitness_scores.append(fitness)\n            \n            # Find best\n            best_idx = np.argmax(fitness_scores)\n            if fitness_scores[best_idx] > best_fitness:\n                best_fitness = fitness_scores[best_idx]\n                best_individual = population[best_idx]\n            \n            print(f\"    Best fitness: {best_fitness:.4f}\")\n            self.fitness_history.append(best_fitness)\n            \n            # Create new population (simplified)\n            new_population = [population[best_idx]]  # Keep best\n            while len(new_population) < self.population_size:\n                new_population.append(self.create_individual())\n            \n            population = new_population\n        \n        return best_individual\n\n# ==================== Main Pipeline ====================\nprint(\"=\"*60)\nprint(\"GENETIC INFORMATION BOTTLENECK (FIXED)\")\nprint(\"=\"*60)\n\n# Load data\ntrain_df = pd.read_parquet(f'{DATA_PATH}train_final.parquet')\ntest_df = pd.read_parquet(f'{DATA_PATH}test_final.parquet')\nprint(f\"Train shape: {train_df.shape}\")\nprint(f\"Test shape: {test_df.shape}\")\n\n# ==================== Feature Engineering ====================\nprint(\"\\n\" + \"=\"*60)\nprint(\"FEATURE ENGINEERING\")\nprint(\"=\"*60)\n\ndef create_market_features(df):\n    \"\"\"Create market microstructure features\"\"\"\n    features = {}\n    \n    if 'bid_qty' in df.columns and 'ask_qty' in df.columns:\n        features['depth_total'] = df['bid_qty'] + df['ask_qty']\n        features['depth_imbalance'] = (df['bid_qty'] - df['ask_qty']) / (features['depth_total'] + 1e-10)\n        \n    if 'buy_qty' in df.columns and 'sell_qty' in df.columns:\n        features['flow_net'] = df['buy_qty'] - df['sell_qty']\n        features['flow_ratio'] = df['buy_qty'] / (df['buy_qty'] + df['sell_qty'] + 1e-10)\n    \n    if 'volume' in df.columns:\n        features['vol_ma10'] = df['volume'].rolling(10, min_periods=1).mean()\n        features['vol_ratio'] = df['volume'] / (features['vol_ma10'] + 1e-10)\n        \n    if all(col in df.columns for col in ['bid_qty', 'ask_qty', 'volume']):\n        features['spread_proxy'] = (df['ask_qty'] - df['bid_qty']) / (df['volume'] + 1)\n        features['liquidity'] = (df['bid_qty'] + df['ask_qty']) / (df['volume'] + 1)\n        \n    return pd.DataFrame(features)\n\nmarket_train = create_market_features(train_df)\nmarket_test = create_market_features(test_df)\nprint(f\"Created {market_train.shape[1]} market features\")\n\n# Combine features\nfeature_cols = [col for col in train_df.columns if col not in ['label', 'timestamp', 'id']]\ny_train = train_df['label'].values\n\nX_train = np.hstack([train_df[feature_cols].values, market_train.values])\nX_test = np.hstack([test_df[feature_cols].values, market_test.values])\n\n# Clean\nX_train = np.nan_to_num(X_train, nan=0.0, posinf=0.0, neginf=0.0)\nX_test = np.nan_to_num(X_test, nan=0.0, posinf=0.0, neginf=0.0)\nprint(f\"Total features: {X_train.shape[1]}\")\n\n# ==================== Robust Scaling ====================\nprint(\"\\n\" + \"=\"*60)\nprint(\"ROBUST SCALING\")\nprint(\"=\"*60)\n\nscaler = RobustScaler()\nX_train_scaled = scaler.fit_transform(X_train)\nX_test_scaled = scaler.transform(X_test)\n\n# Remove constant features\nvariances = np.var(X_train_scaled, axis=0)\nnon_constant = variances > 1e-10\nX_train_scaled = X_train_scaled[:, non_constant]\nX_test_scaled = X_test_scaled[:, non_constant]\nprint(f\"Non-constant features: {X_train_scaled.shape[1]}\")\n\n# ==================== Incremental PCA ====================\nprint(\"\\n\" + \"=\"*60)\nprint(\"INCREMENTAL PCA (FULL DATASET)\")\nprint(\"=\"*60)\n\nn_components = min(150, X_train_scaled.shape[1])\nipca = IncrementalPCA(n_components=n_components, batch_size=5000)\n\n# Fit incrementally\nn_batches = (len(X_train_scaled) + 4999) // 5000\nfor batch_idx in range(n_batches):\n    start_idx = batch_idx * 5000\n    end_idx = min(start_idx + 5000, len(X_train_scaled))\n    X_batch = X_train_scaled[start_idx:end_idx]\n    ipca.partial_fit(X_batch)\n    \n    if batch_idx % 20 == 0:\n        print(f\"  Batch {batch_idx+1}/{n_batches}\")\n\nX_train_pca = ipca.transform(X_train_scaled)\nX_test_pca = ipca.transform(X_test_scaled)\nprint(f\"PCA variance: {ipca.explained_variance_ratio_.sum():.3f}\")\n\n# ==================== Genetic Optimization ====================\nprint(\"\\n\" + \"=\"*60)\nprint(\"GENETIC OPTIMIZATION\")\nprint(\"=\"*60)\n\n# Sample for optimization\nsample_size = min(15000, len(X_train_pca))\nsample_idx = np.random.choice(len(X_train_pca), sample_size, replace=False)\nX_sample = X_train_pca[sample_idx]\ny_sample = y_train[sample_idx]\n\ngenetic_optimizer = GeneticClusterOptimizer(population_size=10, generations=4)\nbest_config = genetic_optimizer.evolve(X_sample, y_sample)\n\nprint(f\"\\nBest configuration:\")\nfor key, value in best_config.items():\n    print(f\"  {key}: {value}\")\n\n# ==================== Apply Configuration ====================\nprint(\"\\n\" + \"=\"*60)\nprint(\"APPLYING CONFIGURATION\")\nprint(\"=\"*60)\n\nn_features = min(best_config['pca_components'], X_train_pca.shape[1])\nfinal_pca = PCA(n_components=n_features, random_state=42)\nX_train_reduced = final_pca.fit_transform(X_train_pca)\nX_test_reduced = final_pca.transform(X_test_pca)\n\nprint(f\"Reduced to {X_train_reduced.shape[1]} features\")\n\n# ==================== Clustering ====================\nprint(\"\\n\" + \"=\"*60)\nprint(\"HIERARCHICAL CLUSTERING\")\nprint(\"=\"*60)\n\n# Calculate MI scores on full data\nprocessor = IncrementalInfoProcessor(batch_size=10000)\nprint(\"Calculating mutual information...\")\nmi_scores = processor.incremental_mutual_info(X_train_reduced, y_train)\n\n# Use sample for correlation (more stable)\nprint(\"Computing correlation matrix...\")\ncorr_matrix = processor.compute_correlation_batch(X_train_reduced, sample_size=30000)\n\n# Create distance matrix\ndist_matrix = 1 - np.abs(corr_matrix)\nnp.fill_diagonal(dist_matrix, 0)\n\n# Ensure it's valid for clustering\ndist_matrix = np.maximum(dist_matrix, 0)\ndist_matrix = (dist_matrix + dist_matrix.T) / 2\n\n# Perform clustering\nprint(\"Performing clustering...\")\ncondensed = squareform(dist_matrix, checks=False)  # Skip checks since we ensured validity\nZ = linkage(condensed, method=best_config['method'])\nclusters = fcluster(Z, best_config['n_clusters'], criterion='maxclust')\n\nprint(f\"Created {len(np.unique(clusters))} clusters\")\n\n# ==================== Select Representatives ====================\nprint(\"\\n\" + \"=\"*60)\nprint(\"SELECTING REPRESENTATIVES\")\nprint(\"=\"*60)\n\nrepresentatives = []\nfor cluster_id in np.unique(clusters):\n    cluster_indices = np.where(clusters == cluster_id)[0]\n    \n    # Select top 2 by MI score\n    cluster_mi = mi_scores[cluster_indices]\n    n_select = min(2, len(cluster_indices))\n    top_indices = cluster_indices[np.argsort(cluster_mi)[-n_select:]]\n    representatives.extend(top_indices)\n\nprint(f\"Selected {len(representatives)} representatives\")\n\n# Additional top MI features\ntop_mi = np.argsort(mi_scores)[-20:]\nall_features = list(set(representatives) | set(top_mi))\n\nX_train_final = X_train_reduced[:, all_features]\nX_test_final = X_test_reduced[:, all_features]\nprint(f\"Final features: {X_train_final.shape[1]}\")\n\n# ==================== Train Models ====================\nprint(\"\\n\" + \"=\"*60)\nprint(\"TRAINING ENSEMBLE\")\nprint(\"=\"*60)\n\nmodels = {}\npredictions = {}\n\n# Model 1: GradientBoosting\nprint(\"1. GradientBoosting...\")\ngb = GradientBoostingRegressor(\n    n_estimators=200,\n    max_depth=5,\n    learning_rate=0.05,\n    subsample=0.8,\n    min_samples_leaf=30,\n    random_state=42\n)\ngb.fit(X_train_final, y_train)\npredictions['gb'] = gb.predict(X_test_final)\n\n# Model 2: RandomForest\nprint(\"2. RandomForest...\")\nrf = RandomForestRegressor(\n    n_estimators=150,\n    max_depth=7,\n    min_samples_leaf=50,\n    random_state=42,\n    n_jobs=-1\n)\nrf.fit(X_train_final, y_train)\npredictions['rf'] = rf.predict(X_test_final)\n\n# Model 3: Ridge\nprint(\"3. Ridge...\")\nridge = Ridge(alpha=10.0, random_state=42)\nridge.fit(X_train_final, y_train)\npredictions['ridge'] = ridge.predict(X_test_final)\n\n# ==================== Ensemble ====================\nprint(\"\\n\" + \"=\"*60)\nprint(\"CREATING ENSEMBLE\")\nprint(\"=\"*60)\n\n# Validation\nval_size = 5000\nval_idx = np.random.choice(len(X_train_final), val_size, replace=False)\n\nval_scores = {}\nfor name, pred_func in [('gb', gb), ('rf', rf), ('ridge', ridge)]:\n    val_pred = pred_func.predict(X_train_final[val_idx])\n    score = np.corrcoef(y_train[val_idx], val_pred)[0, 1]\n    val_scores[name] = score\n    print(f\"{name}: {score:.4f}\")\n\n# Weights\nweights = np.array([val_scores[name] for name in predictions.keys()])\nweights = np.maximum(weights, 0)\nweights = weights / weights.sum()\n\nprint(f\"\\nWeights: {dict(zip(predictions.keys(), weights))}\")\n\nfinal_predictions = sum(weights[i] * predictions[name] \n                       for i, name in enumerate(predictions.keys()))\n\n# ==================== Save ====================\nprint(\"\\n\" + \"=\"*60)\nprint(\"SAVING RESULTS\")\nprint(\"=\"*60)\n\nsample_sub = pd.read_csv(SAMPLE_SUB_PATH)\nif len(final_predictions) != len(sample_sub):\n    final_predictions = final_predictions[:len(sample_sub)]\n\nsubmission = pd.DataFrame({\n    sample_sub.columns[0]: sample_sub.iloc[:, 0],\n    'prediction': final_predictions\n})\n\nsubmission.to_csv(f'{OUTPUT_PATH}submission.csv', index=False)\nprint(f\"✓ Saved to {OUTPUT_PATH}submission.csv\")\n\n# Save models\njoblib.dump({'gb': gb, 'rf': rf, 'ridge': ridge}, f'{OUTPUT_PATH}models.pkl')\njoblib.dump({'scaler': scaler, 'ipca': ipca, 'final_pca': final_pca}, f'{OUTPUT_PATH}preprocessors.pkl')\njoblib.dump(best_config, f'{OUTPUT_PATH}best_config.pkl')\n\nprint(f\"\\n✓ Complete! Processed {len(X_train)} samples\")\nprint(\"=\"*60)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}