{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install flaml ray\nimport random\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.feature_selection import SelectKBest, f_regression\nfrom sklearn.decomposition import PCA\nfrom sklearn.ensemble import RandomForestRegressor\nfrom tqdm import tqdm\nimport warnings\nimport os\nimport time\nimport cupy as cp\nfrom typing import List, Dict, Tuple, Set, Optional\nfrom dataclasses import dataclass, field\nfrom collections import defaultdict\nimport gc\n\nwarnings.filterwarnings(\"ignore\", category=RuntimeWarning)\n\n# ============================================\n# CONFIGURATION SETTINGS\n# ============================================\n\nclass PipelineConfig:\n    \"\"\"\n    Central configuration for the DRW Crypto Prediction Pipeline.\n    All parameters are organized by pipeline stage for easy adjustment.\n    Optimized for memory efficiency while maintaining model quality.\n    \"\"\"\n    \n    # ====================\n    # DATA SAMPLING CONFIGURATION\n    # ====================\n    # Controls how much data is used in different pipeline stages\n    \n    TRAIN_DATA_PERCENTAGE = 40  # Increased from 20% since we're reducing features\n    TRAIN_DATA_RECENT_ONLY = True  # If True, uses most recent data; if False, uses random sampling\n    INTERACTION_DISCOVERY_SAMPLE_SIZE = 50000  # Reduced from 100000\n    \n    # ====================\n    # FEATURE INTERACTION DISCOVERY\n    # ====================\n    # Parameters for the AI-powered feature interaction search\n    \n    # Search Algorithm Settings\n    SEARCH_N_PHASES = 3  # Reduced from 6 - focus on most effective strategies\n    SEARCH_PAIRS_PER_PHASE = 1000  # Reduced from 5000\n    SEARCH_TOP_FEATURES = 100  # Reduced from 300\n    SEARCH_TOP_INTERACTIONS = 300  # Reduced from 2000\n    \n    # Sampling Configuration for Search\n    SEARCH_INITIAL_SAMPLE_SIZE = 1000  # Reduced from 2000\n    SEARCH_MAX_SAMPLE_SIZE = 30000  # Reduced from 100000\n    \n    # Search Quality Thresholds\n    SEARCH_CORRELATION_THRESHOLD = 0.03  # Increased from 0.02 for higher quality\n    SEARCH_MIN_CONFIDENCE = 0.85  # Slightly reduced from 0.9\n    SEARCH_EXPLORATION_BONUS = 0.2  # Unchanged\n    \n    # ====================\n    # FEATURE ENGINEERING\n    # ====================\n    # Controls feature creation and transformation\n    \n    N_INTERACTIONS_TO_CREATE = 200  # Reduced from 800\n    N_INTERACTIONS_TO_SELECT = 150  # Reduced from 600\n    N_PCA_COMPONENTS = 50  # Reduced from 150\n    \n    # ====================\n    # FINAL MODEL PREPARATION\n    # ====================\n    # Parameters for the final feature set and model training\n    \n    N_FINAL_FEATURES = 150  # Reduced from 350\n    \n    # ====================\n    # MODEL TRAINING (FLAML AutoML)\n    # ====================\n    # AutoML training configuration\n    \n    TRAINING_TIME_HOURS = 2  # Unchanged\n    TRAINING_N_CONCURRENT_TRIALS = 4  # Unchanged\n    TRAINING_METRIC = \"mse\"  # Optimization metric (do not change per requirements)\n    TRAINING_TASK = \"regression\"  # Task type (do not change per requirements)\n    TRAINING_FREE_MEM_RATIO = 0.2  # Increased from 0.1 for better memory management\n    \n    # ====================\n    # OUTPUT AND DISPLAY\n    # ====================\n    # Controls what information is displayed and saved\n    \n    # Display Settings\n    N_TOP_SYNERGIES_TO_DISPLAY = 10  # Top synergistic interactions to show\n    N_TOP_FEATURES_TO_DISPLAY = 20  # Top important features to display\n    N_TOP_PCA_TO_DISPLAY = 10  # Top PCA components to show\n    \n    # File I/O Settings\n    INTERACTIONS_FILENAME = 'feature_interactions.csv'  # Saved interaction discovery results\n    SUBMISSION_FILENAME = 'submission_optimized_pipeline.csv'  # Final predictions output\n    \n    # ====================\n    # SYSTEM CONFIGURATION\n    # ====================\n    # Technical settings for processing\n    \n    N_JOBS = -1  # Number of CPU cores to use (-1 = all available)\n    RANDOM_SEED = 42  # Random seed for reproducibility\n\n# Create config instance\nconfig = PipelineConfig()\n\n# Set random seeds using config\nseed = config.RANDOM_SEED\nnp.random.seed(seed)\nrandom.seed(seed)\n\n# ============================================\n# FEATURE INTERACTION DISCOVERY CLASSES\n# ============================================\n\n@dataclass\nclass InteractionCandidate:\n    \"\"\"Store information about a feature interaction candidate\"\"\"\n    feat1_idx: int\n    feat2_idx: int\n    score: float\n    sample_size: int\n    confidence: float\n    exploration_count: int = 1\n    interaction_type: str = 'multiply'\n    metadata: Dict = field(default_factory=dict)\n\nclass WorldClassInteractionSearch:\n    \"\"\"State-of-the-art adaptive search for feature interactions\"\"\"\n    \n    def __init__(self, \n                 initial_sample_size: int = None,\n                 max_sample_size: int = None,\n                 n_top_features: int = None,\n                 n_pairs_per_phase: int = None,\n                 n_phases: int = None,\n                 top_interactions: int = None,\n                 min_confidence: float = None,\n                 exploration_bonus: float = None,\n                 correlation_threshold: float = None,\n                 n_jobs: int = None):\n        # Use config values if not provided\n        self.initial_sample_size = initial_sample_size or config.SEARCH_INITIAL_SAMPLE_SIZE\n        self.max_sample_size = max_sample_size or config.SEARCH_MAX_SAMPLE_SIZE\n        self.n_top_features = n_top_features or config.SEARCH_TOP_FEATURES\n        self.n_pairs_per_phase = n_pairs_per_phase or config.SEARCH_PAIRS_PER_PHASE\n        self.n_phases = n_phases or config.SEARCH_N_PHASES\n        self.top_interactions = top_interactions or config.SEARCH_TOP_INTERACTIONS\n        self.min_confidence = min_confidence or config.SEARCH_MIN_CONFIDENCE\n        self.exploration_bonus = exploration_bonus or config.SEARCH_EXPLORATION_BONUS\n        self.correlation_threshold = correlation_threshold or config.SEARCH_CORRELATION_THRESHOLD\n        self.n_jobs = n_jobs or config.N_JOBS\n        \n        self.candidates = {}\n        self.feature_participation = defaultdict(int)\n        self.feature_stats = {}\n        self.interaction_cache = {}\n    \n    def search(self, X_gpu: cp.ndarray, y_gpu: cp.ndarray, \n               feature_names: List[str]) -> pd.DataFrame:\n        \"\"\"Main search method\"\"\"\n        n_samples, n_features = X_gpu.shape\n        \n        print(f\"Starting world-class adaptive search\")\n        print(f\"Features: {n_features}, Samples: {n_samples:,}\")\n        print(f\"Possible interactions: {n_features * (n_features - 1) // 2:,}\")\n        print(f\"Search depth: {self.n_phases} phases, {self.n_pairs_per_phase:,} pairs/phase\")\n        print(\"-\" * 70)\n        \n        # Preprocess\n        try:\n            X_clean, y_clean, valid_features = self._preprocess_and_cache(X_gpu, y_gpu, feature_names)\n        except Exception as e:\n            print(f\"Error in preprocessing: {e}\")\n            return pd.DataFrame()\n        \n        if len(valid_features) < 2:\n            print(\"Not enough valid features!\")\n            return pd.DataFrame()\n        \n        # Define all strategies\n        all_strategies = [\n            (\"Multi-resolution Sampling\", self._intelligent_sampling),\n            (\"Information-theoretic Discovery\", self._information_theoretic_search),\n            (\"Tree-based Importance\", self._ensemble_importance_search),\n            (\"Evolutionary Search\", self._evolutionary_search),\n            (\"Spectral Analysis\", self._spectral_search),\n            (\"Bayesian Refinement\", self._bayesian_refinement)\n        ]\n        \n        strategies_to_run = all_strategies[:self.n_phases]\n        \n        for phase, (name, method) in enumerate(strategies_to_run, 1):\n            try:\n                print(f\"\\n=== Phase {phase}/{self.n_phases}: {name} ===\")\n                start = time.time()\n                method(X_clean, y_clean, valid_features)\n                elapsed = time.time() - start\n                print(f\"Completed in {elapsed:.1f}s | Total candidates: {len(self.candidates)}\")\n            except Exception as e:\n                print(f\"Warning: {name} failed - {str(e)[:100]}\")\n                continue\n        \n        # Final validation\n        try:\n            print(\"\\n=== Final Validation ===\")\n            final_results = self._final_validation_advanced(X_clean, y_clean, valid_features)\n        except Exception as e:\n            print(f\"Error in final validation: {e}\")\n            final_results = self._emergency_results(valid_features)\n        \n        self._cleanup_memory()\n        return final_results\n    \n    def _preprocess_and_cache(self, X: cp.ndarray, y: cp.ndarray, \n                             feature_names: List[str]) -> Tuple[cp.ndarray, cp.ndarray, List[str]]:\n        \"\"\"Preprocess data and cache statistics\"\"\"\n        print(\"Preprocessing data...\")\n        \n        valid_mask = cp.ones(X.shape[1], dtype=bool)\n        \n        for i in range(X.shape[1]):\n            col = X[:, i]\n            if cp.any(cp.isnan(col)) or cp.any(cp.isinf(col)):\n                col_clean = col[~cp.isnan(col) & ~cp.isinf(col)]\n                if len(col_clean) < X.shape[0] * 0.1:\n                    valid_mask[i] = False\n                    continue\n                median_val = cp.median(col_clean)\n                col = cp.where(cp.isnan(col) | cp.isinf(col), median_val, col)\n                X[:, i] = col\n            \n            if cp.std(col) < 1e-10:\n                valid_mask[i] = False\n                continue\n        \n        valid_indices = cp.where(valid_mask)[0]\n        X_clean = X[:, valid_indices]\n        valid_feature_names = [feature_names[i] for i in cp.asnumpy(valid_indices)]\n        \n        print(f\"Valid features: {len(valid_feature_names)}/{len(feature_names)}\")\n        \n        X_mean = cp.mean(X_clean, axis=0)\n        X_std = cp.std(X_clean, axis=0) + 1e-8\n        X_normalized = (X_clean - X_mean) / X_std\n        \n        y_clean = cp.where(cp.isnan(y) | cp.isinf(y), cp.median(y[~cp.isnan(y) & ~cp.isinf(y)]), y)\n        y_mean = cp.mean(y_clean)\n        y_std = cp.std(y_clean) + 1e-8\n        y_normalized = (y_clean - y_mean) / y_std\n        \n        return X_normalized, y_normalized, valid_feature_names\n    \n    def _intelligent_sampling(self, X: cp.ndarray, y: cp.ndarray, feature_names: List[str]):\n        \"\"\"Multi-resolution sampling with importance weighting\"\"\"\n        n_samples, n_features = X.shape\n        \n        importance_scores = self._compute_feature_importance(X, y)\n        \n        sample_sizes = np.linspace(\n            self.initial_sample_size, \n            min(self.max_sample_size, n_samples // 2),\n            num=4\n        ).astype(int)\n        \n        pairs_per_level = self.n_pairs_per_phase // len(sample_sizes)\n        \n        for level, sample_size in enumerate(sample_sizes):\n            print(f\"Level {level+1}: {sample_size:,} samples, {pairs_per_level:,} pairs\")\n            \n            sample_idx = cp.random.choice(n_samples, size=sample_size, replace=False)\n            X_sample = X[sample_idx]\n            y_sample = y[sample_idx]\n            \n            tested_pairs = set()\n            for _ in range(pairs_per_level * 2):\n                if np.random.random() < 0.7:\n                    top_k = min(self.n_top_features, n_features)\n                    top_indices = np.argsort(importance_scores)[-top_k:]\n                    i = np.random.choice(top_indices)\n                    j = np.random.choice(top_indices)\n                else:\n                    i = np.random.randint(0, n_features)\n                    j = np.random.randint(0, n_features)\n                \n                if i != j:\n                    pair = (min(i, j), max(i, j))\n                    if pair not in tested_pairs:\n                        tested_pairs.add(pair)\n                        \n                        scores = self._compute_all_interaction_scores(\n                            X_sample[:, i], X_sample[:, j], y_sample\n                        )\n                        \n                        best_score = max(scores.values())\n                        if best_score > self.correlation_threshold:\n                            best_type = max(scores, key=scores.get)\n                            self._update_candidate(pair, i, j, best_score, best_type, \n                                                 sample_size, feature_names)\n                        \n                        if len(tested_pairs) >= pairs_per_level:\n                            break\n            \n            if level < len(sample_sizes) - 1:\n                self._adaptive_pruning(0.7 + level * 0.05)\n    \n    def _information_theoretic_search(self, X: cp.ndarray, y: cp.ndarray, feature_names: List[str]):\n        \"\"\"Use mutual information to find complementary features\"\"\"\n        n_samples, n_features = X.shape\n        sample_size = min(self.max_sample_size // 5, n_samples)\n        \n        print(\"Computing mutual information...\")\n        \n        sample_idx = cp.random.choice(n_samples, size=sample_size, replace=False)\n        X_sample = X[sample_idx]\n        y_sample = y[sample_idx]\n        \n        mi_scores = cp.zeros(n_features)\n        for i in range(min(n_features, self.n_top_features * 2)):\n            mi_scores[i] = self._mutual_information(X_sample[:, i], y_sample)\n        \n        mi_scores = cp.nan_to_num(mi_scores, nan=0.0)\n        \n        low_mi = cp.where(mi_scores < cp.percentile(mi_scores, 30))[0]\n        high_mi = cp.where(mi_scores > cp.percentile(mi_scores, 70))[0]\n        \n        if len(low_mi) > 0 and len(high_mi) > 0:\n            n_tests = min(self.n_pairs_per_phase, len(low_mi) * len(high_mi))\n            \n            for _ in range(n_tests):\n                i = int(np.random.choice(cp.asnumpy(low_mi)))\n                j = int(np.random.choice(cp.asnumpy(high_mi)))\n                \n                if i != j:\n                    pair = (min(i, j), max(i, j))\n                    if pair not in self.candidates:\n                        scores = self._compute_all_interaction_scores(\n                            X_sample[:, i], X_sample[:, j], y_sample\n                        )\n                        best_score = max(scores.values())\n                        if best_score > self.correlation_threshold:\n                            best_type = max(scores, key=scores.get)\n                            self._update_candidate(pair, i, j, best_score, best_type,\n                                                 sample_size, feature_names)\n    \n    def _ensemble_importance_search(self, X: cp.ndarray, y: cp.ndarray, feature_names: List[str]):\n        \"\"\"Tree ensemble to find interactions\"\"\"\n        n_samples, n_features = X.shape\n        sample_size = min(self.max_sample_size // 4, n_samples)\n        \n        print(\"Training random forest...\")\n        \n        sample_idx = cp.random.choice(n_samples, size=sample_size, replace=False)\n        X_cpu = cp.asnumpy(X[sample_idx])\n        y_cpu = cp.asnumpy(y[sample_idx])\n        \n        rf = RandomForestRegressor(\n            n_estimators=30,\n            max_depth=3,\n            min_samples_leaf=100,\n            n_jobs=self.n_jobs,\n            random_state=42\n        )\n        rf.fit(X_cpu, y_cpu)\n        \n        interaction_counts = defaultdict(int)\n        \n        for tree in rf.estimators_:\n            tree_structure = tree.tree_\n            \n            def extract_pairs(node=0, features=set()):\n                if tree_structure.feature[node] >= 0:\n                    feature = tree_structure.feature[node]\n                    new_features = features | {feature}\n                    \n                    for f in features:\n                        if f != feature:\n                            pair = (min(f, feature), max(f, feature))\n                            interaction_counts[pair] += 1\n                    \n                    extract_pairs(tree_structure.children_left[node], new_features)\n                    extract_pairs(tree_structure.children_right[node], new_features)\n            \n            extract_pairs()\n        \n        top_patterns = sorted(interaction_counts.items(), \n                            key=lambda x: x[1], \n                            reverse=True)[:self.n_pairs_per_phase]\n        \n        X_sample = X[sample_idx]\n        y_sample = y[sample_idx]\n        \n        for (i, j), _ in top_patterns:\n            if (i, j) not in self.candidates and i < n_features and j < n_features:\n                scores = self._compute_all_interaction_scores(\n                    X_sample[:, i], X_sample[:, j], y_sample\n                )\n                best_score = max(scores.values())\n                if best_score > self.correlation_threshold:\n                    best_type = max(scores, key=scores.get)\n                    self._update_candidate((i, j), i, j, best_score, best_type,\n                                         sample_size, feature_names)\n    \n    def _evolutionary_search(self, X: cp.ndarray, y: cp.ndarray, feature_names: List[str]):\n        \"\"\"Genetic algorithm search\"\"\"\n        n_samples, n_features = X.shape\n        population_size = min(100, self.n_pairs_per_phase // 10)\n        n_generations = 5\n        \n        print(f\"Running genetic algorithm ({n_generations} generations)...\")\n        \n        population = []\n        if self.candidates:\n            top_candidates = sorted(self.candidates.items(), \n                                  key=lambda x: x[1].score, \n                                  reverse=True)[:20]\n            population.extend([pair for pair, _ in top_candidates])\n        \n        while len(population) < population_size:\n            i, j = np.random.randint(0, n_features, 2)\n            if i != j:\n                population.append((min(i, j), max(i, j)))\n        \n        sample_size = min(self.max_sample_size // 4, n_samples)\n        sample_idx = cp.random.choice(n_samples, size=sample_size, replace=False)\n        X_sample = X[sample_idx]\n        y_sample = y[sample_idx]\n        \n        for gen in range(n_generations):\n            fitness = []\n            for pair in population:\n                if pair in self.candidates:\n                    fitness.append(self.candidates[pair].score)\n                else:\n                    score = self._compute_interaction_score(\n                        X_sample[:, pair[0]], X_sample[:, pair[1]], y_sample\n                    )\n                    fitness.append(score)\n            \n            new_population = []\n            while len(new_population) < population_size:\n                idx1, idx2 = np.random.choice(len(population), 2)\n                parent = population[idx1] if fitness[idx1] > fitness[idx2] else population[idx2]\n                \n                if np.random.random() < 0.2:\n                    new_feat = np.random.randint(0, n_features)\n                    if np.random.random() < 0.5:\n                        child = (min(parent[0], new_feat), max(parent[0], new_feat))\n                    else:\n                        child = (min(new_feat, parent[1]), max(new_feat, parent[1]))\n                else:\n                    child = parent\n                \n                if child[0] != child[1]:\n                    new_population.append(child)\n            \n            population = new_population\n            \n            best_indices = np.argsort(fitness)[-10:]\n            for idx in best_indices:\n                if fitness[idx] > self.correlation_threshold:\n                    pair = population[idx]\n                    if pair not in self.candidates:\n                        scores = self._compute_all_interaction_scores(\n                            X_sample[:, pair[0]], X_sample[:, pair[1]], y_sample\n                        )\n                        best_score = max(scores.values())\n                        best_type = max(scores, key=scores.get)\n                        self._update_candidate(pair, pair[0], pair[1], best_score,\n                                             best_type, sample_size, feature_names)\n    \n    def _spectral_search(self, X: cp.ndarray, y: cp.ndarray, feature_names: List[str]):\n        \"\"\"Spectral clustering approach\"\"\"\n        n_samples, n_features = X.shape\n        \n        print(\"Computing spectral affinity...\")\n        \n        sample_size = min(5000, n_samples)\n        sample_idx = cp.random.choice(n_samples, size=sample_size, replace=False)\n        X_sample = X[sample_idx]\n        y_sample = y[sample_idx]\n        \n        target_corrs = cp.zeros(n_features)\n        for i in range(min(n_features, self.n_top_features * 2)):\n            target_corrs[i] = cp.abs(cp.corrcoef(X_sample[:, i], y_sample)[0, 1])\n        \n        n_tests = min(self.n_pairs_per_phase, n_features * 10)\n        \n        for _ in range(n_tests):\n            i = np.random.randint(0, min(n_features, self.n_top_features * 2))\n            j = np.random.randint(0, min(n_features, self.n_top_features * 2))\n            \n            if i != j and abs(float(target_corrs[i] - target_corrs[j])) > 0.1:\n                pair = (min(i, j), max(i, j))\n                \n                if pair not in self.candidates:\n                    scores = self._compute_all_interaction_scores(\n                        X_sample[:, i], X_sample[:, j], y_sample\n                    )\n                    best_score = max(scores.values())\n                    if best_score > self.correlation_threshold:\n                        best_type = max(scores, key=scores.get)\n                        self._update_candidate(pair, i, j, best_score, best_type,\n                                             sample_size, feature_names)\n    \n    def _bayesian_refinement(self, X: cp.ndarray, y: cp.ndarray, feature_names: List[str]):\n        \"\"\"Refine top candidates with larger samples\"\"\"\n        n_samples = X.shape[0]\n        \n        print(\"Bayesian refinement of top candidates...\")\n        \n        candidates_to_refine = sorted(self.candidates.items(), \n                                    key=lambda x: x[1].score * (1 - x[1].confidence),\n                                    reverse=True)[:self.n_pairs_per_phase // 10]\n        \n        for pair, candidate in candidates_to_refine:\n            alpha = candidate.score * candidate.sample_size\n            beta = (1 - candidate.score) * candidate.sample_size\n            sampled_score = np.random.beta(alpha + 1, beta + 1)\n            \n            if sampled_score > candidate.score * 0.9:\n                new_sample_size = min(self.max_sample_size, candidate.sample_size * 2)\n                \n                sample_idx = cp.random.choice(n_samples, size=new_sample_size, replace=False)\n                X_sample = X[sample_idx]\n                y_sample = y[sample_idx]\n                \n                scores = self._compute_all_interaction_scores(\n                    X_sample[:, pair[0]], X_sample[:, pair[1]], y_sample\n                )\n                \n                best_score = max(scores.values())\n                best_type = max(scores, key=scores.get)\n                \n                total_samples = candidate.sample_size + new_sample_size\n                weight_old = candidate.sample_size / total_samples\n                weight_new = new_sample_size / total_samples\n                \n                candidate.score = weight_old * candidate.score + weight_new * best_score\n                candidate.sample_size = total_samples\n                candidate.confidence = min(1.0, candidate.confidence + 0.15)\n                candidate.interaction_type = best_type\n    \n    def _compute_all_interaction_scores(self, feat1: cp.ndarray, feat2: cp.ndarray, \n                                       y: cp.ndarray) -> Dict[str, float]:\n        \"\"\"Compute all interaction types\"\"\"\n        scores = {}\n        \n        try:\n            scores['multiply'] = float(cp.abs(cp.corrcoef(feat1 * feat2, y)[0, 1]))\n            scores['add'] = float(cp.abs(cp.corrcoef(feat1 + feat2, y)[0, 1]))\n            scores['subtract'] = float(cp.abs(cp.corrcoef(feat1 - feat2, y)[0, 1]))\n            scores['divide'] = float(cp.abs(cp.corrcoef(feat1 / (cp.abs(feat2) + 0.1), y)[0, 1]))\n            scores['distance'] = float(cp.abs(cp.corrcoef(cp.abs(feat1 - feat2), y)[0, 1]))\n            scores['max'] = float(cp.abs(cp.corrcoef(cp.maximum(feat1, feat2), y)[0, 1]))\n            scores['min'] = float(cp.abs(cp.corrcoef(cp.minimum(feat1, feat2), y)[0, 1]))\n        except:\n            scores = {'multiply': 0.0}\n        \n        scores = {k: v if not np.isnan(v) else 0.0 for k, v in scores.items()}\n        \n        return scores\n    \n    def _compute_interaction_score(self, feat1: cp.ndarray, feat2: cp.ndarray, \n                                  y: cp.ndarray) -> float:\n        \"\"\"Quick score computation\"\"\"\n        try:\n            return float(cp.abs(cp.corrcoef(feat1 * feat2, y)[0, 1]))\n        except:\n            return 0.0\n    \n    def _compute_feature_importance(self, X: cp.ndarray, y: cp.ndarray) -> np.ndarray:\n        \"\"\"Fast feature importance\"\"\"\n        n_features = X.shape[1]\n        importance = cp.zeros(n_features)\n        \n        for i in range(n_features):\n            try:\n                importance[i] = cp.abs(cp.corrcoef(X[:, i], y)[0, 1])\n            except:\n                importance[i] = 0.0\n        \n        importance = cp.nan_to_num(importance, nan=0.0)\n        importance = importance / (cp.sum(importance) + 1e-8)\n        \n        return cp.asnumpy(importance)\n    \n    def _update_candidate(self, pair: Tuple[int, int], i: int, j: int, \n                         score: float, interaction_type: str, \n                         sample_size: int, feature_names: List[str]):\n        \"\"\"Update or create candidate\"\"\"\n        if pair in self.candidates:\n            old = self.candidates[pair]\n            weight = sample_size / (old.sample_size + sample_size)\n            new_score = old.score * (1 - weight) + score * weight\n            \n            self.candidates[pair] = InteractionCandidate(\n                feat1_idx=i, feat2_idx=j,\n                score=new_score,\n                sample_size=old.sample_size + sample_size,\n                confidence=min(1.0, old.confidence + 0.1),\n                exploration_count=old.exploration_count + 1,\n                interaction_type=interaction_type if score > old.score else old.interaction_type\n            )\n        else:\n            self.candidates[pair] = InteractionCandidate(\n                feat1_idx=i, feat2_idx=j,\n                score=score,\n                sample_size=sample_size,\n                confidence=0.4,\n                interaction_type=interaction_type\n            )\n        \n        self.feature_participation[i] += 1\n        self.feature_participation[j] += 1\n    \n    def _adaptive_pruning(self, confidence_threshold: float):\n        \"\"\"Prune weak candidates\"\"\"\n        if len(self.candidates) < 100:\n            return\n        \n        scores = np.array([c.score for c in self.candidates.values()])\n        score_threshold = np.percentile(scores, 30)\n        \n        new_candidates = {}\n        for pair, candidate in self.candidates.items():\n            if (candidate.score > score_threshold or\n                (candidate.confidence < confidence_threshold and \n                 candidate.score > score_threshold * 0.7) or\n                (self.feature_participation[pair[0]] < 5 or \n                 self.feature_participation[pair[1]] < 5)):\n                new_candidates[pair] = candidate\n        \n        self.candidates = new_candidates\n    \n    def _mutual_information(self, x: cp.ndarray, y: cp.ndarray, n_bins: int = 10) -> float:\n        \"\"\"Fast MI estimation\"\"\"\n        try:\n            if cp.std(x) < 1e-10 or cp.std(y) < 1e-10:\n                return 0.0\n            \n            x_bins = cp.linspace(cp.min(x), cp.max(x) + 1e-10, n_bins + 1)\n            y_bins = cp.linspace(cp.min(y), cp.max(y) + 1e-10, n_bins + 1)\n            \n            x_idx = cp.clip(cp.digitize(x, x_bins) - 1, 0, n_bins - 1)\n            y_idx = cp.clip(cp.digitize(y, y_bins) - 1, 0, n_bins - 1)\n            \n            hist = cp.zeros((n_bins, n_bins))\n            for i in range(len(x_idx)):\n                hist[x_idx[i], y_idx[i]] += 1\n            \n            hist = hist / cp.sum(hist)\n            \n            px = cp.sum(hist, axis=1)\n            py = cp.sum(hist, axis=0)\n            \n            mi = 0.0\n            for i in range(n_bins):\n                for j in range(n_bins):\n                    if hist[i, j] > 1e-10:\n                        mi += hist[i, j] * cp.log(hist[i, j] / (px[i] * py[j] + 1e-10) + 1e-10)\n            \n            return float(max(0, mi))\n        except:\n            return 0.0\n    \n    def _final_validation_advanced(self, X: cp.ndarray, y: cp.ndarray, \n                                  feature_names: List[str]) -> pd.DataFrame:\n        \"\"\"Final validation on large sample\"\"\"\n        n_samples = X.shape[0]\n        \n        final_sample_size = min(n_samples, self.max_sample_size * 2)\n        sample_idx = cp.random.choice(n_samples, size=final_sample_size, replace=False)\n        X_final = X[sample_idx]\n        y_final = y[sample_idx]\n        \n        top_candidates = sorted(self.candidates.items(), \n                               key=lambda x: x[1].score, \n                               reverse=True)[:self.top_interactions]\n        \n        print(f\"Validating {len(top_candidates)} candidates on {final_sample_size:,} samples...\")\n        \n        results = []\n        for (feat1, feat2), candidate in top_candidates:\n            all_scores = self._compute_all_interaction_scores(\n                X_final[:, feat1], X_final[:, feat2], y_final\n            )\n            \n            corr1 = float(cp.abs(cp.corrcoef(X_final[:, feat1], y_final)[0, 1]))\n            corr2 = float(cp.abs(cp.corrcoef(X_final[:, feat2], y_final)[0, 1]))\n            \n            best_score = max(all_scores.values())\n            synergy = best_score - max(corr1, corr2)\n            \n            results.append({\n                'feature_1': feature_names[feat1],\n                'feature_2': feature_names[feat2],\n                'interaction_score': best_score,\n                'interaction_type': max(all_scores, key=all_scores.get),\n                'individual_score_1': corr1,\n                'individual_score_2': corr2,\n                'synergy': synergy,\n                'confidence': candidate.confidence,\n                'samples_tested': candidate.sample_size,\n                'multiply_score': all_scores.get('multiply', 0),\n                'distance_score': all_scores.get('distance', 0),\n                'divide_score': all_scores.get('divide', 0)\n            })\n        \n        results_df = pd.DataFrame(results)\n        results_df = results_df.sort_values('interaction_score', ascending=False)\n        \n        print(f\"\\n=== RESULTS SUMMARY ===\")\n        print(f\"Total interactions: {len(results_df)}\")\n        print(f\"High synergy (>0.02): {len(results_df[results_df['synergy'] > 0.02])}\")\n        print(f\"Very high synergy (>0.05): {len(results_df[results_df['synergy'] > 0.05])}\")\n        if len(results_df) > 0:\n            print(f\"Best interaction score: {results_df['interaction_score'].max():.4f}\")\n            print(f\"Best synergy: {results_df['synergy'].max():.4f}\")\n        \n        return results_df\n    \n    def _emergency_results(self, feature_names: List[str]) -> pd.DataFrame:\n        \"\"\"Fallback if validation fails\"\"\"\n        results = []\n        for (feat1, feat2), candidate in list(self.candidates.items())[:self.top_interactions]:\n            results.append({\n                'feature_1': feature_names[feat1],\n                'feature_2': feature_names[feat2],\n                'interaction_score': candidate.score,\n                'interaction_type': candidate.interaction_type,\n                'confidence': candidate.confidence\n            })\n        return pd.DataFrame(results)\n    \n    def _cleanup_memory(self):\n        \"\"\"Clean GPU memory\"\"\"\n        self.interaction_cache.clear()\n        cp.get_default_memory_pool().free_all_blocks()\n        gc.collect()\n\n\ndef create_all_interaction_features_memory_efficient(train_df: pd.DataFrame, test_df: pd.DataFrame, \n                                                    interactions_df: pd.DataFrame, \n                                                    n_interactions: int = None,\n                                                    batch_size: int = 50) -> Tuple[pd.DataFrame, pd.DataFrame, List[str]]:\n    \"\"\"Create interaction features in batches to manage memory\"\"\"\n    \n    # Use config value if not provided\n    n_interactions = n_interactions or config.N_INTERACTIONS_TO_CREATE\n    \n    print(f\"\\nCreating {n_interactions} interaction features in batches of {batch_size}...\")\n    \n    feature_names = list(train_df.columns)\n    all_interaction_dfs_train = []\n    all_interaction_dfs_test = []\n    all_interaction_names = []\n    \n    # Process in batches\n    for batch_start in range(0, n_interactions, batch_size):\n        batch_end = min(batch_start + batch_size, n_interactions, len(interactions_df))\n        \n        print(f\"Processing batch: interactions {batch_start} to {batch_end}\")\n        \n        interaction_features_train = {}\n        interaction_features_test = {}\n        interaction_names = []\n        \n        # Create interactions for this batch\n        for idx in range(batch_start, batch_end):\n            row = interactions_df.iloc[idx]\n            feat1, feat2 = row['feature_1'], row['feature_2']\n            interaction_type = row.get('interaction_type', 'multiply')\n            \n            if feat1 in feature_names and feat2 in feature_names:\n                feat_name = f'{feat1}_{interaction_type}_{feat2}'\n                \n                # Calculate interaction values based on type\n                if interaction_type == 'multiply':\n                    interaction_features_train[feat_name] = train_df[feat1].values * train_df[feat2].values\n                    interaction_features_test[feat_name] = test_df[feat1].values * test_df[feat2].values\n                elif interaction_type == 'distance':\n                    interaction_features_train[feat_name] = np.abs(train_df[feat1].values - train_df[feat2].values)\n                    interaction_features_test[feat_name] = np.abs(test_df[feat1].values - test_df[feat2].values)\n                elif interaction_type == 'subtract':\n                    interaction_features_train[feat_name] = train_df[feat1].values - train_df[feat2].values\n                    interaction_features_test[feat_name] = test_df[feat1].values - test_df[feat2].values\n                elif interaction_type == 'add':\n                    interaction_features_train[feat_name] = train_df[feat1].values + train_df[feat2].values\n                    interaction_features_test[feat_name] = test_df[feat1].values + test_df[feat2].values\n                elif interaction_type == 'divide':\n                    interaction_features_train[feat_name] = train_df[feat1].values / (np.abs(train_df[feat2].values) + 1e-10)\n                    interaction_features_test[feat_name] = test_df[feat1].values / (np.abs(test_df[feat2].values) + 1e-10)\n                elif interaction_type == 'min':\n                    interaction_features_train[feat_name] = np.minimum(train_df[feat1].values, train_df[feat2].values)\n                    interaction_features_test[feat_name] = np.minimum(test_df[feat1].values, test_df[feat2].values)\n                elif interaction_type == 'max':\n                    interaction_features_train[feat_name] = np.maximum(train_df[feat1].values, train_df[feat2].values)\n                    interaction_features_test[feat_name] = np.maximum(test_df[feat1].values, test_df[feat2].values)\n                else:\n                    interaction_features_train[feat_name] = train_df[feat1].values * train_df[feat2].values\n                    interaction_features_test[feat_name] = test_df[feat1].values * test_df[feat2].values\n                \n                interaction_names.append(feat_name)\n        \n        # Create dataframes for this batch\n        if interaction_names:\n            batch_df_train = pd.DataFrame(interaction_features_train, index=train_df.index)\n            batch_df_test = pd.DataFrame(interaction_features_test, index=test_df.index)\n            \n            all_interaction_dfs_train.append(batch_df_train)\n            all_interaction_dfs_test.append(batch_df_test)\n            all_interaction_names.extend(interaction_names)\n        \n        # Clean up memory\n        del interaction_features_train, interaction_features_test\n        gc.collect()\n    \n    # Combine all batches\n    if all_interaction_dfs_train:\n        interactions_df_train = pd.concat(all_interaction_dfs_train, axis=1)\n        interactions_df_test = pd.concat(all_interaction_dfs_test, axis=1)\n    else:\n        interactions_df_train = pd.DataFrame(index=train_df.index)\n        interactions_df_test = pd.DataFrame(index=test_df.index)\n    \n    print(f\"Created {len(all_interaction_names)} interaction features\")\n    \n    return interactions_df_train, interactions_df_test, all_interaction_names\n\n\ndef select_and_transform_interactions(interactions_train: pd.DataFrame, \n                                     interactions_test: pd.DataFrame,\n                                     train_labels: pd.Series,\n                                     n_select: int = None,\n                                     n_components: int = None) -> Tuple[pd.DataFrame, pd.DataFrame, List[str]]:\n    \"\"\"Apply SelectKBest then PCA to interaction features\"\"\"\n    \n    # Use config values if not provided\n    n_select = n_select or config.N_INTERACTIONS_TO_SELECT\n    n_components = n_components or config.N_PCA_COMPONENTS\n    \n    print(f\"\\nSelecting top {n_select} interactions...\")\n    \n    # Step 1: SelectKBest on interactions\n    selector = SelectKBest(score_func=f_regression, k=min(n_select, interactions_train.shape[1]))\n    selector.fit(interactions_train, train_labels)\n    \n    # Get selected features\n    selected_mask = selector.get_support()\n    selected_columns = interactions_train.columns[selected_mask]\n    \n    interactions_train_selected = interactions_train[selected_columns]\n    interactions_test_selected = interactions_test[selected_columns]\n    \n    print(f\"Selected {len(selected_columns)} best interactions\")\n    \n    # Step 2: Apply PCA to selected interactions\n    print(f\"Applying PCA to reduce to {n_components} components...\")\n    \n    # Scale the data first\n    scaler = StandardScaler()\n    interactions_train_scaled = scaler.fit_transform(interactions_train_selected)\n    interactions_test_scaled = scaler.transform(interactions_test_selected)\n    \n    # Apply PCA\n    n_components_actual = min(n_components, interactions_train_scaled.shape[1])\n    pca = PCA(n_components=n_components_actual, random_state=42)\n    \n    pca_train = pca.fit_transform(interactions_train_scaled)\n    pca_test = pca.transform(interactions_test_scaled)\n    \n    # Create PCA feature names\n    pca_feature_names = [f'AI_PCA_{i+1}' for i in range(n_components_actual)]\n    \n    # Create dataframes\n    pca_df_train = pd.DataFrame(pca_train, columns=pca_feature_names, index=interactions_train.index)\n    pca_df_test = pd.DataFrame(pca_test, columns=pca_feature_names, index=interactions_test.index)\n    \n    print(f\"PCA explained variance ratio: {pca.explained_variance_ratio_.sum():.3f}\")\n    \n    return pca_df_train, pca_df_test, pca_feature_names\n\n\ndef create_manual_features(train_df: pd.DataFrame, test_df: pd.DataFrame, \n                          original_features: List[str]) -> Tuple[pd.DataFrame, pd.DataFrame, List[str]]:\n    \"\"\"Create manual domain-specific features\"\"\"\n    \n    print(\"\\n=== ADDING MANUAL FEATURE INTERACTIONS ===\")\n    \n    manual_features_dict_train = {}\n    manual_features_dict_test = {}\n    manual_features = []\n\n    # 1. Market microstructure interactions (limited to most important)\n    market_features = ['bid_qty', 'ask_qty', 'buy_qty', 'sell_qty', 'volume']\n    existing_market_features = [f for f in market_features if f in original_features]\n\n    # Only create most essential interactions\n    if len(existing_market_features) >= 2:\n        # Just a few key interactions\n        if 'bid_qty' in existing_market_features and 'ask_qty' in existing_market_features:\n            feat_name = 'bid_ask_ratio'\n            manual_features_dict_train[feat_name] = train_df['bid_qty'].values / (train_df['ask_qty'].values + 1e-10)\n            manual_features_dict_test[feat_name] = test_df['bid_qty'].values / (test_df['ask_qty'].values + 1e-10)\n            manual_features.append(feat_name)\n        \n        if 'buy_qty' in existing_market_features and 'sell_qty' in existing_market_features:\n            feat_name = 'buy_sell_ratio'\n            manual_features_dict_train[feat_name] = train_df['buy_qty'].values / (train_df['sell_qty'].values + 1e-10)\n            manual_features_dict_test[feat_name] = test_df['buy_qty'].values / (test_df['sell_qty'].values + 1e-10)\n            manual_features.append(feat_name)\n\n    # 2. Create order flow imbalance features\n    if 'buy_qty' in original_features and 'sell_qty' in original_features:\n        manual_features_dict_train['order_imbalance'] = (train_df['buy_qty'].values - train_df['sell_qty'].values) / (train_df['buy_qty'].values + train_df['sell_qty'].values + 1e-10)\n        manual_features_dict_test['order_imbalance'] = (test_df['buy_qty'].values - test_df['sell_qty'].values) / (test_df['buy_qty'].values + test_df['sell_qty'].values + 1e-10)\n        manual_features.append('order_imbalance')\n\n    if 'bid_qty' in original_features and 'ask_qty' in original_features:\n        manual_features_dict_train['bid_ask_spread'] = (train_df['ask_qty'].values - train_df['bid_qty'].values) / (train_df['ask_qty'].values + train_df['bid_qty'].values + 1e-10)\n        manual_features_dict_test['bid_ask_spread'] = (test_df['ask_qty'].values - test_df['bid_qty'].values) / (test_df['ask_qty'].values + test_df['bid_qty'].values + 1e-10)\n        manual_features.append('bid_ask_spread')\n\n    # Create dataframes\n    if manual_features:\n        manual_df_train = pd.DataFrame(manual_features_dict_train, index=train_df.index)\n        manual_df_test = pd.DataFrame(manual_features_dict_test, index=test_df.index)\n    else:\n        manual_df_train = pd.DataFrame(index=train_df.index)\n        manual_df_test = pd.DataFrame(index=test_df.index)\n\n    print(f\"Created {len(manual_features)} manual interaction features\")\n    \n    return manual_df_train, manual_df_test, manual_features\n\n\n# ============================================\n# MAIN PIPELINE WITH CONFIGURABLE PARAMETERS\n# ============================================\n\nprint(\"=== LOADING DATA ===\")\ntrain = pl.read_parquet(\"/kaggle/input/drw-crypto-market-prediction/train.parquet\")\ntrain = train.to_pandas()\ntest = pl.read_parquet('/kaggle/input/drw-crypto-market-prediction/test.parquet')\ntest = test.to_pandas()\nprint(f\"train.shape: {train.shape}, test.shape: {test.shape}\")\n\nfeatures = [c for c in train.columns if c != 'label']\nlabel_col = 'label'\n\nprint(\"\\n=== CRYPTO TRADING DATA EDA ===\")\nprint(f\"Dataset shape: {train.shape}\")\nprint(f\"Number of features: {len(features)}\")\n\n# Basic preprocessing\nNUNIQUE1 = [c for c in train.columns if train[c].nunique() == 1]\ntrain.drop(NUNIQUE1 + ['timestamp'], axis=1, inplace=True)\ntest.drop(NUNIQUE1 + ['label'], axis=1, inplace=True)\nprint(f\"Removed {len(NUNIQUE1)} features with only 1 unique value\")\n\nfeatures = [c for c in train.columns if c != 'label']\n\n# Store original features before interaction discovery\noriginal_features = features.copy()\n\n# ========== STEP 1: FEATURE INTERACTION DISCOVERY ==========\nprint(\"\\n\" + \"=\"*70)\nprint(\"=== AI-POWERED FEATURE INTERACTION DISCOVERY ===\")\nprint(\"=\"*70)\n\n# Check if we already have saved interactions\nif os.path.exists(config.INTERACTIONS_FILENAME):\n    print(f\"Found existing {config.INTERACTIONS_FILENAME} - loading discovered interactions...\")\n    interaction_results = pd.read_csv(config.INTERACTIONS_FILENAME)\n    print(f\"Loaded {len(interaction_results)} pre-discovered interactions\")\n    if 'synergy' in interaction_results.columns and len(interaction_results) > 0:\n        print(f\"Best synergy score: {interaction_results['synergy'].max():.4f}\")\nelse:\n    print(f\"Running feature interaction discovery using {config.SEARCH_N_PHASES} search phases...\")\n    \n    # Prepare data for GPU\n    X_for_search = train[features].tail(config.INTERACTION_DISCOVERY_SAMPLE_SIZE)\n    y_for_search = train['label'].tail(config.INTERACTION_DISCOVERY_SAMPLE_SIZE)\n    \n    # Convert to GPU\n    X_gpu = cp.asarray(X_for_search.values)\n    y_gpu = cp.asarray(y_for_search.values)\n    \n    # Initialize searcher with config parameters\n    searcher = WorldClassInteractionSearch()\n    \n    # Run search\n    start_time = time.time()\n    interaction_results = searcher.search(X_gpu, y_gpu, features)\n    elapsed = time.time() - start_time\n    print(f\"\\nSearch completed in {elapsed:.1f} seconds ({elapsed/60:.1f} minutes)\")\n    \n    # Save results\n    interaction_results.to_csv(config.INTERACTIONS_FILENAME, index=False)\n    print(f\"Saved interaction results to '{config.INTERACTIONS_FILENAME}'\")\n    \n    # Clean GPU memory\n    cp.get_default_memory_pool().free_all_blocks()\n    gc.collect()\n\n# Display top synergistic interactions\nif len(interaction_results) > 0 and 'synergy' in interaction_results.columns:\n    print(f\"\\n=== TOP {config.N_TOP_SYNERGIES_TO_DISPLAY} SYNERGISTIC INTERACTIONS DISCOVERED ===\")\n    top_synergy = interaction_results.nlargest(min(config.N_TOP_SYNERGIES_TO_DISPLAY, len(interaction_results)), 'synergy')\n    for idx, row in top_synergy.iterrows():\n        print(f\"{row['feature_1']} × {row['feature_2']} ({row.get('interaction_type', 'multiply')}) - \"\n              f\"Synergy: {row['synergy']:.4f}, Score: {row['interaction_score']:.4f}\")\n\n# ========== STEP 2: CREATE ALL INTERACTION FEATURES ==========\ninteractions_train, interactions_test, interaction_feature_names = create_all_interaction_features_memory_efficient(\n    train, test, interaction_results\n)\n\n# ========== STEP 3: SELECT BEST INTERACTIONS AND APPLY PCA ==========\nif interactions_train.shape[1] > 0:\n    pca_train, pca_test, pca_feature_names = select_and_transform_interactions(\n        interactions_train, interactions_test, \n        train['label']\n    )\nelse:\n    print(\"No interactions created, skipping PCA\")\n    pca_train = pd.DataFrame(index=train.index)\n    pca_test = pd.DataFrame(index=test.index)\n    pca_feature_names = []\n\n# Clean up intermediate data\ndel interactions_train, interactions_test\ngc.collect()\n\n# ========== STEP 4: CREATE MANUAL FEATURES ==========\nmanual_train, manual_test, manual_feature_names = create_manual_features(\n    train, test, original_features\n)\n\n# ========== STEP 5: COMBINE ALL FEATURES ==========\nprint(\"\\n=== COMBINING ALL FEATURE TYPES ===\")\n\n# Combine original features, PCA components, and manual features\ntrain_combined = pd.concat([\n    train[original_features],\n    pca_train,\n    manual_train\n], axis=1)\n\ntest_combined = pd.concat([\n    test[original_features],\n    pca_test,\n    manual_test\n], axis=1)\n\n# Add label back to train\ntrain_combined['label'] = train['label']\n\nprint(f\"Combined feature set size: {train_combined.shape[1] - 1} features\")\nprint(f\"  - Original features: {len(original_features)}\")\nprint(f\"  - PCA components: {len(pca_feature_names)}\")\nprint(f\"  - Manual features: {len(manual_feature_names)}\")\n\n# ========== STEP 6: FINAL FEATURE SELECTION ==========\nprint(\"\\n=== FINAL FEATURE SELECTION ===\")\n\n# Combine all feature names\nall_feature_names = original_features + pca_feature_names + manual_feature_names\n\n# Sample data for feature selection to save memory\nsample_size = min(100000, len(train_combined))\nsample_indices = np.random.choice(len(train_combined), sample_size, replace=False)\n\n# Apply final SelectKBest on sample\nk_final = min(config.N_FINAL_FEATURES, len(all_feature_names))\nselector_final = SelectKBest(score_func=f_regression, k=k_final)\nselector_final.fit(train_combined.iloc[sample_indices][all_feature_names], \n                  train_combined.iloc[sample_indices]['label'])\n\n# Get selected features\nselected_indices = selector_final.get_support(indices=True)\nselected_features = [all_feature_names[i] for i in selected_indices]\n\n# Extract only selected features\ntrain_final = train_combined[selected_features + ['label']].copy()\ntest_final = test_combined[selected_features].copy()\n\nprint(f\"\\nFinal feature composition:\")\nprint(f\"Original features: {len([f for f in selected_features if f in original_features])}\")\nprint(f\"PCA components: {len([f for f in selected_features if 'AI_PCA_' in f])}\")\nprint(f\"Manual features: {len([f for f in selected_features if f in manual_feature_names])}\")\nprint(f\"Total features: {len(selected_features)}\")\n\n# Clean up memory\ndel train, test, train_combined, test_combined\ngc.collect()\n\n# ========== DATA SAMPLING FOR TRAINING ==========\nprint(\"\\n=== TRAINING DATA SAMPLING ===\")\n\n# Apply training data percentage sampling if configured\nif config.TRAIN_DATA_PERCENTAGE < 100:\n    n_total_samples = len(train_final)\n    n_samples_to_use = int(n_total_samples * config.TRAIN_DATA_PERCENTAGE / 100)\n    \n    print(f\"Sampling {config.TRAIN_DATA_PERCENTAGE}% of training data...\")\n    print(f\"Total samples: {n_total_samples:,}\")\n    print(f\"Samples to use: {n_samples_to_use:,}\")\n    \n    if config.TRAIN_DATA_RECENT_ONLY:\n        # Use most recent data\n        print(\"Using most recent samples...\")\n        train_final_sampled = train_final.tail(n_samples_to_use).copy()\n    else:\n        # Random sampling\n        print(\"Using random sampling...\")\n        sample_indices = np.random.choice(n_total_samples, size=n_samples_to_use, replace=False)\n        train_final_sampled = train_final.iloc[sample_indices].copy()\nelse:\n    print(\"Using 100% of training data...\")\n    train_final_sampled = train_final\n\nprint(f\"Final training set size: {len(train_final_sampled):,} samples\")\n\n# ========== FLAML TRAINING ==========\nprint(\"\\n=== FLAML MODEL TRAINING ===\")\nfrom flaml import AutoML\nmodel = AutoML()\n\n# Calculate time budget in seconds\ntime_budget_seconds = int(config.TRAINING_TIME_HOURS * 3600)\n\nsettings = {\n    \"time_budget\": time_budget_seconds,\n    \"task\": config.TRAINING_TASK,\n    \"metric\": config.TRAINING_METRIC,\n    \"n_concurrent_trials\": config.TRAINING_N_CONCURRENT_TRIALS,\n    \"free_mem_ratio\": config.TRAINING_FREE_MEM_RATIO,\n}\n\nprint(\"Starting FLAML AutoML training...\")\nprint(f\"Time budget: {config.TRAINING_TIME_HOURS} hours\")\nprint(f\"Number of features: {len(selected_features)}\")\nprint(f\"Training samples: {len(train_final_sampled):,}\")\n\nmodel.fit(X_train=train_final_sampled[selected_features], y_train=train_final_sampled['label'], **settings)\n\nprint(\"\\n=== FLAML RESULTS ===\")\nprint(f\"Best model: {model.best_estimator}\")\nprint(f\"Best MSE: {model.best_loss:.6f}\")\nprint(f\"Best config:\")\nfor key, value in model.best_config.items():\n    print(f\"  {key}: {value}\")\n\n# Make predictions\ntest_preds = model.predict(test_final[selected_features])\n\n# Save submission\nsub = pd.read_csv('/kaggle/input/drw-crypto-market-prediction/sample_submission.csv')\nsub['prediction'] = test_preds\nsub.to_csv(config.SUBMISSION_FILENAME, index=None)\nprint(f\"\\nPredictions saved to '{config.SUBMISSION_FILENAME}'\")\n\n# Save feature importance if available\ntry:\n    if hasattr(model.model.estimator, 'feature_importances_'):\n        feature_importance = pd.DataFrame({\n            'feature': selected_features,\n            'importance': model.model.estimator.feature_importances_\n        }).sort_values('importance', ascending=False)\n        \n        print(f\"\\n=== TOP {config.N_TOP_FEATURES_TO_DISPLAY} MOST IMPORTANT FEATURES ===\")\n        print(feature_importance.head(config.N_TOP_FEATURES_TO_DISPLAY))\n        \n        # Check which feature types are most important\n        pca_importance = feature_importance[feature_importance['feature'].str.contains('AI_PCA_')]\n        if len(pca_importance) > 0:\n            print(f\"\\n=== TOP {config.N_TOP_PCA_TO_DISPLAY} PCA COMPONENTS ===\")\n            print(pca_importance.head(config.N_TOP_PCA_TO_DISPLAY))\nexcept:\n    print(\"\\nFeature importance not available for this model type\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}