{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false},"papermill":{"default_parameters":{},"duration":null,"end_time":null,"environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2025-05-23T19:03:14.791476","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install scikit-learn==1.5.2","metadata":{"_kg_hide-output":true,"papermill":{"duration":9.209373,"end_time":"2025-05-23T19:03:29.658522","exception":false,"start_time":"2025-05-23T19:03:20.449149","status":"completed"},"scrolled":true,"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-05-29T07:52:46.375918Z","iopub.execute_input":"2025-05-29T07:52:46.37624Z","iopub.status.idle":"2025-05-29T07:52:57.129652Z","shell.execute_reply.started":"2025-05-29T07:52:46.376213Z","shell.execute_reply":"2025-05-29T07:52:57.128319Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Imports and configs","metadata":{"papermill":{"duration":0.004193,"end_time":"2025-05-23T19:03:29.66754","exception":false,"start_time":"2025-05-23T19:03:29.663347","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Install required packages\n!pip install scikit-learn pandas numpy xgboost psutil lightgbm scipy joblib -q\n\nimport pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import KFold, TimeSeriesSplit\nfrom sklearn.preprocessing import StandardScaler, RobustScaler\nfrom sklearn.feature_selection import mutual_info_regression, VarianceThreshold, SelectKBest, f_regression\nfrom sklearn.metrics import mutual_info_score\nfrom xgboost import XGBRegressor\nfrom lightgbm import LGBMRegressor\nfrom scipy.stats import pearsonr, entropy\nfrom scipy.spatial.distance import pdist, squareform\nfrom joblib import Parallel, delayed\nimport time\nimport warnings\nimport json\nimport gc\nimport psutil\nimport os\nfrom collections import defaultdict\nfrom functools import lru_cache\nwarnings.filterwarnings('ignore')\n\nclass OptimizedInformationTheoreticFeatureSelector:\n    \"\"\"\n    Memory and runtime optimized Information-Theoretic Feature Selection.\n    \"\"\"\n    \n    def __init__(self, config):\n        self.config = config\n        self.selected_features = []\n        self.feature_scores = {}\n        self.mi_cache = {}  # Cache for MI calculations\n        self.relevance_scores = None\n        \n    def get_memory_usage(self):\n        \"\"\"Monitor current memory usage.\"\"\"\n        process = psutil.Process(os.getpid())\n        return process.memory_info().rss / 1024 / 1024 / 1024\n    \n    def log_memory(self, message):\n        \"\"\"Log memory usage with message.\"\"\"\n        memory_gb = self.get_memory_usage()\n        available_gb = psutil.virtual_memory().available / 1024 / 1024 / 1024\n        print(f\"{message}: Using {memory_gb:.2f} GB, Available: {available_gb:.2f} GB\")\n    \n    def optimize_dtypes(self, df):\n        \"\"\"Aggressively optimize data types.\"\"\"\n        for col in df.columns:\n            col_type = df[col].dtype\n            if col_type != object:\n                # Use float16 where possible for maximum memory savings\n                if str(col_type)[:5] == 'float':\n                    df[col] = df[col].astype(np.float16)\n                elif str(col_type)[:3] == 'int':\n                    c_min = df[col].min()\n                    c_max = df[col].max()\n                    if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                        df[col] = df[col].astype(np.int8)\n                    elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                        df[col] = df[col].astype(np.int16)\n                    else:\n                        df[col] = df[col].astype(np.int32)\n        return df\n    \n    def prescreen_features_variance(self, X, keep_ratio=0.3):\n        \"\"\"Fast variance-based prescreening.\"\"\"\n        print(f\"Prescreening by variance (keeping top {keep_ratio*100:.0f}%)...\")\n        variances = X.var()\n        n_keep = int(len(X.columns) * keep_ratio)\n        top_features = variances.nlargest(n_keep).index.tolist()\n        print(f\"  Reduced from {len(X.columns)} to {len(top_features)} features\")\n        return top_features\n    \n    def prescreen_features_univariate(self, X, y, n_features=100):\n        \"\"\"Fast univariate prescreening.\"\"\"\n        print(f\"Univariate prescreening (top {n_features})...\")\n        selector = SelectKBest(score_func=f_regression, k=min(n_features, len(X.columns)))\n        selector.fit(X, y)\n        selected_features = X.columns[selector.get_support()].tolist()\n        print(f\"  Selected {len(selected_features)} features\")\n        return selected_features\n    \n    def calculate_mi_batch(self, X, y, features, n_neighbors=5, batch_size=10):\n        \"\"\"Calculate MI in batches to save memory.\"\"\"\n        n_features = len(features)\n        mi_scores = np.zeros(n_features)\n        \n        for i in range(0, n_features, batch_size):\n            end_idx = min(i + batch_size, n_features)\n            batch_features = features[i:end_idx]\n            X_batch = X[batch_features]\n            \n            batch_scores = mutual_info_regression(\n                X_batch, y, n_neighbors=n_neighbors, random_state=42\n            )\n            mi_scores[i:end_idx] = batch_scores\n            \n            if (i // batch_size) % 10 == 0:\n                gc.collect()\n        \n        return dict(zip(features, mi_scores))\n    \n    def approximate_mi_matrix(self, X, n_samples=5000, n_pairs=1000):\n        \"\"\"\n        Approximate MI matrix using sampling instead of full computation.\n        \"\"\"\n        print(\"\\nApproximating MI matrix using sampling...\")\n        n_features = len(X.columns)\n        \n        # Sample data points for faster computation\n        if len(X) > n_samples:\n            sample_indices = np.random.choice(len(X), n_samples, replace=False)\n            X_sample = X.iloc[sample_indices]\n        else:\n            X_sample = X\n        \n        # Initialize sparse MI matrix\n        mi_matrix = np.zeros((n_features, n_features))\n        np.fill_diagonal(mi_matrix, 1.0)\n        \n        # Sample random pairs instead of all pairs\n        if n_features > 100:\n            # For large feature sets, sample pairs\n            n_pairs = min(n_pairs, n_features * (n_features - 1) // 2)\n            sampled_pairs = []\n            \n            while len(sampled_pairs) < n_pairs:\n                i = np.random.randint(0, n_features)\n                j = np.random.randint(0, n_features)\n                if i != j and (i, j) not in sampled_pairs and (j, i) not in sampled_pairs:\n                    sampled_pairs.append((i, j))\n            \n            print(f\"  Computing MI for {n_pairs} sampled pairs...\")\n            \n            for idx, (i, j) in enumerate(sampled_pairs):\n                if idx % 100 == 0:\n                    print(f\"    Processed {idx}/{n_pairs} pairs\")\n                \n                # Use correlation as fast approximation for MI\n                corr = np.corrcoef(X_sample.iloc[:, i], X_sample.iloc[:, j])[0, 1]\n                # Convert correlation to approximate MI (simplified)\n                mi_approx = -0.5 * np.log(1 - corr**2) if abs(corr) < 1 else 1.0\n                mi_matrix[i, j] = mi_matrix[j, i] = max(0, mi_approx)\n        else:\n            # For small feature sets, compute all pairs\n            for i in range(n_features):\n                for j in range(i+1, n_features):\n                    corr = np.corrcoef(X_sample.iloc[:, i], X_sample.iloc[:, j])[0, 1]\n                    mi_approx = -0.5 * np.log(1 - corr**2) if abs(corr) < 1 else 1.0\n                    mi_matrix[i, j] = mi_matrix[j, i] = max(0, mi_approx)\n        \n        return mi_matrix\n    \n    def fast_mrmr_selection(self, X, y, n_features_to_select, batch_size=20):\n        \"\"\"\n        Optimized mRMR selection with batching and approximations.\n        \"\"\"\n        print(f\"\\nRunning optimized mRMR selection...\")\n        \n        features = list(X.columns)\n        n_features = len(features)\n        \n        # Calculate relevance scores in batches\n        print(\"Calculating relevance scores...\")\n        if self.relevance_scores is None:\n            self.relevance_scores = self.calculate_mi_batch(X, y, features, batch_size=batch_size)\n        \n        relevance_array = np.array([self.relevance_scores[f] for f in features])\n        \n        # Initialize with most relevant feature\n        selected_indices = []\n        remaining_indices = list(range(n_features))\n        \n        first_idx = np.argmax(relevance_array)\n        selected_indices.append(first_idx)\n        remaining_indices.remove(first_idx)\n        \n        print(f\"Initial feature: {features[first_idx]} (relevance: {relevance_array[first_idx]:.4f})\")\n        \n        # Use correlation matrix as fast approximation for redundancy\n        print(\"Computing correlation matrix for redundancy estimation...\")\n        X_array = X.values.astype(np.float32)\n        corr_matrix = np.corrcoef(X_array.T)\n        \n        # Iterative selection\n        while len(selected_indices) < n_features_to_select and remaining_indices:\n            scores = np.zeros(len(remaining_indices))\n            \n            for idx, candidate_idx in enumerate(remaining_indices):\n                # Relevance\n                relevance = relevance_array[candidate_idx]\n                \n                # Redundancy (using correlation as proxy)\n                redundancy = 0\n                for selected_idx in selected_indices:\n                    redundancy += abs(corr_matrix[candidate_idx, selected_idx])\n                redundancy /= len(selected_indices)\n                \n                # mRMR score (MID)\n                scores[idx] = relevance - redundancy\n            \n            # Select best feature\n            best_idx = remaining_indices[np.argmax(scores)]\n            selected_indices.append(best_idx)\n            remaining_indices.remove(best_idx)\n            \n            if len(selected_indices) % 10 == 0:\n                print(f\"  Selected {len(selected_indices)} features...\")\n        \n        selected_features = [features[idx] for idx in selected_indices]\n        \n        # Store scores\n        self.feature_scores = {\n            feature: {\n                'relevance': self.relevance_scores[feature],\n                'selected': feature in selected_features,\n                'order': selected_features.index(feature) + 1 if feature in selected_features else None\n            }\n            for feature in features\n        }\n        \n        return selected_features\n    \n    def incremental_feature_evaluation(self, X, y, features, max_features=50):\n        \"\"\"\n        Evaluate features incrementally to find optimal number.\n        \"\"\"\n        print(\"\\nIncremental feature evaluation...\")\n        \n        # Use simple train-validation split\n        split_idx = int(0.8 * len(X))\n        X_train = X.iloc[:split_idx]\n        y_train = y.iloc[:split_idx]\n        X_val = X.iloc[split_idx:]\n        y_val = y.iloc[split_idx:]\n        \n        results = []\n        best_score = -np.inf\n        best_n_features = 10\n        \n        # Test different feature counts\n        feature_counts = [10, 20, 30, 40, 50, 75, 100]\n        feature_counts = [n for n in feature_counts if n <= min(len(features), max_features)]\n        \n        for n_features in feature_counts:\n            selected = features[:n_features]\n            \n            # Scale data\n            scaler = StandardScaler()\n            X_train_scaled = scaler.fit_transform(X_train[selected])\n            X_val_scaled = scaler.transform(X_val[selected])\n            \n            # Train simple model\n            model = LGBMRegressor(\n                n_estimators=100,\n                max_depth=5,\n                learning_rate=0.1,\n                subsample=0.8,\n                random_state=42,\n                verbosity=-1\n            )\n            model.fit(X_train_scaled, y_train)\n            \n            # Evaluate\n            pred = model.predict(X_val_scaled)\n            score = pearsonr(y_val, pred)[0]\n            \n            results.append((n_features, score))\n            print(f\"  {n_features} features: {score:.4f}\")\n            \n            if score > best_score:\n                best_score = score\n                best_n_features = n_features\n        \n        print(f\"\\nOptimal features: {best_n_features} (score: {best_score:.4f})\")\n        return results, best_n_features\n\ndef run_optimized_mrmr_pipeline(\n    train_path=\"/kaggle/input/drw-crypto-market-prediction/train.parquet\",\n    test_path=\"/kaggle/input/drw-crypto-market-prediction/test.parquet\",\n    sample_size=30000,  # Reduced from 50000\n    n_features_to_select=50,\n    optimize_count=True\n):\n    \"\"\"\n    Execute optimized mRMR feature selection pipeline.\n    \"\"\"\n    \n    print(\"=\" * 60)\n    print(\"OPTIMIZED INFORMATION-THEORETIC FEATURE SELECTION\")\n    print(\"=\" * 60)\n    \n    selector = OptimizedInformationTheoreticFeatureSelector({})\n    selector.log_memory(\"Initial\")\n    \n    # Load data with sampling\n    print(f\"\\nLoading {sample_size} samples...\")\n    \n    # Use tail to get recent data\n    train_df_full = pd.read_parquet(train_path)\n    if len(train_df_full) > sample_size:\n        train_df = train_df_full.tail(sample_size).reset_index(drop=True)\n    else:\n        train_df = train_df_full\n    \n    del train_df_full\n    gc.collect()\n    \n    # Optimize memory immediately\n    train_df = selector.optimize_dtypes(train_df)\n    \n    # Get features\n    feature_cols = [col for col in train_df.columns if col not in [\"timestamp\", \"label\"]]\n    baseline_features = [\"bid_qty\", \"ask_qty\", \"buy_qty\", \"sell_qty\", \"volume\"]\n    \n    X = train_df[feature_cols]\n    y = train_df[\"label\"].astype(np.float32)\n    \n    del train_df\n    gc.collect()\n    \n    # Clean data\n    X = X.replace([np.inf, -np.inf], np.nan).fillna(0)\n    \n    # Multi-stage feature reduction\n    print(\"\\nMulti-stage feature reduction...\")\n    \n    # Stage 1: Variance screening\n    variance_features = selector.prescreen_features_variance(X, keep_ratio=0.4)\n    X_var = X[variance_features]\n    \n    # Stage 2: Univariate screening  \n    univariate_features = selector.prescreen_features_univariate(X_var, y, n_features=150)\n    X_reduced = X_var[univariate_features]\n    \n    selector.log_memory(\"After prescreening\")\n    \n    # Stage 3: mRMR selection\n    selected_features = selector.fast_mrmr_selection(\n        X_reduced, y, n_features_to_select\n    )\n    \n    # Ensure baseline features\n    for feature in baseline_features:\n        if feature in feature_cols and feature not in selected_features:\n            selected_features.append(feature)\n    \n    # Optimize feature count if requested\n    if optimize_count:\n        results, optimal_n = selector.incremental_feature_evaluation(\n            X_reduced, y, selected_features, max_features=100\n        )\n        selected_features = selected_features[:optimal_n]\n    \n    print(f\"\\nFinal selection: {len(selected_features)} features\")\n    \n    # Save results\n    results = {\n        'selected_features': selected_features,\n        'n_features': len(selected_features),\n        'feature_scores': dict(list(selector.feature_scores.items())[:50])\n    }\n    \n    with open('optimized_mrmr_results.json', 'w') as f:\n        json.dump(results, f, indent=2)\n    \n    selector.log_memory(\"Pipeline complete\")\n    \n    return selected_features\n\ndef train_lightweight_model(\n    selected_features,\n    train_path=\"/kaggle/input/drw-crypto-market-prediction/train.parquet\",\n    test_path=\"/kaggle/input/drw-crypto-market-prediction/test.parquet\",\n    train_size=100000,  # Reduced from 150000\n    submission_path=\"submission.csv\"\n):\n    \"\"\"\n    Train lightweight model with selected features.\n    \"\"\"\n    \n    print(\"\\n\" + \"=\"*60)\n    print(\"TRAINING LIGHTWEIGHT MODEL\")\n    print(\"=\"*60)\n    \n    # Load training data\n    print(f\"\\nLoading {train_size} training samples...\")\n    train_df = pd.read_parquet(train_path)\n    if len(train_df) > train_size:\n        train_df = train_df.tail(train_size).reset_index(drop=True)\n    \n    # Optimize memory\n    for col in selected_features:\n        if col in train_df.columns:\n            train_df[col] = train_df[col].astype(np.float32)\n    \n    X_train = train_df[selected_features]\n    y_train = train_df['label'].astype(np.float32)\n    \n    del train_df\n    gc.collect()\n    \n    # Clean data\n    X_train = X_train.replace([np.inf, -np.inf], np.nan).fillna(X_train.median())\n    \n    # Load test data\n    print(\"Loading test data...\")\n    test_df = pd.read_parquet(test_path)\n    X_test = test_df[selected_features]\n    \n    # Optimize test data memory\n    for col in selected_features:\n        X_test[col] = X_test[col].astype(np.float32)\n    \n    X_test = X_test.replace([np.inf, -np.inf], np.nan).fillna(X_test.median())\n    \n    # Scale features\n    print(\"Scaling features...\")\n    scaler = StandardScaler()\n    X_train_scaled = scaler.fit_transform(X_train)\n    X_test_scaled = scaler.transform(X_test)\n    \n    # Train single efficient model\n    print(\"Training LightGBM model...\")\n    model = LGBMRegressor(\n        n_estimators=300,\n        num_leaves=31,\n        max_depth=6,\n        learning_rate=0.05,\n        subsample=0.8,\n        colsample_bytree=0.8,\n        min_child_samples=20,\n        random_state=42,\n        n_jobs=4,  # Limit parallelism\n        device='cpu',\n        verbosity=-1\n    )\n    \n    model.fit(X_train_scaled, y_train)\n    \n    # Generate predictions\n    predictions = model.predict(X_test_scaled)\n    \n    # Create submission\n    submission = pd.DataFrame({\n        'id': np.arange(1, len(predictions) + 1),\n        'prediction': predictions\n    })\n    \n    submission.to_csv(submission_path, index=False)\n    print(f\"\\nSubmission saved to: {submission_path}\")\n    \n    # Clean up\n    del X_train, X_test, model\n    gc.collect()\n    \n    return submission\n\nif __name__ == \"__main__\":\n    # Run optimized mRMR selection\n    selected_features = run_optimized_mrmr_pipeline(\n        sample_size=30000,  # Reduced sample size\n        n_features_to_select=60,\n        optimize_count=True\n    )\n    \n    # Train model\n    submission = train_lightweight_model(\n        selected_features=selected_features,\n        train_size=100000  # Reduced training size\n    )\n    \n    print(\"\\n\" + \"=\"*60)\n    print(\"OPTIMIZED PIPELINE COMPLETE\")\n    print(\"=\"*60)\n    print(f\"Features selected: {len(selected_features)}\")\n    print(\"Submission saved: submission.csv\")","metadata":{"_kg_hide-output":true,"papermill":{"duration":8.748199,"end_time":"2025-05-23T19:03:38.419964","exception":false,"start_time":"2025-05-23T19:03:29.671765","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-05-29T07:52:57.1318Z","iopub.execute_input":"2025-05-29T07:52:57.13212Z","iopub.status.idle":"2025-05-29T07:53:04.690462Z","shell.execute_reply.started":"2025-05-29T07:52:57.13209Z","shell.execute_reply":"2025-05-29T07:53:04.689512Z"}},"outputs":[],"execution_count":null}]}