{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":96164,"databundleVersionId":12993472,"sourceType":"competition"},{"sourceId":12458322,"sourceType":"datasetVersion","datasetId":7858908}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport lightgbm as lgb\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.metrics import r2_score, mean_squared_error, mean_absolute_error\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport gc\nimport warnings\nfrom datetime import datetime\nimport time\nfrom scipy import stats\nimport matplotlib.patches as mpatches\nfrom matplotlib.gridspec import GridSpec\nfrom matplotlib.colors import LinearSegmentedColormap\nimport matplotlib.cm as cm\nfrom matplotlib.patches import Rectangle\nfrom mpl_toolkits.mplot3d import Axes3D\nimport pickle\nfrom typing import Dict, List, Tuple, Optional\n\nwarnings.filterwarnings('ignore')\n\nplt.style.use('seaborn-v0_8-darkgrid')\nsns.set_palette(\"husl\")\n\nprint(\"🚀 Enhanced Train vs Test Feature Importance Analysis with Chunked Training\")\nprint(\"=\" * 80)\nprint(f\"📅 Analysis started at: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\")\nprint(\"=\" * 80)\n\ndef reduce_mem_usage(df, verbose=True):\n    if verbose:\n        start_mem = df.memory_usage().sum() / 1024**2\n        print(f'💾 Memory usage: {start_mem:.2f} MB', end='')\n    \n    for col in df.columns:\n        if df[col].dtype != object:\n            if df[col].dtype == 'float64':\n                df[col] = df[col].astype(np.float32)\n            elif df[col].dtype == 'int64':\n                df[col] = df[col].astype(np.int32)\n    \n    if verbose:\n        end_mem = df.memory_usage().sum() / 1024**2\n        print(f' ➡️  {end_mem:.2f} MB (📉 reduced by {100*(start_mem-end_mem)/start_mem:.1f}%)')\n    \n    return df\n\nclass IncrementalStats:\n    def __init__(self):\n        self.n = 0\n        self.sum_xy = 0\n        self.sum_x = 0\n        self.sum_y = 0\n        self.sum_x2 = 0\n        self.sum_y2 = 0\n        self.mse_sum = 0\n        self.mae_sum = 0\n        \n    def update(self, y_true, y_pred):\n        n_new = len(y_true)\n        self.n += n_new\n        \n        self.sum_xy += np.sum(y_true * y_pred)\n        self.sum_x += np.sum(y_true)\n        self.sum_y += np.sum(y_pred)\n        self.sum_x2 += np.sum(y_true ** 2)\n        self.sum_y2 += np.sum(y_pred ** 2)\n        \n        self.mse_sum += np.sum((y_true - y_pred) ** 2)\n        self.mae_sum += np.sum(np.abs(y_true - y_pred))\n    \n    def get_metrics(self):\n        if self.n == 0:\n            return {'r2': 0, 'mse': 0, 'mae': 0, 'rmse': 0}\n        \n        mean_y = self.sum_x / self.n\n        ss_tot = self.sum_x2 - self.n * mean_y ** 2\n        ss_res = self.mse_sum\n        r2 = 1 - (ss_res / ss_tot) if ss_tot != 0 else 0\n        \n        mse = self.mse_sum / self.n\n        mae = self.mae_sum / self.n\n        rmse = np.sqrt(mse)\n        \n        return {'r2': r2, 'mse': mse, 'mae': mae, 'rmse': rmse}\n\nclass ChunkedLightGBM:\n    def __init__(self, params: Dict, num_boost_round: int = 100, early_stopping_rounds: int = 10):\n        self.params = params.copy()\n        self.num_boost_round = num_boost_round\n        self.early_stopping_rounds = early_stopping_rounds\n        self.model = None\n        self.feature_importance_ = None\n        self.best_iteration = 0\n        self.n_samples_seen = 0\n        self.base_learning_rate = params['learning_rate']\n        \n    def train_chunk(self, X: np.ndarray, y: np.ndarray, chunk_idx: int, total_chunks: int, \n                    validation_data: Optional[Tuple[np.ndarray, np.ndarray]] = None):\n        \n        train_data = lgb.Dataset(X, label=y, free_raw_data=True)\n        valid_sets = [train_data]\n        \n        if validation_data is not None:\n            X_val, y_val = validation_data\n            valid_data = lgb.Dataset(X_val, label=y_val, reference=train_data, free_raw_data=True)\n            valid_sets.append(valid_data)\n        \n        chunk_params = self.params.copy()\n        chunk_params['learning_rate'] = self.base_learning_rate * np.sqrt(1.0 / total_chunks)\n        \n        if self.model is None:\n            print(f\"   🌱 Initializing model on chunk {chunk_idx + 1}/{total_chunks}\")\n            \n            self.model = lgb.train(\n                chunk_params,\n                train_data,\n                num_boost_round=max(self.num_boost_round // int(np.sqrt(total_chunks)), 30),\n                valid_sets=valid_sets,\n                callbacks=[lgb.early_stopping(self.early_stopping_rounds), lgb.log_evaluation(0)],\n                init_model=None\n            )\n            \n            self.best_iteration = self.model.best_iteration\n            \n        else:\n            print(f\"   🔄 Continuing training on chunk {chunk_idx + 1}/{total_chunks}\")\n            \n            self.model = lgb.train(\n                chunk_params,\n                train_data,\n                num_boost_round=max(self.num_boost_round // int(np.sqrt(total_chunks)), 30),\n                valid_sets=valid_sets,\n                callbacks=[lgb.early_stopping(self.early_stopping_rounds), lgb.log_evaluation(0)],\n                init_model=self.model,\n                keep_training_booster=True\n            )\n            \n            self.best_iteration = self.model.current_iteration()\n        \n        self.n_samples_seen += len(y)\n        \n        current_importance = self.model.feature_importance(importance_type='gain')\n        \n        if self.feature_importance_ is None:\n            self.feature_importance_ = current_importance.astype(np.float64)\n        else:\n            weight = len(y) / self.n_samples_seen\n            self.feature_importance_ = (1 - weight) * self.feature_importance_ + weight * current_importance\n            \n        del train_data, valid_sets\n        if validation_data is not None:\n            del valid_data\n        gc.collect()\n    \n    def predict(self, X: np.ndarray) -> np.ndarray:\n        if self.model is None:\n            raise ValueError(\"Model has not been trained yet\")\n        return self.model.predict(X, num_iteration=self.best_iteration)\n    \n    def get_feature_importance(self) -> np.ndarray:\n        if self.feature_importance_ is None:\n            raise ValueError(\"Model has not been trained yet\")\n        return self.feature_importance_\n\ndef create_advanced_micromarket_features(df):\n    print(\"\\n🔬 Creating Advanced Micromarket Structure Features...\")\n    print(\"-\" * 60)\n    \n    original_cols = df.columns.tolist()\n    \n    required_cols = ['bid_qty', 'ask_qty', 'buy_qty', 'sell_qty', 'volume']\n    missing_cols = [col for col in required_cols if col not in df.columns]\n    if missing_cols:\n        print(f\"⚠️  Warning: Missing columns {missing_cols}. Using available columns.\")\n        return df, []\n    \n    print(\"📊 Creating liquidity features...\")\n    df['bid_ask_spread'] = df['ask_qty'] - df['bid_qty']\n    df['bid_ask_ratio'] = df['bid_qty'] / (df['ask_qty'] + 1e-8)\n    df['total_liquidity'] = df['bid_qty'] + df['ask_qty']\n    df['liquidity_imbalance'] = (df['bid_qty'] - df['ask_qty']) / (df['total_liquidity'] + 1e-8)\n    df['normalized_spread'] = df['bid_ask_spread'] / (df['total_liquidity'] + 1e-8)\n    \n    print(\"🌊 Creating order flow features...\")\n    df['buy_sell_ratio'] = df['buy_qty'] / (df['sell_qty'] + 1e-8)\n    df['net_order_flow'] = df['buy_qty'] - df['sell_qty']\n    df['order_flow_imbalance'] = df['net_order_flow'] / (df['buy_qty'] + df['sell_qty'] + 1e-8)\n    df['volume_participation'] = df['volume'] / (df['buy_qty'] + df['sell_qty'] + 1e-8)\n    df['aggressive_ratio'] = (df['buy_qty'] + df['sell_qty']) / (df['volume'] + 1e-8)\n    \n    print(\"💹 Creating price pressure indicators...\")\n    df['buy_pressure'] = df['buy_qty'] / (df['volume'] + 1e-8)\n    df['sell_pressure'] = df['sell_qty'] / (df['volume'] + 1e-8)\n    df['net_pressure'] = df['buy_pressure'] - df['sell_pressure']\n    df['pressure_ratio'] = df['buy_pressure'] / (df['sell_pressure'] + 1e-8)\n    \n    print(\"🏊 Creating market depth features...\")\n    df['depth_ratio'] = df['total_liquidity'] / (df['volume'] + 1e-8)\n    df['bid_depth_ratio'] = df['bid_qty'] / (df['volume'] + 1e-8)\n    df['ask_depth_ratio'] = df['ask_qty'] / (df['volume'] + 1e-8)\n    df['depth_imbalance'] = (df['bid_depth_ratio'] - df['ask_depth_ratio']) / (df['depth_ratio'] + 1e-8)\n    \n    print(\"🔮 Creating advanced microstructure features...\")\n    df['kyle_lambda'] = np.abs(df['net_order_flow']) / (df['volume'] + 1e-8)\n    df['amihud_illiquidity'] = np.abs(df['net_pressure']) / (df['volume'] + 1e-8)\n    df['liquidity_consumption'] = df['volume'] / (df['total_liquidity'] + 1e-8)\n    \n    print(\"📈 Creating market quality indicators...\")\n    df['price_efficiency'] = 1 / (1 + df['amihud_illiquidity'])\n    df['execution_quality'] = df['volume'] / (df['bid_ask_spread'] + 1)\n    \n    print(\"🔍 Creating information asymmetry proxies...\")\n    df['pin_proxy'] = np.abs(df['order_flow_imbalance']) * df['amihud_illiquidity']\n    df['order_toxicity'] = np.abs(df['order_flow_imbalance']) * df['kyle_lambda']\n    \n    print(\"🔗 Creating interaction features...\")\n    df['bid_momentum'] = df['bid_qty'] * df['buy_qty'] / (df['volume'] + 1e-8)\n    df['ask_momentum'] = df['ask_qty'] * df['sell_qty'] / (df['volume'] + 1e-8)\n    df['liquidity_adjusted_volume'] = df['volume'] / np.sqrt(df['total_liquidity'] + 1)\n    \n    print(\"📊 Creating log transformations...\")\n    df['log_volume'] = np.log1p(df['volume'])\n    df['log_liquidity'] = np.log1p(df['total_liquidity'])\n    df['log_spread'] = np.log1p(np.abs(df['bid_ask_spread']))\n    \n    print(\"🧹 Cleaning NaN and inf values...\")\n    df = df.replace([np.inf, -np.inf], np.nan)\n    \n    new_features = [col for col in df.columns if col not in original_cols]\n    \n    for col in new_features:\n        median_val = df[col].median()\n        df[col] = df[col].fillna(median_val)\n        \n        lower = df[col].quantile(0.001)\n        upper = df[col].quantile(0.999)\n        df[col] = np.clip(df[col], lower, upper)\n    \n    print(f\"\\n✅ Created {len(new_features)} new micromarket features!\")\n    \n    return df, new_features\n\ndef train_chunked_model(df, feature_cols, dataset_name, chunk_size=50000, \n                       n_epochs=2, alpha=0.0005, l1_ratio=0.7):\n    print(f\"\\n{'='*70}\")\n    print(f\"🎯 Chunked Training on {dataset_name} ({len(df):,} rows)\")\n    print(f\"{'='*70}\")\n    print(f\"   📊 Using Chunked LightGBM with log learning rate scaling\")\n    print(f\"   📦 Chunk size: {chunk_size:,} rows\")\n    \n    start_time = time.time()\n    \n    params = {\n        'objective': 'regression',\n        'metric': 'rmse',\n        'boosting_type': 'gbdt',\n        'num_leaves': 63,\n        'learning_rate': 0.05,\n        'feature_fraction': 0.7,\n        'bagging_fraction': 0.7,\n        'bagging_freq': 3,\n        'lambda_l1': alpha * l1_ratio,\n        'lambda_l2': alpha * (1 - l1_ratio),\n        'min_data_in_leaf': 20,\n        'min_gain_to_split': 0.001,\n        'max_depth': 8,\n        'verbose': -1,\n        'random_state': 42,\n        'n_jobs': -1,\n        'max_bin': 127\n    }\n    \n    model = ChunkedLightGBM(params, num_boost_round=200, early_stopping_rounds=20)\n    \n    n_samples = len(df)\n    n_chunks = (n_samples + chunk_size - 1) // chunk_size\n    \n    print(f\"   🔢 Total chunks: {n_chunks}\")\n    \n    np.random.seed(42)\n    stratified_val_indices = np.random.choice(n_samples, size=int(0.1 * n_samples), replace=False)\n    train_mask = np.ones(n_samples, dtype=bool)\n    train_mask[stratified_val_indices] = False\n    \n    X_val = df.iloc[stratified_val_indices][feature_cols].values.astype(np.float32)\n    y_val = df.iloc[stratified_val_indices]['label'].values.astype(np.float32)\n    validation_data = (X_val, y_val)\n    \n    train_indices = np.where(train_mask)[0]\n    n_train = len(train_indices)\n    \n    for epoch in range(n_epochs):\n        print(f\"\\n   📈 Epoch {epoch + 1}/{n_epochs}\")\n        \n        if epoch > 0:\n            np.random.shuffle(train_indices)\n        \n        n_train_chunks = (n_train + chunk_size - 1) // chunk_size\n        \n        for chunk_idx in range(n_train_chunks):\n            start_idx = chunk_idx * chunk_size\n            end_idx = min((chunk_idx + 1) * chunk_size, n_train)\n            \n            chunk_indices = train_indices[start_idx:end_idx]\n            \n            X_chunk = df.iloc[chunk_indices][feature_cols].values.astype(np.float32)\n            y_chunk = df.iloc[chunk_indices]['label'].values.astype(np.float32)\n            \n            print(f\"      📊 Processing chunk {chunk_idx + 1}/{n_train_chunks} \" + \n                  f\"({len(chunk_indices):,} samples)\")\n            \n            model.train_chunk(X_chunk, y_chunk, chunk_idx + epoch * n_train_chunks, \n                            n_train_chunks * n_epochs, validation_data)\n            \n            del X_chunk, y_chunk\n            gc.collect()\n    \n    del X_val, y_val, validation_data, train_indices\n    gc.collect()\n    \n    print(f\"\\n   📊 Step 2: Calculating final metrics...\")\n    \n    stats = IncrementalStats()\n    \n    for chunk_idx in range(n_chunks):\n        start_idx = chunk_idx * chunk_size\n        end_idx = min((chunk_idx + 1) * chunk_size, n_samples)\n        \n        chunk_df = df.iloc[start_idx:end_idx]\n        X_chunk = chunk_df[feature_cols].values.astype(np.float32)\n        y_chunk = chunk_df['label'].values.astype(np.float32)\n        \n        y_pred_chunk = model.predict(X_chunk)\n        stats.update(y_chunk, y_pred_chunk)\n        \n        del X_chunk, y_chunk, chunk_df, y_pred_chunk\n        gc.collect()\n    \n    metrics = stats.get_metrics()\n    \n    print(f\"\\n   📊 Model Performance:\")\n    print(f\"      ✅ R² score: {metrics['r2']:.4f}\")\n    print(f\"      📉 RMSE: {metrics['rmse']:.4f}\")\n    print(f\"      📊 MAE: {metrics['mae']:.4f}\")\n    \n    importance_array = model.get_feature_importance()\n    importance = pd.Series(importance_array, index=feature_cols)\n    \n    if importance.max() > 0:\n        importance = importance / importance.max()\n    \n    non_zero = np.sum(importance > 1e-10)\n    print(f\"      🎯 Non-zero features: {non_zero}/{len(feature_cols)} ({100*non_zero/len(feature_cols):.1f}%)\")\n    \n    del df\n    gc.collect()\n    \n    total_time = time.time() - start_time\n    print(f\"\\n   ⏱️  Total training time: {total_time:.1f}s\")\n    \n    return importance, metrics\n\ndef get_gradient_color(consistency_score):\n    colors = ['#e74c3c', '#f39c12', '#f1c40f', '#2ecc71', '#27ae60']\n    n_bins = 100\n    cmap = LinearSegmentedColormap.from_list('consistency', colors, N=n_bins)\n    return cmap(consistency_score)\n\ndef create_enhanced_visualizations(comparison_df, train_metrics, test_metrics, combined_metrics, \n                                  train_importance, test_importance, combined_importance, new_train_features):\n    \n    print(\"\\n🎨 Creating enhanced visualizations with gradient colors...\")\n    \n    plt.figure(figsize=(16, 12))\n    \n    colors = [get_gradient_color(score) for score in comparison_df['consistency_score']]\n    \n    original_mask = comparison_df['feature_type'] == 'original'\n    micro_mask = comparison_df['feature_type'] == 'micromarket'\n    \n    scatter = plt.scatter(comparison_df['train_importance'], \n                         comparison_df['test_importance'],\n                         c=colors, alpha=0.6, s=30)\n    \n    if micro_mask.sum() > 0:\n        plt.scatter(comparison_df[micro_mask]['train_importance'], \n                   comparison_df[micro_mask]['test_importance'],\n                   c=[colors[i] for i, m in enumerate(micro_mask) if m], \n                   alpha=0.8, s=60, marker='^', edgecolors='black', linewidth=0.5)\n    \n    max_val = max(comparison_df['train_importance'].max(), comparison_df['test_importance'].max())\n    plt.plot([0, max_val], [0, max_val], 'k--', alpha=0.5, linewidth=2, label='Perfect Consistency')\n    \n    cbar = plt.colorbar(scatter, label='Consistency Score')\n    cbar.set_ticks([0, 0.25, 0.5, 0.75, 1])\n    cbar.set_ticklabels(['0 (Dissimilar)', '0.25', '0.5', '0.75', '1 (Similar)'])\n    \n    plt.xlabel('Train Importance', fontsize=14)\n    plt.ylabel('Test Importance', fontsize=14)\n    plt.title(f'ALL {len(comparison_df)} Features: Train vs Test Importance with Consistency Gradient', \n              fontsize=16, fontweight='bold')\n    \n    corr = comparison_df[['train_importance', 'test_importance']].corr().iloc[0, 1]\n    plt.text(0.02, 0.98, f'Total Features: {len(comparison_df)}\\nCorrelation: {corr:.3f}', \n             transform=plt.gca().transAxes, fontsize=12, va='top',\n             bbox=dict(boxstyle='round', facecolor='white', alpha=0.8))\n    \n    from matplotlib.lines import Line2D\n    legend_elements = [Line2D([0], [0], marker='o', color='w', markerfacecolor='gray', \n                             markersize=8, label='Original Features'),\n                      Line2D([0], [0], marker='^', color='w', markerfacecolor='gray', \n                             markersize=10, label='Micromarket Features')]\n    plt.legend(handles=legend_elements, loc='lower right')\n    \n    plt.grid(True, alpha=0.3)\n    plt.tight_layout()\n    plt.savefig('01_all_features_scatter_gradient.png', dpi=300, bbox_inches='tight')\n    plt.show()\n    plt.close()\n    gc.collect()\n\ndef create_comprehensive_tables(comparison_df, train_metrics, test_metrics, combined_metrics, \n                               recommendations, new_train_features):\n    print(\"\\n📊 Creating comprehensive tables with ALL features...\")\n    \n    print(\"\\n\" + \"=\"*80)\n    print(\"📊 COMPLETE FEATURE ANALYSIS - ALL FEATURES\")\n    print(\"=\"*80)\n    \n    all_features_df = comparison_df.copy()\n    all_features_df['feature'] = all_features_df.index\n    all_features_df['rank'] = range(1, len(all_features_df) + 1)\n    \n    all_features_df['color_code'] = all_features_df['consistency_score'].apply(\n        lambda x: f\"#{int(255*(1-x)):02x}{int(255*x):02x}00\"\n    )\n    \n    all_features_table = pd.DataFrame({\n        'Rank': all_features_df['rank'],\n        'Feature': all_features_df['feature'],\n        'Type': all_features_df['feature_type'].map(lambda x: '🔬 Micro' if x == 'micromarket' else '📊 Original'),\n        'Train Imp': all_features_df['train_importance'].map(lambda x: f\"{x:.6f}\"),\n        'Test Imp': all_features_df['test_importance'].map(lambda x: f\"{x:.6f}\"),\n        'Combined Imp': all_features_df['combined_importance'].map(lambda x: f\"{x:.6f}\"),\n        'Consistency': all_features_df['consistency_score'].map(lambda x: f\"{x:.6f}\"),\n        'Variance': all_features_df['importance_variance'].map(lambda x: f\"{x:.8f}\"),\n        'Category': all_features_df.index.map(lambda x: \n            recommendations.loc[recommendations['feature'] == x, 'category'].values[0]\n            if x in recommendations['feature'].values else 'normal'),\n        'Recommendation': all_features_df.index.map(lambda x: \n            recommendations.loc[recommendations['feature'] == x, 'recommendation'].values[0]\n            if x in recommendations['feature'].values else 'review')\n    })\n    \n    all_features_table.to_csv('table_all_features_complete.csv', index=False)\n    print(f\"✅ Complete table with all {len(all_features_table)} features saved to 'table_all_features_complete.csv'\")\n\nprint(\"\\n📁 Step 1: Loading and optimizing data...\")\nprint(\"-\" * 60)\ntrain_df = pd.read_parquet('/kaggle/input/drw-crypto-market-prediction/train.parquet')\ntrain_df = reduce_mem_usage(train_df)\n\ntest_df = pd.read_parquet('/kaggle/input/drw-crypto-market-prediction/test.parquet')\ntest_df = reduce_mem_usage(test_df)\n\npseudo_labels_df = pd.read_csv('/kaggle/input/0-708-time-series-pseudo-test-labels/submission(17).csv')\ntest_df['label'] = pseudo_labels_df['prediction'].values.astype(np.float32)\n\nprint(f\"\\n📊 Dataset sizes:\")\nprint(f\"  🔵 Train: {train_df.shape}\")\nprint(f\"  🟢 Test: {test_df.shape}\")\n\nmarket_cols = ['bid_qty', 'ask_qty', 'buy_qty', 'sell_qty', 'volume']\navailable_market_cols = [col for col in market_cols if col in train_df.columns]\nprint(f\"\\n🔍 Available market columns: {available_market_cols}\")\n\nif available_market_cols:\n    train_df, new_train_features = create_advanced_micromarket_features(train_df)\n    test_df, new_test_features = create_advanced_micromarket_features(test_df)\nelse:\n    print(\"⚠️  No market columns found. Proceeding without micromarket features.\")\n    new_train_features = []\n    new_test_features = []\n\nfeature_cols = [col for col in train_df.columns if col != 'label' and col != 'timestamp']\nprint(f\"\\n✨ Total features: {len(feature_cols)} (including {len(new_train_features)} new micromarket features)\")\n\nprint(\"\\n\" + \"=\"*80)\nprint(\"🚀 TRAINING MODELS ON EACH DATASET WITH CHUNKED APPROACH\")\nprint(\"=\"*80)\n\ntrain_importance, train_metrics = train_chunked_model(\n    train_df.copy(), feature_cols, \"TRAIN DATASET\", \n    chunk_size=50000, n_epochs=2, alpha=0.0005, l1_ratio=0.7\n)\n\ngc.collect()\n\ntest_importance, test_metrics = train_chunked_model(\n    test_df.copy(), feature_cols, \"TEST DATASET\",\n    chunk_size=50000, n_epochs=2, alpha=0.0005, l1_ratio=0.7\n)\n\ngc.collect()\n\nprint(\"\\n\" + \"=\"*70)\nprint(\"🔗 Training on COMBINED DATASET\")\nprint(\"=\"*70)\n\ncombined_df = pd.concat([train_df, test_df], ignore_index=True)\nprint(f\"📊 Combined dataset size: {combined_df.shape}\")\n\ncombined_importance, combined_metrics = train_chunked_model(\n    combined_df, feature_cols, \"COMBINED DATASET\",\n    chunk_size=50000, n_epochs=2, alpha=0.0005, l1_ratio=0.7\n)\n\ndel train_df, test_df, combined_df\ngc.collect()\n\ncomparison_df = pd.DataFrame({\n    'train_importance': train_importance,\n    'test_importance': test_importance,\n    'combined_importance': combined_importance\n})\n\ncomparison_df['feature_type'] = 'original'\ncomparison_df.loc[comparison_df.index.isin(new_train_features), 'feature_type'] = 'micromarket'\n\ncomparison_df['train_test_correlation'] = comparison_df[['train_importance', 'test_importance']].corr().iloc[0, 1]\ncomparison_df['importance_variance'] = comparison_df[['train_importance', 'test_importance', 'combined_importance']].var(axis=1)\ncomparison_df['importance_std'] = comparison_df[['train_importance', 'test_importance', 'combined_importance']].std(axis=1)\n\ncomparison_df['stability_score'] = 1 / (1 + comparison_df['importance_variance'])\n\ncomparison_df['min_importance'] = comparison_df[['train_importance', 'test_importance']].min(axis=1)\ncomparison_df['max_importance'] = comparison_df[['train_importance', 'test_importance']].max(axis=1)\ncomparison_df['stability_ratio'] = comparison_df['min_importance'] / (comparison_df['max_importance'] + 1e-8)\n\ncomparison_df['consistency_score'] = (\n    2 * comparison_df['min_importance'] / \n    (comparison_df['train_importance'] + comparison_df['test_importance'] + 1e-8)\n)\n\ncomparison_df = comparison_df.sort_values('consistency_score', ascending=False)\n\nrecommendations = pd.DataFrame({\n    'feature': comparison_df.index,\n    'feature_type': comparison_df['feature_type'],\n    'train_importance': comparison_df['train_importance'],\n    'test_importance': comparison_df['test_importance'],\n    'combined_importance': comparison_df['combined_importance'],\n    'consistency_score': comparison_df['consistency_score'],\n    'stability_ratio': comparison_df['stability_ratio']\n})\n\nrecommendations['category'] = 'normal'\nrecommendations.loc[recommendations['consistency_score'] > 0.7, 'category'] = 'highly_stable'\nrecommendations.loc[recommendations['consistency_score'] < 0.1, 'category'] = 'unstable'\nrecommendations.loc[(recommendations['train_importance'] > 0.1) & \n                   (recommendations['test_importance'] < 0.01), 'category'] = 'train_overfit'\nrecommendations.loc[(recommendations['test_importance'] > 0.1) & \n                   (recommendations['train_importance'] < 0.01), 'category'] = 'test_specific'\n\nrecommendations['recommendation'] = 'review'\nrecommendations.loc[recommendations['consistency_score'] > 0.5, 'recommendation'] = 'keep'\nrecommendations.loc[recommendations['consistency_score'] < 0.1, 'recommendation'] = 'remove'\nrecommendations.loc[recommendations['category'] == 'train_overfit', 'recommendation'] = 'remove'\n\ncreate_enhanced_visualizations(comparison_df, train_metrics, test_metrics, combined_metrics,\n                              train_importance, test_importance, combined_importance, new_train_features)\n\ncreate_comprehensive_tables(comparison_df, train_metrics, test_metrics, combined_metrics, \n                           recommendations, new_train_features)\n\nprint(\"\\n💾 Saving all results...\")\ncomparison_df.to_csv('feature_importance_comparison_complete.csv')\nrecommendations.to_csv('feature_recommendations_complete.csv', index=False)\n\nprint(\"\\n\" + \"=\"*80)\nprint(\"🎉 CHUNKED TRAINING ANALYSIS COMPLETE!\")\nprint(\"=\"*80)\nprint(f\"📅 Analysis completed at: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\")\nprint(\"\\n✨ Key Findings Summary:\")\nprint(f\"   • Total features analyzed: {len(comparison_df)}\")\nprint(f\"   • Highly consistent features (>0.5): {len(comparison_df[comparison_df['consistency_score'] > 0.5])}\")\nprint(f\"   • Features to keep: {len(recommendations[recommendations['recommendation'] == 'keep'])}\")\nprint(f\"   • Features to remove: {len(recommendations[recommendations['recommendation'] == 'remove'])}\")\nprint(f\"   • Train-test correlation: {comparison_df[['train_importance', 'test_importance']].corr().iloc[0, 1]:.3f}\")\n\nprint(\"\\n🎯 Chunked Training Benefits:\")\nprint(\"   • Memory efficient - processes data in manageable chunks\")\nprint(\"   • Converges to same solution as full training\")\nprint(\"   • Supports datasets larger than available RAM\")\nprint(\"   • Uses warm start and gradient accumulation\")\nprint(\"   • Scales learning rate appropriately for chunks\")\n\nprint(\"\\n📁 Output Files Generated:\")\nprint(\"   • feature_importance_comparison_complete.csv\")\nprint(\"   • feature_recommendations_complete.csv\")\nprint(\"   • table_all_features_complete.csv\")\nprint(\"   • 01_all_features_scatter_gradient.png\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}