{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom scipy import stats\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.decomposition import PCA\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.feature_selection import SelectKBest, f_regression\nfrom sklearn.metrics import mean_squared_error, r2_score\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pd.set_option('display.max_rows', None)\npd.set_option('display.max_columns', None)\nplt.style.use('default')\nsns.set_palette(\"husl\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"1. Dataset Summary","metadata":{}},{"cell_type":"code","source":"df=pd.read_parquet(\"/kaggle/input/drw-crypto-market-prediction/train.parquet\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.head(5)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Dataset Shape: {df.shape[0]:,} rows × {df.shape[1]:,} columns\")\nprint(f\"Memory Usage: {df.memory_usage(deep=True).sum() / 1024**2:.2f} MB\")\nprint(df.dtypes.value_counts())\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.columns","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"missing_summary = df.isnull().sum()\nmissing_percentage = (missing_summary / len(df)) * 100\nprint(missing_summary.sum())\nprint(missing_percentage.sum())","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Time Range: {df.index.min()} to {df.index.max()}\")\ntime_diff = df.index.max() - df.index.min()\nprint(f\"Total Duration: {time_diff}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df[['bid_qty', 'ask_qty', 'buy_qty', 'sell_qty', 'volume', 'label']].describe()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"2. Data Exploration Plan\n\nOBJECTIVE: Understand the crypto market dataset structure, patterns, and relationships to inform feature engineering and model selection for price movement prediction.\n\nEXPLORATION STRATEGY:\n\nDATA QUALITY ASSESSMENT\n- Duplicate records detection and analysis\n- Data consistency checks across timestamps\n- Anomalous patterns identification\n- Missing data patterns analysis (future-proofing)\n- Data integrity validation\n\nTEMPORAL ANALYSIS\n- Examine time series patterns and trends\n- Identify seasonality in minute-level data\n- Analyze trading volume patterns across different time periods\n- Check for any gaps or irregularities in timestamps\n- Intraday patterns analysis (market open/close effects)\n- Weekend vs weekday trading patterns\n\nMARKET MICROSTRUCTURE ANALYSIS\n- Analyze bid-ask spread patterns\n- Examine order book depth (bid_qty vs ask_qty)\n- Study executed trade volumes (buy_qty vs sell_qty)\n- Investigate volume distribution and outliers\n- Liquidity analysis across different time periods\n- Market impact and price discovery mechanisms\n\nTARGET VARIABLE ANALYSIS\n- Distribution analysis of price movement labels\n- Temporal patterns in price movements\n- Correlation with market features\n- Identify extreme movements and their characteristics\n- Persistence/momentum analysis in labels\n- Transition probability matrix for label sequences\n\nFEATURE CORRELATION ANALYSIS\n- Correlation matrix for market features\n- Relationship between anonymized features (X1 to X890)\n- Feature importance preliminary assessment\n- Multicollinearity detection\n- Cross-correlation analysis with lags\n\nSTATISTICAL DISTRIBUTIONS\n- Normality tests for key features\n- Outlier detection and analysis\n- Skewness and kurtosis analysis\n- Feature scaling requirements assessment\n- Heavy-tail behavior identification\n\nMARKET REGIME IDENTIFICATION\n- High vs low volatility periods\n- Bull vs bear market conditions\n- Volume-based market classification\n- Time-of-day effects\n- News/event impact periods\n- Market stress indicators\n\nFEATURE ENGINEERING PLANNING\n- Technical indicators to be created (RSI, MACD, Bollinger Bands, etc.)\n- Rolling statistics strategy (moving averages, volatility)\n- Lag features implementation plan\n- Feature interaction possibilities\n- Derived features from order book data\n\nPREDICTIVE INSIGHTS ANALYSIS\n- Identify patterns predictive of future movements\n- Lead-lag relationships analysis\n- Market efficiency tests\n- Information content assessment of features\n- Predictive power evaluation of different time horizons\n","metadata":{}},{"cell_type":"markdown","source":"3. Exploratory Data Analysis(EDA)","metadata":{}},{"cell_type":"code","source":"df_visual=df.copy()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# 1. TARGET VARIABLE ANALYSIS\n\nfig, axes = plt.subplots(2, 2, figsize=(15, 10))\n\n# Distribution of target variable\naxes[0,0].hist(df_visual['label'], bins=50, alpha=0.7, color='skyblue', edgecolor='black')\naxes[0,0].set_title('Distribution of Target Variable (Label)')\naxes[0,0].set_xlabel('Label Value')\naxes[0,0].set_ylabel('Frequency')\n\n# Box plot for target variable\naxes[0,1].boxplot(df_visual['label'])\naxes[0,1].set_title('Box Plot of Target Variable')\naxes[0,1].set_ylabel('Label Value')\n\n# Q-Q plot for normality check\nstats.probplot(df_visual['label'], dist=\"norm\", plot=axes[1,0])\naxes[1,0].set_title('Q-Q Plot: Label vs Normal Distribution')\n\n# Time series plot of target variable (sample)\nsample_data = df_visual.sample(n=min(10000, len(df_visual))).sort_index()\naxes[1,1].plot(sample_data.index, sample_data['label'], alpha=0.7, linewidth=0.5, color='red')\naxes[1,1].set_title('Time Series Plot of Target Variable (Sample)')\naxes[1,1].set_xlabel('Timestamp')\naxes[1,1].set_ylabel('Label Value')\naxes[1,1].tick_params(axis='x', rotation=45)\n\nplt.tight_layout()\nplt.show()\n\n# Target variable statistics\nprint(f\"\\nTarget Variable Statistics:\")\nprint(f\"Mean: {df_visual['label'].mean():.6f}\")\nprint(f\"Std: {df_visual['label'].std():.6f}\")\nprint(f\"Skewness: {df_visual['label'].skew():.6f}\")\nprint(f\"Kurtosis: {df_visual['label'].kurtosis():.6f}\")\nprint(f\"Range: [{df_visual['label'].min():.3f}, {df_visual['label'].max():.3f}]\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 2. MARKET FEATURES CORRELATION ANALYSIS\n\n# Correlation matrix for market features\nmarket_features = ['bid_qty', 'ask_qty', 'buy_qty', 'sell_qty', 'volume']\ncorrelation_matrix = df_visual[market_features + ['label']].corr()\n\nplt.figure(figsize=(10, 8))\nsns.heatmap(correlation_matrix, annot=True, cmap='coolwarm', center=0, \n            square=True, linewidths=0.5, cbar_kws={\"shrink\": .8})\nplt.title('Correlation Matrix: Market Features vs Target', fontsize=14, pad=20)\nplt.tight_layout()\nplt.show()\n\n# Multicollinearity detection\nprint(\"\\nMulticollinearity Analysis:\")\nhigh_corr_pairs = []\nfor i in range(len(correlation_matrix.columns)):\n    for j in range(i+1, len(correlation_matrix.columns)):\n        corr_val = correlation_matrix.iloc[i, j]\n        if abs(corr_val) > 0.7:  # High correlation threshold\n            high_corr_pairs.append((correlation_matrix.columns[i], \n                                  correlation_matrix.columns[j], corr_val))\n\nfor pair in high_corr_pairs:\n    print(f\"High correlation: {pair[0]} <-> {pair[1]}: {pair[2]:.3f}\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 3. VOLUME AND QUANTITY DISTRIBUTION ANALYSIS\n\nfig, axes = plt.subplots(2, 3, figsize=(18, 12))\n\n# Distribution plots for each market feature (log-transformed)\nfor i, feature in enumerate(market_features):\n    row = i // 3\n    col = i % 3\n    \n    # Log scale for better visualization\n    data_log = np.log1p(df_visual[feature])\n    \n    axes[row, col].hist(data_log, bins=50, alpha=0.7, edgecolor='black', color='lightcoral')\n    axes[row, col].set_title(f'Log Distribution of {feature}')\n    axes[row, col].set_xlabel(f'Log({feature})')\n    axes[row, col].set_ylabel('Frequency')\n    \n    # Add statistics text\n    axes[row, col].text(0.02, 0.98, f'Skew: {data_log.skew():.2f}', \n                       transform=axes[row, col].transAxes, verticalalignment='top',\n                       bbox=dict(boxstyle='round', facecolor='white', alpha=0.8))\n\n# Remove empty subplot\nif len(market_features) < 6:\n    fig.delaxes(axes[1, 2])\n\nplt.tight_layout()\nplt.show()\n\n# Statistical summary\nprint(\"\\nSkewness Analysis (Original Scale):\")\nfor feature in market_features:\n    skew_val = df_visual[feature].skew()\n    print(f\"{feature}: {skew_val:.3f} {'(Highly Skewed)' if abs(skew_val) > 2 else '(Moderately Skewed)' if abs(skew_val) > 1 else '(Normal)'}\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 4. MARKET MICROSTRUCTURE ANALYSIS\n\n# Create market microstructure features\ndf_visual['bid_ask_ratio'] = df_visual['bid_qty'] / (df_visual['ask_qty'] + 1e-8)\ndf_visual['volume_imbalance'] = (df_visual['buy_qty'] - df_visual['sell_qty']) / (df_visual['buy_qty'] + df_visual['sell_qty'] + 1e-8)\ndf_visual['total_order_book'] = df_visual['bid_qty'] + df_visual['ask_qty']\ndf_visual['liquidity_ratio'] = df_visual['volume'] / (df_visual['total_order_book'] + 1e-8)\n\nfig, axes = plt.subplots(2, 2, figsize=(16, 10))\n\n# Bid-Ask Ratio Distribution\naxes[0,0].hist(np.log1p(df_visual['bid_ask_ratio']), bins=50, alpha=0.7, color='lightcoral')\naxes[0,0].set_title('Distribution of Bid-Ask Ratio (Log Scale)')\naxes[0,0].set_xlabel('Log(Bid Qty / Ask Qty)')\naxes[0,0].set_ylabel('Frequency')\n\n# Volume Imbalance Distribution\naxes[0,1].hist(df_visual['volume_imbalance'], bins=50, alpha=0.7, color='lightgreen')\naxes[0,1].set_title('Distribution of Volume Imbalance')\naxes[0,1].set_xlabel('(Buy Qty - Sell Qty) / (Buy Qty + Sell Qty)')\naxes[0,1].set_ylabel('Frequency')\n\n# Liquidity Ratio vs Label\nsample_indices = np.random.choice(len(df_visual), size=min(5000, len(df_visual)), replace=False)\naxes[1,0].scatter(df_visual['liquidity_ratio'].iloc[sample_indices], \n                 df_visual['label'].iloc[sample_indices], alpha=0.5, s=1, color='purple')\naxes[1,0].set_title('Liquidity Ratio vs Target Label')\naxes[1,0].set_xlabel('Liquidity Ratio')\naxes[1,0].set_ylabel('Label')\n\n# Volume vs Label (enhanced)\naxes[1,1].scatter(df_visual['volume'].iloc[sample_indices], \n                 df_visual['label'].iloc[sample_indices], alpha=0.5, s=1, color='orange')\naxes[1,1].set_title('Volume vs Target Label (Sample)')\naxes[1,1].set_xlabel('Volume')\naxes[1,1].set_ylabel('Label')\n\nplt.tight_layout()\nplt.show()\n\n# Market microstructure correlations\nmicrostructure_features = ['bid_ask_ratio', 'volume_imbalance', 'liquidity_ratio']\nmicro_corr = df_visual[microstructure_features + ['label']].corr()['label']\nprint(\"\\nMicrostructure Features Correlation with Target:\")\nfor feature in microstructure_features:\n    print(f\"{feature}: {micro_corr[feature]:.6f}\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 5. TEMPORAL PATTERNS ANALYSIS\n\n# Extract time components\ndf_visual['hour'] = df_visual.index.hour\ndf_visual['minute'] = df_visual.index.minute\ndf_visual['day_of_week'] = df_visual.index.dayofweek\ndf_visual['is_weekend'] = (df_visual.index.dayofweek >= 5).astype(int)\n\n# Advanced hourly analysis\nhourly_stats = df_visual.groupby('hour').agg({\n    'label': ['mean', 'std', 'count'],\n    'volume': ['mean', 'std'],\n    'volume_imbalance': 'mean'\n}).round(6)\n\n# Weekly analysis\nweekly_stats = df_visual.groupby('day_of_week').agg({\n    'label': ['mean', 'std'],\n    'volume': 'mean'\n}).round(6)\n\nfig, axes = plt.subplots(2, 3, figsize=(20, 10))\n\n# Hourly label mean\naxes[0,0].plot(hourly_stats.index, hourly_stats[('label', 'mean')], \n               marker='o', linewidth=2, markersize=6, color='red')\naxes[0,0].set_title('Average Target Label by Hour')\naxes[0,0].set_xlabel('Hour of Day')\naxes[0,0].set_ylabel('Average Label')\naxes[0,0].grid(True, alpha=0.3)\n\n# Hourly volume mean\naxes[0,1].plot(hourly_stats.index, hourly_stats[('volume', 'mean')], \n               marker='o', linewidth=2, markersize=6, color='blue')\naxes[0,1].set_title('Average Volume by Hour')\naxes[0,1].set_xlabel('Hour of Day')\naxes[0,1].set_ylabel('Average Volume')\naxes[0,1].grid(True, alpha=0.3)\n\n# Hourly volume imbalance\naxes[0,2].plot(hourly_stats.index, hourly_stats[('volume_imbalance', 'mean')], \n               marker='o', linewidth=2, markersize=6, color='green')\naxes[0,2].set_title('Average Volume Imbalance by Hour')\naxes[0,2].set_xlabel('Hour of Day')\naxes[0,2].set_ylabel('Volume Imbalance')\naxes[0,2].grid(True, alpha=0.3)\n\n# Day of week patterns\nday_names = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']\naxes[1,0].bar(range(7), weekly_stats[('label', 'mean')].values, color='lightblue', alpha=0.7)\naxes[1,0].set_title('Average Target Label by Day of Week')\naxes[1,0].set_xlabel('Day of Week')\naxes[1,0].set_ylabel('Average Label')\naxes[1,0].set_xticks(range(7))\naxes[1,0].set_xticklabels(day_names)\n\n# Weekend vs Weekday analysis\nweekend_analysis = df_visual.groupby('is_weekend').agg({\n    'label': ['mean', 'std'],\n    'volume': 'mean'\n}).round(6)\n\naxes[1,1].bar(['Weekday', 'Weekend'], weekend_analysis[('label', 'mean')].values, \n              color=['lightcoral', 'lightgreen'], alpha=0.7)\naxes[1,1].set_title('Weekday vs Weekend Label Distribution')\naxes[1,1].set_ylabel('Average Label')\n\n# Hourly volatility\naxes[1,2].plot(hourly_stats.index, hourly_stats[('label', 'std')], \n               marker='s', linewidth=2, markersize=6, color='purple')\naxes[1,2].set_title('Label Volatility by Hour')\naxes[1,2].set_xlabel('Hour of Day')\naxes[1,2].set_ylabel('Label Standard Deviation')\naxes[1,2].grid(True, alpha=0.3)\n\nplt.tight_layout()\nplt.show()\n\n# Statistical significance test for temporal patterns\nfrom scipy.stats import f_oneway\nhourly_groups = [df_visual[df_visual['hour'] == h]['label'].values for h in range(24)]\nf_stat, p_value = f_oneway(*hourly_groups)\nprint(f\"\\nHourly Pattern ANOVA Test:\")\nprint(f\"F-statistic: {f_stat:.4f}, p-value: {p_value:.2e}\")\nprint(f\"Hourly patterns are {'statistically significant' if p_value < 0.05 else 'not significant'}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 6. ADVANCED CORRELATION ANALYSIS\n\n# Identify X features with meaningful correlation\nexclude_cols = ['bid_qty', 'ask_qty', 'buy_qty', 'sell_qty', 'volume', \n                'bid_ask_ratio', 'volume_imbalance', 'total_order_book', 'liquidity_ratio',\n                'hour', 'minute', 'day_of_week', 'is_weekend', 'label']\nx_features_subset = [col for col in df_visual.columns if col not in exclude_cols]\n\n# Calculate correlations with target\ncorrelations = df_visual[x_features_subset + ['label']].corr()['label']\nx_correlations = correlations[:-1]  # Exclude label self-correlation\n\n# Find significant correlations\nabs_corr = x_correlations.abs()\nsignificant_features = abs_corr[abs_corr > 0.05].sort_values(ascending=False)\n\nprint(f\"Features with |correlation| > 0.05:\")\nprint(significant_features)\n\n# Top correlated X features analysis\nif len(significant_features) >= 6:\n    top_x_features = significant_features.head(6).index.tolist()\nelse:\n    top_x_features = significant_features.index.tolist()\n\nif len(top_x_features) > 0:\n    fig, axes = plt.subplots(2, 3, figsize=(18, 12))\n    \n    for i, feature in enumerate(top_x_features):\n        if i >= 6:  # Limit to 6 features\n            break\n            \n        row = i // 3\n        col = i % 3\n        \n        # Distribution plot\n        axes[row, col].hist(df_visual[feature], bins=50, alpha=0.7, edgecolor='black')\n        axes[row, col].set_title(f'Distribution of {feature}\\n(Corr: {x_correlations[feature]:.4f})')\n        axes[row, col].set_xlabel(f'{feature}')\n        axes[row, col].set_ylabel('Frequency')\n    \n    # Remove empty subplots if necessary\n    total_plots = len(top_x_features)\n    if total_plots < 6:\n        for i in range(total_plots, 6):\n            row = i // 3\n            col = i % 3\n            fig.delaxes(axes[row, col])\n    \n    plt.tight_layout()\n    plt.show()\n    \n    # Correlation matrix for top X features\n    if len(top_x_features) > 1:\n        plt.figure(figsize=(10, 8))\n        top_x_corr = df_visual[top_x_features + ['label']].corr()\n        sns.heatmap(top_x_corr, annot=True, cmap='coolwarm', center=0, \n                    square=True, linewidths=0.5)\n        plt.title('Correlation Matrix: Top X Features vs Target')\n        plt.tight_layout()\n        plt.show()\n\nelse:\n    print(\"No X features found with correlation > 0.05\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"4. Data Cleaning & Feature Engineering","metadata":{}},{"cell_type":"code","source":"# Step 4: Data Cleaning & Feature Engineering\nprint(\"DATA CLEANING & FEATURE ENGINEERING\")\n\ndf_processed = df.copy()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 4.1 MEMORY OPTIMIZATION\n\ndef reduce_mem_usage(dataframe, dataset_name):\n    \"\"\"Optimize memory usage by downcasting numeric types\"\"\"\n    print(f'Reducing memory usage for: {dataset_name}')\n    initial_mem_usage = dataframe.memory_usage().sum() / 1024**2\n    \n    for col in dataframe.columns:\n        col_type = dataframe[col].dtype\n        c_min = dataframe[col].min()\n        c_max = dataframe[col].max()\n        \n        if str(col_type)[:3] == 'int':\n            if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                dataframe[col] = dataframe[col].astype(np.int8)\n            elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                dataframe[col] = dataframe[col].astype(np.int16)\n            elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                dataframe[col] = dataframe[col].astype(np.int32)\n            elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                dataframe[col] = dataframe[col].astype(np.int64)\n        else:\n            if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                dataframe[col] = dataframe[col].astype(np.float16)\n            elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                dataframe[col] = dataframe[col].astype(np.float32)\n            else:\n                dataframe[col] = dataframe[col].astype(np.float64)\n    \n    final_mem_usage = dataframe.memory_usage().sum() / 1024**2\n    print(f'--- Memory usage before: {initial_mem_usage:.2f} MB')\n    print(f'--- Memory usage after: {final_mem_usage:.2f} MB')\n    print(f'--- Decreased by: {100 * (initial_mem_usage - final_mem_usage) / initial_mem_usage:.1f}%\\n')\n    return dataframe\n\n# Apply memory optimization\ndf_processed = reduce_mem_usage(df_processed, \"main_dataset\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 4.2 DATA QUALITY ASSESSMENT\n\n# Check for infinite values\nprint(\"Checking for infinite values...\")\ninf_summary = np.isinf(df_processed.select_dtypes(include=[np.number])).sum()\ninf_cols = inf_summary[inf_summary > 0]\n\nif len(inf_cols) > 0:\n    print(f\"Found infinite values in {len(inf_cols)} columns:\")\n    for col, count in inf_cols.items():\n        print(f\"  {col}: {count} infinite values\")\n    \n    # Drop columns with infinite values\n    df_processed = df_processed.drop(columns=inf_cols.index.tolist())\n    print(f\"Dropped {len(inf_cols)} columns with infinite values\")\nelse:\n    print(\"✓ No infinite values found\")\n\n# Check for duplicate timestamps\nduplicate_indices = df_processed.index.duplicated().sum()\nif duplicate_indices > 0:\n    print(f\"Found {duplicate_indices} duplicate timestamps - removing...\")\n    df_processed = df_processed[~df_processed.index.duplicated(keep='first')]\nelse:\n    print(\"✓ No duplicate timestamps\")\n\n# Detect low-variance features (< 10 unique values)\nprint(\"\\nDetecting low-variance features...\")\nlow_variance_cols = []\nfor col in df_processed.columns:\n    if df_processed[col].nunique() < 10:\n        low_variance_cols.append(col)\n\nif len(low_variance_cols) > 0:\n    df_processed = df_processed.drop(columns=low_variance_cols)\n    print(f\"Removed {len(low_variance_cols)} low-variance features\")\nelse:\n    print(\"✓ No low-variance features found\")\n\nprint(f\"Dataset shape after cleaning: {df_processed.shape}\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 4.3 MULTICOLLINEARITY RESOLUTION\n\n# Create derived features before dropping\nmarket_features = ['bid_qty', 'ask_qty', 'buy_qty', 'sell_qty', 'volume']\n\n# Ensure we have these columns\navailable_market_features = [col for col in market_features if col in df_processed.columns]\nprint(f\"Available market features: {available_market_features}\")\n\nif 'buy_qty' in df_processed.columns and 'sell_qty' in df_processed.columns:\n    # Buy-Sell dynamics\n    df_processed['buy_sell_ratio'] = df_processed['buy_qty'] / (df_processed['sell_qty'] + 1e-8)\n    df_processed['volume_imbalance'] = (df_processed['buy_qty'] - df_processed['sell_qty']) / (df_processed['buy_qty'] + df_processed['sell_qty'] + 1e-8)\n    df_processed['net_flow'] = df_processed['buy_qty'] - df_processed['sell_qty']\n    \n    # Now drop the highly correlated originals\n    df_processed = df_processed.drop(columns=['buy_qty', 'sell_qty'])\n    print(\"✓ Created buy_sell_ratio, volume_imbalance, net_flow\")\n    print(\"✓ Dropped buy_qty, sell_qty (high correlation with volume)\")\n\nif 'bid_qty' in df_processed.columns and 'ask_qty' in df_processed.columns:\n    # Order book features\n    df_processed['bid_ask_ratio'] = df_processed['bid_qty'] / (df_processed['ask_qty'] + 1e-8)\n    df_processed['bid_ask_spread'] = df_processed['bid_qty'] - df_processed['ask_qty']\n    df_processed['total_order_book'] = df_processed['bid_qty'] + df_processed['ask_qty']\n    print(\"✓ Created order book features: bid_ask_ratio, bid_ask_spread, total_order_book\")\n\n# Liquidity measures\nif 'volume' in df_processed.columns and 'total_order_book' in df_processed.columns:\n    df_processed['liquidity_ratio'] = df_processed['volume'] / (df_processed['total_order_book'] + 1e-8)\n    print(\"✓ Created liquidity_ratio\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 4.4 OUTLIER DETECTION AND TREATMENT\n\ndef detect_outliers_iqr(data, column, multiplier=1.5):\n    \"\"\"Detect outliers using IQR method\"\"\"\n    Q1 = data[column].quantile(0.25)\n    Q3 = data[column].quantile(0.75)\n    IQR = Q3 - Q1\n    lower_bound = Q1 - multiplier * IQR\n    upper_bound = Q3 + multiplier * IQR\n    return lower_bound, upper_bound\n\n# Define features to check for outliers\noutlier_features = ['volume'] + [col for col in df_processed.columns \n                                if col.startswith('bid_') or col.startswith('ask_') or \n                                   col.endswith('_ratio') or col.endswith('_imbalance')]\n\nprint(\"Applying outlier treatment to key features...\")\noutlier_summary = {}\n\nfor feature in outlier_features:\n    if feature in df_processed.columns:\n        before_count = len(df_processed)\n        lower, upper = detect_outliers_iqr(df_processed, feature, multiplier=2.0)  # Less aggressive\n        \n        # Cap outliers instead of removing (preserve data)\n        df_processed[feature] = df_processed[feature].clip(lower=lower, upper=upper)\n        \n        outlier_summary[feature] = {\n            'lower_bound': lower,\n            'upper_bound': upper,\n            'treatment': 'capped'\n        }\n\nprint(f\"✓ Applied outlier capping to {len(outlier_summary)} features\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 4.5 TEMPORAL FEATURE ENGINEERING (Based on EDA insights)\n\n# Basic time components\ndf_processed['hour'] = df_processed.index.hour\ndf_processed['minute'] = df_processed.index.minute  \ndf_processed['day_of_week'] = df_processed.index.dayofweek\ndf_processed['day_of_month'] = df_processed.index.day\ndf_processed['month'] = df_processed.index.month\n\n# EDA-based temporal patterns\n# Peak hour effect (21:00 showed highest label values)\ndf_processed['is_peak_hour'] = (df_processed['hour'] == 21).astype(int)\n\n# High volume hours (14:00-16:00 from EDA)\ndf_processed['is_high_volume_period'] = ((df_processed['hour'] >= 14) & \n                                        (df_processed['hour'] <= 16)).astype(int)\n\n# High volatility period (13:00-15:00 from EDA)\ndf_processed['is_high_volatility_period'] = ((df_processed['hour'] >= 13) & \n                                            (df_processed['hour'] <= 15)).astype(int)\n\n# Weekend effect\ndf_processed['is_weekend'] = (df_processed['day_of_week'] >= 5).astype(int)\n\n# Cyclical encoding for time features (preserves circular nature)\ndf_processed['hour_sin'] = np.sin(2 * np.pi * df_processed['hour'] / 24)\ndf_processed['hour_cos'] = np.cos(2 * np.pi * df_processed['hour'] / 24)\ndf_processed['minute_sin'] = np.sin(2 * np.pi * df_processed['minute'] / 60)\ndf_processed['minute_cos'] = np.cos(2 * np.pi * df_processed['minute'] / 60)\ndf_processed['day_of_week_sin'] = np.sin(2 * np.pi * df_processed['day_of_week'] / 7)\ndf_processed['day_of_week_cos'] = np.cos(2 * np.pi * df_processed['day_of_week'] / 7)\n\nprint(\"✓ Created temporal features:\")\nprint(\"  - Basic: hour, minute, day_of_week, day_of_month, month\")\nprint(\"  - EDA-based: is_peak_hour, is_high_volume_period, is_high_volatility_period\")\nprint(\"  - Cyclical: hour_sin/cos, minute_sin/cos, day_of_week_sin/cos\")\nprint(\"  - Weekend: is_weekend\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 4.6 ROLLING STATISTICS AND LAG FEATURES\n\n# Define rolling windows (in minutes)\nwindows = [5, 15, 30, 60]  \nkey_features = ['volume', 'bid_ask_ratio', 'volume_imbalance', 'liquidity_ratio']\n\n# Only create rolling features for available columns\navailable_key_features = [col for col in key_features if col in df_processed.columns]\n\nfor window in windows:\n    print(f\"  Creating {window}-minute rolling features...\")\n    \n    for feature in available_key_features:\n        # Rolling mean\n        df_processed[f'{feature}_ma_{window}'] = df_processed[feature].rolling(window=window, min_periods=1).mean()\n        \n        # Rolling standard deviation (volatility)\n        df_processed[f'{feature}_std_{window}'] = df_processed[feature].rolling(window=window, min_periods=1).std()\n        \n        # Rolling min/max for range analysis\n        df_processed[f'{feature}_min_{window}'] = df_processed[feature].rolling(window=window, min_periods=1).min()\n        df_processed[f'{feature}_max_{window}'] = df_processed[feature].rolling(window=window, min_periods=1).max()\n\n# Create lag features (previous values)\nprint(\"Creating lag features...\")\nlag_periods = [1, 3, 5, 10, 30]  # minutes\n\nfor feature in available_key_features:\n    for lag in lag_periods:\n        df_processed[f'{feature}_lag_{lag}'] = df_processed[feature].shift(lag)\n\nprint(f\"✓ Created rolling features for windows: {windows}\")\nprint(f\"✓ Created lag features for periods: {lag_periods}\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 4.7 X FEATURES ENGINEERING (Based on EDA correlation analysis)\n\n# Find all X features\nall_x_features = [col for col in df_processed.columns if col.startswith('X') and col != 'X']\n\n# Significant X features from EDA (update this list based on your actual EDA results)\nsignificant_x_features = ['X21', 'X20', 'X28', 'X19', 'X29', 'X863']\n\n# Keep only significant X features that exist in the dataset\navailable_significant_x = [col for col in significant_x_features if col in df_processed.columns]\n\n# Drop non-significant X features\nx_features_to_drop = [col for col in all_x_features if col not in available_significant_x]\n\nif len(x_features_to_drop) > 0:\n    df_processed = df_processed.drop(columns=x_features_to_drop)\n    print(f\"✓ Dropped {len(x_features_to_drop)} non-significant X features\")\n    print(f\"✓ Kept {len(available_significant_x)} significant X features: {available_significant_x}\")\nelse:\n    print(\"✓ All X features are significant or already filtered\")\n\n# Create X feature combinations (based on EDA showing high inter-correlation)\nif len(available_significant_x) >= 2:\n    print(\"Creating X feature interactions...\")\n    \n    # Create PCA-like combinations for highly correlated X features\n    # Top 3 X features from EDA\n    top_x_features = available_significant_x[:3]\n    \n    if len(top_x_features) >= 2:\n        df_processed['x_feature_sum'] = df_processed[top_x_features].sum(axis=1)\n        df_processed['x_feature_mean'] = df_processed[top_x_features].mean(axis=1)\n        df_processed['x_feature_std'] = df_processed[top_x_features].std(axis=1)\n        \n        print(f\"✓ Created X feature combinations from: {top_x_features}\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 4.8 ADVANCED FEATURE ENGINEERING\n\n# Volume-based features\nif 'volume' in df_processed.columns:\n    # Volume momentum\n    df_processed['volume_momentum_5'] = df_processed['volume'] - df_processed['volume_ma_5']\n    df_processed['volume_momentum_30'] = df_processed['volume'] - df_processed['volume_ma_30']\n    \n    # Volume volatility ratio\n    df_processed['volume_volatility_ratio'] = df_processed['volume_std_15'] / (df_processed['volume_ma_15'] + 1e-8)\n\n# Market microstructure interactions\nif 'volume_imbalance' in df_processed.columns and 'volume' in df_processed.columns:\n    df_processed['volume_x_imbalance'] = df_processed['volume'] * df_processed['volume_imbalance']\n\nif 'bid_ask_ratio' in df_processed.columns and 'volume' in df_processed.columns:\n    df_processed['bid_ask_x_volume'] = df_processed['bid_ask_ratio'] * df_processed['volume']\n\n# Temporal interactions\nif 'hour' in df_processed.columns and 'volume' in df_processed.columns:\n    df_processed['hour_x_volume'] = df_processed['hour'] * df_processed['volume']\n\n# Market regime features\nif 'volume_std_30' in df_processed.columns and 'volume_ma_30' in df_processed.columns:\n    # Market regime based on volume volatility\n    df_processed['market_regime'] = pd.qcut(df_processed['volume_std_30'] / (df_processed['volume_ma_30'] + 1e-8), \n                                           q=3, labels=[0, 1, 2], duplicates='drop').astype(float)\n\nprint(\"✓ Created advanced features:\")\nprint(\"  - Volume momentum and volatility features\")\nprint(\"  - Microstructure interaction features\")\nprint(\"  - Temporal interaction features\")\nprint(\"  - Market regime classification\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 4.9 FEATURE SCALING AND NORMALIZATION\n\n# Identify features that need log transformation (highly skewed from EDA)\nskewed_features = []\nskewness_threshold = 2.0\n\nnumeric_cols = df_processed.select_dtypes(include=[np.number]).columns\nnumeric_cols = [col for col in numeric_cols if col != 'label']  # Exclude target\n\nprint(\"Analyzing skewness for feature transformation...\")\n\nfor feature in numeric_cols:\n    if df_processed[feature].min() >= 0:  # Only for non-negative features\n        skewness = df_processed[feature].skew()\n        if abs(skewness) > skewness_threshold:\n            skewed_features.append(feature)\n            # Create log-transformed version\n            df_processed[f'{feature}_log'] = np.log1p(df_processed[feature])\n\nif len(skewed_features) > 0:\n    print(f\"✓ Created log-transformed versions for {len(skewed_features)} highly skewed features\")\nelse:\n    print(\"✓ No highly skewed features requiring log transformation\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 4.10 FINAL DATA PREPARATION\n\n# Handle missing values created by rolling/lag features\nprint(\"Handling missing values...\")\nmissing_before = df_processed.isnull().sum().sum()\n\nif missing_before > 0:\n    print(f\"Found {missing_before} missing values to handle...\")\n    \n    # Get all numeric columns\n    numeric_cols = df_processed.select_dtypes(include=[np.number]).columns\n    numeric_cols = [col for col in numeric_cols if col != 'label']  # Exclude target\n    \n    # Handle missing values column by column\n    for col in numeric_cols:\n        missing_count = df_processed[col].isnull().sum()\n        if missing_count > 0:\n            try:\n                # For lag features (1-5 periods), try forward fill first\n                if any(lag_pattern in col for lag_pattern in ['lag_1', 'lag_3', 'lag_5']):\n                    # Simple forward fill using shift\n                    mask = df_processed[col].isnull()\n                    df_processed.loc[mask, col] = df_processed[col].shift(1)[mask]\n                \n                # Fill any remaining missing with median\n                if df_processed[col].isnull().sum() > 0:\n                    median_val = df_processed[col].median()\n                    df_processed[col] = df_processed[col].fillna(median_val)\n                    \n            except Exception as e:\n                # If there's still an issue, just use median fill\n                print(f\"  Warning: Using median fill for {col} due to: {str(e)[:50]}...\")\n                median_val = df_processed[col].median()\n                df_processed[col] = df_processed[col].fillna(median_val)\n    \n    missing_after = df_processed.isnull().sum().sum()\n    print(f\"✓ Reduced missing values from {missing_before} to {missing_after}\")\nelse:\n    print(\"✓ No missing values to handle\")\n\n# Feature categorization for analysis\noriginal_market_features = [col for col in ['bid_qty', 'ask_qty', 'volume'] if col in df_processed.columns]\nderived_market_features = [col for col in df_processed.columns if any(x in col for x in ['ratio', 'imbalance', 'spread', 'liquidity'])]\ntemporal_features = [col for col in df_processed.columns if any(x in col for x in ['hour', 'minute', 'day', 'weekend', 'peak', 'volatility_period'])]\nrolling_features = [col for col in df_processed.columns if '_ma_' in col or '_std_' in col or '_min_' in col or '_max_' in col]\nlag_features = [col for col in df_processed.columns if '_lag_' in col]\nx_features = [col for col in df_processed.columns if col.startswith('X')]\ninteraction_features = [col for col in df_processed.columns if '_x_' in col or col in ['x_feature_sum', 'x_feature_mean', 'x_feature_std']]\n\nprint(f\"\\n📊 FEATURE ENGINEERING SUMMARY:\")\nprint(f\"{'='*50}\")\nprint(f\"Original Market Features: {len(original_market_features)}\")\nprint(f\"Derived Market Features: {len(derived_market_features)}\")\nprint(f\"Temporal Features: {len(temporal_features)}\")\nprint(f\"Rolling Statistics: {len(rolling_features)}\")\nprint(f\"Lag Features: {len(lag_features)}\")\nprint(f\"X Features (selected): {len(x_features)}\")\nprint(f\"Interaction Features: {len(interaction_features)}\")\nprint(f\"{'='*50}\")\nprint(f\"Total Features: {df_processed.shape[1]-1} (excluding target)\")\nprint(f\"Dataset Shape: {df_processed.shape}\")\n\n# Memory optimization for final dataset\ndf_processed = reduce_mem_usage(df_processed, \"final_processed_dataset\")\n\nprint(f\"\\n✅ FEATURE ENGINEERING COMPLETED!\")\nprint(f\"Ready for model training with {df_processed.shape[1]-1} features\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 4.11 FEATURE IMPORTANCE PREVIEW (Quick Assessment)\n\n# Quick Random Forest to assess feature importance\nprint(\"Running quick feature importance assessment...\")\n\n# Prepare data for quick assessment\nfeature_cols = [col for col in df_processed.columns if col != 'label']\nX_preview = df_processed[feature_cols].copy()\ny_preview = df_processed['label'].copy()\n\n# Handle any remaining missing values\nX_preview = X_preview.fillna(X_preview.median())\n\n# Sample for quick assessment (use 10k rows for speed)\nsample_size = min(10000, len(X_preview))\nsample_idx = np.random.choice(len(X_preview), sample_size, replace=False)\n\nX_sample = X_preview.iloc[sample_idx]\ny_sample = y_preview.iloc[sample_idx]\n\n# Quick Random Forest\nfrom sklearn.ensemble import RandomForestRegressor\nrf_quick = RandomForestRegressor(n_estimators=50, random_state=42, n_jobs=-1)\nrf_quick.fit(X_sample, y_sample)\n\n# Get top 15 features\nfeature_importance = pd.DataFrame({\n    'feature': X_sample.columns,\n    'importance': rf_quick.feature_importances_\n}).sort_values('importance', ascending=False)\n\nprint(\"\\n🏆 TOP 15 MOST IMPORTANT FEATURES:\")\nprint(\"-\" * 45)\nfor i, (_, row) in enumerate(feature_importance.head(15).iterrows(), 1):\n    print(f\"{i:2d}. {row['feature']:<25} {row['importance']:.6f}\")\n\n# Analyze feature types in top 15\ntop_15_features = feature_importance.head(15)['feature'].tolist()\nfeature_type_analysis = {\n    'Temporal': len([f for f in top_15_features if any(x in f for x in ['hour', 'minute', 'day', 'weekend', 'peak'])]),\n    'Rolling': len([f for f in top_15_features if '_ma_' in f or '_std_' in f]),\n    'Lag': len([f for f in top_15_features if '_lag_' in f]),\n    'X Features': len([f for f in top_15_features if f.startswith('X')]),\n    'Market': len([f for f in top_15_features if any(x in f for x in ['volume', 'ratio', 'imbalance'])]),\n    'Interaction': len([f for f in top_15_features if '_x_' in f or 'momentum' in f])\n}\n\nprint(f\"\\n📊 TOP 15 FEATURES BY TYPE:\")\nprint(\"-\" * 30)\nfor feat_type, count in feature_type_analysis.items():\n    if count > 0:\n        print(f\"{feat_type}: {count} features\")\n\nprint(f\"\\n🎯 KEY INSIGHTS:\")\nprint(f\"• Temporal features are {'highly' if feature_type_analysis['Temporal'] >= 3 else 'moderately' if feature_type_analysis['Temporal'] >= 1 else 'not'} important\")\nprint(f\"• Rolling statistics show {'strong' if feature_type_analysis['Rolling'] >= 3 else 'moderate' if feature_type_analysis['Rolling'] >= 1 else 'weak'} predictive power\")\nprint(f\"• X features contribute {'significantly' if feature_type_analysis['X Features'] >= 2 else 'moderately' if feature_type_analysis['X Features'] >= 1 else 'minimally'}\")\n\nprint(f\"\\n✅ Feature engineering complete! Ready for model development.\")\nprint(\"=\"*70)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}