{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"},{"sourceId":12072255,"sourceType":"datasetVersion","datasetId":7581225}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom sklearn.preprocessing import MinMaxScaler\nimport seaborn as sns\n%pip install mplfinance\n\ntrain_path: str = \"/kaggle/input/drw-crypto-market-prediction/train.parquet\"\ntrain_data = pd.read_parquet(train_path)\n\nraw_data = train_data[['bid_qty', 'ask_qty', 'buy_qty', 'sell_qty', 'volume', 'label']]\ndel train_data","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-06-07T00:29:11.100428Z","iopub.execute_input":"2025-06-07T00:29:11.100673Z","iopub.status.idle":"2025-06-07T00:29:18.049245Z","shell.execute_reply.started":"2025-06-07T00:29:11.100659Z","shell.execute_reply":"2025-06-07T00:29:18.048482Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import mplfinance as mpf\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport numpy as np\nimport warnings\nwarnings.filterwarnings('ignore')\n\ndef method1_label_as_return_signal(labels, initial_price=100.0, signal_strength=0.002):\n    \"\"\"\n    方法1: 将label视为收益率信号（保守策略）\n    \"\"\"\n    returns = labels * signal_strength\n    returns = np.clip(returns, -0.02, 0.02)  # 限制在±2%\n    \n    prices = np.zeros(len(returns))\n    prices[0] = initial_price\n    \n    for i in range(1, len(returns)):\n        prices[i] = prices[i-1] * (1 + returns[i])\n        if prices[i] <= 0:\n            prices[i] = prices[i-1] * 0.999\n    \n    return prices, returns\n\ndef method2_label_as_price_change(labels, initial_price=100.0, scale_factor=0.1):\n    \"\"\"\n    方法2: 将label视为价格变化量（保守策略）\n    \"\"\"\n    price_changes = labels * scale_factor\n    \n    prices = np.zeros(len(labels))\n    prices[0] = initial_price\n    \n    for i in range(1, len(labels)):\n        prices[i] = prices[i-1] + price_changes[i]\n        if prices[i] <= 0:\n            prices[i] = prices[i-1] * 0.99\n    \n    returns = np.diff(prices) / prices[:-1]\n    returns = np.concatenate([[0], returns])\n    \n    return prices, returns\n\ndef method3_label_as_zscore_signal(labels, initial_price=100.0, volatility=0.003):\n    \"\"\"\n    方法3: 将label视为标准化信号（保守策略）\n    \"\"\"\n    returns = labels * volatility\n    returns = np.clip(returns, -0.03, 0.03)  # 限制在±3%\n    \n    prices = np.zeros(len(returns))\n    prices[0] = initial_price\n    \n    for i in range(1, len(returns)):\n        prices[i] = prices[i-1] * (1 + returns[i])\n        if prices[i] <= 0:\n            prices[i] = prices[i-1] * 0.999\n    \n    return prices, returns\n\ndef method4_synthetic_price_from_market_data(raw_data, initial_price=100.0):\n    \"\"\"\n    方法4: 基于市场数据合成价格序列（推荐方法）\n    \"\"\"\n    # 计算买卖压力指标\n    buy_pressure = raw_data['buy_qty'] / (raw_data['buy_qty'] + raw_data['sell_qty'] + 1e-10)\n    buy_pressure = buy_pressure.fillna(0.5)\n    \n    # 计算订单簿不平衡\n    order_imbalance = (raw_data['bid_qty'] - raw_data['ask_qty']) / (raw_data['bid_qty'] + raw_data['ask_qty'] + 1e-10)\n    order_imbalance = order_imbalance.fillna(0)\n    \n    # 标准化label\n    label_normalized = (raw_data['label'] - raw_data['label'].mean()) / (raw_data['label'].std() + 1e-10)\n    label_normalized = np.clip(label_normalized, -2, 2)\n    \n    # 组合信号\n    price_signal = (buy_pressure - 0.5) * 0.3 + order_imbalance * 0.2 + label_normalized * 0.001\n    \n    # 转换为收益率\n    returns = price_signal * 0.002  # 非常小的变化幅度\n    returns = np.clip(returns, -0.01, 0.01)\n    \n    prices = np.zeros(len(returns))\n    prices[0] = initial_price\n    \n    for i in range(1, len(returns)):\n        prices[i] = prices[i-1] * (1 + returns[i])\n    \n    return prices, returns\n\ndef create_ohlcv_data(prices, volume_series, timestamps, resample_freq='2h'):\n    \"\"\"\n    从价格序列和成交量序列创建OHLCV数据\n    \"\"\"\n    try:\n        df = pd.DataFrame({\n            'price': prices,\n            'volume': volume_series,\n            'timestamp': pd.to_datetime(timestamps)\n        })\n        df.set_index('timestamp', inplace=True)\n        \n        # 重采样创建OHLC\n        ohlc = df['price'].resample(resample_freq).ohlc()\n        volume_resampled = df['volume'].resample(resample_freq).sum()\n        \n        # 合并数据\n        ohlcv = pd.concat([ohlc, volume_resampled], axis=1)\n        ohlcv.columns = ['Open', 'High', 'Low', 'Close', 'Volume']\n        \n        # 处理缺失值和异常值\n        ohlcv.dropna(inplace=True)\n        \n        # 确保OHLC逻辑正确\n        ohlcv['High'] = ohlcv[['Open', 'High', 'Low', 'Close']].max(axis=1)\n        ohlcv['Low'] = ohlcv[['Open', 'High', 'Low', 'Close']].min(axis=1)\n        \n        return ohlcv\n    except Exception as e:\n        print(f\"Error creating OHLCV data: {e}\")\n        return pd.DataFrame()\n\ndef plot_professional_chart(ohlcv_data, title, method_name):\n    \"\"\"\n    使用mplfinance创建专业图表（修复版本兼容问题）\n    \"\"\"\n    try:\n        # 创建简化的自定义样式（移除不兼容的参数）\n        mc = mpf.make_marketcolors(\n            up='red', \n            down='green',\n            edge='inherit',\n            wick={'up': 'black', 'down': 'black'},\n            volume='in'\n        )\n        \n        # 简化样式设置\n        style = mpf.make_mpf_style(\n            marketcolors=mc,\n            gridstyle='-',\n            gridcolor='lightgray'\n            # 移除 gridwidth 参数\n        )\n        \n        # 绘制专业图表\n        mpf.plot(\n            ohlcv_data,\n            type='candle',\n            volume=True,\n            style=style,\n            title=f'{title} - {method_name}',\n            ylabel='Price',\n            ylabel_lower='Volume',\n            figsize=(16, 10),\n            panel_ratios=(3, 1),\n            show_nontrading=False\n            # 移除 tight_layout 参数\n        )\n        \n        return True\n    except Exception as e:\n        print(f\"mplfinance plotting error: {e}\")\n        return False\n\ndef plot_simple_manual_chart(ohlcv_data, title, method_name):\n    \"\"\"\n    备用方案：手动绘制简单K线图+成交量\n    \"\"\"\n    try:\n        fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(16, 10), height_ratios=[3, 1])\n        \n        # K线图\n        for i, (date, row) in enumerate(ohlcv_data.iterrows()):\n            open_p, high_p, low_p, close_p = row['Open'], row['High'], row['Low'], row['Close']\n            \n            color = 'red' if close_p >= open_p else 'green'\n            \n            # 影线\n            ax1.plot([i, i], [low_p, high_p], color='black', linewidth=1)\n            \n            # 实体\n            if close_p >= open_p:\n                ax1.bar(i, close_p - open_p, bottom=open_p, color=color, alpha=0.8, width=0.8)\n            else:\n                ax1.bar(i, open_p - close_p, bottom=close_p, color=color, alpha=0.8, width=0.8)\n        \n        ax1.set_title(f'{title} - {method_name}', fontsize=14, fontweight='bold')\n        ax1.set_ylabel('Price', fontsize=12)\n        ax1.grid(True, alpha=0.3)\n        \n        # 成交量图\n        volume_colors = ['red' if row['Close'] >= row['Open'] else 'green' \n                        for _, row in ohlcv_data.iterrows()]\n        \n        ax2.bar(range(len(ohlcv_data)), ohlcv_data['Volume'], \n               color=volume_colors, alpha=0.7, width=0.8)\n        \n        ax2.set_ylabel('Volume', fontsize=12)\n        ax2.set_xlabel('Time', fontsize=12)\n        ax2.grid(True, alpha=0.3)\n        \n        # 设置x轴标签\n        step = max(1, len(ohlcv_data) // 8)\n        tick_positions = list(range(0, len(ohlcv_data), step))\n        tick_labels = [ohlcv_data.index[i].strftime('%m-%d\\n%H:%M') for i in tick_positions]\n        \n        ax2.set_xticks(tick_positions)\n        ax2.set_xticklabels(tick_labels, rotation=0, ha='center')\n        ax1.set_xticklabels([])\n        \n        plt.tight_layout()\n        return True\n        \n    except Exception as e:\n        print(f\"Manual plotting error: {e}\")\n        return False\n\n# ================================\n# 主程序开始\n# ================================\n\n# 加载数据部分（假设raw_data已经存在）\n# 如果需要重新加载，取消下面注释：\n# train_path = \"/kaggle/input/drw-crypto-market-prediction/train.parquet\"\n# train_data = pd.read_parquet(train_path)\n# raw_data = train_data[['bid_qty', 'ask_qty', 'buy_qty', 'sell_qty', 'volume', 'label']]\n# del train_data\n\n# 确保数据完整性\nif 'timestamp' not in raw_data.columns:\n    raw_data = raw_data.reset_index()\n    if 'timestamp' not in raw_data.columns:\n        raw_data['timestamp'] = pd.date_range(start='2023-03-01', periods=len(raw_data), freq='min')\n\nprint(\"=== 🚀 Crypto Market Data Analysis - Complete Version ===\")\nprint(f\"Data shape: {raw_data.shape}\")\nprint(f\"Date range: {raw_data['timestamp'].min()} to {raw_data['timestamp'].max()}\")\nprint(f\"Label stats: min={raw_data['label'].min():.4f}, max={raw_data['label'].max():.4f}, mean={raw_data['label'].mean():.4f}\")\n\n# 选择数据子集进行分析\ndata_points = 10000\nanalysis_data = raw_data.head(data_points).copy()\n\nprint(f\"\\n📊 Analyzing {data_points} data points...\")\nprint(f\"Label distribution:\")\nprint(f\"  • 25th percentile: {analysis_data['label'].quantile(0.25):.4f}\")\nprint(f\"  • 50th percentile: {analysis_data['label'].quantile(0.50):.4f}\")\nprint(f\"  • 75th percentile: {analysis_data['label'].quantile(0.75):.4f}\")\nprint(f\"  • Standard deviation: {analysis_data['label'].std():.4f}\")\n\n# 定义所有方法\nmethods = {\n    \"Method 1: Return Signal (Conservative)\": lambda data: method1_label_as_return_signal(\n        data['label'].values, initial_price=100.0, signal_strength=0.002\n    ),\n    \"Method 2: Price Change (Conservative)\": lambda data: method2_label_as_price_change(\n        data['label'].values, initial_price=100.0, scale_factor=0.1\n    ),\n    \"Method 3: Z-Score Signal (Conservative)\": lambda data: method3_label_as_zscore_signal(\n        data['label'].values, initial_price=100.0, volatility=0.003\n    ),\n    \"Method 4: Market Data Synthesis (Recommended)\": lambda data: method4_synthetic_price_from_market_data(\n        data, initial_price=100.0\n    )\n}\n\nsuccessful_charts = 0\ntotal_methods = len(methods)\n\n# 生成图表\nfor method_name, method_func in methods.items():\n    print(f\"\\n{'='*60}\")\n    print(f\"🎯 {method_name}\")\n    print(f\"{'='*60}\")\n    \n    try:\n        # 应用方法生成价格序列\n        prices, returns = method_func(analysis_data)\n        \n        # 验证价格序列的合理性\n        price_ratio = prices.max() / prices.min() if prices.min() > 0 else float('inf')\n        \n        print(f\"📈 Price Statistics:\")\n        print(f\"  • Price range: {prices.min():.4f} - {prices.max():.4f}\")\n        print(f\"  • Price ratio: {price_ratio:.2f}\")\n        print(f\"  • Total return: {(prices[-1]/prices[0] - 1)*100:.2f}%\")\n        print(f\"  • Return volatility: {np.std(returns)*100:.4f}%\")\n        \n        # 创建OHLCV数据\n        ohlcv_data = create_ohlcv_data(\n            prices,\n            analysis_data['volume'].values,\n            analysis_data['timestamp'].values,\n            resample_freq='2h'\n        )\n        \n        if len(ohlcv_data) > 20:\n            print(f\"📊 Generated {len(ohlcv_data)} candlesticks\")\n            \n            # 首先尝试专业图表\n            print(\"🎨 Attempting professional mplfinance chart...\")\n            success_professional = plot_professional_chart(ohlcv_data, \"Crypto Market Analysis\", method_name)\n            \n            if success_professional:\n                successful_charts += 1\n                plt.show()\n                print(\"✅ Professional chart generated successfully!\")\n                \n                # 显示详细统计\n                print(f\"📋 Chart Statistics:\")\n                print(f\"    • Timespan: {len(ohlcv_data)} periods\")\n                print(f\"    • Average Volume: {ohlcv_data['Volume'].mean():.0f}\")\n                print(f\"    • Price Volatility: {ohlcv_data['Close'].pct_change().std()*100:.2f}%\")\n                max_dd = ((ohlcv_data['Close'] / ohlcv_data['Close'].expanding().max()) - 1).min()*100\n                print(f\"    • Max Drawdown: {max_dd:.2f}%\")\n                \n            else:\n                # 备用方案：手动绘制\n                print(\"⚠️  Professional chart failed, trying manual chart...\")\n                success_manual = plot_simple_manual_chart(ohlcv_data, \"Crypto Market Analysis\", method_name)\n                \n                if success_manual:\n                    successful_charts += 1\n                    plt.show()\n                    print(\"✅ Manual chart generated successfully!\")\n                else:\n                    print(\"❌ Both chart methods failed\")\n                    \n        else:\n            print(f\"❌ Insufficient data points: {len(ohlcv_data)} (need > 20)\")\n            \n    except Exception as e:\n        print(f\"❌ Error in {method_name}: {e}\")\n\n# ================================\n# 最终分析总结\n# ================================\nprint(f\"\\n{'='*80}\")\nprint(\"🎊 FINAL ANALYSIS SUMMARY\")\nprint(f\"{'='*80}\")\n\nprint(f\"\\n📊 Processing Results:\")\nprint(f\"  • Methods tested: {total_methods}\")\nprint(f\"  • Successful charts: {successful_charts}\")\nprint(f\"  • Success rate: {successful_charts/total_methods*100:.1f}%\")\nprint(f\"  • Data points analyzed: {data_points:,}\")\n\nprint(f\"\\n📈 Raw Data Characteristics:\")\nprint(f\"  • Total records: {len(raw_data):,}\")\nprint(f\"  • Time span: {(raw_data['timestamp'].max() - raw_data['timestamp'].min()).days} days\")\nprint(f\"  • Average volume: {raw_data['volume'].mean():.0f}\")\nprint(f\"  • Label range: [{raw_data['label'].min():.4f}, {raw_data['label'].max():.4f}]\")\n\nprint(f\"\\n🏪 Market Structure Analysis:\")\nprint(f\"  • Average bid quantity: {raw_data['bid_qty'].mean():.2f}\")\nprint(f\"  • Average ask quantity: {raw_data['ask_qty'].mean():.2f}\")\nprint(f\"  • Average buy quantity: {raw_data['buy_qty'].mean():.2f}\")\nprint(f\"  • Average sell quantity: {raw_data['sell_qty'].mean():.2f}\")\n\n# 计算买卖压力\nbuy_pressure = raw_data['buy_qty'] / (raw_data['buy_qty'] + raw_data['sell_qty'] + 1e-10)\nprint(f\"  • Average buy pressure: {buy_pressure.mean():.3f}\")\n\nprint(f\"\\n🔍 Label Correlation Analysis:\")\nprint(f\"  • Label vs Volume: {np.corrcoef(raw_data['label'], raw_data['volume'])[0,1]:.4f}\")\nprint(f\"  • Label vs Buy Qty: {np.corrcoef(raw_data['label'], raw_data['buy_qty'])[0,1]:.4f}\")\nprint(f\"  • Label vs Sell Qty: {np.corrcoef(raw_data['label'], raw_data['sell_qty'])[0,1]:.4f}\")\n\nprint(f\"\\n💡 Key Insights & Recommendations:\")\nif successful_charts >= 3:\n    print(f\"  ✅ Multiple methods worked successfully\")\n    print(f\"  ✅ Method 4 (Market Data Synthesis) likely most realistic\")\n    print(f\"  ✅ Label appears to contain meaningful predictive signals\")\n    print(f\"  ✅ Data quality is suitable for candlestick analysis\")\nelse:\n    print(f\"  ⚠️  Limited success - consider parameter adjustment\")\n    print(f\"  💡 Try different resampling frequencies\")\n    print(f\"  💡 Experiment with signal strength parameters\")\n\nprint(f\"\\n🔧 Technical Implementation:\")\nprint(f\"  • Charts use professional mplfinance when possible\")\nprint(f\"  • Fallback to manual matplotlib charts if needed\")\nprint(f\"  • Conservative parameter scaling to prevent extreme values\")\nprint(f\"  • 2-hour resampling for clear candlestick patterns\")\nprint(f\"  • Automatic data validation and error handling\")\n\nprint(f\"\\n🎯 Next Steps:\")\nprint(f\"  1. Fine-tune parameters based on preferred method\")\nprint(f\"  2. Experiment with different time windows\")\nprint(f\"  3. Consider adding technical indicators\")\nprint(f\"  4. Validate against actual market data if available\")\n\nprint(f\"\\n✅ Analysis complete! Generated {successful_charts}/{total_methods} charts successfully\")\n\nif successful_charts == 0:\n    print(f\"\\n🚨 TROUBLESHOOTING TIPS:\")\n    print(f\"  • Check mplfinance version: pip install --upgrade mplfinance\")\n    print(f\"  • Verify data format and completeness\")\n    print(f\"  • Try reducing data_points if memory issues\")\n    print(f\"  • Manual charts should work as fallback\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-07T00:29:21.383585Z","iopub.execute_input":"2025-06-07T00:29:21.383775Z","iopub.status.idle":"2025-06-07T00:29:22.944535Z","shell.execute_reply.started":"2025-06-07T00:29:21.383762Z","shell.execute_reply":"2025-06-07T00:29:22.943905Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"raw_data.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-07T00:29:22.945606Z","iopub.execute_input":"2025-06-07T00:29:22.945766Z","iopub.status.idle":"2025-06-07T00:29:22.954383Z","shell.execute_reply.started":"2025-06-07T00:29:22.945753Z","shell.execute_reply":"2025-06-07T00:29:22.953744Z"}},"outputs":[],"execution_count":null}]}