{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"dockerImageVersionId":31090,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# DRW Crypto Market Prediction - Memory Optimized (Fixed)\nimport numpy as np\nimport pandas as pd\nimport lightgbm as lgb\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.metrics import mean_squared_error\nimport gc\nfrom tqdm import tqdm\n\n# Configuration\nclass Config:\n    LAGS = [1, 5, 15, 60]  # Reduced lags for memory safety\n    CORE_FEATURES = ['bid_qty', 'ask_qty', 'volume']  # Core features only\n    N_FOLDS = 3\n    SAMPLE_FRAC = 0.5  # Use 50% of data if memory constrained\n\n# Memory-safe processing\ndef process_data():\n    \"\"\"Load and process data with memory safeguards\"\"\"\n    # 1. Load data with memory monitoring\n    # print(\"Loading data...\")\n    train = pd.read_parquet(\"/kaggle/input/drw-crypto-market-prediction/train.parquet\")\n    test = pd.read_parquet(\"/kaggle/input/drw-crypto-market-prediction/test.parquet\")\n    \n    # Subsample if needed\n    if Config.SAMPLE_FRAC < 1:\n        train = train.sample(frac=Config.SAMPLE_FRAC, random_state=42)\n    \n    # 2. Create lag features in batches\n    # print(\"Creating lag features...\")\n    for feat in Config.CORE_FEATURES:\n        for lag in Config.LAGS:\n            train[f'{feat}_lag_{lag}'] = train[feat].shift(lag).astype(np.float32)\n            gc.collect()\n    \n    # 3. Drop NA and reduce memory\n    train = train.dropna()\n    for col in train.columns:\n        if train[col].dtype == 'float64':\n            train[col] = train[col].astype(np.float32)\n    \n    return train, test\n\n# Main execution\ndef main():\n    # 1. Process data\n    train, test = process_data()\n    \n    # 2. Feature engineering\n    # print(\"Engineering features...\")\n    train['imbalance'] = (train['bid_qty'] - train['ask_qty']) / (train['bid_qty'] + train['ask_qty'] + 1e-6)\n    features = Config.CORE_FEATURES + [\n        col for col in train.columns \n        if any(feat in col for feat in ['lag_', 'imbalance'])\n    ]\n    \n    # 3. Train model\n    # print(\"Training model...\")\n    models = []\n    kf = GroupKFold(n_splits=Config.N_FOLDS)\n    \n    for fold, (train_idx, val_idx) in enumerate(kf.split(train, groups=np.arange(len(train)))):\n        X_train, X_val = train[features].iloc[train_idx], train[features].iloc[val_idx]\n        y_train, y_val = train['label'].iloc[train_idx], train['label'].iloc[val_idx]\n        \n        model = lgb.LGBMRegressor(\n            objective='regression',\n            n_estimators=300,\n            learning_rate=0.05,\n            max_depth=5,\n            num_leaves=31,\n            min_child_samples=100,\n            random_state=42,\n            verbosity=-1\n        )\n        \n        model.fit(X_train, y_train)\n        val_pred = model.predict(X_val)\n        rmse = mean_squared_error(y_val, val_pred, squared=False)\n        print(f\"Fold {fold+1} RMSE: {rmse:.4f}\")\n        models.append(model)\n        gc.collect()\n    \n    # 4. Create submission\n    print(\"Creating submission...\")\n    test_pred = np.zeros(len(test))\n    for feat in Config.CORE_FEATURES:\n        for lag in Config.LAGS:\n            test[f'{feat}_lag_{lag}'] = train[feat].iloc[-lag:].values[0]  # Safe lag for test\n    \n    test['imbalance'] = (test['bid_qty'] - test['ask_qty']) / (test['bid_qty'] + test['ask_qty'] + 1e-6)\n    test_pred = np.mean([model.predict(test[features]) for model in models], axis=0)\n    \n    submission = pd.DataFrame({\n        'row_id': test.index,\n        'label': test_pred\n    })\n    submission.to_csv('submission.csv', index=False)\n    # # print(\"Submission created successfully!\")\n\nif __name__ == \"__main__\":\n    main()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-07-09T16:45:46.646897Z","iopub.execute_input":"2025-07-09T16:45:46.647291Z","iopub.status.idle":"2025-07-09T16:47:23.459418Z","shell.execute_reply.started":"2025-07-09T16:45:46.647267Z","shell.execute_reply":"2025-07-09T16:47:23.458408Z"}},"outputs":[],"execution_count":null}]}