{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":96164,"databundleVersionId":12993472,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# DRW Crypto Market Prediction - Robust Transformer Implementation\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.preprocessing import StandardScaler\nimport gc\n\n# Configuration\nclass Config:\n    # Only use features that exist in the data\n    CORE_FEATURES = ['bid_qty', 'ask_qty', 'buy_qty', 'sell_qty', 'volume']  # Basic features\n    LAGS = [1, 5, 15, 60]  # 1min to 1hr\n    SEQ_LEN = 60  # Lookback window\n    BATCH_SIZE = 256\n    EPOCHS = 15  # Reduced for faster iteration\n    N_FOLDS = 3\n    LEARNING_RATE = 3e-4\n    DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\nclass CryptoTransformer(nn.Module):\n    def __init__(self, input_dim):\n        super().__init__()\n        self.embedding = nn.Linear(input_dim, 64)\n        encoder_layer = nn.TransformerEncoderLayer(\n            d_model=64, \n            nhead=8, \n            dim_feedforward=256, \n            dropout=0.1\n        )\n        self.transformer = nn.TransformerEncoder(encoder_layer, num_layers=3)\n        self.regressor = nn.Sequential(\n            nn.Linear(64, 32),\n            nn.ReLU(),\n            nn.Linear(32, 1)\n        )\n        \n    def forward(self, x):\n        x = self.embedding(x)\n        x = x.permute(1, 0, 2)  # (seq_len, batch, features)\n        x = self.transformer(x)\n        x = x.mean(dim=0)\n        return self.regressor(x).squeeze()\n\nclass CryptoDataset(Dataset):\n    def __init__(self, data, targets, sequence_length):\n        self.data = data\n        self.targets = targets\n        self.seq_len = sequence_length\n        \n    def __len__(self):\n        return len(self.data) - self.seq_len\n        \n    def __getitem__(self, idx):\n        seq = self.data[idx:idx+self.seq_len]\n        target = self.targets[idx+self.seq_len]\n        return torch.FloatTensor(seq), torch.FloatTensor([target])\n\ndef get_available_features(df, requested_features):\n    \"\"\"Return only features that exist in the dataframe\"\"\"\n    return [f for f in requested_features if f in df.columns]\n\ndef prepare_data():\n    train = pd.read_parquet(\"/kaggle/input/drw-crypto-market-prediction/train.parquet\")\n    test = pd.read_parquet(\"/kaggle/input/drw-crypto-market-prediction/test.parquet\")\n    \n    # Get available features\n    available_features = get_available_features(train, Config.CORE_FEATURES)\n    print(f\"Available features: {available_features}\")\n    \n    # Feature engineering\n    for feat in available_features:\n        for lag in Config.LAGS:\n            train[f'{feat}_lag_{lag}'] = train[feat].shift(lag)\n            test[f'{feat}_lag_{lag}'] = train[feat].iloc[-lag:].values[0]\n    \n    train['imbalance'] = (train['bid_qty'] - train['ask_qty']) / (train['bid_qty'] + train['ask_qty'] + 1e-6)\n    test['imbalance'] = (test['bid_qty'] - test['ask_qty']) / (test['bid_qty'] + test['ask_qty'] + 1e-6)\n    \n    # Find all generated features\n    features = available_features + [col for col in train.columns if 'lag_' in col] + ['imbalance']\n    train = train.dropna()\n    \n    # Normalization\n    scaler = StandardScaler()\n    X_train = scaler.fit_transform(train[features])\n    X_test = scaler.transform(test[features])\n    y_train = train['label'].values\n    \n    return X_train, y_train, X_test, test.index, features\n\ndef main():\n    X_train, y_train, X_test, test_ids, features = prepare_data()\n    print(f\"Final feature set ({len(features)}): {features}\")\n    \n    kf = GroupKFold(n_splits=Config.N_FOLDS)\n    groups = np.arange(len(X_train))\n    test_preds = np.zeros(len(X_test))\n    \n    for fold, (train_idx, val_idx) in enumerate(kf.split(X_train, y_train, groups)):\n        print(f\"\\nFold {fold+1} Training\")\n        \n        # Create datasets\n        train_dataset = CryptoDataset(X_train[train_idx], y_train[train_idx], Config.SEQ_LEN)\n        val_dataset = CryptoDataset(X_train[val_idx], y_train[val_idx], Config.SEQ_LEN)\n        \n        # Initialize model\n        model = CryptoTransformer(input_dim=len(features)).to(Config.DEVICE)\n        optimizer = torch.optim.AdamW(model.parameters(), lr=Config.LEARNING_RATE)\n        criterion = nn.MSELoss()\n        \n        # Data loaders\n        train_loader = DataLoader(train_dataset, batch_size=Config.BATCH_SIZE, shuffle=True)\n        val_loader = DataLoader(val_dataset, batch_size=Config.BATCH_SIZE)\n        \n        # Training loop\n        for epoch in range(Config.EPOCHS):\n            model.train()\n            train_loss = 0\n            for seq, target in train_loader:\n                seq, target = seq.to(Config.DEVICE), target.to(Config.DEVICE)\n                optimizer.zero_grad()\n                output = model(seq)\n                loss = criterion(output, target)\n                loss.backward()\n                optimizer.step()\n                train_loss += loss.item()\n            \n            # Validation\n            model.eval()\n            val_loss = 0\n            with torch.no_grad():\n                for seq, target in val_loader:\n                    seq, target = seq.to(Config.DEVICE), target.to(Config.DEVICE)\n                    output = model(seq)\n                    val_loss += criterion(output, target).item()\n            \n            print(f\"Epoch {epoch+1} | Train Loss: {train_loss/len(train_loader):.4f} | Val Loss: {val_loss/len(val_loader):.4f}\")\n        \n        # Prepare test set for prediction\n        test_dataset = []\n        for i in range(len(X_test) - Config.SEQ_LEN):\n            test_dataset.append(X_test[i:i+Config.SEQ_LEN])\n        test_dataset = torch.FloatTensor(np.array(test_dataset))\n        \n        # Predict\n        model.eval()\n        fold_preds = []\n        with torch.no_grad():\n            for i in range(0, len(test_dataset), Config.BATCH_SIZE):\n                batch = test_dataset[i:i+Config.BATCH_SIZE].to(Config.DEVICE)\n                fold_preds.extend(model(batch).cpu().numpy())\n        \n        # Pad with zeros for the initial SEQ_LEN positions\n        padded_preds = np.zeros(len(X_test))\n        padded_preds[Config.SEQ_LEN:] = np.array(fold_preds)\n        test_preds += padded_preds / Config.N_FOLDS\n    \n    # Create submission\n    submission = pd.DataFrame({\n        'ID': test_ids,\n        'prediction': test_preds\n    })\n    submission.to_csv('submission.csv', index=False)\n    print(\"\\nSubmission created successfully!\")\n\nif __name__ == \"__main__\":\n    main()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-07-11T02:23:42.121457Z","iopub.execute_input":"2025-07-11T02:23:42.122222Z","iopub.status.idle":"2025-07-11T02:59:15.394677Z","shell.execute_reply.started":"2025-07-11T02:23:42.122191Z","shell.execute_reply":"2025-07-11T02:59:15.393844Z"}},"outputs":[],"execution_count":null}]}