{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import random\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\nfrom torch.utils.data import Dataset, DataLoader\nimport torch\nimport torch.nn as nn\nfrom tqdm import tqdm\nfrom scipy.stats import pearsonr\n\nimport copy  # To store best model weights\n\n# Seeds\ntorch.manual_seed(28)\nnp.random.seed(28)\nrandom.seed(28)\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-05-23T10:35:31.263139Z","iopub.execute_input":"2025-05-23T10:35:31.263460Z","iopub.status.idle":"2025-05-23T10:35:36.231807Z","shell.execute_reply.started":"2025-05-23T10:35:31.263435Z","shell.execute_reply":"2025-05-23T10:35:36.231217Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\n\nclass FeatureWiseDataset(Dataset):\n    def __init__(self, df, is_train=True):\n        self.features = df[features].values\n        self.labels = df[label_col].values if is_train else None\n        self.is_train = is_train\n\n    def __len__(self):\n        return len(self.features)\n\n    def __getitem__(self, idx):\n        x = self.features[idx]\n        x = torch.tensor(x, dtype=torch.float32).unsqueeze(1) \n        if self.is_train:\n            y = torch.tensor(self.labels[idx], dtype=torch.float32)\n            return x, y\n        return x","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-23T10:35:36.233105Z","iopub.execute_input":"2025-05-23T10:35:36.233492Z","iopub.status.idle":"2025-05-23T10:35:36.238574Z","shell.execute_reply.started":"2025-05-23T10:35:36.233473Z","shell.execute_reply":"2025-05-23T10:35:36.237977Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class FeatureAttentionModel(nn.Module):\n    def __init__(self, num_features, d_model=256, nhead=8, num_layers=6, dropout=0.1):\n        super().__init__()\n        self.embedding = nn.Linear(1, d_model)\n        self.cls_token = nn.Parameter(torch.randn(1, 1, d_model))\n\n        encoder_layer = nn.TransformerEncoderLayer(\n            d_model=d_model,\n            nhead=nhead,\n            dim_feedforward=4 * d_model,\n            dropout=dropout,\n            batch_first=True\n        )\n        self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_layers=num_layers)\n        self.norm = nn.LayerNorm(d_model)\n\n        self.output_head = nn.Sequential(\n            nn.Linear(d_model, d_model // 2),\n            nn.GELU(),\n            nn.Linear(d_model // 2, 1)\n        )\n\n    def forward(self, x):\n        x = self.embedding(x)\n        cls = self.cls_token.expand(x.size(0), -1, -1)\n        x = torch.cat([cls, x], dim=1)\n\n        encoded = self.transformer_encoder(x)\n        encoded = self.norm(encoded)\n        pooled = encoded[:, 0]\n        return self.output_head(pooled).squeeze(-1) \n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-23T10:35:36.239197Z","iopub.execute_input":"2025-05-23T10:35:36.239374Z","iopub.status.idle":"2025-05-23T10:35:36.255441Z","shell.execute_reply.started":"2025-05-23T10:35:36.239360Z","shell.execute_reply":"2025-05-23T10:35:36.254720Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def plot_predictions(preds, trues, epoch):\n    plt.figure(figsize=(10, 4))\n    plt.plot(trues[:200], label='True', alpha=0.7)\n    plt.plot(preds[:200], label='Pred', alpha=0.7)\n    plt.title(f'Predictions vs Ground Truth (Epoch {epoch})')\n    plt.xlabel('Sample')\n    plt.ylabel('Value')\n    plt.legend()\n    plt.tight_layout()\n    plt.show()\n\nclass NegativePearsonLoss(nn.Module):\n    def forward(self, preds, targets):\n        preds = preds - preds.mean(dim=-1, keepdim=True)\n        targets = targets - targets.mean(dim=-1, keepdim=True)\n\n        numerator = (preds * targets).sum(dim=-1)\n        denominator = torch.norm(preds, dim=-1) * torch.norm(targets, dim=-1)\n        correlation = numerator / (denominator + 1e-8)\n\n        return 1 - correlation.mean()\n\n# --- Evaluation ---\ndef evaluate(model, loader, criterion, device, return_preds=False):\n    model.eval()\n    losses, preds, trues = [], [], []\n    with torch.no_grad():\n        for X, y in loader:\n            X, y = X.to(device), y.to(device)\n            out = model(X).squeeze(-1)\n            loss = criterion(out, y)\n            losses.append(loss.item())\n            preds.extend(out.cpu().numpy())\n            trues.extend(y.cpu().numpy())\n    avg_loss = np.mean(losses)\n\n    preds = np.array(preds)\n    trues = np.array(trues)\n    if np.std(preds) < 1e-6 or np.std(trues) < 1e-6:\n        corr = 0.0\n    else:\n        corr = pearsonr(preds, trues)[0]\n\n    if return_preds:\n        return avg_loss, corr, preds, trues\n    return avg_loss, corr\n\n\ndef train_model(model, train_loader, val_loader, epochs=10, lr=1e-3, device='cuda', save_path='best_model.pt'):\n    model.to(device)\n    optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', patience=2, factor=0.5)\n    criterion = NegativePearsonLoss()\n\n    best_corr = -float('inf')\n    best_model_wts = copy.deepcopy(model.state_dict())\n\n    for epoch in range(1, epochs + 1):\n        model.train()\n        running_loss = 0.0\n        loop = tqdm(train_loader, desc=f\"Training at epoch {epoch}\", position=0, leave=True)\n        for X, y in loop:\n            X, y = X.to(device), y.to(device)\n            optimizer.zero_grad()\n            out = model(X).squeeze(-1)\n            loss = criterion(out, y)\n            loss.backward()\n            optimizer.step()\n            running_loss += loss.item()\n            loop.set_postfix(loss=loss.item())\n\n        val_loss, val_corr, val_preds, val_trues = evaluate(model, val_loader, criterion, device, return_preds=True)\n        plot_predictions(val_preds, val_trues, epoch)\n        scheduler.step(val_loss)\n\n        print(f\"Epoch {epoch}/{epochs} | Train Loss: {running_loss/len(train_loader):.4f} \"\n              f\"| Val Loss: {val_loss:.4f} | Val Corr: {val_corr:.4f}\")\n\n        # Save best model\n        if val_corr > best_corr:\n            best_corr = val_corr\n            best_model_wts = copy.deepcopy(model.state_dict())\n            torch.save(best_model_wts, save_path)\n            print(f\"  ✅ New best model saved (val_corr: {val_corr:.4f})\")\n\n    # Load best weights before returning\n    model.load_state_dict(best_model_wts)\n    return model\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-23T10:35:36.256981Z","iopub.execute_input":"2025-05-23T10:35:36.257185Z","iopub.status.idle":"2025-05-23T10:35:36.275031Z","shell.execute_reply.started":"2025-05-23T10:35:36.257169Z","shell.execute_reply":"2025-05-23T10:35:36.274475Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_parquet('/kaggle/input/drw-crypto-market-prediction/train.parquet')\ntest = pd.read_parquet('/kaggle/input/drw-crypto-market-prediction/test.parquet')\n\ntrain = train.replace([np.inf, -np.inf], np.nan)\nnan_cols = train.columns[train.isna().any()].tolist() # get list of nan columns\n\nfeatures = [col for col in train.columns if col.startswith('X_')] + ['bid_qty', 'ask_qty', 'buy_qty', 'sell_qty', 'volume']\nlabel_col = 'label'\n\nscaler = StandardScaler()\ntrain[features] = scaler.fit_transform(train[features])\ntest[features] = scaler.transform(test[features])\n\ntrain = train.drop(nan_cols, axis=1)\ntest = test.drop(nan_cols, axis=1)\n\nprint(train.shape)\nprint(test.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-23T10:35:36.275605Z","iopub.execute_input":"2025-05-23T10:35:36.275780Z","iopub.status.idle":"2025-05-23T10:36:35.964900Z","shell.execute_reply.started":"2025-05-23T10:35:36.275766Z","shell.execute_reply":"2025-05-23T10:36:35.964133Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Sort chronologically, even if it should already\ntrain = train.sort_values('timestamp').reset_index(drop=True)\n\n# Keep last X% as validation\ntrain_frac = 0.9\nsplit_idx = int(len(train) * train_frac)\ntrain_df = train.iloc[:split_idx].copy()\nval_df = train.iloc[split_idx:].copy()\n\ntrain_dataset = FeatureWiseDataset(train_df, is_train=True)\nval_dataset = FeatureWiseDataset(val_df, is_train=True)\n\ntrain_loader = DataLoader(train_dataset, batch_size=256, shuffle=True)\nval_loader = DataLoader(val_dataset, batch_size=256, shuffle=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-23T10:36:35.965719Z","iopub.execute_input":"2025-05-23T10:36:35.966229Z","iopub.status.idle":"2025-05-23T10:36:45.110685Z","shell.execute_reply.started":"2025-05-23T10:36:35.966210Z","shell.execute_reply":"2025-05-23T10:36:45.109888Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nmodel = FeatureAttentionModel(num_features=len(features))\ntrained_model = train_model(model, train_loader, val_loader, epochs=3, device=device, save_path='best_model.pt')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-23T10:36:45.111566Z","iopub.execute_input":"2025-05-23T10:36:45.111864Z","iopub.status.idle":"2025-05-23T10:39:06.251652Z","shell.execute_reply.started":"2025-05-23T10:36:45.111822Z","shell.execute_reply":"2025-05-23T10:39:06.251005Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_dataset = FeatureWiseDataset(test, is_train=False)\ntest_loader = DataLoader(test_dataset, batch_size=256, shuffle=True)\n\ntrained_model.eval()\ntest_preds = []\n\nwith torch.no_grad():\n    for x_batch in tqdm(test_loader, desc=\"Predicting\"):\n        x_batch = x_batch.to(device)\n        outputs = trained_model(x_batch)\n        test_preds.extend(outputs.cpu().numpy())\n\nsample_submission = pd.read_csv('/kaggle/input/drw-crypto-market-prediction/sample_submission.csv')\nsample_submission['prediction'] = test_preds  # pad the beginning to align sequence\nsample_submission.to_csv('submission.csv', index=False)\nprint(sample_submission.tail())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-23T10:39:06.252492Z","iopub.execute_input":"2025-05-23T10:39:06.252906Z","iopub.status.idle":"2025-05-23T10:39:25.504407Z","shell.execute_reply.started":"2025-05-23T10:39:06.252885Z","shell.execute_reply":"2025-05-23T10:39:25.503717Z"}},"outputs":[],"execution_count":null}]}