{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import random\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\nfrom torch.utils.data import Dataset, DataLoader\nimport torch\nimport torch.nn as nn\nfrom tqdm import tqdm\nimport warnings\nwarnings.simplefilter('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-05-22T15:17:29.075462Z","iopub.execute_input":"2025-05-22T15:17:29.075760Z","iopub.status.idle":"2025-05-22T15:17:29.080546Z","shell.execute_reply.started":"2025-05-22T15:17:29.075721Z","shell.execute_reply":"2025-05-22T15:17:29.079543Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"torch.manual_seed(28)\ntorch.cuda.manual_seed(28)\nnp.random.seed(28)\nrandom.seed(28)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T15:17:29.081717Z","iopub.execute_input":"2025-05-22T15:17:29.081998Z","iopub.status.idle":"2025-05-22T15:17:29.102483Z","shell.execute_reply.started":"2025-05-22T15:17:29.081978Z","shell.execute_reply":"2025-05-22T15:17:29.101707Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def add_new_features(df):\n    df = df.copy()\n    \n    df[\"bid_ask_diff\"] = df[\"bid_qty\"] - df[\"ask_qty\"]\n    df[\"buy_sell_ratio\"] = df[\"buy_qty\"] / (df[\"sell_qty\"] + 1e-6)\n    df[\"bid_ask_ratio\"] = df[\"bid_qty\"] / (df[\"ask_qty\"] + 1e-6)\n    df[\"buy_volume_ratio\"] = df[\"buy_qty\"] / (df[\"volume\"] + 1e-6)\n    df[\"sell_volume_ratio\"] = df[\"sell_qty\"] / (df[\"volume\"] + 1e-6)\n\n    df[\"sell_buy_ratio\"] = df[\"sell_qty\"] / (df[\"buy_qty\"] + df[\"sell_qty\"] + 1e-9)\n    df[\"buy_sell_diff\"] = df[\"buy_qty\"] - df[\"sell_qty\"]\n    df[\"buy_sell_sum\"] = df[\"buy_qty\"] + df[\"sell_qty\"]\n    df[\"ask_bid_ratio\"] = df[\"ask_qty\"] / (df[\"bid_qty\"] + df[\"ask_qty\"] + 1e-9)\n    df[\"bid_ask_sum\"] = df[\"bid_qty\"] + df[\"ask_qty\"]\n\n    df[\"order_pressure\"] = (df[\"buy_qty\"] - df[\"sell_qty\"]) / (df[\"buy_qty\"] + df[\"sell_qty\"] + 1e-6)\n    df[\"quoted_pressure\"] = (df[\"bid_qty\"] - df[\"ask_qty\"]) / (df[\"bid_qty\"] + df[\"ask_qty\"] + 1e-6)\n    df[\"execution_ratio\"] = (df[\"buy_qty\"] + df[\"sell_qty\"]) / (df[\"bid_qty\"] + df[\"ask_qty\"] + 1e-6)\n    df[\"volume_imbalance\"] = (df[\"buy_qty\"] - df[\"sell_qty\"]) / (df[\"volume\"] + 1e-6)\n    df[\"order_book_total\"] = df[\"bid_qty\"] + df[\"ask_qty\"]\n    df[\"execution_total\"] = df[\"buy_qty\"] + df[\"sell_qty\"]\n    df[\"execution_share\"] = df[\"execution_total\"] / (df[\"order_book_total\"] + 1e-6)\n\n    df[\"log_bid_qty\"] = np.log1p(df[\"bid_qty\"])\n    df[\"log_ask_qty\"] = np.log1p(df[\"ask_qty\"])\n    df[\"log_buy_qty\"] = np.log1p(df[\"buy_qty\"])\n    df[\"log_sell_qty\"] = np.log1p(df[\"sell_qty\"])\n    df[\"log_volume\"] = np.log1p(df[\"volume\"])\n\n    df[\"norm_bid_ask_diff\"] = (df[\"bid_qty\"] - df[\"ask_qty\"]) / (df[\"bid_qty\"] + df[\"ask_qty\"] + 1e-6)\n    df[\"norm_buy_sell_diff\"] = (df[\"buy_qty\"] - df[\"sell_qty\"]) / (df[\"buy_qty\"] + df[\"sell_qty\"] + 1e-6)\n\n    window = 5\n    df[\"buy_qty_ma\"] = df[\"buy_qty\"].rolling(window).mean()\n    df[\"sell_qty_ma\"] = df[\"sell_qty\"].rolling(window).mean()\n    df[\"momentum_buy_qty\"] = df[\"buy_qty\"] - df[\"buy_qty\"].shift(window)\n    df[\"ema_buy_qty\"] = df[\"buy_qty\"].ewm(span=window).mean()\n\n    df[\"roc_buy_qty\"] = df[\"buy_qty\"].pct_change(periods=5)\n    df[\"roc_sell_qty\"] = df[\"sell_qty\"].pct_change(periods=5)\n\n    df[\"execution_dominance\"] = df[\"buy_qty\"] / (df[\"sell_qty\"] + 1e-6)\n    df[\"liquidity_ratio\"] = (df[\"buy_qty\"] + df[\"sell_qty\"]) / (df[\"bid_qty\"] + df[\"ask_qty\"] + 1e-6)\n\n    \n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T15:17:29.103097Z","iopub.execute_input":"2025-05-22T15:17:29.103263Z","iopub.status.idle":"2025-05-22T15:17:29.118684Z","shell.execute_reply.started":"2025-05-22T15:17:29.103250Z","shell.execute_reply":"2025-05-22T15:17:29.118020Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_parquet('/kaggle/input/drw-crypto-market-prediction/train.parquet')\ntest = pd.read_parquet('/kaggle/input/drw-crypto-market-prediction/test.parquet')\n\ntrain = add_new_features(train)\ntest = add_new_features(test)\n\nfeatures = [col for col in train.columns if col.startswith('X_')] + ['bid_qty', 'ask_qty', 'buy_qty', 'sell_qty', 'volume']\nlabel_col = 'label'\n\nadditional_features = [\n    \"bid_ask_diff\", \"buy_sell_ratio\", \"bid_ask_ratio\", \"buy_volume_ratio\", \"sell_volume_ratio\",\n    \"sell_buy_ratio\", \"buy_sell_diff\", \"buy_sell_sum\", \"ask_bid_ratio\", \"bid_ask_sum\",\n    \"order_pressure\", \"quoted_pressure\", \"execution_ratio\", \"volume_imbalance\",\n    \"order_book_total\", \"execution_total\", \"execution_share\",\n\n    \"log_bid_qty\", \"log_ask_qty\", \"log_buy_qty\", \"log_sell_qty\", \"log_volume\",\n    \"norm_bid_ask_diff\", \"norm_buy_sell_diff\",\n\n    \"buy_qty_ma\", \"sell_qty_ma\", \"momentum_buy_qty\", \"ema_buy_qty\",\n    \"roc_buy_qty\", \"roc_sell_qty\",\n\n    \"execution_dominance\", \"liquidity_ratio\"\n]\n\nfeatures += additional_features\n\ntrain[features] = train[features].replace([np.inf, -np.inf], np.nan)\ntest[features] = test[features].replace([np.inf, -np.inf], np.nan)\n\ntrain[features] = train[features].fillna(0)\ntest[features] = test[features].fillna(0)\n\nscaler = StandardScaler()\ntrain[features] = scaler.fit_transform(train[features])\ntest[features] = scaler.transform(test[features])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T15:17:29.120156Z","iopub.execute_input":"2025-05-22T15:17:29.120589Z","iopub.status.idle":"2025-05-22T15:17:55.548480Z","shell.execute_reply.started":"2025-05-22T15:17:29.120571Z","shell.execute_reply":"2025-05-22T15:17:55.547471Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"SEQ_LEN = 30  \nTARGET_COL = 'label'\n\nclass CryptoDataset(Dataset):\n    def __init__(self, df, seq_len, is_train=True):\n        self.features = df[features].values\n        self.labels = df[label_col].values if is_train else None\n        self.seq_len = seq_len\n        self.is_train = is_train\n\n    def __len__(self):\n        return len(self.features) - self.seq_len\n\n    def __getitem__(self, idx):\n        x = self.features[idx:idx+self.seq_len]\n        if self.is_train:\n            y = self.labels[idx + self.seq_len]\n            return torch.tensor(x, dtype=torch.float32), torch.tensor(y, dtype=torch.float32)\n        return torch.tensor(x, dtype=torch.float32)\n\ntrain_df, val_df = train_test_split(train, test_size=0.1, shuffle=False)\n\ntrain_dataset = CryptoDataset(train_df, SEQ_LEN, is_train=True)\nval_dataset = CryptoDataset(val_df, SEQ_LEN, is_train=True)\n\ntrain_loader = DataLoader(train_dataset, batch_size=256, shuffle=True)\nval_loader = DataLoader(val_dataset, batch_size=256, shuffle=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T15:17:55.550133Z","iopub.execute_input":"2025-05-22T15:17:55.550352Z","iopub.status.idle":"2025-05-22T15:17:58.381039Z","shell.execute_reply.started":"2025-05-22T15:17:55.550335Z","shell.execute_reply":"2025-05-22T15:17:58.380371Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class GLNRegressor(nn.Module):\n    def __init__(self, input_size, seq_len, hidden_size=128):\n        super(GLNRegressor, self).__init__()\n        self.seq_len = seq_len\n        self.input_size = input_size\n\n        self.gate = nn.Sequential(\n            nn.Linear(input_size, hidden_size),\n            nn.Sigmoid()\n        )\n        self.linear = nn.Linear(input_size, hidden_size)\n\n        self.fc = nn.Sequential(\n            nn.Linear(hidden_size, hidden_size),\n            nn.ReLU(),\n            nn.Linear(hidden_size, 1)\n        )\n\n    def forward(self, x):\n        gate = self.gate(x)                \n        lin = self.linear(x)               \n        gated_out = gate * lin             \n        pooled = gated_out.mean(dim=1)     \n        out = self.fc(pooled)              \n        return out.squeeze()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T15:17:58.381795Z","iopub.execute_input":"2025-05-22T15:17:58.382016Z","iopub.status.idle":"2025-05-22T15:17:58.387622Z","shell.execute_reply.started":"2025-05-22T15:17:58.381999Z","shell.execute_reply":"2025-05-22T15:17:58.386945Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_model(model, train_loader, val_loader, criterion, optimizer, scheduler, device, epochs=10, patience=5):\n    class EarlyStopping:\n        def __init__(self, patience=patience, delta=1e-4):\n            self.patience = patience\n            self.counter = 0\n            self.best_loss = None\n            self.early_stop = False\n            self.delta = delta\n            self.best_model_state = None\n\n        def __call__(self, val_loss, model):\n            if self.best_loss is None or val_loss < self.best_loss - self.delta:\n                self.best_loss = val_loss\n                self.counter = 0\n                self.best_model_state = model.state_dict()\n            else:\n                self.counter += 1\n                if self.counter >= self.patience:\n                    self.early_stop = True\n\n    def evaluate(model, dataloader, device):\n        model.eval()\n        losses = []\n        all_preds = []\n        all_targets = []\n        with torch.no_grad():\n            for x_batch, y_batch in dataloader:\n                x_batch, y_batch = x_batch.to(device), y_batch.to(device)\n                preds = model(x_batch)\n                loss = criterion(preds, y_batch)\n                losses.append(loss.item())\n                all_preds.extend(preds.cpu().numpy())\n                all_targets.extend(y_batch.cpu().numpy())\n        avg_loss = np.mean(losses)\n        try:\n            from scipy.stats import pearsonr\n            corr = pearsonr(all_preds, all_targets)[0]\n        except:\n            corr = 0.0\n        return avg_loss, corr\n\n    early_stopping = EarlyStopping()\n    \n    history = {\n        \"train_loss\": [],\n        \"val_loss\": [],\n        \"val_corr\": [],\n        \"lr\": []\n    }\n    \n    for epoch in range(epochs):\n        model.train()\n        total_loss = 0\n        loop = tqdm(train_loader, desc=f\"Epoch {epoch+1}/{epochs}\")\n        for x_batch, y_batch in loop:\n            x_batch, y_batch = x_batch.to(device), y_batch.to(device)\n\n            optimizer.zero_grad()\n            preds = model(x_batch)\n            loss = criterion(preds, y_batch)\n            loss.backward()\n            optimizer.step()\n\n            total_loss += loss.item()\n            loop.set_postfix(loss=loss.item())\n        \n        train_loss = total_loss / len(train_loader)\n        val_loss, val_corr = evaluate(model, val_loader, device)\n        scheduler.step(val_loss)\n\n        history[\"train_loss\"].append(train_loss)\n        history[\"val_loss\"].append(val_loss)\n        history[\"val_corr\"].append(val_corr)\n        history[\"lr\"].append(optimizer.param_groups[0]['lr'])\n\n        print(f\"Epoch {epoch+1} | Train Loss: {train_loss:.5f} | Val Loss: {val_loss:.5f} | Val Corr: {val_corr:.4f} | LR: {history['lr'][-1]:.6f}\")\n\n        early_stopping(val_loss, model)\n        if early_stopping.early_stop:\n            print(\"Early stopping triggered.\")\n            break\n\n    model.load_state_dict(early_stopping.best_model_state)\n\n    fig, ax1 = plt.subplots(figsize=(10,6))\n\n    ax1.set_xlabel('Epoch')\n    ax1.set_ylabel('Loss', color='tab:blue')\n    ax1.plot(history[\"train_loss\"], label='Train Loss', color='tab:blue', linestyle='-')\n    ax1.plot(history[\"val_loss\"], label='Val Loss', color='tab:blue', linestyle='--')\n    ax1.tick_params(axis='y', labelcolor='tab:blue')\n    ax1.legend(loc='upper left')\n\n    ax2 = ax1.twinx()  \n    ax2.set_ylabel('Learning Rate / Val Corr', color='tab:orange')\n    ax2.plot(history[\"lr\"], label='Learning Rate', color='tab:orange', linestyle='-.')\n    ax2.plot(history[\"val_corr\"], label='Val Corr', color='tab:green', linestyle=':')\n    ax2.tick_params(axis='y', labelcolor='tab:orange')\n    ax2.legend(loc='upper right')\n\n    plt.title('Training Metrics and Learning Rate')\n    plt.show()\n\n    return model, history","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T15:17:58.389577Z","iopub.execute_input":"2025-05-22T15:17:58.389798Z","iopub.status.idle":"2025-05-22T15:17:58.405412Z","shell.execute_reply.started":"2025-05-22T15:17:58.389775Z","shell.execute_reply":"2025-05-22T15:17:58.404868Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nmodel = GLNRegressor(input_size=len(features), seq_len=SEQ_LEN).to(device)\n\ncriterion = nn.MSELoss()\noptimizer = torch.optim.Adam(model.parameters(), lr=1e-3)\nscheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', patience=2, factor=0.5, verbose=True)\n\ntrained_model, history = train_model(model, train_loader, val_loader, criterion, optimizer, scheduler, device, epochs=50, patience=5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T15:17:58.406021Z","iopub.execute_input":"2025-05-22T15:17:58.406267Z","iopub.status.idle":"2025-05-22T15:19:47.911462Z","shell.execute_reply.started":"2025-05-22T15:17:58.406246Z","shell.execute_reply":"2025-05-22T15:19:47.910803Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_dataset = CryptoDataset(test, SEQ_LEN, is_train=False)\ntest_loader = DataLoader(test_dataset, batch_size=256, shuffle=False)\n\nmodel.eval()\npreds = []\n\nwith torch.no_grad():\n    for x_batch in tqdm(test_loader, desc=\"Predicting\"):\n        x_batch = x_batch.to(device)\n        outputs = model(x_batch)\n        preds.extend(outputs.cpu().numpy())\n\nsample_submission = pd.read_csv('/kaggle/input/drw-crypto-market-prediction/sample_submission.csv')\nsample_submission['prediction'] = [0]*SEQ_LEN + preds  \nsample_submission.to_csv('submission.csv', index=False)\nprint(sample_submission.tail())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T15:19:47.912348Z","iopub.execute_input":"2025-05-22T15:19:47.912720Z"}},"outputs":[],"execution_count":null}]}