{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# MLP + Denoising\n~~","metadata":{}},{"cell_type":"markdown","source":"## Data","metadata":{}},{"cell_type":"code","source":"# import packages\nimport random\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\nfrom tqdm import tqdm\n\nimport warnings\nwarnings.filterwarnings(\"ignore\", category=RuntimeWarning)\n\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import train_test_split\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, TensorDataset\n\nfrom scipy.stats import pearsonr\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true,"execution":{"iopub.status.busy":"2025-06-29T00:54:26.207425Z","iopub.execute_input":"2025-06-29T00:54:26.207729Z","iopub.status.idle":"2025-06-29T00:54:26.213205Z","shell.execute_reply.started":"2025-06-29T00:54:26.207710Z","shell.execute_reply":"2025-06-29T00:54:26.212355Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def shrink_dtypes(df):\n    return df.select(\n        pl.col(col).shrink_dtype() for col in df.collect_schema().names()\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-29T00:54:26.214328Z","iopub.execute_input":"2025-06-29T00:54:26.214535Z","iopub.status.idle":"2025-06-29T00:54:26.230773Z","shell.execute_reply.started":"2025-06-29T00:54:26.214521Z","shell.execute_reply":"2025-06-29T00:54:26.230147Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pl.scan_parquet(\"/kaggle/input/drw-crypto-market-prediction/train.parquet\")\n\n# # Select features\nselected_features = [\n    \"X863\", \"X856\", \"X344\", \"X598\", \"X862\", \"X385\", \"X852\", \"X603\", \"X860\", \"X674\",\n    \"X415\", \"X345\", \"X137\", \"X855\", \"X174\", \"X302\", \"X178\", \"X532\", \"X168\", \"X612\",\n    \"bid_qty\", \"ask_qty\", \"buy_qty\", \"sell_qty\", \"volume\", \"label\"\n]\n\ntrain = shrink_dtypes(train.select(selected_features)).collect()\n\ntrain = train.to_pandas()\n\ntrain.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-29T00:54:26.231480Z","iopub.execute_input":"2025-06-29T00:54:26.231712Z","iopub.status.idle":"2025-06-29T00:54:26.361898Z","shell.execute_reply.started":"2025-06-29T00:54:26.231696Z","shell.execute_reply":"2025-06-29T00:54:26.361161Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\nX_train, X_val = train_test_split(train, test_size=0.2, shuffle=False, random_state=42)\n\nY_train = X_train.pop(\"label\")\nY_val = X_val.pop(\"label\")\n\nscaler = StandardScaler()\n\nX_train = scaler.fit_transform(X_train)\nX_val = scaler.transform(X_val)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-29T00:54:26.363760Z","iopub.execute_input":"2025-06-29T00:54:26.364176Z","iopub.status.idle":"2025-06-29T00:54:26.659457Z","shell.execute_reply.started":"2025-06-29T00:54:26.364160Z","shell.execute_reply":"2025-06-29T00:54:26.658663Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(X_train.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-29T00:54:26.660268Z","iopub.execute_input":"2025-06-29T00:54:26.660478Z","iopub.status.idle":"2025-06-29T00:54:26.664464Z","shell.execute_reply.started":"2025-06-29T00:54:26.660462Z","shell.execute_reply":"2025-06-29T00:54:26.663603Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Torch","metadata":{}},{"cell_type":"code","source":"def set_seed(seed=42):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)  # Pour multi-GPU\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    # torch.use_deterministic_algorithms(True, warn_only=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-29T00:54:26.665361Z","iopub.execute_input":"2025-06-29T00:54:26.665619Z","iopub.status.idle":"2025-06-29T00:54:26.677740Z","shell.execute_reply.started":"2025-06-29T00:54:26.665599Z","shell.execute_reply":"2025-06-29T00:54:26.677134Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_activation_function(name):\n    \"\"\"Return the activation function based on the name.\"\"\"\n    if name == None:\n        return None\n    name = name.lower()\n    if name == 'relu':\n        return nn.ReLU()\n    elif name == 'tanh':\n        return nn.Tanh()\n    elif name == 'sigmoid':\n        return nn.Sigmoid()\n    else:\n        raise ValueError(f\"Unsupported activation function: {name}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-29T00:54:26.678391Z","iopub.execute_input":"2025-06-29T00:54:26.678673Z","iopub.status.idle":"2025-06-29T00:54:26.691323Z","shell.execute_reply.started":"2025-06-29T00:54:26.678650Z","shell.execute_reply":"2025-06-29T00:54:26.690637Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class MLP(nn.Module):\n    def __init__(self, dropout_rate=0.6, \n                 layers=[128, 64], activation='relu', last_activation=None):\n        \"\"\"Initialize the neural network with three fully connected layers.\n        Args:\n            input_dim (int): Number of input features.\n            output_dim (int): Number of output features (default is 1 for regression).\n        \"\"\"\n        super(MLP, self).__init__()\n        \n        self.linears = nn.ModuleList()\n        self.activation = get_activation_function(activation)\n        self.last_activation = get_activation_function(last_activation)\n\n\n\n        for i in range(len(layers) - 1):\n            self.linears.append(nn.Linear(layers[i], layers[i + 1]))\n\n\n        self.dropout = nn.Dropout(dropout_rate)\n\n    def forward(self, x):\n        for k in range(len(self.linears) - 1):\n            x = self.activation(self.linears[k](x))\n            x = self.dropout(x)\n        x = self.linears[-1](x)  # Last layer without activation\n        if self.last_activation is not None:\n            x = self.last_activation(x)\n        return x\n    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-29T00:54:26.692071Z","iopub.execute_input":"2025-06-29T00:54:26.692306Z","iopub.status.idle":"2025-06-29T00:54:26.708091Z","shell.execute_reply.started":"2025-06-29T00:54:26.692291Z","shell.execute_reply":"2025-06-29T00:54:26.707330Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class GlobalModel(nn.Module):\n    def __init__(self, ea_layers=[128, 64], \n                 forcast_layers=[64, 256, 64, 1], \n                 dropout_rate=0.6, \n                 activation='relu', \n                 last_activation=None):\n        \"\"\"Initialize the global model with three fully connected layers.\n        Args:\n            input_dim (int): Number of input features.\n            output_dim (int): Number of output features (default is 1 for regression).\n        \"\"\"\n        super(GlobalModel, self).__init__()\n        \n        self.encoder = SimpleNN(\n            layers=ea_layers,\n            dropout_rate=dropout_rate,\n            activation=activation,\n            last_activation=last_activation\n        )\n        \n        self.decoder = SimpleNN(\n            layers=ea_layers[::-1],  # Reverse the order for decoder\n            dropout_rate=dropout_rate,\n            activation=activation,\n            last_activation=None\n        )\n\n        self.forecaster = SimpleNN(\n            layers=forcast_layers,\n            dropout_rate=dropout_rate,\n            activation=activation,\n            last_activation=None\n        )\n\n    def forward(self, x):\n        \"\"\"Forward pass through the model.\"\"\"\n        hidden = self.encoder(x)\n        decoded = self.decoder(hidden)\n        x = self.forecaster(hidden)\n        return x, decoded","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-29T00:54:26.708836Z","iopub.execute_input":"2025-06-29T00:54:26.709122Z","iopub.status.idle":"2025-06-29T00:54:26.722885Z","shell.execute_reply.started":"2025-06-29T00:54:26.709100Z","shell.execute_reply":"2025-06-29T00:54:26.722303Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Checkpointer:\n    def __init__(self, path=\"best_model.pt\"):\n        self.path = path\n        self.best_pearson = -np.inf\n\n    def load(self, model):\n        \"\"\"Load the best model weights.\"\"\"\n        model.load_state_dict(torch.load(self.path))\n        print(f\"Model loaded from {self.path} with best Pearson: {self.best_pearson:.4f}\")\n        return model\n\n    def __call__(self, pearson_coef, model):\n        \"\"\"Call method to save the model if the Pearson coefficient is better than the best one.\"\"\"\n        if pearson_coef > self.best_pearson:\n            self.best_pearson = pearson_coef\n            torch.save(model.state_dict(), self.path)\n            print(f\"✅ New best model saved with Pearson: {pearson_coef:.4f}\")\n    \n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-29T00:54:26.724599Z","iopub.execute_input":"2025-06-29T00:54:26.724836Z","iopub.status.idle":"2025-06-29T00:54:26.739575Z","shell.execute_reply.started":"2025-06-29T00:54:26.724821Z","shell.execute_reply":"2025-06-29T00:54:26.738898Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_dataloaders(X, Y, hparams, device, shuffle=True):\n    \"\"\"Create DataLoader for training and validation datasets.\"\"\"\n    # Convert to tensors\n    X_tensor = torch.tensor(X, dtype=torch.float32, device=device)\n    if Y is not None:\n        Y_tensor = torch.tensor(Y.values, dtype=torch.float32, device=device).unsqueeze(1)\n        dataset = TensorDataset(X_tensor, Y_tensor)\n    else:\n        dataset = TensorDataset(X_tensor)\n    \n    # Create DataLoader\n    dataloader = DataLoader(dataset, batch_size=hparams[\"batch_size\"], shuffle=shuffle, \n                            generator=torch.Generator().manual_seed(hparams[\"seed\"]))\n    return dataloader","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-29T00:54:26.740340Z","iopub.execute_input":"2025-06-29T00:54:26.740517Z","iopub.status.idle":"2025-06-29T00:54:26.758336Z","shell.execute_reply.started":"2025-06-29T00:54:26.740503Z","shell.execute_reply":"2025-06-29T00:54:26.757578Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"hparams = {\n    \"seed\": 42,\n    \"num_epochs\": 20,\n    \"batch_size\": 1024 * 8 * 4,\n    \"learning_rate\": 0.001,\n    \"weight_decay\": 1e-3,\n    # \"weight_decay\": 0,\n    \"dropout_rate\": 0.6,\n    \"layers\": [X_train.shape[1], 256, 64, 1],\n    \"hidden_activation\": None,\n    \"activation\": \"relu\",\n    \"delta\": 5,\n    \"noise_factor\": 0.005\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-29T00:55:49.578275Z","iopub.execute_input":"2025-06-29T00:55:49.578539Z","iopub.status.idle":"2025-06-29T00:55:49.582649Z","shell.execute_reply.started":"2025-06-29T00:55:49.578521Z","shell.execute_reply":"2025-06-29T00:55:49.581842Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"set_seed(hparams[\"seed\"])\n\n\n# Create TensorDataset from PCA-transformed features and labels\ntrain_loader = get_dataloaders(X_train, Y_train, hparams, device, shuffle=True)\nval_loader = get_dataloaders(X_val, Y_val, hparams, device, shuffle=False)\n\n\n# Initialize the model, loss function, and optimizer\ninput_dim = X_train.shape[1]\nmodel = MLP(\n    layers=hparams[\"layers\"],\n    dropout_rate=hparams[\"dropout_rate\"],\n    activation=hparams[\"activation\"],\n    last_activation=hparams[\"hidden_activation\"],\n).to(device)\n\n\ncriterion = nn.HuberLoss(delta=hparams[\"delta\"], reduction='sum')\n\n\n\n\n\noptimizer = optim.AdamW(model.parameters(), lr=hparams[\"learning_rate\"], weight_decay=hparams[\"weight_decay\"])\n\n\ncheckpointer = Checkpointer(path=\"best_model.pt\")\n\n# Training loop\nnum_epochs = hparams[\"num_epochs\"]\nfor epoch in range(num_epochs):\n    model.train()\n    running_ea_loss = 0.0\n    running_forcast_loss = 0.0\n\n    for inputs, targets in tqdm(train_loader, desc=f\"Epoch {epoch+1}/{num_epochs}\"):\n        inputs, targets = inputs.to(device), targets.to(device)\n\n        inputs = inputs + torch.randn_like(inputs) * hparams[\"noise_factor\"]  # Add noise for robustness\n        \n        optimizer.zero_grad()\n        forcast = model(inputs)\n        forcast_loss = criterion(forcast, targets)\n\n        loss = forcast_loss\n        \n        loss.backward()\n\n        torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n        \n        optimizer.step()\n\n\n        running_forcast_loss += forcast_loss.item() * inputs.size(0)\n        \n    running_forcast_loss = running_forcast_loss / len(train_loader.dataset)\n\n\n    print(f\"Training Loss: {running_forcast_loss:.4f}\")\n\n    # Validation phase\n    model.eval()\n    val_ea_loss = 0.0\n    val_forcast_loss = 0.0\n    preds = []\n    trues = []\n    with torch.no_grad():\n        for inputs, targets in tqdm(val_loader, desc=\"Validation\"):\n            inputs, targets = inputs.to(device), targets.to(device)\n            forcast= model(inputs)\n            forcast_loss = criterion(forcast, targets)\n            val_forcast_loss += forcast_loss.item() * inputs.size(0)\n            preds.append(forcast.cpu().numpy())\n            trues.append(targets.cpu().numpy())\n            \n\n    val_forcast_loss /= len(val_loader.dataset)\n    preds = np.concatenate(preds).flatten()\n    trues = np.concatenate(trues).flatten()\n    pearson_coef = pearsonr(preds, trues)[0]\n    print(f\"Validation Pearson Coef: {pearson_coef:.4f} | Loss: {val_forcast_loss:.4f}\")\n\n    checkpointer(pearson_coef, model)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-29T00:55:51.753059Z","iopub.execute_input":"2025-06-29T00:55:51.753802Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test = pl.scan_parquet(\"/kaggle/input/drw-crypto-market-prediction/test.parquet\")\n\n\ntest = shrink_dtypes(test.select(selected_features).drop(\"label\")).collect()\n\ntest = test.to_pandas()\n\nX_test = scaler.transform(test)\n\n\ntest_loader = get_dataloaders(X_test, None, hparams, device, shuffle=False)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-29T00:55:34.516772Z","iopub.execute_input":"2025-06-29T00:55:34.517338Z","iopub.status.idle":"2025-06-29T00:55:34.771484Z","shell.execute_reply.started":"2025-06-29T00:55:34.517316Z","shell.execute_reply":"2025-06-29T00:55:34.770916Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = checkpointer.load(model)\n\nmodel.eval()\npredictions = []\nwith torch.no_grad():\n    for inputs in tqdm(test_loader, desc=\"Predicting\"):\n        inputs = inputs[0].to(device)\n        outputs = model(inputs)\n        predictions.append(outputs.cpu().numpy())\n\npredictions = np.concatenate(predictions).flatten()\n\nprint(predictions)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-29T00:55:34.772097Z","iopub.execute_input":"2025-06-29T00:55:34.772280Z","iopub.status.idle":"2025-06-29T00:55:38.908815Z","shell.execute_reply.started":"2025-06-29T00:55:34.772266Z","shell.execute_reply":"2025-06-29T00:55:38.908046Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission = pd.read_csv(\"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\")\nsubmission[\"prediction\"] = predictions\nsubmission.to_csv(\"submission.csv\", index=False)\nsubmission.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-29T00:55:38.909697Z","iopub.execute_input":"2025-06-29T00:55:38.909952Z","iopub.status.idle":"2025-06-29T00:55:39.824802Z","shell.execute_reply.started":"2025-06-29T00:55:38.909931Z","shell.execute_reply":"2025-06-29T00:55:39.824200Z"}},"outputs":[],"execution_count":null}]}