{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":11305158,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":203900450,"sourceType":"kernelVersion"}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Libs Needed","metadata":{}},{"cell_type":"code","source":"import os # move files\nimport pickle # dump/serilize models\n######################\nimport polars as pl # lazy pipeline data\n####################\nimport numpy as np\nimport pandas as pd\n#####################\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n#######################\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n# import pytorch_lightning as pl\nfrom pytorch_lightning import (LightningDataModule, LightningModule, Trainer)\nfrom pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint, Timer\nfrom pytorch_lightning.loggers import WandbLogger\nimport wandb # w,b tracker\nfrom sklearn.metrics import r2_score\nfrom sklearn.model_selection import train_test_split\nfrom torch.utils.data import Dataset, DataLoader","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T15:17:50.711807Z","iopub.execute_input":"2025-05-22T15:17:50.712454Z","iopub.status.idle":"2025-05-22T15:17:50.717757Z","shell.execute_reply.started":"2025-05-22T15:17:50.712426Z","shell.execute_reply":"2025-05-22T15:17:50.716983Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 1. Configurations (NN)","metadata":{}},{"cell_type":"code","source":"class CFG():\n    use_gpu = True\n    gpu_id = 0\n    seed = 42\n    model = 'nn'\n    use_wandb = False\n    ##################\n    loader_workers = 4\n    batch_size = 8192\n    ################# Key Params\n    lr = 1e-3\n    weight_decay = 5e-4\n    dropouts = [0.1, 0.1]\n    n_hidden = [512, 512, 256]\n    patience = 25\n    #################\n    max_epochs = 2000\n    n_fold = 5\n###########\n# CFG.n_fold","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T15:05:40.885486Z","iopub.execute_input":"2025-05-22T15:05:40.885812Z","iopub.status.idle":"2025-05-22T15:05:40.890391Z","shell.execute_reply.started":"2025-05-22T15:05:40.885780Z","shell.execute_reply":"2025-05-22T15:05:40.889634Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 2. Define PyTorch Dataset","metadata":{}},{"cell_type":"code","source":"#####################################\nclass CustomDataset(Dataset):\n    # this custom dataset will be called by the datamodule deifned below\n    # 1. setup tensor for float values of features, labels,and weights\n    #     - this will be used in NN defined later\n    # 2. easy to split\n    def __init__(self, df, accelerator):\n        self.features = torch.FloatTensor(\n            df[feature_names].values).to(accelerator)\n        self.labels = torch.FloatTensor(\n            df[label_name].values).to(accelerator)\n        self.weights = torch.FloatTensor(\n            df[weight_name].values).to(accelerator)\n    \n    def __len__(self):\n        return len(self.labels)\n    \n    def __getitem__(self, idx):\n        x = self.features[idx]\n        y = self.labels[idx]\n        w = self.weights[idx]\n        return x, y, w\n        \n######################################\nclass DataModule(LightningDataModule): \n    # Here we use PytorchLightening DataModule\n    # to preprocess; It's less verbose than Pytorch\n    \n    def __init__(\n        self, train_df, batch_size, \n        valid_df=None, accelerator='cpu'):\n        super().__init__()\n        self.df = train_df\n        self.batch_size = batch_size\n        self.dates = self.df['date_id'].unique()\n        self.accelerator = accelerator\n        self.train_dataset = None\n        self.valid_df = None\n        if valid_df is not None:\n            self.valid_df = valid_df\n        self.val_dataset = None\n\n    def setup(self, fold=0, N_fold=5, stage=None):\n        # Split dataset\n        selected_dates = [date for ii, date in enumerate(self.dates) if ii % N_fold != fold]\n        df_train = self.df.loc[self.df['date_id'].isin(selected_dates)]\n        self.train_dataset = CustomDataset(df_train, self.accelerator)\n        if self.valid_df is not None:\n            df_valid = self.valid_df\n            self.val_dataset = CustomDataset(df_valid, self.accelerator)\n\n    def train_dataloader(self, n_workers=0):\n        return DataLoader(\n            self.train_dataset, \n            batch_size=self.batch_size, \n            shuffle=True, \n            num_workers=n_workers)\n\n    def val_dataloader(self, n_workers=0):\n        return DataLoader(\n            self.val_dataset, \n            batch_size=self.batch_size, \n            shuffle=False, \n            num_workers=n_workers)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T15:05:30.034985Z","iopub.execute_input":"2025-05-22T15:05:30.035760Z","iopub.status.idle":"2025-05-22T15:05:30.055000Z","shell.execute_reply.started":"2025-05-22T15:05:30.035726Z","shell.execute_reply":"2025-05-22T15:05:30.054357Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 3. Model","metadata":{}},{"cell_type":"code","source":"# Custom R2 metric for validation\ndef r2_val(y_true, y_pred, sample_weight):\n    nom = np.average((y_pred - y_true) ** 2, weights=sample_weight)\n    denom = (np.average((y_true) ** 2, weights=sample_weight) + 1e-38)\n    r2 = 1 -  nom/denom \n    return r2\n\n\nclass NN(LightningModule):\n    # use pytorch lightning\n    \n    def __init__(self, input_dim, hidden_dims, dropouts, lr, weight_decay):\n        super().__init__()\n        self.save_hyperparameters()\n        layers = []\n        in_dim = input_dim\n        \n        for i, hidden_dim in enumerate(hidden_dims):\n            layers.append(nn.BatchNorm1d(in_dim))\n            \n            if i > 0: # skip first layer\n                layers.append(nn.SiLU())\n            \n            if i < len(dropouts): # dropout the first few layers\n                layers.append(nn.Dropout(dropouts[i]))\n            \n            layers.append(nn.Linear(in_dim, hidden_dim))\n            # layers.append(nn.ReLU())\n            in_dim = hidden_dim\n            \n        layers.append(nn.Linear(in_dim, 1)) \n        layers.append(nn.Tanh())\n        self.model = nn.Sequential(*layers)\n        self.lr = lr\n        self.weight_decay = weight_decay\n        self.validation_step_outputs = []\n\n    def forward(self, x):\n        return 5 * self.model(x).squeeze(-1)  \n\n    def training_step(self, batch):\n        x, y, w = batch\n        y_hat = self(x)\n        loss = F.mse_loss(y_hat, y, reduction='none') * w  #\n        loss = loss.mean()\n        self.log(\n            'train_loss', loss, \n            on_step=False, on_epoch=True, \n            batch_size=x.size(0))\n        return loss\n\n    def validation_step(self, batch):\n        x, y, w = batch\n        y_hat = self(x)\n        loss = F.mse_loss(y_hat, y, reduction='none') * w\n        loss = loss.mean()\n        self.log(\n            'val_loss', loss, \n            on_step=False, on_epoch=True, \n            batch_size=x.size(0))\n        self.validation_step_outputs.append((y_hat, y, w))\n        return loss\n\n    def on_validation_epoch_end(self):\n        # Calculate validation WRMSE at each epoch end\n        y = torch.cat(\n            [x[1] for x in self.validation_step_outputs]\n        ).cpu().numpy()\n        \n        if self.trainer.sanity_checking:\n            # do not log the model if sanity_check is not working\n            prob = torch.cat(\n                [x[0] for x in self.validation_step_outputs]\n            ).cpu().numpy()\n        else:\n            prob = torch.cat(\n                [x[0] for x in self.validation_step_outputs]\n            ).cpu().numpy()\n            weights = torch.cat(\n                [x[2] for x in self.validation_step_outputs]\n            ).cpu().numpy()\n            # r2_val\n            val_r_square = r2_val(y, prob, weights)\n            self.log(\n                \"val_r_square\", val_r_square, \n                prog_bar=True, on_step=False, \n                on_epoch=True)\n            \n        self.validation_step_outputs.clear()\n\n    def configure_optimizers(self):\n        \n        optimizer = torch.optim.Adam(\n            self.parameters(), lr=self.lr, \n            weight_decay=self.weight_decay)\n        \n        scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n            optimizer, mode='min', \n            factor=0.5, patience=5,\n            verbose=True)\n        \n        return {\n            'optimizer': optimizer,\n            'lr_scheduler': {\n                'scheduler': scheduler,\n                'monitor': 'val_loss',\n            }\n        }\n\n    def on_train_epoch_end(self):\n        if self.trainer.sanity_checking:\n            return\n        epoch = self.trainer.current_epoch\n        metrics = {\n            k: v.item() if isinstance(v, torch.Tensor) else v \\\n            for k, v in self.trainer.logged_metrics.items()\n        }\n        formatted_metrics = {k: f\"{v:.5f}\" for k, v in metrics.items()}\n        print(f\"Epoch {epoch}: {formatted_metrics}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T15:18:18.261283Z","iopub.execute_input":"2025-05-22T15:18:18.262167Z","iopub.status.idle":"2025-05-22T15:18:18.275510Z","shell.execute_reply.started":"2025-05-22T15:18:18.262132Z","shell.execute_reply":"2025-05-22T15:18:18.274988Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 4. Load data, Run, and Save trained models","metadata":{}},{"cell_type":"code","source":"# input_path = '/kaggle/input/js24-preprocessing-create-lags'\n# df = pl.scan_parquet(f\"{input_path}/training.parquet\").select(\n#     [pl.col(\"date_id\")]).collect().to_pandas()\n##############\n# df[df.date_id > 1500]\n# df.value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T15:09:51.325529Z","iopub.execute_input":"2025-05-22T15:09:51.326138Z","iopub.status.idle":"2025-05-22T15:09:52.156578Z","shell.execute_reply.started":"2025-05-22T15:09:51.326112Z","shell.execute_reply":"2025-05-22T15:09:52.156024Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load data\n\ninput_path = '/kaggle/input/js24-preprocessing-create-lags'\n# Here we use the lagged data from a kaggler\n# Own preprocessed data can be used by uploading to input\n#####################################\nfeature_names = [\n    f\"feature_{i:02d}\" for i in range(79)] + [\n        f\"responder_{idx}_lag_1\" for idx in range(9)]\nlabel_name = 'responder_6'\nweight_name = 'weight'\n###########################\n# Note: We only take samples with date_id > 1500 to have the RAM not overcharged\ndf = pl.scan_parquet(\n    f\"{input_path}/training.parquet\").filter(\n    pl.col(\"date_id\") > 1500).collect().to_pandas()\n\nvalid = pl.scan_parquet(f\"{input_path}/validation.parquet\").filter(\n    pl.col(\"date_id\") > 1500).collect().to_pandas()\n\ndf = pd.concat([df, valid]).reset_index(drop=True)# A trick to boost LB from 0.0045->0.005\n\n# Device\ndevice = torch.device(\n    f'cuda:{CFG.gpu_id}' if torch.cuda.is_available() and CFG.use_gpu else 'cpu'\n)\naccelerator = 'gpu' if torch.cuda.is_available() and CFG.use_gpu else 'cpu'\nloader_device = 'cpu'\n\n\n# Init DataModule\n\ndf[feature_names] = df[feature_names].fillna(\n    method = 'ffill').fillna(0)\nvalid[feature_names] = valid[feature_names].fillna(\n    method = 'ffill').fillna(0)\ndata_module = DataModule(\n    df, batch_size = CFG.batch_size, \n    valid_df = valid, accelerator = loader_device\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T15:32:13.185216Z","iopub.execute_input":"2025-05-22T15:32:13.185496Z","iopub.status.idle":"2025-05-22T15:33:04.691289Z","shell.execute_reply.started":"2025-05-22T15:32:13.185476Z","shell.execute_reply":"2025-05-22T15:33:04.690649Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import gc\ndel df\ngc.collect()\n###############################\n# RUNCV = True # Turn it on when re-train\nRUNCV = False\n###############################\nif RUNCV:\n    \n    for fold in range(CFG.n_fold):\n        data_module.setup(fold, CFG.n_fold)\n        \n        # Obtain input dimension\n        \n        input_dim = data_module.train_dataset.features.shape[1]\n        \n        \n        # Initialize Model\n        model = NN(\n            input_dim = input_dim,\n            hidden_dims = CFG.n_hidden,\n            dropouts = CFG.dropouts,\n            lr = CFG.lr,\n            weight_decay = CFG.weight_decay\n        )\n        \n        # Initialize Callbacks\n        \n        early_stopping = EarlyStopping(\n            'val_loss', \n            patience=CFG.patience, \n            mode='min', verbose=False\n        )\n        checkpoint_callback = ModelCheckpoint(\n            monitor='val_loss', \n            mode='min', \n            save_top_k=1, \n            verbose=False, \n            filename=f\"./models/nn_{fold}.model\"\n        ) \n        timer = Timer()\n        \n        # Initialize Trainer\n        \n        trainer = Trainer(\n            max_epochs=CFG.max_epochs,\n            accelerator=accelerator,\n            devices=\"auto\" if CFG.use_gpu else None,\n            logger=None,\n            callbacks=[early_stopping, checkpoint_callback, timer],\n            enable_progress_bar=True\n        )\n        \n        # Start Training\n        \n        trainer.fit(\n            model, data_module.train_dataloader(CFG.loader_workers), \n            data_module.val_dataloader(CFG.loader_workers)\n        )\n        \n        # You can find trained best model in your local path\n        print(f'Fold-{fold} Training completed in {timer.time_elapsed(\"train\"):.2f}s')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T15:34:22.278417Z","iopub.execute_input":"2025-05-22T15:34:22.278749Z","execution_failed":"2025-05-22T15:38:54.175Z"}},"outputs":[],"execution_count":null}]}