{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Pytorch Lightning Efficient for G2Net","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"based on:\n\n- [G2Net / efficientnet_b7 / baseline [training]](https://www.kaggle.com/yasufuminakama/g2net-efficientnet-b7-baseline-training) \n- [PL 1Fold CQT + DeepSpeed Op Baseline + W&B [.84]](https://www.kaggle.com/ligtfeather/pl-1fold-cqt-deepspeed-op-baseline-w-b-84)","metadata":{}},{"cell_type":"markdown","source":"TODOs:\n\n- add deep speed optimizers - imp.\n- use 16 precision - imp. TESTED: Extremely slow comparing to 32 bit precision. Unknown reason (may need deep speed).\n- add callback for timing - good.\n- model optimization - imp.","metadata":{}},{"cell_type":"markdown","source":"RECORDINGs:\n    \n- highest validation auc in v7 (32 bit precision): 0.853","metadata":{}},{"cell_type":"markdown","source":"FATAL ERROR: something is causing the notebook increase system memory for every batch and I can't find why ...","metadata":{}},{"cell_type":"markdown","source":"## Preparations","metadata":{}},{"cell_type":"code","source":"!pip install -q nnAudio -qq","metadata":{"execution":{"iopub.status.busy":"2021-08-19T15:36:14.148162Z","iopub.execute_input":"2021-08-19T15:36:14.14879Z","iopub.status.idle":"2021-08-19T15:36:24.252443Z","shell.execute_reply.started":"2021-08-19T15:36:14.148668Z","shell.execute_reply":"2021-08-19T15:36:24.251061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport sys\n\n# Path settings.\nOUTPUT_DIR = './'\nif not os.path.exists(OUTPUT_DIR):\n    os.makedirs(OUTPUT_DIR)\n    \nsys.path.append('../input/timm-pytorch-image-models/pytorch-image-models-master')\n\nimport math\nimport time\nimport random\nimport shutil\nfrom pathlib import Path\nfrom contextlib import contextmanager\nfrom collections import defaultdict, Counter\n\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\n\nfrom sklearn import preprocessing\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold, KFold\n\nfrom tqdm.auto import tqdm\nfrom functools import partial\n\nimport cv2\nfrom PIL import Image\n\nimport plotly.express as px\n\nfrom nnAudio.Spectrogram import CQT1992v2 \n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import Adam, SGD\nimport torchvision.models as models\nfrom torch.nn.parameter import Parameter\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, CosineAnnealingLR, ReduceLROnPlateau\nfrom torch.cuda.amp import autocast, GradScaler\n\nimport pytorch_lightning as pl\nfrom pytorch_lightning import seed_everything\nfrom pytorch_lightning.callbacks import ModelCheckpoint, EarlyStopping\nfrom pytorch_lightning.loggers import WandbLogger\n\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom albumentations import ImageOnlyTransform\n\nimport timm\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2021-08-19T15:36:24.254756Z","iopub.execute_input":"2021-08-19T15:36:24.25518Z","iopub.status.idle":"2021-08-19T15:36:33.684537Z","shell.execute_reply.started":"2021-08-19T15:36:24.255133Z","shell.execute_reply":"2021-08-19T15:36:33.683379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# General parameter settings.\nclass CFG:\n    apex=False\n    debug=False\n    print_freq=100\n    num_workers=4\n    model_name='tf_efficientnet_b7_ns'\n    scheduler='CosineAnnealingLR' # ['ReduceLROnPlateau', 'CosineAnnealingLR', 'CosineAnnealingWarmRestarts']\n    epochs=3 \n    precision=32 # 16\n    #factor=0.2 # ReduceLROnPlateau\n    #patience=4 # ReduceLROnPlateau\n    #eps=1e-6 # ReduceLROnPlateau\n    T_max=3 # CosineAnnealingLR\n    #T_0=3 # CosineAnnealingWarmRestarts\n    lr=1e-4\n    min_lr=1e-6\n    batch_size=64 # Tried: 48 for 32 bit precision (memory error), 64 for 16 bit precision.\n    weight_decay=1e-6\n    gradient_accumulation_steps=1\n    max_grad_norm=1000\n    qtransform_params={\"sr\": 2048, \"fmin\": 20, \"fmax\": 1024, \"hop_length\": 32, \"bins_per_octave\": 8}\n    seed=42\n    target_size=1\n    target_col='target'\n    n_fold=5\n    trn_fold=[0] # [0, 1, 2, 3, 4]\n    train=True\n    grad_cam=True\n    \nif CFG.debug:\n    CFG.epochs = 1\n    train = train.sample(n=10000, random_state=CFG.seed).reset_index(drop=True)\n    \ndef class2dict(f):\n    return dict((name, getattr(f, name)) for name in dir(f) if not name.startswith('__'))","metadata":{"execution":{"iopub.status.busy":"2021-08-19T15:36:33.687183Z","iopub.execute_input":"2021-08-19T15:36:33.687673Z","iopub.status.idle":"2021-08-19T15:36:33.699037Z","shell.execute_reply.started":"2021-08-19T15:36:33.687601Z","shell.execute_reply":"2021-08-19T15:36:33.696995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Wandb settigs - login using kaggle secrets.\nfrom kaggle_secrets import UserSecretsClient\nimport wandb\n\nuser_secrets = UserSecretsClient()\nwandb_api = user_secrets.get_secret(\"wandb_api\")\nwandb.login(key=wandb_api)\n\ndef class2dict(f):\n    return dict((name, getattr(f, name)) for name in dir(f) if not name.startswith('__'))\n\nrun = wandb.init(project='G2Net-exp', \n                 group=CFG.model_name, # group changes with model for split cv\n                 config=class2dict(CFG),\n                 job_type='train')","metadata":{"execution":{"iopub.status.busy":"2021-08-19T15:36:33.701589Z","iopub.execute_input":"2021-08-19T15:36:33.702042Z","iopub.status.idle":"2021-08-19T15:36:41.882734Z","shell.execute_reply.started":"2021-08-19T15:36:33.701999Z","shell.execute_reply":"2021-08-19T15:36:41.881401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seed_everything(42)","metadata":{"execution":{"iopub.status.busy":"2021-08-19T15:36:41.884843Z","iopub.execute_input":"2021-08-19T15:36:41.885568Z","iopub.status.idle":"2021-08-19T15:36:41.908047Z","shell.execute_reply.started":"2021-08-19T15:36:41.88552Z","shell.execute_reply":"2021-08-19T15:36:41.906927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Loading & Data Engineering","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv('../input/g2net-gravitational-wave-detection/training_labels.csv')\ntest_df = pd.read_csv('../input/g2net-gravitational-wave-detection/sample_submission.csv')\n\ntrain_dir = '../input/g2net-gravitational-wave-detection/train/'\ntest_dir = '../input/g2net-gravitational-wave-detection/test/'\n\ndef id2path(name, folder=train_dir):\n    path = os.path.join(folder, f'{name[0]}/{name[1]}/{name[2]}/{name}.npy')\n    return path\n\ntrain_df['path'] = train_df['id'].apply(lambda x: id2path(x, train_dir))\ntest_df['path'] = train_df['id'].apply(lambda x: id2path(x, test_dir))","metadata":{"execution":{"iopub.status.busy":"2021-08-19T15:36:41.910101Z","iopub.execute_input":"2021-08-19T15:36:41.911375Z","iopub.status.idle":"2021-08-19T15:36:46.275917Z","shell.execute_reply.started":"2021-08-19T15:36:41.91133Z","shell.execute_reply":"2021-08-19T15:36:46.274713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-19T15:36:46.277794Z","iopub.execute_input":"2021-08-19T15:36:46.278595Z","iopub.status.idle":"2021-08-19T15:36:46.302265Z","shell.execute_reply.started":"2021-08-19T15:36:46.278551Z","shell.execute_reply":"2021-08-19T15:36:46.301098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-19T15:36:46.303991Z","iopub.execute_input":"2021-08-19T15:36:46.304743Z","iopub.status.idle":"2021-08-19T15:36:46.324672Z","shell.execute_reply.started":"2021-08-19T15:36:46.304699Z","shell.execute_reply":"2021-08-19T15:36:46.323234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## DataModule","metadata":{}},{"cell_type":"code","source":"# Transform functions. Add Channel dimension (default 1) for (H, W) input.\ndef get_train_transforms():\n    return A.Compose([\n        ToTensorV2(p=1.0)\n    ])\n\ndef get_valid_transforms():\n    return A.Compose([\n        ToTensorV2(p=1.0)\n    ])\n\ndef get_test_transforms():\n    return A.Compose([\n        ToTensorV2(p=1.0)\n    ])","metadata":{"execution":{"iopub.status.busy":"2021-08-19T15:36:46.331478Z","iopub.execute_input":"2021-08-19T15:36:46.331921Z","iopub.status.idle":"2021-08-19T15:36:46.345058Z","shell.execute_reply.started":"2021-08-19T15:36:46.331878Z","shell.execute_reply":"2021-08-19T15:36:46.343677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Dataset.\nclass BaseDataset(Dataset):\n    \"\"\"Basic dataset for training, validation and testing.\n    \n    Must thoroughly inspect the data dimensions after each transformation. Because you\n    wouldn't know what they will become unless you test with samples.\n    For example, both toTensorV2 and nnAudio CQT transform will add one dimension (out the last)\n    to the input data. We need to squeeze & unsqueeze them a lot.\n    \"\"\"\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.file_names = df['path'].values\n        self.labels = df[CFG.target_col].values\n        self.wave_transform = CQT1992v2(**CFG.qtransform_params)\n        self.transform = transform # image transform\n        \n    def apply_qtransform(self, waves):\n        waves = np.hstack(waves) # 1D data array\n        waves = waves / np.max(waves) # normalization\n        waves = torch.from_numpy(waves).float()\n       \n        image = self.wave_transform(waves) # [1, 46, 385]\n        \n        return image\n    \n    def __getitem__(self, idx): # always idx\n        # Load data.\n        file_path = self.file_names[idx]\n        waves = np.load(file_path)\n        \n        # Wave2Image transformation applied.\n        image = self.apply_qtransform(waves)\n        image = image.squeeze().numpy() # [46, 385]\n        \n        # Image transformation.\n        if self.transform:\n            image = self.transform(image=image)['image'] # [1, 46, 385]\n            \n        label = torch.tensor(self.labels[idx]).float()\n        \n        return image, label\n        \n    def __len__(self):\n        return len(self.file_names)","metadata":{"execution":{"iopub.status.busy":"2021-08-19T15:36:46.350905Z","iopub.execute_input":"2021-08-19T15:36:46.351645Z","iopub.status.idle":"2021-08-19T15:36:46.369586Z","shell.execute_reply.started":"2021-08-19T15:36:46.351599Z","shell.execute_reply":"2021-08-19T15:36:46.368029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# View dataset samples: image - 2d sigle channel.\ntrain_dataset = BaseDataset(train_df, transform=get_train_transforms())\n\nfor i in random.sample(list(range(len(train_dataset))), 3):\n    image, label = train_dataset[i]\n    fig = px.imshow(image.squeeze(), title=f'label: {int(label)}')\n    fig.update_layout(height=200,\n                      title={\n                          'x': 0.5,\n                          'y': 0.9\n                      },\n                      margin=dict(t=10, b=10, l=10, r=10))\n    fig.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-19T15:36:46.371294Z","iopub.execute_input":"2021-08-19T15:36:46.372037Z","iopub.status.idle":"2021-08-19T15:36:48.149432Z","shell.execute_reply.started":"2021-08-19T15:36:46.371992Z","shell.execute_reply":"2021-08-19T15:36:48.147907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Data split before feeding into datloader.\nfold = StratifiedKFold(n_splits=CFG.n_fold, shuffle=True, random_state=CFG.seed)\n\nfor n, (train_idx, val_idx) in enumerate(fold.split(train_df, train_df[CFG.target_col])):\n    train_df.loc[val_idx, 'fold'] = int(n) # Give fold label for validation folds.\n    \ntrain_df['fold'] = train_df['fold'].astype(int)\nprint(train_df.groupby(['fold', 'target']).size())","metadata":{"execution":{"iopub.status.busy":"2021-08-19T15:36:48.15124Z","iopub.execute_input":"2021-08-19T15:36:48.15169Z","iopub.status.idle":"2021-08-19T15:36:48.340726Z","shell.execute_reply.started":"2021-08-19T15:36:48.151638Z","shell.execute_reply":"2021-08-19T15:36:48.339536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# DataModule ready for split. When transfer data from CPU to GPU, use pin_memory to speed up.\nclass G2NetDataModule(pl.LightningDataModule):\n    def __init__(self, df: pd.DataFrame, fold: int = 0):\n        super().__init__()\n        self.df = df\n        self.fold = fold\n       \n    def setup(self):\n        train_idxs = self.df[self.df['fold'] != self.fold].index\n        valid_idxs = self.df[self.df['fold'] == self.fold].index\n        \n        train_fold = self.df.loc[train_idxs].reset_index(drop=True)\n        valid_fold = self.df.loc[valid_idxs].reset_index(drop=True)\n        \n        self.train_dataset = BaseDataset(train_fold, transform=get_train_transforms())\n        self.valid_dataset = BaseDataset(valid_fold, transform=get_valid_transforms())\n        self.test_dataset = BaseDataset(test_df, transform=get_test_transforms())\n        \n    def train_dataloader(self):\n        return DataLoader(self.train_dataset,\n                          batch_size=CFG.batch_size,\n                          shuffle=True,\n                          num_workers=CFG.num_workers,\n                          pin_memory=True)\n    \n    def val_dataloader(self):\n        return DataLoader(self.valid_dataset,\n                          batch_size=CFG.batch_size,\n                          shuffle=False,\n                          num_workers=CFG.num_workers,\n                          pin_memory=True)\n    \n    def test_dataloader(self):\n        return DataLoader(self.test_dataset,\n                          batch_size=CFG.batch_size,\n                          shuffle=False,\n                          num_workers=CFG.num_workers,\n                          pin_memory=True)","metadata":{"execution":{"iopub.status.busy":"2021-08-19T15:36:48.342435Z","iopub.execute_input":"2021-08-19T15:36:48.343051Z","iopub.status.idle":"2021-08-19T15:36:48.356695Z","shell.execute_reply.started":"2021-08-19T15:36:48.343007Z","shell.execute_reply":"2021-08-19T15:36:48.355237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Lightning Module","metadata":{}},{"cell_type":"code","source":"class G2NetLightningModule(pl.LightningModule):\n    def __init__(self, pretrained=True):\n        super().__init__()\n        # Set model. Output: [48, 1], 48 for batches, 1 for output.\n        self.model = timm.create_model(CFG.model_name, pretrained=pretrained, in_chans=1)\n        n_features = self.model.classifier.in_features\n        self.model.classifier = nn.Linear(n_features, CFG.target_size)\n        \n        # Set loss function.\n        self.criterion = nn.BCEWithLogitsLoss()\n    \n    def forward(self, x):\n        output = self.model(x)\n        return output\n    \n    def training_step(self, batch, batch_idx):        \n        x, y = batch # [C, H, W], [labels] from dataloader.\n        y_preds = self.model(x).view(-1) # Reduce one dimension.\n        loss = self.criterion(y_preds, y)\n        score = roc_auc_score(y.cpu().numpy(), y_preds.sigmoid().detach().cpu().numpy())\n        \n        # Log score and loss for each step (batch).\n        self.log('train_auc_step', score, on_step=True, prog_bar=True, logger=True)\n        self.log('train_loss_step', loss, on_step=True, prog_bar=True, logger=True)\n        \n        return dict(loss=loss, predictions=y_preds, labels=y)\n    \n    def training_epoch_end(self, outputs):\n        \"\"\"Compute and log epoch roc-auc score.\n        \"\"\"\n        preds = []\n        labels = []\n        \n        for output in outputs:\n            preds += output['predictions']\n            labels += output['labels']\n            \n        preds = torch.stack(preds)\n        labels = torch.stack(labels)\n        \n        train_epoch_auc = roc_auc_score(labels.cpu().numpy(), preds.sigmoid().detach().cpu().numpy())\n        self.log('train_auc_epoch', train_epoch_auc, prog_bar=True, logger=True)\n        \n    def validation_step(self, batch, batch_idx):\n        x, y = batch\n        y_preds = self.model(x).view(-1)\n        loss = self.criterion(y_preds, y)\n        \n        self.log('val_loss', loss, prog_bar=True, logger=True) # Unable for step log.\n        \n        return dict(predictions=y_preds, labels=y)\n    \n    def validation_epoch_end(self, outputs):\n        \"\"\"Compute and log epoch roc-auc score.\n        \"\"\"\n        preds = []\n        labels = []\n        \n        for output in outputs:\n            preds += output['predictions']\n            labels += output['labels']\n            \n        preds = torch.stack(preds)\n        labels = torch.stack(labels)\n        \n        val_epoch_auc = roc_auc_score(labels.cpu().numpy(), preds.sigmoid().detach().cpu().numpy())\n        self.log('val_auc_epoch', val_epoch_auc, prog_bar=True, logger=True)\n    \n    def configure_optimizers(self):\n        # Optimizer.\n        optimizer = Adam(self.model.parameters(), lr=CFG.lr, weight_decay=CFG.weight_decay)\n        \n        # Scheduler.\n        if CFG.scheduler == 'ReduceLROnPlateau':\n            scheduler = ReduceLROnPlateau(optimizer, \n                                          mode='min', \n                                          factor=CFG.factor, \n                                          patience=CFG.patience, \n                                          verbose=True, \n                                          eps=CFG.eps)\n        elif CFG.scheduler == 'CosineAnnealingLR':\n            scheduler = CosineAnnealingLR(optimizer, \n                                          T_max=CFG.T_max, \n                                          eta_min=CFG.min_lr, \n                                          last_epoch=-1)\n        elif CFG.scheduler == 'CosineAnnealingWarmRestarts':\n            scheduler = CosineAnnealingWarmRestarts(optimizer, \n                                                    T_0=CFG.T_0, \n                                                    T_mult=1,\n                                                    eta_min=CFG.min_lr ,\n                                                    last_epoch=-1)\n        \n        return dict(optimizer=optimizer,\n                    lr_scheduler=scheduler)","metadata":{"execution":{"iopub.status.busy":"2021-08-19T15:36:48.358542Z","iopub.execute_input":"2021-08-19T15:36:48.359079Z","iopub.status.idle":"2021-08-19T15:36:48.38769Z","shell.execute_reply.started":"2021-08-19T15:36:48.359014Z","shell.execute_reply":"2021-08-19T15:36:48.383681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set up callbacks, including checkpoints, early stop, etc.\nearly_stop_callback = EarlyStopping(monitor='val_auc_epoch',\n                                    mode='max',\n                                    patience=5) # Early stop in each epoch.\ncheckpoint_callback = ModelCheckpoint(dirpath='./checkpoints/',\n                                      filename='best-checkpoint-fold{fold}-val_auc{val_auc:.3f}',\n                                      monitor='val_auc_epoch',\n                                      mode='max',\n                                      save_top_k=2,\n                                      verbose=True)","metadata":{"execution":{"iopub.status.busy":"2021-08-19T15:36:48.391376Z","iopub.execute_input":"2021-08-19T15:36:48.392821Z","iopub.status.idle":"2021-08-19T15:36:48.410564Z","shell.execute_reply.started":"2021-08-19T15:36:48.39277Z","shell.execute_reply":"2021-08-19T15:36:48.409259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Logger settings, including runtime logger and wandb setting.\ndef get_logger(fold: int):\n    return WandbLogger(project='G2Net-exp', # same as init\n                       config=class2dict(CFG), # same as init\n                       group=CFG.model_name, # group changes with model\n                       name=f'fold-{fold}', # fold changes\n                       job_type='train')","metadata":{"execution":{"iopub.status.busy":"2021-08-19T15:36:48.41514Z","iopub.execute_input":"2021-08-19T15:36:48.415744Z","iopub.status.idle":"2021-08-19T15:36:48.429179Z","shell.execute_reply.started":"2021-08-19T15:36:48.415701Z","shell.execute_reply":"2021-08-19T15:36:48.427407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train Loop","metadata":{}},{"cell_type":"code","source":"# Initialize model.\nmodel = G2NetLightningModule()\n\nfor fold in range(CFG.n_fold):\n    if fold > 0: break # Debug and Parameter optimization before model selection.\n    \n    # DataModule.\n    data_module = G2NetDataModule(train_df, fold)\n    data_module.setup() # Just to check dataset information by running setup.\n    \n    # Logger.\n    wandb_logger = get_logger(fold)\n    \n    # Trainer.\n    trainer = pl.Trainer(gpus=1,\n                         precision=CFG.precision,\n                         callbacks=[early_stop_callback,\n                                    checkpoint_callback],\n                         max_epochs=CFG.epochs,\n                         logger=wandb_logger,\n                         deterministic=True,\n                         stochastic_weight_avg=True, # https://arxiv.org/abs/1803.05407\n                         progress_bar_refresh_rate=1)\n    \n    # Let's go\n    trainer.fit(model, data_module)\n    \n    print(f'===Fold-{fold} running successful===')","metadata":{"execution":{"iopub.status.busy":"2021-08-19T15:37:26.264254Z","iopub.execute_input":"2021-08-19T15:37:26.264669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}