{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"

EfficientNet CQT pytorch-lightning baseline

\n\nThe main purpose of this kernel is to create a baseline and make a first submission to the competition.\n\nOther objectives are:\n\n\n
\n\nPlease upvote this notebook if you find it useful, or if you copy it.","metadata":{}},{"cell_type":"code","source":"!pip install -q nnAudio\n!pip install -q timm","metadata":{"execution":{"iopub.status.busy":"2021-08-10T08:27:50.229892Z","iopub.execute_input":"2021-08-10T08:27:50.230412Z","iopub.status.idle":"2021-08-10T08:28:06.092339Z","shell.execute_reply.started":"2021-08-10T08:27:50.230291Z","shell.execute_reply":"2021-08-10T08:28:06.091275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Imports\n\nimport numpy as np\nimport pandas as pd\nimport os\nimport random\nimport glob\nimport gc; gc.enable()\n\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom sklearn.metrics import roc_auc_score\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim import Adam\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\n\nfrom pytorch_lightning import LightningModule, LightningDataModule, Trainer\nfrom pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint\n\nimport timm\n\nfrom tqdm.notebook import tqdm\n\nfrom nnAudio.Spectrogram import CQT1992v2\n\npd.options.display.max_colwidth = None","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-08-10T08:28:06.095896Z","iopub.execute_input":"2021-08-10T08:28:06.09616Z","iopub.status.idle":"2021-08-10T08:28:09.751678Z","shell.execute_reply.started":"2021-08-10T08:28:06.09613Z","shell.execute_reply":"2021-08-10T08:28:09.750748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Constants\n\nINPUT_DIR = '../input/g2net-gravitational-wave-detection'\nTRAIN_DIR = '../input/g2net-gravitational-wave-detection/train'\nTEST_DIR = '../input/g2net-gravitational-wave-detection/test'\nOUTPUT_DIR = '/kaggle/working'\n\nSEED = 13\n\nDEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'\n\nconfig = {\n 'seed' : 13,\n 'base_model':'tf_efficientnet_b4',\n 'base_model_classifier':'classifier',\n 'classes' : 1,\n 'precision' : 16,\n 'train_batch_size' : 512,\n 'val_batch_size' : 256,\n 'epochs' : 3,\n 'num_workers' : 8,\n\n # Optimizer and LR scheduling - General\n 'weight_decay': 1e-7,\n 'lr' : 3e-3,\n 'min_lr': 1e-4,\n 'scheduler': 'CosineAnnealingLR',\n\n # CosineAnnealingLR\n 't_max': 5,\n}","metadata":{"execution":{"iopub.status.busy":"2021-08-10T08:28:09.753888Z","iopub.execute_input":"2021-08-10T08:28:09.754255Z","iopub.status.idle":"2021-08-10T08:28:09.804512Z","shell.execute_reply.started":"2021-08-10T08:28:09.754214Z","shell.execute_reply":"2021-08-10T08:28:09.803616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Utils\n\ndef seed_everything(seed):\n random.seed(seed)\n np.random.seed(seed)\n torch.manual_seed(seed)\n torch.cuda.manual_seed_all(seed)\n torch.backends.cudnn.deterministic = True\n torch.backends.cudnn.benchmark = True\n \ndef get_path(x, basedir):\n return f'{basedir}/{x[0]}/{x[1]}/{x[2]}/{x}.npy'","metadata":{"execution":{"iopub.status.busy":"2021-08-10T08:28:09.806728Z","iopub.execute_input":"2021-08-10T08:28:09.807357Z","iopub.status.idle":"2021-08-10T08:28:09.815405Z","shell.execute_reply.started":"2021-08-10T08:28:09.807264Z","shell.execute_reply":"2021-08-10T08:28:09.814498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

Read data and create folds

","metadata":{}},{"cell_type":"code","source":"# Read competition data\nall_data = pd.read_csv(os.path.join(INPUT_DIR, 'training_labels.csv'))\n\n# Add paths to signal files\nall_data['path'] = all_data.id.apply(get_path , basedir=TRAIN_DIR)","metadata":{"execution":{"iopub.status.busy":"2021-08-10T08:28:09.816677Z","iopub.execute_input":"2021-08-10T08:28:09.817176Z","iopub.status.idle":"2021-08-10T08:28:10.770247Z","shell.execute_reply.started":"2021-08-10T08:28:09.817106Z","shell.execute_reply":"2021-08-10T08:28:10.768799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"# Create stratified (on target) folds for the training data. These folds will be inherited by other splits of the data\nkf = StratifiedKFold(n_splits=5, shuffle=True, random_state=SEED)\n \n# Create non-stratified folds\n# kf = KFold(n_splits=5, shuffle=True, random_state=config['seed'])\nfor f, (t_, v_) in enumerate(kf.split(all_data, all_data.target)):\n all_data.loc[v_, 'fold'] = f\n \nall_data['fold'] = all_data['fold'].astype(int) \n\n# The train dataset is large, and training would take too long, so I use just a 10%\n# Further split train_df into a subset for quick experiments \n_, data = train_test_split(all_data, test_size=0.10, \n random_state=SEED, shuffle=True,\n stratify = all_data[['target', 'fold']]) # stratify both by target and fold\n\ndata.reset_index(drop=True, inplace=True)\ndata.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-10T08:28:10.772038Z","iopub.execute_input":"2021-08-10T08:28:10.772657Z","iopub.status.idle":"2021-08-10T08:28:15.494893Z","shell.execute_reply.started":"2021-08-10T08:28:10.772614Z","shell.execute_reply":"2021-08-10T08:28:15.494023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

Lightning modules

","metadata":{}},{"cell_type":"code","source":"class G2NetDataset(Dataset):\n def __init__(self, paths, targets=None):\n self.paths = paths\n self.targets = targets\n self.wave_transform = CQT1992v2(sr=2048, fmin=20, fmax=1024, hop_length=64)\n\n def __len__(self):\n return len(self.paths)\n \n def get_qtransform(self, x):\n image = []\n for i in range(3):\n signal = x[i] / np.max(x[i])\n signal = torch.from_numpy(signal).float()\n channel = self.wave_transform(signal).squeeze()\n image.append(channel)\n \n return torch.stack(image, dim=0)\n \n def __getitem__(self, idx):\n signals = np.load(self.paths[idx])\n image = self.get_qtransform(signals)\n if self.targets is not None:\n return image, torch.tensor(self.targets[idx], dtype=torch.long)\n else:\n return image\n \nclass G2NetDataModule(LightningDataModule):\n\n def __init__(self, train_df, val_df, config):\n super().__init__()\n self.train_df = train_df\n self.val_df = val_df\n self.config = config\n\n def setup(self, stage=None):\n\n # Create train dataset\n self.train_dataset = G2NetDataset(self.train_df.path.values, self.train_df.target.values)\n\n # Create val dataset\n self.val_dataset = G2NetDataset(self.val_df.path.values, self.val_df.target.values)\n\n def train_dataloader(self):\n return DataLoader(self.train_dataset,\n batch_size=self.config['train_batch_size'],\n num_workers=self.config['num_workers'],\n shuffle=True,\n pin_memory=False)\n\n def val_dataloader(self):\n return DataLoader(self.val_dataset,\n batch_size=self.config['val_batch_size'],\n num_workers=self.config['num_workers'],\n shuffle=False,\n pin_memory=False)\n \nclass G2NetClassifier(LightningModule):\n\n def __init__(self, config):\n super().__init__()\n self.save_hyperparameters(config)\n\n self.classifier = timm.create_model(self.hparams.base_model, pretrained=True)\n n_features = self.classifier._modules[self.hparams.base_model_classifier].in_features\n self.classifier._modules[self.hparams.base_model_classifier] = nn.Linear(n_features, self.hparams.classes)\n\n self.loss = nn.BCEWithLogitsLoss()\n \n def forward(self, x):\n out = self.classifier(x)\n return out\n\n def training_step(self, batch, batch_idx):\n images, labels = batch\n logits = self(images).squeeze(1)\n loss = self.loss(logits, labels.float())\n y_true = labels.cpu().numpy()\n y_pred = logits.cpu().detach().numpy()\n score = roc_auc_score(y_true, y_pred)\n self.log('train_loss', loss, on_step=False, on_epoch=True, prog_bar=True)\n self.log('train_score', score, on_step=False, on_epoch=True, prog_bar=True)\n return {'loss': loss, 'train_score': score}\n\n\n def validation_step(self, batch, batch_idx):\n images, labels = batch\n logits = self(images).squeeze(1)\n loss = self.loss(logits, labels.float())\n y_true = labels.cpu().numpy()\n y_pred = logits.cpu().detach().numpy()\n score = roc_auc_score(y_true, y_pred)\n self.log('val_loss', loss, on_step=False, on_epoch=True, prog_bar=True)\n self.log('val_score', score, on_step=False, on_epoch=True, prog_bar=True)\n return {'loss': loss, 'val_score': score}\n\n \n def configure_optimizers(self):\n optimizer = Adam(self.classifier.parameters(),\n lr=self.hparams.lr,\n weight_decay=self.hparams.weight_decay)\n scheduler = CosineAnnealingLR(optimizer, T_max = self.hparams.t_max, eta_min=self.hparams.min_lr, last_epoch=-1)\n return {'optimizer':optimizer, 'scheduler': scheduler}\n ","metadata":{"execution":{"iopub.status.busy":"2021-08-10T08:28:15.496312Z","iopub.execute_input":"2021-08-10T08:28:15.496669Z","iopub.status.idle":"2021-08-10T08:28:15.522583Z","shell.execute_reply.started":"2021-08-10T08:28:15.496638Z","shell.execute_reply":"2021-08-10T08:28:15.521124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

Training

","metadata":{}},{"cell_type":"code","source":"# Train 5 folds\n\ntrain = True\n\nif train:\n for fold in range(5):\n print(f'*** fold {fold} ***')\n\n train_df = data.loc[data.fold != fold]\n val_df = data.loc[data.fold == fold]\n\n seed_everything(config['seed'])\n\n dm = G2NetDataModule(train_df, val_df, config)\n\n filename = f\"{config['base_model']}-f{fold}-{{val_score:.3f}}\"\n\n checkpoint_callback = ModelCheckpoint(monitor='val_score', dirpath=OUTPUT_DIR, mode='max', filename=filename)\n trainer = Trainer(gpus=1,\n max_epochs=config['epochs'],\n precision=config['precision'],\n num_sanity_val_steps=0,\n callbacks=[checkpoint_callback]\n )\n\n model = G2NetClassifier(config)\n trainer.fit(model, datamodule=dm)\n\n del model\n gc.collect()\n","metadata":{"execution":{"iopub.status.busy":"2021-08-10T08:28:15.5252Z","iopub.execute_input":"2021-08-10T08:28:15.525611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

Prediction and submission

","metadata":{}},{"cell_type":"code","source":"# Prediction function\n\ndef predict(model, data_loader):\n \n model.to(DEVICE)\n model.eval()\n model.zero_grad()\n \n predictions = []\n for images in tqdm(data_loader):\n images = images.to(DEVICE)\n logits = model(images)\n logits = logits.squeeze(1)\n predictions.extend(logits.cpu().detach().numpy())\n \n return predictions","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load submission file\nsubmission = pd.read_csv(os.path.join(INPUT_DIR, 'sample_submission.csv'))\n\n# Add paths to signal files\nsubmission['path'] = submission.id.apply(get_path , basedir=TEST_DIR)\n\nsubmission.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Predict on the competition test set\n\nall_predictions = []\n\nfor path in glob.glob(OUTPUT_DIR + '/*.ckpt'):\n print(path) \n model = G2NetClassifier.load_from_checkpoint(path)\n dataset = G2NetDataset(submission.path.values)\n data_loader = DataLoader(dataset, batch_size=config['val_batch_size'],\n num_workers=config['num_workers'], shuffle=False, pin_memory=False)\n \n predictions = predict(model, data_loader)\n all_predictions.append(predictions)\n del model\n gc.collect()\n\npredictions = np.mean(all_predictions, axis=0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save submission file\n\nsubmission['target'] = predictions\nsubmission.drop(columns='path', inplace=True)\nsubmission.to_csv('submission.csv', index = False)\nsubmission.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}