{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Simple starter using Tez\nThis is a very simple starter based on a fantastic trainer [Tez](https://github.com/abhishekkrthakur/tez), which is developed by [@abhishek](https://www.kaggle.com/abhishek) (e.g. https://www.kaggle.com/abhishek/using-tez-in-leaf-disease-classification).\n\nModel: efficientnetv2_rw_s","metadata":{}},{"cell_type":"code","source":"!pip install timm\n!pip install -q nnAudio\n!pip install git+https://github.com/yseeker/tez_custom\n!pip install wandb\nfrom pathlib import Path\n\nimport os\nimport sys\nimport random\nfrom tqdm import tqdm\nimport math\n\nimport pandas as pd\nimport numpy as np\nfrom sklearn import metrics\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn import model_selection as sk_model_selection\nimport torch\nimport torch.nn as nn\nimport torchvision\n\nimport cv2\nfrom PIL import Image\nimport albumentations as A\n\nimport tez\nfrom tez.datasets import ImageDataset\nfrom tez.callbacks import EarlyStopping\nimport timm\nfrom nnAudio.Spectrogram import CQT1992v2","metadata":{"execution":{"iopub.status.busy":"2021-07-04T17:37:22.982941Z","iopub.execute_input":"2021-07-04T17:37:22.9833Z","iopub.status.idle":"2021-07-04T17:38:02.155917Z","shell.execute_reply.started":"2021-07-04T17:37:22.983223Z","shell.execute_reply":"2021-07-04T17:38:02.155042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nclass CFG:\n    project_name = 'project name'\n    pretrained_model_name = 'efficientnetv2_rw_s'\n    lr = 5e-4\n    batch_size= 128\n    wandb_note = f'bs{batch_size}_adamW_default_lr{lr}'\n    pretrained = True\n    prettained_path = '../input/timm_weight/efficientnet_v2s_ra2_288-a6477665.pth'\n    input_channels = 1\n    out_dim = 1\n    colab_or_kaggle = 'colab'\n    wandb_exp_name = f'{pretrained_model_name}_{colab_or_kaggle}_{wandb_note}'\n    monitor = 'valid_roc_auc'\n    epochs = 2\n    num_of_fold = 5\n    seed = 42\n    num_workers = 8\n    fp16 = False\n    checkpoint_path = ''\n    patience_mode = 'max'\n    patience = 10\n    delta = 0.001\n    mixup_alpha = 1.0\n    benchmark = False\n    wandb = False","metadata":{"execution":{"iopub.status.busy":"2021-07-04T17:38:02.157353Z","iopub.execute_input":"2021-07-04T17:38:02.157691Z","iopub.status.idle":"2021-07-04T17:38:02.165052Z","shell.execute_reply.started":"2021-07-04T17:38:02.157655Z","shell.execute_reply":"2021-07-04T17:38:02.164219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_aug = A.Compose(\n    [\n        A.HorizontalFlip(p=0.5),\n        A.VerticalFlip(p=0.5),\n        A.ShiftScaleRotate(p=0.5,\n                           shift_limit = 0.2, \n                           scale_limit=0.2,\n                           rotate_limit=30, \n                           border_mode = cv2.BORDER_REPLICATE),\n        A.OneOf([\n            A.MedianBlur(p=0.3),\n            A.MotionBlur(p=0.3)\n        ]\n        )\n    ]\n)\n\ndf = pd.read_csv('../input/g2net-gravitational-wave-detection/training_labels.csv')\ndf['img_path'] = df['id'].apply(\n    lambda x: f\"../input/g2net-gravitational-wave-detection/train/{x[0]}/{x[1]}/{x[2]}/{x}.npy\"\n)\n\nX = df.img_path.values\nY = df.target.values\n\nskf = StratifiedKFold(n_splits = CFG.num_of_fold)","metadata":{"execution":{"iopub.status.busy":"2021-07-04T17:38:02.166882Z","iopub.execute_input":"2021-07-04T17:38:02.167454Z","iopub.status.idle":"2021-07-04T17:38:02.873435Z","shell.execute_reply.started":"2021-07-04T17:38:02.167415Z","shell.execute_reply":"2021-07-04T17:38:02.872604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def sigmoid(gamma):\n    if gamma < 0:\n        return 1 - 1 / (1 + math.exp(gamma))\n    return 1 / (1 + math.exp(-gamma))\n\n# define vectorized sigmoid\nsigmoid_v = np.vectorize(sigmoid)\n\ndef set_seed(seed = 0):\n    np.random.seed(seed)\n    random_state = np.random.RandomState(seed)\n    random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    return random_state","metadata":{"execution":{"iopub.status.busy":"2021-07-04T17:38:02.875045Z","iopub.execute_input":"2021-07-04T17:38:02.875368Z","iopub.status.idle":"2021-07-04T17:38:02.883Z","shell.execute_reply.started":"2021-07-04T17:38:02.875334Z","shell.execute_reply":"2021-07-04T17:38:02.881992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset and Model","metadata":{}},{"cell_type":"code","source":"class ClassificationDataset():\n    def __init__(self, image_paths, targets, transform = None): \n        self.image_paths = image_paths\n        self.targets = targets\n        self.transform = None\n        self.wave_transform = CQT1992v2(sr=2048, fmin=20, fmax=1024, hop_length=64)\n\n    def __len__(self):\n        return len(self.image_paths)\n    \n    def apply_qtransform(self, waves, transform):\n        waves = np.hstack(waves)\n        waves = waves / np.max(waves)\n        waves = torch.from_numpy(waves).float()\n        image = transform(waves)\n        return image\n    \n    def __getitem__(self, item): \n        targets = self.targets[item]\n        waves = np.load(self.image_paths[item])\n        image = self.apply_qtransform(waves, self.wave_transform)\n\n        return {\n            \"image\": torch.tensor(image, dtype=torch.float),\n            \"targets\": torch.tensor(targets, dtype=torch.float),\n        }\n    \n\nclass CustomNN(tez.Model):\n    def __init__(self):\n        super().__init__()\n        self.model = timm.create_model(CFG.pretrained_model_name, \n                                       pretrained = CFG.pretrained, \n                                       in_chans = CFG.input_channels)\n        if not CFG.pretrained: self.model.load_state_dict(torch.load(CFG.pretrained_path))\n        self.model.classifier = nn.Linear(self.model.classifier.in_features, CFG.out_dim)\n            \n        self.criterion =  nn.BCEWithLogitsLoss()\n\n\n    def forward(self, image, targets = None):\n        outputs = self.model(image)\n        if targets is not None:\n            loss = self.criterion(outputs, targets.view(-1, 1))\n            metrics = self.monitor_metrics(outputs, targets)\n            return outputs, loss, metrics\n        return outputs, None, None\n\n    def epoch_metrics(self, outputs, targets):\n        outputs = sigmoid_v(outputs)\n        roc_auc = metrics.roc_auc_score(targets, outputs)\n        return roc_auc\n\n    def monitor_metrics(self, outputs, targets):\n        outputs = outputs.sigmoid().cpu().detach().numpy()\n        targets = targets.cpu().detach().numpy()\n        if len(np.unique(targets)) > 1: \n            roc_auc = metrics.roc_auc_score(targets, outputs)\n        else: roc_auc = 0.5\n        return {\"roc_auc\": roc_auc}\n\n    def configure_optimizer(self):\n        opt = torch.optim.AdamW(self.parameters(), lr=CFG.lr, weight_decay=0.01)\n        #opt = torch.optim.SGD(self.parameters(), lr=0.001, momentum=0.9)\n        return opt\n    \n    def configure_scheduler(self):\n        sch = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(\n            self.optimizer, T_0=10, T_mult=1, eta_min=1e-6, last_epoch=-1\n        )\n        return sch","metadata":{"execution":{"iopub.status.busy":"2021-07-04T17:39:00.621613Z","iopub.execute_input":"2021-07-04T17:39:00.621977Z","iopub.status.idle":"2021-07-04T17:39:00.643497Z","shell.execute_reply.started":"2021-07-04T17:39:00.621941Z","shell.execute_reply":"2021-07-04T17:39:00.642464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = []\nfor fold_cnt, (train_index, test_index) in enumerate(skf.split(X, Y)):\n    train_images, valid_images = X[train_index], X[test_index]\n    train_targets, valid_targets = Y[train_index], Y[test_index]\n\n    train_dataset = ClassificationDataset(\n        image_paths=train_images, \n        targets=train_targets, \n        transform = None\n    )\n    valid_dataset = ClassificationDataset(\n        image_paths=valid_images, \n        targets=valid_targets, \n        transform = None\n    )\n    model = CustomNN()\n\n    es = EarlyStopping(\n        monitor=CFG.monitor, \n        model_path=CFG.checkpoint_path+f'{CFG.pretrained_model_name}_{fold_cnt}fold_{CFG.wandb_note}.cpt', \n        patience= CFG.patience, \n        mode=CFG.patience_mode,\n        delta = CFG.delta\n    )\n    model.fit(\n        cfg = CFG,\n        train_dataset = train_dataset,\n        valid_dataset = valid_dataset,\n        valid_targets = valid_targets,\n        train_bs=CFG.batch_size,\n        valid_bs=CFG.batch_size,\n        epochs=CFG.epochs,\n        callbacks=[es],\n        n_jobs = CFG.num_workers,\n        fp16=CFG.fp16,\n        benchmark = CFG.benchmark\n    )\n    models.append(model)\n    break","metadata":{"execution":{"iopub.status.busy":"2021-07-04T17:39:01.084896Z","iopub.execute_input":"2021-07-04T17:39:01.085227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv('../input/g2net-gravitational-wave-detection/sample_submission.csv')\nsubmission['img_path'] = submission['id'].apply(\n    lambda x: f\"../input/g2net-gravitational-wave-detection/test/{x[0]}/{x[1]}/{x[2]}/{x}.npy\"\n)\ntest_dataset = ClassificationDataset(\n    image_paths=submission.img_path.values, \n    targets=submission.target.values, \n    transform = None\n)\n\nfinal_preds = None\nnum_of_ave = 1\nouts = []\nfor i, model in enumerate(models):\n    for j in range(num_of_ave):\n        preds = model.predict(test_dataset, batch_size=128, n_jobs=-1)\n        if final_preds is None:\n            final_preds = preds\n        else:\n            final_preds += preds\n    final_preds = final_preds/num_of_ave\n    out = sigmoid_v(final_preds)\n    outs.append(out)\npred = np.mean(np.array(outs), axis=0)\nsubmission.target = pred\nsubmission.drop(['img_path'], axis=1, inplace=True)\nsubmission.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}