{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install  timm > /dev/null\n!pip install pydicom > /dev/null\n!pip install catalyst > /dev/null\n!pip install colorednoise > /dev/null","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport librosa as lb\nimport pandas as pd\nfrom pathlib import Path\nimport torch\nfrom  torch.utils.data import Dataset, DataLoader\nfrom tqdm.notebook import tqdm\nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom IPython.display import Audio\nimport sys    \nimport colorednoise as cn\nfrom skimage.transform import resize\nimport os\nfrom glob import glob\nfrom sklearn.model_selection import GroupKFold, StratifiedKFold\nimport cv2\nfrom skimage import io\nfrom torch import nn\nfrom datetime import datetime\nimport time\nimport random\nimport cv2\nimport torchvision\nfrom torchvision import transforms\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom  torch.cuda.amp import autocast, GradScaler\nimport timm\nimport sklearn\nimport warnings\nimport joblib\nfrom sklearn.metrics import roc_auc_score, log_loss\nfrom sklearn import metrics\nimport warnings\nimport librosa\nimport pydicom\nfrom scipy.ndimage.interpolation import zoom","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CFG = {\n    'fold_num': 5,\n    'seed': 2020,\n    'model_arch': 'resnest50d',\n    'epochs': 30,\n    'train_bs': 16,\n    'valid_bs': 32,\n    'T_0': 10,\n    'lr': 1e-4,\n    'min_lr': 1e-6,\n    'weight_decay':1e-6,\n    'num_workers': 4,\n    'accum_iter': 1, # suppoprt to do batch accumulation for backprop with effectively larger batch size\n    'verbose_step': 1,\n    'device': 'cuda:0',\n    'num_classes': 25,\n    \"sr\": 48000,\n    \"duration\": 10,\n    \"fft\": 2048,\n    \"hop\": 512,\n    \"data_root\": Path(\"../input/rfcx-species-audio-detection\"),\n    \"train_root\": Path(\"../input/rfcx-species-audio-detection/train\"),\n    \"test_root\": Path(\"../input/rfcx-species-audio-detection/test\")\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import timm\n# from pprint import pprint\n# model_names = timm.list_models(pretrained=True)\n# pprint(model_names)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"traint = pd.read_csv( '../input/rfcx-species-audio-detection/train_tp.csv' )\ntrainf = pd.read_csv( '../input/rfcx-species-audio-detection/train_fp.csv' )\ntraint.shape, trainf.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fmin = 24000\nfmax = 0\n\nfor i in range(len(traint)):\n    if fmin > float(traint[\"f_min\"][i]):\n        fmin = float(traint[\"f_min\"][i])\n    if fmax < float(traint[\"f_max\"][i]):\n        fmax = float(traint[\"f_max\"][i])\n# Get some safety margin\nfmin = int(fmin * 0.9)\nfmax = int(fmax * 1.1)\nprint('Minimum frequency: ' + str(fmin) + ', maximum frequency: ' + str(fmax))\nCFG[\"f_min\"], CFG[\"f_max\"] = fmin, fmax","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# DataAugumentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"class AudioTransform:\n    def __init__(self, always_apply=False, p=0.5):\n        self.always_apply = always_apply\n        self.p = p\n\n    def __call__(self, y: np.ndarray):\n        if self.always_apply:\n            return self.apply(y)\n        else:\n            if np.random.rand() < self.p:\n                return self.apply(y)\n            else:\n                return y\n\n    def apply(self, y: np.ndarray):\n        raise NotImplementedError\n\n\nclass wave_Compose:\n    def __init__(self, transforms: list):\n        self.transforms = transforms\n\n    def __call__(self, y: np.ndarray):\n        for trns in self.transforms:\n            y = trns(y)\n        return y\n\n\nclass wave_OneOf:\n    def __init__(self, transforms: list):\n        self.transforms = transforms\n\n    def __call__(self, y: np.ndarray):\n        n_trns = len(self.transforms)\n        trns_idx = np.random.choice(n_trns)\n        trns = self.transforms[trns_idx]\n        return trns(y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class AddGaussianNoise(AudioTransform):\n    def __init__(self, always_apply=False, p=0.5, max_noise_amplitude=0.5, **kwargs):\n        super().__init__(always_apply, p)\n\n        self.noise_amplitude = (0.0, max_noise_amplitude)\n\n    def apply(self, y: np.ndarray, **params):\n        noise_amplitude = np.random.uniform(*self.noise_amplitude)\n        noise = np.random.randn(len(y))\n        augmented = (y + noise * noise_amplitude).astype(y.dtype)\n        return augmented\n    \nclass GaussianNoiseSNR(AudioTransform):\n    def __init__(self, always_apply=False, p=0.5, min_snr=5.0, max_snr=20.0, **kwargs):\n        super().__init__(always_apply, p)\n\n        self.min_snr = min_snr\n        self.max_snr = max_snr\n\n    def apply(self, y: np.ndarray, **params):\n        snr = np.random.uniform(self.min_snr, self.max_snr)\n        a_signal = np.sqrt(y ** 2).max()\n        a_noise = a_signal / (10 ** (snr / 20))\n\n        white_noise = np.random.randn(len(y))\n        a_white = np.sqrt(white_noise ** 2).max()\n        augmented = (y + white_noise * 1 / a_white * a_noise).astype(y.dtype)\n        return augmented\n\nclass PinkNoiseSNR(AudioTransform):\n    def __init__(self, always_apply=False, p=0.5, min_snr=5.0, max_snr=20.0, **kwargs):\n        super().__init__(always_apply, p)\n\n        self.min_snr = min_snr\n        self.max_snr = max_snr\n\n    def apply(self, y: np.ndarray, **params):\n        snr = np.random.uniform(self.min_snr, self.max_snr)\n        a_signal = np.sqrt(y ** 2).max()\n        a_noise = a_signal / (10 ** (snr / 20))\n\n        pink_noise = cn.powerlaw_psd_gaussian(1, len(y))\n        a_pink = np.sqrt(pink_noise ** 2).max()\n        augmented = (y + pink_noise * 1 / a_pink * a_noise).astype(y.dtype)\n        return augmented\n\nclass PitchShift(AudioTransform):\n    def __init__(self, always_apply=False, p=0.3, max_steps=5, sr=32000):\n        super().__init__(always_apply, p)\n\n        self.max_steps = max_steps\n        self.sr = sr\n\n    def apply(self, y: np.ndarray, **params):\n        n_steps = np.random.randint(-self.max_steps, self.max_steps)\n        augmented = librosa.effects.pitch_shift(y, sr=self.sr, n_steps=n_steps)\n        return augmented\n    \nclass TimeStretch(AudioTransform):\n    def __init__(self, always_apply=False, p=0.5, max_rate=1.2):\n        super().__init__(always_apply, p)\n\n        self.max_rate = max_rate\n\n    def apply(self, y: np.ndarray, **params):\n        rate = np.random.uniform(0, self.max_rate)\n        augmented = librosa.effects.time_stretch(y, rate)\n        return augmented\n\nclass TimeShift(AudioTransform):\n    def __init__(self, always_apply=False, p=0.5, max_shift_second=2, sr=32000, padding_mode=\"replace\"):\n        super().__init__(always_apply, p)\n    \n        assert padding_mode in [\"replace\", \"zero\"], \"`padding_mode` must be either 'replace' or 'zero'\"\n        self.max_shift_second = max_shift_second\n        self.sr = sr\n        self.padding_mode = padding_mode\n\n    def apply(self, y: np.ndarray, **params):\n        shift = np.random.randint(-self.sr * self.max_shift_second, self.sr * self.max_shift_second)\n        augmented = np.roll(y, shift)\n        if self.padding_mode == \"zero\":\n            if shift > 0:\n                augmented[:shift] = 0\n            else:\n                augmented[shift:] = 0\n        return augmented\n\nclass VolumeControl(AudioTransform):\n    def __init__(self, always_apply=False, p=0.5, db_limit=10, mode=\"uniform\"):\n        super().__init__(always_apply, p)\n\n        assert mode in [\"uniform\", \"fade\", \"fade\", \"cosine\", \"sine\"], \\\n            \"`mode` must be one of 'uniform', 'fade', 'cosine', 'sine'\"\n\n        self.db_limit= db_limit\n        self.mode = mode\n\n    def apply(self, y: np.ndarray, **params):\n        db = np.random.uniform(-self.db_limit, self.db_limit)\n        if self.mode == \"uniform\":\n            db_translated = 10 ** (db / 20)\n        elif self.mode == \"fade\":\n            lin = np.arange(len(y))[::-1] / (len(y) - 1)\n            db_translated = 10 ** (db * lin / 20)\n        elif self.mode == \"cosine\":\n            cosine = np.cos(np.arange(len(y)) / len(y) * np.pi * 2)\n            db_translated = 10 ** (db * cosine / 20)\n        else:\n            sine = np.sin(np.arange(len(y)) / len(y) * np.pi * 2)\n            db_translated = 10 ** (db * sine / 20)\n        augmented = y * db_translated\n        return augmented","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Utils"},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_duration(audio_name, root=CFG[\"test_root\"]):\n    return lb.get_duration(filename=root.joinpath(audio_name).with_suffix(\".flac\"))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# DataSet"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_tab = traint.copy()\ntrain_tab[\"duration\"] = [get_duration(i, CFG[\"train_root\"]) for i in train_tab[\"recording_id\"]]\ntrain_tab.head(-10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MelSpecComputer:\n    def __init__(self, sr, fft, hop, fmin, fmax):\n        self.sr = sr\n        self.fft = fft\n        self.hop = hop\n        self.fmin = fmin\n        self.fmax = fmax\n\n    def __call__(self, y):\n        melspec = librosa.feature.melspectrogram(y, n_fft=self.fft, hop_length=self.hop, \n                                                  sr=self.sr, fmin=self.fmin, \n                                                  fmax=self.fmax, power=1.5) \n        melspec = lb.power_to_db(melspec).astype(np.float32)\n        return melspec","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def mono_to_color(X, eps=1e-6, mean=None, std=None):\n    X = np.stack([X, X, X], axis=-1)\n\n    # Standardize\n    mean = mean or X.mean()\n    std = std or X.std()\n    X = (X - mean) / (std + eps)\n\n    # Normalize to [0, 255]\n    _min, _max = X.min(), X.max()\n\n    if (_max - _min) > eps:\n        V = np.clip(X, _min, _max)\n        V = 255 * (V - _min) / (_max - _min)\n        V = V.astype(np.uint8)\n    else:\n        V = np.zeros_like(X, dtype=np.uint8)\n\n    return V\n\n\ndef normalize(image, mean=None, std=None):\n    image = image / 255.0\n    if mean is not None and std is not None:\n        image = (image - mean) / std\n    return np.moveaxis(image, 2, 0).astype(np.float32)\n\ndef crop(y, length, t_min, t_max, sr):\n    center = random.randint(0,len(y))\n    start = 0 if (center-length/2) < 0 else center-length/2\n    end = len(y) if (start+length) > len(y) else start+length\n    y = y[int(start) : int(end)] \n    y = y.astype(np.float32, copy=False)\n    if start < t_min*sr and end > t_max*sr:\n        is_label = True\n    elif start > t_min*sr and start < t_max*sr:\n        is_label = True\n    elif end > t_min*sr and end < t_max*sr:\n        is_label = True\n    else:\n        is_label = None\n        \n    return y, is_label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_wave_transforms():\n    return wave_Compose([\n#                     wave_OneOf([\n#                         GaussianNoiseSNR(min_snr=10),\n#                         PinkNoiseSNR(min_snr=10)\n#                         ]),\n                    PitchShift(max_steps=2, sr=CFG[\"sr\"]),\n#                     TimeStretch(),\n#                     TimeShift(sr=CFG[\"sr\"]),\n#                     VolumeControl(mode=\"sine\")\n                    ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import albumentations\nfrom albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, \n    ShiftScaleRotate, CenterCrop, Resize\n)\n\nfrom albumentations.pytorch import ToTensorV2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def get_img_transforms():\n#     return Compose([\n# #             Resize(CFG['img_size'], CFG['img_size']),\n#             Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n#             ToTensorV2(p=1.0),\n#         ], p=1.)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class RFCXDataset(Dataset):\n    def __init__(self, df, data_root, sr, fft, hop, duration, num_classes, fmin=20, \n                 wave_transforms=None, img_transforms=None, output_label=True, fmax=None):\n        \n        super().__init__()\n        self.df = df.reset_index(drop=True).copy()\n        self.wave_transforms = wave_transforms\n        self.img_transforms = img_transforms\n        self.data_root = data_root\n        self.output_label = output_label        \n        self.sr = sr\n        self.fft = fft\n        self.hop = hop\n        self.fmin = fmin\n        self.fmax = fmax \n        self.num_classes = num_classes\n        self.duration = duration\n        self.audio_length = self.duration*self.sr\n        self.mel_spec_computer = MelSpecComputer(sr=self.sr, fft=self.fft, hop=self.hop, fmin=self.fmin, fmax=self.fmax)\n\n    \n    def __len__(self):\n        return self.df.shape[0]\n\n    def read_index(self, index):\n        d = self.df.iloc[index]\n        record = d[\"recording_id\"]\n        y, y_sr = lb.load(self.data_root.joinpath(record).with_suffix(\".flac\").as_posix(),sr=None)\n        if self.wave_transforms is not None:\n            y = self.wave_transforms(y)\n        y, is_label = crop(y, self.audio_length, d[\"t_min\"], d[\"t_max\"], y_sr) \n        return y, is_label\n    \n    def __getitem__(self, index: int):\n                  \n        y, is_label = self.read_index(index)      \n        # get labels\n        if self.output_label:\n            target = np.zeros(CFG[\"num_classes\"])\n            if is_label:\n                label = self.df.iloc[index]['species_id']\n                target[label] = 1\n            else:\n                target[24] = 1\n        melspec = self.mel_spec_computer(y) \n        img = resize(melspec, (224, 400))\n        img = mono_to_color(img)\n        img = normalize(img)\n        \n        # do label smoothing\n        if self.output_label:\n            return img, target\n        else:\n            return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ds = RFCXDataset(train_tab, data_root=CFG[\"train_root\"], sr=CFG[\"sr\"], fft=CFG[\"fft\"], hop=CFG[\"hop\"], \n                 duration=CFG[\"duration\"], num_classes=CFG[\"num_classes\"], \n                 wave_transforms=get_wave_transforms(), img_transforms=None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x, y = ds.__getitem__(0)\nx.shape, y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x = np.moveaxis(x, 0, 1)\nplt.imshow(np.moveaxis(x, 1, 2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ImgClassifier(nn.Module):\n    def __init__(self, model_arch, n_classes, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_arch, pretrained=pretrained)\n        self.classifier = nn.Sequential(\n            nn.ReLU(), nn.Dropout(p=0.2),\n            nn.Linear(1000, 1000), nn.ReLU(), nn.Dropout(p=0.2),\n            nn.Linear(1000, 500), nn.ReLU(), nn.Dropout(p=0.2),\n            nn.Linear(500, n_classes))\n        \n    def forward(self, x):\n        x = self.model(x)\n        x = self.classifier(x)\n#         x = torch.sigmoid(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model = ImgClassifier(CFG['model_arch'], train_tab.species_id.nunique(), pretrained=True)\n# print(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def prepare_dataloader(df, trn_idx, val_idx):\n    \n    from catalyst.data.sampler import BalanceClassSampler\n    train_ = df.iloc[trn_idx].reset_index(drop=True)\n    valid_ = df.iloc[val_idx].reset_index(drop=True)\n    train_ds = RFCXDataset(train_, data_root=CFG[\"train_root\"], sr=CFG[\"sr\"], fft=CFG[\"fft\"], \n                           hop=CFG[\"hop\"], duration=CFG[\"duration\"], num_classes=CFG[\"num_classes\"], \n                           wave_transforms=None, img_transforms=None, output_label=True)\n#     PinkNoiseSNR(max_steps=2, sr=CFG[\"sr\"])\n    valid_ds = RFCXDataset(valid_, data_root=CFG[\"train_root\"], sr=CFG[\"sr\"], fft=CFG[\"fft\"], \n                           hop=CFG[\"hop\"], duration=CFG[\"duration\"], num_classes=CFG[\"num_classes\"], \n                           wave_transforms=None, img_transforms=None, output_label=True)\n    \n    train_loader = torch.utils.data.DataLoader(\n        train_ds,\n        batch_size=CFG['train_bs'],\n        pin_memory=False,\n        drop_last=False,\n        shuffle=True,        \n        num_workers=CFG['num_workers'],\n        #sampler=BalanceClassSampler(labels=train_['label'].values, mode=\"downsampling\")\n    )\n    val_loader = torch.utils.data.DataLoader(\n        valid_ds, \n        batch_size=CFG['valid_bs'],\n        num_workers=CFG['num_workers'],\n        shuffle=False,\n        pin_memory=False,\n    )\n    return train_loader, val_loader\n\ndef train_one_epoch(epoch, model, loss_fn, optimizer, train_loader, device, scheduler=None, schd_batch_update=False):\n    model.train()\n\n    t = time.time()\n    running_loss = None\n\n    pbar = tqdm(enumerate(train_loader), total=len(train_loader))\n    for step, (imgs, image_labels) in pbar:\n        imgs = imgs.to(device)\n        image_labels = image_labels.to(device).float()\n\n        with autocast():\n            image_preds = model(imgs).float()   \n            loss = loss_fn(image_preds, image_labels)\n            \n            scaler.scale(loss).backward()\n\n            if running_loss is None:\n                running_loss = loss.item()\n            else:\n                running_loss = running_loss * .99 + loss.item() * .01\n\n            if ((step + 1) %  CFG['accum_iter'] == 0) or ((step + 1) == len(train_loader)):\n\n                scaler.step(optimizer)\n                scaler.update()\n                optimizer.zero_grad() \n                \n                if scheduler is not None and schd_batch_update:\n                    scheduler.step()\n\n            if ((step + 1) % CFG['verbose_step'] == 0) or ((step + 1) == len(train_loader)):\n                description = f'epoch {epoch} loss: {running_loss:.4f}'\n                pbar.set_description(description)\n                \n    if scheduler is not None and not schd_batch_update:\n        scheduler.step()\n        \ndef valid_one_epoch(epoch, model, loss_fn, val_loader, device, scheduler=None, schd_loss_update=False):\n    model.eval()\n\n    t = time.time()\n    loss_sum = 0\n    sample_num = 0\n    image_preds_all = []\n    image_targets_all = []\n    \n    pbar = tqdm(enumerate(val_loader), total=len(val_loader))\n    for step, (imgs, image_labels) in pbar:\n        imgs = imgs.to(device).float()\n        image_labels = image_labels.to(device).float()\n        \n        image_preds = model(imgs).float()   \n        image_preds_all += [torch.argmax(image_preds, 1).detach().cpu().numpy()]\n        image_targets_all += [torch.argmax(image_labels, 1).cpu().numpy()]\n        \n        loss = loss_fn(image_preds, image_labels)\n        \n        loss_sum += loss.item()*image_labels.shape[0]\n        sample_num += image_labels.shape[0]  \n\n        if ((step + 1) % CFG['verbose_step'] == 0) or ((step + 1) == len(val_loader)):\n            description = f'epoch {epoch} loss: {loss_sum/sample_num:.4f}'\n            pbar.set_description(description)\n    \n    image_preds_all = np.concatenate(image_preds_all)\n    image_targets_all = np.concatenate(image_targets_all)\n    print('validation multi-label accuracy = {:.4f}'.format((image_preds_all==image_targets_all).mean()))\n    \n    if scheduler is not None:\n        if schd_loss_update:\n            scheduler.step(loss_sum/sample_num)\n        else:\n            scheduler.step()\n    \n    return loss_sum/sample_num","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if __name__ == '__main__':\n\n    seed_everything(CFG['seed'])\n    \n    folds = StratifiedKFold(n_splits=CFG['fold_num'], shuffle=True, random_state=CFG['seed']).split(np.arange(train_tab.shape[0]), train_tab.species_id.values)\n    \n    for fold, (trn_idx, val_idx) in enumerate(folds):\n#         # we'll train fold 0 first\n#         if fold > 0:\n#             break \n\n        print('Training with {} started'.format(fold))\n\n        print(len(trn_idx), len(val_idx))\n        train_loader, val_loader = prepare_dataloader(train_tab, trn_idx, val_idx)\n\n        device = torch.device(CFG['device'])\n        \n        model = ImgClassifier(CFG['model_arch'], CFG[\"num_classes\"], pretrained=True).to(device)\n        scaler = GradScaler()   \n        optimizer = torch.optim.Adam(model.parameters(), lr=CFG['lr'], weight_decay=CFG['weight_decay'])\n        #scheduler = torch.optim.lr_scheduler.StepLR(optimizer, gamma=0.1, step_size=CFG['epochs']-1)\n        scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=CFG['T_0'], T_mult=1, eta_min=CFG['min_lr'], last_epoch=-1)\n        loss_fn = nn.BCEWithLogitsLoss().to(device)\n        best_loss = 1e6\n        \n        for epoch in range(CFG['epochs']):\n            train_one_epoch(epoch, model, loss_fn, optimizer, train_loader, device, scheduler=scheduler, schd_batch_update=False)\n\n            with torch.no_grad():\n                loss = valid_one_epoch(epoch, model, loss_fn, val_loader, device, scheduler=None, schd_loss_update=False)\n                os.chdir('/kaggle/working')\n                torch.save(model.state_dict(),'{}_fold_{}_{}'.format(CFG['model_arch'], fold, epoch))\n                os.chdir('../input')\n                \n            if loss <= best_loss:\n                best_loss = loss\n                print('{}_fold_{}_{}'.format(CFG['model_arch'], fold, epoch))\n                \n            \n        del model, optimizer, train_loader, val_loader, scaler, scheduler\n        torch.cuda.empty_cache()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}