{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## About this Code\n\n+ Multi-label classified images with CNN model\n+ Try using TorchLibrosa's SpecAugmentation function\n\nThere are some issues\n+ image size\n  + need resize?\n+ need mono_to_color?","metadata":{}},{"cell_type":"markdown","source":"### References\n\n+ [Training a winning model](https://www.kaggle.com/theoviel/training-a-winning-model/notebook?scriptVersionId=42814701)\n+ [[PyTorch, Training] BirdCLEF2021 Starter](https://www.kaggle.com/hidehisaarai1213/pytorch-training-birdclef2021-starter)\n+ [Cassava / resnext50_32x4d starter [training]](https://www.kaggle.com/yasufuminakama/cassava-resnext50-32x4d-starter-training)\n\nThank you for publishing a great notebook :)","metadata":{}},{"cell_type":"markdown","source":"## Libraries","metadata":{}},{"cell_type":"code","source":"!pip install ../input/torchlibrosa/torchlibrosa-0.0.5-py3-none-any.whl","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append('../input/pytorch-image-models/pytorch-image-models-master')\n\nimport os\nimport math\nimport time\nimport random\n\nimport numpy as np\nimport pandas as pd\nfrom matplotlib import pyplot as plt\nimport seaborn as sns\nimport soundfile as sf\nfrom pathlib import Path\nfrom IPython.display import Audio, IFrame, display \n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import f1_score\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import Adam, SGD\nimport torchvision.models as models\nfrom torch.nn.parameter import Parameter\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, CosineAnnealingLR, ReduceLROnPlateau\n\nfrom torchlibrosa.stft import LogmelFilterBank, Spectrogram\nfrom torchlibrosa.augmentation import SpecAugmentation\n\nimport timm\nimport warnings \n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"OUTPUT_DIR = './'\nif not os.path.exists(OUTPUT_DIR):\n    os.makedirs(OUTPUT_DIR)\n\ntrain = pd.read_csv('../input/birdclef-2021/train_metadata.csv')\ntest = pd.read_csv('../input/birdclef-2021/test.csv')\ntrain = train[['primary_label', 'filename']]\ntrain","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## CFG","metadata":{}},{"cell_type":"code","source":"TARGETS = [\n        'acafly', 'acowoo', 'aldfly', 'ameavo', 'amecro',\n        'amegfi', 'amekes', 'amepip', 'amered', 'amerob',\n        'amewig', 'amtspa', 'andsol1', 'annhum', 'astfly',\n        'azaspi1', 'babwar', 'baleag', 'balori', 'banana',\n        'banswa', 'banwre1', 'barant1', 'barswa', 'batpig1',\n        'bawswa1', 'bawwar', 'baywre1', 'bbwduc', 'bcnher',\n        'belkin1', 'belvir', 'bewwre', 'bkbmag1', 'bkbplo',\n        'bkbwar', 'bkcchi', 'bkhgro', 'bkmtou1', 'bknsti', 'blbgra1',\n        'blbthr1', 'blcjay1', 'blctan1', 'blhpar1', 'blkpho',\n        'blsspa1', 'blugrb1', 'blujay', 'bncfly', 'bnhcow', 'bobfly1',\n        'bongul', 'botgra', 'brbmot1', 'brbsol1', 'brcvir1', 'brebla',\n        'brncre', 'brnjay', 'brnthr', 'brratt1', 'brwhaw', 'brwpar1',\n        'btbwar', 'btnwar', 'btywar', 'bucmot2', 'buggna', 'bugtan',\n        'buhvir', 'bulori', 'burwar1', 'bushti', 'butsal1', 'buwtea',\n        'cacgoo1', 'cacwre', 'calqua', 'caltow', 'cangoo', 'canwar',\n        'carchi', 'carwre', 'casfin', 'caskin', 'caster1', 'casvir',\n        'categr', 'ccbfin', 'cedwax', 'chbant1', 'chbchi', 'chbwre1',\n        'chcant2', 'chispa', 'chswar', 'cinfly2', 'clanut', 'clcrob',\n        'cliswa', 'cobtan1', 'cocwoo1', 'cogdov', 'colcha1', 'coltro1',\n        'comgol', 'comgra', 'comloo', 'commer', 'compau', 'compot1',\n        'comrav', 'comyel', 'coohaw', 'cotfly1', 'cowscj1', 'cregua1',\n        'creoro1', 'crfpar', 'cubthr', 'daejun', 'dowwoo', 'ducfly', 'dusfly',\n        'easblu', 'easkin', 'easmea', 'easpho', 'eastow', 'eawpew', 'eletro',\n        'eucdov', 'eursta', 'fepowl', 'fiespa', 'flrtan1', 'foxspa', 'gadwal',\n        'gamqua', 'gartro1', 'gbbgul', 'gbwwre1', 'gcrwar', 'gilwoo',\n        'gnttow', 'gnwtea', 'gocfly1', 'gockin', 'gocspa', 'goftyr1',\n        'gohque1', 'goowoo1', 'grasal1', 'grbani', 'grbher3', 'grcfly',\n        'greegr', 'grekis', 'grepew', 'grethr1', 'gretin1', 'greyel',\n        'grhcha1', 'grhowl', 'grnher', 'grnjay', 'grtgra', 'grycat',\n        'gryhaw2', 'gwfgoo', 'haiwoo', 'heptan', 'hergul', 'herthr',\n        'herwar', 'higmot1', 'hofwoo1', 'houfin', 'houspa', 'houwre',\n        'hutvir', 'incdov', 'indbun', 'kebtou1', 'killde', 'labwoo', 'larspa',\n        'laufal1', 'laugul', 'lazbun', 'leafly', 'leasan', 'lesgol', 'lesgre1',\n        'lesvio1', 'linspa', 'linwoo1', 'littin1', 'lobdow', 'lobgna5', 'logshr',\n        'lotduc', 'lotman1', 'lucwar', 'macwar', 'magwar', 'mallar3', 'marwre',\n        'mastro1', 'meapar', 'melbla1', 'monoro1', 'mouchi', 'moudov', 'mouela1',\n        'mouqua', 'mouwar', 'mutswa', 'naswar', 'norcar', 'norfli', 'normoc', 'norpar',\n        'norsho', 'norwat', 'nrwswa', 'nutwoo', 'oaktit', 'obnthr1', 'ocbfly1',\n        'oliwoo1', 'olsfly', 'orbeup1', 'orbspa1', 'orcpar', 'orcwar', 'orfpar',\n        'osprey', 'ovenbi1', 'pabspi1', 'paltan1', 'palwar', 'pasfly', 'pavpig2',\n        'phivir', 'pibgre', 'pilwoo', 'pinsis', 'pirfly1', 'plawre1', 'plaxen1',\n        'plsvir', 'plupig2', 'prowar', 'purfin', 'purgal2', 'putfru1', 'pygnut',\n        'rawwre1', 'rcatan1', 'rebnut', 'rebsap', 'rebwoo', 'redcro', 'reevir1',\n        'rehbar1', 'relpar', 'reshaw', 'rethaw', 'rewbla', 'ribgul', 'rinkin1',\n        'roahaw', 'robgro', 'rocpig', 'rotbec', 'royter1', 'rthhum', 'rtlhum',\n        'ruboro1', 'rubpep1', 'rubrob', 'rubwre1', 'ruckin', 'rucspa1', 'rucwar',\n        'rucwar1', 'rudpig', 'rudtur', 'rufhum', 'rugdov', 'rumfly1', 'runwre1',\n        'rutjac1', 'saffin', 'sancra', 'sander', 'savspa', 'saypho', 'scamac1',\n        'scatan', 'scbwre1', 'scptyr1', 'scrtan1', 'semplo', 'shicow', 'sibtan2',\n        'sinwre1', 'sltred', 'smbani', 'snogoo', 'sobtyr1', 'socfly1', 'solsan',\n        'sonspa', 'soulap1', 'sposan', 'spotow', 'spvear1', 'squcuc1', 'stbori',\n        'stejay', 'sthant1', 'sthwoo1', 'strcuc1', 'strfly1', 'strsal1', 'stvhum2',\n        'subfly', 'sumtan', 'swaspa', 'swathr', 'tenwar', 'thbeup1', 'thbkin',\n        'thswar1', 'towsol', 'treswa', 'trogna1', 'trokin', 'tromoc', 'tropar',\n        'tropew1', 'tuftit', 'tunswa', 'veery', 'verdin', 'vigswa', 'warvir',\n        'wbwwre1', 'webwoo1', 'wegspa1', 'wesant1', 'wesblu', 'weskin', 'wesmea',\n        'westan', 'wewpew', 'whbman1', 'whbnut', 'whcpar', 'whcsee1', 'whcspa',\n        'whevir', 'whfpar1', 'whimbr', 'whiwre1', 'whtdov', 'whtspa', 'whwbec1',\n        'whwdov', 'wilfly', 'willet1', 'wilsni1', 'wiltur', 'wlswar', 'wooduc',\n        'woothr', 'wrenti', 'y00475', 'yebcha', 'yebela1', 'yebfly', 'yebori1',\n        'yebsap', 'yebsee1', 'yefgra1', 'yegvir', 'yehbla', 'yehcar1', 'yelgro',\n        'yelwar', 'yeofly1', 'yerwar', 'yeteup1', 'yetvir']","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    seed = 29\n    n_fold = 5\n    trn_fold = [0]\n    target_col = 'primary_label'\n    train_datadir = Path(\"../input/birdclef-2021/train_short_audio\")\n    period = 5\n    img_size = 224\n    criterion ='BCEWithLogitsLoss'\n    model_name = 'tf_efficientnet_b3'\n    target_size = len(TARGETS)\n    # Audio cfg\n    n_mels = 128\n    fmin = 20\n    fmax = 16000\n    n_fft = 2048\n    hop_length = 512\n    sample_rate = 32000\n    epochs = 10\n    # scheduler/optimizer\n    scheduler = 'CosineAnnealingWarmRestarts' \n    T_0=10 \n    lr=1e-4\n    min_lr=1e-6\n    weight_decay=1e-6  \n    # train\n    gradient_accumulation_steps=1\n    apex = False\n    max_grad_norm = 1000\n    print_freq = 100\n    # model\n    pretrained = True\n    in_channels = 1\n    # Split\n    split = \"StratifiedKFold\"\n    split_params = {\n        \"n_splits\": 5,\n        \"shuffle\": True,\n        \"random_state\": 29\n    }\n    # DataLoader\n    loader = {\n        \"train\": {\n            \"batch_size\": 64,\n            \"num_workers\": 4,\n            \"shuffle\": True,\n            \"pin_memory\": True,\n            \"drop_last\": True\n        },\n        \"valid\": {\n            \"batch_size\": 64,\n            \"num_workers\": 4,\n            \"shuffle\": False,\n            \"pin_memory\": True,\n            \"drop_last\": False\n        }\n    }\n    debug = True\n    \nif CFG.debug:\n    CFG.epochs = 5\n    train = train.sample(n=1000, random_state=CFG.seed).reset_index(drop=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Utils","metadata":{}},{"cell_type":"code","source":"def set_seed(seed=42):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\ndef init_logger(log_file='train.log'):\n    from logging import getLogger, INFO, FileHandler,  Formatter,  StreamHandler\n    logger = getLogger(__name__)\n    logger.setLevel(INFO)\n    handler1 = StreamHandler()\n    handler1.setFormatter(Formatter(\"%(message)s\"))\n    handler2 = FileHandler(filename=log_file)\n    handler2.setFormatter(Formatter(\"%(message)s\"))\n    logger.addHandler(handler1)\n    logger.addHandler(handler2)\n    return logger\n\nLOGGER = init_logger()\nset_seed(seed=CFG.seed)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## CV split","metadata":{}},{"cell_type":"code","source":"folds = train.copy()\nFold = StratifiedKFold(**CFG.split_params)\nfor n, (tr_idx, val_idx) in enumerate(Fold.split(folds, folds[CFG.target_col])):\n    folds.loc[val_idx, 'fold'] = int(n)\nfolds['fold'] = folds['fold'].astype(int)\n# # check the propotion\nfold_proportion = pd.pivot_table(folds, index=CFG.target_col, columns=\"fold\", aggfunc=len)\nprint(fold_proportion.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fold_proportion","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset","metadata":{}},{"cell_type":"code","source":"class WaveformDataset(Dataset):\n    def __init__(self,\n                 df: pd.DataFrame,\n                 datadir: Path,\n                 img_size=224,\n                 waveform_transforms=None,\n                 period=20,\n                 validation=False):\n        self.df = df\n        self.datadir = datadir\n        self.img_size = img_size\n        self.waveform_transforms = waveform_transforms\n        self.period = period\n        self.validation = validation\n        self.y = np.array([TARGETS.index(c) for c in df[CFG.target_col]])\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx: int):\n        sample = self.df.loc[idx, :]\n        wav_name = sample['filename']\n        ebird_code = sample[CFG.target_col]\n        \n        y, sr = sf.read(self.datadir / ebird_code / wav_name)\n        \n        len_y = len(y)\n        effective_length = sr * self.period\n        if len_y < effective_length:\n            new_y = np.zeros(effective_length, dtype=y.dtype)\n            if not self.validation:\n                start = np.random.randint(effective_length - len_y)\n            else:\n                start = 0\n            new_y[start:start + len_y] = y\n            y = new_y.astype(np.float32)\n        elif len_y > effective_length:\n            if not self.validation:\n                start = np.random.randint(len_y - effective_length)\n            else:\n                start = 0\n            y = y[start:start + effective_length].astype(np.float32)\n        else:\n            y = y.astype(np.float32)\n\n        y = np.nan_to_num(y)\n\n        if self.waveform_transforms:\n            y = self.waveform_transforms(y)\n\n        y = np.nan_to_num(y)\n        \n        labels = np.zeros(len(TARGETS), dtype=float)\n        labels[TARGETS.index(ebird_code)] = 1.0\n        \n        return{\n            'waveforms': y,\n            'targets': labels\n        }\n        ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = WaveformDataset(train,\n                                CFG.train_datadir,\n                                img_size=CFG.img_size,\n                                waveform_transforms=None,\n                                period=CFG.period,\n                                validation=True)\n\ndata = train_dataset[0]\nprint(data['waveforms'].shape, data['targets'].shape)\nplt.plot(data['waveforms'])\nplt.show()\nAudio(data=data['waveforms'], rate=32000)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## WaveformTransforms","metadata":{}},{"cell_type":"code","source":"# feature works...","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Criterion","metadata":{}},{"cell_type":"code","source":"def get_criterion():\n    if CFG.criterion=='BCEWithLogitsLoss':\n        criterion = nn.BCEWithLogitsLoss(reduction=\"mean\").to(device)\n    else:\n        raise NotImplementedError\n    return criterion","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Scheduler","metadata":{}},{"cell_type":"code","source":"def get_scheduler(optimizer):\n    if CFG.scheduler=='ReduceLROnPlateau':\n        scheduler = ReduceLROnPlateau(optimizer, mode='min', factor=CFG.factor, patience=CFG.patience, verbose=True, eps=CFG.eps)\n    elif CFG.scheduler=='CosineAnnealingLR':\n        scheduler = CosineAnnealingLR(optimizer, T_max=CFG.T_max, eta_min=CFG.min_lr, last_epoch=-1)\n    elif CFG.scheduler=='CosineAnnealingWarmRestarts':\n        scheduler = CosineAnnealingWarmRestarts(optimizer, T_0=CFG.T_0, T_mult=1, eta_min=CFG.min_lr, last_epoch=-1)\n    return scheduler","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check scheduler\nmodel = nn.Sequential(nn.Linear(2, 2), nn.Linear(2, 2))\noptimizer = Adam(model.parameters(), lr=CFG.lr, weight_decay=CFG.weight_decay, amsgrad=False)\nscheduler = get_scheduler(optimizer)\n\nfrom pylab import rcParams\nlrs = []\nfor epoch in range(1, CFG.epochs+1):\n    scheduler.step(epoch-1)\n    lrs.append(optimizer.param_groups[0][\"lr\"])\nrcParams['figure.figsize'] = 20,3\nplt.plot(lrs)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model","metadata":{}},{"cell_type":"code","source":"class CustomEfficientNet(nn.Module):\n    def __init__(self, model_name=CFG.model_name, pretrained=False, in_channels=1):\n        super().__init__()\n        # Spectrogram extractor\n        self.spectrogram_extractor = Spectrogram(n_fft=CFG.n_fft, hop_length=CFG.hop_length,\n                                                 win_length=CFG.n_fft, window=\"hann\", center=True, pad_mode=\"reflect\",\n                                                 freeze_parameters=True)\n\n        # Logmel feature extractor\n        self.logmel_extractor = LogmelFilterBank(sr=CFG.sample_rate, n_fft=CFG.n_fft,\n                                                 n_mels=CFG.n_mels, fmin=CFG.fmin, fmax=CFG.fmax, ref=1.0, amin=1e-10, top_db=None,\n                                                 freeze_parameters=True)\n\n        # Spec augmenter\n        self.spec_augmenter = SpecAugmentation(time_drop_width=64, time_stripes_num=2,\n                                               freq_drop_width=8, freq_stripes_num=2)\n        self.bn0 = nn.BatchNorm2d(CFG.n_mels)\n\n        self.model = timm.create_model(CFG.model_name, pretrained=pretrained,in_chans=in_channels)\n        n_features = self.model.classifier.in_features\n        self.model.classifier = nn.Linear(n_features, CFG.target_size)\n\n    def forward(self, input):\n        \"\"\"\n        Input: (batch_size, data_length)\n        \"\"\"\n        x = self.spectrogram_extractor(input)# (batch_size, 1(channel), time_steps, freq_bins)\n        x = self.logmel_extractor(x)# (batch_size, 1(channel), time_steps, mel_bins)\n        x = x.transpose(1, 3)\n        x = self.bn0(x)\n        x = x.transpose(1, 3)\n        if self.training:\n            x = self.spec_augmenter(x)\n        \n        x = x.transpose(2, 3)\n        # (batch_size, channels, freq, frames)     \n        x = self.model(x)\n        return x","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = CustomEfficientNet(model_name=CFG.model_name, pretrained=False, in_channels=1)\ntrain_dataset = WaveformDataset(train,\n                                CFG.train_datadir,\n                                img_size=CFG.img_size,\n                                waveform_transforms=None,\n                                period=CFG.period,\n                                validation=True)\ntrain_loader = DataLoader(train_dataset, batch_size=4, shuffle=True,\n                          num_workers=4, pin_memory=True, drop_last=True)\nfor data in train_loader:\n    print(data['waveforms'].shape)\n    output = model(data['waveforms'])\n    target = data['targets']\n    break\n    \ncriterion = get_criterion()\nloss = criterion(output, target).item()\nloss","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Helper functions","metadata":{}},{"cell_type":"code","source":"class AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n\n\ndef asMinutes(s):\n    m = math.floor(s / 60)\n    s -= m * 60\n    return '%dm %ds' % (m, s)\n\n\ndef timeSince(since, percent):\n    now = time.time()\n    s = now - since\n    es = s / (percent)\n    rs = es - s\n    return '%s (remain %s)' % (asMinutes(s), asMinutes(rs))\n\n# https://www.kaggle.com/theoviel/training-a-winning-model/notebook?scriptVersionId=42814701\nONE_HOT = np.eye(CFG.target_size)\ndef f1(truth, pred, threshold=0.5, avg=\"samples\"):\n\n    if len(truth.shape) == 1:\n        truth = ONE_HOT[truth]\n    pred = (pred > threshold).astype(int)\n    return f1_score(truth, pred, average=avg)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_one_epoch(train_loader, model, criterion, optimizer, epoch, scheduler, device):\n    batch_time = AverageMeter()\n    data_time = AverageMeter()\n    losses = AverageMeter()\n    scores = AverageMeter()\n    # switch to train mode\n    model.train()\n    start = end = time.time()\n    global_step = 0\n    \n    for step, data in enumerate(train_loader):\n        waveforms = data['waveforms']\n        labels = data['targets']\n        # measure data loading time\n        data_time.update(time.time() - end)\n        \n        waveforms = waveforms.to(device)\n        labels = labels.to(device)\n        batch_size = labels.size(0)\n        y_preds = model(waveforms)\n        loss = criterion(y_preds, labels)\n        # record loss\n        losses.update(loss.item(), batch_size)\n        if CFG.gradient_accumulation_steps > 1:\n            loss = loss / CFG.gradient_accumulation_steps\n        if CFG.apex:\n            with amp.scale_loss(loss, optimizer) as scaled_loss:\n                scaled_loss.backward()\n        else:\n            loss.backward()\n        grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), CFG.max_grad_norm)\n        if (step + 1) % CFG.gradient_accumulation_steps == 0:\n            optimizer.step()\n            optimizer.zero_grad()\n            global_step += 1\n        # measure elapsed time\n        batch_time.update(time.time() - end)\n        end = time.time()\n        if step % CFG.print_freq == 0 or step == (len(train_loader)-1):\n            print('Epoch: [{0}][{1}/{2}] '\n                  'Data {data_time.val:.3f} ({data_time.avg:.3f}) '\n                  'Elapsed {remain:s} '\n                  'Loss: {loss.val:.4f}({loss.avg:.4f}) '\n                  'Grad: {grad_norm:.4f}  '\n                  #'LR: {lr:.6f}  '\n                  .format(\n                   epoch+1, step, len(train_loader), batch_time=batch_time,\n                   data_time=data_time, loss=losses,\n                   remain=timeSince(start, float(step+1)/len(train_loader)),\n                   grad_norm=grad_norm,\n                   #lr=scheduler.get_lr()[0],\n                   ))\n    return losses.avg\n\ndef valid_fn(valid_loader, model, criterion, device):\n    batch_time = AverageMeter()\n    data_time = AverageMeter()\n    losses = AverageMeter()\n    scores = AverageMeter()\n    # switch to evaluation mode\n    model.eval()\n    preds = np.empty((0, CFG.target_size))\n    start = end = time.time()\n    for step, data in enumerate(valid_loader):\n        waveforms = data['waveforms']\n        labels = data['targets']\n        # measure data loading time\n        data_time.update(time.time() - end)\n        waveforms = waveforms.to(device)\n        labels = labels.to(device)\n        batch_size = labels.size(0)\n        # compute loss\n        with torch.no_grad():\n            y_preds = model(waveforms)\n            preds = np.concatenate([preds, torch.sigmoid(y_preds).cpu().numpy()])\n        loss = criterion(y_preds, labels)\n        losses.update(loss.item(), batch_size)\n        if CFG.gradient_accumulation_steps > 1:\n            loss = loss / CFG.gradient_accumulation_steps\n        # measure elapsed time\n        batch_time.update(time.time() - end)\n        end = time.time()\n        if step % CFG.print_freq == 0 or step == (len(valid_loader)-1):\n            print('EVAL: [{0}/{1}] '\n                  'Data {data_time.val:.3f} ({data_time.avg:.3f}) '\n                  'Elapsed {remain:s} '\n                  'Loss: {loss.val:.4f}({loss.avg:.4f}) '\n                  .format(\n                   step, len(valid_loader), batch_time=batch_time,\n                   data_time=data_time, loss=losses,\n                   remain=timeSince(start, float(step+1)/len(valid_loader)),\n                   ))\n    return losses.avg, preds","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train loop","metadata":{}},{"cell_type":"code","source":"def train_loop(folds: pd.DataFrame, fold_num: int = 0):\n    LOGGER.info(f\"========== fold: {fold_num} training ==========\")\n    ### dataset\n    tr_index = folds[folds[\"fold\"] != fold_num].index\n    vl_index = folds[folds[\"fold\"] == fold_num].index\n    \n    train_folds = folds.loc[tr_index].reset_index(drop=True)\n    valid_folds = folds.loc[vl_index].reset_index(drop=True)\n    \n    train_dataset = WaveformDataset(train_folds,\n                                    CFG.train_datadir,\n                                    img_size=CFG.img_size,\n                                    waveform_transforms=None,\n                                    period=CFG.period,\n                                    validation=False)\n    valid_dataset = WaveformDataset(valid_folds,\n                                    CFG.train_datadir,\n                                    img_size=CFG.img_size,\n                                    waveform_transforms=None,\n                                    period=CFG.period,\n                                    validation=True)\n    ### dataloader\n    train_loader = DataLoader(train_dataset, **CFG.loader['train'])\n    valid_loader = DataLoader(valid_dataset, **CFG.loader['valid'])\n    \n    ### model\n    model = CustomEfficientNet(model_name=CFG.model_name, pretrained=CFG.pretrained, in_channels=CFG.in_channels)\n    model.to(device)\n    ### optimizer\n    optimizer = Adam(model.parameters(), lr=CFG.lr, weight_decay=CFG.weight_decay, amsgrad=False)\n    ### get scheduler\n    scheduler = get_scheduler(optimizer)\n    if CFG.apex:\n        model, optimizer = amp.initialize(model, optimizer, opt_level='O1', verbosity=0)\n    ### criterion\n    criterion = get_criterion()\n    \n    # ====================================================\n    # loop\n    # ====================================================\n    best_score = 0.\n    best_loss = np.inf\n    \n    for epoch in range(CFG.epochs):\n        start_time = time.time()\n        # train\n        avg_loss = train_one_epoch(train_loader, model, criterion, optimizer, epoch, scheduler, device)\n        # eval\n        avg_val_loss, preds = valid_fn(valid_loader, model, criterion, device)\n        # scoring\n        print(f'pred max{np.amax(preds)}')\n        micro_f1 = f1(valid_dataset.y, preds, avg=\"micro\")\n        samples_f1 = f1(valid_dataset.y, preds)\n        LOGGER.info(f'micro_f1{micro_f1},samples_f1{samples_f1}')\n        \n        if isinstance(scheduler, ReduceLROnPlateau):\n            scheduler.step(avg_val_loss)\n        elif isinstance(scheduler, CosineAnnealingLR):\n            scheduler.step()\n        elif isinstance(scheduler, CosineAnnealingWarmRestarts):\n            scheduler.step()\n\n        elapsed = time.time() - start_time\n\n        LOGGER.info(f'Epoch {epoch+1} - avg_train_loss: {avg_loss:.4f}  avg_val_loss: {avg_val_loss:.4f}  time: {elapsed:.0f}s')\n\n        if avg_val_loss < best_loss:\n            best_loss = avg_val_loss\n            LOGGER.info(f'Epoch {epoch+1} - Save Best Score: {best_loss:.4f} Model')\n            torch.save({'model': model.state_dict(), \n                        'preds': preds},\n                        OUTPUT_DIR+f'{CFG.model_name}_fold{fold_num}_best.pth')\n    \n    check_point = torch.load(OUTPUT_DIR+f'{CFG.model_name}_fold{fold_num}_best.pth')\n\n    return valid_folds","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def main():\n    for fold in range(CFG.n_fold):\n        if fold in CFG.trn_fold:\n            _oof_df = train_loop(folds, fold)\n\nif __name__ == '__main__':\n    main()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}