{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Training-Birdclef-2021-Pytorch-resnest50","metadata":{}},{"cell_type":"markdown","source":"## About\n\nThis notebook is based on the code published [here](https://www.kaggle.com/ttahara/training-birdsong-baseline-resnest50-fast), which was very well received in the previous \"Cornell Birdcall Identification\" competition, and has been modified so that it can be trained on the \"BirdCLEF 2021 - Birdcall Identification\" data.\nIn addition, I added some modifications that were not included in the original code, such as using the timm library.\nThe modifications were made by referring to the public codes [here](https://www.kaggle.com/theoviel/training-a-winning-model) and [here](https://www.kaggle.com/hidehisaarai1213/pytorch-training-birdclef2021-starter).\n\nThese public codes have helped me a lot.\nI would like to thank @ttahara,@theoviel,@hidehisaarai1213 for making these codes public.\n\nIf there are any shortcomings, I would appreciate it if you could point them out.","metadata":{}},{"cell_type":"markdown","source":"## Prepare","metadata":{}},{"cell_type":"markdown","source":"### import libraries","metadata":{}},{"cell_type":"code","source":"%%bash\npip install ../input/pytorch-pfn-extras/pytorch-pfn-extras-0.4.1/","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-05-30T06:57:15.660432Z","iopub.execute_input":"2021-05-30T06:57:15.660847Z","iopub.status.idle":"2021-05-30T06:57:24.26804Z","shell.execute_reply.started":"2021-05-30T06:57:15.660815Z","shell.execute_reply":"2021-05-30T06:57:24.266652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install timm","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:57:24.270526Z","iopub.execute_input":"2021-05-30T06:57:24.271038Z","iopub.status.idle":"2021-05-30T06:57:31.591948Z","shell.execute_reply.started":"2021-05-30T06:57:24.27099Z","shell.execute_reply":"2021-05-30T06:57:31.590508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install audiomentations","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:57:31.594408Z","iopub.execute_input":"2021-05-30T06:57:31.594767Z","iopub.status.idle":"2021-05-30T06:57:39.899652Z","shell.execute_reply.started":"2021-05-30T06:57:31.594738Z","shell.execute_reply":"2021-05-30T06:57:39.898367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!apt-get -y install sox","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:57:39.904047Z","iopub.execute_input":"2021-05-30T06:57:39.904361Z","iopub.status.idle":"2021-05-30T06:57:42.263674Z","shell.execute_reply.started":"2021-05-30T06:57:39.90433Z","shell.execute_reply":"2021-05-30T06:57:42.262342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport sys\n\nsys.path = [\n    '../input/bird-outputs/src/src/',\n] + sys.path\n","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:57:42.265874Z","iopub.execute_input":"2021-05-30T06:57:42.26636Z","iopub.status.idle":"2021-05-30T06:57:42.272768Z","shell.execute_reply.started":"2021-05-30T06:57:42.266299Z","shell.execute_reply":"2021-05-30T06:57:42.271068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\nimport time\nimport shutil\nimport random\nimport warnings\nimport typing as tp\nfrom pathlib import Path\nfrom contextlib import contextmanager\n\nimport yaml\nfrom joblib import delayed, Parallel\n\nimport cv2\nimport librosa\nimport audioread\nimport soundfile as sf\n\nimport numpy as np\nimport pandas as pd\n\nfrom sklearn.metrics import f1_score\nfrom sklearn.model_selection import StratifiedKFold\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.utils.data as data\n# import resnest.torch as resnest_torch\n\nimport pytorch_pfn_extras as ppe\nfrom pytorch_pfn_extras.training import extensions as ppe_extensions\n\nimport timm\n# from util import f1\nfrom training.mixup import mixup_data\n# from params import NUM_WORKERS, NUM_CLASSES\nfrom training.specaugment import SpecAugmentation\n\npd.options.display.max_rows = 500\npd.options.display.max_columns = 500","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2021-05-30T06:57:42.274829Z","iopub.execute_input":"2021-05-30T06:57:42.275406Z","iopub.status.idle":"2021-05-30T06:57:42.289738Z","shell.execute_reply.started":"2021-05-30T06:57:42.275362Z","shell.execute_reply":"2021-05-30T06:57:42.2885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Path(\"/root/.cache/torch/checkpoints\").mkdir(parents=True, exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:57:42.293382Z","iopub.execute_input":"2021-05-30T06:57:42.2939Z","iopub.status.idle":"2021-05-30T06:57:42.3021Z","shell.execute_reply.started":"2021-05-30T06:57:42.293771Z","shell.execute_reply":"2021-05-30T06:57:42.300818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We will use [this data set](https://www.kaggle.com/theoviel/bird-backgrounds) for BackgroundNoise.\nHowever, the audio file appeared to be corrupted.\nTherefore, convert the wav file to raw file and then convert the raw file to wav file again.","metadata":{}},{"cell_type":"code","source":"wav2raw_dir = '/kaggle/working/wav2raw'\nraw2wav_dir = '/kaggle/working/raw2wav'\n\nPath(wav2raw_dir).mkdir(parents=True, exist_ok=True)\nPath(raw2wav_dir).mkdir(parents=True, exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:57:42.306901Z","iopub.execute_input":"2021-05-30T06:57:42.307212Z","iopub.status.idle":"2021-05-30T06:57:42.315067Z","shell.execute_reply.started":"2021-05-30T06:57:42.307181Z","shell.execute_reply":"2021-05-30T06:57:42.31331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\nimport subprocess\nfrom subprocess import PIPE\n\ntmp_path = Path('../input/bird-backgrounds')\nfor audio in tmp_path.glob('**/*.wav') :\n    raw_file = wav2raw_dir + '/' + audio.name[:-3] + 'raw'\n    wav_file = raw2wav_dir + '/' + audio.name\n    \n    sox2raw = f\"sox {audio} {raw_file}\"\n    proc = subprocess.run(sox2raw, shell=True, stdout=PIPE, stderr=PIPE, text=True)\n#     stdout = proc.stdout\n#     print('STDOUT: {}'.format(stdout))\n\n    raw2sox = f\"sox -t raw -e signed-integer -b 16 -r 32000 {raw_file} -t wav {wav_file}\"\n    proc = subprocess.run(raw2sox, shell=True, stdout=PIPE, stderr=PIPE, text=True)\n#     stdout = proc.stdout\n#     print('STDOUT: {}'.format(stdout))\n","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:57:42.318824Z","iopub.execute_input":"2021-05-30T06:57:42.319408Z","iopub.status.idle":"2021-05-30T06:57:45.534429Z","shell.execute_reply.started":"2021-05-30T06:57:42.319367Z","shell.execute_reply":"2021-05-30T06:57:45.533028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### define utilities","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"}},{"cell_type":"code","source":"def set_seed(seed: int = 42):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)  # type: ignore\n#     torch.backends.cudnn.deterministic = True  # type: ignore\n#     torch.backends.cudnn.benchmark = True  # type: ignore\n    \n\n@contextmanager\ndef timer(name: str) -> None:\n    \"\"\"Timer Util\"\"\"\n    t0 = time.time()\n    print(\"[{}] start\".format(name))\n    yield\n    print(\"[{}] done in {:.0f} s\".format(name, time.time() - t0))","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:57:45.536531Z","iopub.execute_input":"2021-05-30T06:57:45.537052Z","iopub.status.idle":"2021-05-30T06:57:45.545015Z","shell.execute_reply.started":"2021-05-30T06:57:45.537016Z","shell.execute_reply":"2021-05-30T06:57:45.543296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### read data","metadata":{}},{"cell_type":"code","source":"ROOT = Path.cwd().parent\nINPUT_ROOT = ROOT / \"input\"\nRAW_DATA = INPUT_ROOT / \"birdclef-2021\"\nTRAIN_AUDIO_DIR = RAW_DATA / \"train_short_audio\"\nTEST_AUDIO_DIR = RAW_DATA / \"test_soundscapes\"","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:57:45.546705Z","iopub.execute_input":"2021-05-30T06:57:45.547428Z","iopub.status.idle":"2021-05-30T06:57:45.560775Z","shell.execute_reply.started":"2021-05-30T06:57:45.547367Z","shell.execute_reply":"2021-05-30T06:57:45.559496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(RAW_DATA / \"train_metadata.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:57:45.562581Z","iopub.execute_input":"2021-05-30T06:57:45.563385Z","iopub.status.idle":"2021-05-30T06:57:45.815813Z","shell.execute_reply.started":"2021-05-30T06:57:45.563338Z","shell.execute_reply":"2021-05-30T06:57:45.814544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head(3)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-05-30T06:57:45.817514Z","iopub.execute_input":"2021-05-30T06:57:45.81821Z","iopub.status.idle":"2021-05-30T06:57:45.842916Z","shell.execute_reply.started":"2021-05-30T06:57:45.818163Z","shell.execute_reply":"2021-05-30T06:57:45.841223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### settings","metadata":{}},{"cell_type":"code","source":"settings_str = \"\"\"\nglobals:\n  seed: 1213\n  device: cuda\n  num_epochs: 1\n  output_dir: /kaggle/training_output/\n  use_fold: 0\n  target_sr: 32000\n\ndataset:\n  name: SpectrogramDataset\n  params:\n    img_size: 224\n    melspectrogram_parameters:\n      n_mels: 128\n      fmin: 20\n      fmax: 16000\n    \nsplit:\n  name: StratifiedKFold\n  params:\n    n_splits: 5\n    random_state: 42\n    shuffle: True\n\nloader:\n  train:\n    batch_size: 50\n    shuffle: True\n    num_workers: 2\n    pin_memory: True\n    drop_last: True\n  val:\n    batch_size: 100\n    shuffle: False\n    num_workers: 2\n    pin_memory: True\n    drop_last: False\n\nmodel:\n  name: resnest50d_1s4x24d\n  params:\n    pretrained: True\n    n_classes: 397\n\nloss:\n  name: BCEWithLogitsLoss\n  params: {}\n\noptimizer:\n  name: Adam\n  params:\n    lr: 0.001\n\nscheduler:\n  name: CosineAnnealingLR\n  params:\n    T_max: 10\n \naugmentation:\n  params:\n    specaugment_proba: 0.5\n    mixup_proba: 0.5\n    alpha: 5\n\"\"\"","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-05-30T06:57:45.84521Z","iopub.execute_input":"2021-05-30T06:57:45.845701Z","iopub.status.idle":"2021-05-30T06:57:45.855115Z","shell.execute_reply.started":"2021-05-30T06:57:45.845654Z","shell.execute_reply":"2021-05-30T06:57:45.853627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"settings = yaml.safe_load(settings_str)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:57:45.858259Z","iopub.execute_input":"2021-05-30T06:57:45.858675Z","iopub.status.idle":"2021-05-30T06:57:45.878311Z","shell.execute_reply.started":"2021-05-30T06:57:45.858604Z","shell.execute_reply":"2021-05-30T06:57:45.877166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Definition","metadata":{}},{"cell_type":"markdown","source":"### Dataset\n* forked from: https://github.com/koukyo1994/kaggle-birdcall-resnet-baseline-training/blob/master/src/dataset.py\n* modified partialy\n","metadata":{}},{"cell_type":"code","source":"BIRD_CODE = {v: k for k, v in enumerate(train.primary_label.unique())}\nINV_BIRD_CODE = {v: k for k, v in BIRD_CODE.items()}","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:57:45.88026Z","iopub.execute_input":"2021-05-30T06:57:45.8811Z","iopub.status.idle":"2021-05-30T06:57:45.895655Z","shell.execute_reply.started":"2021-05-30T06:57:45.881051Z","shell.execute_reply":"2021-05-30T06:57:45.894474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from audiomentations import *\n\nBACKGROUND_PATH = '/kaggle/working/raw2wav'\n\ndef get_wav_transforms():\n    transforms = Compose(\n        [\n            AddGaussianSNR(max_SNR=0.5, p=0.5),\n            AddBackgroundNoise(\n                sounds_path=BACKGROUND_PATH, min_snr_in_db=0, max_snr_in_db=2, p=0.5\n            ),\n        ]\n    )\n\n    return transforms","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:57:45.9001Z","iopub.execute_input":"2021-05-30T06:57:45.900479Z","iopub.status.idle":"2021-05-30T06:57:45.90991Z","shell.execute_reply.started":"2021-05-30T06:57:45.900405Z","shell.execute_reply":"2021-05-30T06:57:45.90863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PERIOD = 5\n\ndef mono_to_color(\n    X: np.ndarray, mean=None, std=None,\n    norm_max=None, norm_min=None, eps=1e-6\n):\n    # Stack X as [X,X,X]\n    X = np.stack([X, X, X], axis=-1)\n\n    # Standardize\n    mean = mean or X.mean()\n    X = X - mean\n    std = std or X.std()\n    Xstd = X / (std + eps)\n    _min, _max = Xstd.min(), Xstd.max()\n    norm_max = norm_max or _max\n    norm_min = norm_min or _min\n    if (_max - _min) > eps:\n        # Normalize to [0, 255]\n        V = Xstd\n        V[V < norm_min] = norm_min\n        V[V > norm_max] = norm_max\n        V = 255 * (V - norm_min) / (norm_max - norm_min)\n        V = V.astype(np.uint8)\n    else:\n        # Just zero\n        V = np.zeros_like(Xstd, dtype=np.uint8)\n    return V\n\nclass SpectrogramDataset(data.Dataset):\n    def __init__(\n        self,\n        file_list: tp.List[tp.List[str]],img_size=224,train=True,\n        waveform_transforms=None, spectrogram_transforms=None, melspectrogram_parameters={}\n    ):\n        self.file_list = file_list  # list of list: [file_path, ebird_code]\n        self.img_size = img_size\n        self.waveform_transforms =  get_wav_transforms() if train else None\n        self.spectrogram_transforms = spectrogram_transforms\n        self.melspectrogram_parameters = melspectrogram_parameters\n\n    def __len__(self):\n        return len(self.file_list)\n\n    def __getitem__(self, idx: int):\n        wav_path, ebird_code = self.file_list[idx]\n\n        y, sr = sf.read(wav_path)\n\n        if self.waveform_transforms:\n            y = self.waveform_transforms(y, sr)\n        \n        len_y = len(y)\n        effective_length = sr * PERIOD\n        if len_y < effective_length:\n            new_y = np.zeros(effective_length, dtype=y.dtype)\n            if self.train:\n                start = np.random.randint(effective_length - len_y)\n            else:\n                start = 0  \n            new_y[start:start + len_y] = y\n            y = new_y.astype(np.float32)\n        elif len_y > effective_length:\n            start = np.random.randint(len_y - effective_length)\n            y = y[start:start + effective_length].astype(np.float32)\n        else:\n            y = y.astype(np.float32)\n\n        melspec = librosa.feature.melspectrogram(y, sr=sr, **self.melspectrogram_parameters)\n        melspec = librosa.power_to_db(melspec).astype(np.float32)\n\n#         if self.spectrogram_transforms:\n#             melspec = self.spectrogram_transforms(melspec)\n#         else:\n#             pass\n\n        image = mono_to_color(melspec)\n        height, width, _ = image.shape\n        image = cv2.resize(image, (int(width * self.img_size / height), self.img_size))\n        image = np.moveaxis(image, 2, 0)\n        image = (image / 255.0).astype(np.float32)\n        \n        labels = np.zeros(len(BIRD_CODE), dtype=\"f\")\n        labels[BIRD_CODE[ebird_code]] = 1\n\n        return image, labels","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-05-30T06:57:45.911725Z","iopub.execute_input":"2021-05-30T06:57:45.912192Z","iopub.status.idle":"2021-05-30T06:57:45.932831Z","shell.execute_reply.started":"2021-05-30T06:57:45.912147Z","shell.execute_reply":"2021-05-30T06:57:45.931082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Training Utility","metadata":{}},{"cell_type":"code","source":"def get_loaders_for_training(\n    args_dataset: tp.Dict, args_loader: tp.Dict,\n    train_file_list: tp.List[str], val_file_list: tp.List[str]\n):\n    # # make dataset\n    train_dataset = SpectrogramDataset(train_file_list, **args_dataset)\n    val_dataset = SpectrogramDataset(val_file_list, **args_dataset, train=False)\n    # # make dataloader\n    train_loader = data.DataLoader(train_dataset, **args_loader[\"train\"])\n    val_loader = data.DataLoader(val_dataset, **args_loader[\"val\"])\n    \n    return train_loader, val_loader","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-05-30T06:57:45.935049Z","iopub.execute_input":"2021-05-30T06:57:45.935597Z","iopub.status.idle":"2021-05-30T06:57:45.946455Z","shell.execute_reply.started":"2021-05-30T06:57:45.935522Z","shell.execute_reply":"2021-05-30T06:57:45.944858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model(args: tp.Dict):\n    model = timm.create_model(args[\"name\"], pretrained=args[\"params\"][\"pretrained\"])\n    del model.fc\n    model.fc = nn.Sequential(\n        nn.Linear(2048, 1024), nn.ReLU(), nn.Dropout(p=0.2),\n        nn.Linear(1024, 1024), nn.ReLU(), nn.Dropout(p=0.2),\n        nn.Linear(1024, args[\"params\"][\"n_classes\"]))\n    \n    return model","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-05-30T06:57:45.948745Z","iopub.execute_input":"2021-05-30T06:57:45.949858Z","iopub.status.idle":"2021-05-30T06:57:45.958896Z","shell.execute_reply.started":"2021-05-30T06:57:45.949795Z","shell.execute_reply":"2021-05-30T06:57:45.957699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_loop(\n    manager, args, model, device,\n    train_loader, optimizer, scheduler, loss_func\n):\n    spec_augmenter = SpecAugmentation(\n        time_drop_width=16, time_stripes_num=2, freq_drop_width=8, freq_stripes_num=2\n    )\n    \n    \"\"\"Run minibatch training loop\"\"\"\n    while not manager.stop_trigger:\n        model.train()\n        for batch_idx, (data, target) in enumerate(train_loader):\n            with manager.run_iteration():\n                data, target = data.to(device), target.to(device)\n                \n                if np.random.rand() < args[\"augmentation\"][\"params\"][\"specaugment_proba\"]:\n                    data = spec_augmenter(data)\n\n                if np.random.rand() < args[\"augmentation\"][\"params\"][\"mixup_proba\"]:\n                    data, y_a, y_b, _ = mixup_data(data.cuda(), target.cuda(), alpha=args[\"augmentation\"][\"params\"][\"alpha\"])\n                    target = torch.clamp(y_a + y_b, 0, 1)\n\n                optimizer.zero_grad()\n                output = model(data)\n                \n                loss = loss_func(output, target)\n                ppe.reporting.report({'train/loss': loss.item()})\n                loss.backward()\n                optimizer.step()\n                scheduler.step()\n\ndef eval_for_batch(\n    args, model, device,\n    data, target, loss_func, eval_func_dict={}\n):\n    \"\"\"\n    Run evaliation for valid\n    \n    This function is applied to each batch of val loader.\n    \"\"\"\n    model.eval()\n    data, target = data.to(device), target.to(device)\n    output = model(data)\n    # Final result will be average of averages of the same size\n    val_loss = loss_func(output, target).item()\n    ppe.reporting.report({'val/loss': val_loss})\n    \n    for eval_name, eval_func in eval_func_dict.items():\n        eval_value = eval_func(output, target).item()\n        ppe.reporting.report({\"val/{}\".format(eval_aame): eval_value})","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-05-30T06:57:45.960893Z","iopub.execute_input":"2021-05-30T06:57:45.961686Z","iopub.status.idle":"2021-05-30T06:57:45.9783Z","shell.execute_reply.started":"2021-05-30T06:57:45.961642Z","shell.execute_reply":"2021-05-30T06:57:45.977052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_extensions(\n    manager, args, model, device, test_loader, optimizer,\n    loss_func, eval_func_dict={}\n):\n    \"\"\"set extensions for PPE\"\"\"\n        \n    my_extensions = [\n        # # observe, report\n        ppe_extensions.observe_lr(optimizer=optimizer),\n        # ppe_extensions.ParameterStatistics(model, prefix='model'),\n        # ppe_extensions.VariableStatisticsPlot(model),\n        ppe_extensions.LogReport(),\n        ppe_extensions.PlotReport(['train/loss', 'val/loss'], 'epoch', filename='loss.png'),\n        ppe_extensions.PlotReport(['lr',], 'epoch', filename='lr.png'),\n        ppe_extensions.PrintReport([\n            'epoch', 'iteration', 'lr', 'train/loss', 'val/loss', \"elapsed_time\"]),\n#         ppe_extensions.ProgressBar(update_interval=100),\n\n        # # evaluation\n        (\n            ppe_extensions.Evaluator(\n                test_loader, model,\n                eval_func=lambda data, target:\n                    eval_for_batch(args, model, device, data, target, loss_func, eval_func_dict),\n                progress_bar=True),\n            (1, \"epoch\"),\n        ),\n        # # save model snapshot.\n        (\n            ppe_extensions.snapshot(\n                target=model, filename=\"snapshot_epoch_{.updater.epoch}.pth\"),\n            ppe.training.triggers.MinValueTrigger(key=\"val/loss\", trigger=(1, 'epoch'))\n        ),\n    ]\n           \n    # # set extensions to manager\n    for ext in my_extensions:\n        if isinstance(ext, tuple):\n            manager.extend(ext[0], trigger=ext[1])\n        else:\n            manager.extend(ext)\n        \n    return manager","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-05-30T06:57:45.98209Z","iopub.execute_input":"2021-05-30T06:57:45.982666Z","iopub.status.idle":"2021-05-30T06:57:45.995059Z","shell.execute_reply.started":"2021-05-30T06:57:45.982597Z","shell.execute_reply":"2021-05-30T06:57:45.993419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"markdown","source":"### prepare data","metadata":{}},{"cell_type":"markdown","source":"#### get wav file path","metadata":{}},{"cell_type":"code","source":"audio_list = [TRAIN_AUDIO_DIR / row['primary_label'] / row['filename'] for _, row, in train.iterrows()]\ntrain_all = train.assign(file_path=audio_list)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:57:46.001741Z","iopub.execute_input":"2021-05-30T06:57:46.00236Z","iopub.status.idle":"2021-05-30T06:57:55.373295Z","shell.execute_reply.started":"2021-05-30T06:57:46.002315Z","shell.execute_reply":"2021-05-30T06:57:55.372082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_all.head(3)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-05-30T06:57:55.375634Z","iopub.execute_input":"2021-05-30T06:57:55.376101Z","iopub.status.idle":"2021-05-30T06:57:55.400987Z","shell.execute_reply.started":"2021-05-30T06:57:55.376065Z","shell.execute_reply":"2021-05-30T06:57:55.399448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### split data","metadata":{}},{"cell_type":"code","source":"skf = StratifiedKFold(**settings[\"split\"][\"params\"])\n\ntrain_all[\"fold\"] = -1\nfor fold_id, (train_index, val_index) in enumerate(skf.split(train_all, train_all[\"primary_label\"])):\n    train_all.iloc[val_index, -1] = fold_id\n    \n# # check the propotion\nfold_proportion = pd.pivot_table(train_all, index=\"primary_label\", columns=\"fold\", values=\"filename\", aggfunc=len)\nprint(fold_proportion.shape)","metadata":{"_kg_hide-output":false,"execution":{"iopub.status.busy":"2021-05-30T06:57:55.402872Z","iopub.execute_input":"2021-05-30T06:57:55.403573Z","iopub.status.idle":"2021-05-30T06:57:55.552458Z","shell.execute_reply.started":"2021-05-30T06:57:55.403524Z","shell.execute_reply":"2021-05-30T06:57:55.551106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fold_proportion","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-05-30T06:57:55.553969Z","iopub.execute_input":"2021-05-30T06:57:55.55459Z","iopub.status.idle":"2021-05-30T06:57:55.559775Z","shell.execute_reply.started":"2021-05-30T06:57:55.554528Z","shell.execute_reply":"2021-05-30T06:57:55.558368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"use_fold = settings[\"globals\"][\"use_fold\"]\ntrain_file_list = train_all.query(\"fold != @use_fold\")[[\"file_path\", \"primary_label\"]].values.tolist()\nval_file_list = train_all.query(\"fold == @use_fold\")[[\"file_path\", \"primary_label\"]].values.tolist()\n\nprint(\"[fold {}] train: {}, val: {}\".format(use_fold, len(train_file_list), len(val_file_list)))","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:57:55.561797Z","iopub.execute_input":"2021-05-30T06:57:55.562548Z","iopub.status.idle":"2021-05-30T06:57:55.607882Z","shell.execute_reply.started":"2021-05-30T06:57:55.562505Z","shell.execute_reply":"2021-05-30T06:57:55.606539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## run training","metadata":{}},{"cell_type":"code","source":"set_seed(settings[\"globals\"][\"seed\"])\ndevice = torch.device(settings[\"globals\"][\"device\"])\noutput_dir = Path(settings[\"globals\"][\"output_dir\"])\n\n# # # get loader\ntrain_loader, val_loader = get_loaders_for_training(\n    settings[\"dataset\"][\"params\"], settings[\"loader\"], train_file_list, val_file_list)\n\n# # # get model\nmodel = get_model(settings[\"model\"])\nmodel = model.to(device)\n\n# # # get optimizer\noptimizer = getattr(\n    torch.optim, settings[\"optimizer\"][\"name\"]\n)(model.parameters(), **settings[\"optimizer\"][\"params\"])\n\n# # # get scheduler\nscheduler = getattr(\n    torch.optim.lr_scheduler, settings[\"scheduler\"][\"name\"]\n)(optimizer, **settings[\"scheduler\"][\"params\"])\n\n# # # get loss\nloss_func = getattr(nn, settings[\"loss\"][\"name\"])(**settings[\"loss\"][\"params\"])\n\n# # # create training manager\ntrigger = None\n\nmanager = ppe.training.ExtensionsManager(\n    model, optimizer, settings[\"globals\"][\"num_epochs\"],\n    iters_per_epoch=len(train_loader),\n    stop_trigger=trigger,\n    out_dir=output_dir\n)\n\n# # # set manager extensions\nmanager = set_extensions(\n    manager, settings, model, device,\n    val_loader, optimizer, loss_func,\n)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:57:55.609538Z","iopub.execute_input":"2021-05-30T06:57:55.610192Z","iopub.status.idle":"2021-05-30T06:57:56.3877Z","shell.execute_reply.started":"2021-05-30T06:57:55.610147Z","shell.execute_reply":"2021-05-30T06:57:56.386506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # runtraining\ntrain_loop(\n    manager, settings, model, device,\n    train_loader, optimizer, scheduler, loss_func)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T06:57:56.38929Z","iopub.execute_input":"2021-05-30T06:57:56.389702Z","iopub.status.idle":"2021-05-30T09:41:32.508675Z","shell.execute_reply.started":"2021-05-30T06:57:56.389658Z","shell.execute_reply":"2021-05-30T09:41:32.507351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_loader\ndel val_loader\ndel model\ndel optimizer\ndel scheduler\ndel loss_func\ndel manager\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:41:32.514585Z","iopub.execute_input":"2021-05-30T09:41:32.514911Z","iopub.status.idle":"2021-05-30T09:41:32.982721Z","shell.execute_reply.started":"2021-05-30T09:41:32.514877Z","shell.execute_reply":"2021-05-30T09:41:32.981057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## save results","metadata":{}},{"cell_type":"code","source":"%%bash\nls /kaggle/training_output","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:41:32.986557Z","iopub.execute_input":"2021-05-30T09:41:32.987059Z","iopub.status.idle":"2021-05-30T09:41:33.222055Z","shell.execute_reply.started":"2021-05-30T09:41:32.986998Z","shell.execute_reply":"2021-05-30T09:41:33.219365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for f_name in [\"log\",\"loss.png\", \"lr.png\"]:\n    shutil.copy(output_dir / f_name, f_name)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:41:33.230112Z","iopub.execute_input":"2021-05-30T09:41:33.231206Z","iopub.status.idle":"2021-05-30T09:41:33.244324Z","shell.execute_reply.started":"2021-05-30T09:41:33.231115Z","shell.execute_reply":"2021-05-30T09:41:33.24277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"log = pd.read_json(\"log\")\nbest_epoch = log[\"val/loss\"].idxmin() + 1\nlog.iloc[[best_epoch - 1],]","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:41:33.246987Z","iopub.execute_input":"2021-05-30T09:41:33.247709Z","iopub.status.idle":"2021-05-30T09:41:34.486532Z","shell.execute_reply.started":"2021-05-30T09:41:33.247662Z","shell.execute_reply":"2021-05-30T09:41:34.485443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shutil.copy(output_dir / \"snapshot_epoch_{}.pth\".format(best_epoch), \"best_model.pth\")","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:41:34.488183Z","iopub.execute_input":"2021-05-30T09:41:34.488653Z","iopub.status.idle":"2021-05-30T09:41:34.647977Z","shell.execute_reply.started":"2021-05-30T09:41:34.488605Z","shell.execute_reply":"2021-05-30T09:41:34.646623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"m = get_model({\n    'name': settings[\"model\"][\"name\"],\n    'params': {\n        'pretrained': settings[\"model\"][\"params\"][\"pretrained\"], \n        'n_classes': settings[\"model\"][\"params\"][\"n_classes\"]\n    }})\nstate_dict = torch.load('best_model.pth')\nm.load_state_dict(state_dict)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:41:34.650083Z","iopub.execute_input":"2021-05-30T09:41:34.650586Z","iopub.status.idle":"2021-05-30T09:41:35.690659Z","shell.execute_reply.started":"2021-05-30T09:41:34.650509Z","shell.execute_reply":"2021-05-30T09:41:35.689083Z"},"trusted":true},"execution_count":null,"outputs":[]}]}