{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install audiomentations","metadata":{"execution":{"iopub.status.busy":"2021-12-05T06:30:40.089764Z","iopub.execute_input":"2021-12-05T06:30:40.090036Z","iopub.status.idle":"2021-12-05T06:30:47.579182Z","shell.execute_reply.started":"2021-12-05T06:30:40.089999Z","shell.execute_reply":"2021-12-05T06:30:47.578358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom  ast import literal_eval\nimport librosa\nimport librosa.display as lbd\nimport soundfile as sf\nfrom  soundfile import SoundFile\nfrom  IPython.display import Audio\nfrom pathlib import Path\n\nimport torch\nfrom torch import nn, optim\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset, WeightedRandomSampler\nfrom torch.cuda.amp import autocast, GradScaler\nfrom skimage.transform import rescale, resize, downscale_local_mean\nfrom audiomentations import Compose, AddGaussianSNR, AddGaussianNoise, PitchShift, AddBackgroundNoise, AddShortNoises, Gain\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold\nfrom sklearn.metrics import f1_score\nimport random\nimport time\n\nfrom scipy.special import logit, expit\nimport argparse\n\n# from resnest.torch import resnest50\nfrom matplotlib import pyplot as plt\nimport os, random, gc\nimport re, time, json\nfrom tqdm import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-12-05T06:30:47.581139Z","iopub.execute_input":"2021-12-05T06:30:47.581788Z","iopub.status.idle":"2021-12-05T06:30:48.981550Z","shell.execute_reply.started":"2021-12-05T06:30:47.581746Z","shell.execute_reply":"2021-12-05T06:30:48.980493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append('../input/timm-pytorch-image-models/pytorch-image-models-master')\nimport timm","metadata":{"execution":{"iopub.status.busy":"2021-12-05T06:30:48.983313Z","iopub.execute_input":"2021-12-05T06:30:48.983623Z","iopub.status.idle":"2021-12-05T06:30:48.987747Z","shell.execute_reply.started":"2021-12-05T06:30:48.983579Z","shell.execute_reply":"2021-12-05T06:30:48.987035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.environ['CUDA_VISIBLE_DEVICES'] = \"0\"\nPath('./checkpoints').mkdir()\n\nfname = 'stft_tranformer'\nFOLD = 0\nfname = fname + '_' + str(FOLD)\ncheckpoint_path = Path('./checkpoints') / fname\n\n","metadata":{"execution":{"iopub.status.busy":"2021-12-05T06:30:48.990289Z","iopub.execute_input":"2021-12-05T06:30:48.990713Z","iopub.status.idle":"2021-12-05T06:30:49.000942Z","shell.execute_reply.started":"2021-12-05T06:30:48.990677Z","shell.execute_reply":"2021-12-05T06:30:49.000194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUM_CLASSES = 398\nSR = 32_000\nDURATION = 7\n\nMAX_READ_SAMPLES = 5\n\n# PERIOD = 5\n# IMAGE_HEIGHT = 256\n# IMAGE_WIDTH = 576\n\n# POSWEIGHT=10\n# SR=32000\n\n# pd.options.display.max_columns = 100","metadata":{"execution":{"iopub.status.busy":"2021-12-05T06:30:49.002451Z","iopub.execute_input":"2021-12-05T06:30:49.002719Z","iopub.status.idle":"2021-12-05T06:30:49.009250Z","shell.execute_reply.started":"2021-12-05T06:30:49.002684Z","shell.execute_reply":"2021-12-05T06:30:49.008569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def seed_torch(seed_value):\n#     random.seed(seed_value) # Python\n#     np.random.seed(seed_value) # cpu vars\n#     torch.manual_seed(seed_value) # cpu  vars    \n#     if torch.cuda.is_available(): \n#         torch.cuda.manual_seed(seed_value)\n#         torch.cuda.manual_seed_all(seed_value) # gpu vars\n#     if torch.backends.cudnn.is_available:\n#         torch.backends.cudnn.deterministic = True\n#         torch.backends.cudnn.benchmark = False\n        \ndef seed_everything(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\nseed_everything()","metadata":{"execution":{"iopub.status.busy":"2021-12-05T06:30:49.010166Z","iopub.execute_input":"2021-12-05T06:30:49.012816Z","iopub.status.idle":"2021-12-05T06:30:49.021435Z","shell.execute_reply.started":"2021-12-05T06:30:49.012787Z","shell.execute_reply":"2021-12-05T06:30:49.020815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Get melspec meta data for training data","metadata":{}},{"cell_type":"code","source":"INPUT = Path('../input/')\nMEL_PATHS = sorted(INPUT.glob(\"kkiller-birdclef-mels-computer-d7-part?/rich_train_metadata.csv\"))\nTRAIN_LABEL_PATHS = sorted(INPUT.glob(\"kkiller-birdclef-mels-computer-d7-part?/LABEL_IDS.json\"))\nMEL_PATHS","metadata":{"execution":{"iopub.status.busy":"2021-12-05T06:30:49.024166Z","iopub.execute_input":"2021-12-05T06:30:49.024621Z","iopub.status.idle":"2021-12-05T06:30:49.062886Z","shell.execute_reply.started":"2021-12-05T06:30:49.024539Z","shell.execute_reply":"2021-12-05T06:30:49.062201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_df(mel_paths=MEL_PATHS, train_label_paths=TRAIN_LABEL_PATHS):\n  df = None\n  LABEL_IDS = {}\n    \n  for file_path in mel_paths:\n    temp = pd.read_csv(str(file_path), index_col=0)\n    temp[\"impath\"] = temp.apply(lambda row: file_path.parent/\"audio_images/{}/{}.npy\".format(row.primary_label, row.filename), axis=1) \n    df = temp if df is None else df.append(temp)\n    \n  df[\"secondary_labels\"] = df[\"secondary_labels\"].apply(literal_eval)\n\n  for file_path in train_label_paths:\n    with open(str(file_path)) as f:\n      LABEL_IDS.update(json.load(f))\n\n  return LABEL_IDS, df\n\n\nLABEL_IDS, df = get_df()\ndf.rename(columns={'frames':'length'}, inplace=True)\nprint(df.shape)\ndf.to_csv('train_melspec_meta.csv', index=False)\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-05T06:30:49.063908Z","iopub.execute_input":"2021-12-05T06:30:49.064474Z","iopub.status.idle":"2021-12-05T06:30:54.393759Z","shell.execute_reply.started":"2021-12-05T06:30:49.064438Z","shell.execute_reply":"2021-12-05T06:30:54.393087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Get melspec meta data for ff1010 data","metadata":{}},{"cell_type":"code","source":"ff1010 = pd.read_csv('../input/ff1010bird-duration7/rich_metadata.csv')\nff1010 = ff1010[ff1010['primary_label']=='nocall']\nff1010['secondary_labels'] = [[]] * len(ff1010)\nff1010['filename'] = ['%d.wav.npy' % (itemid) for itemid in ff1010['itemid']]\nff1010['label_id'] = 397\nff1010['impath'] = ['../input/ff1010bird-duration7/nocall/%s' %(fname) for fname in ff1010['filename'] ]\nff1010.rename(columns={'frames':'length'}, inplace=True)\n\n# ff1010 = pd.read_csv('../input/freefield1010-nocall-preprocessed/train_ff1010.csv')\n# ff1010['primary_label'] = 'nocall'\n# ff1010['impath'] = ['../input/freefield1010-nocall-preprocessed/%s' %(fname) for fname in ff1010['filename'] ]\nprint(ff1010.shape)\nff1010.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-05T06:30:54.395204Z","iopub.execute_input":"2021-12-05T06:30:54.395685Z","iopub.status.idle":"2021-12-05T06:30:54.455229Z","shell.execute_reply.started":"2021-12-05T06:30:54.395648Z","shell.execute_reply":"2021-12-05T06:30:54.454447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from  sklearn.model_selection  import StratifiedKFold\n# skf = StratifiedKFold(n_splits=5, random_state=666, shuffle=True)\n# splits = skf.split(np.arange(len(ff1010)), y=ff1010.primary_label.values)\n# ff1010[\"fold\"] = -1\n\n# for fold, (train_set, val_set) in enumerate(splits):\n\n#     ff1010.loc[ff1010.index[val_set], \"fold\"] = fold\n\n# ff1010.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-05T06:30:54.457962Z","iopub.execute_input":"2021-12-05T06:30:54.458211Z","iopub.status.idle":"2021-12-05T06:30:54.461962Z","shell.execute_reply.started":"2021-12-05T06:30:54.458178Z","shell.execute_reply":"2021-12-05T06:30:54.461255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Combine these two meta data","metadata":{}},{"cell_type":"code","source":"columns = ['length', 'primary_label', 'secondary_labels', 'filename', 'impath', 'fold', 'label_id']\ndf = pd.concat((df[columns], ff1010[columns])).reset_index(drop=True)\nprint(df.shape)\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-05T06:30:54.463454Z","iopub.execute_input":"2021-12-05T06:30:54.463825Z","iopub.status.idle":"2021-12-05T06:30:54.511812Z","shell.execute_reply.started":"2021-12-05T06:30:54.463785Z","shell.execute_reply":"2021-12-05T06:30:54.511082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.label_id.min(), df.label_id.max()","metadata":{"execution":{"iopub.status.busy":"2021-12-05T06:30:54.513091Z","iopub.execute_input":"2021-12-05T06:30:54.513501Z","iopub.status.idle":"2021-12-05T06:30:54.521637Z","shell.execute_reply.started":"2021-12-05T06:30:54.513464Z","shell.execute_reply":"2021-12-05T06:30:54.520867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# primary_labels = set(train.primary_label.unique())\n# secondary_labels = set([s for labels in train.secondary_labels for s in eval(str(labels))])\n# len(primary_labels), len(secondary_labels), len(secondary_labels - primary_labels), secondary_labels-primary_labels","metadata":{"execution":{"iopub.status.busy":"2021-12-05T06:30:54.522776Z","iopub.execute_input":"2021-12-05T06:30:54.523197Z","iopub.status.idle":"2021-12-05T06:30:54.527187Z","shell.execute_reply.started":"2021-12-05T06:30:54.523160Z","shell.execute_reply":"2021-12-05T06:30:54.526423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# res = [[label for label in eval(str(secondary_label)) if label != 'rocpig1'] \n#                              for secondary_label in train['secondary_labels']]\n# train['secondary_labels'] = res\n# train.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-05T06:30:54.528607Z","iopub.execute_input":"2021-12-05T06:30:54.529059Z","iopub.status.idle":"2021-12-05T06:30:54.535286Z","shell.execute_reply.started":"2021-12-05T06:30:54.529025Z","shell.execute_reply":"2021-12-05T06:30:54.534541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# BIRD_CODE = {}\n# INV_BIRD_CODE = {}\n# for i,label in enumerate(sorted(primary_labels)):\n#     BIRD_CODE[label] = i\n#     INV_BIRD_CODE[i] = label\n\n# NOCALL_CODE = BIRD_CODE['nocall']\n# NOCALL_CODE","metadata":{"execution":{"iopub.status.busy":"2021-12-05T06:30:54.536479Z","iopub.execute_input":"2021-12-05T06:30:54.536741Z","iopub.status.idle":"2021-12-05T06:30:54.543185Z","shell.execute_reply.started":"2021-12-05T06:30:54.536706Z","shell.execute_reply":"2021-12-05T06:30:54.542523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train['class'] = [BIRD_CODE[label] for label in train.primary_label]\n# train['weight'] = train.groupby('class')['class'].transform('count')\n# train['weight'] = 1 / np.sqrt(train['weight'])\n# train['weight'] /= train['weight'].mean()\n# train.loc[train.primary_label == 'nocall', 'weight'] = 1\n# train.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-05T06:30:54.546013Z","iopub.execute_input":"2021-12-05T06:30:54.546202Z","iopub.status.idle":"2021-12-05T06:30:54.551559Z","shell.execute_reply.started":"2021-12-05T06:30:54.546178Z","shell.execute_reply":"2021-12-05T06:30:54.550672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"!pip install -q pysndfx SoundFile audiomentations pretrainedmodels efficientnet_pytorch resnest","metadata":{"execution":{"iopub.status.busy":"2021-12-05T06:30:54.553176Z","iopub.execute_input":"2021-12-05T06:30:54.553444Z","iopub.status.idle":"2021-12-05T06:31:02.830175Z","shell.execute_reply.started":"2021-12-05T06:30:54.553412Z","shell.execute_reply":"2021-12-05T06:31:02.829301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from efficientnet_pytorch import EfficientNet\nimport pretrainedmodels\nimport resnest.torch as resnest_torch\nimport torch\nfrom torch import nn, optim\nfrom  torch.utils.data import Dataset, DataLoader\n\nfrom resnest.torch import resnest50\nfrom matplotlib import pyplot as plt\n\nimport os, random, gc\nimport re, time, json\nfrom  ast import literal_eval\n\n\nfrom IPython.display import Audio\nfrom sklearn.metrics import label_ranking_average_precision_score\n\nfrom tqdm.notebook import tqdm\nimport joblib","metadata":{"execution":{"iopub.status.busy":"2021-12-05T06:31:02.832413Z","iopub.execute_input":"2021-12-05T06:31:02.833115Z","iopub.status.idle":"2021-12-05T06:31:02.839781Z","shell.execute_reply.started":"2021-12-05T06:31:02.833071Z","shell.execute_reply":"2021-12-05T06:31:02.839069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model(name, num_classes=NUM_CLASSES):\n    \"\"\"\n    Loads a pretrained model. \n    Supports ResNest, ResNext-wsl, EfficientNet, ResNext and ResNet.\n\n    Arguments:\n        name {str} -- Name of the model to load\n\n    Keyword Arguments:\n        num_classes {int} -- Number of classes to use (default: {1})\n\n    Returns:\n        torch model -- Pretrained model\n    \"\"\"\n    if \"resnest\" in name:\n#         model = getattr(resnest_torch, name)(pretrained=True)\n        model = getattr(timm.models.resnest, name)(pretrained=True)\n    elif \"wsl\" in name:\n        model = torch.hub.load(\"facebookresearch/WSL-Images\", name)\n    elif name.startswith(\"resnext\") or  name.startswith(\"resnet\"):\n        model = torch.hub.load(\"pytorch/vision:v0.6.0\", name, pretrained=True)\n    elif name.startswith(\"tf_efficientnet_b\"):\n        model = getattr(timm.models.efficientnet, name)(pretrained=True)\n    elif \"efficientnet-b\" in name:\n        model = EfficientNet.from_pretrained(name)\n    else:\n        model = pretrainedmodels.__dict__[name](pretrained='imagenet')\n\n    if hasattr(model, \"fc\"):\n        nb_ft = model.fc.in_features\n        model.fc = nn.Linear(nb_ft, num_classes)\n    elif hasattr(model, \"_fc\"):\n        nb_ft = model._fc.in_features\n        model._fc = nn.Linear(nb_ft, num_classes)\n    elif hasattr(model, \"classifier\"):\n        nb_ft = model.classifier.in_features\n        model.classifier = nn.Linear(nb_ft, num_classes)\n    elif hasattr(model, \"last_linear\"):\n        nb_ft = model.last_linear.in_features\n        model.last_linear = nn.Linear(nb_ft, num_classes)\n\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-12-05T06:31:02.843760Z","iopub.execute_input":"2021-12-05T06:31:02.844280Z","iopub.status.idle":"2021-12-05T06:31:02.854721Z","shell.execute_reply.started":"2021-12-05T06:31:02.844229Z","shell.execute_reply":"2021-12-05T06:31:02.854014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import joblib\ndef load_data(df):\n    def load_row(row):\n        # impath = TRAIN_IMAGES_ROOT/f\"{row.primary_label}/{row.filename}.npy\"\n        return row.filename, np.load(str(row.impath))[:MAX_READ_SAMPLES]\n    pool = joblib.Parallel(4)\n    mapper = joblib.delayed(load_row)\n    tasks = [mapper(row) for row in df.itertuples(False)]\n    res = pool(tqdm(tasks))\n    res = dict(res)\n    return res","metadata":{"execution":{"iopub.status.busy":"2021-12-05T06:31:02.855834Z","iopub.execute_input":"2021-12-05T06:31:02.856644Z","iopub.status.idle":"2021-12-05T06:31:02.867780Z","shell.execute_reply.started":"2021-12-05T06:31:02.856607Z","shell.execute_reply":"2021-12-05T06:31:02.866997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\naudio_image_store = load_data(df)\nlen(audio_image_store)","metadata":{"execution":{"iopub.status.busy":"2021-12-05T06:31:02.870725Z","iopub.execute_input":"2021-12-05T06:31:02.871152Z","iopub.status.idle":"2021-12-05T06:34:43.942985Z","shell.execute_reply.started":"2021-12-05T06:31:02.871119Z","shell.execute_reply":"2021-12-05T06:34:43.941703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"shape:\", next(iter(audio_image_store.values())).shape)\nlbd.specshow(next(iter(audio_image_store.values()))[0])","metadata":{"execution":{"iopub.status.busy":"2021-12-05T06:34:43.947080Z","iopub.execute_input":"2021-12-05T06:34:43.948443Z","iopub.status.idle":"2021-12-05T06:34:44.127143Z","shell.execute_reply.started":"2021-12-05T06:34:43.948396Z","shell.execute_reply":"2021-12-05T06:34:44.126514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.Series([len(x) for x in audio_image_store.values()]).value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-12-05T06:34:44.128070Z","iopub.execute_input":"2021-12-05T06:34:44.128376Z","iopub.status.idle":"2021-12-05T06:34:44.196873Z","shell.execute_reply.started":"2021-12-05T06:34:44.128339Z","shell.execute_reply":"2021-12-05T06:34:44.196099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"class BirdClefDataset(Dataset):\n\n    def __init__(self, audio_image_store, meta, sr=SR, is_train=True, num_classes=NUM_CLASSES, duration=DURATION):\n        \n        self.audio_image_store = audio_image_store\n        self.meta = meta.copy().reset_index(drop=True)\n        self.sr = sr\n        self.is_train = is_train\n        self.num_classes = num_classes\n        self.duration = duration\n        self.audio_length = self.duration*self.sr\n    \n    @staticmethod\n    def normalize(image):\n        image = image.astype(\"float32\", copy=False) / 255.0\n        image = np.stack([image, image, image])\n        return image\n\n    def __len__(self):\n        return len(self.meta)\n    \n    def __getitem__(self, idx):\n        row = self.meta.iloc[idx]\n        image = self.audio_image_store[row.filename]\n\n        image = image[np.random.choice(len(image))]\n        image = self.normalize(image)\n        \n        \n        t = np.zeros(self.num_classes, dtype=np.float32) + 0.0025 # Label smoothing\n        t[row.label_id] = 0.995\n        \n        return image, t","metadata":{"execution":{"iopub.status.busy":"2021-12-05T06:34:44.198331Z","iopub.execute_input":"2021-12-05T06:34:44.198580Z","iopub.status.idle":"2021-12-05T06:34:44.208610Z","shell.execute_reply.started":"2021-12-05T06:34:44.198546Z","shell.execute_reply":"2021-12-05T06:34:44.207097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ds = BirdClefDataset(audio_image_store, meta=df, sr=SR, duration=DURATION, is_train=True)\n# len(df)","metadata":{"execution":{"iopub.status.busy":"2021-12-05T06:34:44.210030Z","iopub.execute_input":"2021-12-05T06:34:44.210318Z","iopub.status.idle":"2021-12-05T06:34:44.216973Z","shell.execute_reply.started":"2021-12-05T06:34:44.210283Z","shell.execute_reply":"2021-12-05T06:34:44.216285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# x, y = ds[np.random.choice(len(ds))]\n# # x, y = ds[0]\n# x.shape, y.shape, np.where(y >= 0.5)","metadata":{"execution":{"iopub.status.busy":"2021-12-05T06:34:44.217829Z","iopub.execute_input":"2021-12-05T06:34:44.219443Z","iopub.status.idle":"2021-12-05T06:34:44.225132Z","shell.execute_reply.started":"2021-12-05T06:34:44.219418Z","shell.execute_reply":"2021-12-05T06:34:44.224503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"def one_step( xb, yb, net, criterion, optimizer, scheduler=None):\n  xb, yb = xb.to(DEVICE), yb.to(DEVICE)\n        \n  optimizer.zero_grad()\n  o = net(xb)\n  loss = criterion(o, yb)\n  loss.backward()\n  optimizer.step()\n  \n  with torch.no_grad():\n      l = loss.item()\n\n      o = o.sigmoid()\n      yb = (yb > 0.5 )*1.0\n      lrap = label_ranking_average_precision_score(yb.cpu().numpy(), o.cpu().numpy())\n\n      o = (o > 0.5)*1.0\n\n      prec = (o*yb).sum()/(1e-6 + o.sum())\n      rec = (o*yb).sum()/(1e-6 + yb.sum())\n      f1 = 2*prec*rec/(1e-6+prec+rec)\n\n  if  scheduler is not None:\n    scheduler.step()\n\n  return l, lrap, f1.item(), rec.item(), prec.item()","metadata":{"execution":{"iopub.status.busy":"2021-12-05T06:34:44.226020Z","iopub.execute_input":"2021-12-05T06:34:44.227282Z","iopub.status.idle":"2021-12-05T06:34:44.236815Z","shell.execute_reply.started":"2021-12-05T06:34:44.227230Z","shell.execute_reply":"2021-12-05T06:34:44.235889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@torch.no_grad()\ndef evaluate(net, criterion, val_laoder):\n    net.eval()\n\n    os, y = [], []\n    val_laoder = tqdm(val_laoder, leave = False, total=len(val_laoder))\n\n    for icount, (xb, yb) in  enumerate(val_laoder):\n\n        y.append(yb.to(DEVICE))\n\n        xb = xb.to(DEVICE)\n        o = net(xb)\n\n        os.append(o)\n\n    y = torch.cat(y)\n    o = torch.cat(os)\n\n    l = criterion(o, y).item()\n    \n    o = o.sigmoid()\n    y = (y > 0.5)*1.0\n\n    lrap = label_ranking_average_precision_score(y.cpu().numpy(), o.cpu().numpy())\n\n    o = (o > 0.5)*1.0\n\n    prec = ((o*y).sum()/(1e-6 + o.sum())).item()\n    rec = ((o*y).sum()/(1e-6 + y.sum())).item()\n    f1 = 2*prec*rec/(1e-6+prec+rec)\n\n    return l, lrap, f1, rec, prec, ","metadata":{"execution":{"iopub.status.busy":"2021-12-05T06:34:44.237880Z","iopub.execute_input":"2021-12-05T06:34:44.238295Z","iopub.status.idle":"2021-12-05T06:34:44.249102Z","shell.execute_reply.started":"2021-12-05T06:34:44.238259Z","shell.execute_reply":"2021-12-05T06:34:44.248371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def one_epoch(net, criterion, optimizer, scheduler, train_laoder, val_laoder):\n  net.train()\n  l, lrap, prec, rec, f1, icount = 0.,0.,0.,0., 0., 0\n  train_laoder = tqdm(train_laoder, leave = False)\n  epoch_bar = train_laoder\n  \n  for (xb, yb) in  epoch_bar:\n      # epoch_bar.set_description(\"----|----|----|----|---->\")\n      _l, _lrap, _f1, _rec, _prec = one_step(xb, yb, net, criterion, optimizer)\n      l += _l\n      lrap += _lrap\n      f1 += _f1\n      rec += _rec\n      prec += _prec\n\n      icount += 1\n        \n      if hasattr(epoch_bar, \"set_postfix\") and not icount%10:\n          epoch_bar.set_postfix(\n            loss=\"{:.6f}\".format(l/icount),\n            lrap=\"{:.3f}\".format(lrap/icount),\n            prec=\"{:.3f}\".format(prec/icount),\n            rec=\"{:.3f}\".format(rec/icount),\n            f1=\"{:.3f}\".format(f1/icount),\n          )\n  \n  scheduler.step()\n\n  l /= icount\n  lrap /= icount\n  f1 /= icount\n  rec /= icount\n  prec /= icount\n  \n  l_val, lrap_val, f1_val, rec_val, prec_val = evaluate(net, criterion, val_laoder)\n  \n  return (l, l_val), (lrap, lrap_val), (f1, f1_val), (rec, rec_val), (prec, prec_val)","metadata":{"execution":{"iopub.status.busy":"2021-12-05T06:34:44.250270Z","iopub.execute_input":"2021-12-05T06:34:44.250950Z","iopub.status.idle":"2021-12-05T06:34:44.261879Z","shell.execute_reply.started":"2021-12-05T06:34:44.250909Z","shell.execute_reply":"2021-12-05T06:34:44.261173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AutoSave:\n  def __init__(self, top_k=2, metric=\"f1\", mode=\"min\", root=None, name=\"ckpt\"):\n    self.top_k = top_k\n    self.logs = []\n    self.metric = metric\n    self.mode = mode\n    self.root = Path(root or MODEL_ROOT)\n    assert self.root.exists()\n    self.name = name\n\n    self.top_models = []\n    self.top_metrics = []\n\n  def log(self, model, metrics):\n    metric = metrics[self.metric]\n    rank = self.rank(metric)\n\n    self.top_metrics.insert(rank+1, metric)\n    if len(self.top_metrics) > self.top_k:\n      self.top_metrics.pop(0)\n\n    self.logs.append(metrics)\n    self.save(model, metric, rank, metrics[\"epoch\"])\n\n\n  def save(self, model, metric, rank, epoch):\n    t = time.strftime(\"%Y%m%d%H%M%S\")\n    name = \"{}_epoch_{:02d}_{}_{:.04f}_{}\".format(self.name, epoch, self.metric, metric, t)\n    name = re.sub(r\"[^\\w_-]\", \"\", name) + \".pth\"\n    path = self.root.joinpath(name)\n\n    old_model = None\n    self.top_models.insert(rank+1, name)\n    if len(self.top_models) > self.top_k:\n      old_model = self.root.joinpath(self.top_models[0])\n      self.top_models.pop(0)      \n\n    torch.save(model.state_dict(), path.as_posix())\n\n    if old_model is not None:\n      old_model.unlink()\n\n    self.to_json()\n\n\n  def rank(self, val):\n    r = -1\n    for top_val in self.top_metrics:\n      if val <= top_val:\n        return r\n      r += 1\n\n    return r\n  \n  def to_json(self):\n    # t = time.strftime(\"%Y%m%d%H%M%S\")\n    name = \"{}_logs\".format(self.name)\n    name = re.sub(r\"[^\\w_-]\", \"\", name) + \".json\"\n    path = self.root.joinpath(name)\n\n    with path.open(\"w\") as f:\n      json.dump(self.logs, f, indent=2)","metadata":{"execution":{"iopub.status.busy":"2021-12-05T06:34:44.264130Z","iopub.execute_input":"2021-12-05T06:34:44.264643Z","iopub.status.idle":"2021-12-05T06:34:44.279292Z","shell.execute_reply.started":"2021-12-05T06:34:44.264611Z","shell.execute_reply":"2021-12-05T06:34:44.278571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def one_fold(model_name, fold, train_set, val_set, epochs=20, save=True, save_root=None):\n\n  save_root = Path(save_root) or MODEL_ROOT\n\n  saver = AutoSave(root=save_root, name=f\"birdclef_{model_name}_fold{fold}\", metric=\"f1_val\")\n\n  net = get_model(model_name).to(DEVICE)\n#   model = getattr(resnest_torch, 'resnest50')(pretrained=True)\n\n  criterion = nn.BCEWithLogitsLoss()\n\n  optimizer = optim.Adam(net.parameters(), lr=8e-4)\n  scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, eta_min=1e-5, T_max=epochs)\n\n  train_data = BirdClefDataset(audio_image_store, meta=df.iloc[train_set].reset_index(drop=True),\n                           sr=SR, duration=DURATION, is_train=True)\n  train_laoder = DataLoader(train_data, batch_size=TRAIN_BATCH_SIZE, num_workers=TRAIN_NUM_WORKERS, shuffle=True, pin_memory=True)\n\n  val_data = BirdClefDataset(audio_image_store, meta=df.iloc[val_set].reset_index(drop=True),  sr=SR, duration=DURATION, is_train=False)\n  val_laoder = DataLoader(val_data, batch_size=VAL_BATCH_SIZE, num_workers=VAL_NUM_WORKERS, shuffle=False)\n\n  epochs_bar = tqdm(list(range(epochs)), leave=False)\n  for epoch  in epochs_bar:\n    epochs_bar.set_description(f\"--> [EPOCH {epoch:02d}]\")\n    net.train()\n\n    (l, l_val), (lrap, lrap_val), (f1, f1_val), (rec, rec_val), (prec, prec_val) = one_epoch(\n        net=net,\n        criterion=criterion,\n        optimizer=optimizer,\n        scheduler=scheduler,\n        train_laoder=train_laoder,\n        val_laoder=val_laoder,\n      )\n\n    epochs_bar.set_postfix(\n    loss=\"({:.6f}, {:.6f})\".format(l, l_val),\n    prec=\"({:.3f}, {:.3f})\".format(prec, prec_val),\n    rec=\"({:.3f}, {:.3f})\".format(rec, rec_val),\n    f1=\"({:.3f}, {:.3f})\".format(f1, f1_val),\n    lrap=\"({:.3f}, {:.3f})\".format(lrap, lrap_val),\n    )\n\n    print(\n        \"[{epoch:02d}] loss: {loss} lrap: {lrap} f1: {f1} rec: {rec} prec: {prec}\".format(\n            epoch=epoch,\n            loss=\"({:.6f}, {:.6f})\".format(l, l_val),\n            prec=\"({:.3f}, {:.3f})\".format(prec, prec_val),\n            rec=\"({:.3f}, {:.3f})\".format(rec, rec_val),\n            f1=\"({:.3f}, {:.3f})\".format(f1, f1_val),\n            lrap=\"({:.3f}, {:.3f})\".format(lrap, lrap_val),\n        )\n    )\n\n    if save:\n      metrics = {\n          \"loss\": l, \"lrap\": lrap, \"f1\": f1, \"rec\": rec, \"prec\": prec,\n          \"loss_val\": l_val, \"lrap_val\": lrap_val, \"f1_val\": f1_val, \"rec_val\": rec_val, \"prec_val\": prec_val,\n          \"epoch\": epoch,\n      }\n\n      saver.log(net, metrics)","metadata":{"execution":{"iopub.status.busy":"2021-12-05T06:34:44.283759Z","iopub.execute_input":"2021-12-05T06:34:44.284418Z","iopub.status.idle":"2021-12-05T06:34:44.299589Z","shell.execute_reply.started":"2021-12-05T06:34:44.284380Z","shell.execute_reply":"2021-12-05T06:34:44.298962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(model_name, epochs=20, save=True, n_splits=5, seed=177, save_root=None, suffix=\"\", folds=None):\n  gc.collect()\n  torch.cuda.empty_cache()\n\n  save_root = save_root or MODEL_ROOT/f\"{model_name}{suffix}\"\n  save_root.mkdir(exist_ok=True, parents=True)\n  \n  fold_bar = tqdm(df.reset_index().groupby(\"fold\").index.apply(list).items(), total=df.fold.max()+1)\n  \n  for fold, val_set in fold_bar:\n      if folds and not fold in folds:\n        continue\n      \n      print(f\"\\n############################### [FOLD {fold}]\")\n      fold_bar.set_description(f\"[FOLD {fold}]\")\n      train_set = np.setdiff1d(df.index, val_set)\n        \n      one_fold(model_name, fold=fold, train_set=train_set , val_set=val_set , epochs=epochs, save=save, save_root=save_root)\n    \n      gc.collect()\n      torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2021-12-05T06:34:44.301004Z","iopub.execute_input":"2021-12-05T06:34:44.301290Z","iopub.status.idle":"2021-12-05T06:34:44.311796Z","shell.execute_reply.started":"2021-12-05T06:34:44.301258Z","shell.execute_reply":"2021-12-05T06:34:44.311016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MODEL_NAMES = [\n      \"resnest50d\",\n] \nMODEL_ROOT = Path(\".\")\nDATA_ROOT = Path(\"../input/birdclef-2021\")","metadata":{"execution":{"iopub.status.busy":"2021-12-05T06:34:44.313298Z","iopub.execute_input":"2021-12-05T06:34:44.313813Z","iopub.status.idle":"2021-12-05T06:34:44.320876Z","shell.execute_reply.started":"2021-12-05T06:34:44.313777Z","shell.execute_reply":"2021-12-05T06:34:44.320201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nTRAIN_BATCH_SIZE = 100\nTRAIN_NUM_WORKERS = 2\n\nVAL_BATCH_SIZE = 128\nVAL_NUM_WORKERS = 2\n\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(DEVICE)\n\nfor model_name in MODEL_NAMES:\n  print(\"\\n\\n###########################################\", model_name.upper())\n  try:\n    train(model_name, epochs=20, suffix=f\"_sr{SR}_d{DURATION}_v1_v1\", folds=[0])\n  except Exception as e:\n    # print(f\"Error {model_name} : \\n{e}\")\n    raise ValueError() from  e","metadata":{"execution":{"iopub.status.busy":"2021-12-05T06:34:44.322667Z","iopub.execute_input":"2021-12-05T06:34:44.324369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}