{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"try:\n    import resnest\nexcept ModuleNotFoundError:\n    !pip install -q \"../input/resnest50-fast-package/resnest-0.0.6b20200701/resnest\"\n\ntry:\n    import efficientnet_pytorch\nexcept ModuleNotFoundError:\n    !pip install -q \"../input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master\"","metadata":{"execution":{"iopub.status.busy":"2021-05-31T04:47:49.212066Z","iopub.execute_input":"2021-05-31T04:47:49.217275Z","iopub.status.idle":"2021-05-31T04:47:49.750876Z","shell.execute_reply.started":"2021-05-31T04:47:49.217085Z","shell.execute_reply":"2021-05-31T04:47:49.749905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\nimport sys \nimport time\nsys.path.append('../input/timm-pytorch-image-models/pytorch-image-models-master')\n\nimport cv2\nfrom efficientnet_pytorch import EfficientNet\nimport librosa as lb\nimport numpy as np\nimport pandas as pd\nfrom pathlib import Path\nfrom resnest.torch import resnest50\nimport soundfile as sf\nimport timm\nimport torch\nfrom torch import nn\nfrom torch.nn import functional as F\nfrom  torch.utils.data import Dataset, DataLoader\nfrom tqdm.notebook import tqdm","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","id":"kSCcqxf0c_O7","papermill":{"duration":4.153186,"end_time":"2021-02-16T15:59:57.894303","exception":false,"start_time":"2021-02-16T15:59:53.741117","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-31T04:47:49.752683Z","iopub.execute_input":"2021-05-31T04:47:49.753115Z","iopub.status.idle":"2021-05-31T04:47:51.492098Z","shell.execute_reply.started":"2021-05-31T04:47:49.753072Z","shell.execute_reply":"2021-05-31T04:47:51.491035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUM_CLASSES = 397\nSR = 32_000\nDURATION = 5\nTHRESH = 0.12\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(\"DEVICE:\", DEVICE)\nTEST_AUDIO_ROOT = Path(\"../input/birdclef-2021/test_soundscapes\")\nSAMPLE_SUB_PATH = \"../input/birdclef-2021/sample_submission.csv\"\nTARGET_PATH = None\n\nif not len(list(TEST_AUDIO_ROOT.glob(\"*.ogg\"))):\n    TEST_AUDIO_ROOT = Path(\"../input/birdclef-2021/train_soundscapes\")\n    SAMPLE_SUB_PATH = None\n    # SAMPLE_SUB_PATH = \"../input/birdclef-2021/sample_submission.csv\"\n    TARGET_PATH = Path(\"../input/birdclef-2021/train_soundscape_labels.csv\")","metadata":{"id":"rJhYZVIDc_O9","papermill":{"duration":0.440614,"end_time":"2021-02-16T15:59:59.86408","exception":false,"start_time":"2021-02-16T15:59:59.423466","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-31T04:47:51.493943Z","iopub.execute_input":"2021-05-31T04:47:51.494410Z","iopub.status.idle":"2021-05-31T04:47:51.532304Z","shell.execute_reply.started":"2021-05-31T04:47:51.494369Z","shell.execute_reply":"2021-05-31T04:47:51.530746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MelSpecComputer:\n    def __init__(self, sr, n_mels, fmin, fmax, **kwargs):\n        self.sr = sr\n        self.n_mels = n_mels\n        self.fmin = fmin\n        self.fmax = fmax\n        kwargs[\"n_fft\"] = kwargs.get(\"n_fft\", self.sr//10)\n        kwargs[\"hop_length\"] = kwargs.get(\"hop_length\", self.sr//(10*4))\n        self.kwargs = kwargs\n\n    def __call__(self, y):\n        melspec = lb.feature.melspectrogram(\n            y, sr=self.sr, n_mels=self.n_mels, fmin=self.fmin, fmax=self.fmax, **self.kwargs,\n        )\n        melspec = lb.power_to_db(melspec).astype(np.float32)\n        return melspec\n\n\ndef mono_to_color(X, eps=1e-6, mean=None, std=None):\n    mean = mean or X.mean()\n    std = std or X.std()\n    X = (X - mean) / (std + eps)\n    \n    _min, _max = X.min(), X.max()\n\n    if (_max - _min) > eps:\n        V = np.clip(X, _min, _max)\n        V = 255 * (V - _min) / (_max - _min)\n        V = V.astype(np.uint8)\n    else:\n        V = np.zeros_like(X, dtype=np.uint8)\n\n    return V\n\n\ndef crop_or_pad(y, length):\n    if len(y) < length:\n        y = np.concatenate([y, length - np.zeros(len(y))])\n    elif len(y) > length:\n        y = y[:length]\n    return y\n\n\nclass BirdCLEFDataset(Dataset):\n    def __init__(self, data, sr=SR, n_mels=128, fmin=0, fmax=None, duration=DURATION, step=None, res_type=\"kaiser_fast\", resample=True):\n        \n        self.data = data\n        \n        self.sr = sr\n        self.n_mels = n_mels\n        self.fmin = fmin\n        self.fmax = fmax or self.sr//2\n\n        self.duration = duration\n        self.audio_length = self.duration*self.sr\n        self.step = step or self.audio_length\n        \n        self.res_type = res_type\n        self.resample = resample\n\n        self.mel_spec_computer = MelSpecComputer(sr=self.sr, n_mels=self.n_mels, fmin=self.fmin,\n                                                 fmax=self.fmax)\n    def __len__(self):\n        return len(self.data)\n    \n    @staticmethod\n    def normalize(image):\n        image = image.astype(\"float32\", copy=False) / 255.0\n        image = np.stack([image, image, image])\n        return image\n    \n    def audio_to_image(self, audio):\n        melspec = self.mel_spec_computer(audio)\n        \n        image = mono_to_color(melspec)\n        image = self.normalize(image)\n        return image\n\n    def read_file(self, filepath):\n        audio, orig_sr = sf.read(filepath, dtype=\"float32\")\n\n        if self.resample and orig_sr != self.sr:\n            audio = lb.resample(audio, orig_sr, self.sr, res_type=self.res_type)\n          \n        audios = []\n        for i in range(self.audio_length, len(audio) + self.step, self.step):\n            start = max(0, i - self.audio_length)\n            end = start + self.audio_length\n            audios.append(audio[start:end])\n            \n        if len(audios[-1]) < self.audio_length:\n            audios = audios[:-1]\n            \n        images = [self.audio_to_image(audio) for audio in audios]\n        images = np.stack(images)\n        \n        return images\n        \n    def __getitem__(self, idx):\n        return self.read_file(self.data.loc[idx, \"filepath\"])\n\n\nclass densenet(nn.Module):\n    def __init__(self, n_class=397, model_name='densenet121', pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained,num_classes=0)\n        self.dropout = torch.nn.Dropout(p=0.2)\n        self.classifier = nn.Linear(1024, n_class)\n\n    def forward(self, x):\n        x = self.model(x)\n        x = self.dropout(x)\n        x = self.classifier(x)\n        return x\n\n\nclass nfnet(nn.Module):\n    def __init__(self, n_class=397, model_name='eca_nfnet_l0', pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained,num_classes=0)\n        self.dropout = torch.nn.Dropout(p=0.4)\n        self.classifier = nn.Linear(2304, n_class)\n\n    def forward(self, x):\n        x = self.model(x)\n        x = self.dropout(x)\n        x = self.classifier(x)\n        \n        return x\n\n\ndef load_densenet(checkpoint_path, num_classes=NUM_CLASSES):\n    net = densenet(pretrained=False)\n    dummy_device = torch.device(\"cpu\")\n    d = torch.load(checkpoint_path, map_location=dummy_device)\n#     for key in list(d.keys()):\n#         d[key.replace(\"model.\", \"\")] = d.pop(key)\n    net.load_state_dict(d)\n    net = net.to(DEVICE)\n    net = net.eval()\n    return net\n\n\ndef load_resnest50dnet(checkpoint_path, num_classes=NUM_CLASSES):\n    net = timm.create_model('resnest50d',pretrained=False)\n    net.fc = nn.Linear(net.fc.in_features, num_classes)\n    dummy_device = torch.device(\"cpu\")\n    d = torch.load(checkpoint_path, map_location=dummy_device)\n    for key in list(d.keys()):\n        d[key.replace(\"model.\", \"\")] = d.pop(key)\n    net.load_state_dict(d)\n    net = net.to(DEVICE)\n    net = net.eval()\n    return net\n\n\ndef load_resnestnet(checkpoint_path, num_classes=NUM_CLASSES):\n    net = resnest50(pretrained=False)\n    net.fc = nn.Linear(net.fc.in_features, num_classes)\n    dummy_device = torch.device(\"cpu\")\n    d = torch.load(checkpoint_path, map_location=dummy_device)\n    for key in list(d.keys()):\n        d[key.replace(\"model.\", \"\")] = d.pop(key)\n    net.load_state_dict(d)\n    net = net.to(DEVICE)\n    net = net.eval()\n    return net\n\n\ndef load_nfnet(checkpoint_path, num_classes=NUM_CLASSES):\n    net = nfnet(pretrained=False)\n    dummy_device = torch.device(\"cpu\")\n    d = torch.load(checkpoint_path, map_location=dummy_device)\n#     for key in list(d.keys()):\n#         d[key.replace(\"model.\", \"\")] = d.pop(key)\n    net.load_state_dict(d)\n    net = net.to(DEVICE)\n    net = net.eval()\n    return net\n\n\ndef load_efficient_net(checkpoint_path, num_classes=NUM_CLASSES):\n    net = EfficientNet.from_name('efficientnet-b5')\n    net._fc = nn.Linear(net._fc.in_features, num_classes)\n    #model._fc = nn.Linear(in_features=model._fc.in_features, out_features=6)\n    dummy_device = torch.device(\"cpu\")\n    d = torch.load(checkpoint_path, map_location=dummy_device)\n    for key in list(d.keys()):\n        d[key.replace(\"model.\", \"\")] = d.pop(key)\n    #model.load_state_dict(torch.load('../input/pytorch-efficientnet/best_model.pth'))\n    net.load_state_dict(d)\n    net = net.to(DEVICE)\n    net = net.eval()\n    return net\n\n\n@torch.no_grad()\ndef get_thresh_preds(out, thresh=None):\n    thresh = thresh or THRESH\n    o = (-out).argsort(1)\n    npreds = (out > thresh).sum(1)\n    preds = []\n    for oo, npred in zip(o, npreds):\n        preds.append(oo[:npred].cpu().numpy().tolist())\n    return preds\n\n\ndef get_bird_names(preds):\n    bird_names = []\n    for pred in preds:\n        if not pred:\n            bird_names.append(\"nocall\")\n        else:\n            bird_names.append(\" \".join([INV_LABEL_IDS[bird_id] for bird_id in pred]))\n    return bird_names\n\n\ndef predict(nets, test_data, names=True):\n    preds = []\n    x = [0.04, 0.05, 0.05]\n    with torch.no_grad():\n        for idx in  tqdm(list(range(len(test_data)))):\n            xb = torch.from_numpy(test_data[idx]).to(DEVICE)\n            pred = 0.\n            for model_id, net in enumerate(nets):\n                o = net(xb)\n                o = torch.sigmoid(o)\n                if model_id == 0:\n                    pred += o * (1 - x[0] * 5 - x[1] * 4 - x[2])\n                elif model_id in (1, 2, 3, 4, 5):\n                    pred += o * x[0]\n                elif model_id in (6, 7, 8, 9):\n                    pred += o * x[1]\n                elif model_id == 10:\n                    pred += o * x[2]\n\n            pred /= 1\n            \n            if names:\n                pred = get_bird_names(get_thresh_preds(pred))\n\n            preds.append(pred)\n    return preds\n\n\ndef preds_as_df(data, preds):\n    sub = {\n        \"row_id\": [],\n        \"birds\": [],\n    }\n    \n    for row, pred in zip(data.itertuples(False), preds):\n        row_id = [f\"{row.id}_{row.site}_{5*i}\" for i in range(1, len(pred)+1)]\n        sub[\"birds\"] += pred\n        sub[\"row_id\"] += row_id\n        \n    sub = pd.DataFrame(sub)\n    \n    if SAMPLE_SUB_PATH:\n        sample_sub = pd.read_csv(SAMPLE_SUB_PATH, usecols=[\"row_id\"])\n        sub = sample_sub.merge(sub, on=\"row_id\", how=\"left\")\n        sub[\"birds\"] = sub[\"birds\"].fillna(\"nocall\")\n    return sub","metadata":{"id":"4fwNhdbJc_O-","papermill":{"duration":0.056975,"end_time":"2021-02-16T16:00:00.157168","exception":false,"start_time":"2021-02-16T16:00:00.100193","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-31T04:47:51.534556Z","iopub.execute_input":"2021-05-31T04:47:51.535018Z","iopub.status.idle":"2021-05-31T04:47:51.588947Z","shell.execute_reply.started":"2021-05-31T04:47:51.534962Z","shell.execute_reply":"2021-05-31T04:47:51.587631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.DataFrame(\n     [(path.stem, *path.stem.split(\"_\"), path) for path in Path(TEST_AUDIO_ROOT).glob(\"*.ogg\")],\n    columns = [\"filename\", \"id\", \"site\", \"date\", \"filepath\"]\n)\ndf_train = pd.read_csv(\"../input/birdclef-2021/train_metadata.csv\")\nLABEL_IDS = {label: label_id for label_id,label in enumerate(sorted(df_train[\"primary_label\"].unique()))}\nINV_LABEL_IDS = {val: key for key,val in LABEL_IDS.items()}\ntest_data = BirdCLEFDataset(data=data)\n\ncheckpoint_paths = [\n    Path(\"../input/kkiller-birdclef-models-public/birdclef_resnest50_fold0_epoch_10_f1_val_06471_20210417161101.pth\"),\n    Path(\"../input/densenet/birdclef_densenet121_fold0_epoch_16_f1_val_06185_20210516040431.pth\"),\n    Path(\"../input/densenet/birdclef_densenet121_fold1_epoch_17_f1_val_06234_20210516072613.pth\"),\n    Path(\"../input/densenet/birdclef_densenet121_fold2_epoch_16_f1_val_06259_20210517151626.pth\"),\n    Path(\"../input/densenet/birdclef_densenet121_fold3_epoch_16_f1_val_06143_20210517180831.pth\"),\n    Path(\"../input/densenet/birdclef_densenet121_fold4_epoch_16_f1_val_06165_20210517210136.pth\"),\n    Path('../input/bridclef-resnest50-weight/birdclef_resnest50_fold4_epoch_29_f1_val_07694_20210513205331.pth'),\n    Path('../input/bridclef-resnest50-weight/birdclef_resnest50_fold1_epoch_18_f1_val_07636_20210512152028.pth'),\n    Path('../input/bridclef-resnest50-weight/birdclef_resnest50_fold2_epoch_24_f1_val_07728_20210512202556.pth'),\n    Path('../input/bridclef-resnest50-weight/birdclef_resnest50_fold3_epoch_28_f1_val_07609_20210513105835.pth'),\n    Path(\"../input/birdclefefficientnetb5-fold0-epoch-09-10/birdclef_efficientnet-b5_fold0_epoch_09_f1_val_07398_20210522154537.pth\"),\n]\n\nnets = [\n    load_resnestnet(checkpoint_paths[0]),\n    load_densenet(checkpoint_paths[1]),\n    load_densenet(checkpoint_paths[2]),\n    load_densenet(checkpoint_paths[3]),\n    load_densenet(checkpoint_paths[4]),\n    load_densenet(checkpoint_paths[5]),\n    load_resnestnet(checkpoint_paths[6]),\n    load_resnestnet(checkpoint_paths[7]),\n    load_resnestnet(checkpoint_paths[8]),\n    load_resnestnet(checkpoint_paths[9]),\n    load_efficient_net(checkpoint_paths[10]),\n]\n\npred_probas = predict(nets, test_data, names=False)\npreds = [get_bird_names(get_thresh_preds(pred, thresh=THRESH)) for pred in pred_probas]\nsub = preds_as_df(data, preds)\nsub.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-05-31T04:47:51.591207Z","iopub.execute_input":"2021-05-31T04:47:51.591857Z","iopub.status.idle":"2021-05-31T04:49:44.434506Z","shell.execute_reply.started":"2021-05-31T04:47:51.591811Z","shell.execute_reply":"2021-05-31T04:49:44.433600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_metrics(s_true, s_pred):\n    s_true = set(s_true.split())\n    s_pred = set(s_pred.split())\n    n, n_true, n_pred = len(s_true.intersection(s_pred)), len(s_true), len(s_pred)\n    \n    prec = n/n_pred\n    rec = n/n_true\n    f1 = 2*prec*rec/(prec + rec) if prec + rec else 0\n    \n    return {\"f1\": f1, \"prec\": prec, \"rec\": rec, \"n_true\": n_true, \"n_pred\": n_pred, \"n\": n}\n\n\nif TARGET_PATH:\n    sub_target = pd.read_csv(TARGET_PATH)\n    sub_target = sub_target.merge(sub, how=\"left\", on=\"row_id\")\n    assert sub_target[\"birds_x\"].notnull().all()\n    assert sub_target[\"birds_y\"].notnull().all()\n    \n    df_metrics = pd.DataFrame([get_metrics(s_true, s_pred) for s_true, s_pred in zip(sub_target.birds_x, sub_target.birds_y)])\n\n    print(df_metrics.mean())","metadata":{"execution":{"iopub.status.busy":"2021-05-31T04:49:44.436926Z","iopub.execute_input":"2021-05-31T04:49:44.437295Z","iopub.status.idle":"2021-05-31T04:49:44.480567Z","shell.execute_reply.started":"2021-05-31T04:49:44.437264Z","shell.execute_reply":"2021-05-31T04:49:44.479367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}