{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"try:\n    import resnest\nexcept ModuleNotFoundError:\n    !pip install -q \"../input/resnest-package/resnest-0.0.6b20200701/resnest\"","metadata":{"execution":{"iopub.status.busy":"2021-06-01T19:40:54.112417Z","iopub.execute_input":"2021-06-01T19:40:54.112775Z","iopub.status.idle":"2021-06-01T19:41:22.632568Z","shell.execute_reply.started":"2021-06-01T19:40:54.112695Z","shell.execute_reply":"2021-06-01T19:41:22.631606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\nimport cv2\nimport time\nimport torch\nimport numpy as np\nimport pandas as pd\nimport librosa as lb\nfrom torch import nn\nimport soundfile as sf\nfrom pathlib import Path\nfrom tqdm.notebook import tqdm\nfrom collections import defaultdict\nfrom resnest.torch import resnest50, resnest101\nfrom  torch.utils.data import Dataset, DataLoader","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","id":"kSCcqxf0c_O7","papermill":{"duration":4.153186,"end_time":"2021-02-16T15:59:57.894303","exception":false,"start_time":"2021-02-16T15:59:53.741117","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-01T19:41:22.634439Z","iopub.execute_input":"2021-06-01T19:41:22.634785Z","iopub.status.idle":"2021-06-01T19:41:26.519401Z","shell.execute_reply.started":"2021-06-01T19:41:22.634756Z","shell.execute_reply":"2021-06-01T19:41:26.518578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Configs","metadata":{"papermill":{"duration":0.046996,"end_time":"2021-02-16T15:59:59.377834","exception":false,"start_time":"2021-02-16T15:59:59.330838","status":"completed"},"tags":[]}},{"cell_type":"code","source":"NUM_CLASSES = 397\nSR = 32_000\nDURATION = 5\n\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(\"DEVICE:\", DEVICE)\n\nTEST_AUDIO_ROOT = Path(\"../input/birdclef-2021/test_soundscapes\")\nSAMPLE_SUB_PATH = \"../input/birdclef-2021/sample_submission.csv\"\nTARGET_PATH = None\n    \nif not len(list(TEST_AUDIO_ROOT.glob(\"*.ogg\"))):\n    TEST_AUDIO_ROOT = Path(\"../input/birdclef-2021/train_soundscapes\")\n    SAMPLE_SUB_PATH = None\n    TARGET_PATH = Path(\"../input/birdclef-2021/train_soundscape_labels.csv\")","metadata":{"id":"rJhYZVIDc_O9","papermill":{"duration":0.440614,"end_time":"2021-02-16T15:59:59.86408","exception":false,"start_time":"2021-02-16T15:59:59.423466","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-01T19:41:26.520744Z","iopub.execute_input":"2021-06-01T19:41:26.521085Z","iopub.status.idle":"2021-06-01T19:41:26.597843Z","shell.execute_reply.started":"2021-06-01T19:41:26.521051Z","shell.execute_reply":"2021-06-01T19:41:26.596975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Functions","metadata":{"papermill":{"duration":0.046026,"end_time":"2021-02-16T16:00:00.054107","exception":false,"start_time":"2021-02-16T16:00:00.008081","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class MelSpecComputer:\n    def __init__(self, sr, n_mels, fmin, fmax, **kwargs):\n        self.sr = sr\n        self.n_mels = n_mels\n        self.fmin = fmin\n        self.fmax = fmax\n        kwargs[\"n_fft\"] = kwargs.get(\"n_fft\", self.sr//10)\n        kwargs[\"hop_length\"] = kwargs.get(\"hop_length\", self.sr//(10*4))\n        self.kwargs = kwargs\n\n    def __call__(self, y):\n\n        melspec = lb.feature.melspectrogram(\n            y, sr=self.sr, n_mels=self.n_mels, fmin=self.fmin, fmax=self.fmax, **self.kwargs,\n        )\n\n        melspec = lb.power_to_db(melspec).astype(np.float32)\n        return melspec\n    \ndef mono_to_color(X, eps=1e-6, mean=None, std=None):\n    mean = mean or X.mean()\n    std = std or X.std()\n    X = (X - mean) / (std + eps)\n    \n    _min, _max = X.min(), X.max()\n\n    if (_max - _min) > eps:\n        V = np.clip(X, _min, _max)\n        V = 255 * (V - _min) / (_max - _min)\n        V = V.astype(np.uint8)\n    else:\n        V = np.zeros_like(X, dtype=np.uint8)\n\n    return V\n\ndef crop_or_pad(y, length):\n    if len(y) < length:\n        y = np.concatenate([y, length - np.zeros(len(y))])\n    elif len(y) > length:\n        y = y[:length]\n    return y\n\nclass BirdCLEFDataset(Dataset):\n    def __init__(self, data, sr=SR, n_mels=128, fmin=0, fmax=None, duration=DURATION, step=None, res_type=\"kaiser_fast\", resample=True):\n        \n        self.data = data\n        \n        self.sr = sr\n        self.n_mels = n_mels\n        self.fmin = fmin\n        self.fmax = fmax or self.sr//2\n\n        self.duration = duration\n        self.audio_length = self.duration*self.sr\n        self.step = step or self.audio_length\n        \n        self.res_type = res_type\n        self.resample = resample\n\n        self.mel_spec_computer = MelSpecComputer(sr=self.sr, n_mels=self.n_mels, fmin=self.fmin,\n                                                 fmax=self.fmax)\n    def __len__(self):\n        return len(self.data)\n    \n    @staticmethod\n    def normalize(image):\n        image = image.astype(\"float32\", copy=False) / 255.0\n        image = np.stack([image, image, image])\n        return image\n    \n    def audio_to_image(self, audio):\n        melspec = self.mel_spec_computer(audio) \n        image = mono_to_color(melspec)\n        image = self.normalize(image)\n        return image\n\n    def read_file(self, filepath):\n        audio, orig_sr = sf.read(filepath, dtype=\"float32\")\n\n        if self.resample and orig_sr != self.sr:\n            audio = lb.resample(audio, orig_sr, self.sr, res_type=self.res_type)\n          \n        audios = []\n        for i in range(self.audio_length, len(audio) + self.step, self.step):\n            start = max(0, i - self.audio_length)\n            end = start + self.audio_length\n            audios.append(audio[start:end])\n            \n        if len(audios[-1]) < self.audio_length:\n            audios = audios[:-1]\n            \n        images = [self.audio_to_image(audio) for audio in audios]\n        images = np.stack(images)\n        \n        return images\n    \n        \n    def __getitem__(self, idx):\n        return self.read_file(self.data.loc[idx, \"filepath\"])\n    \ndef load_net(checkpoint_path, num_classes=NUM_CLASSES):\n    \n    if 'resnest50' in checkpoint_path:\n        net = resnest50(pretrained=False)\n    else:\n        net = resnest101(pretrained=False)\n\n    net.fc = nn.Linear(net.fc.in_features, num_classes)\n    dummy_device = torch.device(\"cpu\")\n    d = torch.load(checkpoint_path, map_location=dummy_device)\n    for key in list(d.keys()):\n        d[key.replace(\"model.\", \"\")] = d.pop(key)\n    net.load_state_dict(d)\n    net = net.to(DEVICE)\n    net = net.eval()\n    return net\n\n@torch.no_grad()\ndef get_thresh_preds(out, thresh=None):\n    thresh = thresh or THRESH\n    o = (-out).argsort(1)\n    npreds = (out > thresh).sum(1)\n    preds = []\n    for oo, npred in zip(o, npreds):\n        preds.append(oo[:npred].cpu().numpy().tolist())\n    return preds\n\ndef get_bird_names(preds):\n    bird_names = []\n    for pred in preds:\n        if not pred:\n            bird_names.append(\"nocall\")\n        else:\n            bird_names.append(\" \".join([INV_LABEL_IDS[bird_id] for bird_id in pred]))\n    return bird_names\n\n\ndef predict(nets, test_data, names=True):\n    preds = []\n    with torch.no_grad():\n        for idx in  tqdm(list(range(len(test_data)))):\n            xb = torch.from_numpy(test_data[idx]).to(DEVICE)\n            pred = 0.\n            for net in nets:\n                o = net(xb)\n                o = torch.sigmoid(o)\n\n                pred += o\n\n            pred /= len(nets)\n            \n            if names:\n                pred = get_bird_names(get_thresh_preds(pred))\n\n            preds.append(pred)\n    return preds\n\n\ndef preds_as_df(data, preds):\n    sub = {\n        \"row_id\": [],\n        \"birds\": [],\n    }\n    \n    for row, pred in zip(data.itertuples(False), preds):\n        row_id = [f\"{row.id}_{row.site}_{5*i}\" for i in range(1, len(pred)+1)]\n        sub[\"birds\"] += pred\n        sub[\"row_id\"] += row_id\n        \n    sub = pd.DataFrame(sub)\n    \n    if SAMPLE_SUB_PATH:\n        sample_sub = pd.read_csv(SAMPLE_SUB_PATH, usecols=[\"row_id\"])\n        sub = sample_sub.merge(sub, on=\"row_id\", how=\"left\")\n        sub[\"birds\"] = sub[\"birds\"].fillna(\"nocall\")\n    return sub\n\ndef get_metrics(s_true, s_pred):\n    s_true = set(s_true.split())\n    s_pred = set(s_pred.split())\n    n, n_true, n_pred = len(s_true.intersection(s_pred)), len(s_true), len(s_pred)\n    \n    prec = n/n_pred\n    rec = n/n_true\n    f1 = 2*prec*rec/(prec + rec) if prec + rec else 0\n    \n    return {\"f1\": f1, \"prec\": prec, \"rec\": rec, \"n_true\": n_true, \"n_pred\": n_pred, \"n\": n}\n\ndef def_value():\n    return 1.0","metadata":{"id":"4fwNhdbJc_O-","papermill":{"duration":0.056975,"end_time":"2021-02-16T16:00:00.157168","exception":false,"start_time":"2021-02-16T16:00:00.100193","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-01T19:41:26.599809Z","iopub.execute_input":"2021-06-01T19:41:26.6004Z","iopub.status.idle":"2021-06-01T19:41:26.634185Z","shell.execute_reply.started":"2021-06-01T19:41:26.60035Z","shell.execute_reply":"2021-06-01T19:41:26.633179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.DataFrame(\n     [(path.stem, *path.stem.split(\"_\"), path) for path in Path(TEST_AUDIO_ROOT).glob(\"*.ogg\")],\n    columns = [\"filename\", \"id\", \"site\", \"date\", \"filepath\"]\n)\nprint(data.shape)\ndata.head()","metadata":{"id":"rVwlOrbxc_PD","outputId":"9b53ec99-634a-4b30-f9b5-75036184482b","papermill":{"duration":0.162256,"end_time":"2021-02-16T16:00:01.102152","exception":false,"start_time":"2021-02-16T16:00:00.939896","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-01T19:41:26.637394Z","iopub.execute_input":"2021-06-01T19:41:26.63788Z","iopub.status.idle":"2021-06-01T19:41:26.677046Z","shell.execute_reply.started":"2021-06-01T19:41:26.637844Z","shell.execute_reply":"2021-06-01T19:41:26.676181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv(\"../input/birdclef-2021/train_metadata.csv\")\n\nLABEL_IDS = {label: label_id for label_id,label in enumerate(sorted(df_train[\"primary_label\"].unique()))}\nINV_LABEL_IDS = {val: key for key,val in LABEL_IDS.items()}","metadata":{"execution":{"iopub.status.busy":"2021-06-01T19:41:26.679322Z","iopub.execute_input":"2021-06-01T19:41:26.679655Z","iopub.status.idle":"2021-06-01T19:41:27.025505Z","shell.execute_reply.started":"2021-06-01T19:41:26.679621Z","shell.execute_reply":"2021-06-01T19:41:27.024744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{"id":"BNsivZZZc_PG","papermill":{"duration":0.048675,"end_time":"2021-02-16T16:00:08.424781","exception":false,"start_time":"2021-02-16T16:00:08.376106","status":"completed"},"tags":[]}},{"cell_type":"code","source":"test_data = BirdCLEFDataset(data=data)\nlen(test_data), test_data[0].shape","metadata":{"id":"AzlOErOmc_PH","papermill":{"duration":0.058312,"end_time":"2021-02-16T16:00:08.531199","exception":false,"start_time":"2021-02-16T16:00:08.472887","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-01T19:41:27.026968Z","iopub.execute_input":"2021-06-01T19:41:27.027327Z","iopub.status.idle":"2021-06-01T19:41:30.050409Z","shell.execute_reply.started":"2021-06-01T19:41:27.027291Z","shell.execute_reply":"2021-06-01T19:41:30.049338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_preds = {}","metadata":{"execution":{"iopub.status.busy":"2021-06-01T19:41:30.051718Z","iopub.execute_input":"2021-06-01T19:41:30.052067Z","iopub.status.idle":"2021-06-01T19:41:30.056666Z","shell.execute_reply.started":"2021-06-01T19:41:30.052031Z","shell.execute_reply":"2021-06-01T19:41:30.055758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# MODEL 1\n","metadata":{}},{"cell_type":"code","source":"MODEL1 = [f'../input/kkiller-birdclef-models-public/birdclef_resnest50_fold0_epoch_10_f1_val_06471_20210417161101.pth']\n\nfor m in MODEL1:\n    checkpoint_paths = [Path(m)]\n\nnets = [load_net(checkpoint_path.as_posix()) for checkpoint_path in checkpoint_paths]\npred_probas = predict(nets, test_data, names=False)\nraw_preds['MODEL1'] = pred_probas","metadata":{"execution":{"iopub.status.busy":"2021-06-01T19:41:30.05798Z","iopub.execute_input":"2021-06-01T19:41:30.058538Z","iopub.status.idle":"2021-06-01T19:42:38.241197Z","shell.execute_reply.started":"2021-06-01T19:41:30.058501Z","shell.execute_reply":"2021-06-01T19:42:38.240269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# MODEL 2","metadata":{}},{"cell_type":"code","source":"MODEL2 = [\n    Path(f'../input/birdclef-models/resnest101_sr32000_d7_v11/resnest101_sr32000_d7_v11/birdclef_resnest101_fold0_epoch_16_f1_val_07591_20210519181210.pth'),\n    Path(f'../input/birdclef-models/resnest101_sr32000_d7_v11/resnest101_sr32000_d7_v11/birdclef_resnest101_fold1_epoch_13_f1_val_07365_20210519220652.pth'),\n    Path(f'../input/birdclef-models/resnest101_sr32000_d7_v11/resnest101_sr32000_d7_v11/birdclef_resnest101_fold2_epoch_14_f1_val_07521_20210520024418.pth'),\n    Path(f'../input/birdclef-models/resnest101_sr32000_d7_v11/resnest101_sr32000_d7_v11/birdclef_resnest101_fold3_epoch_15_f1_val_07583_20210520072224.pth'),\n    Path(f'../input/birdclef-models/resnest101_sr32000_d7_v11/resnest101_sr32000_d7_v11/birdclef_resnest101_fold4_epoch_10_f1_val_07331_20210520105725.pth'),\n  \n    Path(f'../input/birdclef-models/resnest101_sr32000_d7_v8/resnest101_sr32000_d7_v8/birdclef_resnest101_fold0_epoch_16_f1_val_07734_20210518233713.pth'),\n    Path(f'../input/birdclef-models/resnest101_sr32000_d7_v8/resnest101_sr32000_d7_v8/birdclef_resnest101_fold1_epoch_16_f1_val_07675_20210519030243.pth'),\n    Path(f'../input/birdclef-models/resnest101_sr32000_d7_v8/resnest101_sr32000_d7_v8/birdclef_resnest101_fold2_epoch_12_f1_val_07466_20210519054713.pth'),\n    Path(f'../input/birdclef-models/resnest101_sr32000_d7_v8/resnest101_sr32000_d7_v8/birdclef_resnest101_fold3_epoch_13_f1_val_07424_20210519092316.pth'),\n    Path(f'../input/birdclef-models/resnest101_sr32000_d7_v8/resnest101_sr32000_d7_v8/birdclef_resnest101_fold4_epoch_18_f1_val_07755_20210519134027.pth'), \n]\n\ncheckpoint_paths = MODEL2\n\nnets = [load_net(checkpoint_path.as_posix()) for checkpoint_path in checkpoint_paths]\npred_probas = predict(nets, test_data, names=False)\nraw_preds['MODEL2'] = pred_probas","metadata":{"execution":{"iopub.status.busy":"2021-06-01T19:42:38.2428Z","iopub.execute_input":"2021-06-01T19:42:38.243377Z","iopub.status.idle":"2021-06-01T19:44:57.304162Z","shell.execute_reply.started":"2021-06-01T19:42:38.243328Z","shell.execute_reply":"2021-06-01T19:44:57.303263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# MODEL 3","metadata":{}},{"cell_type":"code","source":"MODEL3 = [\n    Path(f'../input/birdclef-models/resnest101_sr32000_d7_v27/resnest101_sr32000_d7_v27/birdclef_resnest101_fold1_epoch_09_f1_val_05461_20210526015608.pth'),\n    Path(f'../input/birdclef-models/resnest101_sr32000_d7_v27/resnest101_sr32000_d7_v27/birdclef_resnest101_fold1_epoch_11_f1_val_05736_20210526020742.pth'),\n    Path(f'../input/birdclef-models/resnest101_sr32000_d7_v27/resnest101_sr32000_d7_v27/birdclef_resnest101_fold2_epoch_13_f1_val_05963_20210526045546.pth'),\n    Path(f'../input/birdclef-models/resnest101_sr32000_d7_v27/resnest101_sr32000_d7_v27/birdclef_resnest101_fold3_epoch_11_f1_val_05636_20210526072043.pth'),\n    Path(f'../input/birdclef-models/resnest101_sr32000_d7_v27/resnest101_sr32000_d7_v27/birdclef_resnest101_fold4_epoch_14_f1_val_05889_20210526101445.pth'),\n    \n    Path(f'../input/birdclef-models/resnest101_sr32000_d7_v26/resnest101_sr32000_d7_v26/birdclef_resnest101_fold0_epoch_14_f1_val_06002_20210525035058.pth'),\n    Path(f'../input/birdclef-models/resnest101_sr32000_d7_v26/resnest101_sr32000_d7_v26/birdclef_resnest101_fold1_epoch_11_f1_val_05715_20210525075316.pth'),\n    Path(f'../input/birdclef-models/resnest101_sr32000_d7_v26/resnest101_sr32000_d7_v26/birdclef_resnest101_fold2_epoch_11_f1_val_05670_20210525122558.pth'),\n    Path(f'../input/birdclef-models/resnest101_sr32000_d7_v26/resnest101_sr32000_d7_v26/birdclef_resnest101_fold3_epoch_11_f1_val_05598_20210525165840.pth'),\n]\n\ncheckpoint_paths = MODEL3\n\nnets = [load_net(checkpoint_path.as_posix()) for checkpoint_path in checkpoint_paths]\npred_probas = predict(nets, test_data, names=False)\nraw_preds['MODEL3'] = pred_probas","metadata":{"execution":{"iopub.status.busy":"2021-06-01T19:44:57.305535Z","iopub.execute_input":"2021-06-01T19:44:57.306037Z","iopub.status.idle":"2021-06-01T19:47:15.255254Z","shell.execute_reply.started":"2021-06-01T19:44:57.305997Z","shell.execute_reply":"2021-06-01T19:47:15.25434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# MODEL 4","metadata":{}},{"cell_type":"code","source":"MODEL4 = [\n    Path(f'../input/birdclef-models/resnest101_sr32000_d7_v32/resnest101_sr32000_d7_v32/birdclef_resnest101_fold0_epoch_02_f1_val_05618_20210530142121.pth'),\n    Path(f'../input/birdclef-models/resnest101_sr32000_d7_v32/resnest101_sr32000_d7_v32/birdclef_resnest101_fold1_epoch_02_f1_val_05536_20210530181637.pth'),\n    Path(f'../input/birdclef-models/resnest101_sr32000_d7_v32/resnest101_sr32000_d7_v32/birdclef_resnest101_fold2_epoch_02_f1_val_05802_20210530220945.pth'),\n    Path(f'../input/birdclef-models/resnest101_sr32000_d7_v32/resnest101_sr32000_d7_v32/birdclef_resnest101_fold3_epoch_02_f1_val_05935_20210531020315.pth'),\n    Path(f'../input/birdclef-models/resnest101_sr32000_d7_v32/resnest101_sr32000_d7_v32/birdclef_resnest101_fold4_epoch_02_f1_val_05595_20210531055634.pth'),\n    \n    Path(f'../input/birdclef-models/resnest101_sr32000_d7_v32/resnest101_sr32000_d7_v32/birdclef_resnest101_fold0_epoch_05_f1_val_07128_20210530150823.pth'),\n    Path(f'../input/birdclef-models/resnest101_sr32000_d7_v32/resnest101_sr32000_d7_v32/birdclef_resnest101_fold1_epoch_05_f1_val_07131_20210530190322.pth'),\n    Path(f'../input/birdclef-models/resnest101_sr32000_d7_v32/resnest101_sr32000_d7_v32/birdclef_resnest101_fold2_epoch_05_f1_val_07029_20210530225631.pth'),\n    Path(f'../input/birdclef-models/resnest101_sr32000_d7_v32/resnest101_sr32000_d7_v32/birdclef_resnest101_fold3_epoch_05_f1_val_07148_20210531024952.pth'),\n    Path(f'../input/birdclef-models/resnest101_sr32000_d7_v32/resnest101_sr32000_d7_v32/birdclef_resnest101_fold4_epoch_05_f1_val_07087_20210531064317.pth'),\n]\n\n\ncheckpoint_paths = MODEL4\n\nnets = [load_net(checkpoint_path.as_posix()) for checkpoint_path in checkpoint_paths]\npred_probas = predict(nets, test_data, names=False)\nraw_preds['MODEL4'] = pred_probas","metadata":{"execution":{"iopub.status.busy":"2021-06-01T19:47:49.861283Z","iopub.execute_input":"2021-06-01T19:47:49.86163Z","iopub.status.idle":"2021-06-01T19:50:10.090898Z","shell.execute_reply.started":"2021-06-01T19:47:49.861598Z","shell.execute_reply":"2021-06-01T19:50:10.089634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# MODEL 5","metadata":{}},{"cell_type":"code","source":"MODEL5 = [\n    Path(f'../input/birdclef-models/resnest101_sr32000_d7_v28/resnest101_sr32000_d7_v28/birdclef_resnest101_fold0_epoch_12_f1_val_05806_20210526212606.pth'),\n    Path(f'../input/birdclef-models/resnest101_sr32000_d7_v28/resnest101_sr32000_d7_v28/birdclef_resnest101_fold1_epoch_12_f1_val_05777_20210526232211.pth'),\n    Path(f'../input/birdclef-models/resnest101_sr32000_d7_v28/resnest101_sr32000_d7_v28/birdclef_resnest101_fold2_epoch_09_f1_val_05546_20210527010045.pth'),\n    Path(f'../input/birdclef-models/resnest101_sr32000_d7_v28/resnest101_sr32000_d7_v28/birdclef_resnest101_fold3_epoch_12_f1_val_05634_20210527031422.pth'),\n    Path(f'../input/birdclef-models/resnest101_sr32000_d7_v28/resnest101_sr32000_d7_v28/birdclef_resnest101_fold4_epoch_09_f1_val_05711_20210527045319.pth'),\n]\n\ncheckpoint_paths = MODEL5\n\nnets = [load_net(checkpoint_path.as_posix()) for checkpoint_path in checkpoint_paths]\npred_probas = predict(nets, test_data, names=False)\nraw_preds['MODEL5'] = pred_probas","metadata":{"execution":{"iopub.status.busy":"2021-06-01T19:50:10.096339Z","iopub.execute_input":"2021-06-01T19:50:10.096858Z","iopub.status.idle":"2021-06-01T19:51:45.396104Z","shell.execute_reply.started":"2021-06-01T19:50:10.096809Z","shell.execute_reply":"2021-06-01T19:51:45.395227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# MODEL 6","metadata":{}},{"cell_type":"code","source":"MODEL6 = [\n    Path(f'../input/birdclef-models/resnest50_sr32000_d7_v9/resnest50_sr32000_d7_v9/birdclef_resnest50_fold0_epoch_22_f1_val_07582_20210519030656.pth'),\n    Path(f'../input/birdclef-models/resnest50_sr32000_d7_v9/resnest50_sr32000_d7_v9/birdclef_resnest50_fold1_epoch_29_f1_val_07637_20210519053244.pth'),\n    Path(f'../input/birdclef-models/resnest50_sr32000_d7_v9/resnest50_sr32000_d7_v9/birdclef_resnest50_fold2_epoch_22_f1_val_07535_20210519070947.pth'),\n    Path(f'../input/birdclef-models/resnest50_sr32000_d7_v9/resnest50_sr32000_d7_v9/birdclef_resnest50_fold3_epoch_23_f1_val_07607_20210519091446.pth'),\n    Path(f'../input/birdclef-models/resnest50_sr32000_d7_v9/resnest50_sr32000_d7_v9/birdclef_resnest50_fold4_epoch_26_f1_val_07666_20210519112640.pth'),\n    \n    Path(f'../input/birdclef-models/resnest50_sr32000_d7_v3/resnest50_sr32000_d7_v3/birdclef_resnest50_fold2_epoch_22_f1_val_07690_20210518082321.pth'),\n]\n\ncheckpoint_paths = MODEL6\n\nnets = [load_net(checkpoint_path.as_posix()) for checkpoint_path in checkpoint_paths]\npred_probas = predict(nets, test_data, names=False)\nraw_preds['MODEL6'] = pred_probas","metadata":{"execution":{"iopub.status.busy":"2021-06-01T19:51:45.397575Z","iopub.execute_input":"2021-06-01T19:51:45.398077Z","iopub.status.idle":"2021-06-01T19:53:05.351146Z","shell.execute_reply.started":"2021-06-01T19:51:45.398036Z","shell.execute_reply":"2021-06-01T19:53:05.350264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"squared_models = {}","metadata":{"execution":{"iopub.status.busy":"2021-06-01T19:53:05.352475Z","iopub.execute_input":"2021-06-01T19:53:05.352974Z","iopub.status.idle":"2021-06-01T19:53:05.356584Z","shell.execute_reply.started":"2021-06-01T19:53:05.352934Z","shell.execute_reply":"2021-06-01T19:53:05.355742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# COMBINING MODELS","metadata":{}},{"cell_type":"code","source":"consider_models = ['MODEL1', 'MODEL2', 'MODEL3', 'MODEL4', 'MODEL5', 'MODEL6']\n\nsquared_preds = {}\nfor model in consider_models:\n    \n    res_t = []\n    for file in raw_preds[model]:\n        res_t.append(torch.square(file))\n    squared_preds[model] = res_t\n\nmean_preds = []\n\nfor t in range(len(squared_preds[consider_models[0]])):\n    \n    squared_list = []\n    for model in squared_preds:\n        squared_list.append(squared_preds[model][t])\n\n        \n    mean_preds.append(torch.mean(torch.stack(squared_list),0))\n    \nfinal_preds = []\n\nfor t in mean_preds:\n    final_preds.append(torch.sqrt(t))\n    \n    \nsquared_models['_'.join(consider_models)] = final_preds","metadata":{"execution":{"iopub.status.busy":"2021-06-01T19:53:05.359199Z","iopub.execute_input":"2021-06-01T19:53:05.359723Z","iopub.status.idle":"2021-06-01T19:53:05.409597Z","shell.execute_reply.started":"2021-06-01T19:53:05.359686Z","shell.execute_reply":"2021-06-01T19:53:05.40895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"th = 0.165\npreds = [get_bird_names(get_thresh_preds(pred, thresh=th)) for pred in final_preds]\nsubmission_df = preds_as_df(data, preds)\nprint(submission_df.shape)\nsubmission_df","metadata":{"execution":{"iopub.status.busy":"2021-06-01T19:54:40.280015Z","iopub.execute_input":"2021-06-01T19:54:40.280372Z","iopub.status.idle":"2021-06-01T19:54:40.421307Z","shell.execute_reply.started":"2021-06-01T19:54:40.280341Z","shell.execute_reply":"2021-06-01T19:54:40.409117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-01T19:54:42.851523Z","iopub.execute_input":"2021-06-01T19:54:42.851844Z","iopub.status.idle":"2021-06-01T19:54:43.236395Z","shell.execute_reply.started":"2021-06-01T19:54:42.851813Z","shell.execute_reply":"2021-06-01T19:54:43.235615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Small validation","metadata":{}},{"cell_type":"code","source":"if TARGET_PATH:\n    sub_target = pd.read_csv(TARGET_PATH)\n    sub_target = sub_target.merge(submission_df, how=\"left\", on=\"row_id\")\n    \n    print(sub_target[\"birds_x\"].notnull().sum(), sub_target[\"birds_x\"].notnull().sum())\n    assert sub_target[\"birds_x\"].notnull().all()\n    assert sub_target[\"birds_y\"].notnull().all()\n    \n    df_metrics = pd.DataFrame([get_metrics(s_true, s_pred) for s_true, s_pred in zip(sub_target.birds_x, sub_target.birds_y)])\n    \n    print(df_metrics.mean())","metadata":{"execution":{"iopub.status.busy":"2021-06-01T19:54:44.394286Z","iopub.execute_input":"2021-06-01T19:54:44.394614Z","iopub.status.idle":"2021-06-01T19:54:44.438727Z","shell.execute_reply.started":"2021-06-01T19:54:44.394581Z","shell.execute_reply":"2021-06-01T19:54:44.437209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_metrics = pd.DataFrame([get_metrics(s_true, s_pred) for s_true, s_pred in zip(sub_target[sub_target.birds_y != \"nocall\"].birds_x, sub_target[sub_target.birds_y != \"nocall\"].birds_y)])\nprint(df_metrics.mean())\n\ndisplay(sub_target[sub_target.birds_y != \"nocall\"])","metadata":{"execution":{"iopub.status.busy":"2021-06-01T19:54:46.311959Z","iopub.execute_input":"2021-06-01T19:54:46.312295Z","iopub.status.idle":"2021-06-01T19:54:46.343752Z","shell.execute_reply.started":"2021-06-01T19:54:46.312259Z","shell.execute_reply":"2021-06-01T19:54:46.34299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_metrics = pd.DataFrame([get_metrics(s_true, s_pred) for s_true, s_pred in zip(sub_target[sub_target.birds_x != \"nocall\"].birds_x, sub_target[sub_target.birds_x != \"nocall\"].birds_y)])\nprint(df_metrics.mean())\n\n\ndisplay(sub_target[sub_target.birds_x != \"nocall\"])","metadata":{"execution":{"iopub.status.busy":"2021-06-01T19:54:49.130131Z","iopub.execute_input":"2021-06-01T19:54:49.130508Z","iopub.status.idle":"2021-06-01T19:54:49.164536Z","shell.execute_reply.started":"2021-06-01T19:54:49.130474Z","shell.execute_reply":"2021-06-01T19:54:49.163862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_metrics = pd.DataFrame([get_metrics(s_true, s_pred) for s_true, s_pred in zip(sub_target[sub_target.birds_y == \"nocall\"].birds_x, sub_target[sub_target.birds_y == \"nocall\"].birds_y)])\nprint(df_metrics.mean())\n\ndisplay(sub_target[sub_target.birds_y == \"nocall\"])","metadata":{"execution":{"iopub.status.busy":"2021-06-01T19:55:00.439173Z","iopub.execute_input":"2021-06-01T19:55:00.43953Z","iopub.status.idle":"2021-06-01T19:55:00.472596Z","shell.execute_reply.started":"2021-06-01T19:55:00.439498Z","shell.execute_reply":"2021-06-01T19:55:00.471654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_metrics = pd.DataFrame([get_metrics(s_true, s_pred) for s_true, s_pred in zip(sub_target[sub_target.birds_x == \"nocall\"].birds_x, sub_target[sub_target.birds_x == \"nocall\"].birds_y)])\nprint(df_metrics.mean())\n\n\ndisplay(sub_target[sub_target.birds_x == \"nocall\"])","metadata":{"execution":{"iopub.status.busy":"2021-06-01T19:55:02.362007Z","iopub.execute_input":"2021-06-01T19:55:02.362339Z","iopub.status.idle":"2021-06-01T19:55:02.393324Z","shell.execute_reply.started":"2021-06-01T19:55:02.362306Z","shell.execute_reply":"2021-06-01T19:55:02.392299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}