{"cells":[{"metadata":{},"cell_type":"markdown","source":"# ResNet50 baseline inference\nmodified from: https://www.kaggle.com/ttahara/inference-birdsong-baseline-resnest50-fast?rvi=1","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Thank @hidehisaarai1213 and @ttahara very much for their great notebooks, which make me easy to get start.\nhttps://www.kaggle.com/hidehisaarai1213/inference-pytorch-birdcall-resnet-baseline  \nhttps://www.kaggle.com/ttahara/training-birdsong-baseline-resnest50-fast  \nhttps://www.kaggle.com/ttahara/inference-birdsong-baseline-resnest50-fast","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# How I trained","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"I used @hidehisaarai1213 's training code and modified a little bit.  \n\n**for those audios with duration less than 5s, most people try to pad 0. My idea is resample another audio from the same bird and \nconcatenate with the one has duration less than 5s, then random crop 5s.**","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"see code blew:","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nwhile n_samples <= opt.N_SAMPLES:\n    sub_df = self.df[self.df['ebird_code'] == ebird_code]\n    aid = random.randrange(0, len(sub_df))\n    _sample = sub_df.iloc[aid]\n    _wav_name = _sample[\"resampled_filename\"]\n    _ebird_code = _sample[\"ebird_code\"]\n    assert _ebird_code == ebird_code, \"Err.\"\n    _wav_pth = os.path.join(self.pth, _ebird_code, _wav_name)\n    _y, _ = librosa.load(_wav_pth)\n    y.append(_y)\n    n_samples += len(_y)\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"just demostrate the core part I changed. If you think it's useful try it.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Prepare","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### import libraries","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport gc\nimport time\nimport math\nimport shutil\nimport random\nimport warnings\nimport typing as tp\nfrom pathlib import Path\nfrom contextlib import contextmanager\n\nimport yaml\nfrom joblib import delayed, Parallel\n\nimport cv2\nimport librosa\nimport audioread\nimport soundfile as sf\n\nimport numpy as np\nimport pandas as pd\n\nfrom fastprogress import progress_bar\nfrom sklearn.metrics import f1_score\nfrom sklearn.model_selection import StratifiedKFold\nfrom torchvision import models\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.nn import Conv2d, Module, Linear, BatchNorm2d, ReLU\nfrom torch.nn.modules.utils import _pair\nimport torch.utils.data as data\n\npd.options.display.max_rows = 500\npd.options.display.max_columns = 500","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":false},"cell_type":"markdown","source":"### define utilities","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def set_seed(seed: int = 42):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)  # type: ignore\n#     torch.backends.cudnn.deterministic = True  # type: ignore\n#     torch.backends.cudnn.benchmark = True  # type: ignore\n    \n\n@contextmanager\ndef timer(name: str) -> None:\n    \"\"\"Timer Util\"\"\"\n    t0 = time.time()\n    print(\"[{}] start\".format(name))\n    yield\n    print(\"[{}] done in {:.0f} s\".format(name, time.time() - t0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# logger = get_logger(\"main.log\")\nset_seed(1213)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### read data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"ROOT = Path.cwd().parent\nINPUT_ROOT = ROOT / \"input\"\nRAW_DATA = INPUT_ROOT / \"birdsong-recognition\"\nTRAIN_AUDIO_DIR = RAW_DATA / \"train_audio\"\n# TRAIN_RESAMPLED_AUDIO_DIRS = [\n#   INPUT_ROOT / \"birdsong-resampled-train-audio-{:0>2}\".format(i)  for i in range(5)\n# ]\nTEST_AUDIO_DIR = RAW_DATA / \"test_audio\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(RAW_DATA / \"train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if not TEST_AUDIO_DIR.exists():\n    TEST_AUDIO_DIR = INPUT_ROOT / \"birdcall-check\" / \"test_audio\"\n    test = pd.read_csv(INPUT_ROOT / \"birdcall-check\" / \"test.csv\")\nelse:\n    test = pd.read_csv(RAW_DATA / \"test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv(\"../input/birdsong-recognition/sample_submission.csv\")\nsub.to_csv(\"submission.csv\", index=False)  # this will be overwritten if everything goes well","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### set parameters","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"TARGET_SR = 32000\n\nmodel_config = {\n    \"base_model_name\": \"resnet50\",\n    \"pretrained\": False,\n    \"num_classes\": 264,\n    \"trained_weights\": '/kaggle/input/001resnet50/best.pth'\n}\n\nmelspectrogram_parameters = {\n    \"n_mels\": 128,\n    \"fmin\": 20,\n    \"fmax\": 16000\n}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Definition","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Dataset\n\nFor `site_3`, I decided to use the same procedure as I did for `site_1` and `site_2`, which is, crop 5 seconds out of the clip and provide prediction on that short clip.\nThe only difference is that I crop 5 seconds short clip from start to the end of the `site_3` clip and aggeregate predictions for each short clip after I did prediction for all those short clips.","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"BIRD_CODE = {\n    'aldfly': 0, 'ameavo': 1, 'amebit': 2, 'amecro': 3, 'amegfi': 4,\n    'amekes': 5, 'amepip': 6, 'amered': 7, 'amerob': 8, 'amewig': 9,\n    'amewoo': 10, 'amtspa': 11, 'annhum': 12, 'astfly': 13, 'baisan': 14,\n    'baleag': 15, 'balori': 16, 'banswa': 17, 'barswa': 18, 'bawwar': 19,\n    'belkin1': 20, 'belspa2': 21, 'bewwre': 22, 'bkbcuc': 23, 'bkbmag1': 24,\n    'bkbwar': 25, 'bkcchi': 26, 'bkchum': 27, 'bkhgro': 28, 'bkpwar': 29,\n    'bktspa': 30, 'blkpho': 31, 'blugrb1': 32, 'blujay': 33, 'bnhcow': 34,\n    'boboli': 35, 'bongul': 36, 'brdowl': 37, 'brebla': 38, 'brespa': 39,\n    'brncre': 40, 'brnthr': 41, 'brthum': 42, 'brwhaw': 43, 'btbwar': 44,\n    'btnwar': 45, 'btywar': 46, 'buffle': 47, 'buggna': 48, 'buhvir': 49,\n    'bulori': 50, 'bushti': 51, 'buwtea': 52, 'buwwar': 53, 'cacwre': 54,\n    'calgul': 55, 'calqua': 56, 'camwar': 57, 'cangoo': 58, 'canwar': 59,\n    'canwre': 60, 'carwre': 61, 'casfin': 62, 'caster1': 63, 'casvir': 64,\n    'cedwax': 65, 'chispa': 66, 'chiswi': 67, 'chswar': 68, 'chukar': 69,\n    'clanut': 70, 'cliswa': 71, 'comgol': 72, 'comgra': 73, 'comloo': 74,\n    'commer': 75, 'comnig': 76, 'comrav': 77, 'comred': 78, 'comter': 79,\n    'comyel': 80, 'coohaw': 81, 'coshum': 82, 'cowscj1': 83, 'daejun': 84,\n    'doccor': 85, 'dowwoo': 86, 'dusfly': 87, 'eargre': 88, 'easblu': 89,\n    'easkin': 90, 'easmea': 91, 'easpho': 92, 'eastow': 93, 'eawpew': 94,\n    'eucdov': 95, 'eursta': 96, 'evegro': 97, 'fiespa': 98, 'fiscro': 99,\n    'foxspa': 100, 'gadwal': 101, 'gcrfin': 102, 'gnttow': 103, 'gnwtea': 104,\n    'gockin': 105, 'gocspa': 106, 'goleag': 107, 'grbher3': 108, 'grcfly': 109,\n    'greegr': 110, 'greroa': 111, 'greyel': 112, 'grhowl': 113, 'grnher': 114,\n    'grtgra': 115, 'grycat': 116, 'gryfly': 117, 'haiwoo': 118, 'hamfly': 119,\n    'hergul': 120, 'herthr': 121, 'hoomer': 122, 'hoowar': 123, 'horgre': 124,\n    'horlar': 125, 'houfin': 126, 'houspa': 127, 'houwre': 128, 'indbun': 129,\n    'juntit1': 130, 'killde': 131, 'labwoo': 132, 'larspa': 133, 'lazbun': 134,\n    'leabit': 135, 'leafly': 136, 'leasan': 137, 'lecthr': 138, 'lesgol': 139,\n    'lesnig': 140, 'lesyel': 141, 'lewwoo': 142, 'linspa': 143, 'lobcur': 144,\n    'lobdow': 145, 'logshr': 146, 'lotduc': 147, 'louwat': 148, 'macwar': 149,\n    'magwar': 150, 'mallar3': 151, 'marwre': 152, 'merlin': 153, 'moublu': 154,\n    'mouchi': 155, 'moudov': 156, 'norcar': 157, 'norfli': 158, 'norhar2': 159,\n    'normoc': 160, 'norpar': 161, 'norpin': 162, 'norsho': 163, 'norwat': 164,\n    'nrwswa': 165, 'nutwoo': 166, 'olsfly': 167, 'orcwar': 168, 'osprey': 169,\n    'ovenbi1': 170, 'palwar': 171, 'pasfly': 172, 'pecsan': 173, 'perfal': 174,\n    'phaino': 175, 'pibgre': 176, 'pilwoo': 177, 'pingro': 178, 'pinjay': 179,\n    'pinsis': 180, 'pinwar': 181, 'plsvir': 182, 'prawar': 183, 'purfin': 184,\n    'pygnut': 185, 'rebmer': 186, 'rebnut': 187, 'rebsap': 188, 'rebwoo': 189,\n    'redcro': 190, 'redhea': 191, 'reevir1': 192, 'renpha': 193, 'reshaw': 194,\n    'rethaw': 195, 'rewbla': 196, 'ribgul': 197, 'rinduc': 198, 'robgro': 199,\n    'rocpig': 200, 'rocwre': 201, 'rthhum': 202, 'ruckin': 203, 'rudduc': 204,\n    'rufgro': 205, 'rufhum': 206, 'rusbla': 207, 'sagspa1': 208, 'sagthr': 209,\n    'savspa': 210, 'saypho': 211, 'scatan': 212, 'scoori': 213, 'semplo': 214,\n    'semsan': 215, 'sheowl': 216, 'shshaw': 217, 'snobun': 218, 'snogoo': 219,\n    'solsan': 220, 'sonspa': 221, 'sora': 222, 'sposan': 223, 'spotow': 224,\n    'stejay': 225, 'swahaw': 226, 'swaspa': 227, 'swathr': 228, 'treswa': 229,\n    'truswa': 230, 'tuftit': 231, 'tunswa': 232, 'veery': 233, 'vesspa': 234,\n    'vigswa': 235, 'warvir': 236, 'wesblu': 237, 'wesgre': 238, 'weskin': 239,\n    'wesmea': 240, 'wessan': 241, 'westan': 242, 'wewpew': 243, 'whbnut': 244,\n    'whcspa': 245, 'whfibi': 246, 'whtspa': 247, 'whtswi': 248, 'wilfly': 249,\n    'wilsni1': 250, 'wiltur': 251, 'winwre3': 252, 'wlswar': 253, 'wooduc': 254,\n    'wooscj2': 255, 'woothr': 256, 'y00475': 257, 'yebfly': 258, 'yebsap': 259,\n    'yehbla': 260, 'yelwar': 261, 'yerwar': 262, 'yetvir': 263\n}\n\nINV_BIRD_CODE = {v: k for k, v in BIRD_CODE.items()}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def mono_to_color(X: np.ndarray,\n                  mean=None,\n                  std=None,\n                  norm_max=None,\n                  norm_min=None,\n                  eps=1e-6):\n    \"\"\"\n    Code from https://www.kaggle.com/daisukelab/creating-fat2019-preprocessed-data\n    \"\"\"\n    # Stack X as [X,X,X]\n    X = np.stack([X, X, X], axis=-1)\n\n    # Standardize\n    mean = mean or X.mean()\n    X = X - mean\n    std = std or X.std()\n    Xstd = X / (std + eps)\n    _min, _max = Xstd.min(), Xstd.max()\n    norm_max = norm_max or _max\n    norm_min = norm_min or _min\n    if (_max - _min) > eps:\n        # Normalize to [0, 255]\n        V = Xstd\n        V[V < norm_min] = norm_min\n        V[V > norm_max] = norm_max\n        V = 255 * (V - norm_min) / (norm_max - norm_min)\n        V = V.astype(np.uint8)\n    else:\n        # Just zero\n        V = np.zeros_like(Xstd, dtype=np.uint8)\n    return V\n\n\nclass TestDataset(data.Dataset):\n    def __init__(self, df: pd.DataFrame, clip: np.ndarray,\n                 img_size=224, melspectrogram_parameters={}):\n        self.df = df\n        self.clip = clip\n        self.img_size = img_size\n        self.melspectrogram_parameters = melspectrogram_parameters\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx: int):\n        SR = 32000\n        sample = self.df.loc[idx, :]\n        site = sample.site\n        row_id = sample.row_id\n        \n        if site == \"site_3\":\n            y = self.clip.astype(np.float32)\n            len_y = len(y)\n            start = 0\n            end = SR * 5\n            images = []\n            while len_y > start:\n                y_batch = y[start:end].astype(np.float32)\n                if len(y_batch) != (SR * 5):\n                    break\n                start = end\n                end = end + SR * 5\n                \n                melspec = librosa.feature.melspectrogram(y_batch,\n                                                         sr=SR,\n                                                         **self.melspectrogram_parameters)\n                melspec = librosa.power_to_db(melspec).astype(np.float32)\n                image = mono_to_color(melspec)\n                height, width, _ = image.shape\n                image = cv2.resize(image, (int(width * self.img_size / height), self.img_size))\n                image = np.moveaxis(image, 2, 0)\n                image = (image / 255.0).astype(np.float32)\n                images.append(image)\n            images = np.asarray(images)\n            return images, row_id, site\n        else:\n            end_seconds = int(sample.seconds)\n            start_seconds = int(end_seconds - 5)\n            \n            start_index = SR * start_seconds\n            end_index = SR * end_seconds\n            \n            y = self.clip[start_index:end_index].astype(np.float32)\n\n            melspec = librosa.feature.melspectrogram(y, sr=SR, **self.melspectrogram_parameters)\n            melspec = librosa.power_to_db(melspec).astype(np.float32)\n\n            image = mono_to_color(melspec)\n            height, width, _ = image.shape\n            image = cv2.resize(image, (int(width * self.img_size / height), self.img_size))\n            image = np.moveaxis(image, 2, 0)\n            image = (image / 255.0).astype(np.float32)\n\n            return image, row_id, site","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### load model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# just copy from: https://github.com/koukyo1994/kaggle-birdcall-resnet-baseline-training/blob/master/src/models.py#L7\nclass ResNet(nn.Module):\n    def __init__(self, base_model_name: str, pretrained=False,\n                 num_classes=264):\n        super().__init__()\n        base_model = models.__getattribute__(base_model_name)(\n            pretrained=pretrained)\n        layers = list(base_model.children())[:-2]\n        layers.append(nn.AdaptiveMaxPool2d(1))\n        self.encoder = nn.Sequential(*layers)\n\n        in_features = base_model.fc.in_features\n        self.classifier = nn.Sequential(\n            nn.Linear(in_features, 1024), nn.ReLU(), nn.Dropout(p=0.2),\n            nn.Linear(1024, 1024), nn.ReLU(), nn.Dropout(p=0.2),\n            nn.Linear(1024, num_classes))\n\n    def forward(self, x):\n        batch_size = x.size(0)\n        x = self.encoder(x)\n        x = x.view(batch_size, -1)\n        x = self.classifier(x)\n        multiclass_proba = F.softmax(x, dim=1)\n        multilabel_proba = F.sigmoid(x)\n        return {\n            \"logits\": x,\n            \"multiclass_proba\": multiclass_proba,\n            \"multilabel_proba\": multilabel_proba\n        }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_model(args: tp.Dict):\n    \n    model = ResNet('resnet50', num_classes=264)\n    state_dict = torch.load(args[\"trained_weights\"])\n    model.load_state_dict(state_dict['model_state_dict'])\n    device = torch.device(\"cuda\")\n    model.to(device)\n    model.eval()\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Prediction loop","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def prediction_for_clip(test_df: pd.DataFrame, \n                        clip: np.ndarray, \n                        model: ResNet, \n                        mel_params: dict, \n                        threshold=0.5):\n\n    dataset = TestDataset(df=test_df, \n                          clip=clip,\n                          img_size=224,\n                          melspectrogram_parameters=mel_params)\n    loader = data.DataLoader(dataset, batch_size=1, shuffle=False)\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    \n    model.eval()\n    prediction_dict = {}\n    for image, row_id, site in progress_bar(loader):\n        site = site[0]\n        row_id = row_id[0]\n        if site in {\"site_1\", \"site_2\"}:\n            image = image.to(device)\n\n            with torch.no_grad():\n                #prediction = F.sigmoid(model(image)['logits'])\n                prediction = model(image)[\"multilabel_proba\"]\n                proba = prediction.detach().cpu().numpy().reshape(-1)\n\n            events = proba >= threshold\n            labels = np.argwhere(events).reshape(-1).tolist()\n\n        else:\n            # to avoid prediction on large batch\n            image = image.squeeze(0)\n            batch_size = 16\n            whole_size = image.size(0)\n            if whole_size % batch_size == 0:\n                n_iter = whole_size // batch_size\n            else:\n                n_iter = whole_size // batch_size + 1\n                \n            all_events = set()\n            for batch_i in range(n_iter):\n                batch = image[batch_i * batch_size:(batch_i + 1) * batch_size]\n                if batch.ndim == 3:\n                    batch = batch.unsqueeze(0)\n\n                batch = batch.to(device)\n                with torch.no_grad():\n                    #prediction = F.sigmoid(model(batch)['logits'])\n                    prediction = model(batch)[\"multilabel_proba\"]\n                    proba = prediction.detach().cpu().numpy()\n                    \n                events = proba >= threshold\n                for i in range(len(events)):\n                    event = events[i, :]\n                    labels = np.argwhere(event).reshape(-1).tolist()\n                    for label in labels:\n                        all_events.add(label)\n                        \n            labels = list(all_events)\n        if len(labels) == 0:\n            prediction_dict[row_id] = \"nocall\"\n        else:\n            labels_str_list = list(map(lambda x: INV_BIRD_CODE[x], labels))\n            label_string = \" \".join(labels_str_list)\n            prediction_dict[row_id] = label_string\n    return prediction_dict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def prediction(test_df: pd.DataFrame,\n               test_audio: Path,\n               model_config: dict,\n               mel_params: dict,\n               target_sr: int,\n               threshold=0.5):\n    model = get_model(model_config)\n    unique_audio_id = test_df.audio_id.unique()\n\n    warnings.filterwarnings(\"ignore\")\n    prediction_dfs = []\n    for audio_id in unique_audio_id:\n        with timer(f\"Loading {audio_id}\"):\n            clip, _ = librosa.load(test_audio / (audio_id + \".mp3\"),\n                                   sr=target_sr,\n                                   mono=True,\n                                   res_type=\"kaiser_fast\")\n        \n        test_df_for_audio_id = test_df.query(\n            f\"audio_id == '{audio_id}'\").reset_index(drop=True)\n        with timer(f\"Prediction on {audio_id}\"):\n            prediction_dict = prediction_for_clip(test_df_for_audio_id,\n                                                  clip=clip,\n                                                  model=model,\n                                                  mel_params=mel_params,\n                                                  threshold=threshold)\n        row_id = list(prediction_dict.keys())\n        birds = list(prediction_dict.values())\n        prediction_df = pd.DataFrame({\n            \"row_id\": row_id,\n            \"birds\": birds\n        })\n        prediction_dfs.append(prediction_df)\n    \n    prediction_df = pd.concat(prediction_dfs, axis=0, sort=False).reset_index(drop=True)\n    return prediction_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Prediction","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"submission = prediction(test_df=test,\n                        test_audio=TEST_AUDIO_DIR,\n                        model_config=model_config,\n                        mel_params=melspectrogram_parameters,\n                        target_sr=TARGET_SR,\n                        threshold=0.6)\nsubmission.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"submission","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"seems like 0.568 it's not easy to beat. haha  \nThank you for reading my kernel! plz give me a upvote.","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}