{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Multi-label dataset\n\nThis notebook provides a multi-label dataset in pytorch, with extra labels extracted from `secondary_labels` or `background` meta-data in `train.csv`. Also, there are some simple statistics about multi-labels.  \n\nSince the test dataset is multi-labeled, maybe training with multi-labels can help. But the main problem for this competition is noise and the exact time range corresponding to each label, so this may not be that helpful.\n\n### References:  \nMulti-label idea: https://www.kaggle.com/maxwell110/mluticlass-to-muiltilabel  \nDataset scripts: https://www.kaggle.com/ttahara/training-birdsong-baseline-resnest50-fast  ","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport gc\nimport time\nimport shutil\nimport random\nimport warnings\nimport typing as tp\nfrom pathlib import Path\nfrom glob import glob\nfrom datetime import datetime\n\nimport cv2\nimport librosa\nimport audioread\nimport soundfile as sf\nimport matplotlib.pyplot as plt\n\nimport numpy as np\nimport pandas as pd\n\nfrom sklearn.metrics import f1_score, accuracy_score\nfrom sklearn.model_selection import StratifiedKFold\n\nimport torch","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"VIS = True\n\nN_SPLITS = 5\nFOLD = 0\nIMSIZE = 224\nWORKSPACE = '.'\nSEED = 42","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ROOT = Path.cwd().parent\nINPUT_ROOT = ROOT / \"input\"\nRAW_DATA = INPUT_ROOT / \"birdsong-recognition\"\nTRAIN_AUDIO_DIR = RAW_DATA / \"train_audio\"\nTRAIN_RESAMPLED_AUDIO_DIRS = [INPUT_ROOT / \"birdsong-resampled-train-audio-{:0>2}\".format(i)  for i in range(5)]\nTEST_AUDIO_DIR = RAW_DATA / \"test_audio\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(TRAIN_RESAMPLED_AUDIO_DIRS[0] / \"train_mod.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# use sci_name to extract secondary labels  \nSPS_CODE = dict(zip(train.sci_name, train.ebird_code))\nINV_SPS_CODE = {v: k for k, v in SPS_CODE.items()}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BIRD_CODE = {\n    'aldfly': 0, 'ameavo': 1, 'amebit': 2, 'amecro': 3, 'amegfi': 4,\n    'amekes': 5, 'amepip': 6, 'amered': 7, 'amerob': 8, 'amewig': 9,\n    'amewoo': 10, 'amtspa': 11, 'annhum': 12, 'astfly': 13, 'baisan': 14,\n    'baleag': 15, 'balori': 16, 'banswa': 17, 'barswa': 18, 'bawwar': 19,\n    'belkin1': 20, 'belspa2': 21, 'bewwre': 22, 'bkbcuc': 23, 'bkbmag1': 24,\n    'bkbwar': 25, 'bkcchi': 26, 'bkchum': 27, 'bkhgro': 28, 'bkpwar': 29,\n    'bktspa': 30, 'blkpho': 31, 'blugrb1': 32, 'blujay': 33, 'bnhcow': 34,\n    'boboli': 35, 'bongul': 36, 'brdowl': 37, 'brebla': 38, 'brespa': 39,\n    'brncre': 40, 'brnthr': 41, 'brthum': 42, 'brwhaw': 43, 'btbwar': 44,\n    'btnwar': 45, 'btywar': 46, 'buffle': 47, 'buggna': 48, 'buhvir': 49,\n    'bulori': 50, 'bushti': 51, 'buwtea': 52, 'buwwar': 53, 'cacwre': 54,\n    'calgul': 55, 'calqua': 56, 'camwar': 57, 'cangoo': 58, 'canwar': 59,\n    'canwre': 60, 'carwre': 61, 'casfin': 62, 'caster1': 63, 'casvir': 64,\n    'cedwax': 65, 'chispa': 66, 'chiswi': 67, 'chswar': 68, 'chukar': 69,\n    'clanut': 70, 'cliswa': 71, 'comgol': 72, 'comgra': 73, 'comloo': 74,\n    'commer': 75, 'comnig': 76, 'comrav': 77, 'comred': 78, 'comter': 79,\n    'comyel': 80, 'coohaw': 81, 'coshum': 82, 'cowscj1': 83, 'daejun': 84,\n    'doccor': 85, 'dowwoo': 86, 'dusfly': 87, 'eargre': 88, 'easblu': 89,\n    'easkin': 90, 'easmea': 91, 'easpho': 92, 'eastow': 93, 'eawpew': 94,\n    'eucdov': 95, 'eursta': 96, 'evegro': 97, 'fiespa': 98, 'fiscro': 99,\n    'foxspa': 100, 'gadwal': 101, 'gcrfin': 102, 'gnttow': 103, 'gnwtea': 104,\n    'gockin': 105, 'gocspa': 106, 'goleag': 107, 'grbher3': 108, 'grcfly': 109,\n    'greegr': 110, 'greroa': 111, 'greyel': 112, 'grhowl': 113, 'grnher': 114,\n    'grtgra': 115, 'grycat': 116, 'gryfly': 117, 'haiwoo': 118, 'hamfly': 119,\n    'hergul': 120, 'herthr': 121, 'hoomer': 122, 'hoowar': 123, 'horgre': 124,\n    'horlar': 125, 'houfin': 126, 'houspa': 127, 'houwre': 128, 'indbun': 129,\n    'juntit1': 130, 'killde': 131, 'labwoo': 132, 'larspa': 133, 'lazbun': 134,\n    'leabit': 135, 'leafly': 136, 'leasan': 137, 'lecthr': 138, 'lesgol': 139,\n    'lesnig': 140, 'lesyel': 141, 'lewwoo': 142, 'linspa': 143, 'lobcur': 144,\n    'lobdow': 145, 'logshr': 146, 'lotduc': 147, 'louwat': 148, 'macwar': 149,\n    'magwar': 150, 'mallar3': 151, 'marwre': 152, 'merlin': 153, 'moublu': 154,\n    'mouchi': 155, 'moudov': 156, 'norcar': 157, 'norfli': 158, 'norhar2': 159,\n    'normoc': 160, 'norpar': 161, 'norpin': 162, 'norsho': 163, 'norwat': 164,\n    'nrwswa': 165, 'nutwoo': 166, 'olsfly': 167, 'orcwar': 168, 'osprey': 169,\n    'ovenbi1': 170, 'palwar': 171, 'pasfly': 172, 'pecsan': 173, 'perfal': 174,\n    'phaino': 175, 'pibgre': 176, 'pilwoo': 177, 'pingro': 178, 'pinjay': 179,\n    'pinsis': 180, 'pinwar': 181, 'plsvir': 182, 'prawar': 183, 'purfin': 184,\n    'pygnut': 185, 'rebmer': 186, 'rebnut': 187, 'rebsap': 188, 'rebwoo': 189,\n    'redcro': 190, 'redhea': 191, 'reevir1': 192, 'renpha': 193, 'reshaw': 194,\n    'rethaw': 195, 'rewbla': 196, 'ribgul': 197, 'rinduc': 198, 'robgro': 199,\n    'rocpig': 200, 'rocwre': 201, 'rthhum': 202, 'ruckin': 203, 'rudduc': 204,\n    'rufgro': 205, 'rufhum': 206, 'rusbla': 207, 'sagspa1': 208, 'sagthr': 209,\n    'savspa': 210, 'saypho': 211, 'scatan': 212, 'scoori': 213, 'semplo': 214,\n    'semsan': 215, 'sheowl': 216, 'shshaw': 217, 'snobun': 218, 'snogoo': 219,\n    'solsan': 220, 'sonspa': 221, 'sora': 222, 'sposan': 223, 'spotow': 224,\n    'stejay': 225, 'swahaw': 226, 'swaspa': 227, 'swathr': 228, 'treswa': 229,\n    'truswa': 230, 'tuftit': 231, 'tunswa': 232, 'veery': 233, 'vesspa': 234,\n    'vigswa': 235, 'warvir': 236, 'wesblu': 237, 'wesgre': 238, 'weskin': 239,\n    'wesmea': 240, 'wessan': 241, 'westan': 242, 'wewpew': 243, 'whbnut': 244,\n    'whcspa': 245, 'whfibi': 246, 'whtspa': 247, 'whtswi': 248, 'wilfly': 249,\n    'wilsni1': 250, 'wiltur': 251, 'winwre3': 252, 'wlswar': 253, 'wooduc': 254,\n    'wooscj2': 255, 'woothr': 256, 'y00475': 257, 'yebfly': 258, 'yebsap': 259,\n    'yehbla': 260, 'yelwar': 261, 'yerwar': 262, 'yetvir': 263\n}\n\nINV_BIRD_CODE = {v: k for k, v in BIRD_CODE.items()}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true,"collapsed":true},"cell_type":"code","source":"import re\n\n# background\ndef get_sps_from_bg(s):\n    if type(s) != str: s = str(s)\n    return re.findall('\\((.*?)\\)', s)\n\n# secondary\ndef get_sps_from_sec(l):\n    return [re.split('_', s[1:-1])[0] for s in re.split(',\\s*', l[1:-1])]\n\ndef map_sps_to_code(l):\n    res = []\n    for s in l:\n        if s == '':\n            continue\n        elif s in SPS_CODE:\n            res.append(SPS_CODE[s])\n        else:\n            # there are species not included in the classification targets (thus no ebird_code)\n            print(f\"{s} doesn't exist.\")\n    return res\n\ntrain['labels_bg'] = train['background'].apply(get_sps_from_bg).apply(map_sps_to_code)\ntrain['labels_sec'] = train['secondary_labels'].apply(get_sps_from_sec).apply(map_sps_to_code)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train[['ebird_code', 'labels_sec', 'labels_bg']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Simple statistics\nprint(f\"Total Entries                  : {len(train)}\")\nprint(f\"Entries with secondary_labels  : {len(train[train['labels_sec'].str.len() > 0])}\")\nprint(f\"Entries with background        : {len(train[train['labels_bg'].str.len() > 0])}\")\nprint(f\"Average secondary_labels length: {train['labels_sec'].apply(len).mean()}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"# Simple check shows some bg has more labels than sec. (most are the same)\nfor i in range(len(train)):\n    if set(train.labels_bg.iloc[i]) != set(train.labels_sec.iloc[i]):\n        print(train.labels_bg.iloc[i], train.labels_sec.iloc[i])","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"PERIOD = 5 # 5s\n\ndef mono_to_color(X, mean=None, std=None, eps=1e-6):\n    \n    X = np.stack([X, X, X], axis=-1)\n\n    # Standardize\n    mean = mean or X.mean()\n    std = std or X.std()\n    X = (X - mean)/(std + eps)\n    \n    # Normalization\n    mx = X.max()\n    mn = X.min()\n    if mx - mn > eps:\n        X = (X - X.min()) / (X.max() - X.min())\n        X = (X * 255).astype(np.uint8)\n    else:\n        X = np.zeros_like(X, dtype=np.uint8)\n        \n    return X\n\n\nclass SpectrogramDataset(torch.utils.data.Dataset):\n    def __init__(self, file_list, img_size=IMSIZE, waveform_transforms=None, spectrogram_transforms=None):\n        self.file_list = file_list  # list of list: [file_path, ebird_code]\n        self.img_size = img_size\n        self.waveform_transforms = waveform_transforms\n        self.spectrogram_transforms = spectrogram_transforms\n\n    def __len__(self):\n        return len(self.file_list)\n\n    def __getitem__(self, idx: int):\n        wav_path, ebird_code, labels_sec = self.file_list[idx]\n\n        y, sr = sf.read(wav_path) # sample rate\n\n        if self.waveform_transforms:\n            y = self.waveform_transforms(y)\n        else:\n            len_y = len(y)\n            effective_length = sr * PERIOD\n            # padding\n            if len_y < effective_length:\n                new_y = np.zeros(effective_length, dtype=y.dtype)\n                start = np.random.randint(effective_length - len_y)\n                new_y[start:start + len_y] = y\n                y = new_y.astype(np.float32)\n            # random clip\n            elif len_y > effective_length:\n                start = np.random.randint(len_y - effective_length)\n                y = y[start:start + effective_length].astype(np.float32)\n            else:\n                y = y.astype(np.float32)\n\n        melspec = librosa.feature.melspectrogram(y, sr=sr, n_mels=128, fmin=20, fmax=16000)\n        melspec = librosa.power_to_db(melspec).astype(np.float32)\n\n        if self.spectrogram_transforms:\n            melspec = self.spectrogram_transforms(melspec)\n        else:\n            pass\n        \n        image = mono_to_color(melspec)  \n            \n        height, width, _ = image.shape\n        image = cv2.resize(image, (int(width * self.img_size / height), self.img_size))\n        image = np.moveaxis(image, 2, 0)\n        image = (image / 255.0).astype(np.float32)\n        \n        # one-hot label\n        labels = np.zeros(len(BIRD_CODE), dtype=\"f\")\n        labels[BIRD_CODE[ebird_code]] = 1\n        for sec in labels_sec:\n            labels[BIRD_CODE[sec]] = 1\n\n        return image, labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tmp_list = []\nfor audio_d in TRAIN_RESAMPLED_AUDIO_DIRS:\n    if not audio_d.exists():\n        continue\n    for ebird_d in audio_d.iterdir():\n        if ebird_d.is_file():\n            continue\n        for wav_f in ebird_d.iterdir():\n            tmp_list.append([ebird_d.name, wav_f.name, wav_f.as_posix()])\n            \ntrain_wav_path_exist = pd.DataFrame(tmp_list, columns=[\"ebird_code\", \"resampled_filename\", \"file_path\"])\n\ndel tmp_list\n\ntrain_all = pd.merge(train, train_wav_path_exist, on=[\"ebird_code\", \"resampled_filename\"], how=\"inner\")\n\nprint(train.shape)\nprint(train_wav_path_exist.shape)\nprint(train_all.shape)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":false,"trusted":true},"cell_type":"code","source":"skf = StratifiedKFold(n_splits=N_SPLITS, random_state=SEED, shuffle=True)\n\ntrain_all[\"fold\"] = -1\nfor fold_id, (train_index, val_index) in enumerate(skf.split(train_all, train_all[\"ebird_code\"])):\n    train_all.iloc[val_index, -1] = fold_id\n    \n# # check the propotion\nfold_proportion = pd.pivot_table(train_all, index=\"ebird_code\", columns=\"fold\", values=\"xc_id\", aggfunc=len)\nprint(fold_proportion.shape, fold_proportion)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"use_fold = FOLD\ntrain_file_list = train_all.query(\"fold != @use_fold\")[[\"file_path\", \"ebird_code\", \"labels_sec\"]].values.tolist()\nval_file_list = train_all.query(\"fold == @use_fold\")[[\"file_path\", \"ebird_code\", \"labels_sec\"]].values.tolist()\n\nprint(\"[fold {}] train: {}, val: {}\".format(use_fold, len(train_file_list), len(val_file_list)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = SpectrogramDataset(train_file_list)\n\nfor i in range(3):\n    img, lbl = train_dataset[i]\n    print(np.argwhere(lbl))\n    plt.imshow(img.transpose(1,2,0))\n    plt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}