{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install ../input/torchlibrosa/torchlibrosa-0.0.5-py3-none-any.whl > /dev/null","metadata":{"execution":{"iopub.status.busy":"2022-06-21T09:42:49.162639Z","iopub.execute_input":"2022-06-21T09:42:49.162963Z","iopub.status.idle":"2022-06-21T09:42:57.920438Z","shell.execute_reply.started":"2022-06-21T09:42:49.162879Z","shell.execute_reply":"2022-06-21T09:42:57.919518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install colorednoise > /dev/null","metadata":{"execution":{"iopub.status.busy":"2022-06-21T09:42:57.922658Z","iopub.execute_input":"2022-06-21T09:42:57.922924Z","iopub.status.idle":"2022-06-21T09:43:07.395474Z","shell.execute_reply.started":"2022-06-21T09:42:57.922887Z","shell.execute_reply":"2022-06-21T09:43:07.394591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport ast\nimport numpy as np\nimport warnings\nfrom sklearn import metrics\nfrom torch.utils.data import Dataset\nimport pandas as pd\nimport torchaudio\nimport glob\nimport colorednoise as cn\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold\nimport os\nfrom torch.utils.data import Dataset\nimport pandas as pd\nimport torchaudio\nfrom sklearn.preprocessing import MultiLabelBinarizer\nfrom fastai.vision.all import *\nfrom typing import Optional,Tuple,List\nsys.path.append('../input/timm-pytorch-image-models/pytorch-image-models-master')\nimport timm\nfrom timm import create_model\nimport audioread\nimport librosa\nimport soundfile as sf\nfrom albumentations.core.transforms_interface import ImageOnlyTransform\nfrom torchlibrosa.stft import LogmelFilterBank, Spectrogram\nfrom torchlibrosa.augmentation import SpecAugmentation\n\nimport albumentations as A\nimport albumentations.pytorch.transforms as T\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2022-06-21T09:43:07.397405Z","iopub.execute_input":"2022-06-21T09:43:07.397671Z","iopub.status.idle":"2022-06-21T09:43:14.150331Z","shell.execute_reply.started":"2022-06-21T09:43:07.397638Z","shell.execute_reply":"2022-06-21T09:43:14.149324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AllParams:\n    seed = 71\n    target_columns = 'afrsil1 akekee akepa1 akiapo akikik amewig aniani apapan arcter \\\n                      barpet bcnher belkin1 bkbplo bknsti bkwpet blkfra blknod bongul \\\n                      brant brnboo brnnod brnowl brtcur bubsan buffle bulpet burpar buwtea \\\n                      cacgoo1 calqua cangoo canvas caster1 categr chbsan chemun chukar cintea \\\n                      comgal1 commyn compea comsan comwax coopet crehon dunlin elepai ercfra eurwig \\\n                      fragul gadwal gamqua glwgul gnwtea golphe grbher3 grefri gresca gryfra gwfgoo \\\n                      hawama hawcoo hawcre hawgoo hawhaw hawpet1 hoomer houfin houspa hudgod iiwi incter1 \\\n                      jabwar japqua kalphe kauama laugul layalb lcspet leasan leater1 lessca lesyel lobdow lotjae \\\n                      madpet magpet1 mallar3 masboo mauala maupar merlin mitpar moudov norcar norhar2 normoc norpin \\\n                      norsho nutman oahama omao osprey pagplo palila parjae pecsan peflov perfal pibgre pomjae puaioh \\\n                      reccar redava redjun redpha1 refboo rempar rettro ribgul rinduc rinphe rocpig rorpar rudtur ruff \\\n                      saffin sander semplo sheowl shtsan skylar snogoo sooshe sooter1 sopsku1 sora spodov sposan \\\n                      towsol wantat1 warwhe1 wesmea wessan wetshe whfibi whiter whttro wiltur yebcar yefcan zebdov'.split()\n\n    img_size = 224 # 128\n    main_metric = \"epoch_f1_at_03\"\n    USE_SEC = 30\n    # Melspectrogram\n    period = 5\n    duration = period\n    n_mels = 224 # 128\n    fmin = 20\n    fmax = 16000\n    n_fft = 2048\n    hop_length = 512\n    sample_rate = 32000\n    sr = sample_rate\n    melspectrogram_parameters = {\n        \"n_mels\": 224, # 128,\n        \"fmin\": 20,\n        \"fmax\": 16000\n    }\n    \n    # train\n    epochs = 9\n    folds = [0] # [0, 1, 2, 3, 4]\n    N_FOLDS = 5\n    LR = 1e-3\n    ETA_MIN = 1e-6\n    WEIGHT_DECAY = 0.0001\n    batch_size = 32 # 32\n    base_model_name = \"tf_efficientnet_b0_ns\"\n    EARLY_STOPPING = True\n    DEBUG = False # True\n    EVALUATION = 'AUC'\n    apex = True\n    pooling = \"max\"\n    pretrained = True\n    num_classes = 152\n    in_channels = 3\n    mode = 'train'\n    mixup_or_cutmix_epochs = 6\n    mixup_or_cutmix = 'mixup'\n    current_epoch = 0\n    \nparams = AllParams","metadata":{"execution":{"iopub.status.busy":"2022-06-21T09:43:14.152682Z","iopub.execute_input":"2022-06-21T09:43:14.153162Z","iopub.status.idle":"2022-06-21T09:43:14.165541Z","shell.execute_reply.started":"2022-06-21T09:43:14.153115Z","shell.execute_reply":"2022-06-21T09:43:14.164723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\nOUTPUT_DIR = f'./'\nif not os.path.exists(OUTPUT_DIR):\n    os.makedirs(OUTPUT_DIR)\n   \n    \ndef set_seed(seed=42):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    \nset_seed(params.seed)\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"execution":{"iopub.status.busy":"2022-06-21T09:43:14.166785Z","iopub.execute_input":"2022-06-21T09:43:14.167294Z","iopub.status.idle":"2022-06-21T09:43:14.181299Z","shell.execute_reply.started":"2022-06-21T09:43:14.167252Z","shell.execute_reply":"2022-06-21T09:43:14.180582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# loading training data","metadata":{}},{"cell_type":"code","source":"import glob\n\nall_path = glob.glob('../input/birdclef-2022/train_audio/*/*.ogg')\n\nlen(all_path)","metadata":{"execution":{"iopub.status.busy":"2022-06-21T09:43:14.183222Z","iopub.execute_input":"2022-06-21T09:43:14.183770Z","iopub.status.idle":"2022-06-21T09:43:18.189696Z","shell.execute_reply.started":"2022-06-21T09:43:14.183731Z","shell.execute_reply":"2022-06-21T09:43:18.189011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import ast\n\n\ntrain = pd.read_csv('../input/birdclef-2022/train_metadata.csv')\n\n\ntrain['new_target'] = train['primary_label'] + ' ' + train['secondary_labels'].map(lambda x: ' '.join(ast.literal_eval(x)))\ntrain['len_new_target'] = train['new_target'].map(lambda x: len(x.split()))\n# train['len_new_target'].value_counts()\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-21T09:43:18.190996Z","iopub.execute_input":"2022-06-21T09:43:18.191433Z","iopub.status.idle":"2022-06-21T09:43:18.418122Z","shell.execute_reply.started":"2022-06-21T09:43:18.191394Z","shell.execute_reply":"2022-06-21T09:43:18.417304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path_df = pd.DataFrame(all_path, columns=['file_path'])\npath_df['filename'] = path_df['file_path'].map(lambda x: x.split('/')[-2]+'/'+x.split('/')[-1])\npath_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-21T09:43:18.419491Z","iopub.execute_input":"2022-06-21T09:43:18.419763Z","iopub.status.idle":"2022-06-21T09:43:18.448254Z","shell.execute_reply.started":"2022-06-21T09:43:18.419728Z","shell.execute_reply":"2022-06-21T09:43:18.447474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.merge(train, path_df, on='filename')\nprint(train.shape)\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-21T09:43:18.449527Z","iopub.execute_input":"2022-06-21T09:43:18.449901Z","iopub.status.idle":"2022-06-21T09:43:18.493135Z","shell.execute_reply.started":"2022-06-21T09:43:18.449858Z","shell.execute_reply":"2022-06-21T09:43:18.492428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train = train[train.primary_label.isin(scored_classes)]\n#train.reset_index(inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-06-21T09:43:18.496419Z","iopub.execute_input":"2022-06-21T09:43:18.496994Z","iopub.status.idle":"2022-06-21T09:43:18.501438Z","shell.execute_reply.started":"2022-06-21T09:43:18.496963Z","shell.execute_reply":"2022-06-21T09:43:18.500623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Fold = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\nfor n, (trn_index, val_index) in enumerate(Fold.split(train, train['primary_label'])):\n    train.loc[val_index, 'kfold'] = int(n)\ntrain['kfold'] = train['kfold'].astype(int)","metadata":{"execution":{"iopub.status.busy":"2022-06-21T09:43:18.502851Z","iopub.execute_input":"2022-06-21T09:43:18.503212Z","iopub.status.idle":"2022-06-21T09:43:18.536467Z","shell.execute_reply.started":"2022-06-21T09:43:18.503149Z","shell.execute_reply":"2022-06-21T09:43:18.535594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.to_csv('train_folds.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-06-21T09:43:18.537696Z","iopub.execute_input":"2022-06-21T09:43:18.538400Z","iopub.status.idle":"2022-06-21T09:43:18.757393Z","shell.execute_reply.started":"2022-06-21T09:43:18.538362Z","shell.execute_reply":"2022-06-21T09:43:18.756625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating dataset","metadata":{}},{"cell_type":"code","source":"class NoiseInjection(Transform):\n    def __init__(self, always_apply=False, p=0.5, max_noise_level=0.5):\n        self.noise_level = (0.0, max_noise_level)\n        self.p = p\n        self.always_apply = always_apply\n        \n    def encodes(self, signal: np.ndarray):\n        noise_level = np.random.uniform(*self.noise_level)\n        noise = np.random.randn(len(signal))\n        if self.always_apply == True:\n                signal = (signal + noise * noise_level).astype(signal.dtype)\n            \n        else:\n            if np.random.rand() < self.p:\n                signal = (signal + noise * noise_level).astype(signal.dtype) \n        return signal\n\n\nclass GaussianNoise(Transform):\n    def __init__(self, always_apply=False, p=0.5, min_snr=5, max_snr=20):\n        self.always_apply = always_apply\n        self.p = p\n        self.min_snr = min_snr\n        self.max_snr = max_snr\n\n    def encodes(self, signal: np.ndarray):\n        snr = np.random.uniform(self.min_snr, self.max_snr)\n        a_signal = np.sqrt(signal ** 2).max()\n        a_noise = a_signal / (10 ** (snr / 20))\n        white_noise = np.random.randn(len(signal))\n        a_white = np.sqrt(white_noise ** 2).max()\n        \n        if self.always_apply == True:\n            signal = (signal + white_noise * 1 / a_white * a_noise).astype(signal.dtype)\n            \n        else:\n            if np.random.rand() < self.p:\n                signal = (signal + white_noise * 1 / a_white * a_noise).astype(signal.dtype)\n        return signal\n\n\nclass PinkNoise(Transform):\n    def __init__(self, always_apply=False, p=0.5, min_snr=5, max_snr=20):\n        self.always_apply = always_apply\n        self.p = p\n        self.min_snr = min_snr\n        self.max_snr = max_snr\n\n    def encodes(self, signal: np.ndarray):\n        snr = np.random.uniform(self.min_snr, self.max_snr)\n        a_signal = np.sqrt(signal ** 2).max()\n        a_noise = a_signal / (10 ** (snr / 20))\n\n        pink_noise = cn.powerlaw_psd_gaussian(1, len(signal))\n        a_pink = np.sqrt(pink_noise ** 2).max()\n        if self.always_apply == True:\n            signal = (signal + pink_noise * 1 / a_pink * a_noise).astype(signal.dtype)\n            \n        else:\n            if np.random.rand() < self.p:\n                signal = (signal + pink_noise * 1 / a_pink * a_noise).astype(signal.dtype)\n        return signal\n\nclass RandomVolume(Transform):\n    def __init__(self,always_apply=False, p=0.2, limit=12):\n        self.limit = limit\n        self.p = p\n        self.always_apply = always_apply\n    def encodes(self, signal: np.ndarray):\n        db = np.random.uniform(-self.limit, self.limit)\n        if self.always_apply == True:\n            if db >= 0:\n                return self.volume_up(signal, db)\n            else:\n                return self.volume_down(signal, db)\n        else:\n            if np.random.rand() < self.p:\n                if db >= 0:\n                    return self.volume_up(signal, db)\n                else:\n                    return self.volume_down(signal, db)      \n            return signal\n\n    def volume_down(self,signal: np.ndarray, db: float):\n        signal = signal * self._db2float(-db)\n        return signal\n    \n    def volume_up(self,signal: np.ndarray, db: float):\n        signal = signal * self._db2float(db)\n        return signal\n    \n    def _db2float(self,db: float, amplitude=True):\n        if amplitude:\n            return 10 ** (db / 20)\n        else:\n            return 10 ** (db / 10)\n\nclass TimeShift(Transform):\n    def __init__(self, always_apply=False, p=0.5, max_shift_second=2,padding_mode=\"zero\"):\n        self.p = p\n        self.max_shift_second = max_shift_second\n        self.padding_mode = padding_mode\n        self.always_apply = always_apply\n        self.sr = params.sr\n        assert padding_mode in [\"replace\", \"zero\"], \"`padding_mode` must be either 'replace' or 'zero'\"\n        \n    def encodes(self, signal: np.ndarray):\n        shift = np.random.randint(-self.sr * self.max_shift_second, self.sr * self.max_shift_second)\n        augmented = np.roll(signal, shift)\n        if self.always_apply == True:\n            if self.padding_mode == \"zero\":\n                if shift > 0:\n                    augmented[:shift] = 0\n                else:\n                    augmented[shift:] = 0\n            return augmented\n        else:\n            if np.random.rand() < self.p:\n                if self.padding_mode == \"zero\":\n                    if shift > 0:\n                        augmented[:shift] = 0\n                    else:\n                        augmented[shift:] = 0\n                    return augmented\n        return signal   \n\nclass PitchShift(Transform):\n    def __init__(self, always_apply=False, p=0.5, max_steps=5):\n        self.p = p\n        self.always_apply = always_apply\n        self.max_steps = max_steps\n        self.sr = params.sr\n\n    def encodes(self, signal: np.ndarray):\n        n_steps = np.random.randint(-self.max_steps, self.max_steps)\n        if self.always_apply == True:\n            augmented = librosa.effects.pitch_shift(signal, sr=self.sr, n_steps=n_steps)\n            return augmented\n        elif np.random.rand() < self.p:\n            augmented = librosa.effects.pitch_shift(signal, sr=self.sr, n_steps=n_steps)\n            return augmented          \n        else:\n            return signal","metadata":{"execution":{"iopub.status.busy":"2022-06-21T09:43:18.758622Z","iopub.execute_input":"2022-06-21T09:43:18.758877Z","iopub.status.idle":"2022-06-21T09:43:18.792355Z","shell.execute_reply.started":"2022-06-21T09:43:18.758842Z","shell.execute_reply":"2022-06-21T09:43:18.791630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nmean = (0.485, 0.456, 0.406) # RGB\nstd = (0.229, 0.224, 0.225) # RGB\n\nalbu_transforms = {\n    'train' : A.Compose([\n            A.HorizontalFlip(p=0.5),\n            A.OneOf([\n                A.Cutout(max_h_size=5, max_w_size=16),\n                A.CoarseDropout(max_holes=4),\n            ], p=0.5),\n            A.Normalize(mean, std),\n    ]),\n    'valid' : A.Compose([\n            A.Normalize(mean, std),\n    ]),\n}","metadata":{"execution":{"iopub.status.busy":"2022-06-21T09:43:18.793589Z","iopub.execute_input":"2022-06-21T09:43:18.794121Z","iopub.status.idle":"2022-06-21T09:43:18.804574Z","shell.execute_reply.started":"2022-06-21T09:43:18.794083Z","shell.execute_reply":"2022-06-21T09:43:18.803009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class OneOf(Transform):\n    def __init__(self, transforms,p=0.5):\n        self.p = p\n        self.transforms = transforms\n        \n        transforms_ps = [t.p for t in transforms]\n        s = sum(transforms_ps)\n        self.transforms_ps = [t / s for t in transforms_ps]\n        \n    def encodes(self, signal: np.ndarray):\n        if params.mode == \"train\":\n            if self.transforms_ps and (random.random() < self.p):\n                random_state = np.random.RandomState(random.randint(0, 2 ** 32 - 1))\n                t = random_state.choice(self.transforms, p=self.transforms_ps)\n                signal = t(signal)\n\n            signal = RandomVolume(p=0.2, limit=12)(signal)\n        signal =  self.Normalize(signal=signal,p=1)\n        signal = np.concatenate([signal, signal, signal])[:params.duration * params.sr] \n\n        if params.mode == \"train\":\n            train = True\n        else:\n            train = False\n        signal = self.crop_or_pad(signal, params.duration * params.sr, sr=params.sr, train=train, probs=None)\n        return signal\n    \n    def crop_or_pad(self,signal, length, sr, train, probs=None):\n        if len(signal) <= length:\n            signal = np.concatenate([signal, np.zeros(length - len(signal))])\n        else:\n            if train == False:\n                start = 0\n            elif probs is None:\n                start = np.random.randint(len(signal) - length)\n            else:\n                start = (\n                        np.random.choice(np.arange(len(probs)), p=probs) + np.random.random()\n                )\n                start = int(sr * (start))\n\n            signal = signal[start: start + length]\n\n        return signal.astype(np.float32) \n    \n    def Normalize(self,signal,always_apply=False, p=1):\n        max_vol = np.abs(signal).max()\n        if always_apply == True:\n             signal_vol = signal * 1 / max_vol\n\n        else:\n            if np.random.rand() < p:\n                 signal_vol = signal * 1 / max_vol \n        return np.array(signal_vol)\n    \n    \n    \nclass get_audio_sample_path(Transform):\n    def encodes(self, x):\n        return x.file_path\n    \nclass get_audio_sample_label(Transform):       \n    def encodes(self,x):\n        labels = x.new_target\n        targets = np.zeros(len(params.target_columns), dtype=float)\n        for ebird_code in labels.split():\n            targets[params.target_columns.index(ebird_code)] = 1.0\n        return targets\n\nclass load_signal(Transform):\n    def __init__(self,device='cpu'):self.device=device\n    def encodes(self, x:str):\n        sr = params.sample_rate\n        signal, sr = sf.read(x, always_2d=True)\n        #signal = np.load(x)\n        signal = np.mean(signal, 1) # there is (X, 2) array\n        if len(signal) > sr * params.USE_SEC:\n            signal = signal[sr:-sr]\n            signal = signal[:sr * params.USE_SEC]\n\n        #if len(signal) > 0:\n        #    signal = signal[:params.duration*sr]\n        #else:\n        #    signal = np.array([0.0001 for x in range( params.USE_SEC*sr)])\n        return signal\n\n\nclass mel_spec(Transform):\n    def encodes(self, x):\n        melspec = librosa.feature.melspectrogram(\n            y=x, sr=params.sample_rate, n_mels=params.n_mels, fmin=params.fmin, fmax=params.fmax,\n        )\n        image = librosa.power_to_db(melspec).astype(np.float32)\n        image = self.mono_to_color(image)\n        image = image.astype(np.uint8)\n        # image = np.load(wav_path) # (224, 313, 3)\n        image = albu_transforms[params.mode](image=image)['image']\n        image = image.T\n        return image\n\n    def mono_to_color(self,X, eps=1e-6, mean=None, std=None):\n        X = np.stack([X, X, X], axis=-1)\n\n        # Standardize\n        mean = mean or X.mean()\n        std = std or X.std()\n        X = (X - mean) / (std + eps)\n\n        # Normalize to [0, 255]\n        _min, _max = X.min(), X.max()\n\n        if (_max - _min) > eps:\n            V = np.clip(X, _min, _max)\n            V = 255 * (V - _min) / (_max - _min)\n            V = V.astype(np.uint8)\n        else:\n            V = np.zeros_like(X, dtype=np.uint8)\n        return V","metadata":{"execution":{"iopub.status.busy":"2022-06-21T09:43:18.806083Z","iopub.execute_input":"2022-06-21T09:43:18.806716Z","iopub.status.idle":"2022-06-21T09:43:18.833450Z","shell.execute_reply.started":"2022-06-21T09:43:18.806669Z","shell.execute_reply":"2022-06-21T09:43:18.832699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if torch.cuda.is_available():\n        device = \"cuda\"\nelse:\n        device = \"cpu\"\nprint(f\"Using {device}\")","metadata":{"execution":{"iopub.status.busy":"2022-06-21T09:43:18.835863Z","iopub.execute_input":"2022-06-21T09:43:18.836060Z","iopub.status.idle":"2022-06-21T09:43:18.846432Z","shell.execute_reply.started":"2022-06-21T09:43:18.836037Z","shell.execute_reply":"2022-06-21T09:43:18.845383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_fold = 0\ndef get_dls(val_fold=0,bs=64):\n    splits = [train[train.kfold != val_fold].index.tolist() , train[train.kfold == val_fold].index.tolist()]\n    #x_tfms = [get_audio_sample_path,load_signal(device),resample_if_necessary(32000),\n    #          mix_down_if_necessary,cut_if_necessary(),right_pad_if_necessary,\n    #          mel_spec,mono_to_color]\n    x_tfms = [get_audio_sample_path,load_signal,OneOf([   \n                            NoiseInjection(p=1, max_noise_level=0.04),\n                            #GaussianNoise(p=1, min_snr=5, max_snr=20),\n                            #PinkNoise(p=1, min_snr=5, max_snr=20),\n                            PitchShift(max_steps=4, p=0.1),\n                            TimeShift(max_shift_second=0.5, p=0.1)\n                        ],p=1.),mel_spec]\n    \n    y_tfms = [get_audio_sample_label]\n\n    dsets = Datasets(items = train ,tfms=[x_tfms, y_tfms],splits=splits)\n\n    dls = dsets.dataloaders(bs=bs)   \n    return dls","metadata":{"execution":{"iopub.status.busy":"2022-06-21T09:43:18.847907Z","iopub.execute_input":"2022-06-21T09:43:18.848557Z","iopub.status.idle":"2022-06-21T09:43:18.856503Z","shell.execute_reply.started":"2022-06-21T09:43:18.848519Z","shell.execute_reply":"2022-06-21T09:43:18.855743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xb, yb = get_dls(val_fold=0,bs=params.batch_size).one_batch()\nxb.shape, yb.shape","metadata":{"execution":{"iopub.status.busy":"2022-06-21T09:43:18.858066Z","iopub.execute_input":"2022-06-21T09:43:18.858664Z","iopub.status.idle":"2022-06-21T09:43:28.177016Z","shell.execute_reply.started":"2022-06-21T09:43:18.858552Z","shell.execute_reply":"2022-06-21T09:43:28.176231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls = get_dls(val_fold=0,bs=params.batch_size)\ndls.vocab = params.target_columns","metadata":{"execution":{"iopub.status.busy":"2022-06-21T09:43:28.178217Z","iopub.execute_input":"2022-06-21T09:43:28.178855Z","iopub.status.idle":"2022-06-21T09:43:28.586905Z","shell.execute_reply.started":"2022-06-21T09:43:28.178817Z","shell.execute_reply":"2022-06-21T09:43:28.586030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating the model","metadata":{}},{"cell_type":"code","source":"def interpolate(x: torch.Tensor, ratio: int):\n    (batch_size, time_steps, classes_num) = x.shape\n    upsampled = x[:, :, None, :].repeat(1, 1, ratio, 1)\n    upsampled = upsampled.reshape(batch_size, time_steps * ratio, classes_num)\n    return upsampled\n\n\ndef pad_framewise_output(framewise_output: torch.Tensor, frames_num: int):\n    output = F.interpolate(\n        framewise_output.unsqueeze(1),\n        size=(frames_num, framewise_output.size(2)),\n        align_corners=True,\n        mode=\"bilinear\").squeeze(1)\n\n    return output","metadata":{"execution":{"iopub.status.busy":"2022-06-21T09:43:28.588059Z","iopub.execute_input":"2022-06-21T09:43:28.588469Z","iopub.status.idle":"2022-06-21T09:43:28.597353Z","shell.execute_reply.started":"2022-06-21T09:43:28.588432Z","shell.execute_reply":"2022-06-21T09:43:28.596200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def init_layer(layer):\n    nn.init.xavier_uniform_(layer.weight)\n\n    if hasattr(layer, \"bias\"):\n        if layer.bias is not None:\n            layer.bias.data.fill_(0.)\n\n\ndef init_bn(bn):\n    bn.bias.data.fill_(0.)\n    bn.weight.data.fill_(1.0)\n\nclass AttBlockV2(nn.Module):\n    def __init__(self,\n                 in_features: int,\n                 out_features: int,\n                 activation=\"linear\"):\n        super().__init__()\n\n        self.activation = activation\n        self.att = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n        \n        self.cla = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n\n        self.init_weights()\n\n    def init_weights(self):\n        init_layer(self.att)\n        init_layer(self.cla)\n\n    def forward(self, x):\n        # x: (n_samples, n_in, n_time)\n        norm_att = torch.softmax(torch.tanh(self.att(x)), dim=-1)\n        cla = self.nonlinear_transform(self.cla(x))\n        x = torch.sum(norm_att * cla, dim=2)\n        return x, norm_att, cla\n\n    def nonlinear_transform(self, x):\n        if self.activation == 'linear':\n            return x\n        elif self.activation == 'sigmoid':\n            return torch.sigmoid(x)","metadata":{"execution":{"iopub.status.busy":"2022-06-21T09:43:28.601532Z","iopub.execute_input":"2022-06-21T09:43:28.602457Z","iopub.status.idle":"2022-06-21T09:43:28.629918Z","shell.execute_reply.started":"2022-06-21T09:43:28.602409Z","shell.execute_reply":"2022-06-21T09:43:28.629101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\nclass building_model(Module):\n    def __init__(self,num_classes:int,arch:str = params.base_model_name, pretrained:bool=False):\n        self.spec_augmenter = SpecAugmentation(time_drop_width=64//2, time_stripes_num=2,\n                                                   freq_drop_width=8//2, freq_stripes_num=2)\n        self.spec_time_augmenter = torchaudio.transforms.TimeMasking(24)\n        self.spec_freq_augmenter = torchaudio.transforms.FrequencyMasking(96)\n        self.bn0 = nn.BatchNorm2d(params.n_mels)\n        self.base_model = create_model(arch, pretrained=pretrained)\n        self.linear = nn.Linear(self.base_model.get_classifier().out_features, num_classes)\n        self.layers = list(self.base_model.children())[:-2]\n        self.encoder = nn.Sequential(*self.layers)\n        self.in_features = self.base_model.classifier.in_features\n        self.fc1 = nn.Linear(self.in_features, self.in_features, bias=True)\n        self.fc2 = nn.Linear(num_classes,num_classes, bias=False)\n        self.att_block = AttBlockV2(self.in_features, num_classes, activation=\"sigmoid\")  \n        self.init_weight()\n        self.mode = 'train'\n    def init_weight(self):\n        init_bn(self.bn0)\n        init_layer(self.fc1)\n        \n    def change_mode(self,mode='train'):\n        self.mode = mode\n\n    def forward(self,input_data):\n        frames_num = input_data.shape[2]\n        x = input_data # (batch_size, 3, time_steps, mel_bins)\n        x = x.transpose(1, 3)\n        x = self.bn0(x)\n        x = x.transpose(1, 3)\n\n        if self.training:\n            if random.random() < 0.75:\n                x = self.spec_augmenter(x)\n                #x = self.spec_time_augmenter(x)\n                #x = self.spec_freq_augmenter(x)\n                \n        x = x.transpose(2, 3)\n        x = self.encoder(x)  \n        x = torch.mean(x, dim=3)\n        x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x = x1 + x2\n        x = F.dropout(x, p=0.4, training=self.training)\n        x = x.transpose(1, 2)\n        x = F.relu_(self.fc1(x))\n        x = x.transpose(1, 2)\n        x = F.dropout(x, p=0.4, training=self.training)\n        (clipwise_output, norm_att, segmentwise_output) = self.att_block(x)\n        logit = torch.sum(norm_att * self.att_block.cla(x), dim=2)\n        segmentwise_logit = self.att_block.cla(x).transpose(1, 2)\n        segmentwise_output = segmentwise_output.transpose(1, 2)\n\n        interpolate_ratio = frames_num // segmentwise_output.size(1)\n\n        # Get framewise output\n        framewise_output = interpolate(segmentwise_output,\n                                       interpolate_ratio)\n        framewise_output = pad_framewise_output(framewise_output, frames_num)\n\n        framewise_logit = interpolate(segmentwise_logit, interpolate_ratio)\n        framewise_logit = pad_framewise_output(framewise_logit, frames_num)\n\n        output_dict = {\n            'framewise_output': framewise_output,\n            'clipwise_output': clipwise_output,\n            'logit': logit,\n            'framewise_logit': framewise_logit,\n        }\n\n        return output_dict    ","metadata":{"execution":{"iopub.status.busy":"2022-06-21T09:43:28.635514Z","iopub.execute_input":"2022-06-21T09:43:28.637655Z","iopub.status.idle":"2022-06-21T09:43:28.671198Z","shell.execute_reply.started":"2022-06-21T09:43:28.637611Z","shell.execute_reply":"2022-06-21T09:43:28.670273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loss","metadata":{}},{"cell_type":"code","source":"# https://www.kaggle.com/c/rfcx-species-audio-detection/discussion/213075\nclass BCEFocalLoss(nn.Module):\n    def __init__(self, alpha=0.25, gamma=2.0):\n        super().__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n\n    def forward(self, preds, targets):\n        bce_loss = nn.BCEWithLogitsLoss(reduction='none')(preds, targets)\n        probas = torch.sigmoid(preds)\n        loss = targets * self.alpha * \\\n            (1. - probas)**self.gamma * bce_loss + \\\n            (1. - targets) * probas**self.gamma * bce_loss\n        loss = loss.mean()\n        return loss\n\n\nclass BCEFocal2WayLoss(nn.Module):\n    def __init__(self, weights=[1, 1], class_weights=None):\n        super().__init__()\n\n        self.focal = BCEFocalLoss()\n\n        self.weights = weights\n\n    def forward(self, input, target):\n        input_ = input[\"logit\"]\n        target = target.float()\n\n        framewise_output = input[\"framewise_logit\"]\n        clipwise_output_with_max, _ = framewise_output.max(dim=1)\n\n        loss = self.focal(input_, target)\n        aux_loss = self.focal(clipwise_output_with_max, target)\n\n        return self.weights[0] * loss + self.weights[1] * aux_loss","metadata":{"execution":{"iopub.status.busy":"2022-06-21T09:43:28.677402Z","iopub.execute_input":"2022-06-21T09:43:28.679985Z","iopub.status.idle":"2022-06-21T09:43:28.694601Z","shell.execute_reply.started":"2022-06-21T09:43:28.679942Z","shell.execute_reply":"2022-06-21T09:43:28.693582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def loss_fn(logits, targets):\n    if params.mode == \"valid\":\n        loss_fct = BCEFocal2WayLoss()\n        loss = loss_fct(logits, targets)\n        return loss\n    elif (params.mixup_or_cutmix == \"mixup\" and params.current_epoch < params.mixup_or_cutmix_epochs):\n        loss = mixup_criterion(logits, targets)\n        return loss\n    elif (params.mixup_or_cutmix == \"cutmix\" and params.current_epoch < params.mixup_or_cutmix_epochs):\n        loss = cutmix_criterion(logits, targets)\n        return loss\n    else:\n        loss_fct = BCEFocal2WayLoss()\n        loss = loss_fct(logits, targets)\n        return loss        \n        \ndef cutmix_criterion(preds, new_targets):\n    targets1, targets2, lam = new_targets[0], new_targets[1], new_targets[2]\n    criterion = BCEFocal2WayLoss()\n    return lam * criterion(preds, targets1) + (1 - lam) * criterion(preds, targets2)\n\ndef mixup_criterion(preds, new_targets):\n    targets1, targets2, lam = new_targets[0], new_targets[1], new_targets[2]\n    criterion = BCEFocal2WayLoss()\n    return lam * criterion(preds, targets1) + (1 - lam) * criterion(preds, targets2)\n","metadata":{"execution":{"iopub.status.busy":"2022-06-21T09:43:28.700756Z","iopub.execute_input":"2022-06-21T09:43:28.703434Z","iopub.status.idle":"2022-06-21T09:43:28.713899Z","shell.execute_reply.started":"2022-06-21T09:43:28.703391Z","shell.execute_reply":"2022-06-21T09:43:28.713096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# metric","metadata":{}},{"cell_type":"code","source":"def MetricMeter(y_pred, y_true):\n    with torch.no_grad():\n        y_true = y_true.cpu().detach().numpy()\n        y_pred = y_pred[\"clipwise_output\"].cpu().detach().numpy()\n        f1_03 = metrics.f1_score(np.array(y_true),np.array(y_pred) > 0.3, average=\"micro\")        \n        return f1_03","metadata":{"execution":{"iopub.status.busy":"2022-06-21T09:43:28.714957Z","iopub.execute_input":"2022-06-21T09:43:28.715242Z","iopub.status.idle":"2022-06-21T09:43:28.725368Z","shell.execute_reply.started":"2022-06-21T09:43:28.715204Z","shell.execute_reply":"2022-06-21T09:43:28.724523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport sklearn.metrics\n\ndef comp_metric(y_pred, y_true, epsilon=1e-9):\n    with torch.no_grad():\n        y_true = y_true.cpu().numpy()\n        y_pred = y_pred[\"clipwise_output\"].cpu().numpy()\n    # Get representative confusion matrices for each label\n    mlbl_cms = sklearn.metrics.multilabel_confusion_matrix(y_true, np.array(y_pred) > 0.3)\n\n    # Get two scores (TP and TN SCORES)\n    tp_scores = np.array([\n        mlbl_cm[1, 1]/(epsilon+mlbl_cm[:, 1].sum()) \\\n        for mlbl_cm in mlbl_cms\n        ])\n    tn_scores = np.array([\n        mlbl_cm[0, 0]/(epsilon+mlbl_cm[:, 0].sum()) \\\n        for mlbl_cm in mlbl_cms\n        ])\n\n    # Get average\n    tp_mean = tp_scores.mean()\n    tn_mean = tn_scores.mean()\n\n    return round((tp_mean+tn_mean)/2, 8)\n","metadata":{"execution":{"iopub.status.busy":"2022-06-21T09:43:28.726959Z","iopub.execute_input":"2022-06-21T09:43:28.727406Z","iopub.status.idle":"2022-06-21T09:43:28.737600Z","shell.execute_reply.started":"2022-06-21T09:43:28.727368Z","shell.execute_reply":"2022-06-21T09:43:28.736818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class train_val_clb(Callback):\n        def after_epoch(self):\n            params.current_epoch = params.current_epoch + 1\n            print(params.current_epoch)\n            \n        def after_fit(self):\n            params.current_epoch = 0\n            \n        def after_validate(self):\n            self.learn.model.change_mode('train')\n            params.mode = 'train'\n            \n        def before_validate(self):\n            self.learn.model.change_mode('valid')\n            params.mode = 'valid'\n            \n        def before_batch(self):\n            if params.mode == \"train\":\n                self.learn.dls.bs = 32\n                data = self.learn.xb\n                targets = self.learn.yb\n                if params.current_epoch < params.mixup_or_cutmix_epochs:\n                    if np.random.rand()<0.5:\n                        new_data, new_targets = self.mixup(data[0],targets[0],0.4)\n                        self.learn.xb = (new_data,)\n                        self.learn.yb = (new_targets,)\n                        params.mixup_or_cutmix = \"mixup\"\n\n                    else:\n                        new_data, new_targets = self.cutmix(data[0], targets[0],0.4)\n                        self.learn.xb = (new_data,)\n                        self.learn.yb = (new_targets,)\n                        params.mixup_or_cutmix = \"cutmix\"\n \n            else:\n                self.learn.dls.bs = 32\n\n        def cutmix(self,data, targets, alpha):\n            indices = torch.randperm(data.size(0))\n            shuffled_data = data[indices]\n            shuffled_targets = targets[indices]\n\n            lam = np.random.beta(alpha, alpha)\n            bbx1, bby1, bbx2, bby2 = self.rand_bbox(data.size(), lam)\n            data[:, :, bbx1:bbx2, bby1:bby2] = data[indices, :, bbx1:bbx2, bby1:bby2]\n            # adjust lambda to exactly match pixel ratio\n            lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (data.size()[-1] * data.size()[-2]))\n\n            new_targets = [targets, shuffled_targets, lam]\n            return data, new_targets\n\n        def mixup(self,data, targets, alpha):\n            indices = torch.randperm(data.size(0))\n            shuffled_data = data[indices]\n            shuffled_targets = targets[indices]\n\n            lam = np.random.beta(alpha, alpha)\n            new_data = data * lam + shuffled_data * (1 - lam)\n            new_targets = [targets, shuffled_targets, lam]\n            return new_data, new_targets\n        \n        def rand_bbox(self,size, lam):\n            W = size[2]\n            H = size[3]\n            cut_rat = np.sqrt(1. - lam)\n            cut_w = int(W * cut_rat)\n            cut_h = int(H * cut_rat)\n\n            # uniform\n            cx = np.random.randint(W)\n            cy = np.random.randint(H)\n\n            bbx1 = np.clip(cx - cut_w // 2, 0, W)\n            bby1 = np.clip(cy - cut_h // 2, 0, H)\n            bbx2 = np.clip(cx + cut_w // 2, 0, W)\n            bby2 = np.clip(cy + cut_h // 2, 0, H)\n\n            return bbx1, bby1, bbx2, bby2","metadata":{"execution":{"iopub.status.busy":"2022-06-21T09:43:28.739159Z","iopub.execute_input":"2022-06-21T09:43:28.739560Z","iopub.status.idle":"2022-06-21T09:43:28.760018Z","shell.execute_reply.started":"2022-06-21T09:43:28.739526Z","shell.execute_reply":"2022-06-21T09:43:28.759307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = building_model(num_classes=len(dls.vocab),pretrained=params.pretrained)\n#model = resnet18(num_classes=152)\nlearn = Learner(dls,model,loss_func=loss_fn,metrics=MetricMeter,cbs=[train_val_clb,SaveModelCallback(monitor='MetricMeter',comp=np.greater)])\n#learn.summary()","metadata":{"execution":{"iopub.status.busy":"2022-06-21T09:43:28.764354Z","iopub.execute_input":"2022-06-21T09:43:28.764554Z","iopub.status.idle":"2022-06-21T09:43:39.403510Z","shell.execute_reply.started":"2022-06-21T09:43:28.764531Z","shell.execute_reply":"2022-06-21T09:43:39.402756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#learn.lr_find()","metadata":{"execution":{"iopub.status.busy":"2022-06-21T09:43:39.404729Z","iopub.execute_input":"2022-06-21T09:43:39.404985Z","iopub.status.idle":"2022-06-21T09:43:39.409681Z","shell.execute_reply.started":"2022-06-21T09:43:39.404951Z","shell.execute_reply":"2022-06-21T09:43:39.409014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.model = load_learner(\"../input/fastai-effb0-base-model/model.pkl\").model","metadata":{"execution":{"iopub.status.busy":"2022-06-21T09:43:39.410819Z","iopub.execute_input":"2022-06-21T09:43:39.411211Z","iopub.status.idle":"2022-06-21T09:43:39.869554Z","shell.execute_reply.started":"2022-06-21T09:43:39.411154Z","shell.execute_reply":"2022-06-21T09:43:39.868740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.fit_one_cycle(10,lr_max = params.LR,wd=params.WEIGHT_DECAY)","metadata":{"execution":{"iopub.status.busy":"2022-06-21T09:43:39.870876Z","iopub.execute_input":"2022-06-21T09:43:39.871378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.export(fname='model.pkl')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# references","metadata":{}},{"cell_type":"markdown","source":"1- https://www.kaggle.com/code/kaerunantoka/birdclef2022-audio-to-numpy-1-4/notebook\n\n2- https://www.kaggle.com/code/kaerunantoka/birdclef2022-n001-training/notebook","metadata":{}}]}