{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install ../input/torchlibrosa/torchlibrosa-0.0.5-py3-none-any.whl > /dev/null","metadata":{"execution":{"iopub.status.busy":"2023-04-04T15:31:41.630682Z","iopub.execute_input":"2023-04-04T15:31:41.631147Z","iopub.status.idle":"2023-04-04T15:31:50.101059Z","shell.execute_reply.started":"2023-04-04T15:31:41.631013Z","shell.execute_reply":"2023-04-04T15:31:50.100188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install colorednoise > /dev/null","metadata":{"execution":{"iopub.status.busy":"2023-04-04T15:31:50.103576Z","iopub.execute_input":"2023-04-04T15:31:50.104058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport ast\nimport numpy as np\nimport warnings\nfrom sklearn import metrics\nfrom torch.utils.data import Dataset\nimport pandas as pd\nimport torchaudio\nimport glob\nfrom fastai.distributed import *\nimport colorednoise as cn\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold\nimport os\nfrom torch.utils.data import Dataset\nimport pandas as pd\nimport torchaudio\nfrom sklearn.preprocessing import MultiLabelBinarizer\nfrom fastai.vision.all import *\nfrom typing import Optional,Tuple,List\nsys.path.append('../input/timm-pytorch-image-models/pytorch-image-models-master')\nimport timm\nfrom timm import create_model\nimport audioread\nimport librosa\nimport soundfile as sf\nfrom albumentations.core.transforms_interface import ImageOnlyTransform\nfrom torchlibrosa.stft import LogmelFilterBank, Spectrogram\nfrom torchlibrosa.augmentation import SpecAugmentation\n\nimport albumentations as A\nimport albumentations.pytorch.transforms as T\nimport matplotlib.pyplot as plt\nimport torchaudio\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AllParams:\n    seed = 71\n    target_columns = ['yetgre1', 'moccha1', 'rostur1', 'walsta1', 'ratcis1', 'norfis1', \n                      'macshr1', 'brrwhe3', 'crefra2', 'pabspa1', 'sltnig1', 'cabgre1', \n                      'equaka1', 'sobfly1', 'rindov', 'wlwwar', 'brwwar1', 'gnbcam2',\n                      'carcha1', 'abethr1', 'yertin1', 'spewea1', 'varsun2', 'yebduc1', \n                      'eubeat1', 'hadibi1', 'brcale1', 'litwea1', 'sincis1', 'whbcro2', \n                      'thrnig1', 'bubwar2', 'kvbsun1', 'blbpuf2', 'blakit1', 'colsun2', \n                      'bltapa1', 'gycwar3', 'joygre1', 'greegr', 'vibsta2', 'wtbeat1', \n                      'afrgos1', 'rebfir2', 'yebgre1', 'comsan', 'pygbat1', 'meypar1', \n                      'yelbis1', 'norbro1', 'ndcsun2', 'gybfis1', 'reftin1', 'brobab1', \n                      'refwar2', 'norcro1', 'yebapa1', 'yewgre1', 'palfly2', 'gargan',\n                      'darter3', 'rerswa1', 'augbuz1', 'gyhbus1', 'refcro1', 'witswa1', \n                      'gryapa1', 'pitwhy', 'eaywag1', 'blhgon1', 'yebsto1', 'hipbab1', 'whcpri2', \n                      'spemou2', 'gobsta5', 'blksaw1', 'afecuc1', 'spepig1', 'mabeat1', 'rewsta1', \n                      'rebhor1', 'brtcha1', 'blacuc1', 'brican1', 'rehblu1', 'gobbun1', 'supsta1', \n                      'bkfruw1', 'litswi1', 'spmthr1', 'spwlap1', 'quailf1', 'golher1', 'didcuc1',\n                      'gytbar1', 'klacuc1', 'afbfly1', 'brcsta1', 'bawhor2', 'whihel1', 'yespet1',\n                      'dotbar1', 'luebus1', 'yeccan1', 'tafpri1', 'chespa1', 'blacra1', 'scthon1',\n                      'whbcou1', 'ccbeat1', 'libeat1', 'whctur2', 'butapa1', 'norpuf1', 'blwlap1',\n                      'afmdov1', 'hartur1', 'beasun2', 'vimwea1', 'squher1', 'yebbar1', 'bltori1',\n                      'sccsun2', 'piecro1', 'chibat1', 'marsto1', 'afpfly1', 'bcbeat1', 'wbswea1',\n                      'yebere1', 'rbsrob1', 'brcwea1', 'bswdov1', 'kerspa2', 'slcbou1', 'fislov1',\n                      'cohmar1', 'lesmaw1', 'cibwar1', 'woosan', 'shesta1', 'reccor', 'gnhsun1', \n                      'chucis1', 'fatrav1', 'slbgre1', 'afghor1', 'afrjac1', 'abhori1', 'wbgbir1',\n                      'subbus1', 'bawman1', 'whrshr1', 'hoopoe', 'lessts1', 'rocmar2', 'lotlap1',\n                      'tamdov1', 'rufcha2', 'palpri1', 'reboxp1', 'chewea1', 'malkin1', 'vilwea1',\n                      'reccuc1', 'bltbar1', 'trobou1', 'abythr1', 'broman1', 'easmog1', 'spfbar1',\n                      'afpwag1', 'refbar2', 'strher', 'whhsaw1', 'grbcam1', 'sichor1', 'crheag1',\n                      'wookin1', 'helgui', 'strsee1', 'chtapa3', 'grccra1', 'brubru1', 'wbrcha2',\n                      'bkctch1', 'yesbar1', 'scrcha1', 'affeag1', 'grwpyt1', 'whbtit5', 'spfwea1',\n                      'brosun1', 'combuz1', 'tacsun1', 'darbar1', 'grewoo2', 'purgre2', 'grecor',\n                      'whbcan1', 'afrgrp1', 'mouwag1', 'bagwea1', 'eswdov1', 'blfbus1', 'soucit1',\n                      'blnmou1', 'gbesta1', 'whbwhe3', 'somgre1', 'afrthr1', 'carwoo1', 'yenspu1',\n                      'gobwea1', 'wfbeat1', 'blnwea1', 'soufis1', 'hunsun2', 'nobfly1', 'gyhkin1',\n                      'nubwoo1', 'afpkin1', 'marsun2', 'gabgos2', 'yefcan', 'btweye2', 'huncis1',\n                      'raybar1', 'dutdov1', 'gyhneg1', 'stusta1', 'wheslf1', 'somtit4', 'mcptit1',\n                      'whbwea1', 'lawgol', 'combul2', 'gyhspa1', 'ruegls1', 'fotdro5', 'afdfly1',\n                      'sacibi2', 'hamerk1', 'piekin1', 'afgfly1', 'reisee2', 'amesun2', 'laudov1',\n                      'grywrw1', 'blhher1', 'loceag1', 'crohor1', 'lotcor1', 'brctch1', 'barswa',\n                      'categr', 'reedov1', 'blaplo1', 'litegr', 'egygoo', 'rehwea1', 'fatwid1',\n                      'blcapa2', 'edcsun3']\n    \n    img_size = 224 # 128\n    main_metric = \"epoch_f1_at_03\"\n    USE_SEC = 30\n    # Melspectrogram\n    period = 5\n    duration = period\n    n_mels = 224 # 128\n    fmin = 20\n    fmax = 16000\n    n_fft = 2048\n    hop_length = 512\n    sample_rate = 32000\n    sr = sample_rate\n    melspectrogram_parameters = {\n        \"n_mels\": 224, # 128,\n        \"fmin\": 20,\n        \"fmax\": 16000\n    }\n    \n    # train\n    epochs = 9\n    folds = [0] # [0, 1, 2, 3, 4]\n    N_FOLDS = 5\n    LR = 0.001\n    ETA_MIN = 1e-6\n    WEIGHT_DECAY = 0.00005\n    batch_size = 64 # 32\n    base_model_name = \"tf_efficientnet_b0_ns\"\n    EARLY_STOPPING = True\n    DEBUG = False # True\n    EVALUATION = 'AUC'\n    apex = True\n    pooling = \"max\"\n    pretrained = True\n    num_classes = 152\n    in_channels = 3\n    mode = 'train'\n    mixup_or_cutmix_epochs = 6\n    mixup_or_cutmix = 'mixup'\n    current_epoch = 0\n    all_validation_preds = np.array([])\n    all_validation_targets = np.array([])\n    padding_factor = 5\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nparams = AllParams","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\nOUTPUT_DIR = f'./'\nif not os.path.exists(OUTPUT_DIR):\n    os.makedirs(OUTPUT_DIR)\n   \n    \ndef set_seed(seed=42):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    \nset_seed(params.seed)\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# loading training data","metadata":{}},{"cell_type":"code","source":"import glob\n\nall_path = glob.glob('/kaggle/input/birdclef-2023/train_audio/*/*.ogg')\n\nlen(all_path)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n\n\ntrain = pd.read_csv('/kaggle/input/birdclef-2023/train_metadata.csv')\ntrain['new_target'] = train['primary_label'] + ' ' + train['secondary_labels'].map(lambda x: ' '.join(ast.literal_eval(x)))\ntrain['len_new_target'] = train['new_target'].map(lambda x: len(x.split()))\n# train['len_new_target'].value_counts()\ntrain.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path_df = pd.DataFrame(all_path, columns=['file_path'])\npath_df['filename'] = path_df['file_path'].map(lambda x: x.split('/')[-2]+'/'+x.split('/')[-1])\npath_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.merge(train, path_df, on='filename')\nprint(train.shape)\ntrain.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Fold = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\nfor n, (trn_index, val_index) in enumerate(Fold.split(train, train['primary_label'])):\n    train.loc[val_index, 'kfold'] = int(n)\n\ntrain['kfold'] = train['kfold'].astype(int)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"counts = train.primary_label.value_counts()\ncond = train.primary_label.isin(counts[counts<5].index.tolist())\ntrain.loc[cond, 'kfold'] = 1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_dist = train['primary_label'].value_counts()\nthr = 50\ndown_classes = class_dist[class_dist < thr].index.tolist()\nup_dfs = []\nfor c in down_classes:\n    # get the dataframe for the current class\n    class_df = train.query(\"primary_label==@c\")\n    # find number of samples to add\n    num_up = thr - class_df.shape[0]\n    # upsample the dataframe\n    class_df = class_df.sample(n=num_up, replace=True, random_state=42)\n    # append the upsampled dataframe to the list\n    up_dfs.append(class_df)\nup_df = pd.concat([train] + up_dfs, axis=0, ignore_index=True)\ntrain = up_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train = train[train.primary_label.isin(scored_classes)]\n#train.reset_index(inplace=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.to_csv('train_folds.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating dataset","metadata":{}},{"cell_type":"code","source":"class NoiseInjection(Transform):\n    def __init__(self, always_apply=False, p=0.5, max_noise_level=0.5):\n        self.noise_level = (0.0, max_noise_level)\n        self.p = p\n        self.always_apply = always_apply\n        \n    def encodes(self, signal: np.ndarray):\n        noise_level = np.random.uniform(*self.noise_level)\n        noise = np.random.randn(len(signal))\n        if self.always_apply == True:\n                signal = (signal + noise * noise_level).astype(signal.dtype)\n            \n        else:\n            if np.random.rand() < self.p:\n                signal = (signal + noise * noise_level).astype(signal.dtype) \n        return signal\n\n\nclass GaussianNoise(Transform):\n    def __init__(self, always_apply=False, p=0.5, min_snr=5, max_snr=20):\n        self.always_apply = always_apply\n        self.p = p\n        self.min_snr = min_snr\n        self.max_snr = max_snr\n\n    def encodes(self, signal: np.ndarray):\n        snr = np.random.uniform(self.min_snr, self.max_snr)\n        a_signal = np.sqrt(signal ** 2).max()\n        a_noise = a_signal / (10 ** (snr / 20))\n        white_noise = np.random.randn(len(signal))\n        a_white = np.sqrt(white_noise ** 2).max()\n        \n        if self.always_apply == True:\n            signal = (signal + white_noise * 1 / a_white * a_noise).astype(signal.dtype)\n            \n        else:\n            if np.random.rand() < self.p:\n                signal = (signal + white_noise * 1 / a_white * a_noise).astype(signal.dtype)\n        return signal\n\n\nclass PinkNoise(Transform):\n    def __init__(self, always_apply=False, p=0.5, min_snr=5, max_snr=20):\n        self.always_apply = always_apply\n        self.p = p\n        self.min_snr = min_snr\n        self.max_snr = max_snr\n\n    def encodes(self, signal: np.ndarray):\n        snr = np.random.uniform(self.min_snr, self.max_snr)\n        a_signal = np.sqrt(signal ** 2).max()\n        a_noise = a_signal / (10 ** (snr / 20))\n\n        pink_noise = cn.powerlaw_psd_gaussian(1, len(signal))\n        a_pink = np.sqrt(pink_noise ** 2).max()\n        if self.always_apply == True:\n            signal = (signal + pink_noise * 1 / a_pink * a_noise).astype(signal.dtype)\n            \n        else:\n            if np.random.rand() < self.p:\n                signal = (signal + pink_noise * 1 / a_pink * a_noise).astype(signal.dtype)\n        return signal\n\nclass RandomVolume(Transform):\n    def __init__(self,always_apply=False, p=0.2, limit=12):\n        self.limit = limit\n        self.p = p\n        self.always_apply = always_apply\n    def encodes(self, signal: np.ndarray):\n        db = np.random.uniform(-self.limit, self.limit)\n        if self.always_apply == True:\n            if db >= 0:\n                return self.volume_up(signal, db)\n            else:\n                return self.volume_down(signal, db)\n        else:\n            if np.random.rand() < self.p:\n                if db >= 0:\n                    return self.volume_up(signal, db)\n                else:\n                    return self.volume_down(signal, db)      \n            return signal\n\n    def volume_down(self,signal: np.ndarray, db: float):\n        signal = signal * self._db2float(-db)\n        return signal\n    \n    def volume_up(self,signal: np.ndarray, db: float):\n        signal = signal * self._db2float(db)\n        return signal\n    \n    def _db2float(self,db: float, amplitude=True):\n        if amplitude:\n            return 10 ** (db / 20)\n        else:\n            return 10 ** (db / 10)\n\nclass TimeShift(Transform):\n    def __init__(self, always_apply=False, p=0.5, max_shift_second=2,padding_mode=\"zero\"):\n        self.p = p\n        self.max_shift_second = max_shift_second\n        self.padding_mode = padding_mode\n        self.always_apply = always_apply\n        self.sr = params.sr\n        assert padding_mode in [\"replace\", \"zero\"], \"`padding_mode` must be either 'replace' or 'zero'\"\n        \n    def encodes(self, signal: np.ndarray):\n        shift = np.random.randint(-self.sr * self.max_shift_second, self.sr * self.max_shift_second)\n        augmented = np.roll(signal, shift)\n        if self.always_apply == True:\n            if self.padding_mode == \"zero\":\n                if shift > 0:\n                    augmented[:shift] = 0\n                else:\n                    augmented[shift:] = 0\n            return augmented\n        else:\n            if np.random.rand() < self.p:\n                if self.padding_mode == \"zero\":\n                    if shift > 0:\n                        augmented[:shift] = 0\n                    else:\n                        augmented[shift:] = 0\n                    return augmented\n        return signal   \n\nclass PitchShift(Transform):\n    def __init__(self, always_apply=False, p=0.5, max_steps=5):\n        self.p = p\n        self.always_apply = always_apply\n        self.max_steps = max_steps\n        self.sr = params.sr\n\n    def encodes(self, signal: np.ndarray):\n        n_steps = np.random.randint(-self.max_steps, self.max_steps)\n        if self.always_apply == True:\n            augmented = librosa.effects.pitch_shift(signal, sr=self.sr, n_steps=n_steps)\n            return augmented\n        elif np.random.rand() < self.p:\n            augmented = librosa.effects.pitch_shift(signal, sr=self.sr, n_steps=n_steps)\n            return augmented          \n        else:\n            return signal","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nmean = (0.485, 0.456, 0.406) # RGB\nstd = (0.229, 0.224, 0.225) # RGB\n\nalbu_transforms = {\n    'train' : A.Compose([\n            A.HorizontalFlip(p=0.5),\n            A.OneOf([\n                A.Cutout(max_h_size=5, max_w_size=16),\n                A.CoarseDropout(max_holes=4),\n            ], p=0.5),\n            A.Normalize(mean, std),\n    ]),\n    'valid' : A.Compose([\n            A.Normalize(mean, std),\n    ]),\n}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nclass audio_aug_pipline(torch.nn.Module):\n    def __init__(\n        self,\n        input_freq=16000,\n        sample_rate=params.sr,\n        stretch_factor=0.8,\n    ):\n        super().__init__()\n\n        self.PitchShift_layer = torchaudio.transforms.PitchShift(sample_rate, 4)\n        self.TimeStretch_layer = torchaudio.transforms.TimeStretch(stretch_factor, fixed_rate=True)\n        self.Change_Volume = torchaudio.transforms.Vol(12,gain_type=\"amplitude\")\n    def forward(self, waveform: torch.Tensor) -> torch.Tensor:\n\n        aug_data = \n\n        return mel\n'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class OneOf(Transform):\n    def __init__(self, transforms,p=0.5):\n        self.p = p\n        self.transforms = transforms\n        \n        transforms_ps = [t.p for t in transforms]\n        s = sum(transforms_ps)\n        self.transforms_ps = [t / s for t in transforms_ps]\n        \n    def encodes(self, signal: np.ndarray):\n        if params.mode == \"train\":\n            if self.transforms_ps and (random.random() < self.p):\n                random_state = np.random.RandomState(random.randint(0, 2 ** 32 - 1))\n                t = random_state.choice(self.transforms, p=self.transforms_ps)\n                signal = t(signal)\n\n            signal = RandomVolume(p=0.2, limit=12)(signal)\n        signal =  self.Normalize(signal=signal,p=1)\n        signal = np.concatenate([signal, signal, signal])[:params.duration * params.sr] \n\n        if params.mode == \"train\":\n            train = True\n        else:\n            train = False\n        signal = self.crop_or_pad(signal, params.duration * params.sr, sr=params.sr, train=train, probs=None)\n        return signal\n    \n    def crop_or_pad(self,signal, length, sr, train, probs=None):\n        if len(signal) <= length:\n            signal = np.concatenate([signal, np.zeros(length - len(signal))])\n        else:\n            if train == False:\n                start = 0\n            elif probs is None:\n                start = np.random.randint(len(signal) - length)\n            else:\n                start = (\n                        np.random.choice(np.arange(len(probs)), p=probs) + np.random.random()\n                )\n                start = int(sr * (start))\n\n            signal = signal[start: start + length]\n\n        return signal.astype(np.float32) \n    \n    def Normalize(self,signal,always_apply=False, p=1):\n        max_vol = np.abs(signal).max()\n        if always_apply == True:\n             signal_vol = signal * 1 / max_vol\n\n        else:\n            if np.random.rand() < p:\n                 signal_vol = signal * 1 / max_vol \n        return np.array(signal_vol)\n    \n    \n    \nclass get_audio_sample_path(Transform):\n    def encodes(self, x):\n        return x.file_path\n    \nclass get_audio_sample_label(Transform):       \n    def encodes(self,x):\n        labels = x.new_target\n        targets = np.zeros(len(params.target_columns), dtype=float)\n        for ebird_code in labels.split():\n            targets[params.target_columns.index(ebird_code)] = 1.0\n        return targets\n\nclass load_signal(Transform):\n    def encodes(self, x:str):\n        sr = params.sample_rate\n        signal, sr = sf.read(x, always_2d=True)\n        #signal = np.load(x)\n        signal = np.mean(signal, 1) # there is (X, 2) array\n        if len(signal) > sr * params.USE_SEC:\n            signal = signal[sr:-sr]\n            signal = signal[:sr * params.USE_SEC]\n\n        #if len(signal) > 0:\n        #    signal = signal[:params.duration*sr]\n        #else:\n        #    signal = np.array([0.0001 for x in range( params.USE_SEC*sr)])\n        return signal\n\n\nclass mel_spec(Transform):\n    def encodes(self, x):\n        #melspec_transform = librosa.feature.melspectrogram(\n        #     sr=params.sample_rate, n_mels=params.n_mels, fmin=params.fmin, fmax=params.fmax,\n        #)\n        #x = x.to(params.device)\n        if x.shape[1]!= len(params.target_columns):\n            melspec_transformer = torchaudio.transforms.MelSpectrogram(sample_rate=params.sample_rate,\n                                                                      f_min=params.fmin,f_max=params.fmax,\n                                                                      n_mels=params.n_mels,\n                                                                      hop_length=params.hop_length).to(x.get_device())\n            melspec_batch = melspec_transformer(x)\n            #power_to_db_transform = torchaudio.transforms.AmplitudeToDB(\"power\")\n            #melspec_batch = power_to_db_transform(melspec_batch)\n            melspec_batch = melspec_batch.to(\"cpu\").numpy()\n            image_batch = list()\n            image_final_batch = list()\n            for i in range(melspec_batch.shape[0]):\n                mel_spec = librosa.power_to_db(melspec_batch[i]).astype(np.float32)\n                image_batch.append(self.mono_to_color(mel_spec))\n                image_batch[i] = image_batch[i].astype(np.uint8)\n                # image = np.load(wav_path) # (224, 313, 3)\n                image_final_batch.append(albu_transforms[params.mode](image=image_batch[i])['image'].T)\n\n            image_final_batch = np.stack(image_final_batch,axis=0)\n            image_final_batch  = torch.tensor(image_final_batch)\n            image_final_batch.type(torch.FloatTensor)\n            return image_final_batch.to(x.get_device())\n        else:\n            return x\n    def mono_to_color(self,X, eps=1e-6, mean=None, std=None):\n        X = np.stack([X, X, X], axis=-1)\n        # Standardize\n        mean = mean or X.mean()\n        std = std or X.std()\n        X = (X - mean) / (std + eps)\n\n        # Normalize to [0, 255]\n        _min, _max = X.min(), X.max()\n\n        if (_max - _min) > eps:\n            V = np.clip(X, _min, _max)\n            V = 255 * (V - _min) / (_max - _min)\n            V = V.astype(np.uint8)\n        else:\n            V = np.zeros_like(X, dtype=np.uint8)\n        return V","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if torch.cuda.is_available():\n        device = \"cuda\"\nelse:\n        device = \"cpu\"\nprint(f\"Using {device}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_fold = 0\ndef get_dls(val_fold=0,bs=64):\n    splits = [train[train.kfold != val_fold].index.tolist() , train[train.kfold == val_fold].index.tolist()]\n    #x_tfms = [get_audio_sample_path,load_signal(device),resample_if_necessary(32000),\n    #          mix_down_if_necessary,cut_if_necessary(),right_pad_if_necessary,\n    #          mel_spec,mono_to_color]\n    x_tfms = [get_audio_sample_path,load_signal,OneOf([   \n                            NoiseInjection(p=1, max_noise_level=0.04),\n                            #GaussianNoise(p=0.01, min_snr=5, max_snr=20),\n                            #PinkNoise(p=1, min_snr=5, max_snr=20),\n                            PitchShift(max_steps=4, p=0.1),\n                            TimeShift(max_shift_second=0.5, p=0.1)\n                        ],p=1.)]\n    \n    y_tfms = [get_audio_sample_label]\n\n    dsets = Datasets(items = train ,tfms=[x_tfms, y_tfms],splits=splits)\n\n    dls = dsets.dataloaders(bs=bs ,after_batch=[mel_spec,ToTensor()])   \n    return dls","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xb, yb = get_dls(val_fold=0,bs=params.batch_size).one_batch()\nxb.shape, yb.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls = get_dls(val_fold=0,bs=params.batch_size)\ndls.vocab = params.target_columns","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(dls.valid)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating the model","metadata":{}},{"cell_type":"code","source":"def interpolate(x: torch.Tensor, ratio: int):\n    (batch_size, time_steps, classes_num) = x.shape\n    upsampled = x[:, :, None, :].repeat(1, 1, ratio, 1)\n    upsampled = upsampled.reshape(batch_size, time_steps * ratio, classes_num)\n    return upsampled\n\n\ndef pad_framewise_output(framewise_output: torch.Tensor, frames_num: int):\n    output = F.interpolate(\n        framewise_output.unsqueeze(1),\n        size=(frames_num, framewise_output.size(2)),\n        align_corners=True,\n        mode=\"bilinear\").squeeze(1)\n\n    return output","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def init_layer(layer):\n    nn.init.xavier_uniform_(layer.weight)\n\n    if hasattr(layer, \"bias\"):\n        if layer.bias is not None:\n            layer.bias.data.fill_(0.)\n\n\ndef init_bn(bn):\n    bn.bias.data.fill_(0.)\n    bn.weight.data.fill_(1.0)\n\nclass AttBlockV2(nn.Module):\n    def __init__(self,\n                 in_features: int,\n                 out_features: int,\n                 activation=\"linear\"):\n        super().__init__()\n\n        self.activation = activation\n        self.att = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n        \n        self.cla = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n\n        self.init_weights()\n\n    def init_weights(self):\n        init_layer(self.att)\n        init_layer(self.cla)\n\n    def forward(self, x):\n        # x: (n_samples, n_in, n_time)\n        norm_att = torch.softmax(torch.tanh(self.att(x)), dim=-1)\n        cla = self.nonlinear_transform(self.cla(x))\n        x = torch.sum(norm_att * cla, dim=2)\n        return x, norm_att, cla\n\n    def nonlinear_transform(self, x):\n        if self.activation == 'linear':\n            return x\n        elif self.activation == 'sigmoid':\n            return torch.sigmoid(x)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\nclass building_model(Module):\n    def __init__(self,num_classes:int,arch:str = params.base_model_name, pretrained:bool=False):\n        self.spec_augmenter = SpecAugmentation(time_drop_width=64//2, time_stripes_num=2,\n                                                   freq_drop_width=8//2, freq_stripes_num=2)\n        self.spec_time_augmenter = torchaudio.transforms.TimeMasking(24)\n        self.spec_freq_augmenter = torchaudio.transforms.FrequencyMasking(96)\n        self.bn0 = nn.BatchNorm2d(params.n_mels)\n        self.base_model = create_model(arch, pretrained=pretrained)\n        self.linear = nn.Linear(self.base_model.get_classifier().out_features, num_classes)\n        self.layers = list(self.base_model.children())[:-2]\n        self.encoder = nn.Sequential(*self.layers)\n        self.in_features = self.base_model.classifier.in_features\n        self.fc1 = nn.Linear(self.in_features, self.in_features, bias=True)\n        self.fc2 = nn.Linear(num_classes,num_classes, bias=False)\n        self.att_block = AttBlockV2(self.in_features, num_classes, activation=\"sigmoid\")  \n        self.init_weight()\n        self.mode = 'train'\n    def init_weight(self):\n        init_bn(self.bn0)\n        init_layer(self.fc1)\n        \n    def change_mode(self,mode='train'):\n        self.mode = mode\n\n    def forward(self,input_data):\n        frames_num = input_data.shape[2]\n        x = input_data # (batch_size, 3, time_steps, mel_bins)\n        x = x.transpose(1, 3)\n        x = self.bn0(x)\n        x = x.transpose(1, 3)\n\n        if self.training:\n            if random.random() < 0.75:\n                x = self.spec_augmenter(x)\n                #x = self.spec_time_augmenter(x)\n                #x = self.spec_freq_augmenter(x)\n                \n        x = x.transpose(2, 3)\n        x = self.encoder(x)  \n        x = torch.mean(x, dim=3)\n        x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x = x1 + x2\n        x = F.dropout(x, p=0.4, training=self.training)\n        x = x.transpose(1, 2)\n        x = F.relu_(self.fc1(x))\n        x = x.transpose(1, 2)\n        x = F.dropout(x, p=0.4, training=self.training)\n        (clipwise_output, norm_att, segmentwise_output) = self.att_block(x)\n        logit = torch.sum(norm_att * self.att_block.cla(x), dim=2)\n        segmentwise_logit = self.att_block.cla(x).transpose(1, 2)\n        segmentwise_output = segmentwise_output.transpose(1, 2)\n\n        interpolate_ratio = frames_num // segmentwise_output.size(1)\n\n        # Get framewise output\n        framewise_output = interpolate(segmentwise_output,\n                                       interpolate_ratio)\n        framewise_output = pad_framewise_output(framewise_output, frames_num)\n\n        framewise_logit = interpolate(segmentwise_logit, interpolate_ratio)\n        framewise_logit = pad_framewise_output(framewise_logit, frames_num)\n\n        output_dict = {\n            'framewise_output': framewise_output,\n            'clipwise_output': clipwise_output,\n            'logit': logit,\n            'framewise_logit': framewise_logit,\n        }\n\n        return output_dict    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loss","metadata":{}},{"cell_type":"code","source":"# https://www.kaggle.com/c/rfcx-species-audio-detection/discussion/213075\nclass BCEFocalLoss(nn.Module):\n    def __init__(self, alpha=0.25, gamma=2.0):\n        super().__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n\n    def forward(self, preds, targets):\n        bce_loss = nn.BCEWithLogitsLoss(reduction='none')(preds, targets)\n        probas = torch.sigmoid(preds)\n        loss = targets * self.alpha * \\\n            (1. - probas)**self.gamma * bce_loss + \\\n            (1. - targets) * probas**self.gamma * bce_loss\n        loss = loss.mean()\n        return loss\n\n\nclass BCEFocal2WayLoss(nn.Module):\n    def __init__(self, weights=[1, 1], class_weights=None):\n        super().__init__()\n\n        self.focal = BCEFocalLoss()\n\n        self.weights = weights\n\n    def forward(self, input, target):\n        input_ = input[\"logit\"]\n        target = target.float()\n\n        framewise_output = input[\"framewise_logit\"]\n        clipwise_output_with_max, _ = framewise_output.max(dim=1)\n\n        loss = self.focal(input_, target)\n        aux_loss = self.focal(clipwise_output_with_max, target)\n\n        return self.weights[0] * loss + self.weights[1] * aux_loss","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def loss_fn(logits, targets):\n\n    if params.mode == \"valid\":\n        loss_fct = BCEFocal2WayLoss()\n        loss = loss_fct(logits, targets)\n        return loss\n    elif (params.mixup_or_cutmix == \"mixup\" and params.current_epoch < params.mixup_or_cutmix_epochs):\n        loss = mixup_criterion(logits, targets)\n        return loss\n    elif (params.mixup_or_cutmix == \"cutmix\" and params.current_epoch < params.mixup_or_cutmix_epochs):\n        loss = cutmix_criterion(logits, targets)\n        return loss\n    else:\n        loss_fct = BCEFocal2WayLoss()\n        loss = loss_fct(logits, targets)\n        return loss        \n        \ndef cutmix_criterion(preds, new_targets):\n    targets1, targets2, lam = new_targets[0], new_targets[1], new_targets[2]\n    criterion = BCEFocal2WayLoss()\n    return lam * criterion(preds, targets1) + (1 - lam) * criterion(preds, targets2)\n\ndef mixup_criterion(preds, new_targets):\n    targets1, targets2, lam = new_targets[0], new_targets[1], new_targets[2]\n    criterion = BCEFocal2WayLoss()\n    return lam * criterion(preds, targets1) + (1 - lam) * criterion(preds, targets2)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# metric","metadata":{}},{"cell_type":"code","source":"def MetricMeter(y_pred,y_true):\n    with torch.no_grad():\n        y_true = y_true.cpu().detach().numpy()\n        y_pred = y_pred[\"clipwise_output\"].cpu().detach().numpy()\n\n        if len(params.all_validation_preds.shape) == 1:\n            params.all_validation_preds  = y_pred\n            params.all_validation_targets = y_true\n            \n        else:\n            params.all_validation_preds = np.concatenate((params.all_validation_preds,y_pred),axis=0)\n            params.all_validation_targets = np.concatenate((params.all_validation_targets,y_true),axis=0)\n        \n        f1_03 = metrics.f1_score(np.array(y_true),np.array(y_pred) > 0.3, average=\"micro\") \n        \n        return f1_03","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport sklearn.metrics\n\ndef comp_metric(y_pred, y_true, epsilon=1e-9):\n    with torch.no_grad():\n        y_true = y_true.cpu().numpy()\n        y_pred = y_pred[\"clipwise_output\"].cpu().numpy()\n    # Get representative confusion matrices for each label\n    mlbl_cms = sklearn.metrics.multilabel_confusion_matrix(y_true, np.array(y_pred) > 0.3)\n\n    # Get two scores (TP and TN SCORES)\n    tp_scores = np.array([\n        mlbl_cm[1, 1]/(epsilon+mlbl_cm[:, 1].sum()) \\\n        for mlbl_cm in mlbl_cms\n        ])\n    tn_scores = np.array([\n        mlbl_cm[0, 0]/(epsilon+mlbl_cm[:, 0].sum()) \\\n        for mlbl_cm in mlbl_cms\n        ])\n\n    # Get average\n    tp_mean = tp_scores.mean()\n    tn_mean = tn_scores.mean()\n\n    return round((tp_mean+tn_mean)/2, 8)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class train_val_clb(Callback):\n        def after_epoch(self):\n            params.current_epoch = params.current_epoch + 1\n            print(params.current_epoch)\n            \n        def after_fit(self):\n            params.current_epoch = 0\n            \n        def after_validate(self):\n            self.learn.model.change_mode('train')\n            params.mode = 'train'\n            if len(params.all_validation_targets.shape) > 1:\n                num_classes = params.all_validation_preds.shape[1]\n                pad_rows = np.array([[1]*num_classes]*params.padding_factor)\n                params.all_validation_preds = np.concatenate((params.all_validation_preds,pad_rows),axis=0)\n                params.all_validation_targets = np.concatenate((params.all_validation_targets,pad_rows),axis=0)\n                score = sklearn.metrics.average_precision_score(params.all_validation_targets ,params.all_validation_preds , average='macro')\n                print(\"cmAP score on validation data = \",score)\n                params.all_validation_targets = np.array([])\n                params.all_validation_preds = np.array([])\n                \n        def before_validate(self):\n            self.learn.model.change_mode('valid')\n            params.mode = 'valid'\n            \n        def before_batch(self):\n            if params.mode == \"train\":\n                self.learn.dls.bs = params.batch_size\n                data = self.learn.xb\n                targets = self.learn.yb\n                if params.current_epoch < params.mixup_or_cutmix_epochs:\n                    if np.random.rand()<0.5:\n                        new_data, new_targets = self.mixup(data[0],targets[0],0.4)\n                        self.learn.xb = (new_data,)\n                        self.learn.yb = (new_targets,)\n                        params.mixup_or_cutmix = \"mixup\"\n\n                    else:\n                        new_data, new_targets = self.cutmix(data[0], targets[0],0.4)\n                        self.learn.xb = (new_data,)\n                        self.learn.yb = (new_targets,)\n                        params.mixup_or_cutmix = \"cutmix\"\n \n            else:\n                self.learn.dls.bs = params.batch_size\n                \n        def cutmix(self,data, targets, alpha):\n            indices = torch.randperm(data.size(0))\n            shuffled_data = data[indices]\n            shuffled_targets = targets[indices]\n\n            lam = np.random.beta(alpha, alpha)\n            bbx1, bby1, bbx2, bby2 = self.rand_bbox(data.size(), lam)\n            data[:, :, bbx1:bbx2, bby1:bby2] = data[indices, :, bbx1:bbx2, bby1:bby2]\n            # adjust lambda to exactly match pixel ratio\n            lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (data.size()[-1] * data.size()[-2]))\n\n            new_targets = [targets, shuffled_targets, lam]\n            return data, new_targets\n\n        def mixup(self,data, targets, alpha):\n            indices = torch.randperm(data.size(0))\n            shuffled_data = data[indices]\n            shuffled_targets = targets[indices]\n\n            lam = np.random.beta(alpha, alpha)\n            new_data = data * lam + shuffled_data * (1 - lam)\n            new_targets = [targets, shuffled_targets, lam]\n            return new_data, new_targets\n        \n        def rand_bbox(self,size, lam):\n            W = size[2]\n            H = size[3]\n            cut_rat = np.sqrt(1. - lam)\n            cut_w = int(W * cut_rat)\n            cut_h = int(H * cut_rat)\n\n            # uniform\n            cx = np.random.randint(W)\n            cy = np.random.randint(H)\n\n            bbx1 = np.clip(cx - cut_w // 2, 0, W)\n            bby1 = np.clip(cy - cut_h // 2, 0, H)\n            bbx2 = np.clip(cx + cut_w // 2, 0, W)\n            bby2 = np.clip(cy + cut_h // 2, 0, H)\n\n            return bbx1, bby1, bbx2, bby2","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = building_model(num_classes=len(dls.vocab),pretrained=params.pretrained)\n#model = resnet18(num_classes=152)\nlearn = Learner(dls,model,loss_func=loss_fn,metrics=MetricMeter,cbs=[train_val_clb,SaveModelCallback(monitor='MetricMeter',comp=np.greater)])\n#learn.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.model.load_state_dict(torch.load('/kaggle/input/fastai-effb0-base-model-birdclef2023/audio_model.pt'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#learn.model = load_learner(\"../input/fastai-effb0-base-model/model.pkl\").model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.fit_flat_cos(15,lr = params.LR ,wd=params.WEIGHT_DECAY)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#with learn.parallel_ctx(): learn.fit_flat_cos(10,lr = params.LR ,wd=params.WEIGHT_DECAY)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.save(learn.model.state_dict(), \"audio_model.pt\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#learn.export(fname='model.pkl')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# references","metadata":{}},{"cell_type":"markdown","source":"1- https://www.kaggle.com/code/kaerunantoka/birdclef2022-audio-to-numpy-1-4/notebook\n\n2- https://www.kaggle.com/code/kaerunantoka/birdclef2022-n001-training/notebook","metadata":{}}]}