{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":44224,"databundleVersionId":5188730,"sourceType":"competition"},{"sourceId":5148212,"sourceType":"datasetVersion","datasetId":2991134},{"sourceId":5184098,"sourceType":"datasetVersion","datasetId":3014066},{"sourceId":7150058,"sourceType":"datasetVersion","datasetId":4128151},{"sourceId":7229630,"sourceType":"datasetVersion","datasetId":4163945,"isSourceIdPinned":true}],"dockerImageVersionId":30407,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Baseline for Pytorch Lightning based submission \n\n**Step 1: For generating spectrograms :** https://www.kaggle.com/code/nischaydnk/split-creating-melspecs-stage-1\n\n**Step 2: Training Notebook with Pytorch Lightning:** https://www.kaggle.com/code/nischaydnk/birdclef-2023-pytorch-lightning-training-w-cmap\n\nFeel free to reach out in comments incase you find bugs or have doubts!!","metadata":{}},{"cell_type":"code","source":"!pip install --no-index --find-links /kaggle/input/onnxruntime/ onnxruntime","metadata":{"execution":{"iopub.status.busy":"2023-12-18T11:37:33.091527Z","iopub.execute_input":"2023-12-18T11:37:33.092025Z","iopub.status.idle":"2023-12-18T11:37:47.406956Z","shell.execute_reply.started":"2023-12-18T11:37:33.091980Z","shell.execute_reply":"2023-12-18T11:37:47.405598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport torch\nimport os\nimport pytorch_lightning as pl\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn import model_selection\nimport torchvision.transforms as transforms\nimport torchvision.io \nimport torchaudio\nimport torchaudio.functional as F\nimport torch.nn.functional as Fnn\nimport librosa\nfrom PIL import Image\nimport albumentations as alb\nimport torch.multiprocessing as mp\nimport warnings\n\nwarnings.filterwarnings('ignore')\nfrom pytorch_lightning.callbacks import ModelCheckpoint, BackboneFinetuning, EarlyStopping\nimport torch.nn as nn\nfrom torch.nn.functional import cross_entropy\nimport torchmetrics\nimport timm\nfrom pathlib import Path\n\nfrom tqdm.notebook import tqdm\nimport onnxruntime\nimport concurrent","metadata":{"execution":{"iopub.status.busy":"2023-12-18T11:37:47.409539Z","iopub.execute_input":"2023-12-18T11:37:47.409955Z","iopub.status.idle":"2023-12-18T11:38:06.462644Z","shell.execute_reply.started":"2023-12-18T11:37:47.409901Z","shell.execute_reply":"2023-12-18T11:38:06.461073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc","metadata":{"execution":{"iopub.status.busy":"2023-12-18T11:38:06.464720Z","iopub.execute_input":"2023-12-18T11:38:06.465328Z","iopub.status.idle":"2023-12-18T11:38:06.471863Z","shell.execute_reply.started":"2023-12-18T11:38:06.465281Z","shell.execute_reply":"2023-12-18T11:38:06.470318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import librosa as lb\nimport librosa.display as lbd\nimport soundfile as sf\nfrom  soundfile import SoundFile ","metadata":{"execution":{"iopub.status.busy":"2023-12-18T11:38:06.474853Z","iopub.execute_input":"2023-12-18T11:38:06.475244Z","iopub.status.idle":"2023-12-18T11:38:06.506131Z","shell.execute_reply.started":"2023-12-18T11:38:06.475208Z","shell.execute_reply":"2023-12-18T11:38:06.504725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config:\n    num_classes = 264\n    batch_size = 12\n    PRECISION = 32    \n    seed = 2023\n    model = \"tf_efficientnet_b1_ns\"\n    pretrained = False\n    use_mixup = False\n    mixup_alpha = 0.2   \n    DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')    \n    \n    nfft = 1024\n    power = 2\n    hop = nfft // 2\n    fmin = 50\n    fmax = None\n    constPres = 2e-3\n    minAbsPres = 2e-5\n    maxAbsPres = 20.0\n    normalize_wrt = \"meanPresBand\"\n    bandpass = True\n    bandpass_min = 1500\n    bandpass_max = 15000\n    bandpass_order = 2\n\n    data_root = \"/kaggle/input/birdclef-2023/\"\n    train_path = \"/kaggle/input/birdclef-2023/train_metadata.csv\"\n    #train_path = '/kaggle/input/bc2023-train-val-df/train.csv'\n    #valid_path = \"/kaggle/input/melspecs-v8/valid_meta.csv\"\n    \n    test_path = '/kaggle/input/birdclef-2023/test_soundscapes/'\n    SR = 32000\n    DURATION = 5\n    Audio_length = SR * DURATION\n    LR = 5e-4\n    \n    model_output = ['/kaggle/input/sed-callsong-onnx-v14/sed_callsong_v16.onnx']\n    #model_ckpt = '/kaggle/input/b23-sed-selfsuper-v3/exp1/tf_efficientnet_b1_ns_ap.ckpt'\n    ","metadata":{"papermill":{"duration":0.099568,"end_time":"2022-04-22T06:00:18.542447","exception":false,"start_time":"2022-04-22T06:00:18.442879","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-12-18T11:38:06.508334Z","iopub.execute_input":"2023-12-18T11:38:06.508831Z","iopub.status.idle":"2023-12-18T11:38:06.522005Z","shell.execute_reply.started":"2023-12-18T11:38:06.508769Z","shell.execute_reply":"2023-12-18T11:38:06.520385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"song_classes = np.load('/kaggle/input/sed-callsong-onnx-v14/song_classes.npy')\nsong_classes = list(song_classes)","metadata":{"execution":{"iopub.status.busy":"2023-12-18T11:38:06.526619Z","iopub.execute_input":"2023-12-18T11:38:06.527095Z","iopub.status.idle":"2023-12-18T11:38:06.541438Z","shell.execute_reply.started":"2023-12-18T11:38:06.527052Z","shell.execute_reply":"2023-12-18T11:38:06.540195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Config.num_classes = 264 + len(song_classes)","metadata":{"execution":{"iopub.status.busy":"2023-12-18T11:38:06.543010Z","iopub.execute_input":"2023-12-18T11:38:06.543390Z","iopub.status.idle":"2023-12-18T11:38:06.551069Z","shell.execute_reply.started":"2023-12-18T11:38:06.543355Z","shell.execute_reply":"2023-12-18T11:38:06.549725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pl.seed_everything(Config.seed, workers=True)","metadata":{"execution":{"iopub.status.busy":"2023-12-18T11:38:06.552469Z","iopub.execute_input":"2023-12-18T11:38:06.552813Z","iopub.status.idle":"2023-12-18T11:38:06.573266Z","shell.execute_reply.started":"2023-12-18T11:38:06.552781Z","shell.execute_reply":"2023-12-18T11:38:06.571527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def config_to_dict(cfg):\n    return dict((name, getattr(cfg, name)) for name in dir(cfg) if not name.startswith('__'))","metadata":{"papermill":{"duration":0.033041,"end_time":"2022-04-22T06:00:18.664481","exception":false,"start_time":"2022-04-22T06:00:18.63144","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-12-18T11:38:06.574999Z","iopub.execute_input":"2023-12-18T11:38:06.575450Z","iopub.status.idle":"2023-12-18T11:38:06.583126Z","shell.execute_reply.started":"2023-12-18T11:38:06.575409Z","shell.execute_reply":"2023-12-18T11:38:06.581619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def compute_melspec(y, sr, n_mels, fmin, fmax):\n    \"\"\"\n    Computes a mel-spectrogram and puts it at decibel scale\n    Arguments:\n        y {np array} -- signal\n        params {AudioParams} -- Parameters to use for the spectrogram. Expected to have the attributes sr, n_mels, f_min, f_max\n    Returns:\n        np array -- Mel-spectrogram\n    \"\"\"\n    melspec = lb.feature.melspectrogram(\n        y=y, sr=sr, n_mels=n_mels, fmin=fmin, fmax=fmax,\n    )\n\n    melspec = lb.power_to_db(melspec).astype(np.float32)\n    return melspec\n\ndef mono_to_color(X, eps=1e-6, mean=None, std=None):\n    mean = mean or X.mean()\n    std = std or X.std()\n    X = (X - mean) / (std + eps)\n    \n    _min, _max = X.min(), X.max()\n\n    if (_max - _min) > eps:\n        V = np.clip(X, _min, _max)\n        V = 255 * (V - _min) / (_max - _min)\n        V = V.astype(np.uint8)\n    else:\n        V = np.zeros_like(X, dtype=np.uint8)\n\n    return V\n\ndef crop_or_pad(y, length, is_train=True, start=None):\n    if len(y) < length:\n        y = np.concatenate([y, np.zeros(length - len(y))])\n        \n        n_repeats = length // len(y)\n        epsilon = length % len(y)\n        \n        y = np.concatenate([y]*n_repeats + [y[:epsilon]])\n        \n    elif len(y) > length:\n        if not is_train:\n            start = start or 0\n        else:\n            start = start or np.random.randint(len(y) - length)\n\n        y = y[start:start + length]\n\n    return y\n","metadata":{"papermill":{"duration":58.466679,"end_time":"2022-04-22T06:01:17.158088","exception":false,"start_time":"2022-04-22T06:00:18.691409","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-12-18T11:38:06.589390Z","iopub.execute_input":"2023-12-18T11:38:06.590179Z","iopub.status.idle":"2023-12-18T11:38:06.608139Z","shell.execute_reply.started":"2023-12-18T11:38:06.590137Z","shell.execute_reply":"2023-12-18T11:38:06.607065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def compute_melspec4(y, sr, n_mels, fmin, fmax, power, nfft,hoplength,constPres,minAbsPres, maxAbsPres, melfb):\n    \"\"\"\n    Computes a mel-spectrogram and puts it at decibel scale\n    Arguments:\n        y {np array} -- signal\n        params {AudioParams} -- Parameters to use for the spectrogram. Expected to have the attributes sr, n_mels, f_min, f_max\n    Returns:\n        np array -- Mel-spectrogram\n    \"\"\"\n    #melfb = librosa.filters.mel(sr=sr, n_fft=nfft)\n    \n    #spec = librosa.stft(y=y, n_fft=nfft, hop_length=hoplength, win_length=nfft, window='hann')\n    window = torch.hann_window(nfft)\n    \n    spec = torch.stft(input=y, n_fft=nfft, hop_length=hoplength, win_length=nfft, window=window, return_complex=True)\n    \n    spec = torch.abs(spec)\n    meanPres = torch.mean(spec)\n    spec = (spec/meanPres) * constPres \n    spec[spec < minAbsPres] = minAbsPres\n    spec = spec / maxAbsPres\n    spec = torch.pow(spec,power)\n    \n\n    melspec1 = torch.matmul(melfb,spec)\n    db_melspec = 10.0 * torch.log10(melspec1)\n    \n    db_melspec -= torch.min(db_melspec)\n    db_melspec /= torch.max(db_melspec)\n    \n    #melspec = lb.feature.melspectrogram(y=y, sr=sr, n_mels=n_mels, fmin=fmin, fmax=fmax)\n    #db_melspec = lb.power_to_db(melspec).astype(np.float32)\n    \n    return db_melspec","metadata":{"execution":{"iopub.status.busy":"2023-12-18T11:38:06.609620Z","iopub.execute_input":"2023-12-18T11:38:06.610443Z","iopub.status.idle":"2023-12-18T11:38:06.626074Z","shell.execute_reply.started":"2023-12-18T11:38:06.610401Z","shell.execute_reply":"2023-12-18T11:38:06.624715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv(Config.train_path)\nConfig.num_classes = len(df_train.primary_label.unique())","metadata":{"papermill":{"duration":0.035353,"end_time":"2022-04-22T06:01:17.283888","exception":false,"start_time":"2022-04-22T06:01:17.248535","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-12-18T11:38:06.627574Z","iopub.execute_input":"2023-12-18T11:38:06.628005Z","iopub.status.idle":"2023-12-18T11:38:06.808865Z","shell.execute_reply.started":"2023-12-18T11:38:06.627966Z","shell.execute_reply":"2023-12-18T11:38:06.807627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = pd.DataFrame(\n     [(path.stem, *path.stem.split(\"_\"), path) for path in Path(Config.test_path).glob(\"*.ogg\")],\n    columns = [\"filename\", \"name\" ,\"id\", \"path\"]\n)\nprint(df_test.shape)\ndf_test.head()","metadata":{"execution":{"iopub.status.busy":"2023-12-18T11:38:06.810623Z","iopub.execute_input":"2023-12-18T11:38:06.811168Z","iopub.status.idle":"2023-12-18T11:38:06.851328Z","shell.execute_reply.started":"2023-12-18T11:38:06.811114Z","shell.execute_reply":"2023-12-18T11:38:06.850210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_test = pd.concat([df_test,df_test,df_test]).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-12-18T11:38:06.852868Z","iopub.execute_input":"2023-12-18T11:38:06.853524Z","iopub.status.idle":"2023-12-18T11:38:06.858189Z","shell.execute_reply.started":"2023-12-18T11:38:06.853486Z","shell.execute_reply":"2023-12-18T11:38:06.856620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.signal import butter, sosfilt, lfilter\n\ndef butter_bandpass(lowcut, highcut, fs, order=5):\n    nyq = 0.5 * fs\n    low = lowcut / nyq\n    high = highcut / nyq\n    b, a = butter(order, [low, high], btype='band')\n    return b, a\n\n\ndef butter_bandpass_filter(data, lowcut, highcut, fs, order=5):\n    b, a = butter_bandpass(lowcut, highcut, fs, order=order)\n    y = lfilter(b, a, data)\n    return y","metadata":{"execution":{"iopub.status.busy":"2023-12-18T11:38:06.859918Z","iopub.execute_input":"2023-12-18T11:38:06.860324Z","iopub.status.idle":"2023-12-18T11:38:06.870734Z","shell.execute_reply.started":"2023-12-18T11:38:06.860285Z","shell.execute_reply":"2023-12-18T11:38:06.869456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import albumentations as A\ndef get_train_transform():\n    return A.Compose([\n        A.HorizontalFlip(p=0.5),\n        A.OneOf([\n                A.Cutout(max_h_size=5, max_w_size=16),\n                A.CoarseDropout(max_holes=4),\n            ], p=0.5),\n    ])","metadata":{"execution":{"iopub.status.busy":"2023-12-18T11:38:06.872702Z","iopub.execute_input":"2023-12-18T11:38:06.873969Z","iopub.status.idle":"2023-12-18T11:38:06.888170Z","shell.execute_reply.started":"2023-12-18T11:38:06.873921Z","shell.execute_reply":"2023-12-18T11:38:06.886635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class BirdDataset(torch.utils.data.Dataset):\n\n    def __init__(self, data, sr = Config.SR, duration = Config.DURATION, cfg = Config, train = False,\n                res_type=\"kaiser_fast\", resample=True,):\n        \n        self.data = data\n        self.sr = sr\n        \n        \n        self.res_type = res_type\n        self.resample = resample\n\n        self.duration = duration\n        self.audio_length = Config.Audio_length\n        self.step = self.audio_length\n        \n        self.n_mels = 128\n        self.fmin = cfg.fmin\n        self.fmax = cfg.fmax or self.sr//2\n        self.power = cfg.power\n        self.nfft = cfg.nfft\n        self.hoplength = cfg.hop\n        self.constPres = cfg.constPres\n        self.minAbsPres = cfg.minAbsPres\n        self.maxAbsPres = cfg.maxAbsPres\n        #self.melfb = librosa.filters.mel(sr=Config.SR, n_fft=Config.nfft)\n        fbanks = F.melscale_fbanks(int(Config.nfft // 2 + 1),n_mels=128,f_min=0.,f_max=cfg.SR/2.,\n                        sample_rate=cfg.SR,\n                        norm='slaney')\n        \n        self.melfb = torch.transpose(fbanks,1,0)\n        \n        #if train:\n        #    self.img_dir = Config.train_images\n        #else:\n        #    self.img_dir = Config.valid_images\n\n    def __len__(self):\n        return len(self.data)\n\n    @staticmethod\n    def normalize(image):\n        image = image / 255.0\n        #image = torch.stack([image, image, image])\n        return image\n    \n    def audio_to_image(self, audio):\n        \n        melspec = compute_melspec4(audio,self.sr,self.n_mels,self.fmin,self.fmax,self.power,self.nfft, \n                                   self.hoplength,self.constPres,self.minAbsPres,self.maxAbsPres,self.melfb) \n        \n        image = melspec\n        \n        return image\n    \n    def read_file(self, filepath):\n        audio, orig_sr = sf.read(filepath, dtype=\"float32\")\n        #audio, orig_sr = torchaudio.load(filepath, frame_offset=0)\n        #print(audio.shape)\n\n        if self.resample and orig_sr != self.sr:\n            audio = lb.resample(audio, orig_sr, self.sr, res_type=self.res_type)\n            \n        if Config.bandpass:\n            audio = butter_bandpass_filter(audio,Config.bandpass_min,Config.bandpass_max,fs=orig_sr,order=Config.bandpass_order)\n        \n        audio = torch.Tensor(audio)\n        \n        audios = []\n        for i in range(self.audio_length, len(audio) + self.step, self.step):\n            start = max(0, i - self.audio_length)\n            end = start + self.audio_length\n            audios.append(audio[start:end])\n            \n        if len(audios[-1]) < self.audio_length:\n            audios = audios[:-1]\n            \n        images = []\n            \n        for audio in audios:\n            #image = self.audio_to_image(torch.from_numpy(audio))\n            image = self.audio_to_image(audio)\n            #print(image.shape)\n            image = torch.stack([image, image, image])\n            #print(image.shape)\n            images.append(image)\n\n        images = np.stack(images)\n        \n        \n        return images\n    \n        \n    def __getitem__(self, idx):\n        return self.read_file(self.data.loc[idx, \"path\"])","metadata":{"execution":{"iopub.status.busy":"2023-12-18T11:38:06.890217Z","iopub.execute_input":"2023-12-18T11:38:06.890956Z","iopub.status.idle":"2023-12-18T11:38:06.916316Z","shell.execute_reply.started":"2023-12-18T11:38:06.890904Z","shell.execute_reply":"2023-12-18T11:38:06.914926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds_test = BirdDataset(\n    df_test, \n    sr = Config.SR,\n    duration = Config.DURATION,\n    cfg = Config,\n)","metadata":{"papermill":{"duration":0.036289,"end_time":"2022-04-22T06:01:17.539606","exception":false,"start_time":"2022-04-22T06:01:17.503317","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-12-18T11:38:06.918373Z","iopub.execute_input":"2023-12-18T11:38:06.919077Z","iopub.status.idle":"2023-12-18T11:38:07.024762Z","shell.execute_reply.started":"2023-12-18T11:38:06.918845Z","shell.execute_reply":"2023-12-18T11:38:07.022985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds_test[0].shape","metadata":{"execution":{"iopub.status.busy":"2023-12-18T11:38:07.027521Z","iopub.execute_input":"2023-12-18T11:38:07.028958Z","iopub.status.idle":"2023-12-18T11:38:08.492645Z","shell.execute_reply.started":"2023-12-18T11:38:07.028870Z","shell.execute_reply":"2023-12-18T11:38:08.490982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_batch(img_ds, num_items, num_rows, num_cols, predict_arr=None):\n    fig = plt.figure(figsize=(12, 6))    \n    img_index = np.random.randint(0, 120, size=num_items)\n    index=0\n    for ind in img_index:  # list first 9 images\n        img = img_ds[0][ind]\n        \n        ax = fig.add_subplot(num_rows, num_cols, index + 1, xticks=[], yticks=[])\n        if isinstance(img, torch.Tensor):\n            img = img.detach().numpy()\n        if isinstance(img, np.ndarray):\n            #img = img.transpose(1, 2, 0)\n            #ax.imshow(img)     \n            #print(img.shape)\n            ax.pcolormesh(img[0,:,:])\n            \n        title = f\"Spec\"\n        ax.set_title(title)  \n        index+=1","metadata":{"execution":{"iopub.status.busy":"2023-12-18T11:38:08.494415Z","iopub.execute_input":"2023-12-18T11:38:08.494938Z","iopub.status.idle":"2023-12-18T11:38:08.507203Z","shell.execute_reply.started":"2023-12-18T11:38:08.494861Z","shell.execute_reply":"2023-12-18T11:38:08.505521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#show_batch(ds_test, 4, 2, 2)","metadata":{"papermill":{"duration":0.584852,"end_time":"2022-04-22T06:01:18.338238","exception":false,"start_time":"2022-04-22T06:01:17.753386","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-12-18T11:38:08.509308Z","iopub.execute_input":"2023-12-18T11:38:08.510612Z","iopub.status.idle":"2023-12-18T11:38:08.521662Z","shell.execute_reply.started":"2023-12-18T11:38:08.510563Z","shell.execute_reply":"2023-12-18T11:38:08.520566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.optim.lr_scheduler import CosineAnnealingLR, CosineAnnealingWarmRestarts, ReduceLROnPlateau, OneCycleLR\n\ndef get_optimizer(lr, params):\n    model_optimizer = torch.optim.Adam(\n            filter(lambda p: p.requires_grad, params), \n            lr=lr,\n            weight_decay=Config.weight_decay\n        )\n    interval = \"epoch\"\n    \n    lr_scheduler = CosineAnnealingWarmRestarts(\n                            model_optimizer, \n                            T_0=Config.epochs, \n                            T_mult=1, \n                            eta_min=1e-6, \n                            last_epoch=-1\n                        )\n\n    return {\n        \"optimizer\": model_optimizer, \n        \"lr_scheduler\": {\n            \"scheduler\": lr_scheduler,\n            \"interval\": interval,\n            \"monitor\": \"val_loss\",\n            \"frequency\": 1\n        }\n    }","metadata":{"papermill":{"duration":0.048043,"end_time":"2022-04-22T06:01:22.109544","exception":false,"start_time":"2022-04-22T06:01:22.061501","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-12-18T11:38:08.523633Z","iopub.execute_input":"2023-12-18T11:38:08.525094Z","iopub.status.idle":"2023-12-18T11:38:08.535702Z","shell.execute_reply.started":"2023-12-18T11:38:08.525037Z","shell.execute_reply":"2023-12-18T11:38:08.534608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sklearn.metrics\n\ndef padded_cmap(solution, submission, padding_factor=5):\n    solution = solution#.drop(['row_id'], axis=1, errors='ignore')\n    submission = submission#.drop(['row_id'], axis=1, errors='ignore')\n    new_rows = []\n    for i in range(padding_factor):\n        new_rows.append([1 for i in range(len(solution.columns))])\n    new_rows = pd.DataFrame(new_rows)\n    new_rows.columns = solution.columns\n    padded_solution = pd.concat([solution, new_rows]).reset_index(drop=True).copy()\n    padded_submission = pd.concat([submission, new_rows]).reset_index(drop=True).copy()\n    score = sklearn.metrics.average_precision_score(\n        padded_solution.values,\n        padded_submission.values,\n        average='macro',\n    )\n    return score\n\ndef map_score(solution, submission):\n    solution = solution#.drop(['row_id'], axis=1, errors='ignore')\n    submission = submission#.drop(['row_id'], axis=1, errors='ignore')\n    score = sklearn.metrics.average_precision_score(\n        solution.values,\n        submission.values,\n        average='micro',\n    )\n    return score","metadata":{"execution":{"iopub.status.busy":"2023-12-18T11:38:08.536996Z","iopub.execute_input":"2023-12-18T11:38:08.538106Z","iopub.status.idle":"2023-12-18T11:38:08.550811Z","shell.execute_reply.started":"2023-12-18T11:38:08.538061Z","shell.execute_reply":"2023-12-18T11:38:08.549212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def init_layer(layer):\n    nn.init.xavier_uniform_(layer.weight)\n\n    if hasattr(layer, \"bias\"):\n        if layer.bias is not None:\n            layer.bias.data.fill_(0.)\n\n\ndef init_bn(bn):\n    bn.bias.data.fill_(0.)\n    bn.weight.data.fill_(1.0)\n\n\ndef interpolate(x: torch.Tensor, ratio: int):\n    \"\"\"Interpolate data in time domain. This is used to compensate the\n    resolution reduction in downsampling of a CNN.\n\n    Args:\n      x: (batch_size, time_steps, classes_num)\n      ratio: int, ratio to interpolate\n    Returns:\n      upsampled: (batch_size, time_steps * ratio, classes_num)\n    \"\"\"\n    (batch_size, time_steps, classes_num) = x.shape\n    upsampled = x[:, :, None, :].repeat(1, 1, ratio, 1)\n    upsampled = upsampled.reshape(batch_size, time_steps * ratio, classes_num)\n    return upsampled\n\n\ndef pad_framewise_output(framewise_output: torch.Tensor, frames_num: int):\n    \"\"\"Pad framewise_output to the same length as input frames. The pad value\n    is the same as the value of the last frame.\n    Args:\n      framewise_output: (batch_size, frames_num, classes_num)\n      frames_num: int, number of frames to pad\n    Outputs:\n      output: (batch_size, frames_num, classes_num)\n    \"\"\"\n    pad = framewise_output[:, -1:, :].repeat(\n        1, frames_num - framewise_output.shape[1], 1)\n    \"\"\"tensor for padding\"\"\"\n\n    output = torch.cat((framewise_output, pad), dim=1)\n    \"\"\"(batch_size, frames_num, classes_num)\"\"\"\n\n    return output","metadata":{"execution":{"iopub.status.busy":"2023-12-18T11:38:08.552432Z","iopub.execute_input":"2023-12-18T11:38:08.553868Z","iopub.status.idle":"2023-12-18T11:38:08.568759Z","shell.execute_reply.started":"2023-12-18T11:38:08.553820Z","shell.execute_reply":"2023-12-18T11:38:08.567679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Identity(nn.Module):\n    def __init__(self):\n        super().__init__()\n        \n    def forward(self, x):\n        return x\n\nclass AttBlock(nn.Module):\n    def __init__(self,\n                 in_features: int,\n                 out_features: int,\n                 activation=\"linear\",\n                 temperature=1.0):\n        super().__init__()\n\n        self.activation = activation\n        self.temperature = temperature\n        self.att = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n        self.cla = nn.Conv1d(\n            in_channels=in_features,\n            out_channels=out_features,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            bias=True)\n\n        self.bn_att = nn.BatchNorm1d(out_features)\n        self.init_weights()\n\n    def init_weights(self):\n        init_layer(self.att)\n        init_layer(self.cla)\n        init_bn(self.bn_att)\n\n    def forward(self, x):\n        # x: (n_samples, n_in, n_time)\n        norm_att = torch.softmax(torch.clamp(self.att(x), -10, 10), dim=-1)\n        cla = self.nonlinear_transform(self.cla(x))\n        x = torch.sum(norm_att * cla, dim=2)\n        return x, norm_att, cla\n\n    def nonlinear_transform(self, x):\n        if self.activation == 'linear':\n            return x\n        elif self.activation == 'sigmoid':\n            return torch.sigmoid(x)","metadata":{"execution":{"iopub.status.busy":"2023-12-18T11:38:08.571228Z","iopub.execute_input":"2023-12-18T11:38:08.571858Z","iopub.status.idle":"2023-12-18T11:38:08.591794Z","shell.execute_reply.started":"2023-12-18T11:38:08.571807Z","shell.execute_reply":"2023-12-18T11:38:08.590485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class BirdClefModel(pl.LightningModule):\n    def __init__(self, model_name=Config.model, num_classes = Config.num_classes, pretrained = Config.pretrained):\n        super().__init__()\n        self.num_classes = num_classes\n        \n        self.interpolate_ratio = 32\n\n        self.backbone = timm.create_model(model_name, pretrained=pretrained)\n\n        self.backbone.global_pool = nn.Sequential()\n        self.backbone.classifier = nn.Sequential()\n\n        #if 'res' in model_name:\n        #    self.in_features = self.backbone.fc.in_features\n        #    self.backbone.fc = nn.Linear(self.in_features, num_classes)\n        #elif 'dense' in model_name:\n        #    self.in_features = self.backbone.classifier.in_features\n        #    self.backbone.classifier = nn.Linear(self.in_features, num_classes)\n        #elif 'efficientnet' in model_name:\n        #    self.in_features = self.backbone.classifier.in_features\n        #    self.backbone.classifier = nn.Sequential(\n        #        nn.Linear(self.in_features, num_classes)\n        #    )\n        \n        self.fc1 = nn.Linear(1280, 2048, bias=True)\n        self.att_block = AttBlock(2048, self.num_classes, activation='linear')\n        \n        self.loss_function = nn.BCEWithLogitsLoss() \n\n    def forward(self,images):\n        #print(images.shape)\n        frames_num = images.shape[3]\n        x = self.backbone(images)\n        #print(x.shape)\n        \n        # Aggregate in frequency axis\n        x = torch.mean(x, dim=3)\n        \n        x1 = Fnn.max_pool1d(x, kernel_size=3, stride=1, padding=1)\n        x2 = Fnn.avg_pool1d(x, kernel_size=3, stride=1, padding=1)\n        \n        x = x1 + x2\n\n        x = Fnn.dropout(x, p=0.5, training=self.training)\n        x = x.transpose(1, 2)\n        x = Fnn.relu_(self.fc1(x))\n        x = x.transpose(1, 2)\n        x = Fnn.dropout(x, p=0.5, training=self.training)\n\n        (clipwise_output, norm_att, segmentwise_output) = self.att_block(x)\n        #print(clipwise_output.shape, segmentwise_output.shape)\n        segmentwise_output = segmentwise_output.transpose(1, 2)\n        #print(clipwise_output.shape, segmentwise_output.shape)\n\n        # Get framewise output\n        framewise_output = interpolate(segmentwise_output,\n                                       self.interpolate_ratio)\n        \n        #print(framewise_output.shape, segmentwise_output.shape)\n        \n        framewise_output = pad_framewise_output(framewise_output, frames_num)\n\n        output_dict = {\n            'framewise_output': framewise_output,\n            'clipwise_output': clipwise_output\n        }\n\n        return output_dict\n        \n    def configure_optimizers(self):\n        return get_optimizer(lr=Config.LR, params=self.parameters())\n\n    def training_step(self, batch, batch_idx):\n        image, target = batch        \n\n        y_pred = self(image)\n        loss = self.loss_function(y_pred['clipwise_output'],target)\n\n        self.log(\"train_loss\", loss, on_step=True, on_epoch=True, prog_bar=True)\n        return loss        \n\n    def validation_step(self, batch, batch_idx):\n        image, target = batch     \n        y_pred = self(image)\n        val_loss = self.loss_function(y_pred['clipwise_output'], target)\n        self.log(\"val_loss\", val_loss, on_step=True, on_epoch=True, logger=True, prog_bar=True)\n        \n        return {\"val_loss\": val_loss, \"logits\": y_pred['clipwise_output'], \"targets\": target}\n    \n    def train_dataloader(self):\n        return self._train_dataloader \n    \n    def validation_dataloader(self):\n        return self._validation_dataloader\n    \n    def validation_epoch_end(self,outputs):\n        avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean()\n        output_val = torch.cat([x['logits'] for x in outputs],dim=0).sigmoid().cpu().detach().numpy()\n        target_val = torch.cat([x['targets'] for x in outputs],dim=0).cpu().detach().numpy()\n        \n        # print(output_val.shape)\n        val_df = pd.DataFrame(target_val, columns = birds)\n        pred_df = pd.DataFrame(output_val, columns = birds)\n        \n        avg_score = padded_cmap(val_df, pred_df, padding_factor = 5)\n        avg_score2 = padded_cmap(val_df, pred_df, padding_factor = 3)\n        avg_score3 = sklearn.metrics.label_ranking_average_precision_score(target_val,output_val)\n        \n#         competition_metrics(output_val,target_val)\n        print(f'epoch {self.current_epoch} validation loss {avg_loss}')\n        print(f'epoch {self.current_epoch} validation C-MAP score pad 5 {avg_score}')\n        print(f'epoch {self.current_epoch} validation C-MAP score pad 3 {avg_score2}')\n        print(f'epoch {self.current_epoch} validation AP score {avg_score3}')\n        \n        \n        val_df.to_pickle('val_df.pkl')\n        pred_df.to_pickle('pred_df.pkl')\n        \n        \n        return {'val_loss': avg_loss,'val_cmap':avg_score}\n    ","metadata":{"papermill":{"duration":0.156714,"end_time":"2022-04-22T06:01:22.301564","exception":false,"start_time":"2022-04-22T06:01:22.14485","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-12-18T11:38:08.594351Z","iopub.execute_input":"2023-12-18T11:38:08.594963Z","iopub.status.idle":"2023-12-18T11:38:08.629401Z","shell.execute_reply.started":"2023-12-18T11:38:08.594896Z","shell.execute_reply":"2023-12-18T11:38:08.627904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_encoder_output(idx):\n    images = ds_test[idx]\n    print(images.shape)\n    with torch.no_grad():\n        logits = encoder_session.run([], {'input': images})[1]\n        print(logits.shape)\n        predictions[idx] = logits\n    return True","metadata":{"execution":{"iopub.status.busy":"2023-12-18T11:38:08.631069Z","iopub.execute_input":"2023-12-18T11:38:08.632116Z","iopub.status.idle":"2023-12-18T11:38:08.648644Z","shell.execute_reply.started":"2023-12-18T11:38:08.632071Z","shell.execute_reply":"2023-12-18T11:38:08.647188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def my_sigmoid(x):\n    return 1 / (1 + np.exp(-x))","metadata":{"execution":{"iopub.status.busy":"2023-12-18T11:38:08.658310Z","iopub.execute_input":"2023-12-18T11:38:08.658743Z","iopub.status.idle":"2023-12-18T11:38:08.665064Z","shell.execute_reply.started":"2023-12-18T11:38:08.658708Z","shell.execute_reply":"2023-12-18T11:38:08.663507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"en = range(len(ds_test))","metadata":{"execution":{"iopub.status.busy":"2023-12-18T11:38:08.666978Z","iopub.execute_input":"2023-12-18T11:38:08.667498Z","iopub.status.idle":"2023-12-18T11:38:08.676314Z","shell.execute_reply.started":"2023-12-18T11:38:08.667443Z","shell.execute_reply":"2023-12-18T11:38:08.674944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(Config.model_output)","metadata":{"execution":{"iopub.status.busy":"2023-12-18T11:38:08.678122Z","iopub.execute_input":"2023-12-18T11:38:08.678574Z","iopub.status.idle":"2023-12-18T11:38:08.691206Z","shell.execute_reply.started":"2023-12-18T11:38:08.678534Z","shell.execute_reply":"2023-12-18T11:38:08.689801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for encoder_state in tqdm(Config.model_output):\n    #encoder_state = Config.model_output\n    encoder_session = onnxruntime.InferenceSession(encoder_state, providers=['CPUExecutionProvider'])\n    print('Encoder from: ', encoder_state)\n    \n    \n    predictions = {}\n    with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:\n        _ = executor.map(get_encoder_output, en)\n        \n                \n    preds = []\n    for i in range(len(ds_test)):\n        preds.append(my_sigmoid(predictions[i]))\n        #preds = np.concatenate(preds)\n    \n    print('\\n\\n')\n    \n    del encoder_session, predictions\n    gc.collect()\n","metadata":{"execution":{"iopub.status.busy":"2023-12-18T11:38:08.693193Z","iopub.execute_input":"2023-12-18T11:38:08.693746Z","iopub.status.idle":"2023-12-18T11:38:43.382005Z","shell.execute_reply.started":"2023-12-18T11:38:08.693704Z","shell.execute_reply":"2023-12-18T11:38:43.380250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2023-12-18T11:38:43.383733Z","iopub.execute_input":"2023-12-18T11:38:43.384374Z","iopub.status.idle":"2023-12-18T11:38:43.390002Z","shell.execute_reply.started":"2023-12-18T11:38:43.384332Z","shell.execute_reply":"2023-12-18T11:38:43.388985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filenames = df_test.filename.values.tolist()\n\nbird_cols = list(pd.get_dummies(df_train['primary_label']).columns) \nclmns = ['row_id'] + bird_cols + song_classes\nsub_df = pd.DataFrame(columns=clmns)","metadata":{"papermill":{"duration":0.052364,"end_time":"2022-04-22T06:01:22.708806","exception":false,"start_time":"2022-04-22T06:01:22.656442","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-12-18T11:38:43.392015Z","iopub.execute_input":"2023-12-18T11:38:43.392392Z","iopub.status.idle":"2023-12-18T11:38:43.431551Z","shell.execute_reply.started":"2023-12-18T11:38:43.392355Z","shell.execute_reply":"2023-12-18T11:38:43.430504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df","metadata":{"execution":{"iopub.status.busy":"2023-12-18T11:38:43.432699Z","iopub.execute_input":"2023-12-18T11:38:43.433083Z","iopub.status.idle":"2023-12-18T11:38:43.453565Z","shell.execute_reply.started":"2023-12-18T11:38:43.433047Z","shell.execute_reply":"2023-12-18T11:38:43.451968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Generate Submission csv","metadata":{}},{"cell_type":"code","source":"for i, file in enumerate(filenames):\n    pred = preds[i]\n    num_rows = len(pred)\n    row_ids = [f'{file}_{(i+1)*5}' for i in range(num_rows)]\n    df = pd.DataFrame(columns=clmns)\n    \n    df['row_id'] = row_ids\n    df[bird_cols + song_classes] = pred\n    \n    sub_df = pd.concat([sub_df,df]).reset_index(drop=True)\n    ","metadata":{"execution":{"iopub.status.busy":"2023-12-18T11:38:43.455607Z","iopub.execute_input":"2023-12-18T11:38:43.456041Z","iopub.status.idle":"2023-12-18T11:38:43.719202Z","shell.execute_reply.started":"2023-12-18T11:38:43.456001Z","shell.execute_reply":"2023-12-18T11:38:43.717723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# max pooling for call and song classes\nfor song in song_classes:\n    sub_df[song[:-5]] = sub_df[[song[:-5], song]].max(axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-12-18T11:38:43.720780Z","iopub.execute_input":"2023-12-18T11:38:43.721217Z","iopub.status.idle":"2023-12-18T11:38:43.888264Z","shell.execute_reply.started":"2023-12-18T11:38:43.721177Z","shell.execute_reply":"2023-12-18T11:38:43.887267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df = sub_df.drop(columns=song_classes)","metadata":{"execution":{"iopub.status.busy":"2023-12-18T11:38:43.889714Z","iopub.execute_input":"2023-12-18T11:38:43.890464Z","iopub.status.idle":"2023-12-18T11:38:43.897370Z","shell.execute_reply.started":"2023-12-18T11:38:43.890422Z","shell.execute_reply":"2023-12-18T11:38:43.895844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df","metadata":{"execution":{"iopub.status.busy":"2023-12-18T11:38:43.899190Z","iopub.execute_input":"2023-12-18T11:38:43.899672Z","iopub.status.idle":"2023-12-18T11:38:43.946638Z","shell.execute_reply.started":"2023-12-18T11:38:43.899627Z","shell.execute_reply":"2023-12-18T11:38:43.945294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2023-12-18T11:38:43.948393Z","iopub.execute_input":"2023-12-18T11:38:43.948768Z","iopub.status.idle":"2023-12-18T11:38:44.008853Z","shell.execute_reply.started":"2023-12-18T11:38:43.948731Z","shell.execute_reply":"2023-12-18T11:38:44.007796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}