{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Thanks to https://github.com/YuanGongND/ssast","metadata":{}},{"cell_type":"code","source":"# !pip download timm===0.4.5 -d ./timm/\n# !pip download torchinfo -d ./torchinfo/","metadata":{"execution":{"iopub.status.busy":"2022-05-15T13:43:44.644056Z","iopub.execute_input":"2022-05-15T13:43:44.644356Z","iopub.status.idle":"2022-05-15T13:43:44.64793Z","shell.execute_reply.started":"2022-05-15T13:43:44.644323Z","shell.execute_reply":"2022-05-15T13:43:44.64722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir ./timm\n!mkdir ./ssastbasepatch400\n!cp ../input/birdclef2022-ssast/timm/timm-0.4.5-py3-none-any.whl ./timm/timm-0.4.5-py3-none-any.whl\n!cp ../input/birdclef2022-ssast/ssastbasepatch400/SSAST-Base-Patch-400.pth ./ssastbasepatch400/SSAST-Base-Patch-400.pth","metadata":{"execution":{"iopub.status.busy":"2022-05-15T13:43:44.66212Z","iopub.execute_input":"2022-05-15T13:43:44.662708Z","iopub.status.idle":"2022-05-15T13:43:48.302105Z","shell.execute_reply.started":"2022-05-15T13:43:44.662672Z","shell.execute_reply":"2022-05-15T13:43:48.300984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install ../input/birdclef2022-ssast/timm/Pillow-9.1.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\n# !pip install ../input/birdclef2022-ssast/timm/certifi-2021.10.8-py2.py3-none-any.whl\n# !pip install ../input/birdclef2022-ssast/timm/charset_normalizer-2.0.12-py3-none-any.whl\n# !pip install ../input/birdclef2022-ssast/timm/idna-3.3-py3-none-any.whl\n# !pip install ../input/birdclef2022-ssast/timm/numpy-1.21.6-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl\n# !pip install ../input/birdclef2022-ssast/timm/requests-2.27.1-py2.py3-none-any.whl\n!pip install ./timm/timm-0.4.5-py3-none-any.whl\n# !pip install ../input/birdclef2022-ssast/timm/torch-1.11.0-cp37-cp37m-manylinux1_x86_64.whl\n# !pip install ../input/birdclef2022-ssast/timm/torchvision-0.12.0-cp37-cp37m-manylinux1_x86_64.whl\n# !pip install ../input/birdclef2022-ssast/timm/typing_extensions-4.2.0-py3-none-any.whl\n# !pip install ../input/birdclef2022-ssast/timm/urllib3-1.26.9-py2.py3-none-any.whl\n# !pip install ../input/birdclef2022-ssast/torchinfo/torchinfo-1.6.5-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2022-05-15T13:43:48.304697Z","iopub.execute_input":"2022-05-15T13:43:48.305013Z","iopub.status.idle":"2022-05-15T13:44:16.898342Z","shell.execute_reply.started":"2022-05-15T13:43:48.304971Z","shell.execute_reply":"2022-05-15T13:44:16.897392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install timm===0.4.5\n# !pip install torchinfo","metadata":{"execution":{"iopub.status.busy":"2022-05-15T13:44:16.899984Z","iopub.execute_input":"2022-05-15T13:44:16.900302Z","iopub.status.idle":"2022-05-15T13:44:16.906447Z","shell.execute_reply.started":"2022-05-15T13:44:16.900258Z","shell.execute_reply":"2022-05-15T13:44:16.905275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.nn as nn\nimport torch\nimport torchaudio\nfrom torchaudio import transforms\nimport sys\nimport os\nfrom timm.models.layers import trunc_normal_\nimport timm\nimport numpy as np\nfrom timm.models.layers import to_2tuple\nfrom random import randrange\nimport pandas as pd\nfrom matplotlib import pyplot as plt\nimport random\nfrom IPython.display import Audio, display\nfrom ast import literal_eval\n# from torchinfo import summary\n\n#from tqdm import tqdm\nfrom collections import defaultdict","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-15T13:44:16.910235Z","iopub.execute_input":"2022-05-15T13:44:16.910832Z","iopub.status.idle":"2022-05-15T13:44:16.918948Z","shell.execute_reply.started":"2022-05-15T13:44:16.910787Z","shell.execute_reply":"2022-05-15T13:44:16.917973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from os.path import exists\n# import requests\n\n# pretrained_mdl_path = './ssastbasepatch400/SSAST-Base-Patch-400.pth'\n# os.makedirs(os.path.dirname('./ssastbasepatch400/'), exist_ok=True)\n# file_exists = exists(pretrained_mdl_path)\n\n# if file_exists == False:\n#     url = 'https://www.dropbox.com/s/ewrzpco95n9jdz6/SSAST-Base-Patch-400.pth?dl=1'\n\n#     r = requests.get(url, allow_redirects=True)\n\n#     open(pretrained_mdl_path, 'wb').write(r.content)\n\npretrained_mdl_path = \"../input/birdclef2022-ssast/ssastbasepatch400/SSAST-Base-Patch-400.pth\"\n","metadata":{"execution":{"iopub.status.busy":"2022-05-15T13:44:16.920848Z","iopub.execute_input":"2022-05-15T13:44:16.92131Z","iopub.status.idle":"2022-05-15T13:44:16.928319Z","shell.execute_reply.started":"2022-05-15T13:44:16.921197Z","shell.execute_reply":"2022-05-15T13:44:16.927396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sys.path.append(\"/data/sls/scratch/yuangong/aed-trans/src/models/\")\n# sys.path.append(\"/data/sls/scratch/yuangong/aed-trans/src/\")","metadata":{"execution":{"iopub.status.busy":"2022-05-15T13:44:16.930292Z","iopub.execute_input":"2022-05-15T13:44:16.930676Z","iopub.status.idle":"2022-05-15T13:44:16.939858Z","shell.execute_reply.started":"2022-05-15T13:44:16.930633Z","shell.execute_reply":"2022-05-15T13:44:16.939061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2022-05-15T13:44:16.941205Z","iopub.execute_input":"2022-05-15T13:44:16.941689Z","iopub.status.idle":"2022-05-15T13:44:16.949585Z","shell.execute_reply.started":"2022-05-15T13:44:16.941645Z","shell.execute_reply":"2022-05-15T13:44:16.948701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"root_dir = '../input/birdclef-2022'\ndf = pd.read_csv(root_dir + '/train_metadata.csv')\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-15T13:44:16.950778Z","iopub.execute_input":"2022-05-15T13:44:16.951247Z","iopub.status.idle":"2022-05-15T13:44:17.041529Z","shell.execute_reply.started":"2022-05-15T13:44:16.951206Z","shell.execute_reply":"2022-05-15T13:44:17.040685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns_of_interest = ['primary_label', 'filename', 'secondary_labels', 'rating']\ndf = df[columns_of_interest]\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-15T13:44:17.042803Z","iopub.execute_input":"2022-05-15T13:44:17.043131Z","iopub.status.idle":"2022-05-15T13:44:17.059171Z","shell.execute_reply.started":"2022-05-15T13:44:17.04309Z","shell.execute_reply":"2022-05-15T13:44:17.058307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import json\n\n# with open('../input/birdclef-2022/scored_birds.json') as sbfile:\n#     scored_birds = json.load(sbfile)\n#     df = df[df['primary_label'].isin(scored_birds)]","metadata":{"execution":{"iopub.status.busy":"2022-05-15T13:44:17.063352Z","iopub.execute_input":"2022-05-15T13:44:17.063614Z","iopub.status.idle":"2022-05-15T13:44:17.069183Z","shell.execute_reply.started":"2022-05-15T13:44:17.063573Z","shell.execute_reply":"2022-05-15T13:44:17.06841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = df.primary_label.unique().tolist()\nlabel2id = {labels[i]: i for i in range(len(labels))}","metadata":{"execution":{"iopub.status.busy":"2022-05-15T13:44:17.070934Z","iopub.execute_input":"2022-05-15T13:44:17.071218Z","iopub.status.idle":"2022-05-15T13:44:17.079492Z","shell.execute_reply.started":"2022-05-15T13:44:17.071182Z","shell.execute_reply":"2022-05-15T13:44:17.07878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_audio_dir = '../input/birdclef-2022/train_audio'","metadata":{"execution":{"iopub.status.busy":"2022-05-15T13:44:17.081286Z","iopub.execute_input":"2022-05-15T13:44:17.081632Z","iopub.status.idle":"2022-05-15T13:44:17.087642Z","shell.execute_reply.started":"2022-05-15T13:44:17.081558Z","shell.execute_reply":"2022-05-15T13:44:17.086838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for i in tqdm(range(len(df))):\n#     filename = df.loc[i,'filename']\n#     sig, sr = torchaudio.load(os.path.join(train_audio_dir, filename))\n#     df.loc[i, 'num_channels'] = sig.shape[0]  # number of audio channels (mono/stereo)\n#     df.loc[i, 'signal_len'] = sig.shape[1]  # signal length\n#     df.loc[i, 'sampling_rate'] = sr\n#     df.loc[i, 'duration'] = sig.shape[1] // sr","metadata":{"execution":{"iopub.status.busy":"2022-05-15T13:44:17.089492Z","iopub.execute_input":"2022-05-15T13:44:17.089883Z","iopub.status.idle":"2022-05-15T13:44:17.097394Z","shell.execute_reply.started":"2022-05-15T13:44:17.089843Z","shell.execute_reply":"2022-05-15T13:44:17.09651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# os.mkdir('./audio_16k/')\n# for i in tqdm(range(len(train_df))):\n#     filename = train_df.loc[i,'filename']\n#     print('sox ' + train_audio_dir + '/audio/' + filename + ' -r 16000 ' + './audio_16k/' + filename)\n#     os.system('sox ' + train_audio_dir + '/audio/' + filename + ' -r 16000 ' + './audio_16k/' + filename)","metadata":{"execution":{"iopub.status.busy":"2022-05-15T13:44:17.100689Z","iopub.execute_input":"2022-05-15T13:44:17.10092Z","iopub.status.idle":"2022-05-15T13:44:17.107129Z","shell.execute_reply.started":"2022-05-15T13:44:17.100886Z","shell.execute_reply":"2022-05-15T13:44:17.106185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df['num_channels'] = df['num_channels'].astype('int64')\n# df['signal_len'] = df['signal_len'].astype('int64')\n# df['sampling_rate'] = df['sampling_rate'].astype('int64')\n# df.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-15T13:44:17.108615Z","iopub.execute_input":"2022-05-15T13:44:17.108992Z","iopub.status.idle":"2022-05-15T13:44:17.119774Z","shell.execute_reply.started":"2022-05-15T13:44:17.108924Z","shell.execute_reply":"2022-05-15T13:44:17.118786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(df['duration'].max(), df['duration'].min())","metadata":{"execution":{"iopub.status.busy":"2022-05-15T13:44:17.121386Z","iopub.execute_input":"2022-05-15T13:44:17.121741Z","iopub.status.idle":"2022-05-15T13:44:17.128218Z","shell.execute_reply.started":"2022-05-15T13:44:17.121704Z","shell.execute_reply":"2022-05-15T13:44:17.127425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_test_split(df, frac=0.2):\n    \n    # get random sample \n    test = df.sample(frac=frac, axis=0, random_state=333)\n\n    # get everything but the test sample\n    train = df.drop(index=test.index)\n    \n    train.reset_index(drop=True, inplace=True)\n    test.reset_index(drop=True, inplace=True)\n\n    return train, test","metadata":{"execution":{"iopub.status.busy":"2022-05-15T13:44:17.129916Z","iopub.execute_input":"2022-05-15T13:44:17.130236Z","iopub.status.idle":"2022-05-15T13:44:17.13927Z","shell.execute_reply.started":"2022-05-15T13:44:17.130179Z","shell.execute_reply":"2022-05-15T13:44:17.138489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data, test_data = train_test_split(df, frac=0.1)\n#train_data = df","metadata":{"execution":{"iopub.status.busy":"2022-05-15T13:44:17.14044Z","iopub.execute_input":"2022-05-15T13:44:17.142002Z","iopub.status.idle":"2022-05-15T13:44:17.153057Z","shell.execute_reply.started":"2022-05-15T13:44:17.141972Z","shell.execute_reply":"2022-05-15T13:44:17.152228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# DataLoader","metadata":{}},{"cell_type":"code","source":"class AudioDataset():\n    def __init__(self, audio_conf, pd_data, label2id, audio_dir):\n        \"\"\"\n        Dataset that manages audio recordings\n        :param audio_conf: Dictionary containing the audio loading and preprocessing settings\n        :param dataset_json_file\n        \"\"\"\n        self.pd_data = pd_data\n        self.data = pd_data.to_dict('index')\n        self.audio_conf = audio_conf\n        self.audio_dir = audio_dir\n        \n        \n        self.melbins = self.audio_conf.get('num_mel_bins')\n        self.freqm = self.audio_conf.get('freqm')\n        self.timem = self.audio_conf.get('timem')\n        self.mixup = self.audio_conf.get('mixup')\n        self.secondary_label = self.audio_conf.get('secondary_label')\n        \n        # dataset spectrogram mean and std, used to normalize the input\n        self.norm_mean = self.audio_conf.get('mean')\n        self.norm_std = self.audio_conf.get('std')\n        # skip_norm is a flag that if you want to skip normalization to compute the normalization stats using src/get_norm_stats.py, if Ture, input normalization will be skipped for correctly calculating the stats.\n        # set it as True ONLY when you are getting the normalization stats.\n        self.skip_norm = self.audio_conf.get('skip_norm') if self.audio_conf.get('skip_norm') else False\n        if self.skip_norm:\n            print('now skip normalization (use it ONLY when you are computing the normalization stats).')\n        else:\n            print('use dataset mean {:.3f} and std {:.3f} to normalize the input.'.format(self.norm_mean, self.norm_std))\n        # if add noise for data augmentation\n        self.noise = self.audio_conf.get('noise')\n        if self.noise == True:\n            print('now use noise augmentation')\n\n        self.index_dict = label2id\n        self.label_num = len(self.index_dict)\n        print('number of classes is {:d}'.format(self.label_num))\n        \n    def _monoToStereo(self, aud):\n        sig, sr = aud\n        if sig.shape[0] == 2:\n            return aud\n        else:\n            stereo_sig = torch.cat([sig, sig])\n\n        return (stereo_sig, sr)\n    \n    def _stereoToMono(self, aud):\n        sig, sr = aud\n        if sig.shape[0] == 1:\n            return aud\n        else:\n            mono_sig = torch.mean(sig, dim=0, keepdim=True)\n        \n        return (mono_sig, sr)\n    \n    def _channelsNumber(self, aud, num_channel=1):\n#         sig, sr = aud\n#         transform = torchaudio.transforms.Resample(sr, 16*1000)\n#         sig = transform(sig)\n#         aud = (sig, 16*1000)\n        if num_channel == 1:\n            return self._stereoToMono(aud)\n        else:\n            return self._monoToStereo(aud)\n\n    def _wav2fbank(self, filename, filename2=None):\n        # mixup\n        path1 = os.path.join(self.audio_dir, filename)\n        if filename2 == None:\n            aud = torchaudio.load(path1)\n            waveform, sr = self._channelsNumber(aud)\n            waveform = waveform - waveform.mean()\n        # mixup\n        else:\n            aud1 = torchaudio.load(os.path.join(self.audio_dir, filename))\n            aud2 = torchaudio.load(os.path.join(self.audio_dir, filename2))\n            waveform1, sr = self._channelsNumber(aud1)\n            waveform2, _ = self._channelsNumber(aud2)\n\n            waveform1 = waveform1 - waveform1.mean()\n            waveform2 = waveform2 - waveform2.mean()\n\n            if waveform1.shape[1] != waveform2.shape[1]:\n                if waveform1.shape[1] > waveform2.shape[1]:\n                    # padding\n                    p = waveform1.shape[1] - waveform2.shape[1]\n                    l = random.randint(0, p)\n                    temp_wav = torch.zeros(1, waveform1.shape[1])\n                    temp_wav[0, l:(waveform2.shape[1] + l)] = waveform2\n                    waveform2 = temp_wav\n                else:\n                    # cutting\n                    p = waveform2.shape[1] - waveform1.shape[1]\n                    l = random.randint(0, p)\n                    waveform2 = waveform2[0, l:(waveform1.shape[1] + l)]\n\n            # sample lambda from uniform distribution\n            #mix_lambda = random.random()\n            # sample lambda from beta distribtion\n            mix_lambda = np.random.beta(10, 10)\n\n            mix_waveform = mix_lambda * waveform1 + (1 - mix_lambda) * waveform2\n            waveform = mix_waveform - mix_waveform.mean()\n\n        fbank = torchaudio.compliance.kaldi.fbank(waveform, htk_compat=True, sample_frequency=sr, use_energy=False,\n                                                  window_type='hanning', num_mel_bins=self.melbins, dither=0.0, frame_shift=10)\n\n        target_length = self.audio_conf.get('target_length')\n        n_frames = fbank.shape[0]\n\n        p = target_length - n_frames\n\n        # cut and pad\n        if p > 0:\n            l = random.randint(0, p)\n            m = torch.nn.ZeroPad2d((0, 0, l, p - l))\n            fbank = m(fbank)\n        elif p < 0:\n            k = -p\n            j = random.randint(0, k)\n            fbank = fbank[j:j+target_length, :]\n\n        if filename2 == None:\n            return fbank, 0\n        else:\n            return fbank, mix_lambda\n\n    def __getitem__(self, index):\n        \"\"\"\n        returns: image, audio, nframes\n        where image is a FloatTensor of size (3, H, W)\n        audio is a FloatTensor of size (N_freq, N_frames) for spectrogram, or (N_frames) for waveform\n        nframes is an integer\n        \"\"\"\n        # do mix-up for this sample (controlled by the given mixup rate)\n        secon_lab = 0.5\n        if random.random() < self.mixup:\n            datum = self.data[index]\n            # find another sample to mix, also do balance sampling\n            # sample the other sample from the multinomial distribution, will make the performance worse\n            # mix_sample_idx = np.random.choice(len(self.data), p=self.sample_weight_file)\n            # sample the other sample from the uniform distribution\n            mix_sample_idx = random.randint(0, len(self.data)-1)\n            mix_datum = self.data[mix_sample_idx]\n            \n            score_coe = 0.5 if datum['rating'] <= 2.5 else 1\n            mix_score_coe = 0.5 if mix_datum['rating'] <= 2.5 else 1\n            # get the mixed fbank\n            fbank, mix_lambda = self._wav2fbank(datum['filename'], mix_datum['filename'])\n            # initialize the label\n            label_indices = np.zeros(self.label_num)\n            # add sample 1 labels\n            label_indices[int(self.index_dict[datum['primary_label']])] += mix_lambda * score_coe\n            if self.secondary_label:\n                for secondary_label in literal_eval(datum['secondary_labels']):\n                    label_indices[int(self.index_dict[secondary_label])] = mix_lambda * secon_lab * score_coe\n            # add sample 2 labels\n            label_indices[int(self.index_dict[mix_datum['primary_label']])] += (1.0-mix_lambda) * mix_score_coe\n            if self.secondary_label:\n                for secondary_label in literal_eval(mix_datum['secondary_labels']):\n                    label_indices[int(self.index_dict[secondary_label])] = (1.0-mix_lambda) * secon_lab * mix_score_coe\n            \n            label_indices = torch.FloatTensor(label_indices)\n        # if not do mixup\n        else:\n            datum = self.data[index]\n            score_coe = 0.5 if datum['rating'] <= 2.5 else 1\n            \n            label_indices = np.zeros(self.label_num)\n            fbank, mix_lambda = self._wav2fbank(datum['filename'])\n            label_indices[int(self.index_dict[datum['primary_label']])] = 1.0 * score_coe\n            if self.secondary_label:\n                for secondary_label in literal_eval(datum['secondary_labels']):\n                    label_indices[int(self.index_dict[secondary_label])] = secon_lab * score_coe\n\n            label_indices = torch.FloatTensor(label_indices)\n\n\n        # normalize the input for both training and test\n        if not self.skip_norm:\n            fbank = (fbank - self.norm_mean) / (self.norm_std * 2)\n        # skip normalization the input if you are trying to get the normalization stats.\n        else:\n            pass\n        \n        if random.random() < 0.5:\n            if self.noise == True:\n                fbank = fbank + torch.rand(fbank.shape[0], fbank.shape[1]) * np.random.rand() / 10\n                fbank = torch.roll(fbank, np.random.randint(-10, 10), 0)\n\n        # the output fbank shape is [time_frame_num, frequency_bins], e.g., [1024, 128]\n        return fbank, label_indices\n\n    def __len__(self):\n        return len(self.pd_data)","metadata":{"execution":{"iopub.status.busy":"2022-05-15T13:44:17.154716Z","iopub.execute_input":"2022-05-15T13:44:17.15525Z","iopub.status.idle":"2022-05-15T13:44:17.19739Z","shell.execute_reply.started":"2022-05-15T13:44:17.155208Z","shell.execute_reply":"2022-05-15T13:44:17.196563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_length = 500\nnum_mel_bins = 128\nbatch_size = 12\npin_memory = True","metadata":{"execution":{"iopub.status.busy":"2022-05-15T13:44:17.198754Z","iopub.execute_input":"2022-05-15T13:44:17.199197Z","iopub.status.idle":"2022-05-15T13:44:17.210847Z","shell.execute_reply.started":"2022-05-15T13:44:17.199123Z","shell.execute_reply":"2022-05-15T13:44:17.209932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# set skip_norm as True only when you are computing the normalization stats\n# audio_conf = {'num_mel_bins': num_mel_bins, 'target_length': target_length, 'freqm': 0, 'timem': 0, 'mixup': 0.75, 'skip_norm': True, 'mode': 'train'}\n# ad = AudioDataset(audio_conf, train_data, label2id, train_audio_dir)       \n\n# train_loader = torch.utils.data.DataLoader(ad, batch_size=256, shuffle=True, num_workers=0, pin_memory=True)\n# mean=[]\n# std=[]\n# for i in range(3):\n#     for (audio_input, labels) in train_loader:\n#         cur_mean = torch.mean(audio_input)\n#         cur_std = torch.std(audio_input)\n#         mean.append(cur_mean)\n#         std.append(cur_std)\n#     print(i+1)\n\n# print(np.mean(mean), np.mean(std))","metadata":{"execution":{"iopub.status.busy":"2022-05-15T13:44:17.212388Z","iopub.execute_input":"2022-05-15T13:44:17.212729Z","iopub.status.idle":"2022-05-15T13:44:17.222355Z","shell.execute_reply.started":"2022-05-15T13:44:17.212688Z","shell.execute_reply":"2022-05-15T13:44:17.221437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import seaborn as sns\n\n# sns.kdeplot([f for f in frames if f < 10000], x=\"x\")\n# # Add labels\n# plt.title('Histogram of Frames')\n# plt.xlabel('x')\n# plt.ylabel('y')","metadata":{"execution":{"iopub.status.busy":"2022-05-15T13:44:17.225885Z","iopub.execute_input":"2022-05-15T13:44:17.226135Z","iopub.status.idle":"2022-05-15T13:44:17.236582Z","shell.execute_reply.started":"2022-05-15T13:44:17.226099Z","shell.execute_reply":"2022-05-15T13:44:17.235654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sampler_rate 32k\nmean = -6.5556197 #-4.8866115 #-3.6730156\nstd = 3.9615002 #4.4127903 #4.336685\n\n# sampler_rate 16k\n# mean = -7.348659\n# std = 4.028582","metadata":{"execution":{"iopub.status.busy":"2022-05-15T13:44:17.238073Z","iopub.execute_input":"2022-05-15T13:44:17.238446Z","iopub.status.idle":"2022-05-15T13:44:17.247238Z","shell.execute_reply.started":"2022-05-15T13:44:17.238404Z","shell.execute_reply":"2022-05-15T13:44:17.246412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"audio_conf = {'num_mel_bins': num_mel_bins, 'target_length': target_length, 'freqm': 0, 'timem': 0, 'mixup': 0.5,\n              'mode': 'train', 'mean': mean, 'std': std, 'noise': True, 'secondary_label': True}\n\nval_audio_conf = {'num_mel_bins': num_mel_bins, 'target_length': target_length, 'freqm': 0, 'timem': 0, 'mixup': 0,\n                  'mode': 'evaluation', 'mean': mean, 'std': std, 'noise': False, 'secondary_label': False}","metadata":{"execution":{"iopub.status.busy":"2022-05-15T13:44:17.248645Z","iopub.execute_input":"2022-05-15T13:44:17.249469Z","iopub.status.idle":"2022-05-15T13:44:17.25818Z","shell.execute_reply.started":"2022-05-15T13:44:17.249426Z","shell.execute_reply":"2022-05-15T13:44:17.257431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_audioset = AudioDataset(audio_conf, train_data, label2id, train_audio_dir)\ntest_audioset = AudioDataset(val_audio_conf, test_data, label2id, train_audio_dir)","metadata":{"execution":{"iopub.status.busy":"2022-05-15T13:44:17.259907Z","iopub.execute_input":"2022-05-15T13:44:17.260245Z","iopub.status.idle":"2022-05-15T13:44:17.33826Z","shell.execute_reply.started":"2022-05-15T13:44:17.260205Z","shell.execute_reply":"2022-05-15T13:44:17.337443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataloader = torch.utils.data.DataLoader(train_audioset, batch_size=batch_size, shuffle=True, num_workers=0, pin_memory=pin_memory)\ntest_dataloader = torch.utils.data.DataLoader(test_audioset, batch_size=batch_size, shuffle=False, num_workers=0, pin_memory=pin_memory)","metadata":{"execution":{"iopub.status.busy":"2022-05-15T13:44:17.339798Z","iopub.execute_input":"2022-05-15T13:44:17.340307Z","iopub.status.idle":"2022-05-15T13:44:17.351384Z","shell.execute_reply.started":"2022-05-15T13:44:17.340264Z","shell.execute_reply":"2022-05-15T13:44:17.350499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# AST model","metadata":{}},{"cell_type":"code","source":"# override the timm package to relax the input shape constraint.\nclass PatchEmbed(nn.Module):\n    \"\"\" Image to Patch Embedding\n    \"\"\"\n    def __init__(self, img_size=224, patch_size=16, in_chans=3, embed_dim=768):\n        super().__init__()\n        img_size = to_2tuple(img_size)\n        patch_size = to_2tuple(patch_size)\n        num_patches = (img_size[1] // patch_size[1]) * (img_size[0] // patch_size[0])\n        self.img_size = img_size\n        self.patch_size = patch_size\n        self.num_patches = num_patches\n\n        self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size)\n\n    def forward(self, x):\n        x = self.proj(x).flatten(2).transpose(1, 2)\n        return x\n\ndef get_sinusoid_encoding(n_position, d_hid):\n    ''' Sinusoid position encoding table '''\n\n    def get_position_angle_vec(position):\n        return [position / np.power(10000, 2 * (hid_j // 2) / d_hid) for hid_j in range(d_hid)]\n\n    sinusoid_table = np.array([get_position_angle_vec(pos_i) for pos_i in range(n_position)])\n    sinusoid_table[:, 0::2] = np.sin(sinusoid_table[:, 0::2])  # dim 2i\n    sinusoid_table[:, 1::2] = np.cos(sinusoid_table[:, 1::2])  # dim 2i+1\n\n    return torch.FloatTensor(sinusoid_table).unsqueeze(0)\n\nclass ASTModel(nn.Module):\n    def __init__(self, label_dim=527,\n                 fshape=128, tshape=2, fstride=128, tstride=2,\n                 input_fdim=128, input_tdim=1024, model_size='base',\n                 pretrain_stage=True, load_pretrained_mdl_path=None):\n\n        super(ASTModel, self).__init__()\n        assert timm.__version__ == '0.4.5', 'Please use timm == 0.4.5, the code might not be compatible with newer versions.'\n        \n        # override timm input shape restriction\n        timm.models.vision_transformer.PatchEmbed = PatchEmbed\n        \n        # pretrain the AST models\n        if pretrain_stage == True:\n            if load_pretrained_mdl_path != None:\n                raise ValueError('Setting load_pretrained_mdl_path at pretraining stage is useless, pretraining is always from scratch, please change it to None.')\n            if fstride != fshape or tstride != tshape:\n                raise ValueError('fstride != fshape or tstride != tshape, they must be same at the pretraining stage, patch split overlapping is not supported.')\n\n            # if AudioSet pretraining is not used (but ImageNet pretraining may still apply)\n            if model_size == 'tiny':\n                self.v = timm.create_model('vit_deit_tiny_distilled_patch16_224', pretrained=False)\n                self.heads, self.depth = 3, 12\n                self.cls_token_num = 2\n            elif model_size == 'small':\n                self.v = timm.create_model('vit_deit_small_distilled_patch16_224', pretrained=False)\n                self.heads, self.depth = 6, 12\n                self.cls_token_num = 2\n            elif model_size == 'base':\n                self.v = timm.create_model('vit_deit_base_distilled_patch16_384', pretrained=False)\n                self.heads, self.depth = 12, 12\n                self.cls_token_num = 2\n            elif model_size == 'base_nokd':\n                self.v = timm.create_model('vit_deit_base_patch16_384', pretrained=False)\n                self.heads, self.depth = 12, 12\n                self.cls_token_num = 1\n            else:\n                raise Exception('Model size must be one of tiny, small, base, base_nokd')\n\n            self.original_num_patches = self.v.patch_embed.num_patches\n            self.oringal_hw = int(self.original_num_patches ** 0.5)\n            self.original_embedding_dim = self.v.pos_embed.shape[2]\n\n            # SSL Pretraining Code\n            self.softmax = nn.Softmax(dim=-1)\n            self.lsoftmax = nn.LogSoftmax(dim=-1)\n            self.fshape, self.tshape = fshape, tshape\n            self.fstride, self.tstride = fstride, tstride\n            self.input_fdim, self.input_tdim = input_fdim, input_tdim\n            # this is a trick to make state_dict to track pretraining input_fdim and input_tdim and save them by using torch.save\n            self.p_input_fdim, self.p_input_tdim = nn.Parameter(torch.tensor(input_fdim), requires_grad=False), nn.Parameter(torch.tensor(input_tdim), requires_grad=False)\n\n            # masked patch classification (discriminative objective) layer\n            # we use two layers for pretext task, but using a single layer has similar performance.\n            # we map the output of transformer (768-dim for base models) to 256-dim patch input space, and then dot product with flattened patch input (also 256-dim) to calculate loss.\n            # alternatively, you can map the output of transformer to 768-dim patch embedding space, and dot product with patch embedding. Performance-wise they are similar, but map to 256 space is more efficient.\n            self.cpredlayer = nn.Sequential(nn.Linear(self.original_embedding_dim, self.original_embedding_dim), nn.ReLU(), nn.Linear(self.original_embedding_dim, 256))\n            # masked patch reconstruction (generative objective) layer\n            self.gpredlayer = nn.Sequential(nn.Linear(self.original_embedding_dim, self.original_embedding_dim), nn.ReLU(), nn.Linear(self.original_embedding_dim, 256))\n            self.unfold = torch.nn.Unfold(kernel_size=(fshape, tshape), stride=(fstride, tstride))\n\n            # we use learnable mask embedding (follow the BEIT paper), but using a fixed mask embedding (e.g., 0) leads to same performance.\n            self.mask_embed = nn.Parameter(torch.zeros([1, 1, self.original_embedding_dim]))\n            self.mask_embed = torch.nn.init.xavier_normal_(self.mask_embed)\n\n            # get the intermediate shape\n            self.p_f_dim, self.p_t_dim = self.get_shape(fstride, tstride, input_fdim, input_tdim, fshape, tshape)\n            num_patches = self.p_f_dim * self.p_t_dim\n            self.num_patches = num_patches\n            self.v.patch_embed.num_patches = num_patches\n            print('pretraining patch split stride: frequency={:d}, time={:d}'.format(fstride, tstride))\n            print('pretraining patch shape: frequency={:d}, time={:d}'.format(fshape, tshape))\n            print('pretraining patch array dimension: frequency={:d}, time={:d}'.format(self.p_f_dim, self.p_t_dim))\n            print('pretraining number of patches={:d}'.format(num_patches))\n\n            # the linear patch projection layer, use 1 channel for spectrogram rather than the original 3 channels for RGB images.\n            new_proj = torch.nn.Conv2d(1, self.original_embedding_dim, kernel_size=(fshape, tshape), stride=(fstride, tstride))\n            self.v.patch_embed.proj = new_proj\n\n            # use trainable positional embedding\n            new_pos_embed = nn.Parameter(torch.zeros(1, self.v.patch_embed.num_patches + self.cls_token_num, self.original_embedding_dim))\n            self.v.pos_embed = new_pos_embed\n            trunc_normal_(self.v.pos_embed, std=.02)\n\n        # use a pretrained models for finetuning\n        elif pretrain_stage == False:\n            device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n            if load_pretrained_mdl_path == None:\n                raise ValueError('Please set load_pretrained_mdl_path to load a pretrained models.')\n            sd = torch.load(load_pretrained_mdl_path, map_location=device)\n            # get the fshape and tshape, input_fdim and input_tdim in the pretraining stage\n            try:\n                p_fshape, p_tshape = sd['module.v.patch_embed.proj.weight'].shape[2], sd['module.v.patch_embed.proj.weight'].shape[3]\n                p_input_fdim, p_input_tdim = sd['module.p_input_fdim'].item(), sd['module.p_input_tdim'].item()\n            except:\n                raise  ValueError('The model loaded is not from a torch.nn.Dataparallel object. Wrap it with torch.nn.Dataparallel and try again.')\n\n            print('now load a SSL pretrained models from ' + load_pretrained_mdl_path)\n            # during pretraining, fstride=fshape and tstride=tshape because no patch overlapping is used\n            # here, input_fdim and input_tdim should be that used in pretraining, not that in the fine-tuning.\n            # we need to know input_fdim and input_tdim to do positional embedding cut/interpolation.\n            # generally it should be better to use same input_fdim during pretraining and finetuning, but input_tdim can be safely different\n            audio_model = ASTModel(fstride=p_fshape, tstride=p_tshape, fshape=p_fshape, tshape=p_tshape,\n                                   input_fdim=p_input_fdim, input_tdim=p_input_tdim, pretrain_stage=True, model_size=model_size)\n            audio_model = torch.nn.DataParallel(audio_model)\n            audio_model.load_state_dict(sd, strict=False)\n\n            self.v = audio_model.module.v\n            self.original_embedding_dim = self.v.pos_embed.shape[2]\n            self.cls_token_num = audio_model.module.cls_token_num\n\n            # mlp head for fine-tuning\n            self.mlp_head = nn.Sequential(nn.LayerNorm(self.original_embedding_dim),\n                                          nn.Linear(self.original_embedding_dim, label_dim))\n\n            f_dim, t_dim = self.get_shape(fstride, tstride, input_fdim, input_tdim, fshape, tshape)\n            # patch array dimension during pretraining\n            p_f_dim, p_t_dim = audio_model.module.p_f_dim, audio_model.module.p_t_dim\n            num_patches = f_dim * t_dim\n            p_num_patches = p_f_dim * p_t_dim\n            self.v.patch_embed.num_patches = num_patches\n            print('fine-tuning patch split stride: frequncey={:d}, time={:d}'.format(fstride, tstride))\n            print('fine-tuning number of patches={:d}'.format(num_patches))\n\n            # patch shape should be same for pretraining and fine-tuning\n            if fshape != p_fshape or tshape != p_tshape:\n                raise ValueError('The patch shape of pretraining and fine-tuning is not consistant, pretraining: f={:d}, t={:d}, finetuning: f={:d}, t={:d}'.format(p_fshape, p_tshape, fshape, tshape))\n\n            # patch split stride generally should be different for pretraining and fine-tuning, as patch split overlapping is only used in finetuning\n            # during pretraining, p_fshape = p_fstride and p_tshape = p_tstride\n            if fstride != p_fshape or tstride != p_tshape:\n                # initialize a new patch embedding layer with desired new stride.\n                new_proj = torch.nn.Conv2d(1, self.original_embedding_dim, kernel_size=(fshape, tshape), stride=(fstride, tstride))\n                # but the weights of patch embedding layer is still got from the pretrained models\n                new_proj.weight = torch.nn.Parameter(torch.sum(self.v.patch_embed.proj.weight, dim=1).unsqueeze(1))\n                new_proj.bias = self.v.patch_embed.proj.bias\n                self.v.patch_embed.proj = new_proj\n\n            new_pos_embed = self.v.pos_embed[:, self.cls_token_num:, :].detach().reshape(1, p_num_patches, self.original_embedding_dim).transpose(1, 2).reshape(1, self.original_embedding_dim, p_f_dim, p_t_dim)\n            # cut or interpolate the positional embedding\n            if t_dim < p_t_dim:\n                new_pos_embed = new_pos_embed[:, :, :, int(p_t_dim/2) - int(t_dim / 2): int(p_t_dim/2) - int(t_dim / 2) + t_dim]\n            else:\n                new_pos_embed = torch.nn.functional.interpolate(new_pos_embed, size=(8, t_dim), mode='bilinear')\n            if f_dim < p_f_dim:\n                new_pos_embed = new_pos_embed[:, :, int(p_f_dim/2) - int(f_dim / 2): int(p_f_dim/2) - int(f_dim / 2) + t_dim, :]\n            else:\n                new_pos_embed = torch.nn.functional.interpolate(new_pos_embed, size=(f_dim, t_dim), mode='bilinear')\n\n            new_pos_embed = new_pos_embed.reshape(1, self.original_embedding_dim, num_patches).transpose(1, 2)\n            self.v.pos_embed = nn.Parameter(torch.cat([self.v.pos_embed[:, :self.cls_token_num, :].detach(), new_pos_embed], dim=1))\n\n    # get the shape of intermediate representation.\n    def get_shape(self, fstride, tstride, input_fdim, input_tdim, fshape, tshape):\n        test_input = torch.randn(1, 1, input_fdim, input_tdim)\n        test_proj = nn.Conv2d(1, self.original_embedding_dim, kernel_size=(fshape, tshape), stride=(fstride, tstride))\n        test_out = test_proj(test_input)\n        f_dim = test_out.shape[2]\n        t_dim = test_out.shape[3]\n        return f_dim, t_dim\n\n\n    def finetuningavgtok(self, x):\n        B = x.shape[0]\n        x = self.v.patch_embed(x)\n        if self.cls_token_num == 2:\n            cls_tokens = self.v.cls_token.expand(B, -1, -1)\n            dist_token = self.v.dist_token.expand(B, -1, -1)\n            x = torch.cat((cls_tokens, dist_token, x), dim=1)\n        else:\n            cls_tokens = self.v.cls_token.expand(B, -1, -1)\n            x = torch.cat((cls_tokens, x), dim=1)\n        x = x + self.v.pos_embed\n        x = self.v.pos_drop(x)\n\n        for blk_id, blk in enumerate(self.v.blocks):\n            x = blk(x)\n        x = self.v.norm(x)\n\n        # average output of all tokens except cls token(s)\n        x = torch.mean(x[:, self.cls_token_num:, :], dim=1)\n        x = self.mlp_head(x)\n        return x\n\n    def finetuningcls(self, x):\n        B = x.shape[0]\n        x = self.v.patch_embed(x)\n        if self.cls_token_num == 2:\n            cls_tokens = self.v.cls_token.expand(B, -1, -1)\n            dist_token = self.v.dist_token.expand(B, -1, -1)\n            x = torch.cat((cls_tokens, dist_token, x), dim=1)\n        else:\n            cls_tokens = self.v.cls_token.expand(B, -1, -1)\n            x = torch.cat((cls_tokens, x), dim=1)\n        x = x + self.v.pos_embed\n        x = self.v.pos_drop(x)\n\n        for blk_id, blk in enumerate(self.v.blocks):\n            x = blk(x)\n        x = self.v.norm(x)\n\n        # if models has two cls tokens (DEIT), average as the clip-level representation\n        if self.cls_token_num == 2:\n            x = (x[:, 0] + x[:, 1]) / 2\n        else:\n            x = x[:, 0]\n        x = self.mlp_head(x)\n        return x\n\n    \n    \n    def forward(self, x, task, cluster=True, mask_patch=400):\n        # expect input x = (batch_size, time_frame_num, frequency_bins), e.g., (12, 1024, 128)\n        x = x.unsqueeze(1)\n        x = x.transpose(2, 3)\n\n        # finetuning (ft), use the mean of all token (patch) output as clip-level representation.\n        # this is default for SSAST fine-tuning as during pretraining, supervision signal is given to each token, not the [cls] token\n        if task == 'ft_avgtok':\n            return self.finetuningavgtok(x)\n        # alternatively, use the [cls] token output as clip-level representation.\n        elif task == 'ft_cls':\n            return self.finetuningcls(x)\n        else:\n            raise Exception('Task unrecognized.')\n","metadata":{"execution":{"iopub.status.busy":"2022-05-15T13:44:17.353563Z","iopub.execute_input":"2022-05-15T13:44:17.354021Z","iopub.status.idle":"2022-05-15T13:44:17.420542Z","shell.execute_reply.started":"2022-05-15T13:44:17.353969Z","shell.execute_reply":"2022-05-15T13:44:17.419481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utils","metadata":{}},{"cell_type":"code","source":"import numpy as np\nfrom scipy import stats\nfrom sklearn import metrics\nimport torch\n\ndef d_prime(auc):\n    standard_normal = stats.norm()\n    d_prime = standard_normal.ppf(auc) * np.sqrt(2.0)\n    return d_prime\n\ndef calculate_stats(output, target, multi_label=False):\n    \"\"\"Calculate statistics including mAP, AUC, etc.\n    Args:\n      output: 2d array, (samples_num, classes_num)\n      target: 2d array, (samples_num, classes_num)\n    Returns:\n      stats: list of statistic of each class.\n    \"\"\"\n\n    classes_num = target.shape[-1]\n    stats = []\n\n    # Accuracy, only used for single-label classification such as esc-50, not for multiple label one such as AudioSet\n    acc = metrics.accuracy_score(np.argmax(target, 1), np.argmax(output, 1))\n\n    # Class-wise statistics\n    if multi_label == True:\n        for k in range(classes_num):\n            if target[:, k].sum() / target.shape[0] == target[0][k]:\n                continue\n\n            # Average precision\n            avg_precision = metrics.average_precision_score(\n                target[:, k], output[:, k], average=None)\n\n            # AUC\n            auc = metrics.roc_auc_score(target[:, k], output[:, k], average=None)\n\n            # Precisions, recalls\n            (precisions, recalls, thresholds) = metrics.precision_recall_curve(\n                target[:, k], output[:, k])\n\n            # FPR, TPR\n            (fpr, tpr, thresholds) = metrics.roc_curve(target[:, k], output[:, k])\n\n            save_every_steps = 1000     # Sample statistics to reduce size\n            dict = {'precisions': precisions[0::save_every_steps],\n                    'recalls': recalls[0::save_every_steps],\n                    'AP': avg_precision,\n                    'fpr': fpr[0::save_every_steps],\n                    'fnr': 1. - tpr[0::save_every_steps],\n                    'auc': auc,\n                    # note acc is not class-wise, this is just to keep consistent with other metrics\n                    'acc': acc\n                    }\n            stats.append(dict)\n    else:\n        stats.append({'acc': acc})\n    \n    return stats","metadata":{"execution":{"iopub.status.busy":"2022-05-15T13:44:17.425941Z","iopub.execute_input":"2022-05-15T13:44:17.426188Z","iopub.status.idle":"2022-05-15T13:44:17.442463Z","shell.execute_reply.started":"2022-05-15T13:44:17.426161Z","shell.execute_reply":"2022-05-15T13:44:17.441356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# train function","metadata":{}},{"cell_type":"code","source":"import sys\nimport os\nimport datetime\n# sys.path.append(os.path.dirname(os.path.dirname(sys.path[0])))\nimport time\nimport torch\nfrom torch import nn\nimport numpy as np\nfrom torch.cuda.amp import autocast,GradScaler\n\nvalid_loss_list = []\nvalid_acc_list = []\n\ntrain_loss_list = []\n\ndef train(audio_model, train_loader, test_loader, args, pretrained_path=None):\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    print('running on ' + str(device))\n    torch.set_grad_enabled(True)\n\n    # best_cum_mAP is checkpoint ensemble from the first epoch to the best epoch\n    best_epoch, best_cum_epoch, best_mAP, best_acc, best_cum_mAP = 0, 0, -np.inf, -np.inf, -np.inf\n    best_loss = 10000\n    global_step, epoch = 0, 0\n    start_time = time.time()\n\n    if not isinstance(audio_model, nn.DataParallel):\n        audio_model = nn.DataParallel(audio_model)\n        \n    if pretrained_path:\n        checkpoint = torch.load(pretrained_path)\n        audio_model.load_state_dict(checkpoint['model_state_dict'])\n        # best_loss = checkpoint['best_loss']\n\n    audio_model = audio_model.to(device)\n    # Set up the optimizer\n    trainables = [p for p in audio_model.parameters() if p.requires_grad]\n    print('Total parameter number is : {:.3f} million'.format(sum(p.numel() for p in audio_model.parameters()) / 1e6))\n    print('Total trainable parameter number is : {:.3f} million'.format(sum(p.numel() for p in trainables) / 1e6))\n\n    # diff lr optimizer\n    mlp_list = ['mlp_head.0.weight', 'mlp_head.0.bias', 'mlp_head.1.weight', 'mlp_head.1.bias']\n    mlp_params = list(filter(lambda kv: kv[0] in mlp_list, audio_model.module.named_parameters()))\n    base_params = list(filter(lambda kv: kv[0] not in mlp_list, audio_model.module.named_parameters()))\n    mlp_params = [i[1] for i in mlp_params]\n    base_params = [i[1] for i in base_params]\n    # only finetuning small/tiny models on balanced audioset uses different learning rate for mlp head\n    print('The mlp header uses {:d} x larger lr'.format(args[\"head_lr\"]))\n    if args[\"optimizer\"] == 'adam':\n        optimizer = torch.optim.Adam(\n            [\n                {'params': base_params, 'lr': args[\"lr\"]},\n                {'params': mlp_params, 'lr': args[\"lr\"] * args[\"head_lr\"]}\n            ], weight_decay=5e-7, betas=(0.95, 0.999)\n        )\n    elif args[\"optimizer\"] == \"adamw\":\n        optimizer = torch.optim.AdamW(\n            [\n                {'params': base_params, 'lr': args[\"lr\"]},\n                {'params': mlp_params, 'lr': args[\"lr\"] * args[\"head_lr\"]}\n            ], weight_decay=5e-7, amsgrad=True\n        )\n    else:\n        optimizer = torch.optim.SGD(\n            [\n                {'params': base_params, 'lr': args[\"lr\"]},\n                {'params': mlp_params, 'lr': args[\"lr\"] * args[\"head_lr\"]}\n            ], momentum=0.9, nesterov=True, weight_decay=5e-7\n        )\n    \n    if pretrained_path:\n        optimizer.load_state_dict(checkpoint['optimizer_state_dict'])\n        epoch = checkpoint['epoch']\n    \n    mlp_lr = optimizer.param_groups[1]['lr']\n    lr_list = [args[\"lr\"], mlp_lr]\n\n    print('Total mlp parameter number is : {:.3f} million'.format(sum(p.numel() for p in mlp_params) / 1e6))\n    print('Total base parameter number is : {:.3f} million'.format(sum(p.numel() for p in base_params) / 1e6))\n\n    if args[\"adaptschedule\"] == True:\n        scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', factor=0.5, patience=args[\"lr_patience\"], verbose=True)\n        print('now use adaptive learning rate scheduler.')\n    else:\n        scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, list(range(args[\"lrscheduler_start\"], 1000, args[\"lrscheduler_step\"])),gamma=args[\"lrscheduler_decay\"])\n    main_metrics = args[\"metrics\"]\n    if args[\"loss\"] == 'BCE':\n        loss_fn = nn.BCEWithLogitsLoss()\n    elif args[\"loss\"] == 'CE':\n        loss_fn = nn.CrossEntropyLoss()\n    args[\"loss_fn\"] = loss_fn\n\n    \n    epoch += 1\n\n    print(\"current #steps=%s, #epochs=%s\" % (global_step, epoch))\n    print(\"start training...\")\n    result = np.zeros([args[\"n_epochs\"], 10])\n    audio_model.train()\n    while epoch < args[\"n_epochs\"] + 1:\n        begin_time = time.time()\n        audio_model.train()\n        print('---------------')\n        print(datetime.datetime.now())\n        print(\"current #epochs=%s, #steps=%s\" % (epoch, global_step))\n        epoch_loss = []\n\n        for i, (audio_input, labels) in enumerate(train_loader):\n\n            B = audio_input.size(0)\n            audio_input = audio_input.to(device, non_blocking=True)\n            labels = labels.to(device, non_blocking=True)\n\n            # first several steps for warm-up\n            if global_step <= 1000 and global_step % 50 == 0 and args[\"warmup\"] == True:\n                for group_id, param_group in enumerate(optimizer.param_groups):\n                    warm_lr = (global_step / 1000) * lr_list[group_id]\n                    param_group['lr'] = warm_lr\n                    print('warm-up learning rate is {:f}'.format(param_group['lr']))\n\n            audio_output = audio_model(audio_input, args[\"task\"])\n            if isinstance(loss_fn, torch.nn.CrossEntropyLoss):\n                loss = loss_fn(audio_output, torch.argmax(labels.long(), axis=1))\n            else:\n                loss = loss_fn(audio_output, labels)\n\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n\n            print_step = global_step % args[\"n_print_steps\"] == 0\n            early_print_step = epoch == 0 and global_step % (args[\"n_print_steps\"]/10) == 0\n            print_step = print_step or early_print_step\n\n            if print_step and global_step != 0:\n                print(f'epoch {epoch}, loss {float(loss):f}')\n\n            global_step += 1\n            epoch_loss.append(float(loss))\n\n        print('start validation')\n        stats, valid_loss = validate(audio_model, test_loader, args, epoch)\n        \n        acc = stats[0]['acc']\n        \n#         if main_metrics == 'mAP':\n#             mAP = np.mean([stat['AP'] for stat in stats])\n#             mAUC = np.mean([stat['auc'] for stat in stats])\n            \n#             middle_ps = [stat['precisions'][int(len(stat['precisions'])/2)] for stat in stats]\n#             middle_rs = [stat['recalls'][int(len(stat['recalls'])/2)] for stat in stats]\n#             average_precision = np.mean(middle_ps)\n#             average_recall = np.mean(middle_rs)\n        \n        valid_loss_list.append(valid_loss)\n        valid_acc_list.append(acc)\n\n        if main_metrics == 'mAP':\n            print(\"mAP: {:.6f}\".format(mAP))\n        else:\n            print(\"acc: {:.6f}\".format(acc))\n#         print(\"AUC: {:.6f}\".format(mAUC))\n#         print(\"Avg Precision: {:.6f}\".format(average_precision))\n#         print(\"Avg Recall: {:.6f}\".format(average_recall))\n#         print(\"d_prime: {:.6f}\".format(d_prime(mAUC)))\n#         print(\"valid_loss: {:.6f}\".format(valid_loss))\n\n        print('validation finished')\n\n#         if mAP > best_mAP:\n#             best_mAP = mAP\n#             if main_metrics == 'mAP':\n#                 best_epoch = epoch\n\n        if acc > best_acc:\n            best_acc = acc\n            if main_metrics == 'acc':\n                best_epoch = epoch\n                \n        epoch_loss_mean = np.mean(epoch_loss)\n        print(\"epoch loss: {:.6f}\".format(epoch_loss_mean))\n        if epoch_loss_mean < best_loss:\n            best_loss = epoch_loss_mean\n            if main_metrics == 'loss':\n                best_epoch = epoch\n        \n        if best_epoch == epoch:\n            torch.save({\n                'epoch': epoch,\n                'model_state_dict': audio_model.state_dict(),\n                'optimizer_state_dict': optimizer.state_dict(),\n                'best_loss': best_loss,\n            }, './model-stride-10-adamw-scored-just-scored.pth')\n\n        if isinstance(scheduler, torch.optim.lr_scheduler.ReduceLROnPlateau):\n            print('adaptive learning rate scheduler step')\n            scheduler.step(mAP)\n        else:\n            print('normal learning rate scheduler step')\n            scheduler.step()\n\n        print('Epoch-{0} lr: {1}'.format(epoch, optimizer.param_groups[0]['lr']))\n        print('Epoch-{0} lr: {1}'.format(epoch, optimizer.param_groups[1]['lr']))\n\n        finish_time = time.time()\n        print('epoch {:d} training time: {:.3f}'.format(epoch, finish_time-begin_time))\n\n        epoch += 1\n\ndef validate(audio_model, val_loader, args, epoch):\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    \n    if not isinstance(audio_model, nn.DataParallel):\n        audio_model = nn.DataParallel(audio_model)\n    audio_model = audio_model.to(device)\n    # switch to evaluate mode\n    audio_model.eval()\n\n    A_predictions = []\n    A_targets = []\n    A_loss = []\n    # args[\"loss_fn\"] = nn.BCEWithLogitsLoss()\n    with torch.no_grad():\n        for i, (audio_input, labels) in enumerate(val_loader):\n            audio_input = audio_input.to(device)\n\n            # compute output\n            audio_output = audio_model(audio_input, args[\"task\"])\n            audio_output_for_loss = audio_output\n            if isinstance(args[\"loss_fn\"], torch.nn.CrossEntropyLoss):\n                audio_output = torch.nn.functional.softmax(audio_output, dim=-1)\n            else:\n                audio_output = torch.sigmoid(audio_output)\n            predictions = audio_output.to('cpu').detach()\n\n            A_predictions.append(predictions)\n            A_targets.append(labels)\n\n            # compute the loss\n            labels = labels.to(device)\n            if isinstance(args[\"loss_fn\"], torch.nn.CrossEntropyLoss):\n                loss = args[\"loss_fn\"](audio_output_for_loss, torch.argmax(labels.long(), axis=1))\n            else:\n                loss = args[\"loss_fn\"](audio_output_for_loss, labels)\n            A_loss.append(loss.to('cpu').detach())\n\n        audio_output = torch.cat(A_predictions)\n        target = torch.cat(A_targets)\n        loss = np.mean(A_loss)\n        stats = calculate_stats(audio_output, target, False)\n\n    return stats, loss","metadata":{"execution":{"iopub.status.busy":"2022-05-15T13:44:17.445343Z","iopub.execute_input":"2022-05-15T13:44:17.44605Z","iopub.status.idle":"2022-05-15T13:44:17.495713Z","shell.execute_reply.started":"2022-05-15T13:44:17.445974Z","shell.execute_reply":"2022-05-15T13:44:17.494681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"args = {\n    \"exp_dir\": \"\",\n    \"head_lr\": 1,\n    \"lr\": 0.0001,\n    \"lrscheduler_start\": 7,\n    \"lrscheduler_step\": 5,\n    \"lrscheduler_decay\": 0.1,\n    \"metrics\": [\"mAP\", \"acc\"][1],\n    \"loss\": [\"BCE\", \"CE\"][0],\n    \"n_epochs\": 50,\n    \"warmup\": True,\n    \"task\": [\"ft_avgtok\", \"ft_cls\"][1],\n    \"n_print_steps\": 100,\n    \"wa\": True,\n    \"wa_start\": 1,\n    \"wa_znd\": 15,\n    \"save_model\": False,\n    \"adaptschedule\": False,\n    \"optimizer\": [\"adam\", \"adamw\", \"sgd\"][1]\n}\n\naudio_model = ASTModel(label_dim=len(labels),\n                 fshape=16, tshape=16, fstride=10, tstride=10,\n                 input_fdim=128, input_tdim=target_length, model_size='base',\n                 pretrain_stage=False,\n                 load_pretrained_mdl_path=pretrained_mdl_path)\n\nif not isinstance(audio_model, torch.nn.DataParallel):\n    audio_model = torch.nn.DataParallel(audio_model)","metadata":{"execution":{"iopub.status.busy":"2022-05-15T13:44:17.497232Z","iopub.execute_input":"2022-05-15T13:44:17.498218Z","iopub.status.idle":"2022-05-15T13:44:19.402485Z","shell.execute_reply.started":"2022-05-15T13:44:17.498174Z","shell.execute_reply":"2022-05-15T13:44:19.401662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# summary(audio_model, input_size=(batch_size, target_length, num_mel_bins), task=args[\"task\"])","metadata":{"execution":{"iopub.status.busy":"2022-05-15T13:44:19.403918Z","iopub.execute_input":"2022-05-15T13:44:19.405526Z","iopub.status.idle":"2022-05-15T13:44:19.409619Z","shell.execute_reply.started":"2022-05-15T13:44:19.405481Z","shell.execute_reply":"2022-05-15T13:44:19.40864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# stats, valid_loss = validate(audio_model, test_dataloader, args, 1)","metadata":{"execution":{"iopub.status.busy":"2022-05-15T13:44:19.411053Z","iopub.execute_input":"2022-05-15T13:44:19.41162Z","iopub.status.idle":"2022-05-15T13:44:19.421462Z","shell.execute_reply.started":"2022-05-15T13:44:19.411561Z","shell.execute_reply":"2022-05-15T13:44:19.420536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train(audio_model, train_dataloader, test_dataloader,  args)","metadata":{"execution":{"iopub.status.busy":"2022-05-15T13:44:19.424985Z","iopub.execute_input":"2022-05-15T13:44:19.425222Z","iopub.status.idle":"2022-05-15T13:44:19.431312Z","shell.execute_reply.started":"2022-05-15T13:44:19.425194Z","shell.execute_reply":"2022-05-15T13:44:19.430494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submit","metadata":{}},{"cell_type":"code","source":"import os\nimport json\nimport numpy as np\nimport pandas as pd\nimport librosa\nimport math\n\ndef get_score(audio_model):\n    test_audio_dir = '../input/birdclef-2022/test_soundscapes/'\n    #test_audio_dir = '../input/birdclef-2022/train_audio/akiapo/'\n    file_list = [f.split('.')[0] for f in sorted(os.listdir(test_audio_dir))]\n    with open('../input/birdclef-2022/scored_birds.json') as sbfile:\n        scored_birds = json.load(sbfile)\n        threshold = 0.7\n    pred = {'row_id': [], 'target': []}\n    for afile in file_list:\n        path = test_audio_dir + afile + '.ogg'\n        \n        waveform, sr = torchaudio.load(path)\n        waveform = waveform - waveform.mean()\n        \n        length = waveform.shape[1] // sr\n        chunks = [[] for i in range(math.ceil(length/5))] #length//5\n        for i in range(len(chunks)):        \n            chunk_end_time = (i + 1) * 5\n            fbank = torchaudio.compliance.kaldi.fbank(waveform[:, i*5*sr:(i+1)*5*sr], htk_compat=True, sample_frequency=sr, use_energy=False,\n                                                  window_type='hanning', num_mel_bins=num_mel_bins, dither=0.0, frame_shift=10)\n            n_frames = fbank.shape[0]\n            p = target_length - n_frames\n            if p > 0:\n                m = torch.nn.ZeroPad2d((0, 0, 0, p))\n                fbank = m(fbank)\n            elif p < 0:\n                fbank = fbank[0:target_length, :]\n            fbank = (fbank - mean) / (std * 2)\n            aud = torch.unsqueeze(fbank, 0)\n            with torch.no_grad():\n                scores = audio_model(aud, args[\"task\"])\n            #scores = torch.nn.functional.softmax(scores, dim=-1)\n            scores = torch.sigmoid(scores)\n            \n            for bird in scored_birds:\n                idx = label2id[bird]\n                score = scores[0][idx]\n                row_id = afile + '_' + bird + '_' + str(chunk_end_time)\n                pred['row_id'].append(row_id)\n                pred['target'].append(True if score > threshold else False)\n    results = pd.DataFrame(pred, columns = ['row_id', 'target'])\n    return results","metadata":{"execution":{"iopub.status.busy":"2022-05-15T13:44:19.433477Z","iopub.execute_input":"2022-05-15T13:44:19.433808Z","iopub.status.idle":"2022-05-15T13:44:19.450911Z","shell.execute_reply.started":"2022-05-15T13:44:19.433767Z","shell.execute_reply":"2022-05-15T13:44:19.450039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def submit(pretrained_path=None):\n    if pretrained_path:\n        checkpoint = torch.load(pretrained_path)\n        #checkpoint = torch.load(pretrained_path, map_location=torch.device('cpu'))\n        audio_model.load_state_dict(checkpoint['model_state_dict'])\n    audio_model.eval()\n    return get_score(audio_model)","metadata":{"execution":{"iopub.status.busy":"2022-05-15T13:44:19.452574Z","iopub.execute_input":"2022-05-15T13:44:19.453245Z","iopub.status.idle":"2022-05-15T13:44:19.462658Z","shell.execute_reply.started":"2022-05-15T13:44:19.453202Z","shell.execute_reply":"2022-05-15T13:44:19.46177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = submit(\"../input/birdclef2022-epoch10/model-stride-10-adamw-scored.pth\")\n# pd.options.display.max_rows = None\n# print(results)\nresults.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-05-15T13:44:19.464186Z","iopub.execute_input":"2022-05-15T13:44:19.464521Z","iopub.status.idle":"2022-05-15T13:44:35.458921Z","shell.execute_reply.started":"2022-05-15T13:44:19.464483Z","shell.execute_reply":"2022-05-15T13:44:35.45814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#print(np.mean([t.cpu() for t in tresholds]))","metadata":{"execution":{"iopub.status.busy":"2022-05-15T13:44:35.46016Z","iopub.execute_input":"2022-05-15T13:44:35.46048Z","iopub.status.idle":"2022-05-15T13:44:35.46625Z","shell.execute_reply.started":"2022-05-15T13:44:35.460437Z","shell.execute_reply":"2022-05-15T13:44:35.464772Z"},"trusted":true},"execution_count":null,"outputs":[]}]}