{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":70203,"databundleVersionId":8068726,"sourceType":"competition"}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"#
1. Initial Setting
","metadata":{}},{"cell_type":"code","source":"import os\nos.environ['CUDA_IS_VISIBLE'] = '0'\nfrom collections import defaultdict # defaultdict(list)\nimport copy # copy.deepcopy(model.state_dict())\nimport sys\nfrom tqdm import tqdm\nimport ctypes\nimport gc\nimport random\nimport pickle\nfrom glob import glob\n\nimport pandas as pd, numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import KFold, GroupKFold\n\n# Pytorch Imports \nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import Dataset,DataLoader\n\nimport pytorch_lightning as pl\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nDEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'\nLOAD_MODELS_FROM = None","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:03:09.484056Z","iopub.execute_input":"2024-05-29T12:03:09.484415Z","iopub.status.idle":"2024-05-29T12:03:20.213919Z","shell.execute_reply.started":"2024-05-29T12:03:09.484384Z","shell.execute_reply":"2024-05-29T12:03:20.212869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n SEED = 2024\n VER = 1\n BASE_PATH = '../input/birdclef-2024/'\n AUDIO_PATH = os.path.join(BASE_PATH + 'train_audio')\n TRAIN_BATCH = 16\n EVAL_BATCH = 32","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:03:20.215767Z","iopub.execute_input":"2024-05-29T12:03:20.216435Z","iopub.status.idle":"2024-05-29T12:03:20.221593Z","shell.execute_reply.started":"2024-05-29T12:03:20.216406Z","shell.execute_reply":"2024-05-29T12:03:20.220159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_memory():\n ctypes.CDLL('libc.so.6').malloc_trim(0)\n gc.collect()\n \nclean_memory() ","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:03:20.222748Z","iopub.execute_input":"2024-05-29T12:03:20.223191Z","iopub.status.idle":"2024-05-29T12:03:20.403615Z","shell.execute_reply.started":"2024-05-29T12:03:20.223168Z","shell.execute_reply":"2024-05-29T12:03:20.402574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# To proudce simliar result in each run \ndef seed_everything():\n random.seed(CFG.SEED)\n os.environ['PYTHONHASHSEED'] = str(CFG.SEED)\n np.random.seed(CFG.SEED)\n torch.manual_seed(CFG.SEED)\n torch.cuda.manual_seed(CFG.SEED)\n torch.backends.cudnn.deterministic = True\n torch.backends.cudnn.benchmark = True\n \nseed_everything()","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:03:20.405913Z","iopub.execute_input":"2024-05-29T12:03:20.406201Z","iopub.status.idle":"2024-05-29T12:03:20.417802Z","shell.execute_reply.started":"2024-05-29T12:03:20.406177Z","shell.execute_reply":"2024-05-29T12:03:20.417043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#
2. Road and Read Data
","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv(CFG.BASE_PATH + 'train_metadata.csv')\n\nprint('Shape of Train: ', df_train.shape)\nprint(display(df_train.head()))","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:03:20.418999Z","iopub.execute_input":"2024-05-29T12:03:20.419280Z","iopub.status.idle":"2024-05-29T12:03:20.641840Z","shell.execute_reply.started":"2024-05-29T12:03:20.419256Z","shell.execute_reply":"2024-05-29T12:03:20.640933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.info()","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:03:20.643164Z","iopub.execute_input":"2024-05-29T12:03:20.643493Z","iopub.status.idle":"2024-05-29T12:03:20.686758Z","shell.execute_reply.started":"2024-05-29T12:03:20.643467Z","shell.execute_reply":"2024-05-29T12:03:20.685859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub = pd.read_csv(CFG.BASE_PATH + 'sample_submission.csv')\n\nprint('Shape of Sample Submission: ', sample_sub.shape)\nprint(display(sample_sub.head()))","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:03:20.688046Z","iopub.execute_input":"2024-05-29T12:03:20.688428Z","iopub.status.idle":"2024-05-29T12:03:20.728854Z","shell.execute_reply.started":"2024-05-29T12:03:20.688400Z","shell.execute_reply":"2024-05-29T12:03:20.727891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Listen Bird Songs!","metadata":{}},{"cell_type":"code","source":"import IPython.display as ipd\naudio_asbfly = glob('/kaggle/input/birdclef-2024/train_audio/asbfly/*')","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:03:20.730239Z","iopub.execute_input":"2024-05-29T12:03:20.730972Z","iopub.status.idle":"2024-05-29T12:03:20.742744Z","shell.execute_reply.started":"2024-05-29T12:03:20.730935Z","shell.execute_reply":"2024-05-29T12:03:20.741939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ipd.Audio(audio_asbfly[0])","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:03:20.744075Z","iopub.execute_input":"2024-05-29T12:03:20.744368Z","iopub.status.idle":"2024-05-29T12:03:20.771668Z","shell.execute_reply.started":"2024-05-29T12:03:20.744343Z","shell.execute_reply":"2024-05-29T12:03:20.770706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import IPython.display as ipd\naudio_barfly1 = glob('/kaggle/input/birdclef-2024/train_audio/barfly1/*')","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:03:20.775996Z","iopub.execute_input":"2024-05-29T12:03:20.776277Z","iopub.status.idle":"2024-05-29T12:03:20.783111Z","shell.execute_reply.started":"2024-05-29T12:03:20.776254Z","shell.execute_reply":"2024-05-29T12:03:20.781948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ipd.Audio(audio_barfly1[0])","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:03:20.784384Z","iopub.execute_input":"2024-05-29T12:03:20.784780Z","iopub.status.idle":"2024-05-29T12:03:20.802411Z","shell.execute_reply.started":"2024-05-29T12:03:20.784750Z","shell.execute_reply":"2024-05-29T12:03:20.801581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Quick EDA","metadata":{}},{"cell_type":"code","source":"import plotly.express as px\nimport plotly.graph_objects as go","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:03:20.803436Z","iopub.execute_input":"2024-05-29T12:03:20.803705Z","iopub.status.idle":"2024-05-29T12:03:21.348514Z","shell.execute_reply.started":"2024-05-29T12:03:20.803684Z","shell.execute_reply":"2024-05-29T12:03:21.347677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tmp = df_train[df_train['secondary_labels'] != '[]'].iloc[:50]\nfig = px.sunburst(tmp, path=['primary_label', 'secondary_labels'] )\nfig.update_layout(\n title=\"Primary & Secondary Labels \",\n )\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:03:21.349643Z","iopub.execute_input":"2024-05-29T12:03:21.349955Z","iopub.status.idle":"2024-05-29T12:03:22.851468Z","shell.execute_reply.started":"2024-05-29T12:03:21.349929Z","shell.execute_reply":"2024-05-29T12:03:22.850501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tmp = df_train.copy()\n\nfig = px.scatter_mapbox(\n tmp, \n lat=\"latitude\", \n lon=\"longitude\", \n color=\"primary_label\",\n zoom=0.1,\n title='Bird Recordings Loaction'\n)\n\nfig.update_layout(mapbox_style=\"open-street-map\")\nfig.update_layout(margin={\"r\":0,\"t\":30,\"l\":0,\"b\":0})\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:03:22.852715Z","iopub.execute_input":"2024-05-29T12:03:22.853055Z","iopub.status.idle":"2024-05-29T12:03:23.468672Z","shell.execute_reply.started":"2024-05-29T12:03:22.853031Z","shell.execute_reply":"2024-05-29T12:03:23.467776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Label Encoding","metadata":{}},{"cell_type":"code","source":"for label in df_train['primary_label'].unique():\n df_train[label] = np.where(df_train['primary_label'] == label, 1, 0) ","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:03:23.470103Z","iopub.execute_input":"2024-05-29T12:03:23.470779Z","iopub.status.idle":"2024-05-29T12:03:24.291540Z","shell.execute_reply.started":"2024-05-29T12:03:23.470746Z","shell.execute_reply":"2024-05-29T12:03:24.290690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\nencoder = LabelEncoder()\ndf_train['primary_label_encoded'] = encoder.fit_transform(df_train['primary_label'])","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:03:24.292613Z","iopub.execute_input":"2024-05-29T12:03:24.292912Z","iopub.status.idle":"2024-05-29T12:03:24.304157Z","shell.execute_reply.started":"2024-05-29T12:03:24.292886Z","shell.execute_reply":"2024-05-29T12:03:24.303076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#
3. DataLoader
","metadata":{}},{"cell_type":"markdown","source":"### Butter High Pass(>1Hz)& Gaussian Filtering","metadata":{}},{"cell_type":"code","source":"import librosa","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:03:24.305623Z","iopub.execute_input":"2024-05-29T12:03:24.305929Z","iopub.status.idle":"2024-05-29T12:03:24.318169Z","shell.execute_reply.started":"2024-05-29T12:03:24.305904Z","shell.execute_reply":"2024-05-29T12:03:24.317370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.signal import butter, lfilter\nfrom scipy.ndimage import gaussian_filter\n\ndef filtering(data, cutoff_freq=1000, sampling_rate=32000, order=3):\n nyquist = 0.5 * sampling_rate\n normal_cutoff = cutoff_freq / nyquist\n \n b,a = butter(order, normal_cutoff, btype='high', analog=False)\n filtered_data = lfilter(b, a, data, axis=0)\n filtered_data = gaussian_filter(filtered_data, sigma=1, order=2, mode='reflect')\n \n \n return filtered_data","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:03:24.319413Z","iopub.execute_input":"2024-05-29T12:03:24.319702Z","iopub.status.idle":"2024-05-29T12:03:24.387421Z","shell.execute_reply.started":"2024-05-29T12:03:24.319679Z","shell.execute_reply":"2024-05-29T12:03:24.386439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,6))\nplt.subplot(1,2,1)\nplt.title('Before Filtering')\noffset = 0\nfor i in range(5):\n y, _ = librosa.load(os.path.join(CFG.AUDIO_PATH, df_train.iloc[i]['filename']), sr=None)\n pd.Series(y+offset).plot()\n offset += 1\n \nplt.subplot(1,2,2)\nplt.title('After Filtering')\noffset = 0\nfor i in range(5):\n y, _ = librosa.load(os.path.join(CFG.AUDIO_PATH, df_train.iloc[i]['filename']), sr=None)\n y = filtering(y)\n pd.Series(y+offset).plot()\n offset += 1\n \nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:03:24.388699Z","iopub.execute_input":"2024-05-29T12:03:24.389005Z","iopub.status.idle":"2024-05-29T12:03:35.347731Z","shell.execute_reply.started":"2024-05-29T12:03:24.388981Z","shell.execute_reply":"2024-05-29T12:03:35.346744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torchaudio\nprint(torchaudio.__version__)","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:03:35.348853Z","iopub.execute_input":"2024-05-29T12:03:35.349373Z","iopub.status.idle":"2024-05-29T12:03:35.597567Z","shell.execute_reply.started":"2024-05-29T12:03:35.349347Z","shell.execute_reply":"2024-05-29T12:03:35.596640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Audio I/O","metadata":{}},{"cell_type":"code","source":"TARGETS = df_train.columns[-182:]\n\nclass BirdDataset(Dataset):\n def __init__(self, data, mode='train', downsample=True, augment=False):\n super(BirdDataset, self).__init__()\n self.data = data\n self.mode = mode\n self.downsample = downsample\n self.augment = augment\n \n def __len__(self):\n return len(self.data)\n \n \n def __getitem__(self, idx):\n X, y = self.generate_data(idx)\n if self.augment:\n X = self.__augment(X)\n # DownSampling\n if self.downsample:\n X = X[::1000,]\n \n return X,y\n \n def generate_data(self, index):\n row = self.data.iloc[index]\n audio_path = os.path.join(CFG.AUDIO_PATH, self.data.iloc[index]['filename'])\n # Torchaudio is faster than librosa when loading bid data \n audio, _ = torchaudio.load(audio_path)\n audio = audio.squeeze().float() # torch.float32\n \n # Fill Nan\n m = torch.nanmean(audio)\n audio = torch.nan_to_num(audio, nan=m)\n \n # standardization is already none from competition host\n # So, we don't need to remove ouliter \n \n # Filtering\n # audio = filtering(audio)\n \n X = audio\n \n if self.mode != 'test':\n y = torch.tensor(row[TARGETS].astype(float).values, dtype=torch.float32)\n \n return X, y\n \n # Masking(10%)\n def __augment(self, audio):\n mask_length = int(len(audio) * 0.10) \n start_idx = random.randint(0, len(audio) - mask_length)\n \n if random.random() <= 0.5:\n audio[start_idx:start_idx + mask_length] = 0\n \n if random.random() <= 0.5:\n audio = torch.flip(audio, dims=[0])\n \n \n return audio","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:03:35.598654Z","iopub.execute_input":"2024-05-29T12:03:35.598953Z","iopub.status.idle":"2024-05-29T12:03:35.611509Z","shell.execute_reply.started":"2024-05-29T12:03:35.598927Z","shell.execute_reply":"2024-05-29T12:03:35.610608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Customizing Collate_fn\n\nAs you know, Audio Lengths are different, so I'm gonna fix audio length by each batch","metadata":{}},{"cell_type":"code","source":"from torch.nn.utils.rnn import pad_sequence\n\ndef collate_fn(batch):\n\n # A data tuple has the form\n\n audios, targets = [], []\n\n for audio, target in batch:\n audios.append(audio)\n targets.append(target)\n\n audios = pad_sequence(audios, batch_first=True, padding_value=0).unsqueeze(1)\n targets = torch.stack(targets)\n\n return audios, targets\n ","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:03:35.612909Z","iopub.execute_input":"2024-05-29T12:03:35.613235Z","iopub.status.idle":"2024-05-29T12:03:35.627399Z","shell.execute_reply.started":"2024-05-29T12:03:35.613205Z","shell.execute_reply":"2024-05-29T12:03:35.626562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### No Audio Augmention","metadata":{}},{"cell_type":"code","source":"ROWS = 4\nCOLS = 4\nBATCH = 1\n\nfor label in df_train['primary_label'].unique()[:1]:\n print('#'*25)\n print(f'### Primary Label is {label}')\n \n tmp = df_train[df_train['primary_label'] == label]\n train_ds = BirdDataset(tmp)\n train_loader = DataLoader(train_ds, batch_size = CFG.TRAIN_BATCH, drop_last=True, collate_fn=collate_fn,\n num_workers=1, shuffle=False, pin_memory=True)\n \n for i, (X,y) in enumerate(train_loader):\n print(f\"Batch {i}:\")\n print(f\"X shape: {X.shape}\")\n print(f\"y shape: {y.shape}\")\n for j in range(ROWS):\n for i in range(COLS):\n file_name = tmp.iloc[j*COLS+i+1]['filename'].split('/')[0]\n plt.subplot(ROWS,COLS,j*COLS+i+1)\n pd.Series(X[j*COLS+i,0,:].numpy()).plot()\n plt.title(f'{file_name}', size=10)\n \n plt.tight_layout()\n plt.show() \n break","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:03:35.628519Z","iopub.execute_input":"2024-05-29T12:03:35.628865Z","iopub.status.idle":"2024-05-29T12:03:39.985634Z","shell.execute_reply.started":"2024-05-29T12:03:35.628803Z","shell.execute_reply":"2024-05-29T12:03:39.984546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Audio Augmentation\nIn this case, I just add `Masking(ratio=0.10) & p = 0.5` & `Time shift & p=0.5`","metadata":{}},{"cell_type":"code","source":"ROWS = 4\nCOLS = 4\nBATCH = 1\n\nfor label in df_train['primary_label'].unique()[:1]:\n print('#'*25)\n print(f'### Primary Label is {label}')\n \n tmp = df_train[df_train['primary_label'] == label]\n train_ds = BirdDataset(tmp, augment=True)\n train_loader = DataLoader(train_ds, batch_size = CFG.TRAIN_BATCH, drop_last=True, collate_fn=collate_fn,\n num_workers=1, shuffle=False, pin_memory=True)\n for i, (X,y) in enumerate(train_loader):\n print(f\"Batch {i}:\")\n print(f\"X shape: {X.shape}\")\n print(f\"y shape: {y.shape}\")\n for j in range(ROWS):\n for i in range(COLS):\n file_name = tmp.iloc[j*COLS+i+1]['filename'].split('/')[0]\n plt.subplot(ROWS,COLS,j*COLS+i+1)\n pd.Series(X[j*COLS+i,0,:].numpy()).plot(color='orange')\n plt.title(f'{file_name}', size=10)\n \n plt.tight_layout()\n plt.show() \n break","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:03:39.987081Z","iopub.execute_input":"2024-05-29T12:03:39.987352Z","iopub.status.idle":"2024-05-29T12:03:43.516914Z","shell.execute_reply.started":"2024-05-29T12:03:39.987326Z","shell.execute_reply":"2024-05-29T12:03:43.515795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#
4. Build WaveNet Model
","metadata":{}},{"cell_type":"markdown","source":"### What's Conv1D\nClick [**Pytorch Document**](https://pytorch.org/docs/stable/generated/torch.nn.Conv1d.html)\n\n nn.Conv1d(in_channels, out_channels, kernel_size, stride=1, padding=0, dilation=1, groups=1, bias=True, padding_mode='zeros',device=None, dtype=None)","metadata":{}},{"cell_type":"markdown","source":"* `in_channels`: Number of channels in the input image\n* `out_channels`: Number of channles produced by the convolution \n* `kernel_size`: Size of the convolving kernel \n* `stride`: Stride of the convolution. Default:1\n* `padding`: Padding added to both sides of the input, Default: 0\n* `padding_mode`: 'zeros','reflect','replicate', 'circular', Default: 'zeros'\n* `dilation`: Spacing between kernel elements. Default: 1\n* `groups`: Number of blocked connections from input channels to output channels, Default: 1\n* `bias`: If True, adds a learnable bias to the output. Default: True","metadata":{}},{"cell_type":"markdown","source":"There is the simplesst case\n\nN is a batch size, C denotes a number of channels, L is a length of signal sequence \n\n**Input Size**: (Batch size, Number of Channel, length of signal)\n\n**Output Size**: (Batch size, Number of Channel, length of signal)\n![](https://www.googleapis.com/download/storage/v1/b/kaggle-forum-message-attachments/o/inbox%2F16438831%2Fbfc63b2c99fc1142d06a6916e76bb80f%2Fsignal.JPG?generation=1716949013985473&alt=media)\n\n","metadata":{}},{"cell_type":"code","source":"audio, _ = torchaudio.load(os.path.join(CFG.AUDIO_PATH, df_train.iloc[0]['filename']))","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:03:43.518667Z","iopub.execute_input":"2024-05-29T12:03:43.519540Z","iopub.status.idle":"2024-05-29T12:03:43.550001Z","shell.execute_reply.started":"2024-05-29T12:03:43.519497Z","shell.execute_reply":"2024-05-29T12:03:43.549255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Dilation Effect**","metadata":{}},{"cell_type":"markdown","source":"![](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAANsAAADmCAMAAABruQABAAABI1BMVEX////3y6v4y6zDw8Pu7u7BwcH39/fFxMXR0dG8u7zLy8v9AAD+ZWb+6Oj/z874AAD40bHguJv+eXr9S0v/rq3Yspf+b27RsZbWnIX+mZjnvqHQqpD/2dr5Zlb8EA//+fgAAAC1lH74qo/FpIv8VkblqI77eWb/i4vc3Nz8XEzvNSy5q6Dk5ui+oZD7oojnu5zIhXKurq5xcXFeXl58fHw/Pz//0bCdnZ02NjaampqpqKhlZWWJiYlycnJEQ0MrKyoeHh5PT0/sVkihjH7yIhrXgG3ijnnvb2/fv6K7m4TKkn7tbGiwmIj5aVnIrJP6IhfnRz3qWVj9oqEYGBf9v7/aZ17keXbraVjfs7PS4eKVgm/fln+2dGGsnZSpinbVpqSkcmE5NZW0AAANQElEQVR4nO1diX/bthUGKQg8kjZRNyWt1YOuupXtmKWkKFM8ZFnuFWdtl03beqzz9v//FQNI4pAFXSRliyq+/H4xBEEAPr4H4PGBxAOR0Tk0jATcAUz2bXX/XhoRsO+2ewB0VriZ99CqBYx7aEVxaxqKW10obk1DcasLxa1pKG51obg1DcWtLhS3pqG41YXiBt56vyo+klV3VNweP6mIR2/Lqjsubtf9QYn+ly5DMmC46bJc85YVHrSB2wdP9RIXL3i2rWk0t++xXDS8KHM1rR3cNB3zwP+J3IyXRR7+13dZLhroZW6LuGkybtoqt6Fe5p4kN61t3HS5TuoyueW57RlvJe5wK7EqtwIt4XavcvtoBzTITSa3My43vgbAwUVtub33iONDefr3TXG77lG8/E74df8lzR0KcjtnhXtVuX361Z8ZvubJb76hqY+b4/btz8zs8ITC3ERxEc8Wcitz+/wpw1BjyX6PJZvj9um2itagBjem7kOm4Vq/R1Nt56YXq+ewNH30gluRq7gt45i4UZ0szJyT0sl8idSp3PRSbmVuy7ktye205hJ2wzjUWLLf08sPreb2iq3/532WHA5o6tWT5tZujwHybMRzhRUdCLnVuT3vU9wOWPJmSFMNcrtm1fe/5Nld3uo5t7ngLS/ciE7qDFgn9cZ1kvlLtAvRntToML9YtpVpDyrbyoQbnTYOPZcI9wEit5d0mGvCPQ5s4P6Nrt18DdAOtgYQbvmN2hI37lO4w628q6vP7T7sEjm3zlpudf0lAre7dol+IJ2UcCvvsFd0sj63+7JLSp/Cqk6yUS7RyRo+hfu0SzbOJVK51Z5LJGsAtkvoFHyINUBflhtfeVbkluNga/fzBtdu3pSwdncOuHZLba6b4UnYXE/1ctYgOllMJlphl5zCPU7JgswlRfKU7t90qc3V+Pr2wP4SrdzzOh1/yVpuevu53dN9wMPOJfrpzSWlIZSb3kUS2yUnILev/sDwWki+y5LNrd1/cU0KVyzMIex1CLmVuT3aiubsybOXFKKt3GO5A3GPiuW+rLy3+NZWNL23qK/sCUvuTVG+t1jM2i3ZE6bjWbrfrbV/v3tVbuXDMyfwnIKEW6F+LX92hvqCdpLbKXNroU6unUva/OzM9/yxgT/xbFt4gkB4xvAHXrgN3P76GcObdxj+JuQ+Y7nP3vDsJ8fP7e/PquGdZ+/Lqjsqbg1DcasLxa1pKG51obg1DcWtLhS3pqG41YXi1jR+u9w8JH6CxT7tmh9s6u4GbuLmL8735OVL7CAHyOvbxA2OR1eC6zoczWL8B03kpd35+gbXc5uPFl2e7c/neS0LdPcHBSbL+SheKTEdXUU0vYkbQiAjv+6SYrg/ADkkdwxcF9gWAq5nIRP6+EJ5vgvc0dqKNnBDoEt+10HAQKQJ4BDJzRDsANf3AHINz4UWvsLIwj8ZF9xMaOFvXeDPcC+63aWKgbvYhRtAcE76kPpWkH9MyB906c1gNk3GIHJiOJvHDjDH1tisxg3AKTm8Kwq8Wd6Emf+ZoIltX1oTz3UyM7uKHBfN/DSh3MKryA6shUe4jeKYNJwEQVCcAmZe7sQtno/JVUSzXA3d0SzJP80gcGx74kZRriVjGMZ2HHqVuFmjWX5A2Ogqbykd51dxPO6AUWKHMVH0sANi3w9sf8y44T5DOzXQHHgL215AMlDpYxszduDYlnnSGJP/J+Pyo0OUYzy3gWP5Por8nNscBpnvuxXlBqBD/g+ccigFZLiMwwzMI1JpipnYIPKT1Pdtxs0E9tgf2ZibO7N8rJjAD8PQJ9+l/Ay3TdzsLgzJZcwiMuxQ4pkOJNzgAs47yAVxUnDzkimCsBI3L4E+0cJuapGf+643I2Vn6LITZ8hDpNKcm3sJ8Rw6ASYZ+VMTRBm6tNGVhxYuIvMdlVuIu7ILNxiOYnxJUAgQoeinAakFZaAToTDNgIWln+H5BoIEf+WtTlrbuYF4FJK+4Dqm+K8dpKTvIMNtgzidQlJp4pKWumnawfkxaQXnoHAau7hd6Aap2PA0HY2oD7Nta/d4exGGtnGD24swtI3bPlDc6kJxaxoWiDrdQ6MjORP14I1KWlVQUFBQUFBQUFBQUFBQUFBQUFBQUGgA6D7wQK1Gpn1wSHzmh2/UjE56r0NxqwfFrWkobnWhuDWN3wq33zWMfxwRt0fXf2T453sy/IsX+FFa4L0feQnhCIkj4PZa+iqrAOGt1ueetMQL4QXXo+L2BX8FeQ03dn6JeCK8iBfsveSj48bO31onN34+l/ypsRc6PcTx6Lhtk5u2VW76scrtgh4MtF5uOpXbOm60hmPj9lQrDxTbIDd9q9xWjyQ7Dm50tGwcbxu5He14Y8c/btDJ8iTINdxYIKUK3N5/uyq2c/uVny70Hzk3firRUM7tO1bg1f7cHm8/dmkNtnP7yZWFyxCAhBgZ0gJCEI2f97dLHl8/Z/iFJ29/YMmbG5b84ZYl/70DN+mJC5Xx4f7chHOxz9mp1xdDeiq8eBKrfsbOxX768Q7c5GpbFZW4sXhU5xfssJnhGT/DqqfTmGJn7EydFnHTd+emt4sbNYzOL+iRT4RbaQzl3Mo1+IydqdMabsX5Rjk3etYRkVuRq9GIDFRuxVlIbeF2R26aVG5aS+UmH2+6ZLxRCbaFG5v4qU7eOY27z7LPeHYruF3zY6pve7ITq4fsxOregMcR22XtfnhuPXqKN+bGk+zM797NDTvnG3OjyVZwW9JJXaaTMrukHTrJzjHUdTLeyqMzxbW7zwIitNAuKafBizXceqfCrTSq2Nqt07Wbcmvl2q1L126t7Wt3IQwuN10qt9baXHy8aaVOUrt5ebyVuW2Rm3QNELw0whrAY1zvwu0nbw+fgvwYDq+mT4Ev2DwahxglRFy7WXKXtftXHjpV7gsyWGC03o2cPTeVXlXhdjib6ws2N63d69Co72KtD6+Gz3wXu4T5Tvb0l2zf69Cp23iD77UcBNW4iXPJhjVA33sNKP3KBGu5lW7jzX5lrZpf+c7aXTqxuV1SrgGV7BK+H1CXW2WdXOYm2CVSbnv4gnbTSW2LTpY11NDJQ9glO+ybatQXc4j9tzt2Cd0Qa8QuYXLboJM7yq26Th5Kbnzerb2Po9deA7i/RG6X7LcG/DpgWLOPM2QFzuXcvmQFnleyS+iv+7/0WU23QqvnLDm8Zcld1u6fpGG8BKyJ6CV2mtdQyxd0dnu2xS7pt8wXxPY6dO4zL+yScgeE+cwLW7ll993F3Ed95tz3Ks6TfK+jXfdv4h6VxvY6lv0l7d2jOuH9N66T2qnpJDWMTtUXdKJzydZ7U5p90ba9fG6XDH7hNlIjdsmDc/vg1RnFLUud3fRYcjhgyf4NS7aD213/ZHFvqrMk90+20S451D3OUXDbba+jjfOkVqrfae5RScbbqXDTT1knDzSXPPgzht/L7Elsl9C0EFX1Aq8B+3D7r/AiqrQ07LACHfk+jlBDBX/Jm3cZ/idNvn7Nkzz76118QRp7IneNn4s/9DvY9kxvFT/XAZ/p3b6PwxR/7XsddfZxtkdSXYdduLE3F+q++3CEz9BvlZu+TW5H+87KVp+5RncyWvnuwy7ctFZy0073fZwTfo9q6zuZpvBO5pq55Fjfyfz8E4Y30mCb17zAJ4+lJd6w7z87Km7vNAxuej88t8NBcWsailtdKG5N47fBDS4bGkXotzXPUQJzeywXsb59uKHl5wTwJ1SxI4xbNp8EQtuzlMSeArEv/9n0jrSjlSb8yzkL6LUXt+wyEDviBCkpm1ny0sEGBWDc8MVx8B/XJNHdyKc8OlJsAR9AH5cxzC50O6QFy0ckRlTesmt50O8Cdxa7wEzEa4tr4IHx9uJWfEU6glOwjHKVdUlHEtqRLu3IdBdumAGpxnPAjHQRennot7ibZWhiTSOwSH3TiUcxSONkRLnZTmxOrSDyJonnp9YES89Ng3RKtAjaFeU2D8mX0EETj/wJ80PZMiPvSBABJ/Vxs6MIBHEy340bDMYh+Zs4eV1ROiFXPboMgZ8a/gxcIWAHwJ0jxzYcSLkR7bHjAIw8MPONkU/i4cEi5OF0njJX2D7coGfmsft8Jw8u6HpT0q8YdyRJjWRCOmJMgTtCjmEsYLiT3ACYkGTXKTXbJgHtonAMSOg1A8xwTkiqXFi+xeQWAjSLsinhdpX4vpfLLQgLUgGLfLjnPDklXTCccqznEf2y8BJEAYkBhz8ZuCMpcvKO7MItcvPLBReQBHmzLS8P/RZ3k9SbeJ6by21KAkZembhjoY2ygpu3QBmWmw/DCJJnz1H5kHpiujNGaR9ukWvnYeaKjhhdN4+BlhlR4I4h6QiujlxkMLNxLTvpZGeaTzlWF1gWCfKW5hfdwjQ8OwgMgKm4CYAxgCH+ync9Eq3M9UlYuCTBl9EGcZoJ/lg3THmQx324dYOQjAY8W5CO4NZy8fm4I9AIAjvvCJ5WoqIjyZq1YYnbvtikDCt46LV7T+wTiK1t3PaC4tY0LJAYnYNjxXJD99CqoeKIKSjcI/4P07aUj42rpTwAAAAASUVORK5CYII=)","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(12,6))\nfor i in range(5):\n plt.subplot(1,5,i+1)\n plt.title(f'Dilations: {2**i}')\n conv1d = nn.Conv1d(in_channels=1, out_channels=1, kernel_size=3, dilation=2**i, padding='same', bias=True)(audio)\n pd.Series(conv1d[0].detach().numpy()).plot(color='pink')\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:03:43.551069Z","iopub.execute_input":"2024-05-29T12:03:43.551318Z","iopub.status.idle":"2024-05-29T12:03:45.227757Z","shell.execute_reply.started":"2024-05-29T12:03:43.551290Z","shell.execute_reply":"2024-05-29T12:03:45.226807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Wave Block","metadata":{}},{"cell_type":"code","source":"class WaveBlock(nn.Module):\n def __init__(self, in_channels, filters, kernel_size, n):\n super(WaveBlock, self).__init__()\n self.filters = filters\n self.kernel_size = kernel_size\n self.n = n\n self.dilation_rates = [2 ** i for i in range(n)]\n \n self.init_conv = nn.Conv1d(in_channels=in_channels, out_channels=filters, kernel_size=1, padding='same')\n self.residual_convs = nn.ModuleList()\n self.tanh_convs = nn.ModuleList()\n self.sigm_convs = nn.ModuleList()\n self.final_convs = nn.ModuleList()\n \n for dilation_rate in self.dilation_rates:\n self.tanh_convs.append(nn.Conv1d(in_channels=filters, out_channels=filters, kernel_size=kernel_size, padding='same', dilation=dilation_rate))\n self.sigm_convs.append(nn.Conv1d(in_channels=filters, out_channels=filters, kernel_size=kernel_size, padding='same', dilation=dilation_rate))\n self.final_convs.append(nn.Conv1d(in_channels=filters, out_channels=filters, kernel_size=1, padding='same'))\n \n def forward(self, x):\n x = self.init_conv(x)\n res_x = x\n \n for tanh_conv, sigm_conv, final_conv in zip(self.tanh_convs, self.sigm_convs, self.final_convs):\n tanh_out = torch.tanh(tanh_conv(x))\n sigm_out = torch.sigmoid(sigm_conv(x))\n x = tanh_out * sigm_out\n x = final_conv(x)\n res_x = res_x + x\n \n return res_x","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:03:45.228948Z","iopub.execute_input":"2024-05-29T12:03:45.229227Z","iopub.status.idle":"2024-05-29T12:03:45.240722Z","shell.execute_reply.started":"2024-05-29T12:03:45.229204Z","shell.execute_reply":"2024-05-29T12:03:45.239872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Display WaveBlock","metadata":{}},{"cell_type":"code","source":"wave = WaveBlock(1, filters=4, kernel_size=3, n=5)(audio)\n\nplt.subplot(2,2,1)\nplt.title('Applying Wave Block: Channel 0', size=10)\npd.Series(wave[0].detach().numpy()).plot()\nplt.subplot(2,2,2)\nplt.title('Applying Wave Block: Channel 1', size=10)\npd.Series(wave[1].detach().numpy()).plot()\nplt.subplot(2,2,3)\nplt.title('Applying Wave Block: Channel 2', size=10)\npd.Series(wave[2].detach().numpy()).plot()\nplt.subplot(2,2,4)\nplt.title('Applying Wave Block: Channel 3', size=10)\npd.Series(wave[3].detach().numpy()).plot()\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:03:45.246024Z","iopub.execute_input":"2024-05-29T12:03:45.246316Z","iopub.status.idle":"2024-05-29T12:03:47.242937Z","shell.execute_reply.started":"2024-05-29T12:03:45.246293Z","shell.execute_reply":"2024-05-29T12:03:47.241830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Bird_WaveNet(pl.LightningModule):\n def __init__(self):\n super(Bird_WaveNet, self).__init__()\n \n # WaveBlock layers\n self.waveblock1 = WaveBlock(in_channels=1, filters=4, kernel_size=3, n=5) # 1,2,4,8,16\n self.waveblock2 = WaveBlock(in_channels=4, filters=8, kernel_size=3, n=4) # 1,2,4,8\n self.waveblock3 = WaveBlock(in_channels=8, filters=16, kernel_size=3, n=3) # 1,2,4\n self.waveblock4 = WaveBlock(in_channels=16, filters=32, kernel_size=3, n=1) # 1(Default)\n \n self.dropout = nn.Dropout(p=0.10)\n \n # RNN Block\n self.rnn = nn.GRU(input_size=32, hidden_size=32, \n num_layers=2, batch_first=True, bidirectional=True)\n \n # Dense layers\n self.fc1 = nn.Linear(in_features=64, out_features=128) \n self.fc2 = nn.Linear(in_features=128, out_features=182)\n \n def forward(self, x):\n x = self.waveblock1(x) # (Batch Size X 4 X Signal Length)\n x = self.waveblock2(x) # (Batch Size X 8 X Signal Length)\n x = self.waveblock3(x) # (Batch Size X 16 X Signal Length)\n x = self.waveblock4(x) # (Batch Size X 32 X Signal Length)\n \n x = x.transpose(1, 2) # Change shape from (batch, channels, seq_len) to (batch, seq_len, channels) for RNN\n x, _= self.rnn(x)\n \n x = F.relu(self.fc1(x[:, -1, :]))\n output = self.fc2(x)\n\n return output\n \n def training_step(self, batch, batch_idx):\n x, y = batch \n out = self.forward(x)\n out = F.log_softmax(out, dim=-1)\n loss_fn = nn.CrossEntropyLoss(reduction='mean')\n loss = loss_fn(out, y)\n return loss\n \n def predict_step(self, batch, batch_idx, dataloader_idx=0):\n return F.softmax(self(batch), dim=-1)\n \n \n def configure_optimizers(self):\n optimizer = torch.optim.AdamW(self.parameters(), lr=1e-3)\n \n return optimizer","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:03:47.244264Z","iopub.execute_input":"2024-05-29T12:03:47.244542Z","iopub.status.idle":"2024-05-29T12:03:47.257638Z","shell.execute_reply.started":"2024-05-29T12:03:47.244518Z","shell.execute_reply":"2024-05-29T12:03:47.256746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#
4. Train WaveNet Model
","metadata":{}},{"cell_type":"markdown","source":"
\n \nšŸ“Œ **Check out** \n \n - I made this notebook for study\n - Audio Data is much data in this competition\n - So, for studying, I did oversampling[::1000], sampling: 2000\n - So, If you want to make 1D CNN, you have to control Out of Memory\n ","metadata":{}},{"cell_type":"code","source":"df_train = df_train.sample(2000)","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:03:47.258923Z","iopub.execute_input":"2024-05-29T12:03:47.259619Z","iopub.status.idle":"2024-05-29T12:03:47.292547Z","shell.execute_reply.started":"2024-05-29T12:03:47.259594Z","shell.execute_reply":"2024-05-29T12:03:47.291612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_oof = []\nall_true = []\nvalid_loaders = []\n\ngkf = GroupKFold(n_splits=5)\nfor i, (train_index, valid_index) in enumerate(gkf.split(df_train, df_train[TARGETS], df_train['primary_label_encoded'])):\n print('#'*25)\n print(f'### Fold {i+1}')\n \n train_ds = BirdDataset(df_train.iloc[train_index], augment=True)\n train_loader = DataLoader(train_ds, batch_size = CFG.TRAIN_BATCH, drop_last=True, collate_fn=collate_fn,\n num_workers=2, shuffle=True, pin_memory=True)\n \n valid_ds = BirdDataset(df_train.iloc[valid_index], augment=True)\n valid_loader = DataLoader(train_ds, batch_size = CFG.EVAL_BATCH, drop_last=True, collate_fn=collate_fn,\n num_workers=2, shuffle=False, pin_memory=True)\n \n print(f'### Train Size: {len(train_index)}, Valid size: {len(valid_index)}')\n print('#'*25)\n \n trainer = pl.Trainer(max_epochs=1, precision=16)\n model = Bird_WaveNet()\n model.to(DEVICE)\n model.train()\n if LOAD_MODELS_FROM is None: \n trainer.fit(model=model, train_dataloaders=train_loader)\n \n trainer.save_checkpoint(f'Bird_WaveNet_v{CFG.VER}_f{i}.ckpt')\n \n valid_loaders.append(valid_loader)\n all_true.append(df_train.iloc[valid_index][TARGETS].values)\n \n del trainer, model\n \n clean_memory()\n torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:03:47.293949Z","iopub.execute_input":"2024-05-29T12:03:47.294631Z"},"trusted":true},"execution_count":null,"outputs":[]}]}