{"cells":[{"metadata":{},"cell_type":"markdown","source":"# What are Transformers?\n\nTransformers are neural networks which make use of a concept called \"attention layers\" from NLP which means paying attention to only those parts of the inputs which are important. Through multiple attention layers Transformers learn the necessary parts of inputs and are able to learn better contextual embeddings in the case of text.\n\nIn recent times, Transformers are being heavily applied to Computer vision tasks. In this notebook I will try to experiment by applying it to audio data and see if we get results that are good enough.\n\nFramework of choice is definitely Pytorch Lightning since it lets me set up a baseline and keep modifying it with ease. "},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install pytorch_lightning","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport librosa\nimport IPython.display as ipd\nimport torchaudio.transforms as a_trans\nimport soundfile as sf\nimport torch\nimport torchaudio\nimport pytorch_lightning as pl\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torchvision.models import resnext50_32x4d\nfrom torch.utils.data import Dataset,DataLoader\nimport sklearn\nfrom torchvision.transforms import RandomResizedCrop\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"tp_train = pd.read_csv(\"../input/rfcx-species-audio-detection/train_fp.csv\")\nfp_train = pd.read_csv(\"../input/rfcx-species-audio-detection/train_fp.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tp_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Describe train dataframe ####\ntp_train.describe()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Multiple species are recorded in a single audio clip. I convert them into a one hot vector with ones at indices based on which sound is detected"},{"metadata":{"trusted":true},"cell_type":"code","source":"### Convert problem to multiclass ###\nids = tp_train.recording_id.unique()\nmultilabel_list = []\nfor filename in ids:\n    req = tp_train[tp_train.recording_id==filename]\n    labels = req.species_id\n    multilabel_list.append(labels.values)\nnew_df = pd.DataFrame()\nnew_df[\"rec_id\"] = ids\nnew_df[\"labels\"] = multilabel_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#### Check species distribution ###\nfig = plt.figure(figsize=(10,8))\nsns.countplot(tp_train.species_id)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def onehot(df,num_classes):\n    onehot_labels = []\n    for indices in df.labels:\n        vec = np.zeros(num_classes)\n        vec[indices-1] = 1\n        onehot_labels.append(vec)\n    return onehot_labels\nlabs = onehot(new_df,23)\nnew_df[\"onehot\"] = labs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Check distribution of time and frequency limits ###\nfig,ax = plt.subplots(2,2,figsize=(8,8))\nsns.distplot(tp_train.t_min,ax=ax[0][0])\nsns.distplot(tp_train.t_max,ax=ax[0][1])\nsns.distplot(tp_train.f_min,ax=ax[1][0])\nsns.distplot(tp_train.f_max,ax=ax[1][1])\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import librosa.display","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This metric is lwlwrap, which is a version of sklearn's lrap. More details can be found here: https://scikit-learn.org/stable/modules/generated/sklearn.metrics.label_ranking_average_precision_score.html"},{"metadata":{"trusted":true},"cell_type":"code","source":"def _one_sample_positive_class_precisions(scores, truth):\n    \"\"\"Calculate precisions for each true class for a single sample.\n\n    Args:\n    scores: np.array of (num_classes,) giving the individual classifier scores.\n    truth: np.array of (num_classes,) bools indicating which classes are true.\n\n    Returns:\n    pos_class_indices: np.array of indices of the true classes for this sample.\n    pos_class_precisions: np.array of precisions corresponding to each of those\n      classes.\n    \"\"\"\n    num_classes = scores.shape[0]\n    pos_class_indices = np.flatnonzero(truth > 0)\n    # Only calculate precisions if there are some true classes.\n    if not len(pos_class_indices):\n        return pos_class_indices, np.zeros(0)\n    # Retrieval list of classes for this sample. \n    retrieved_classes = np.argsort(scores)[::-1]\n    # class_rankings[top_scoring_class_index] == 0 etc.\n    class_rankings = np.zeros(num_classes, dtype=np.int)\n    class_rankings[retrieved_classes] = range(num_classes)\n    # Which of these is a true label?\n    retrieved_class_true = np.zeros(num_classes, dtype=np.bool)\n    retrieved_class_true[class_rankings[pos_class_indices]] = True\n    # Num hits for every truncated retrieval list.\n    retrieved_cumulative_hits = np.cumsum(retrieved_class_true)\n    # Precision of retrieval list truncated at each hit, in order of pos_labels.\n    precision_at_hits = (\n      retrieved_cumulative_hits[class_rankings[pos_class_indices]] / \n      (1 + class_rankings[pos_class_indices].astype(np.float)))\n    return pos_class_indices, precision_at_hits\n\n\n# All-in-one calculation of per-class lwlrap.\n\ndef calculate_per_class_lwlrap(truth, scores):\n    \"\"\"Calculate label-weighted label-ranking average precision.\n\n    Arguments:\n    truth: np.array of (num_samples, num_classes) giving boolean ground-truth\n      of presence of that class in that sample.\n    scores: np.array of (num_samples, num_classes) giving the classifier-under-\n      test's real-valued score for each class for each sample.\n\n    Returns:\n    per_class_lwlrap: np.array of (num_classes,) giving the lwlrap for each \n      class.\n    weight_per_class: np.array of (num_classes,) giving the prior of each \n      class within the truth labels.  Then the overall unbalanced lwlrap is \n      simply np.sum(per_class_lwlrap * weight_per_class)\n    \"\"\"\n    assert truth.shape == scores.shape\n    num_samples, num_classes = scores.shape\n    # Space to store a distinct precision value for each class on each sample.\n    # Only the classes that are true for each sample will be filled in.\n    precisions_for_samples_by_classes = np.zeros((num_samples, num_classes))\n    for sample_num in range(num_samples):\n        pos_class_indices, precision_at_hits = (\n          _one_sample_positive_class_precisions(scores[sample_num, :], \n                                                truth[sample_num, :]))\n        precisions_for_samples_by_classes[sample_num, pos_class_indices] = (\n            precision_at_hits)\n    labels_per_class = np.sum(truth > 0, axis=0)\n    weight_per_class = labels_per_class / float(np.sum(labels_per_class))\n    # Form average of each column, i.e. all the precisions assigned to labels in\n    # a particular class.\n    per_class_lwlrap = (np.sum(precisions_for_samples_by_classes, axis=0) / \n                      np.maximum(1, labels_per_class))\n    # overall_lwlrap = simple average of all the actual per-class, per-sample precisions\n    #                = np.sum(precisions_for_samples_by_classes) / np.sum(precisions_for_samples_by_classes > 0)\n    #           also = weighted mean of per-class lwlraps, weighted by class label prior across samples\n    #                = np.sum(per_class_lwlrap * weight_per_class)\n    return per_class_lwlrap, weight_per_class\n\n\ndef calculate_overall_lwlrap_sklearn(truth, scores):\n    \"\"\"Calculate the overall lwlrap using sklearn.metrics.lrap.\"\"\"\n    # sklearn doesn't correctly apply weighting to samples with no labels, so just skip them.\n    sample_weight = np.sum(truth > 0, axis=1)\n    nonzero_weight_sample_indices = np.flatnonzero(sample_weight > 0)\n    overall_lwlrap = sklearn.metrics.label_ranking_average_precision_score(\n      truth[nonzero_weight_sample_indices, :] > 0, \n      scores[nonzero_weight_sample_indices, :], \n      sample_weight=sample_weight[nonzero_weight_sample_indices])\n    return overall_lwlrap\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"import math\nclass SpecDataset(Dataset):\n    def __init__(self,csv,transforms=None):\n        self.csv = csv\n        self.csv.reset_index(inplace=True)\n        \n    \n    def __len__(self):\n        return len(self.csv)\n\n    def __getitem__(self,idx):\n        if torch.is_tensor(idx):\n            idx = idx.tolist()\n        \n        sig,sr =  torchaudio.load(\"../input/rfcx-species-audio-detection/train/\"+self.csv.recording_id[idx]+\".flac\",normalization=True)\n        label = self.csv.species_id[idx]\n        f_min_1 = self.csv.f_min[idx]\n        f_max_1 = self.csv.f_max[idx]\n        t_min_1 = self.csv.t_min[idx]\n        t_max_1 = self.csv.t_max[idx]\n        #print(\"B SIG: \",sig.size())\n        sig = sig[:,math.floor((t_min_1*sr)):math.floor(((t_min_1+4)*sr))]\n        \n        if sig.size()[1]!=4*sr:\n            \n            #print(\"SIG: \",sig.size()) \n            if sig.size()[1]>4*sr:\n                sig = sig[:,:4*sr]\n            else:\n                tens = torch.zeros(1,4*sr)\n                tens[:,:sig.size(1)] = sig\n                sig = tens\n        \n                \n        spec_fn = a_trans.MelSpectrogram(sample_rate=sr,n_fft=2048,hop_length=512,f_min=f_min_1,f_max=f_max_1)\n        S = spec_fn(sig)\n        S_db = a_trans.AmplitudeToDB().forward(S)\n        #print(S_db.shape)\n        return {\"X\": S_db,\"Y\": label}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"I set up a data class to convert audio clips to their subsequent spectrograms. I use torchaudio because:\n\na) It allows me to generate spectrograms using a GPU, thus making it faster.<br>\nb) Librosa is too damn slow :')\n\nI convert the input to a 256 x 256 shaped tensor for easily inputting it to our backbone"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Transformer_Data(Dataset):\n    def __init__(self,csv,transform=None):\n        self.csv = csv\n        self.csv.reset_index(inplace=True,drop=True)\n        \n    \n    def __len__(self):\n        return len(self.csv)\n    \n    def __getitem__(self,idx):\n        if torch.is_tensor(idx):\n            idx = idx.to_list()\n        sig,sr =  torchaudio.load(\"../input/rfcx-species-audio-detection/train/\"+self.csv.rec_id[idx]+\".flac\",normalization=True)\n        \n        label = self.csv.onehot[idx]\n        spec_fn = a_trans.MelSpectrogram(sample_rate=sr,n_fft=2048,hop_length=512)\n        S = spec_fn(sig)\n        S_db = a_trans.AmplitudeToDB().forward(S)\n        #S_db = S_db.reshape([S_db.shape[1],S_db.shape[2],S_db.shape[0]])\n        crop = RandomResizedCrop((256,256))\n        S_db = crop(S_db)\n        #print(S_db.shape)\n        return {\"X\": S_db,\"y\": label}\n        \n        \n        ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## The backbone\n\nI use the Vision Transformer, which was a version of the transformer network used for image classification. More details about the same can be found in this paper: https://arxiv.org/abs/2010.11929\n\nI did not implement it from scratch since another implementation was already available. Thanks to this amazing repo: https://github.com/lucidrains/vit-pytorch"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install vit-pytorch","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train,X_test = train_test_split(new_df,test_size=0.2,random_state=42)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Set up the datasets"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = Transformer_Data(X_train)\nval_dataset = Transformer_Data(X_test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Set up the dataloaders"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataloader = DataLoader(train_dataset,batch_size=32,num_workers=4,shuffle=True)\nval_dataloader = DataLoader(val_dataset,batch_size=32,num_workers=4,shuffle=True)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Set up the model class. This class now makes it easy to train and can be modified at will for other networks. See how easy lightning makes it"},{"metadata":{"trusted":true},"cell_type":"code","source":"from vit_pytorch import ViT\nclass AudioClassifier(pl.LightningModule):\n    def __init__(self):\n        super().__init__()\n        #self.backbone = resnext50_32x4d()\n        self.backbone = ViT(\n                    image_size = 256,\n                    patch_size = 32,\n                    num_classes = 23,\n                    dim = 1024,\n                    depth = 6,\n                    heads = 16,\n                    mlp_dim = 2048,\n                    dropout = 0.1,\n                    emb_dropout = 0.1,\n                    channels= 1\n                )\n\n        #self.backbone.conv1 = nn.Conv2d(1, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n        #self.fc = nn.Linear(1000,24)\n        \n    \n    def forward(self,x):\n        out = self.backbone(x)\n        #out = self.fc(x)\n        return out\n    \n    def training_step(self,batch,batch_idx):\n        X = batch[\"X\"]\n        y = batch[\"y\"]\n        y_hat = self.forward(X)\n        #y_hat = y_hat.float()\n        #y = y.float()\n        loss = F.binary_cross_entropy_with_logits(y_hat,y)\n        self.log(\"Training loss \",loss,logger=True,prog_bar=True)\n        \n        return loss\n    \n    def validation_step(self,batch,batch_idx):\n        X = batch[\"X\"]\n        y = batch[\"y\"]\n        #X = torch.squeeze(X,1)\n        #print(\"Y SHAPE: \",y.shape)\n        #y_oh= F.one_hot(y,24)\n        y_hat = self.forward(X)\n        lwlwrap = calculate_overall_lwlrap_sklearn(y.detach().cpu().numpy(),y_hat.detach().cpu().numpy())\n        #print(\"DType: \",y_hat.dtype)\n        #print(\"ODtype: \",y.dtype)\n        #y_hat = y_hat.float()\n        #y = y.float()\n        loss = F.binary_cross_entropy_with_logits(y_hat,y)\n        self.log(\"Val loss \",loss,logger=True,prog_bar=True)\n        return {\"loss\":loss,\"lwlwrap\": lwlwrap}\n    \n    def validation_epoch_end(self,val_ops):\n        metric_list = [op[\"lwlwrap\"] for op in val_ops]\n        val_lwlwrap = sum(metric_list)/len(metric_list)\n        self.log(\"Val lwlwrap \",val_lwlwrap,logger=True,prog_bar=True)\n        return val_lwlwrap\n    \n    \n        \n        \n    def configure_optimizers(self):\n        return torch.optim.Adam(self.parameters(),lr=1e-4)\n        \n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Train the model\nI train on a gpu currently since I am facing issues with the tpu. 16 bit precision is used to put less pressure on the memory"},{"metadata":{"trusted":true},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\nmodel = AudioClassifier()\ntrainer = pl.Trainer(gpus=1,precision=16,max_epochs=5)\ntrainer.fit(model,train_dataloader,val_dataloader)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Load the model from checkpoint for inference"},{"metadata":{"trusted":true},"cell_type":"code","source":"### LOAD MODEL FROM CHECKPOINT ###\nmodel = AudioClassifier.load_from_checkpoint(\"lightning_logs/version_0/checkpoints/epoch=4.ckpt\")\nmodel.eval()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Make Inference on the test data using this function. This is currently facing memory issues, will fix it in the upcoming versions"},{"metadata":{"trusted":true},"cell_type":"code","source":"import glob\ndef inference(model,test_folder):\n    all_preds = []\n    model.eval()\n    for ii,filename in enumerate(glob.glob(test_folder+\"/*\")):\n        print(\"Iteration: \",ii)\n        sig,sr =  torchaudio.load(filename,normalization=True)\n        spec_fn = a_trans.MelSpectrogram(sample_rate=sr,n_fft=2048,hop_length=512)\n        S = spec_fn(sig)\n        S_db = a_trans.AmplitudeToDB().forward(S)\n        #S_db = S_db.reshape([S_db.shape[1],S_db.shape[2],S_db.shape[0]])\n        crop = RandomResizedCrop((256,256))\n        S_db = crop(S_db)\n        preds = F.sigmoid(model(S_db.unsqueeze(0)))\n        all_preds.append(preds)\n    \n        \n        \n        \n        \n    all_preds = torch.cat(all_preds)    \n    return all_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''import time\nstart = time.time()\npreds = inference(model,\"../input/rfcx-species-audio-detection/test\")\nprint(\"End: \",time.time()-start)'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}