{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Pytorch Audio Classification (In Progress)","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Imports","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install torchsummary","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from __future__ import print_function, division\n\nimport pathlib\nimport os\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nfrom torchsummary import summary\n\nimport torch\nimport torchaudio\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.nn.init as I\nimport torch.optim as optim\nimport torchvision\n\nfrom torch.utils.data import Dataset, DataLoader\nimport multiprocessing\nfrom tqdm import tqdm\nimport pandas as pd\nimport librosa\n\nimport shutil\nfrom pathlib import Path\nimport gc","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Analysing CSV","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"example_test_audio_metadata = pd.read_csv('../input/birdsong-recognition/example_test_audio_metadata.csv')\nexample_test_audio_metadata.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"example_test_audio_summary = pd.read_csv('../input/birdsong-recognition/example_test_audio_summary.csv')\nexample_test_audio_summary.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('../input/birdsong-recognition/test.csv')\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/birdsong-recognition/train.csv')\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Constants","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"EPOCHS = 50\nDURATION = 5\nTEST = True\nNO_OF_TEST_CLASSES = 5\ndata_dir = pathlib.Path('/kaggle/input/birdsong-recognition/train_audio/')\n# Image size of spectrogram for 5 seconds \nIMG_SIZE=(552, 128)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f'Running on : {device}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train Data Processing","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Creating a new data frame with the required columns for analysis. For training filtering the audio which has a duration of more than 5 seconds, and then calculating offset to read 5 seconds of audio data in the middle.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df[[\"ebird_code\", \"channels\", \"duration\", \"filename\", \"species\",\"bird_seen\",\"latitude\", \"longitude\"]]\ntrain_df = train_df[train_df[\"duration\"] >= DURATION]\ntrain_df['offset'] = train_df.apply(lambda row: int((row.duration - DURATION) / 2) ,axis=1)\ntrain_df['path']= train_df.apply(lambda row: f'/kaggle/input/birdsong-recognition/train_audio/{row.ebird_code}/{row.filename}' ,axis=1)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CLASS_NAMES = [item.name for item in data_dir.glob('*')]\nif TEST:\n    CLASS_NAMES = CLASS_NAMES[:NO_OF_TEST_CLASSES]\n    print(f\"Test running on {NO_OF_TEST_CLASSES} classes\")\n    print(f\"Class Names\", CLASS_NAMES)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CLASS_MAP = {x: CLASS_NAMES.index(x) for x in CLASS_NAMES}\nprint(f\"Total Classes: {len(CLASS_NAMES)}\")\nprint(f\"Classes: \", CLASS_MAP)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Parsing class name.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Creating a batch of data","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Creating batches of data. If split_window = [0, 0.25, 0.5, 1], the data splitted into 0%-25%, 25%-50% and 50%-100%. Created it for test run. Here I am creating 100% of split, Because I am training only for only 10 classes in test purpose","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"training_data = train_df[train_df.ebird_code.isin(CLASS_NAMES)]\ntraining_data = training_data[['path', 'offset']]\ntraining_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"paths = training_data.values\npaths = [[pathlib.Path(item[0]), item[1]] for item in paths]\n\ngroups = [[[y[0], y[1]] for y in paths if y[0].parts[-2]==x] for x in CLASS_NAMES]\n\nplot_data = []\n\n# Percentage window split. if split_window = [0, 0.25, 0.5, 1], the data splitted into 0%-25%, 25%-50% and 50%-100%\nsplit_window = [0, 1]  \nbatches = [[] for _ in range(len(split_window)-1)]\n\nfor group in groups:\n    plot_data.append(group[1]) \n    tr_b = [group[int(len(group) * split_window[i]) : int(len(group) * split_window[i+1])] for i in range(len(split_window)-1)]\n    batches = [tr_b[i] + batches[i] for i in range(len(batches))]\n    \nprint(\"Trainning audio batches count: \", [len(l) for l in batches])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Analysis","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Pie chart, where the slices will be ordered and plotted counter-clockwise:\nlabels = 'Bird Seen', 'Bird Unseen'\nbird_seen_count = train_df[train_df.bird_seen == 'yes'].shape[0]\nsizes = [bird_seen_count, train_df.shape[0] - bird_seen_count]\nexplode = (0, 0.1) \n\nfig1, ax1 = plt.subplots()\nax1.pie(sizes, explode=explode, labels=labels, autopct='%1.1f%%',\n        shadow=True, startangle=90)\nax1.axis('equal')  # Equal aspect ratio ensures that pie is drawn as a circle.\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Pie chart, where the slices will be ordered and plotted counter-clockwise:\nlabels = 'Stereo Audio Files', 'Mono Audio Files'\nmono_audio_count = train_df[train_df.channels == '1 (mono)'].shape[0]\nsizes = [train_df.shape[0] - mono_audio_count, mono_audio_count]\nexplode = (0, 0.1) \n\nfig1, ax1 = plt.subplots()\nax1.pie(sizes, explode=explode, labels=labels, autopct='%1.1f%%',\n        shadow=True, startangle=90)\nax1.axis('equal')  # Equal aspect ratio ensures that pie is drawn as a circle.\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Test Mode Data\n\nData distribution in test mode","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"if TEST:\n    labels = CLASS_NAMES\n    distribution = train_df[train_df.ebird_code.isin(CLASS_NAMES)]\n    distribution = distribution.groupby(['ebird_code'])['ebird_code'].value_counts().to_frame()\n    distribution = distribution.rename(columns={ distribution.columns[0]: \"count\" })\n    sizes = list(distribution['count'].values)\n\n    fig1, ax1 = plt.subplots()\n    ax1.pie(sizes, labels=labels, autopct='%1.1f%%',\n            shadow=True, startangle=90)\n    ax1.axis('equal')  # Equal aspect ratio ensures that pie is drawn as a circle.\n\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Plotting audio samples","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"[**MelSpectrogram:**](https://medium.com/analytics-vidhya/understanding-the-mel-spectrogram-fca2afa2ce53#:~:text=A%20mel%20spectrogram%20is%20a,converted%20to%20the%20mel%20scale) A mel spectrogram is a spectrogram where the frequencies are converted to the mel scaleMelSpectrogram: ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(5):\n    data = plot_data[i]\n    path = data[0]\n    name = path.parts[-2]\n    offset = data[1]\n    S, sr = librosa.load(path, mono=True, duration=DURATION, offset=offset)\n    specgram = torchaudio.transforms.MelSpectrogram()(torch.tensor(S))\n    img = torchvision.transforms.ToPILImage()(specgram).convert(\"RGB\")\n    img = img.resize(IMG_SIZE, Image.ANTIALIAS)    \n    plt.figure()\n    plt.subplot(2, 1, 1)\n    plt.plot(S)\n    plt.title(name)\n    plt.subplot(2, 1, 2)\n    plt.imshow(img)\n    plt.title(name + \": \"+ 'Mel spectrogram')\n    plt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Converting Audio To Images","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Here the audio files are read from the offset with required durations(5 seconds) and converted to MelSpectrogram. Reading audio as mono channel audio. The MelSpectrogram images are resized into 552, 128 (Image size of spectrogram for 5 seconds) and saved as images. The audio to image conversion happens in a multiprocess pool to speed up the process.\n\nThe conversion processing happens as mini-batches, each mini-batch processing 100 audio files.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def delete_previous_image_dir():\n    if os.path.exists('/kaggle/working/train_images/'):\n        shutil.rmtree('/kaggle/working/train_images/')\n    \ndef create_dir():\n    if not os.path.exists('train_images'):\n        os.makedirs('train_images')\n    for name in CLASS_NAMES:\n        p = Path(f\"/kaggle/working/train_images/{name}/\")\n        if not os.path.exists(p):\n            p.mkdir(parents=True)\n    \n\ndef convert_audio_to_image(data):\n    path = data[0]\n    offset = data[1]\n    class_name = path.parts[-2]\n    name = path.parts[-1].split(\".\")[0]\n    try:\n        S, sr = librosa.load(path, mono=True, duration=DURATION, offset=offset)\n        specgram = torchaudio.transforms.MelSpectrogram()(torch.tensor(S))\n        img = torchvision.transforms.ToPILImage()(specgram).convert(\"RGB\")\n        img.save(f\"/kaggle/working/train_images/{class_name}/{name}.png\")\n        \n        del S\n        del specgram\n        del img\n    except Exception as e:\n        print(f\"Exception in reading: {path}\")\n    gc.collect()\n\ndef process_audio(paths):\n    for path in tqdm(paths):\n        convert_audio_to_image(path)\n        \ndef process_audio_in_pool(data):\n    pool = multiprocessing.Pool(multiprocessing.cpu_count())\n    pool.map(convert_audio_to_image, data)  \n    pool.close()\n    pool.join()\n\ndelete_previous_image_dir()\ncreate_dir()\n\nbatch = batches[0]\ntotal = int(len(batch) / 100) + (1 if len(batch) % 100 > 1 else 0)\ncurrent = 0\nwhile len(batch) > 0:\n    # Batch contain 100 images\n    print(f\"Processing mini batch(may take some time): {current + 1} / {total}\")\n    process_audio_in_pool(batch[:100])\n    batch = batch[100:]\n    current += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_dir = pathlib.Path('/kaggle/working/train_images/')\npaths = list(data_dir.glob('*/*.png'))\nimage_count = len(paths)\nprint(f\"Total Image: {image_count}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"groups = [[y for y in paths if y.parts[-2]==x] for x in CLASS_NAMES]\n\ntrain_path = []\ntest_path = []\n\nfor group in groups:\n    l = len(group)\n    l = int(l * 0.8)\n    train_path += group[:l]\n    test_path += group[l:]\n\n    \nprint(\"Trainning images: \", len(train_path))\nprint(\"Testing audio: \", len(test_path))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Input Pipeline","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class AudioDataset(Dataset):\n\n    paths = []\n    \n    def __init__(self, paths):\n        self.paths = paths\n        \n    def __len__(self):\n        return len(self.paths)\n\n    def __getitem__(self, idx):\n        path = self.paths[idx]\n        img = Image.open(path)\n        img = img.resize(IMG_SIZE, Image.ANTIALIAS)    \n        pil_to_tensor = torchvision.transforms.ToTensor()(img)\n        return pil_to_tensor, CLASS_MAP[path.parts[-2]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data = AudioDataset(paths=train_path)\ntest_data = AudioDataset(paths=test_path)\n\ntrain_loader = DataLoader(train_data, batch_size=16, shuffle=True)\ntest_loader = DataLoader(test_data, batch_size=16, shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Network","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# Transfer Learning","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Creating transfer learning network with resnet18 as base model.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"net = torchvision.models.resnet18()\nnet = net.cuda() if torch.cuda.is_available() else net\nnet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"criterion = nn.CrossEntropyLoss()\noptimizer = optim.SGD(net.parameters(), lr=0.0001, momentum=0.9)\n\n\nnum_ftrs = net.fc.in_features\nnet.fc = nn.Linear(num_ftrs, len(CLASS_NAMES))\nnet.fc = net.fc.cuda() if torch.cuda.is_available() else net.fc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"summary(net, (3, IMG_SIZE[0], IMG_SIZE[1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for epoch in range(EPOCHS):\n    print(f\"Running Epoch {epoch + 1}\")\n    # loop over the dataset multiple times\n    running_loss = 0.0\n    no_of_batches = 0\n    tk0 = tqdm(train_loader, total=int(len(train_loader)))\n    for i, data in enumerate(tk0):\n        # get the inputs; data is a list of [inputs, labels]\n        inputs, labels = data\n        inputs, labels = inputs.to(device), labels.to(device)\n\n        # zero the parameter gradients\n        optimizer.zero_grad()\n\n        # forward + backward + optimize\n        outputs = net(inputs)\n        loss = criterion(outputs, labels)\n        loss.backward()\n        optimizer.step()\n\n        # print statistics\n        running_loss += loss.item()\n        no_of_batches += 1\n    correct = 0\n    total = 0\n    with torch.no_grad():\n        tk1 = tqdm(test_loader, total=int(len(test_loader)))\n        for i, data in enumerate(tk1):\n            inputs, labels = data\n            inputs, labels = inputs.to(device), labels.to(device)\n            outputs = net(inputs)\n            _, predicted = torch.max(outputs.data, 1)\n            total += labels.size(0)\n            correct += (predicted == labels).sum().item()\n\n    print('[Epoch - %d] loss: %.3f   Accuracy: %d %%' % (epoch + 1, running_loss / no_of_batches, 100 * correct / total))\nprint('Finished Training')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Ref: https://pytorch.org/tutorials/beginner/audio_preprocessing_tutorial.html","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}