{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":70203,"databundleVersionId":8068726,"sourceType":"competition"},{"sourceId":171789695,"sourceType":"kernelVersion"}],"dockerImageVersionId":30683,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport random\nimport time\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport timm\nimport torch\nfrom torch import nn\nimport torch.nn.functional as F  # Import functional module for softmax\n\nimport PIL\nfrom PIL import Image\n\n#for handling super-wide images\nPIL.Image.MAX_IMAGE_PIXELS = 933120000\n\nfrom torchvision import transforms\n\nfrom torch.utils.data import Dataset\nfrom torch.utils.data import DataLoader\nfrom torch.utils.data import random_split\nfrom torch.optim.lr_scheduler import StepLR\n\nimport librosa\n\nfrom scipy.signal import butter, filtfilt","metadata":{"execution":{"iopub.status.busy":"2024-04-15T23:42:55.191015Z","iopub.execute_input":"2024-04-15T23:42:55.191733Z","iopub.status.idle":"2024-04-15T23:43:04.911342Z","shell.execute_reply.started":"2024-04-15T23:42:55.191691Z","shell.execute_reply":"2024-04-15T23:43:04.910313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training an ImageNet on contiguous spectrograms generated here:\n## https://www.kaggle.com/code/richolson/birdclef-2024-contiguous-mel-spectrogram-generator\n\n### Many 5-second segments of OGGS didn't include bird calls - so training based on randomly-selected 10-second segments\n### 10-second subsamples of spectrogram (1024x224) scaled to imagenet_input_size x imagenet_input_size for model input\n\n### See \"Load model\" cell for information on ImageNet configuration for this version...","metadata":{}},{"cell_type":"markdown","source":"# Misc. setup....","metadata":{}},{"cell_type":"code","source":"#plausible training could be done on CPU (much faster on GPU)\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\n\n#contiguous spectrograms - a single PNG file per OGG input\nmel_dir = \"/kaggle/input/birdclef-2024-contiguous-mel-spectrogram-generator/train_images/\"\n\naudio_dir = \"/kaggle/input/birdclef-2024/train_audio/\"\n\ntrain_meta = pd.read_csv(\"/kaggle/input/birdclef-2024/train_metadata.csv\")\n\nsample_submit = pd.read_csv(\"/kaggle/input/birdclef-2024/sample_submission.csv\")\n\n\n#bandpass filter for audio (Hz)\nlow_cut = 400\nhigh_cut = 10000\n\n#imagenet input is 224x224\nimagenet_input_size = 224\n\n#based on spectrogram generation\nspectrogram_width_per_5sec = 512\nspectrogram_height = 224\n\n#we are training the model on sample segment this long\nmodel_sample_time_sec = 10\n\nspectrogram_image_sample_width = (model_sample_time_sec / 5) * spectrogram_width_per_5sec\n\nbatch_size = 32","metadata":{"execution":{"iopub.status.busy":"2024-04-15T23:43:04.913243Z","iopub.execute_input":"2024-04-15T23:43:04.913763Z","iopub.status.idle":"2024-04-15T23:43:05.140610Z","shell.execute_reply.started":"2024-04-15T23:43:04.913735Z","shell.execute_reply":"2024-04-15T23:43:05.139488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Get target classes from sample_submit (assures correct order)","metadata":{}},{"cell_type":"code","source":"species = sample_submit.columns[1:].tolist()\nprint(f\"{len(species)} species:\\n {species}\")","metadata":{"execution":{"iopub.status.busy":"2024-04-15T23:43:05.147695Z","iopub.execute_input":"2024-04-15T23:43:05.148004Z","iopub.status.idle":"2024-04-15T23:43:05.153729Z","shell.execute_reply.started":"2024-04-15T23:43:05.147977Z","shell.execute_reply":"2024-04-15T23:43:05.152785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"available_models = timm.list_models()\n\n# Print models that contain 'efficientnetv2' in their name\nefficientnetv2_models = [model for model in available_models if 'efficientnetv2' in model]\nprint(efficientnetv2_models)","metadata":{"execution":{"iopub.status.busy":"2024-04-15T23:43:05.154923Z","iopub.execute_input":"2024-04-15T23:43:05.155282Z","iopub.status.idle":"2024-04-15T23:43:05.174335Z","shell.execute_reply.started":"2024-04-15T23:43:05.155251Z","shell.execute_reply":"2024-04-15T23:43:05.173492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load model / setup classification layer","metadata":{}},{"cell_type":"code","source":"from torchinfo import summary\n\n#model = timm.create_model('efficientnet_b1', pretrained=True)\nmodel = timm.create_model('tf_efficientnetv2_b2', pretrained=True)\nmodel.classifier = nn.Linear(model.classifier.in_features, out_features=len(species))\n\nmodel.to(device)\n\nsummary(model)","metadata":{"execution":{"iopub.status.busy":"2024-04-15T23:43:05.175465Z","iopub.execute_input":"2024-04-15T23:43:05.175826Z","iopub.status.idle":"2024-04-15T23:43:07.041861Z","shell.execute_reply.started":"2024-04-15T23:43:05.175792Z","shell.execute_reply":"2024-04-15T23:43:07.040754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Image transforms for resize / conversion to tensor / augment","metadata":{}},{"cell_type":"code","source":"resize_transform = transforms.Compose([\n    transforms.Resize((imagenet_input_size, imagenet_input_size)),  # Resize the image to 224x224\n    transforms.Grayscale(num_output_channels=3),  # Convert grayscale to RGB by replicating channels\n    transforms.ToTensor(),  # Convert the image to a PyTorch tensor\n])\n\n\nclass AddGaussianNoise(object):\n    def __init__(self, mean=0., std_range=(0.05, 0.2)):\n        self.mean = mean\n        self.std_range = std_range\n\n    def __call__(self, tensor):\n        std = random.uniform(*self.std_range)  # Randomly choose a std deviation each time\n        # Generate noise only for one channel\n        single_channel_noise = torch.randn(tensor[0, :, :].size()) * std + self.mean\n        # Expand noise to all channels\n        noise = single_channel_noise.repeat(tensor.size(0), 1, 1)\n        return tensor + noise\n\n    def __repr__(self):\n        return f'{self.__class__.__name__}(mean={self.mean}, std_range={self.std_range})'\n    \nresize_and_augment_transform = transforms.Compose([\n    transforms.Resize((imagenet_input_size, imagenet_input_size)),  # Resize the image to 224x224\n    transforms.Grayscale(num_output_channels=3),  # Convert grayscale to RGB by replicating channels\n    transforms.ColorJitter(brightness=(0.5, 1.5), contrast=(0.5, 1.5)),  # Randomly change the brightness and contrast\n    transforms.ToTensor(),  # Convert the image to a PyTorch tensor\n    AddGaussianNoise(0., (0.02, 0.1))  # Add Gaussian noise with random std deviation between...\n\n])","metadata":{"execution":{"iopub.status.busy":"2024-04-15T23:43:07.043445Z","iopub.execute_input":"2024-04-15T23:43:07.043910Z","iopub.status.idle":"2024-04-15T23:43:07.055628Z","shell.execute_reply.started":"2024-04-15T23:43:07.043872Z","shell.execute_reply":"2024-04-15T23:43:07.054382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creates random crops of N-second intervals / rescales to imagenet_input_size","metadata":{}},{"cell_type":"code","source":"class SpectrogramDataset(Dataset):\n    def __init__(self, files, transform=resize_transform):\n        self.files = files\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.files)\n\n    def __getitem__(self, idx):\n        image_path, label = self.files[idx]\n        image = Image.open(image_path)\n\n        if image.width > spectrogram_image_sample_width:\n            max_x = image.width - spectrogram_image_sample_width\n            x_start = random.randint(0, max_x)\n            image = image.crop((x_start, 0, x_start + spectrogram_image_sample_width, spectrogram_height))\n\n        # Apply transformations\n        if self.transform:\n            image = self.transform(image)\n\n        # Convert label to tensor\n        label_tensor = torch.tensor(label, dtype=torch.long)\n\n        return image, label_tensor","metadata":{"execution":{"iopub.status.busy":"2024-04-15T23:43:07.057004Z","iopub.execute_input":"2024-04-15T23:43:07.057324Z","iopub.status.idle":"2024-04-15T23:43:07.068944Z","shell.execute_reply.started":"2024-04-15T23:43:07.057296Z","shell.execute_reply":"2024-04-15T23:43:07.067877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Split files across train / validate + duplicate or reduce data for some classes\n* Limits # of OGG files processed for a class to multiple median sample-per-class count\n* Duplicates sample files of classes under certain threshold\n* Prevents drastic overrepresentation of some classes\n* Note - this is by file count - not total time (which would probably make more sense...)","metadata":{}},{"cell_type":"code","source":"def load_and_limit_data(directory, species, split_ratio=0.8):\n    all_files = []\n    class_counts = {}\n\n    # Gather all files and their class counts\n    for label in species:\n        path = os.path.join(directory, label)\n        label_files = [(os.path.join(path, file), species.index(label)) for file in os.listdir(path) if file.endswith('.png')]\n        class_counts[label] = len(label_files)\n        all_files.extend(label_files)\n\n    # Calculate the median size and determine the limit\n    median_size = sorted(class_counts.values())[len(species) // 2]\n    limit_size = 3 * median_size\n    duplication_threshold = 0.3 * median_size \n\n    # Process files per class, with duplication for small classes and limiting for large classes\n    limited_files = []\n    for label in species:\n        path = os.path.join(directory, label)\n        label_files = [(os.path.join(path, file), species.index(label)) for file in os.listdir(path) if file.endswith('.png')]\n        if len(label_files) < duplication_threshold:\n        #add label_files to itself until we meet threshold\n            while len(label_files) < duplication_threshold:\n                label_files += label_files\n        # Limit the number of files if above the predefined limit size\n        random.shuffle(label_files)  # Shuffle to randomize the files that are included\n        limited_files.extend(label_files[:min(len(label_files), limit_size)])\n\n    # Shuffle all files to randomize input\n    random.shuffle(limited_files)\n    train_size = int(len(limited_files) * split_ratio)\n    train_files = limited_files[:train_size]\n    val_files = limited_files[train_size:]\n\n    return train_files, val_files\n\ntrain_files, val_files = load_and_limit_data(mel_dir, species, split_ratio=0.8)","metadata":{"execution":{"iopub.status.busy":"2024-04-15T23:43:07.070270Z","iopub.execute_input":"2024-04-15T23:43:07.070587Z","iopub.status.idle":"2024-04-15T23:43:10.358645Z","shell.execute_reply.started":"2024-04-15T23:43:07.070562Z","shell.execute_reply":"2024-04-15T23:43:10.357280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Get rid of any training files with secondary labels\n* Function can also remove low-quality recordings (not doing)\n* Secondary birds we are trying to target seems like it would be confusing to the model\n* Yes - this messes with the train / validate split a bit","metadata":{}},{"cell_type":"code","source":"def filter_files_by_metadata(file_label_pairs):\n    filtered_files = []\n    dropped_for_quality = 0\n    dropped_for_secondary = 0\n    \n    for file_path, label in file_label_pairs:\n        file_name = os.path.basename(file_path)  # Extract filename from path\n        species_folder = os.path.basename(os.path.dirname(file_path))  # Extract folder/species name\n        file_name = file_name.replace(\".png\",\".ogg\")\n\n        # Look up the file in the dataframe\n        file_meta = train_meta[train_meta['filename'] == f\"{species_folder}/{file_name}\"]\n        \n        if not file_meta.empty:\n            rating = file_meta['rating'].iloc[0]\n            secondary_labels = file_meta['secondary_labels'].iloc[0]\n            \n            #if rating < 3 and rating != 0:\n            #    dropped_for_quality += 1\n            #    continue  # Skip adding this file\n            \n            if secondary_labels != \"[]\":\n                dropped_for_secondary += 1\n                continue  # Skip adding this file\n        \n        # If the file passes both checks, add it to the list\n        filtered_files.append((file_path, label))\n    \n    print(\"Dropped due to low quality:\", dropped_for_quality)\n    print(\"Dropped due to secondary labels:\", dropped_for_secondary)\n    print(\"Files remaining:\", len(filtered_files))\n\n    return filtered_files\n\ntrain_files = filter_files_by_metadata(train_files)","metadata":{"execution":{"iopub.status.busy":"2024-04-15T23:43:10.361864Z","iopub.execute_input":"2024-04-15T23:43:10.362195Z","iopub.status.idle":"2024-04-15T23:44:16.832513Z","shell.execute_reply.started":"2024-04-15T23:43:10.362167Z","shell.execute_reply":"2024-04-15T23:44:16.831424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Initialize our datasets / loaders","metadata":{}},{"cell_type":"code","source":"train_dataset = SpectrogramDataset(train_files, transform=resize_and_augment_transform)\nval_dataset = SpectrogramDataset(val_files, transform=resize_transform)\n\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\nval_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2024-04-15T23:44:16.833834Z","iopub.execute_input":"2024-04-15T23:44:16.834129Z","iopub.status.idle":"2024-04-15T23:44:16.840530Z","shell.execute_reply.started":"2024-04-15T23:44:16.834104Z","shell.execute_reply":"2024-04-15T23:44:16.839523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preview data loader output","metadata":{}},{"cell_type":"code","source":"images, labels = next(iter(train_loader))\nfor i in range(0,3):\n    image_tensor = images[i]\n    image_tensor = image_tensor.clamp(0, 1)  # Clamp values to [0, 1]\n    image_tensor = image_tensor.permute(1, 2, 0).permute(2, 0, 1)  # Permute to (C, H, W) order\n    image_pil = transforms.ToPILImage()(image_tensor)\n    display(image_pil)","metadata":{"execution":{"iopub.status.busy":"2024-04-15T23:44:16.841930Z","iopub.execute_input":"2024-04-15T23:44:16.842240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train the model","metadata":{}},{"cell_type":"code","source":"learning_rate = 0.001\nnum_epochs = 10\n\nquick_test = False\nquick_test_steps = 50\n\ncriterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(model.parameters(), lr=learning_rate) \nscheduler = StepLR(optimizer, step_size=2, gamma=0.15)  # Decays the learning rate\n\ntotal_batches = len(train_loader)\nprint(total_batches, \"batches per epoch\")\n\n#scoring mechanism that considers predictions relative ranking to other classes\ndef rank_based_accuracy(outputs, labels):\n    batch_size = outputs.size(0)\n    probabilities, sorted_indices = outputs.sort(dim=1, descending=True)\n    correct_indices = labels.view(-1, 1).expand_as(sorted_indices)\n    ranks = (sorted_indices == correct_indices).nonzero(as_tuple=True)[1]\n\n    # Calculate score using reciprocal of rank + 1\n    scores = 1.0 / (ranks.float() + 1.0)\n\n    return 100 * scores.sum().item() / batch_size\n\nfor epoch in range(num_epochs):\n    model.train()  # Training mode\n    training_loss = 0.0\n\n    for i, (images, labels) in enumerate(train_loader):\n        images, labels = images.to(device), labels.to(device)\n        optimizer.zero_grad()  # Zero the gradients\n        outputs = model(images)\n        loss = criterion(outputs, labels)\n        loss.backward()  # Backpropagation\n        optimizer.step()  # Update weights\n        training_loss += loss.item() * images.size(0)\n        \n        if (i + 1) % (total_batches // 10) == 0:  # Print 10 times per epoch\n            print(f\"Progress: {100 * (i + 1) / total_batches:.2f}%\", end=\"\\r\")\n        if quick_test and i == quick_test_steps: break\n                \n    model.eval()\n    total_rank_score = 0\n    total, correct = 0, 0\n\n    with torch.no_grad():\n        for images, labels in val_loader:\n            images, labels = images.to(device), labels.to(device)\n            outputs = model(images)\n            _, predicted = torch.max(outputs, 1)\n            total += labels.size(0)\n            correct += (predicted == labels).sum().item()\n            total_rank_score += rank_based_accuracy(outputs, labels)\n    \n\n    # Calculate average dynamic rank-based accuracy over all batches\n    accuracy = 100 * correct / total\n    rank_accuracy_score = total_rank_score / len(val_loader)\n    print(f'Epoch [{epoch+1}/{num_epochs}], Loss: {training_loss / total:.4f}')\n    print(f'Rank-Based Accuracy: {rank_accuracy_score:.2f}%,  Validation Accuracy: {accuracy:.2f}%')\n    \n    scheduler.step()  # Step the scheduler at the end of each epoch\n    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluate inference behavior / sanity check\n* This code is mostly duplicated into the \"Run\" notebook\n* Make sure spectrograms are being generated\n* Verify predictions look like probabilities / aren't the same for all classes\n* Test file is of an \"ASBFLY\" (first class) - if that column has a higher values that others - it's a good sign!","metadata":{}},{"cell_type":"code","source":"#test data gets same bandpass as when spectrograms were generated\ndef bandpass_filter(data, lowcut, highcut, sr, order=5):\n    nyquist = 0.5 * sr\n    low = lowcut / nyquist\n    high = highcut / nyquist\n    b, a = butter(order, [low, high], btype='band')\n    y = filtfilt(b, a, data)\n    return y\n\n#using same hop length as dataset creation\n#(results int 1024x224 for 10-seconds / gets rescaled for model input)\nhop_length = int((160400 / spectrogram_width_per_5sec))\n\ndef evaluate_audio_file_segments(audio_path):\n    \n    start_time = time.time()\n\n    # Load the audio file\n    audio, sr = librosa.load(audio_path, sr=None)\n    audio = bandpass_filter(audio, low_cut, high_cut, sr)\n\n    # Calculate the number of samples per segment\n    samples_per_segment = sr * model_sample_time_sec\n\n    # Split the audio into segments\n    total_samples = len(audio)\n    segments = [audio[i:i + samples_per_segment] for i in range(0, total_samples, samples_per_segment) if i + samples_per_segment <= total_samples]\n\n    predictions = []  # Store predictions for each segment\n    total_time = 0\n    \n    images = []\n\n    for segment in segments:\n\n        # Process each segment into a spectrogram\n        spectrogram = librosa.feature.melspectrogram(y=segment, sr=sr, hop_length=hop_length, n_mels=224, fmin=low_cut, fmax=high_cut)\n        spectrogram_db = librosa.amplitude_to_db(spectrogram, ref=np.max)\n\n        # Normalize spectrogram for image display\n        spectrogram_norm = (spectrogram_db - spectrogram_db.min()) / (spectrogram_db.max() - spectrogram_db.min()) * 255\n        spectrogram_image = Image.fromarray(spectrogram_norm.astype(np.uint8))                \n                \n        # Convert the PIL Image to a tensor\n        spectrogram_image_tensor = resize_transform(spectrogram_image)\n        spectrogram_image_tensor = spectrogram_image_tensor.to(device)\n\n        model.eval()\n        \n        with torch.no_grad():\n            final_tensor = spectrogram_image_tensor.repeat(1, 1, 1, 1)\n            logits = model(final_tensor)  # Assuming model expects 3-channel input\n            \n        # Convert logits to probabilities\n        probabilities = F.softmax(logits, dim=1)  # Apply softmax to convert logits to probabilities\n        predictions.append(probabilities.cpu().numpy())\n        \n        # Convert the tensor to a PIL Image\n        spectrogram_image_tensor = spectrogram_image_tensor.clamp(0, 1)  # Clamp values to [0, 1]\n        spectrogram_image_tensor = spectrogram_image_tensor.permute(1, 2, 0).permute(2, 0, 1)  # Permute to (C, H, W) order\n        image_pil = transforms.ToPILImage()(spectrogram_image_tensor)        \n        \n        images.append(image_pil)\n        \n    #needs to be < 6.5s\n    print (\"Time:\", time.time()- start_time)\n        \n    return predictions, images\n\npredictions, images = evaluate_audio_file_segments(\"/kaggle/input/birdclef-2024/train_audio/asbfly/XC629847.ogg\")\n\n#make sure predictions and images look OK\nfor i in range(0,2):\n    display(images[i])\n    print(\"Asbfly odds:\", predictions[i][0][0])\n    print(predictions[i])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Save model","metadata":{}},{"cell_type":"code","source":"torch.save(model, \"model.pth\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}