{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":90860,"databundleVersionId":10740331,"sourceType":"competition"}],"dockerImageVersionId":30822,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Pixel Play 25' - Armaan Mahajan 24113022 (Submission)","metadata":{}},{"cell_type":"markdown","source":"**(1.0) Pre-requisites**","metadata":{}},{"cell_type":"code","source":"# ==============================\n# CELL 1: INSTALL CLIP LIBRARY\n# ==============================\n# Install the CLIP library from its GitHub repository\n!pip install git+https://github.com/openai/CLIP.git","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T06:14:52.485353Z","iopub.execute_input":"2025-01-12T06:14:52.485682Z","iopub.status.idle":"2025-01-12T06:15:01.343767Z","shell.execute_reply.started":"2025-01-12T06:14:52.485657Z","shell.execute_reply":"2025-01-12T06:15:01.342858Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==============================\n# CELL 2: DEFINE PATH VARIABLES\n# ==============================\n# Set up the directories for training data, testing data, saving models, and saving predictions\ntraining_directory = '/kaggle/input/vlg-recruitment-24-challenge/vlg-dataset/vlg-dataset/train'  # Path to training image directories\ntesting_directory = '/kaggle/input/vlg-recruitment-24-challenge/vlg-dataset/vlg-dataset/test'    # Path to testing images\n\nhots_directory = '/kaggle/input/vlg-recruitment-24-challenge/Special-Package/Special-Package/final_examples' # Not for final model but for extra report\n\nmodel_save_directory = '/kaggle/working/'                                            # Path to save the trained model\nprediction_csv_path = '/kaggle/working/save_predictions(enhanced).csv'               # Name for predictions file\n\n\nhots_prediction_csv_path = '/kaggle/working/save_hots_predictions.csv' \nhots_file_path = '/kaggle/input/vlg-recruitment-24-challenge/Special-Package/Special-Package/labels.txt'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T06:15:01.344948Z","iopub.execute_input":"2025-01-12T06:15:01.345223Z","iopub.status.idle":"2025-01-12T06:15:01.350031Z","shell.execute_reply.started":"2025-01-12T06:15:01.345181Z","shell.execute_reply":"2025-01-12T06:15:01.348986Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =============================================\n# CELL 3: IMPORTING LIBRARIES AND DEVICE SETUP\n# =============================================\n# Import necessary Python libraries for file handling, data processing, and machine learning\nimport os  # For operating system dependent functionality\nimport glob  # For file pattern matching\nimport csv  # For reading and writing CSV files\n\nimport torch  # Main PyTorch library for tensor operations\nimport torchvision  # PyTorch library for computer vision\nimport torchvision.transforms as transforms  # For image transformations\nfrom torch.utils.data import Dataset, DataLoader  # For handling datasets and loading data\n\nfrom PIL import Image  # For image processing\nimport clip  # CLIP model Architecture for project\nimport numpy as np  # For numerical operations - mostly arrays\n\nimport torch.nn as nn  # For building neural network layers\nimport torch.optim as optim  # For optimization algorithms\nfrom torch.optim.lr_scheduler import CyclicLR  # For learning rate scheduling\n\n# Check if a GPU is available and set the device accordingly\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\nprint(\"Using device: \" + device)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T06:15:01.352253Z","iopub.execute_input":"2025-01-12T06:15:01.352529Z","iopub.status.idle":"2025-01-12T06:15:06.518235Z","shell.execute_reply.started":"2025-01-12T06:15:01.352494Z","shell.execute_reply":"2025-01-12T06:15:06.517384Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =======================================================\n# CELL 4: DEFINING ANIMAL CLASSES AND TEXT DESCRIPTIONS\n# =======================================================\n\n# List of 40 known animal classes that the model will be trained to recognize\nknown_animal_classes = [\n    'antelope', 'bat', 'beaver', 'blue+whale', 'bobcat', 'buffalo', 'chihuahua', 'cow',\n    'dalmatian', 'deer', 'dolphin', 'elephant', 'german+shepherd', 'giant+panda',\n    'giraffe', 'grizzly+bear', 'hamster', 'hippopotamus', 'humpback+whale', 'killer+whale',\n    'leopard', 'lion', 'mole', 'mouse', 'otter', 'ox', 'persian+cat', 'pig',\n    'polar+bear', 'raccoon', 'rat', 'seal', 'siamese+cat', 'skunk', 'spider+monkey',\n    'tiger', 'walrus', 'weasel', 'wolf', 'zebra'\n]\n\n# List of 10 additional animal classes that the model cannot see during training\nunknown_animal_classes = [\n    'horse', 'moose', 'gorilla', 'fox', 'sheep', \n    'chimpanzee', 'squirrel', 'rhinoceros', 'rabbit', 'collie'\n]\n\n# Textual prompts for each known class; helps text embeddings distinguish potential similarity between known and unknown classes\nknown_class_descriptions = {\n    'antelope': 'A graceful, slender animal that can run incredibly fast and is found in Africa and Eurasia',\n    'bat': 'A small, nocturnal mammal capable of sustained flight using echolocation',\n    'beaver': 'A large, amphibious rodent known for building dams and lodges with branches and mud',\n    'blue+whale': 'The largest animal on Earth, a massive marine mammal with a long, slender body',\n    'bobcat': 'A medium-sized North American cat with spotted fur and short tail',\n    'buffalo': 'A robust bovine species with large horns, often found in herds on grasslands',\n    'chihuahua': 'A tiny dog breed with a lively personality and an apple-domed skull',\n    'cow': 'A domesticated bovine widely raised for milk, meat and farm work',\n    'dalmatian': 'A distinctively spotted, medium-sized dog breed known as a carriage or fire dog',\n    'deer': 'A hoofed ruminant mammal of the family Cervidae, often with antlers in males',\n    'dolphin': 'An intelligent marine mammal known for its playful behavior and high intelligence',\n    'elephant': 'A massive herbivorous mammal with a trunk and ivory tusks, found in Africa and Asia',\n    'german+shepherd': 'A large-sized breed of dog known for its intelligence and herding or guard abilities',\n    'giant+panda': 'A black-and-white bear native to China that feeds mostly on bamboo',\n    'giraffe': 'The tallest living terrestrial animal, easily recognized by its extremely long neck',\n    'grizzly+bear': 'A large subspecies of brown bear with a muscular hump and strong forelimbs',\n    'hamster': 'A small rodent often kept as a pet, known for storing food in its cheek pouches',\n    'hippopotamus': 'A large, mostly herbivorous mammal with a barrel-shaped torso and enormous jaws',\n    'humpback+whale': 'A large whale famous for its acrobatic behavior and complex whale songs',\n    'killer+whale': 'Also known as orca, a highly social marine predator belonging to the dolphin family',\n    'leopard': 'A big cat known for its spotted coat and climbing ability, widely distributed in Africa and Asia',\n    'lion': 'A social big cat with males possessing a prominent mane, known as the king of the jungle',\n    'mole': 'A small burrowing mammal with velvety fur and tiny eyes, adapted for underground life',\n    'mouse': 'A small rodent with a pointed nose, furry round body, and a long tail',\n    'otter': 'A playful, aquatic member of the weasel family with a streamlined body and webbed feet',\n    'ox': 'A large, domesticated bovine used primarily for draft work in many parts of the world',\n    'persian+cat': 'A long-haired cat breed characterized by its round face and shortened muzzle',\n    'pig': 'A highly intelligent, domesticated omnivorous mammal with a snout used for foraging',\n    'polar+bear': 'A large, white-furred bear living in the Arctic, highly adapted to cold climates',\n    'raccoon': 'A medium-sized mammal with distinct facial mask markings and dexterous front paws',\n    'rat': 'A rodent known for its adaptability, intelligence, and long, scaly tail',\n    'seal': 'A marine mammal with a streamlined body, flippers, and typically whiskered face',\n    'siamese+cat': 'A sleek, short-haired cat breed with distinctive point coloration and blue almond eyes',\n    'skunk': 'A small to medium-sized mammal known for its ability to spray a foul-smelling liquid',\n    'spider+monkey': 'A New World monkey with a prehensile tail and long limbs, known for agile movement in trees',\n    'tiger': 'A powerful, striped big cat native to Asia and regarded as the largest cat species',\n    'walrus': 'A large, flippered marine mammal with tusks, whiskers, and a bulky body',\n    'weasel': 'A small, active predator with a slender body, known for its quick movements',\n    'wolf': 'A wild canine with a pack-based social structure and a highly expressive face',\n    'zebra': 'An African equid with distinctive black-and-white striped coats'\n}\n\n# Detailed descriptions for each unknown class to help the model recognize them without prior training - An attempt for Zero-Shot Training\nunknown_class_descriptions = {\n    'horse': 'A strong, four-legged mammal with a mane and tail, known for its speed and use by humans',\n    'moose': 'A tall, long-legged animal with broad antlers, found in northern forests and wetlands',\n    'gorilla': 'A powerful, predominantly herbivorous ape with a large body and gentle demeanor',\n    'fox': 'A small to medium-sized omnivorous mammal with a bushy tail and elongated snout',\n    'sheep': 'A wool-producing ruminant farm animal, often kept in flocks, used for meat and fiber',\n    'chimpanzee': 'An intelligent ape closely related to humans, known for tool use and complex social groups',\n    'squirrel': 'A small rodent with a bushy tail, known for climbing and storing nuts in trees',\n    'rhinoceros': 'A huge herbivore with thick protective skin and one or two horns on its snout',\n    'rabbit': 'A small mammal with long ears, known for rapid reproduction and hopping locomotion',\n    'collie': 'An active, intelligent herding dog breed with a thick double coat and pointed snout'\n}\n\n# Combine class names with their descriptions to create text prompts for the CLIP model\ntext_descriptions = {}\nfor animal in known_animal_classes:\n    # Replaces '+' with space for better readability in descriptions\n    animal_name = animal.replace('+', ' ')\n    # Create a detailed description for each known class\n    text_descriptions[animal] = (\n        f\"A highly detailed professional photograph capturing the {animal_name}. \"\n        f\"{known_class_descriptions[animal]}\"\n    )\n\nfor animal in unknown_animal_classes:\n    # Replace '+' again\n    animal_name = animal.replace('+', ' ')\n    # Create a detailed description for each unknown class\n    text_descriptions[animal] = (\n        f\"A highly detailed professional photograph capturing the {animal_name}. \"\n        f\"{unknown_class_descriptions[animal]}\"\n    )\n\n# Create a final list containing all 50 animal classes\nall_animal_classes = known_animal_classes + unknown_animal_classes\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T06:15:06.519410Z","iopub.execute_input":"2025-01-12T06:15:06.519852Z","iopub.status.idle":"2025-01-12T06:15:06.528080Z","shell.execute_reply.started":"2025-01-12T06:15:06.519826Z","shell.execute_reply":"2025-01-12T06:15:06.527138Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**(2.0) Data Preparation**","metadata":{}},{"cell_type":"markdown","source":"(2.1) Loading Data","metadata":{}},{"cell_type":"code","source":"# ================================================\n# CELL 5: LOAD TRAINING DATASET FOR KNOWN CLASSES\n# ================================================\n# Define a custom dataset class for loading known animal images\nclass KnownAnimalsDataset(Dataset):\n    def __init__(self, folder_path, transform=None):\n        # Initialize with the path to the training data and any image transformations\n        self.transform = transform\n        self.image_samples = []  # List to store image file paths and their labels\n        self.class_list = known_animal_classes  \n\n        # Loop through each class and collect file paths of each image\n        for class_name in self.class_list:\n            \n            class_folder = os.path.join(folder_path, class_name)\n            \n            image_files = glob.glob(class_folder + '/*.*') # Using glob for easier file retrieval\n            \n            for image_file in image_files:\n                if os.path.isfile(image_file):  # Checks for correct file type to prevent error\n                   \n                    self.image_samples.append((image_file, class_name)) # Creates the required image-label pair\n\n    def __len__(self):\n        # Return the total number of samples\n        return len(self.image_samples)\n\n    def __getitem__(self, index):\n        # Get the image path and label for the given index\n        image_path, label = self.image_samples[index]\n        # Open the image and convert it to RGB format\n        image = Image.open(image_path).convert(\"RGB\")\n        if self.transform:\n            # Apply required transformations to the image\n            image = self.transform(image)\n        # Return the transformed image and the index of its class\n        return image, self.class_list.index(label)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T06:15:06.529027Z","iopub.execute_input":"2025-01-12T06:15:06.529372Z","iopub.status.idle":"2025-01-12T06:15:06.548423Z","shell.execute_reply.started":"2025-01-12T06:15:06.529346Z","shell.execute_reply":"2025-01-12T06:15:06.547529Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"(2.2) Data Augmentation","metadata":{}},{"cell_type":"code","source":"# ====================================\n# CELL 6: DEFINE DATA AUGMENTATIONS\n# ====================================\n# I apply various random transformations to training images to make the dataset diverse\n\nimage_transformations = transforms.Compose([\n    # Randomly crop the image to 336x336 pixels with slight scale variations\n    transforms.RandomResizedCrop((336,336), scale=(0.8, 1.0)),\n    # Randomly flip the image horizontally with 50% chance\n    transforms.RandomHorizontalFlip(p = 0.5),\n    # Randomly change the brightness, contrast, saturation, and hue of the image\n    transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.2),\n    # Randomly rotate the image by up to 15 degrees\n    transforms.RandomRotation(degrees=15),\n    # Randomly grayscale picture with 20% probability\n    transforms.RandomGrayscale(p=0.2),\n    # Convert the PIL image to a PyTorch tensor for uniform processing\n    transforms.ToTensor(),\n\n    \n    # Randomly erase a part of the image with a 10% probability\n    transforms.RandomApply([transforms.RandomErasing(p=1.0)], p=0.1),\n\n    \n    # Normalize the image tensor with mean and standard deviation values\n    transforms.Normalize([0.485, 0.456, 0.406],  # Using ImageNet statistics\n                         [0.229, 0.224, 0.225])\n])\n\n# Creates the dataset with the training directory and applied transformations for an augmented data\nfull_training_dataset = KnownAnimalsDataset(training_directory, transform=image_transformations)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T06:15:06.549268Z","iopub.execute_input":"2025-01-12T06:15:06.549518Z","iopub.status.idle":"2025-01-12T06:15:13.088457Z","shell.execute_reply.started":"2025-01-12T06:15:06.549497Z","shell.execute_reply":"2025-01-12T06:15:13.087513Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"(2.3) Training & Validation Partititons","metadata":{}},{"cell_type":"code","source":"# ====================================================\n# CELL 7: SPLITTING DATASET INTO TRAIN AND VALIDATION\n# ====================================================\n\n# Define the size of the validation set (15% of the full training dataset)\nvalidation_split = 0.15\nvalidation_size = int(len(full_training_dataset) * validation_split)\ntraining_size = len(full_training_dataset) - validation_size\n\n# Splitting the dataset\ntraining_dataset, validation_dataset = torch.utils.data.random_split(\n    full_training_dataset,\n    [training_size, validation_size],\n    generator=torch.Generator().manual_seed(13)  # Manual Seed for reproducibility every time model is trained\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T06:15:13.089266Z","iopub.execute_input":"2025-01-12T06:15:13.089513Z","iopub.status.idle":"2025-01-12T06:15:13.120506Z","shell.execute_reply.started":"2025-01-12T06:15:13.089492Z","shell.execute_reply":"2025-01-12T06:15:13.119828Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"(2.4) Dataloaders","metadata":{}},{"cell_type":"code","source":"# ==============================\n# CELL 8: CREATING DATALOADERS \n# ==============================\n\n# DataLoader for training data\ntraining_loader = DataLoader(\n    training_dataset,\n    batch_size=32,  \n    shuffle=True,    # Shuffles the data at every epoch\n    num_workers=4    # Kaggle processing limit is 4\n)\n\n# DataLoader for validation data\nvalidation_loader = DataLoader(\n    validation_dataset,\n    batch_size=32,  \n    shuffle=False,  # No need to shuffle validation data\n    num_workers=4    \n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T06:15:13.121241Z","iopub.execute_input":"2025-01-12T06:15:13.121431Z","iopub.status.idle":"2025-01-12T06:15:13.125281Z","shell.execute_reply.started":"2025-01-12T06:15:13.121414Z","shell.execute_reply":"2025-01-12T06:15:13.124556Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**(3.0) Model Setup**","metadata":{}},{"cell_type":"markdown","source":"(3.1) Defining Model Parameters","metadata":{}},{"cell_type":"code","source":"# ====================================\n# CELL 9: LOAD AND PREPARE THE PRETRAINED CLIP MODEL\n# ====================================\n# Load the CLIP model and its preprocessing steps\nclip_model, clip_preprocess = clip.load(\"ViT-L/14@336px\", device=device)  \n\n# Freeze all layers in the model except for specific ones to allow partial fine-tuning\nfor layer_name, layer_parameters in clip_model.named_parameters():\n    # Only allow gradients for certain layers to fine-tune \n    if (\n        \"visual.transformer.resblocks\" in layer_name\n        or \"visual.ln_post\" in layer_name\n        or \"visual.proj\" in layer_name\n    ):\n        layer_parameters.requires_grad = True  # These layers will be updated during training\n    else:\n        layer_parameters.requires_grad = False  # These layers will remain frozen\n\n# Determine the size of the image embedding by passing a dummy image through the model\nwith torch.no_grad():\n    dummy_image = torch.zeros(1, 3, 336, 336).to(device)  # Create a dummy image tensor\n    dummy_features = clip_model.encode_image(dummy_image)  # Get image embeddings\nimage_embedding_size = dummy_features.shape[1]  \n\n# Add a new classifier with additional layers to map image embeddings to the number of known classes\n\n\nclassifier_layer = nn.Sequential(\n    nn.Linear(image_embedding_size, 512),         # First linear layer from embedding to 512 neurons\n    nn.ReLU(),                                    # ReLU activation function for non-linearity\n    nn.BatchNorm1d(512),                          # Batch normalization for stable training\n    nn.Dropout(0.3),                              # Dropout layer to prevent overfitting\n    nn.Linear(512, 256),                          # Second linear layer from 512 to 256 neurons\n    nn.ReLU(),                                    # ReLU activation function\n    nn.Dropout(0.3),                              # Additional dropout\n    nn.Linear(256, len(known_animal_classes))    # Third linear layer to number of classes\n).to(device)\n\n\n# Define the optimizer to update the model's parameters during training\n\noptimizer_parameters = [\n    {\"params\": [param for name, param in clip_model.named_parameters() if param.requires_grad], \"lr\": 1e-5},\n    {\"params\": classifier_layer.parameters(), \"lr\": 3e-4}\n]\noptimizer = optim.AdamW(optimizer_parameters, weight_decay=1e-4)\n\n\nlearning_rate_scheduler = CyclicLR(\n    optimizer,\n    base_lr=3e-5,  # Lower bound of the learning rate\n    max_lr=3e-4,    # Upper bound of the learning rate\n    step_size_up = 100,  # Number of iterations to increase the learning rate\n    mode=\"triangular2\",  # Shape of the learning rate cycle\n    cycle_momentum=False  # Whether to cycle momentum\n)\n\n# Define the loss function to measure how well the model is performing\nloss_function = nn.CrossEntropyLoss()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T06:15:13.127537Z","iopub.execute_input":"2025-01-12T06:15:13.127723Z","iopub.status.idle":"2025-01-12T06:15:42.076136Z","shell.execute_reply.started":"2025-01-12T06:15:13.127707Z","shell.execute_reply":"2025-01-12T06:15:42.075444Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"(3.2) Training the Model","metadata":{}},{"cell_type":"code","source":"# ====================================\n# CELL 10: TRAIN THE MODEL WITH VALIDATION AND EARLY STOPPING\n# ====================================\n# Import tqdm for progress bars to better visualise the processing part\nfrom tqdm import tqdm\n\n# Setting the epochs\nnumber_of_epochs = 10  \n\n# Parameters for Early Stopping\nearly_stopping_patience = 5  # Number of epochs to wait for improvement before stopping\nbest_validation_loss = float('inf')  # Initialize the best validation loss as infinity\nepochs_without_improvement = 0  # Counter for epochs without improvement for early stopping\n\n# Start the training loop\nfor current_epoch in range(number_of_epochs):\n    # Setting the model and classifier to training mode\n    clip_model.train()\n    classifier_layer.train()\n    \n    total_training_loss = 0.0  # To calculate loss over the training epoch\n    total_training_correct = 0  # To count correct predictions during training\n    total_training_samples = 0  # To count total training samples processed\n\n    # Use tqdm to display a progress bar for the training batches\n    training_batches = tqdm(training_loader, desc=f\"Epoch {current_epoch + 1}/{number_of_epochs} - Training\", unit=\"batch\")\n    \n    # Loop through each batch of data in the training loader\n    for batch_images, batch_labels in training_batches:\n        # Move images and labels to the selected device (GPU or CPU)\n        batch_images = batch_images.to(device)\n        batch_labels = batch_labels.to(device)\n\n        # Encode the images with text to get their features using the CLIP model\n        with torch.no_grad():\n            image_features = clip_model.encode_image(batch_images)\n            image_features = image_features.float()  \n\n        # Pass the image features through the classifier to get predictions\n        predictions = classifier_layer(image_features)\n        \n        # Calculate the loss between predictions and actual labels\n        loss = loss_function(predictions, batch_labels)\n\n        # Clear previous gradients\n        optimizer.zero_grad()\n        # Backpropagate the loss to compute gradients\n        loss.backward()\n        # Update the model parameters based on gradients\n        optimizer.step()\n        # Update the learning rate using the scheduler\n        learning_rate_scheduler.step()\n\n        # Accumulate the loss\n        total_training_loss += loss.item() * batch_labels.size(0)\n        # Get the predicted classes by selecting the highest probability\n        _, predicted_classes = torch.max(predictions.detach(), 1)\n        \n        # Count how many predictions were correct\n        total_training_correct += (predicted_classes == batch_labels).sum().item()\n        # Update the total number of samples processed\n        total_training_samples += batch_labels.size(0)\n        \n        # Calculate current loss and accuracy for the batch\n        current_training_loss = total_training_loss / total_training_samples\n        current_training_accuracy = 100.0 * total_training_correct / total_training_samples\n        # Update the tqdm progress bar with current loss and accuracy\n        training_batches.set_postfix(loss=current_training_loss, accuracy=f\"{current_training_accuracy:.2f}%\")\n\n    # Calculate the average loss and accuracy for the training epoch\n    average_training_loss = total_training_loss / total_training_samples\n    training_accuracy_percentage = 100.0 * total_training_correct / total_training_samples\n\n###########################################################################################################################\n    \n    # Set the model and classifier to evaluation mode\n    clip_model.eval()\n    classifier_layer.eval()\n    \n    total_validation_loss = 0.0  # To calculate loss over the validation epoch\n    total_validation_correct = 0  # To count correct predictions during validation\n    total_validation_samples = 0  # To count total validation samples processed\n\n    \n    validation_batches = tqdm(validation_loader, desc=f\"Epoch {current_epoch + 1}/{number_of_epochs} - Validation\", unit=\"batch\")\n    \n    # Disable gradient calculations for validation to speed up computation\n    with torch.no_grad():\n        # Loop through each batch of data in the validation loader\n        for val_images, val_labels in validation_batches:\n            # Move images and labels to the selected device (GPU or CPU)\n            val_images = val_images.to(device)\n            val_labels = val_labels.to(device)\n\n            # Encode the images to get their features using the CLIP model\n            image_features = clip_model.encode_image(val_images)\n            image_features = image_features.float()  \n\n            # Pass the image features through the classifier to get predictions\n            val_predictions = classifier_layer(image_features)\n            \n            # Calculate the loss between predictions and actual labels\n            val_loss = loss_function(val_predictions, val_labels)\n\n            # Accumulate the loss\n            total_validation_loss += val_loss.item() * val_labels.size(0)\n\n            \n            # Get the predicted classes by selecting the highest probability\n            _, val_predicted_classes = torch.max(val_predictions, 1)\n            # Count how many predictions were correct\n            total_validation_correct += (val_predicted_classes == val_labels).sum().item()\n            # Update the total number of samples processed\n            total_validation_samples += val_labels.size(0)\n            \n            # Calculate current loss and accuracy for the batch\n            current_validation_loss = total_validation_loss / total_validation_samples\n            current_validation_accuracy = 100.0 * total_validation_correct / total_validation_samples\n            # Update the tqdm progress bar with current loss and accuracy\n            validation_batches.set_postfix(loss=current_validation_loss, accuracy=f\"{current_validation_accuracy:.2f}%\")\n    \n    # Calculate the average loss and accuracy for the validation epoch\n    average_validation_loss = total_validation_loss / total_validation_samples\n    validation_accuracy_percentage = 100.0 * total_validation_correct / total_validation_samples\n\n   \n    # Check if the validation loss has improved\n    if average_validation_loss < best_validation_loss:\n        best_validation_loss = average_validation_loss  # Update the best validation loss\n        epochs_without_improvement = 0  # Reset the counter\n        # Save the best model weights\n        torch.save({\n            'clip_model_weights': clip_model.state_dict(),\n            'classifier_weights': classifier_layer.state_dict()\n        }, os.path.join(model_save_directory, 'best_clip_finetuned.pth'))\n        print(f\"Validation loss improved to {best_validation_loss:.4f}. Model saved.\")\n    else:\n        epochs_without_improvement += 1  # Increase the counter\n        print(f\"No improvement in validation loss for {epochs_without_improvement} epoch(s).\")\n        # Check if model reached the patience limit\n        if epochs_without_improvement >= early_stopping_patience:\n            print(\"Early stopping triggered. Training halted.\")\n            break  # Exit the training loop\n\n    # Printing the results for the current epoch\n    print(f\"Epoch [{current_epoch + 1}/{number_of_epochs}], \"\n          f\"Training Loss: {average_training_loss:.4f}, Training Accuracy: {training_accuracy_percentage:.2f}%, \"\n          f\"Validation Loss: {average_validation_loss:.4f}, Validation Accuracy: {validation_accuracy_percentage:.2f}%\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T06:15:42.077162Z","iopub.execute_input":"2025-01-12T06:15:42.077379Z","iopub.status.idle":"2025-01-12T07:10:25.882489Z","shell.execute_reply.started":"2025-01-12T06:15:42.077361Z","shell.execute_reply":"2025-01-12T07:10:25.881449Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"(3.3) Saving the best weights","metadata":{}},{"cell_type":"code","source":"# ====================================\n# CELL 11: SAVE THE TRAINED MODEL\n# ====================================\n# Check if the save directory exists; if not, create it\nif not os.path.exists(model_save_directory):\n    os.makedirs(model_save_directory, exist_ok=True)\n\n# Save the state dictionaries of both the CLIP model and the classifier layer\ntorch.save({\n    'clip_model_weights': clip_model.state_dict(),\n    'classifier_weights': classifier_layer.state_dict()\n}, os.path.join(model_save_directory, 'clip_finetuned_final.pth'))\n\n\nprint(\"Training Completed, Model saved succesfully\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T07:10:25.883761Z","iopub.execute_input":"2025-01-12T07:10:25.884445Z","iopub.status.idle":"2025-01-12T07:10:27.165944Z","shell.execute_reply.started":"2025-01-12T07:10:25.884412Z","shell.execute_reply":"2025-01-12T07:10:27.165138Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**(4.0) Final Predictions**","metadata":{}},{"cell_type":"code","source":"# ====================================\n# CELL 12: TEST THE MODEL AND MAKE PREDICTIONS\n# ====================================\n# Precompute the text embeddings for all 50 classes using CLIP's text encoder\n\n# Tokenize all text prompts and move them to the selected device\ntext_tokens = torch.cat([clip.tokenize(text_descriptions[class_name]) for class_name in all_animal_classes]).to(device)\n\n# Encode the text prompts to get their embeddings\nwith torch.no_grad():\n    text_features = clip_model.encode_text(text_tokens)\n    # Normalize the text embeddings to have unit length\n    text_features /= text_features.norm(dim=-1, keepdim=True)\n\n# Define a custom dataset class again but for loading test images this time\nclass TestImagesDataset(Dataset):\n    def __init__(self, folder_path, transform=None):\n        # Initialize with the path to the test data and any image transformations\n        self.transform = transform\n        # Get all image file paths in the test directory\n        self.image_files = sorted(glob.glob(os.path.join(folder_path, '*.*')))\n        # Extract image filenames from the file paths\n        self.image_names = [os.path.basename(file_path) for file_path in self.image_files]\n\n    def __len__(self):\n        # Return the total number of test images\n        return len(self.image_files)\n\n    def __getitem__(self, index):\n        # Get the image path and name for the given index\n        image_path = self.image_files[index]\n        image_name = self.image_names[index]\n        # Open the image and convert it to RGB format\n        image = Image.open(image_path).convert(\"RGB\")\n        if self.transform:\n            # Apply any transformations to the image\n            image = self.transform(image)\n        # Return the transformed image and its filename\n        return image, image_name\n\n# Define transformations for test images to match the preprocessing used during training\n# No Random augmentations used however \ntest_image_transformations = transforms.Compose([\n    # Resize the image to 336 x 336 pixels\n    transforms.Resize((336, 336)),\n    # Convert the PIL image to a PyTorch tensor\n    transforms.ToTensor(),\n    # Normalize the image tensor with mean and standard deviation values\n    transforms.Normalize(\n        mean=(0.485, 0.456, 0.406),  # Same as training normalization\n        std=(0.229, 0.224, 0.225)\n    )\n])\n\n# Create an instance of the test dataset with the testing directory and transformations\ntest_dataset = TestImagesDataset(testing_directory, transform=test_image_transformations)\n\n\n\n\ntest_loader = DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=4)\n\n# Set the model and classifier to evaluation mode \nclip_model.eval()\nclassifier_layer.eval()\n\n# Initialize a list to store predictions\nall_predictions = []\n\n# Disable gradient calculations for faster processing\nwith torch.no_grad():\n    # Loop through each batch of test data\n    for test_images, test_image_names in tqdm(test_loader, desc=\"Testing\", unit=\"batch\"):\n        # Move test images to the selected device (GPU or CPU)\n        test_images = test_images.to(device)\n\n        # Encode the test images to get their features using the CLIP model\n        image_features = clip_model.encode_image(test_images)\n        # Normalize the image embeddings to have unit length\n        image_features /= image_features.norm(dim=-1, keepdim=True)\n\n        # Calculate similarity between image features and all text embeddings\n        # This results in a similarity score for each class per image\n        similarity_scores = image_features @ text_features.T  # Matrix multiplication\n\n        # Apply softmax to get probabilities for each class\n        probability_scores = similarity_scores.softmax(dim=-1)\n        # Get the index of the class with the highest probability for each image\n        top_class_indices = probability_scores.argmax(dim=-1).cpu().numpy()\n\n        # Loop through each prediction and store the results\n        for i, class_index in enumerate(top_class_indices):\n            predicted_class = all_animal_classes[class_index]  # Get class name from index\n            all_predictions.append((test_image_names[i], predicted_class))  # Append to predictions list\n\nprint(\"Testing Complete\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T07:10:27.166790Z","iopub.execute_input":"2025-01-12T07:10:27.167012Z","iopub.status.idle":"2025-01-12T07:12:10.412417Z","shell.execute_reply.started":"2025-01-12T07:10:27.166992Z","shell.execute_reply":"2025-01-12T07:12:10.411432Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"(4.1) Saving the results","metadata":{}},{"cell_type":"code","source":"# ====================================\n# CELL 13: SAVE PREDICTIONS TO CSV FILE\n# ====================================\n# Sort the predictions by image filename for consistency\nall_predictions.sort(key=lambda x: x[0])\n\n# Open the CSV file in write mode\nwith open(prediction_csv_path, mode='w', newline='') as csv_file:\n    csv_writer = csv.writer(csv_file)\n    # Write the header row\n    csv_writer.writerow(['image_id', 'class'])\n    # Write each prediction as a new row in the CSV\n    for image_name, predicted_class in all_predictions:\n        csv_writer.writerow([image_name, predicted_class])\n\nprint(\"The predictions are now saved in 'save_predictions.csv' in the specified directory\")\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-01-12T07:12:10.413396Z","iopub.execute_input":"2025-01-12T07:12:10.413671Z","iopub.status.idle":"2025-01-12T07:12:10.421567Z","shell.execute_reply.started":"2025-01-12T07:12:10.413644Z","shell.execute_reply":"2025-01-12T07:12:10.420658Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"(4.2) HOTS Analysis","metadata":{}},{"cell_type":"code","source":"# =============================================\n# CELL 14: HOTS SAMPLE TESTING WITH SAME CODE\n# =============================================\n\n# Create an instance of the test dataset with the testing directory and transformations\ntest_dataset = TestImagesDataset(hots_directory, transform=test_image_transformations)\n\n\n\ntest_loader = DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=4)\n\n# Set the model and classifier to evaluation mode \nclip_model.eval()\nclassifier_layer.eval()\n\n# Initialize a list to store predictions\nall_hots_predictions = []\n\n# Disable gradient calculations for faster processing\nwith torch.no_grad():\n    # Loop through each batch of test data\n    for test_images, test_image_names in tqdm(test_loader, desc=\"Testing HOTS\", unit=\"batch\"):\n        # Move test images to the selected device (GPU or CPU)\n        test_images = test_images.to(device)\n\n        # Encode the test images to get their features using the CLIP model\n        image_features = clip_model.encode_image(test_images)\n        # Normalize the image embeddings to have unit length\n        image_features /= image_features.norm(dim=-1, keepdim=True)\n\n        # Calculate similarity between image features and all text embeddings\n        # This results in a similarity score for each class per image\n        similarity_scores = image_features @ text_features.T  # Matrix multiplication\n\n        # Apply softmax to get probabilities for each class\n        probability_scores = similarity_scores.softmax(dim=-1)\n        # Get the index of the class with the highest probability for each image\n        top_class_indices = probability_scores.argmax(dim=-1).cpu().numpy()\n\n        # Loop through each prediction and store the results\n        for i, class_index in enumerate(top_class_indices):\n            predicted_class = all_animal_classes[class_index]  # Get class name from index\n            all_hots_predictions.append((test_image_names[i], predicted_class))  # Append to predictions list\n\nprint(\"Testing Complete\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T07:12:10.422376Z","iopub.execute_input":"2025-01-12T07:12:10.422635Z","iopub.status.idle":"2025-01-12T07:12:14.503192Z","shell.execute_reply.started":"2025-01-12T07:12:10.422604Z","shell.execute_reply":"2025-01-12T07:12:14.502224Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ======================================\n# CELL 15: SAVE PREDICTIONS TO CSV FILE\n# ======================================\n# Sort the predictions by image filename for consistency\nall_hots_predictions.sort(key=lambda x: x[0])\n\n# Open the CSV file in write mode\nwith open(hots_prediction_csv_path, mode='w', newline='') as csv_file:\n    csv_writer = csv.writer(csv_file)\n    # Write the header row\n    csv_writer.writerow(['image_id', 'class'])\n    # Write each prediction as a new row in the CSV\n    for image_name, predicted_class in all_hots_predictions:\n        csv_writer.writerow([image_name, predicted_class])\n\nprint(\"The predictions are now saved in the specified directory\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T07:12:14.504037Z","iopub.execute_input":"2025-01-12T07:12:14.504359Z","iopub.status.idle":"2025-01-12T07:12:14.510432Z","shell.execute_reply.started":"2025-01-12T07:12:14.504322Z","shell.execute_reply":"2025-01-12T07:12:14.509747Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =================================\n# CELL 16: CHECKING HOTS ACCURACY\n# =================================\n\n# Open the txt file and read each line, remove any extra spaces, and append to the list\nwith open(hots_file_path, mode='r') as label_file:\n    accurate_result = []\n    for line in label_file:\n        accurate_result.append(line.strip())\n\ncounter = 0\n\n# Compare each value in all_hots_predictions with the accurate_result list\nfor index in range(len(accurate_result)):\n    if all_hots_predictions[index][1] == accurate_result[index]:\n        counter += 1\n\n# Calculate accuracy as a percentage\naccuracy = (counter / len(accurate_result)) * 100\n\n# Print the accuracy result\nprint(\"Accuracy for HOTS: \", accuracy, \"%\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T07:12:14.511247Z","iopub.execute_input":"2025-01-12T07:12:14.511442Z","iopub.status.idle":"2025-01-12T07:12:14.536900Z","shell.execute_reply.started":"2025-01-12T07:12:14.511425Z","shell.execute_reply":"2025-01-12T07:12:14.536143Z"}},"outputs":[],"execution_count":null}]}