{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":21154,"databundleVersionId":1243559,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import cv2\nimport numpy as np\nimport tensorflow as tf\nimport os\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport torchvision\nimport torch\nfrom torch.utils.data import Dataset\nimport os\nfrom torch.utils.data import Dataset\nfrom torchvision import datasets,transforms\nfrom PIL import Image\nfrom torch.utils.data import random_split\nimport torch.optim as optim","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-06-08T20:06:11.982906Z","iopub.execute_input":"2024-06-08T20:06:11.983718Z","iopub.status.idle":"2024-06-08T20:06:29.805133Z","shell.execute_reply.started":"2024-06-08T20:06:11.983682Z","shell.execute_reply":"2024-06-08T20:06:29.804231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_feature_description = {\n    'image': tf.io.FixedLenFeature([], tf.string, default_value=''),\n    'id': tf.io.FixedLenFeature([], tf.string, default_value=''),\n    'class': tf.io.FixedLenFeature([], tf.int64, default_value=0),\n}\n\ndef _parse_image_function(example_proto):\n    return tf.io.parse_single_example(example_proto, train_feature_description)\n\ndef preprocess_image(image):\n    image = tf.io.decode_image(image, channels=3)\n    return image\n\nBASE_DIR = \"/kaggle/input/tpu-getting-started/\"\ntfrec_dir_train = \"tfrecords-jpeg-224x224/train/\"\ntfrec_dir_test = \"tfrecords-jpeg-224x224/test/\"\ntfrec_dir_val = \"tfrecords-jpeg-224x224/val/\"\n\npath_to_train_images = '/kaggle/working/train/'\npath_to_test_images = '/kaggle/working/test/'\npath_to_valid_images = '/kaggle/working/valid/'\n\ndef convert(tfrec_dir, output_dir):\n    for tfName in os.listdir(os.path.join(BASE_DIR, tfrec_dir))[:]:\n        train_image_dataset = tf.data.TFRecordDataset(BASE_DIR+tfrec_dir+tfName)\n        train_images = train_image_dataset.map(_parse_image_function)\n\n        for image_features in train_images:\n            image_raw = preprocess_image(image_features['image'])\n            image_raw_int = image_raw.numpy()\n            image_name = image_features['id'].numpy().decode(\"utf-8\") + '.jpg'\n            cl = image_features['class'].numpy()\n            name_cl = str(cl) + '/'\n            path_to_class = output_dir + name_cl\n            if not os.path.isdir(path_to_class):\n                os.mkdir(path_to_class)\n            path_to_img = path_to_class + image_name\n            #print(path_to_img)\n            cv2.imwrite(path_to_img, cv2.cvtColor(image_raw_int, cv2.COLOR_BGR2RGB))\n            \n            #print(image_name, image_features['class'].numpy(), counter)\n","metadata":{"execution":{"iopub.status.busy":"2024-06-08T20:06:29.806793Z","iopub.execute_input":"2024-06-08T20:06:29.807299Z","iopub.status.idle":"2024-06-08T20:06:29.818211Z","shell.execute_reply.started":"2024-06-08T20:06:29.807272Z","shell.execute_reply":"2024-06-08T20:06:29.817312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.mkdir(path_to_train_images)\nos.mkdir(path_to_test_images)\nos.mkdir(path_to_valid_images)\nconvert(tfrec_dir_train, path_to_train_images)\nconvert(tfrec_dir_test, path_to_test_images)\nconvert(tfrec_dir_val, path_to_valid_images)","metadata":{"execution":{"iopub.status.busy":"2024-06-08T20:06:29.819197Z","iopub.execute_input":"2024-06-08T20:06:29.819438Z","iopub.status.idle":"2024-06-08T20:07:07.605373Z","shell.execute_reply.started":"2024-06-08T20:06:29.819417Z","shell.execute_reply":"2024-06-08T20:07:07.604498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\n\ndef extract_number(folder_name):\n    match = re.search(r'\\d+', folder_name)\n    return int(match.group()) if match else None\n\ndef sort_folders(folders):\n    return sorted(folders, key=extract_number)\n\npaths=[]\n\nfor i in os.listdir('/kaggle/working/train/'):\n    img_dir = '/kaggle/working/train/' + i + '/'\n    paths.append(i)\n        \nsorted_folders = sort_folders(paths)\nprint(sorted_folders)\n","metadata":{"execution":{"iopub.status.busy":"2024-06-08T20:07:07.607432Z","iopub.execute_input":"2024-06-08T20:07:07.607722Z","iopub.status.idle":"2024-06-08T20:07:07.615654Z","shell.execute_reply.started":"2024-06-08T20:07:07.607697Z","shell.execute_reply":"2024-06-08T20:07:07.614659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MY_DATASET(Dataset):\n  def __init__(self,path,transform):\n    self.labels = []\n    self.img_pths = []\n\n    for i in os.listdir(path):\n        img_dir = path + '/' + i + '/'\n        for j in os.listdir(img_dir):\n            self.img_pths.append(img_dir+j)\n            self.labels.append(i)\n    predlabel_map = sorted(list(set(self.labels)),key=extract_number)\n    #print(predlabel_map)\n    self.label_map = {}\n    it = 0\n    for i in predlabel_map:\n      self.label_map[i] = it\n      it+=1\n    for i in range(len(self.labels)):\n        self.labels[i] = self.label_map[self.labels[i]]\n    self.transform = transform\n\n  def __len__(self):\n      return len(self.labels)\n\n  def __getitem__(self, idx):\n      img_path = self.img_pths[idx]\n      image = Image.open(img_path).convert('RGB')\n      w, h = image.size\n      label = self.labels[idx]\n      if(self.transform == transform):\n        trans_loc = transforms.CenterCrop((min(w, h), min(w, h)))\n        image = trans_loc(image)\n      if self.transform:\n        image = self.transform(image)\n      return image, label\n\ntransform = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.4487185, 0.41564807, 0.3029375 ], std=[0.2823133, 0.245806, 0.27172017])\n])\ndataset  = MY_DATASET('/kaggle/working/train/', transform)\nprint(len(dataset))","metadata":{"execution":{"iopub.status.busy":"2024-06-08T20:07:07.616901Z","iopub.execute_input":"2024-06-08T20:07:07.617272Z","iopub.status.idle":"2024-06-08T20:07:07.651994Z","shell.execute_reply.started":"2024-06-08T20:07:07.617240Z","shell.execute_reply":"2024-06-08T20:07:07.651117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport shutil\n\ndef copy_folders_with_few_images(base_path, dest_path, threshold = 100):\n    if not os.path.exists(base_path):\n        print(f\"Path '{base_path}' does not exist.\")\n        return\n    \n    if not os.path.exists(dest_path):\n        os.makedirs(dest_path)\n\n    for folder_name in os.listdir(base_path):\n        folder_path = os.path.join(base_path, folder_name)\n\n        if os.path.isdir(folder_path):\n            num_images = sum(1 for item in os.listdir(folder_path) if item.lower().endswith(('.png', '.jpg', '.jpeg', '.gif', '.bmp', '.tiff')))\n            \n            if num_images < threshold:\n                dest_folder_path = os.path.join(dest_path, folder_name)\n                \n                \n                shutil.copytree(folder_path, dest_folder_path)\n                print(f\"Copied folder '{folder_name}' to '{dest_path}' (contains {num_images} images).\")\n\n\nbase_path = '/kaggle/working/train/'\ndest_path = '/kaggle/working/aug/'\ncopy_folders_with_few_images(base_path, dest_path)\n\ntransform_aug = transforms.Compose([\n    transforms.RandomResizedCrop(224),\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomVerticalFlip(),\n    transforms.RandomRotation(30),\n    transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.1),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n])\n","metadata":{"execution":{"iopub.status.busy":"2024-06-08T20:07:07.653068Z","iopub.execute_input":"2024-06-08T20:07:07.653389Z","iopub.status.idle":"2024-06-08T20:07:08.031699Z","shell.execute_reply.started":"2024-06-08T20:07:07.653366Z","shell.execute_reply":"2024-06-08T20:07:08.030770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport shutil\nfrom random import choice\n\ndef duplicate_images_in_folders(base_path, min_images=30):\n    if not os.path.exists(base_path):\n        print(f\"Path '{base_path}' does not exist.\")\n        return\n\n    for folder_name in os.listdir(base_path):\n        folder_path = os.path.join(base_path, folder_name)\n\n        if os.path.isdir(folder_path):\n            images = [img for img in os.listdir(folder_path) if img.lower().endswith(('.png', '.jpg', '.jpeg', '.gif', '.bmp', '.tiff'))]\n            num_images = len(images)\n            \n            if num_images < min_images:\n                print(f\"Folder '{folder_name}' has only {num_images} images. Duplicating images to reach {min_images} images.\")\n                \n                origin_images = set(images)\n                duplicated_images = set()\n\n                while num_images < min_images:\n                    if not origin_images:\n                        origin_images, duplicated_images = duplicated_images, set()\n                    \n                    img_to_duplicate = choice(list(origin_images))\n                    original_path = os.path.join(folder_path, img_to_duplicate)\n                    new_image_name = f\"copy_{num_images}_{img_to_duplicate}\"\n                    new_image_path = os.path.join(folder_path, new_image_name)\n                    \n                    shutil.copy2(original_path, new_image_path)\n                    \n                    images.append(new_image_name)\n                    origin_images.remove(img_to_duplicate)\n                    duplicated_images.add(img_to_duplicate)\n                    num_images += 1\n\nbase_path = '/kaggle/working/aug/'\nduplicate_images_in_folders(base_path, 60)\ndata_aug  = MY_DATASET('/kaggle/working/aug/', transform_aug)","metadata":{"execution":{"iopub.status.busy":"2024-06-08T20:07:08.032900Z","iopub.execute_input":"2024-06-08T20:07:08.033262Z","iopub.status.idle":"2024-06-08T20:07:08.208236Z","shell.execute_reply.started":"2024-06-08T20:07:08.033236Z","shell.execute_reply":"2024-06-08T20:07:08.207324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfrom torchvision import datasets, transforms\nfrom torch.utils.data import random_split, ConcatDataset, DataLoader\n\ntrain_dataset = dataset\n\n\nfrom collections import defaultdict\nclass_counts = defaultdict(int)\n\nfor _, label in train_dataset:\n    class_counts[label] += 1\ntotal_samples = len(train_dataset)\n\nclass_weights = [total_samples / class_counts[label] for label in range(104)]\nweight_tensor = torch.tensor(class_weights, dtype=torch.float)\nfor label, weight in enumerate(class_weights):\n    print(f\"Class {label}: Weight {weight}\")\ntrain_dataset = ConcatDataset([train_dataset, data_aug])","metadata":{"execution":{"iopub.status.busy":"2024-06-08T20:07:08.209610Z","iopub.execute_input":"2024-06-08T20:07:08.210000Z","iopub.status.idle":"2024-06-08T20:07:25.070893Z","shell.execute_reply.started":"2024-06-08T20:07:08.209967Z","shell.execute_reply":"2024-06-08T20:07:25.069804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataloader = DataLoader(train_dataset, batch_size=12, shuffle=True)\nval_dataset = MY_DATASET('/kaggle/working/valid/', transform)\nval_dataloader = DataLoader(val_dataset, batch_size=12, shuffle=True)\ntest_dataset  = MY_DATASET('/kaggle/working/test/', transform)\ntest_dataloader = DataLoader(test_dataset, batch_size=12, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2024-06-08T20:07:25.072191Z","iopub.execute_input":"2024-06-08T20:07:25.072554Z","iopub.status.idle":"2024-06-08T20:07:25.096111Z","shell.execute_reply.started":"2024-06-08T20:07:25.072519Z","shell.execute_reply":"2024-06-08T20:07:25.095385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from typing import Tuple\nfrom random import randrange\n# from torchsummary import summary\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision\n\nimport matplotlib.pyplot as plt\n\nclass MY_CNN(nn.Module):\n    def __init__(self, num_classes):\n      super(MY_CNN, self).__init__()\n      self.conv1 = nn.Conv2d(3, 3, kernel_size=3, stride =1, padding=1)\n      self.conv1_2 = nn.Conv2d(3, 16, kernel_size=1, stride =1, padding=1)\n      self.conv2 = nn.Conv2d(16, 16, kernel_size=3, stride=1, padding=1)\n      self.conv2_2 = nn.Conv2d(16, 32, kernel_size=1, stride=1, padding=1)\n      self.conv3 = nn.Conv2d(32, 32, kernel_size=3, stride=1, padding=1)\n      self.conv3_2 = nn.Conv2d(32, 64, kernel_size=1, stride=1, padding=1)\n      self.conv4 = nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1)\n      self.conv4_2 = nn.Conv2d(64, 128, kernel_size=1, stride=1, padding=1)\n      self.conv5 = nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1)\n      self.conv5_2 = nn.Conv2d(128, 256, kernel_size=1, stride=1, padding=1)\n      self.pool = nn.MaxPool2d((3, 3), padding = 0)\n      self.adaptpool =  nn.AdaptiveAvgPool2d((4, 4))\n      self.fc1 = nn.Linear(4096, 2048)\n      #self.fc2 = nn.Linear(4096, 512)\n      self.fc3 = nn.Linear(2048, num_classes)\n      self.BN1 = nn.BatchNorm2d(16)\n      self.BN2 = nn.BatchNorm2d(32)\n      self.BN3 = nn.BatchNorm2d(64)\n      self.BN4 = nn.BatchNorm2d(128)\n      self.BN5 = nn.BatchNorm2d(256)\n      self.relu = nn.ReLU()\n      self.drop = nn.Dropout(p=0.4)\n      self.num_classes = num_classes\n      \n\n    def forward(self, x: torch.Tensor):\n        x = self.relu(self.BN1(self.conv1_2(self.conv1(x)))) # 256 x 256 x 16\n        x = self.pool(x)\n        #print(x.shape, \"\\n\")\n        x = self.relu(self.BN2(self.conv2_2(self.conv2(x))))\n        x = self.pool(x)\n        #print(x.shape, \"\\n\")\n        x = self.relu(self.BN3(self.conv3_2(self.conv3(x))))\n        x = self.pool(x)\n        #print(x.shape, \"\\n\")\n        x = self.relu(self.BN4(self.conv4_2(self.conv4(x))))\n        x = self.pool(x)\n        x = self.relu(self.BN5(self.conv5_2(self.conv5(x))))\n        x = self.adaptpool(x)\n        #print(x.shape, \"\\n\")\n        x = self.drop(x)\n        x = self.relu(self.fc1(x.view(-1, 4096)))\n        #x = self.relu(self.fc2(x))\n        x = self.fc3(x)\n        #print(x.shape, \"\\n\")\n        return x\n","metadata":{"execution":{"iopub.status.busy":"2024-06-08T20:07:25.098759Z","iopub.execute_input":"2024-06-08T20:07:25.099046Z","iopub.status.idle":"2024-06-08T20:07:25.115897Z","shell.execute_reply.started":"2024-06-08T20:07:25.099024Z","shell.execute_reply":"2024-06-08T20:07:25.114961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = MY_CNN(num_classes=104).cuda()\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(model.parameters(), lr=0.001, betas=(0.9, 0.999), eps=1e-8, weight_decay=0.00001, amsgrad=False)","metadata":{"execution":{"iopub.status.busy":"2024-06-08T20:07:25.117033Z","iopub.execute_input":"2024-06-08T20:07:25.117357Z","iopub.status.idle":"2024-06-08T20:07:25.281324Z","shell.execute_reply.started":"2024-06-08T20:07:25.117327Z","shell.execute_reply":"2024-06-08T20:07:25.280181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def model_training(model,train_loader,optimizer,criterion):\n    running_loss = 0.0\n    model.train()\n    for inputs,labels in train_loader:\n        optimizer.zero_grad()\n        inputs,labels = inputs.cuda() , labels.cuda()\n        out = model(inputs)\n        #print(out[0], \"hi\", labels[0], out.shape, labels.shape)\n        loss = criterion(out,labels)\n        loss.backward()\n        optimizer.step()\n        #print(loss.item())\n        running_loss += loss.item()\n    \n    #print(running_loss, len(train_loader))\n\n    return running_loss / len(train_loader)","metadata":{"execution":{"iopub.status.busy":"2024-06-08T20:07:25.285568Z","iopub.execute_input":"2024-06-08T20:07:25.285840Z","iopub.status.idle":"2024-06-08T20:07:25.292738Z","shell.execute_reply.started":"2024-06-08T20:07:25.285818Z","shell.execute_reply":"2024-06-08T20:07:25.291669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import f1_score\ndef model_validating(model, val_loader, criterion):\n    true_positives = 0\n    predicted_positives = 0\n    actual_positives = 0\n    model.eval()\n    all_predictions = []\n    all_labels = []\n    running_loss = 0.0\n    \n    for inputs, labels in val_loader:\n        with torch.no_grad():\n            inputs, labels = inputs.cuda(), labels.cuda()\n            out = model(inputs)\n            \n            _, predicted = torch.max(out, 1)\n            all_predictions.extend(predicted.cpu().numpy())\n            all_labels.extend(labels.cpu().numpy())\n            \n            true_positives += torch.sum(predicted * labels).cpu().item()\n            predicted_positives += torch.sum(predicted).cpu().item()\n            actual_positives += torch.sum(labels).cpu().item()\n            \n            loss = criterion(out, labels)\n            running_loss += loss.item()\n    \n    precision = true_positives / (predicted_positives + 1e-10)\n    recall = true_positives / (actual_positives + 1e-10)\n    f1 = 2 * (precision * recall) / (precision + recall + 1e-10)\n    \n    avg_loss = running_loss / len(val_loader)\n    \n    return avg_loss, f1_score(all_labels, all_predictions, average='weighted')","metadata":{"execution":{"iopub.status.busy":"2024-06-08T20:07:25.294410Z","iopub.execute_input":"2024-06-08T20:07:25.295013Z","iopub.status.idle":"2024-06-08T20:07:26.035758Z","shell.execute_reply.started":"2024-06-08T20:07:25.294982Z","shell.execute_reply":"2024-06-08T20:07:26.034964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install torchsummary","metadata":{"execution":{"iopub.status.busy":"2024-06-08T20:07:26.036975Z","iopub.execute_input":"2024-06-08T20:07:26.037269Z","iopub.status.idle":"2024-06-08T20:07:39.468064Z","shell.execute_reply.started":"2024-06-08T20:07:26.037244Z","shell.execute_reply":"2024-06-08T20:07:39.466815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torchsummary import summary\nsummary(model, (3, 224, 224))","metadata":{"execution":{"iopub.status.busy":"2024-06-08T20:07:39.469746Z","iopub.execute_input":"2024-06-08T20:07:39.470105Z","iopub.status.idle":"2024-06-08T20:07:40.181081Z","shell.execute_reply.started":"2024-06-08T20:07:39.470073Z","shell.execute_reply":"2024-06-08T20:07:40.180032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def save_arrays_to_file(filename, arr):\n    with open(filename, 'w') as file:\n        for i in arr:\n            file.write(f\"{i}\\n\")\n\nimport torch\nif not os.path.exists('/kaggle/working/vesa/'):\n    os.mkdir('/kaggle/working/vesa/')\nvalb = 0\nnum_epochs = 30\ntrain_loss_arr = []\nval_loss_arr = []\nval_loss_acc = []\nfor epoch in range(0, num_epochs):\n    train_loss = model_training(model, train_dataloader, optimizer, criterion)\n    val_loss, val_acc = model_validating(model, val_dataloader, criterion)\n\n    if(valb < val_acc):\n        torch.save({'model_state_dict': model.state_dict(),'optimizer_state_dict': optimizer.state_dict(),}, \"/kaggle/working/vesa/im_in_real_problem_30_60.pth\")\n        valb = val_acc\n    if(epoch%10 ==0):\n        torch.save({'model_state_dict': model.state_dict(),'optimizer_state_dict': optimizer.state_dict(),}, (\"/kaggle/working/vesa/im_in_real_problem_30_60_epoch_\"+str(epoch)+\".pth\"))\n    if(epoch%30 ==0):\n        save_arrays_to_file('/kaggle/working/vesa/train_loss_0_60_epoch_'+str(epoch)+'.txt', train_loss_arr)\n        save_arrays_to_file('/kaggle/working/vesa/val_loss_0_60_epoch_'+str(epoch)+'.txt', val_loss_arr)\n        save_arrays_to_file('/kaggle/working/vesa/val_acc_0_60_epoch_'+str(epoch)+'.txt', val_loss_acc)\n    print(f'Epoch [{epoch+1}/{num_epochs}], Train Loss: {train_loss:.4f}, Val Loss: {val_loss:.4f} ', val_acc)\n    train_loss_arr.append(train_loss)\n    val_loss_arr.append(val_loss)\n    val_loss_acc.append(val_acc)\n\n\nsave_arrays_to_file('/kaggle/working/vesa/train_loss_0_60_2.txt', train_loss_arr)\nsave_arrays_to_file('/kaggle/working/vesa/val_loss_0_60_2.txt', val_loss_arr)\nsave_arrays_to_file('/kaggle/working/vesa/val_acc_0_60_2.txt', val_loss_acc)","metadata":{"execution":{"iopub.status.busy":"2024-06-08T20:07:40.182252Z","iopub.execute_input":"2024-06-08T20:07:40.182559Z","iopub.status.idle":"2024-06-08T20:45:49.294949Z","shell.execute_reply.started":"2024-06-08T20:07:40.182535Z","shell.execute_reply":"2024-06-08T20:45:49.293877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import Dataset, DataLoader\nfrom PIL import Image\nimport os\n\nclass CustomDataset(Dataset):\n    def __init__(self, image_folder, transform=None):\n        self.image_folder = image_folder\n        self.image_filenames = [os.path.join(image_folder, fname) for fname in os.listdir(image_folder)]\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.image_filenames)\n\n    def __getitem__(self, idx):\n        img_name = self.image_filenames[idx]\n        image = Image.open(img_name)\n        if self.transform:\n            image = self.transform(image)\n        return image, os.path.basename(img_name)\n\n# Example usage:\ndataset = CustomDataset(image_folder='/kaggle/working/test/0/', transform=transform)\ntest_dataloaders = DataLoader(dataset, batch_size=12, shuffle=False)\n# model_testing(model, test_loader, 'submission.csv')\n","metadata":{"execution":{"iopub.status.busy":"2024-06-08T20:45:49.296257Z","iopub.execute_input":"2024-06-08T20:45:49.296601Z","iopub.status.idle":"2024-06-08T20:45:49.324651Z","shell.execute_reply.started":"2024-06-08T20:45:49.296572Z","shell.execute_reply":"2024-06-08T20:45:49.323951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport pandas as pd\nimport os\n\ndef model_testing(model, test_loader, output_csv_path):\n    model.eval()\n    all_predictions = []\n    all_ids = []\n\n    for inputs, ids in test_loader:\n        with torch.no_grad():\n            inputs = inputs.cuda()\n            out = model(inputs)\n            \n            _, predicted = torch.max(out, 1)\n            all_predictions.extend(predicted.cpu().numpy())\n            all_ids.extend([os.path.splitext(filename)[0] for filename in ids])  # Remove file extension\n    \n    results_df = pd.DataFrame({'id': all_ids, 'label': all_predictions})\n    \n    # Ensure the output path is correct for Kaggle\n    output_path = os.path.join('/kaggle/working', output_csv_path)\n    results_df.to_csv(output_csv_path, index=False)\n    \n    # Verify file creation by listing directory contents\n    print(\"Contents of /kaggle/working directory:\")\n    print(os.listdir('/kaggle/working'))\n\n# Example usage:\n# Assuming test_loader yields (inputs, ids) where ids are filenames\nmodel_testing(model, test_dataloaders, 'submission.csv')\n","metadata":{"execution":{"iopub.status.busy":"2024-06-08T20:45:49.325639Z","iopub.execute_input":"2024-06-08T20:45:49.325924Z","iopub.status.idle":"2024-06-08T20:46:01.645441Z","shell.execute_reply.started":"2024-06-08T20:45:49.325887Z","shell.execute_reply":"2024-06-08T20:46:01.644513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport shutil\n\ntry:\n    shutil.rmtree('/kaggle/working/aug/')\nexcept OSError as e:\n    print(e)\n\ntry:\n    shutil.rmtree('/kaggle/working/test/')\nexcept OSError as e:\n    print(e)\n    \ntry:\n    shutil.rmtree('/kaggle/working/train/')\nexcept OSError as e:\n    print(e)\n\ntry:\n    shutil.rmtree('/kaggle/working/valid/')\nexcept OSError as e:\n    print(e)\n    \ntry:\n    shutil.rmtree('/kaggle/working/vesa/')\nexcept OSError as e:\n    print(e)\n    \nprint(os.listdir('/kaggle/working'))","metadata":{"execution":{"iopub.status.busy":"2024-06-08T20:46:01.647138Z","iopub.execute_input":"2024-06-08T20:46:01.647549Z","iopub.status.idle":"2024-06-08T20:46:02.646809Z","shell.execute_reply.started":"2024-06-08T20:46:01.647506Z","shell.execute_reply":"2024-06-08T20:46:02.645874Z"},"trusted":true},"execution_count":null,"outputs":[]}]}