{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":85240,"databundleVersionId":9622164,"sourceType":"competition"}],"dockerImageVersionId":30805,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport numpy as np\nimport torch\nimport torch.nn as nn\nimport torchvision.transforms as T\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision.models.detection import retinanet_resnet50_fpn_v2\nfrom torchvision.ops import box_convert\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom PIL import Image\nfrom matplotlib.image import imread\nfrom torchvision.ops.boxes import box_convert\n\n# Class names mapping\nCLASS_NAMES = {\n    0: \"aegypti\",\n    1: \"albopictus\",\n    2: \"anopheles\",\n    3: \"culex\",\n    4: \"culiseta\",\n    5: \"japonicus/koreicus\",\n}\n\n# Dataset class\nfrom torchvision.io import read_image\n\nclass MosquitoDataset(Dataset):\n    def __init__(self, img_dir, label_dir=None, transform=None, device='cuda'):\n        self.img_dir = img_dir\n        self.label_dir = label_dir\n        self.transform = transform\n        self.device = device\n        self.img_files = [f for f in os.listdir(img_dir) if f.endswith('.jpeg')]\n\n    def __len__(self):\n        return len(self.img_files)\n\n    def __getitem__(self, idx):\n        img_name = self.img_files[idx]\n        img_path = os.path.join(self.img_dir, img_name)\n\n        # Load image using torchvision's read_image\n        img = imread(img_path)\n\n        # Load labels\n        if self.label_dir:\n            label_path = os.path.join(self.label_dir, img_name.replace('.jpeg', '.txt'))\n            bboxes, labels = [], []\n            with open(label_path, 'r') as file:\n                for line in file.readlines():\n                    label, x_center, y_center, width, height = map(float, line.strip().split())\n                    x_min = (x_center - width / 2)*512\n                    y_min = (y_center - height / 2)*512\n                    x_max = (x_center + width / 2)*512\n                    y_max = (y_center + height / 2)*512\n                    bboxes.append([x_min, y_min, x_max, y_max])\n                    labels.append(int(label))\n    \n            # Apply transformations\n            if self.transform:\n                transformed = self.transform(\n                    image=img)\n                img = transformed['image']\n                bboxes = torch.tensor(bboxes, dtype=torch.float32).clone().detach()\n                labels = torch.tensor(labels, dtype=torch.int64).clone().detach()\n    \n            return img, {\"boxes\": bboxes, \"labels\": labels}\n        else:\n            if self.transform:\n                transformed = self.transform(\n                    image=img)\n                img = transformed['image']\n            return img,img_name.split(\".\")[0]\n            \ntransform = A.Compose(\n    [\n        # A.Resize(512, 512),\n        A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),\n        # A.HorizontalFlip(p=0.5),\n        # A.VerticalFlip(p=0.5),\n        # A.ColorJitter(p=0.2),\n        ToTensorV2(),\n    ]\n)\n\n# Custom loss function\nclass FocalLoss(nn.Module):\n    def __init__(self, alpha=0.25, gamma=2):\n        super(FocalLoss, self).__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n\n    def forward(self, logits, targets):\n        prob = torch.sigmoid(logits)\n        p_t = prob * targets + (1 - prob) * (1 - targets)\n        focal_weight = (1 - p_t) ** self.gamma\n        alpha_weight = self.alpha * targets + (1 - self.alpha) * (1 - targets)\n        loss = -alpha_weight * focal_weight * torch.log(p_t + 1e-8)\n        return loss.mean()\n  ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T10:50:19.05267Z","iopub.execute_input":"2024-12-04T10:50:19.05304Z","iopub.status.idle":"2024-12-04T10:50:22.201461Z","shell.execute_reply.started":"2024-12-04T10:50:19.053004Z","shell.execute_reply":"2024-12-04T10:50:22.200698Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import os\n# import shutil\n# import random\n\n# # Define directories\n# kaggle_dir = '/kaggle/input/dlp-object-detection/final_dlp_data/final_dlp_data/train/images'  # Replace with your YOLO dataset folder path\n# output_dir = '/kaggle/working/'  # Replace with your desired output folder\n\n# # Train-test split ratio\n# train_ratio = 0.8\n\n# # Create output directories\n# train_dir = os.path.join(output_dir, 'train')\n# test_dir = os.path.join(output_dir, 'test')\n# train_im_dir = os.path.join(train_dir, 'images')\n# train_lb_dir = os.path.join(train_dir, 'labels')\n# test_im_dir = os.path.join(test_dir, 'images')\n# test_lb_dir = os.path.join(test_dir, 'labels')\n# os.makedirs(train_dir, exist_ok=True)\n# os.makedirs(test_dir, exist_ok=True)\n# os.makedirs(train_im_dir, exist_ok=True)\n# os.makedirs(train_lb_dir, exist_ok=True)\n# os.makedirs(test_im_dir, exist_ok=True)\n# os.makedirs(test_lb_dir, exist_ok=True)\n\n# # Get all files\n# all_files = [f for f in os.listdir(kaggle_dir) if f.endswith('.jpeg')]\n\n# # Shuffle and split data\n# random.shuffle(all_files)\n# train_files = all_files[:int(len(all_files) * train_ratio)]\n# test_files = all_files[int(len(all_files) * train_ratio):]\n\n# def move_files(file_list, destination_dir):\n#     image_dir = os.path.join(destination_dir, 'images')\n#     label_dir = os.path.join(destination_dir, 'labels')\n#     for file in file_list:\n#         image_path = os.path.join(kaggle_dir, file)\n#         label_path = os.path.join(kaggle_dir, file.replace('.jpeg', '.txt')).replace('images','labels')\n#         # Move image\n#         shutil.copy(image_path, image_dir)\n        \n#         # Move corresponding label file if exists\n#         if os.path.exists(label_path):\n#             shutil.copy(label_path, label_dir)\n\n# # Move files to respective directories\n# move_files(train_files, train_dir)\n# move_files(test_files, test_dir)\n\n# print(f\"Train files moved to: {train_dir}\")\n# print(f\"Test files moved to: {test_dir}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T10:50:22.202917Z","iopub.execute_input":"2024-12-04T10:50:22.203341Z","iopub.status.idle":"2024-12-04T10:50:22.208297Z","shell.execute_reply.started":"2024-12-04T10:50:22.203313Z","shell.execute_reply":"2024-12-04T10:50:22.207352Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ! ls /kaggle/working/train/labels |wc -l","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T10:50:22.209087Z","iopub.execute_input":"2024-12-04T10:50:22.209369Z","iopub.status.idle":"2024-12-04T10:50:22.223152Z","shell.execute_reply.started":"2024-12-04T10:50:22.209343Z","shell.execute_reply":"2024-12-04T10:50:22.222336Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from torchvision.models.detection.retinanet import retinanet_resnet50_fpn_v2\nfrom torchvision.models.detection import fasterrcnn_resnet50_fpn\n\n# Model\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# Modify the model for custom classes\nmodel = fasterrcnn_resnet50_fpn(num_classes=6)\n\n# Move model to device\nmodel = model.to(device)\n\n# Optimizer and Loss\noptimizer = torch.optim.AdamW(model.parameters(), lr=1e-4, weight_decay=1e-4)\n\n# Dataset and DataLoader\ntrain_dataset = MosquitoDataset(\n    img_dir=\"/kaggle/working/train/images\",\n    label_dir=\"/kaggle/working/train/labels\",\n    transform=transform,\n    device=device\n)\nval_dataset = MosquitoDataset(\n    img_dir=\"/kaggle/working/test/images\",\n    label_dir=\"/kaggle/working/test/labels\",\n    transform=transform,\n    device=device\n)\ntest_dataset = MosquitoDataset(\n    img_dir=\"/kaggle/input/dlp-object-detection/final_dlp_data/final_dlp_data/test/images\",\n    transform=transform,\n    device=device\n)\ntrain_loader = DataLoader(train_dataset, batch_size=8, shuffle=True, collate_fn=lambda x: tuple(zip(*x)),num_workers=4)\nval_loader = DataLoader(val_dataset, batch_size=4, shuffle=False, collate_fn=lambda x: tuple(zip(*x)),num_workers=4)\ntest_loader= DataLoader(test_dataset, batch_size=4, shuffle=False,collate_fn=lambda x: tuple(zip(*x)),num_workers=4)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T10:50:22.426383Z","iopub.execute_input":"2024-12-04T10:50:22.427257Z","iopub.status.idle":"2024-12-04T10:50:23.341965Z","shell.execute_reply.started":"2024-12-04T10:50:22.427222Z","shell.execute_reply":"2024-12-04T10:50:23.341257Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Training loop\nfrom tqdm import tqdm\ndef train(model, data_loader, optimizer, device, num_epochs=10):\n    model.train()\n    for epoch in range(num_epochs):\n        epoch_loss = 0\n        model.train()\n        for images, targets in tqdm(train_loader):\n            images = list(img.to(device) for img in images)\n            targets = [{k: v.to(device) for k, v in t.items()} for t in targets]\n\n            optimizer.zero_grad()\n            losses = model(images, targets)\n            loss = sum(loss for loss in losses.values())\n            loss.backward()\n            optimizer.step()\n            epoch_loss += loss.item()\n        predictions = []\n        val_labels = []\n        model.eval()\n        with torch.no_grad():  # Disable gradient calculation\n            val_loss=0\n            for images, targets in tqdm(val_loader):\n                images = list(img.to(device) for img in images)\n                targets = [{k: v.to(device) for k, v in t.items()} for t in targets]\n                val_losses = model(images, targets)\n                val_loss += sum(loss for loss in val_losses.values())\n        print(f\"Epoch [{epoch+1}/{num_epochs}], Loss: {epoch_loss:.4f}, Val: {val_loss}\")\n\ntrain(model, train_loader, optimizer, device, num_epochs=10)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T11:01:55.049611Z","iopub.execute_input":"2024-12-04T11:01:55.049968Z","iopub.status.idle":"2024-12-04T11:02:33.430024Z","shell.execute_reply.started":"2024-12-04T11:01:55.04994Z","shell.execute_reply":"2024-12-04T11:02:33.428698Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"torch.save(model.state_dict(), \"/kaggle/working/model.pt\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T11:01:12.908627Z","iopub.execute_input":"2024-12-04T11:01:12.909661Z","iopub.status.idle":"2024-12-04T11:01:13.184394Z","shell.execute_reply.started":"2024-12-04T11:01:12.909606Z","shell.execute_reply":"2024-12-04T11:01:13.183682Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\noutput_results = []\nmodel.eval()\nwith torch.no_grad():\n    for images, image_ids in test_loader:\n        images = list(image.to(device) for image in images)\n        outputs = model(images)\n        \n        for i, output in enumerate(outputs):\n            _,h,w=images[i].cpu().shape\n            box = box_convert(output['boxes'].cpu(), 'xyxy', 'cxcywh')[0]\n            score = output['scores'].cpu()[0]\n            label = output['labels'].cpu()[0]\n            image_id = image_ids[i]\n            output_results.append([\n                len(output_results),image_id, CLASS_NAMES[label.item()], score.item(),\n                box[0].item()/w, box[1].item()/h, box[2].item()/w, box[3].item()/h\n            ])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T10:50:27.330933Z","iopub.execute_input":"2024-12-04T10:50:27.331598Z","iopub.status.idle":"2024-12-04T10:52:18.086602Z","shell.execute_reply.started":"2024-12-04T10:50:27.331562Z","shell.execute_reply":"2024-12-04T10:52:18.085591Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nresults_df = pd.DataFrame(output_results, columns=[\n    'id''ImageID', 'LabelName', 'Conf', 'xcenter', 'ycenter', 'bbx_width', 'bbx_height'\n])\nresults_df.to_csv('predictions.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T10:52:18.387914Z","iopub.execute_input":"2024-12-04T10:52:18.38829Z","iopub.status.idle":"2024-12-04T10:52:18.399444Z","shell.execute_reply.started":"2024-12-04T10:52:18.388255Z","shell.execute_reply":"2024-12-04T10:52:18.39861Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"results_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T10:52:18.401185Z","iopub.execute_input":"2024-12-04T10:52:18.401567Z","iopub.status.idle":"2024-12-04T10:52:18.423755Z","shell.execute_reply.started":"2024-12-04T10:52:18.401526Z","shell.execute_reply":"2024-12-04T10:52:18.422916Z"}},"outputs":[],"execution_count":null}]}