{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":85240,"databundleVersionId":9622164,"sourceType":"competition"}],"dockerImageVersionId":30805,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"! pip install ultralytics","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T06:45:42.789451Z","iopub.execute_input":"2024-12-04T06:45:42.789770Z","iopub.status.idle":"2024-12-04T06:45:52.996604Z","shell.execute_reply.started":"2024-12-04T06:45:42.789744Z","shell.execute_reply":"2024-12-04T06:45:52.995524Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nbase_dir = \"/kaggle/input/dlp-object-detection/final_dlp_data/final_dlp_data\"\n\ntrain_dir = os.path.join(base_dir,'train')\ntest_dir = os.path.join(base_dir,'test')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T06:45:52.998257Z","iopub.execute_input":"2024-12-04T06:45:52.998564Z","iopub.status.idle":"2024-12-04T06:45:53.003235Z","shell.execute_reply.started":"2024-12-04T06:45:52.998537Z","shell.execute_reply":"2024-12-04T06:45:53.002442Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport torch\nfrom PIL import Image\nfrom torch.utils.data import Dataset\n\ndevice = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n\nclass Train(Dataset):\n    def __init__(self, img_dir, labels_dir, transform=None):\n        self.img_dir = img_dir\n        self.labels_dir = labels_dir\n        self.img_files = sorted(os.listdir(img_dir))\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.img_files)\n\n    def __getitem__(self, idx):\n        # Load image\n        img_path = os.path.join(self.img_dir, self.img_files[idx])\n        image = Image.open(img_path).convert(\"RGB\")\n\n        # Load labels\n\n        label_file = os.path.splitext(self.img_files[idx])[0] + \".txt\"\n        label_path = os.path.join(self.labels_dir, label_file)\n        boxes = []\n        labels = []\n        with open(label_path, \"r\") as f:\n            parts = f.read().strip().split()\n            class_id = int(parts[0])\n            x_min, y_min, x_max, y_max = map(float, parts[1:])\n            boxes.append([x_min, y_min, x_max, y_max])\n            labels.append(class_id)\n\n        # Convert to tensor\n        print(boxes,labels)\n        boxes = torch.tensor(boxes, dtype=torch.float32)\n        labels = torch.tensor(labels, dtype=torch.int64)\n\n        target = {\n            \"boxes\": boxes,\n            \"labels\": labels,\n        }\n\n        return image, target","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T06:45:55.457224Z","iopub.execute_input":"2024-12-04T06:45:55.457572Z","iopub.status.idle":"2024-12-04T06:45:58.478582Z","shell.execute_reply.started":"2024-12-04T06:45:55.457543Z","shell.execute_reply":"2024-12-04T06:45:58.477781Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import ShuffleSplit\nfrom torch.utils.data import Subset\n\ntrain_dataset = Train(os.path.join(train_dir,'images'),os.path.join(train_dir,'labels'))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T02:21:48.392071Z","iopub.execute_input":"2024-12-04T02:21:48.392412Z","iopub.status.idle":"2024-12-04T02:21:49.786890Z","shell.execute_reply.started":"2024-12-04T02:21:48.392381Z","shell.execute_reply":"2024-12-04T02:21:49.785994Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport copy\nimport os\nimport cv2\nfrom torchvision.transforms import ToTensor\n\ndef show(img, boxes):\n    # Convert image to tensor if it's not already\n    if not isinstance(img, torch.Tensor):\n        img = ToTensor()(img)\n    \n    boxes = boxes.detach().numpy().astype(np.int32)\n    sample = img.permute(1, 2, 0).numpy().copy()\n\n    print(boxes)\n\n    for box in boxes:\n        cv2.rectangle(sample, (box[0], box[1]), (box[2], box[3]), (220, 0, 0), 3)\n\n    plt.axis(\"off\")\n    plt.imshow(sample)\n\n\nplt.figure(figsize=(8,8))\nimg,target=next(iter(train_dataset))\nshow(img,target[\"boxes\"])\nplt.savefig(\"1.png\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T06:46:03.714648Z","iopub.execute_input":"2024-12-04T06:46:03.715481Z","iopub.status.idle":"2024-12-04T06:46:07.944918Z","shell.execute_reply.started":"2024-12-04T06:46:03.715447Z","shell.execute_reply":"2024-12-04T06:46:07.943406Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Starting off with YOLO From here\n\noriginal_dataset_path = '/kaggle/input/dlp-object-detection/final_dlp_data/final_dlp_data'\nyolo_dataset_dir = '/kaggle/working/yolo_dataset'\n\nos.makedirs(os.path.join(yolo_dataset_dir, 'images', 'train'), exist_ok=True)\nos.makedirs(os.path.join(yolo_dataset_dir, 'images', 'val'), exist_ok=True)\nos.makedirs(os.path.join(yolo_dataset_dir, 'labels', 'train'), exist_ok=True)\nos.makedirs(os.path.join(yolo_dataset_dir, 'labels', 'val'), exist_ok=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T06:46:16.925807Z","iopub.execute_input":"2024-12-04T06:46:16.926140Z","iopub.status.idle":"2024-12-04T06:46:16.932085Z","shell.execute_reply.started":"2024-12-04T06:46:16.926109Z","shell.execute_reply":"2024-12-04T06:46:16.931249Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nimport shutil\n\ndef copy_files(file_list, src_images, src_labels, dest_images, dest_labels):\n    for filename in file_list:\n        # Copy image\n        shutil.copy(os.path.join(src_images, filename), os.path.join(dest_images, filename))\n        \n        # Copy label\n        label_filename = filename.replace('.jpeg', '.txt')\n        shutil.copy(os.path.join(src_labels, label_filename), os.path.join(dest_labels, label_filename))\n\ntrain_images_path = os.path.join(original_dataset_path, 'train', 'images')\ntrain_labels_path = os.path.join(original_dataset_path, 'train', 'labels')\n\nimage_filenames = [f for f in os.listdir(train_images_path) if f.endswith('.jpeg')]\n\ntrain_files, val_files = train_test_split(image_filenames, test_size=0.1, random_state=42)\n\ncopy_files(train_files, train_images_path, train_labels_path,\n           os.path.join(yolo_dataset_dir, 'images', 'train'),\n           os.path.join(yolo_dataset_dir, 'labels', 'train'))\n\n# Copy validation files\ncopy_files(val_files, train_images_path, train_labels_path,\n           os.path.join(yolo_dataset_dir, 'images', 'val'),\n           os.path.join(yolo_dataset_dir, 'labels', 'val'))\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T06:46:19.234736Z","iopub.execute_input":"2024-12-04T06:46:19.235061Z","iopub.status.idle":"2024-12-04T06:48:31.159449Z","shell.execute_reply.started":"2024-12-04T06:46:19.235035Z","shell.execute_reply":"2024-12-04T06:48:31.158679Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"CLASS_LABELS = {\n    0: \"aegypti\",\n    1: \"albopictus\",\n    2: \"anopheles\",\n    3: \"culex\",\n    4: \"culiseta\",\n    5: \"japonicus/koreicus\"\n}\n\nbase_path = '/kaggle/input/dlp-object-detection/final_dlp_data/final_dlp_data'\n\nimage_filename = '0000c8c4-e87a-44b8-84d4-8bebcf75645c.jpeg'\nlabel_filename = image_filename.replace('.jpeg', '.txt')\n\ndef visualize_with_opencv(image_path, label_path, class_labels):\n    image = cv2.imread(image_path)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\n    if not os.path.exists(label_path):\n        print(f\"Label file not found: {label_path}\")\n        return\n\n    with open(label_path, 'r') as file:\n        lines = file.readlines()\n\n    for line in lines:\n        parts = line.strip().split()\n        if len(parts) != 5:\n            print(f\"Skipping invalid label line: {line}\")\n            continue\n        class_id, x_center, y_center, width, height = parts\n        try:\n            class_id = int(class_id)\n            x_center = float(x_center)\n            y_center = float(y_center)\n            width = float(width)\n            height = float(height)\n        except ValueError:\n            print(f\"Invalid values in label line: {line}\")\n            continue\n\n        img_height, img_width, _ = image.shape\n        x_center_pixel = x_center * img_width\n        y_center_pixel = y_center * img_height\n        width_pixel = width * img_width\n        height_pixel = height * img_height\n\n        x1 = int(x_center_pixel - (width_pixel / 2))\n        y1 = int(y_center_pixel - (height_pixel / 2))\n        x2 = int(x_center_pixel + (width_pixel / 2))\n        y2 = int(y_center_pixel + (height_pixel / 2))\n\n        box_color = (0, 255, 0)  # Green\n        text_color = (255, 255, 255)  # White\n\n        cv2.rectangle(image, (x1, y1), (x2, y2), box_color, 2)\n\n        label = class_labels.get(class_id, \"Unknown\")\n\n        (text_width, text_height), baseline = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.6, 2)\n\n        cv2.rectangle(image, (x1, y1 - text_height - baseline - 10), \n                              (x1 + text_width + 10, y1), box_color, -1)\n\n        cv2.putText(image, label, (x1 + 5, y1 - 5), cv2.FONT_HERSHEY_SIMPLEX, 0.6, text_color, 2)\n\n    plt.figure(figsize=(12, 8))\n    plt.imshow(image)\n\n    plt.show()\n\n\n# Example image and label filename\nimage_filename = '0000c8c4-e87a-44b8-84d4-8bebcf75645c.jpeg'\nlabel_filename = image_filename.replace('.jpeg', '.txt')\n\nimage_path = os.path.join(train_images_path, image_filename)\nlabel_path = os.path.join(train_labels_path, label_filename)\n\n# Visualize\nvisualize_with_opencv(image_path, label_path, CLASS_LABELS)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T06:51:02.878936Z","iopub.execute_input":"2024-12-04T06:51:02.879330Z","iopub.status.idle":"2024-12-04T06:51:03.438349Z","shell.execute_reply.started":"2024-12-04T06:51:02.879286Z","shell.execute_reply":"2024-12-04T06:51:03.437492Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"yaml_content = \"\"\"\npath: /kaggle/working/yolo_dataset \ntrain: images/train \nval: images/val \n\n# Classes\nnames:\n  0: aegypti\n  1: albopictus\n  2: anopheles\n  3: culex\n  4: culiseta\n  5: japonicus/koreicus\n  \n\"\"\"\n\nwith open('/kaggle/working/dataset.yaml', 'w') as f:\n    f.write(yaml_content)\n\nprint(\"dataset.yaml file has been created.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T06:51:11.335226Z","iopub.execute_input":"2024-12-04T06:51:11.335590Z","iopub.status.idle":"2024-12-04T06:51:11.341442Z","shell.execute_reply.started":"2024-12-04T06:51:11.335558Z","shell.execute_reply":"2024-12-04T06:51:11.340503Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from ultralytics import YOLO\nmodel = YOLO(\"yolo11n.pt\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T06:51:13.905871Z","iopub.execute_input":"2024-12-04T06:51:13.906700Z","iopub.status.idle":"2024-12-04T06:51:14.941289Z","shell.execute_reply.started":"2024-12-04T06:51:13.906664Z","shell.execute_reply":"2024-12-04T06:51:14.940403Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.train(\n    data=\"/kaggle/working/dataset.yaml\", \n    amp=True,\n    epochs=100,                            \n    imgsz=640,                           \n    batch=32,                             \n    workers=4,\n    augment=False,  \n    hsv_h=0.015,   \n    hsv_s=0.7,     \n    hsv_v=0.4,     \n    name=\"mosquito_detection\",           \n    project=\"/kaggle/working/yolo_models\",\n    device='0'\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T06:51:23.144627Z","iopub.execute_input":"2024-12-04T06:51:23.145208Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def convert_yolo_label_to_absolute(label_path, image_width, image_height):\n    absolute_bboxes = []\n    with open(label_path, 'r') as file:\n        lines = file.readlines()\n    \n    for line in lines:\n        parts = line.strip().split()\n        if len(parts) != 5:\n            continue  # Skip malformed lines\n        class_id = int(parts[0])\n        x_center_norm = float(parts[1])\n        y_center_norm = float(parts[2])\n        width_norm = float(parts[3])\n        height_norm = float(parts[4])\n        \n        # Convert normalized coordinates to absolute pixels\n        x_center = x_center_norm * image_width\n        y_center = y_center_norm * image_height\n        width = width_norm * image_width\n        height = height_norm * image_height\n        \n        # Calculate top-left corner\n        x1 = x_center - (width / 2)\n        y1 = y_center - (height / 2)\n        \n        absolute_bboxes.append({\n            'class_id': class_id,\n            'x1': x1,\n            'y1': y1,\n            'width': width,\n            'height': height\n        })\n    \n    return absolute_bboxes","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T04:46:13.669294Z","iopub.execute_input":"2024-12-04T04:46:13.669618Z","iopub.status.idle":"2024-12-04T04:46:13.676121Z","shell.execute_reply.started":"2024-12-04T04:46:13.669591Z","shell.execute_reply":"2024-12-04T04:46:13.675229Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def convert_absolute_to_normalized(x_center, y_center, width, height, image_width, image_height):\n    \"\"\"\n    Normalize bounding box coordinates.\n\n    Args:\n        x_center (float): Absolute x-coordinate of the bounding box center.\n        y_center (float): Absolute y-coordinate of the bounding box center.\n        width (float): Absolute width of the bounding box.\n        height (float): Absolute height of the bounding box.\n        image_width (int): Width of the image in pixels.\n        image_height (int): Height of the image in pixels.\n\n    Returns:\n        tuple: Normalized (x_center, y_center, width, height)\n    \"\"\"\n    x_center_norm = x_center / image_width\n    y_center_norm = y_center / image_height\n    width_norm = width / image_width\n    height_norm = height / image_height\n    return x_center_norm, y_center_norm, width_norm, height_norm\n\ndef process_image(model, image_path, conf_threshold=0.25):\n\n    img = cv2.imread(image_path)\n    if img is None:\n        print(f\"Warning: Unable to load image at {image_path}\")\n        return {\n            'LabelName': 'albopictus',\n            'Conf': 0,\n            'xcenter': 0.5,\n            'ycenter': 0.5,\n            'bbx_width': 0.5,\n            'bbx_height': 0.5\n        }\n    original_height, original_width, _ = img.shape\n\n    results = model.predict(\n        source=image_path,\n        conf=conf_threshold,\n        save=False  # Don't save predictions\n    )\n\n    if not results:\n        print(f\"No predictions for image: {image_path}\")\n        return {\n            'LabelName': 'albopictus',\n            'Conf': 0,\n            'xcenter': 0.5,\n            'ycenter': 0.5,\n            'bbx_width': 0.5,\n            'bbx_height': 0.5\n        }\n\n\n    result = results[0]\n\n    if not result.boxes:\n        print(f\"No detections with confidence >= {conf_threshold} for image: {image_path}\")\n        return {\n            'LabelName': 'albopictus',\n            'Conf': 0,\n            'xcenter': 0.5,\n            'ycenter': 0.5,\n            'bbx_width': 0.5,\n            'bbx_height': 0.5\n        }\n\n    predictions = []\n    for box in result.boxes:\n        cls_id = int(box.cls[0].cpu().numpy())\n        conf = float(box.conf[0].cpu().numpy())\n        x1, y1, x2, y2 = box.xyxy[0].cpu().numpy().astype(float)\n        width = x2 - x1\n        height = y2 - y1\n        x_center = x1 + (width / 2)\n        y_center = y1 + (height / 2)\n\n        area = width * height\n\n        predictions.append({\n            'class_id': cls_id,\n            'Conf': conf,\n            'x_center': x_center,\n            'y_center': y_center,\n            'width': width,\n            'height': height,\n            'area': area\n        })\n\n    predictions_sorted = sorted(predictions, key=lambda x: (-x['area'], -x['Conf']))\n    best_pred = predictions_sorted[0]\n\n    x_center_norm, y_center_norm, width_norm, height_norm = convert_absolute_to_normalized(\n        best_pred['x_center'], best_pred['y_center'], best_pred['width'], best_pred['height'],\n        original_width, original_height\n    )\n\n    class_name = model.names[best_pred['class_id']]\n\n    return {\n        'LabelName': class_name,\n        'Conf': round(best_pred['Conf'], 4),\n        'xcenter': round(x_center_norm, 6),\n        'ycenter': round(y_center_norm, 6),\n        'bbx_width': round(width_norm, 6),\n        'bbx_height': round(height_norm, 6)\n    }\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T04:46:44.384073Z","iopub.execute_input":"2024-12-04T04:46:44.384405Z","iopub.status.idle":"2024-12-04T04:46:44.396347Z","shell.execute_reply.started":"2024-12-04T04:46:44.384377Z","shell.execute_reply":"2024-12-04T04:46:44.395365Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\ndef generate_predictions_csv(model, test_dir, output_csv_path, conf_threshold=0.25):\n    predictions_data = []\n\n    image_filenames = sorted([\n        f for f in os.listdir(test_dir)\n        if f.lower().endswith(('.png', '.jpg', '.jpeg', '.bmp', '.tiff'))\n    ])\n\n    total_images = len(image_filenames)\n    print(f\"Total images found: {total_images}\")\n\n    # Iterate over each image and process predictions\n    for idx, image_filename in enumerate(image_filenames):\n        image_path = os.path.join(test_dir, image_filename)\n        prediction = process_image(model, image_path, conf_threshold)\n\n        predictions_data.append({\n            'id': idx,\n            'ImageID': image_filename,\n            'LabelName': prediction['LabelName'],\n            'Conf': prediction['Conf'],\n            'xcenter': prediction['xcenter'],\n            'ycenter': prediction['ycenter'],\n            'bbx_width': prediction['bbx_width'],\n            'bbx_height': prediction['bbx_height']\n        })\n\n        if (idx + 1) % 50 == 0 or (idx + 1) == total_images:\n            print(f\"Processed {idx + 1}/{total_images} images.\")\n\n    df = pd.DataFrame(predictions_data, columns=[\n        'id', 'ImageID', 'LabelName', 'Conf', 'xcenter', 'ycenter', 'bbx_width', 'bbx_height'\n    ])\n\n    df.to_csv(output_csv_path, index=False)\n    print(f\"Predictions CSV saved at: {output_csv_path}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T04:45:38.869179Z","iopub.execute_input":"2024-12-04T04:45:38.869529Z","iopub.status.idle":"2024-12-04T04:45:38.876975Z","shell.execute_reply.started":"2024-12-04T04:45:38.869498Z","shell.execute_reply":"2024-12-04T04:45:38.875945Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_directory = \"/kaggle/input/dlp-object-detection/final_dlp_data/final_dlp_data/test/images\"\noutput_csv = \"/kaggle/working/yolo_predictions1.csv\"  \n\ngenerate_predictions_csv(\n    model=model,\n    test_dir=test_directory,\n    output_csv_path=output_csv,\n    conf_threshold=0.25\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T04:46:47.520768Z","iopub.execute_input":"2024-12-04T04:46:47.521085Z","iopub.status.idle":"2024-12-04T04:47:45.773072Z","shell.execute_reply.started":"2024-12-04T04:46:47.521060Z","shell.execute_reply":"2024-12-04T04:47:45.772337Z"}},"outputs":[],"execution_count":null}]}