{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":85240,"databundleVersionId":9622164,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport shutil\nimport random\nimport re\n\ndef replace_str(text:str, old_folder:str=\"images\", new_folder:str=\"\")->str:\n    # Define the regex pattern to match '/images/'\n    pattern = rf'/{old_folder}'\n    # Replace all occurrences of the pattern with '/labels/'\n    replaced_text = re.sub(pattern, f'/{new_folder}', text)\n    return replaced_text\n\ndef change_extension(filename:str, new_extension:str)->str:\n    \"\"\"\n    Change the previous extension to new.\n    \"\"\"\n    # Split the filename into the base and the extension\n    base = os.path.splitext(filename)[0]\n    # Create the new filename with the new extension\n    new_filename = f\"{base}.{new_extension}\"\n    return new_filename\n\ndef create_directory(path):\n    if not os.path.exists(path):\n        os.makedirs(path)\n\ndef split_data(source_dir, destination_dir, train_pct, test_pct, image_folder:str=\"images\", label_folder:str=\"labels\"):\n    \n    # for destination of image\n    train_dir = os.path.join(destination_dir, \"train\")   \n    test_dir = os.path.join(destination_dir, \"test\")   \n    val_dir = os.path.join(destination_dir, \"val\")   \n    # Create target directories\n    create_directory(train_dir)\n    create_directory(test_dir)\n    create_directory(val_dir)\n    \n    # for destination of labels\n    train_dir_label = os.path.join(replace_str(destination_dir, new_folder=\"labels\"), \"train\")   \n    test_dir_label = os.path.join(replace_str(destination_dir, new_folder=\"labels\"), \"test\")   \n    val_dir_label = os.path.join(replace_str(destination_dir, new_folder=\"labels\"), \"val\") \n    # print(\"@@@@@@@ train_dir_label: \", train_dir_label)  \n    # Create target directories\n    create_directory(train_dir_label)\n    create_directory(test_dir_label)\n    create_directory(val_dir_label)\n\n\n    # Get list of all files in the source directory\n    files = [f for f in os.listdir(source_dir) if os.path.isfile(os.path.join(source_dir, f))]\n    random.shuffle(files)  # Shuffle the files randomly\n\n    total_files = len(files)\n    train_count = int(total_files * train_pct / 100)\n    test_count = int(total_files * test_pct / 100)\n    val_count = total_files - train_count - test_count  # Remaining files go to validation\n\n    train_files = files[:train_count]\n    test_files = files[train_count:train_count + test_count]\n    val_files = files[train_count + test_count:]\n\n    # copy files to corresponding directories\n    for file in train_files:\n        shutil.copy(os.path.join(source_dir, file), os.path.join(train_dir, file))\n\n        shutil.copy(os.path.join(replace_str(source_dir, old_folder=image_folder, new_folder=label_folder), change_extension(file, \"txt\")), \n                    os.path.join(replace_str(train_dir_label, new_folder=\"labels\"), change_extension(file, \"txt\")))\n    for file in test_files:\n        shutil.copy(os.path.join(source_dir, file), os.path.join(test_dir, file))\n        shutil.copy(os.path.join(replace_str(source_dir, old_folder=image_folder, new_folder=label_folder), change_extension(file, \"txt\")), \n                    os.path.join(replace_str(test_dir_label, new_folder=\"labels\"), change_extension(file, \"txt\")))\n\n    for file in val_files:\n        shutil.copy(os.path.join(source_dir, file), os.path.join(val_dir, file))\n        shutil.copy(os.path.join(replace_str(source_dir, old_folder=image_folder, new_folder=label_folder), change_extension(file, \"txt\")), \n                    os.path.join(replace_str(val_dir_label, new_folder=\"labels\"), change_extension(file, \"txt\")))\n\n\n# Define paths\nsource_dir = '/kaggle/input/dlp-object-detection/final_dlp_data/final_dlp_data/train/images'\ndestination_dir = '/kaggle/working/images'\n\n# Define percentages\ntrain_pct = 95\ntest_pct = 0\nval_pct = 5\n\n# Split the data\nsplit_data(source_dir, destination_dir, train_pct, test_pct, image_folder = \"images\", label_folder = \"labels\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T05:14:33.86462Z","iopub.execute_input":"2024-12-06T05:14:33.864863Z","iopub.status.idle":"2024-12-06T05:16:40.360064Z","shell.execute_reply.started":"2024-12-06T05:14:33.864839Z","shell.execute_reply":"2024-12-06T05:16:40.359094Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install ultralytics","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T05:16:40.361307Z","iopub.execute_input":"2024-12-06T05:16:40.361972Z","iopub.status.idle":"2024-12-06T05:16:51.150274Z","shell.execute_reply.started":"2024-12-06T05:16:40.361932Z","shell.execute_reply":"2024-12-06T05:16:51.149192Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_yaml_path = \"/kaggle/working/config.yaml\"\nwith open(data_yaml_path, \"w\") as f:\n    f.write(\nf\"\"\"\npath: /kaggle/working/\n\ntrain: images/train\ntest: images/test\nval: images/val\n\n# classes\nnames:\n  0: aegypti\n  1: albopictus\n  2: anopheles\n  3: culex\n  4: culiseta\n  5: japonicus/koreicus\n \n\"\"\")\nprint(f\"config.yaml file created at {data_yaml_path}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T05:16:51.174023Z","iopub.execute_input":"2024-12-06T05:16:51.17439Z","iopub.status.idle":"2024-12-06T05:16:51.18762Z","shell.execute_reply.started":"2024-12-06T05:16:51.174343Z","shell.execute_reply":"2024-12-06T05:16:51.186787Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from ultralytics import YOLO\n\n# Load YOLO model\nconfig_path = \"config.yaml\"\n# model_checkpoint = \"yolov8m.pt\"  # Pretrained YOLOv8 model\nmodel_checkpoint = \"yolov5l.pt\"  # Pretrained YOLOv8 model\nmodel = YOLO(model_checkpoint)\n\nfor i, layer in enumerate(model.model.model):  # Access backbone layers\n    if i >= len(model.model.model) - 3:  # Unfreeze last 3 layers of the backbone\n        for param in layer.parameters():\n            param.requires_grad = True\n    else:  # Keep other layers frozen\n        for param in layer.parameters():\n            param.requires_grad = False\n\n# Set training with updated parameters\nmodel.train(\n    data=config_path,          # Path to data configuration\n    epochs=7,                 # Increase epochs for better learning\n    batch=16,                  # Moderate batch size\n    imgsz=640,                 # Input image size\n    device=0,                  # GPU\n    lr0=5e-4,                  # Reduced initial learning rate for stability\n    lrf=1e-6,                  # Smaller final learning rate for cosine annealing\n    weight_decay=1e-4,         # Lower weight decay to reduce over-regularization\n    label_smoothing=0.05,      # Slightly reduced label smoothing\n    augment=True,              # Enable augmentations\n    hsv_h=0.02,                # Adjusted HSV-Hue augmentation\n    hsv_s=0.5,                 # Moderate HSV-Saturation\n    hsv_v=0.3,                 # Moderate HSV-Value\n    flipud=0.2,                # Lower probability for vertical flips\n    fliplr=0.5,                # Retain horizontal flip\n    mosaic=0.5,                # Reduced mosaic augmentation\n    mixup=0.05,                # Reduced MixUp augmentation\n    project=\"YOLOv8_Training\",\n    name=\"yolov8_optimized\",\n)\n# Evaluate the trained model\nresults = model.val(data=config_path, conf=0.1, iou=0.5, task=\"val\")\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T08:43:14.258759Z","iopub.execute_input":"2024-12-06T08:43:14.259112Z","iopub.status.idle":"2024-12-06T09:27:20.917684Z","shell.execute_reply.started":"2024-12-06T08:43:14.25908Z","shell.execute_reply":"2024-12-06T09:27:20.916468Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Access mAP metrics\nmap50 = results.box.map50  # mAP@0.5\nmap50_95 = results.box.map  # mAP@0.5:0.95","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T09:27:50.896252Z","iopub.execute_input":"2024-12-06T09:27:50.897003Z","iopub.status.idle":"2024-12-06T09:27:50.902996Z","shell.execute_reply.started":"2024-12-06T09:27:50.896962Z","shell.execute_reply":"2024-12-06T09:27:50.902048Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from ultralytics import YOLO\nimport cv2\nimport numpy as np\nfrom matplotlib import pyplot as plt\n\n# Load the trained YOLO model\nmodel = YOLO(\"runs/detect/train/weights/best.pt\")  # Path to your trained weights\n\npath = \"/kaggle/input/dlp-object-detection/final_dlp_data/final_dlp_data/test/images/c8a84ab5-6414-4519-bc1c-d871e3f94d82.jpeg\"\n# Perform inference on an image\nresults = model.predict(source=path, conf=0.25)\n\n# Access the predictions from the results\npredictions = results[0].boxes  # You can access `boxes` to get the predicted bounding boxes\n\n# Read the input image\ninput_image = cv2.imread(path)\ninput_image = cv2.cvtColor(input_image, cv2.COLOR_BGR2RGB)\n\n# Class names (use the model's `names` attribute for mapping)\nclass_names = model.names  # This is a dictionary mapping class IDs to class names\n\n# Display the image with bounding boxes and class names\nplt.figure(figsize=(10, 10))\nplt.imshow(input_image)\nplt.axis('off')\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T21:00:32.777224Z","iopub.execute_input":"2024-12-05T21:00:32.777673Z","iopub.status.idle":"2024-12-05T21:00:33.963419Z","shell.execute_reply.started":"2024-12-05T21:00:32.777628Z","shell.execute_reply":"2024-12-05T21:00:33.962433Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!ls -l /kaggle/input/dlp-object-detection/final_dlp_data/final_dlp_data/test/images| wc -l","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T10:33:08.061267Z","iopub.execute_input":"2024-12-05T10:33:08.061655Z","iopub.status.idle":"2024-12-05T10:33:09.195482Z","shell.execute_reply.started":"2024-12-05T10:33:08.061622Z","shell.execute_reply":"2024-12-05T10:33:09.194252Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"path = \"/kaggle/input/dlp-object-detection/final_dlp_data/final_dlp_data/test/images\"\ntest_images = list()\nfor dirname, _, filenames in os.walk(path):\n\n    for file in filenames:\n        test_images.append(os.path.join(dirname, file))\n\n    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T08:33:33.191197Z","iopub.execute_input":"2024-12-06T08:33:33.191588Z","iopub.status.idle":"2024-12-06T08:33:33.200624Z","shell.execute_reply.started":"2024-12-06T08:33:33.191556Z","shell.execute_reply":"2024-12-06T08:33:33.199746Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from PIL import Image  # To get image dimensions\n\n# Prepare submission\nsubmission_data = []\ninvalid_images = list()\nerror_lst = list()\ni = 0\nmodel = YOLO(\"YOLOv8_Training/yolov8_cosine_label_smooth2/weights/best.pt\")\nfor image_path in test_images:\n    try:\n        # Assuming `image_path` is the path to the image\n        img = Image.open(image_path)\n        img_width, img_height = img.size  # Get image dimensions\n        results = model(image_path)\n        box = results[0].boxes.xywh[0].cpu().tolist()\n        conf = results[0].boxes.conf[0].item()\n        cls = results[0].boxes.cls[0].item()\n        # for *box, conf, cls in zip(*results[0].boxes.xywh[0].cpu().tolist(), results[0].boxes.conf[0].item(), results[0].boxes.cls[0].item()):\n        submission_data.append({\n            'id': i,\n            'ImageID': image_path.split('.')[0].split('/')[-1],\n            'LabelName': model.names[int(cls)],\n            'Conf': conf,\n            'xcenter': box[0]/ img_width,\n            'ycenter': box[1]/ img_height,\n            'bbx_width': box[2]/ img_width,\n            'bbx_height': box[3]/ img_height\n        })\n    except Exception as e:\n        print(e)\n        print(image_path)\n        invalid_images.append(image_path)\n        error_lst.append(e)\n        temp = submission_data[-1].copy()\n        temp['id'] += 1\n        temp['ImageID'] = image_path.split('.')[0].split('/')[-1]\n        submission_data.append(temp)\n    i += 1\n# Save the results to CSV\nimport pandas as pd\nsubmission_df = pd.DataFrame(submission_data)\nsubmission_df.to_csv('21F1001709.csv', index=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"results[0].boxes.xywh[0].cpu().tolist()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"invalid_images","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T21:05:12.364541Z","iopub.execute_input":"2024-12-05T21:05:12.36492Z","iopub.status.idle":"2024-12-05T21:05:12.371665Z","shell.execute_reply.started":"2024-12-05T21:05:12.364891Z","shell.execute_reply":"2024-12-05T21:05:12.370774Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Read the input image\ninput_image = cv2.imread(invalid_images[2])\ninput_image = cv2.cvtColor(input_image, cv2.COLOR_BGR2RGB)\n\nplt.figure(figsize=(10, 10))\nplt.imshow(input_image)\nplt.axis('off')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T10:45:29.901364Z","iopub.execute_input":"2024-12-05T10:45:29.901748Z","iopub.status.idle":"2024-12-05T10:45:31.62887Z","shell.execute_reply.started":"2024-12-05T10:45:29.901716Z","shell.execute_reply":"2024-12-05T10:45:31.627962Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Loop through the images and plot them\nplt.figure(figsize=(20, 15))  # Adjust the figure size\nfor i, img_path in enumerate(invalid_images):\n    img = cv2.imread(img_path)  # Read the image\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)  # Convert BGR to RGB (for correct colors in Matplotlib)\n    \n    # Add a subplot for each image\n    plt.subplot(4, 4, i + 1)  # Create a grid (4 rows x 4 columns) for subplots\n    plt.imshow(img)  # Display the image\n    plt.axis('off')  # Turn off the axes\n    plt.title(f\"Image {i+1}\")  # Optional: Add a title\n\nplt.tight_layout()  # Adjust layout for better spacing\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T10:49:02.770674Z","iopub.execute_input":"2024-12-05T10:49:02.771461Z","iopub.status.idle":"2024-12-05T10:49:13.988519Z","shell.execute_reply.started":"2024-12-05T10:49:02.771427Z","shell.execute_reply":"2024-12-05T10:49:13.987458Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}