{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":10386,"databundleVersionId":862046,"sourceType":"competition"}],"dockerImageVersionId":31154,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =====================================================\n# 🧠 STEP 1 — IMPORTS & SETUP\n# =====================================================\nimport os\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport cv2\nfrom tqdm import tqdm\nfrom PIL import Image\n\nsns.set(style=\"whitegrid\")\nplt.rcParams[\"figure.figsize\"] = (10, 5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T16:40:48.968074Z","iopub.execute_input":"2025-10-16T16:40:48.968376Z","iopub.status.idle":"2025-10-16T16:40:50.366519Z","shell.execute_reply.started":"2025-10-16T16:40:48.968354Z","shell.execute_reply":"2025-10-16T16:40:50.365607Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =====================================================\n# 🧾 QUICK LOOK\n# =====================================================\nprint(\"\\n🔹 Bounding Boxes:\")\ndisplay(train_boxes.head())\nprint(\"\\n🔹 Human Labels:\")\ndisplay(train_labels.head())\nprint(\"\\n🔹 Class Descriptions:\")\ndisplay(classes.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T16:43:16.07454Z","iopub.execute_input":"2025-10-16T16:43:16.075208Z","iopub.status.idle":"2025-10-16T16:43:16.115396Z","shell.execute_reply.started":"2025-10-16T16:43:16.07518Z","shell.execute_reply":"2025-10-16T16:43:16.114493Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =====================================================\n# 📸 EDA — SAMPLE IMAGE WITH BOXES\n# =====================================================\n# Let's visualize 1 random image with its bounding boxes\n\n# Image folder (upload the images from stage_1_test_images.zip or train subset)\nimg_dir = \"/kaggle/input/inclusive-images-challenge/stage_1_test_images\"  # example path; adjust to your folder\n\nsample_id = train_boxes[\"ImageID\"].sample(1).values[0]\nsample_boxes = train_boxes[train_boxes[\"ImageID\"] == sample_id]\n\nimg_path = os.path.join(img_dir, f\"{sample_id}.jpg\")\nif os.path.exists(img_path):\n    img = cv2.imread(img_path)\n    if img is not None:\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        h, w, _ = img.shape\n        for _, row in sample_boxes.iterrows():\n            x1, y1, x2, y2 = int(row[\"XMin\"]*w), int(row[\"YMin\"]*h), int(row[\"XMax\"]*w), int(row[\"YMax\"]*h)\n            cv2.rectangle(img, (x1, y1), (x2, y2), (255,0,0), 2)\n            cv2.putText(img, row[\"DisplayName\"][:15], (x1, y1-5),\n                        cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255,255,0), 1, cv2.LINE_AA)\n        plt.imshow(img)\n        plt.title(f\"Sample image: {sample_id}\")\n        plt.axis(\"off\")\n        plt.show()\n    else:\n        print(\"⚠️ Image not found in given folder.\")\nelse:\n    print(\"⚠️ Sample image not found (please upload your image directory to Kaggle).\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T16:45:14.413412Z","iopub.execute_input":"2025-10-16T16:45:14.414195Z","iopub.status.idle":"2025-10-16T16:45:16.186606Z","shell.execute_reply.started":"2025-10-16T16:45:14.414168Z","shell.execute_reply":"2025-10-16T16:45:16.185886Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =====================================================\n# 🧠 STEP 1 — SETUP ENVIRONMENT\n# =====================================================\n!pip install ultralytics==8.2.64 roboflow opencv-python matplotlib seaborn -q\n\nimport os\nimport cv2\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom ultralytics import YOLO\n\n# Make project directories\nos.makedirs(\"/kaggle/working/inclusive_yolo/images/train\", exist_ok=True)\nos.makedirs(\"/kaggle/working/inclusive_yolo/images/val\", exist_ok=True)\nos.makedirs(\"/kaggle/working/inclusive_yolo/labels/train\", exist_ok=True)\nos.makedirs(\"/kaggle/working/inclusive_yolo/labels/val\", exist_ok=True)\nprint(\"✅ Directories created successfully!\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T16:45:48.241913Z","iopub.execute_input":"2025-10-16T16:45:48.242535Z","iopub.status.idle":"2025-10-16T16:47:25.796252Z","shell.execute_reply.started":"2025-10-16T16:45:48.242512Z","shell.execute_reply":"2025-10-16T16:47:25.79532Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =====================================================\n# 📁 LOAD DATA\n# =====================================================\n# Make sure you added the dataset in Kaggle under \"Add Data\" → choose Inclusive Images dataset\n# Example paths (adjust if different)\ntrain_boxes = pd.read_csv(\"/kaggle/input/inclusive-images-challenge/train_bounding_boxes.csv\")\ntrain_labels = pd.read_csv(\"/kaggle/input/inclusive-images-challenge/train_human_labels.csv\")\nclasses = pd.read_csv(\"/kaggle/input/inclusive-images-challenge/class-descriptions.csv\")\n\nprint(\"✅ Dataset loaded successfully!\")\nprint(\"Train boxes:\", train_boxes.shape)\nprint(\"Train labels:\", train_labels.shape)\nprint(\"Classes:\", classes.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T16:51:44.279175Z","iopub.execute_input":"2025-10-16T16:51:44.280116Z","iopub.status.idle":"2025-10-16T16:52:08.031658Z","shell.execute_reply.started":"2025-10-16T16:51:44.280086Z","shell.execute_reply":"2025-10-16T16:52:08.030799Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =====================================================\n# 🔍 STEP 3 — BASIC EDA\n# =====================================================\nprint(train_boxes.head())\nprint(train_labels.head())\n\nplt.figure(figsize=(10,4))\ntop_classes = train_labels['LabelName'].value_counts().head(10)\nsns.barplot(x=top_classes.index, y=top_classes.values)\nplt.xticks(rotation=90)\nplt.title(\"Top 10 Most Common Classes\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T16:52:11.06296Z","iopub.execute_input":"2025-10-16T16:52:11.063293Z","iopub.status.idle":"2025-10-16T16:52:11.996694Z","shell.execute_reply.started":"2025-10-16T16:52:11.063263Z","shell.execute_reply":"2025-10-16T16:52:11.995749Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =====================================================\n# 🧩 STEP 5 — CREATE DATA.YAML CONFIG\n# =====================================================\nyaml_content = \"\"\"\ntrain: /kaggle/working/inclusive_yolo/images/train\nval: /kaggle/working/inclusive_yolo/images/val\n\nnc: 10\nnames: ['person', 'car', 'bicycle', 'dog', 'cat', 'bus', 'chair', 'tree', 'sign', 'bag']\n\"\"\"\n\nwith open(\"/kaggle/working/inclusive_yolo/data.yaml\", \"w\") as f:\n    f.write(yaml_content)\n\nprint(\"✅ data.yaml created!\")\n\n# =====================================================\n# 🧠 STEP 6 — TRAIN YOLOv8n MODEL\n# =====================================================\nmodel = YOLO(\"yolov8n.pt\")  # nano version (lightweight)\nmodel.train(\n    data=\"/kaggle/working/inclusive_yolo/data.yaml\",\n    epochs=20,\n    imgsz=640,\n    batch=16,\n    device=0\n)\n\n# =====================================================\n# 📊 STEP 7 — EVALUATE MODEL\n# =====================================================\nmetrics = model.val()\nprint(metrics)\n\n# Show prediction sample\nresults = model.predict(source=\"/kaggle/working/inclusive_yolo/images/val\", save=True, conf=0.3)\n\n# =====================================================\n# 🔄 STEP 8 — EXPORT TO TFLITE\n# =====================================================\nexport_path = model.export(format=\"tflite\", dynamic=True, optimize=True)\nprint(\"✅ Model exported to TFLite:\", export_path)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T16:55:01.513271Z","iopub.execute_input":"2025-10-16T16:55:01.514031Z","iopub.status.idle":"2025-10-16T17:01:36.236055Z","shell.execute_reply.started":"2025-10-16T16:55:01.514005Z","shell.execute_reply":"2025-10-16T17:01:36.234628Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Install necessary libraries\n!pip install google-cloud-storage ultralytics opencv-python-headless pandas matplotlib numpy scikit-learn tensorflow\n\n# Authenticate with Google Cloud (if needed for gsutil)\n# You'll need to upload your Google Cloud service account key (JSON file) to Kaggle for authentication.\n# Upload it as a dataset or directly in the notebook files section.\nfrom google.cloud import storage\n!echo \"Your Google Cloud credentials JSON file should be uploaded to /kaggle/input/your-credentials.json\"\n!gcloud auth activate-service-account --key-file=/kaggle/input/your-credentials.json  # Replace with your file path\n\n# Verify installations\nimport ultralytics\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport cv2\nimport numpy as np\nimport tensorflow as tf\nprint(\"Libraries installed successfully!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T17:02:13.507715Z","iopub.execute_input":"2025-10-16T17:02:13.508438Z","iopub.status.idle":"2025-10-16T17:02:52.31746Z","shell.execute_reply.started":"2025-10-16T17:02:13.50841Z","shell.execute_reply":"2025-10-16T17:02:52.316329Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport shutil\n\n# Specify your dataset path (replace 'your_dataset_name' with the actual name, e.g., 'inclusive-images-challenge')\ndataset_input_path = '/kaggle/input/inclusive-images-challenge'  # Example: '/kaggle/input/inclusive-images-challenge/'\n\n# Create working directory\n!mkdir -p /kaggle/working/dataset\n!mkdir -p /kaggle/working/dataset/images  # For image files, including stage_1_test_images\n\n# Copy key files and folders from input to working directory\nkey_files = ['train_bounding_boxes.csv', 'train_human_labels.csv', 'class-descriptions.csv']  # List of files\nkey_folders = ['stage_1_test_images']  # Assuming this is a folder with JPG images\n\n# Copy files\nfor file in key_files:\n    source_file = os.path.join(dataset_input_path, file)\n    if os.path.exists(source_file):\n        shutil.copy(source_file, '/kaggle/working/dataset/')\n    else:\n        print(f\"Warning: {file} not found in {dataset_input_path}\")\n\n# Copy folder (e.g., stage_1_test_images)\nfor folder in key_folders:\n    source_folder = os.path.join(dataset_input_path, folder)\n    if os.path.exists(source_folder) and os.path.isdir(source_folder):\n        # Copy the entire folder to /kaggle/working/dataset/images/\n        shutil.copytree(source_folder, os.path.join('/kaggle/working/dataset/images/', folder))\n        print(f\"Copied {folder} to /kaggle/working/dataset/images/\")\n    else:\n        print(f\"Warning: {folder} not found or not a folder in {dataset_input_path}\")\n\n# Verify the files and folders in your working directory\nprint(\"Files and folders in /kaggle/working/dataset:\")\n!ls /kaggle/working/dataset\nprint(\"Contents of /kaggle/working/dataset/images:\")\n!ls /kaggle/working/dataset/images\n\nprint(\"Dataset access and setup complete!\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T17:16:41.268692Z","iopub.execute_input":"2025-10-16T17:16:41.269163Z","iopub.status.idle":"2025-10-16T17:21:06.625167Z","shell.execute_reply.started":"2025-10-16T17:16:41.269136Z","shell.execute_reply":"2025-10-16T17:21:06.624001Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport os\nimport cv2\nimport shutil\nimport random\n\n# Load CSV files from the working directory\nbbox_df = pd.read_csv('/kaggle/working/dataset/train_bounding_boxes.csv')\nlabels_df = pd.read_csv('/kaggle/working/dataset/class-descriptions.csv')\n\nprint(f\"Loaded bbox_df with columns: {bbox_df.columns.tolist()}\")\nprint(f\"Loaded labels_df with columns: {labels_df.columns.tolist()}\")\n\n# Create directories for YOLO format\n!mkdir -p /kaggle/working/yolo_data/images/train\n!mkdir -p /kaggle/working/yolo_data/images/val\n!mkdir -p /kaggle/working/yolo_data/labels/train\n!mkdir -p /kaggle/working/yolo_data/labels/val\n\n# Map class descriptions to IDs\nif 'description' in labels_df.columns:\n    class_mapping = {row['description']: i for i, row in labels_df.iterrows()}\nelse:\n    raise ValueError(\"Column 'description' not found in class-descriptions.csv.\")\n\ndef convert_to_yolo_bbox(row, img_width, img_height):\n    x_min, x_max, y_min, y_max = row['XMin'], row['XMax'], row['YMin'], row['YMax']\n    x_center = (x_min + x_max) / 2.0\n    y_center = (y_min + y_max) / 2.0\n    width = x_max - x_min\n    height = y_max - y_min\n    return x_center, y_center, width, height\n\n# Limit to the first 100 ImageIDs to avoid crashes\nunique_image_ids = bbox_df['ImageID'].unique()[:100]\nprocessed_images = 0\n\nfor image_id in unique_image_ids:\n    # Adjust this path to your actual subfolder, e.g., 'stage_1_test_images/'\n    img_path = f'/kaggle/working/dataset/images/stage_1_test_images/{image_id}.jpg'  # Update based on your structure\n    print(f\"Checking for image: {img_path}\")\n    if os.path.exists(img_path):\n        img = cv2.imread(img_path)\n        if img is not None:\n            processed_images += 1\n            print(f\"Processing image: {image_id}.jpg\")\n            img_height, img_width = img.shape[:2]\n            sub_df = bbox_df[bbox_df['ImageID'] == image_id]\n            yolo_labels = []\n            for _, row in sub_df.iterrows():\n                if 'LabelName' in row:  # Ensure the column exists\n                    class_id = class_mapping.get(row['LabelName'], -1)\n                    if class_id != -1:\n                        x_center, y_center, width, height = convert_to_yolo_bbox(row, img_width, img_height)\n                        x_center /= img_width\n                        y_center /= img_height\n                        width /= img_width\n                        height /= img_height\n                        yolo_labels.append(f\"{class_id} {x_center} {y_center} {width} {height}\")\n                else:\n                    print(f\"Warning: 'LabelName' column not found for image {image_id}\")\n            if yolo_labels:\n                label_path = f'/kaggle/working/yolo_data/labels/train/{image_id}.txt'\n                with open(label_path, 'w') as f:\n                    for label in yolo_labels:\n                        f.write(label + '\\n')\n                shutil.copy(img_path, f'/kaggle/working/yolo_data/images/train/{image_id}.jpg')\n                print(f\"Copied {image_id}.jpg to /kaggle/working/yolo_data/images/train/\")\n    else:\n        print(f\"Warning: Image {image_id}.jpg not found at {img_path}\")\n\nprint(f\"Processed {processed_images} images. If this is 0, double-check your dataset upload and paths.\")\n\n# Split data into train/val only if there are images\nimages = [f for f in os.listdir('/kaggle/working/yolo_data/images/train') if f.endswith('.jpg')]\nprint(f\"Found {len(images)} images in /kaggle/working/yolo_data/images/train for splitting.\")\nif len(images) > 0:\n    random.seed(42)\n    random.shuffle(images)\n    split_idx = int(0.8 * len(images))\n    train_images = images[:split_idx]\n    val_images = images[split_idx:]\n\n    for img in train_images:\n        shutil.move(f'/kaggle/working/yolo_data/images/train/{img}', f'/kaggle/working/yolo_data/images/train_final/{img}')\n        shutil.move(f'/kaggle/working/yolo_data/labels/train/{img.replace(\".jpg\", \".txt\")}', f'/kaggle/working/yolo_data/labels/train_final/{img.replace(\".jpg\", \".txt\")}')\n        print(f\"Moved {img} to train_final\")\n\n    for img in val_images:\n        shutil.move(f'/kaggle/working/yolo_data/images/train/{img}', f'/kaggle/working/yolo_data/images/val/{img}')\n        shutil.move(f'/kaggle/working/yolo_data/labels/train/{img.replace(\".jpg\", \".txt\")}', f'/kaggle/working/yolo_data/labels/val/{img.replace(\".jpg\", \".txt\")}')\n        print(f\"Moved {img} to val\")\nelse:\n    print(\"No images found for splitting. You may need to manually verify and upload a subset of your dataset.\")\n\nprint(\"Preprocessing complete! Double-check the directories.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T18:40:48.157531Z","iopub.execute_input":"2025-10-16T18:40:48.157893Z","iopub.status.idle":"2025-10-16T18:41:09.162413Z","shell.execute_reply.started":"2025-10-16T18:40:48.157861Z","shell.execute_reply":"2025-10-16T18:41:09.161378Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport glob\n\n# Load the CSV to get ImageIDs\nbbox_df = pd.read_csv('/kaggle/working/dataset/train_bounding_boxes.csv')\nimage_ids = set(bbox_df['ImageID'].unique())  # Get unique ImageIDs\n\n# Scan for .jpg files in /kaggle/working/dataset/ and its subfolders\ndataset_path = '/kaggle/working/dataset/'  # Adjust if your dataset is in a different location\njpg_files = glob.glob(dataset_path + '/**/*.jpg', recursive=True)  # Recursive search for all .jpg files\n\nprint(f\"Found {len(jpg_files)} .jpg files in {dataset_path} and subfolders:\")\n\nmatching_files = []\nnon_matching_files = []\n\nfor file in jpg_files:\n    filename = os.path.basename(file)  # Get just the filename, e.g., '123456.jpg'\n    image_id = os.path.splitext(filename)[0]  # Remove .jpg to get '123456'\n    if image_id in image_ids:\n        matching_files.append(file)\n        print(f\"Matching file found: {file} (ID: {image_id})\")\n    else:\n        non_matching_files.append(file)\n        print(f\"Non-matching file: {file} (ID: {image_id} not in CSV)\")\n\nif matching_files:\n    print(f\"\\nTotal matching images: {len(matching_files)}\")\n    print(\"Example matching paths:\")\n    for path in matching_files[:5]:  # Show the first 5 for brevity\n        print(path)\n    print(\"Use one of these paths (e.g., the directory containing matching images) in Cell 3.\")\nelse:\n    print(\"No matching images found. Double-check your dataset upload or CSV.\")\n\nif non_matching_files:\n    print(f\"\\nWarning: {len(non_matching_files)} non-matching files found. These don't correspond to ImageIDs in your CSV.\")\n\nprint(\"If no files were found, verify your dataset upload in Kaggle's file browser.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T18:44:46.739381Z","iopub.execute_input":"2025-10-16T18:44:46.739718Z","iopub.status.idle":"2025-10-16T18:45:04.918086Z","shell.execute_reply.started":"2025-10-16T18:44:46.739693Z","shell.execute_reply":"2025-10-16T18:45:04.917085Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport os\n\nbbox_df = pd.read_csv('/kaggle/working/dataset/train_bounding_boxes.csv')\nimage_ids = set(bbox_df['ImageID'].unique())\nprint(f\"Unique ImageIDs from CSV: {list(image_ids)[:10]}... (first 10 for brevity)\")\n\n# Now, list the first 10 image filenames from your directory\nimage_directory = '/kaggle/working/dataset/images/stage_1_test_images/'  # Based on your output\nif os.path.exists(image_directory):\n    images = [f for f in os.listdir(image_directory) if f.endswith('.jpg')]\n    print(f\"First 10 image filenames: {images[:10]}\")\nelse:\n    print(f\"Directory {image_directory} not found.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T18:48:11.68074Z","iopub.execute_input":"2025-10-16T18:48:11.681125Z","iopub.status.idle":"2025-10-16T18:48:31.607179Z","shell.execute_reply.started":"2025-10-16T18:48:11.681102Z","shell.execute_reply":"2025-10-16T18:48:31.6064Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport os\nimport glob\nimport base64\nimport re  # For safe decoding\n\n# Load the CSV to get ImageIDs\nbbox_df = pd.read_csv('/kaggle/working/dataset/train_bounding_boxes.csv')\nimage_ids = set(bbox_df['ImageID'].unique())  # Get unique ImageIDs\nprint(f\"Unique ImageIDs from CSV: {list(image_ids)[:10]}... (first 10 for brevity)\")\n\n# Scan for .jpg files in /kaggle/working/dataset/ and subfolders\ndataset_path = '/kaggle/working/dataset/'  # Adjust if needed\njpg_files = glob.glob(dataset_path + '/**/*.jpg', recursive=True)\nprint(f\"Found {len(jpg_files)} .jpg files.\")\n\nmatching_files = []\nnon_matching_files = []\ndecoded_matches = []  # For potential decoded matches\n\nfor file in jpg_files:\n    filename = os.path.basename(file)  # e.g., '453277514e67452f42536f3d.jpg'\n    base_name = os.path.splitext(filename)[0]  # e.g., '453277514e67452f42536f3d'\n    \n    # Try base64 decoding the base_name\n    try:\n        decoded = base64.b64decode(base_name + '==').decode('utf-8')  # Add padding if needed\n        if decoded in image_ids:\n            matching_files.append(file)\n            decoded_matches.append((file, decoded))\n            print(f\"Matching file after decoding: {file} (Decoded ID: {decoded}, matches CSV ID)\")\n        else:\n            non_matching_files.append(file)\n            print(f\"Non-matching after decoding: {file} (Decoded to: {decoded}, not in CSV)\")\n    except (base64.b64DecodeError, UnicodeDecodeError):\n        non_matching_files.append(file)\n        print(f\"Decoding failed for: {file}\")\n\nif matching_files:\n    print(f\"\\nTotal matching images: {len(matching_files)}\")\n    print(\"Example matching paths:\")\n    for path, decoded_id in decoded_matches[:5]:\n        print(f\"Path: {path}, Decoded ID: {decoded_id}\")\n    print(\"Use the directory from matching paths in Cell 3 and adjust for decoding if needed.\")\nelse:\n    print(\"No matching images found even after decoding. The encoding might be different or not base64.\")\n\nif non_matching_files:\n    print(f\"\\nWarning: {len(non_matching_files)} non-matching files found. The filenames might use a different encoding.\")\n\nprint(\"If no matches, try adjusting the decoding or check for other patterns in the filenames.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T18:49:57.280384Z","iopub.execute_input":"2025-10-16T18:49:57.280687Z","iopub.status.idle":"2025-10-16T18:50:15.984569Z","shell.execute_reply.started":"2025-10-16T18:49:57.280667Z","shell.execute_reply":"2025-10-16T18:50:15.983425Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nprint(\"Contents of /kaggle/working/yolo_data/images/:\")\n!ls /kaggle/working/yolo_data/images/\nprint(\"Contents of /kaggle/working/yolo_data/images/train_final (if it exists):\")\n!ls /kaggle/working/yolo_data/images/train_final  # This might error if the directory doesn't exist","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T18:31:35.556842Z","iopub.execute_input":"2025-10-16T18:31:35.557254Z","iopub.status.idle":"2025-10-16T18:31:36.018625Z","shell.execute_reply.started":"2025-10-16T18:31:35.55722Z","shell.execute_reply":"2025-10-16T18:31:36.017636Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nprint(\"Contents of /kaggle/working/yolo_data/images/train:\")\n!ls /kaggle/working/yolo_data/images/train\nprint(\"Contents of /kaggle/working/yolo_data/images/val:\")\n!ls /kaggle/working/yolo_data/images/val","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T18:31:44.560114Z","iopub.execute_input":"2025-10-16T18:31:44.561001Z","iopub.status.idle":"2025-10-16T18:31:45.026221Z","shell.execute_reply.started":"2025-10-16T18:31:44.560959Z","shell.execute_reply":"2025-10-16T18:31:45.025227Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nfrom collections import Counter\nimport cv2\n\nbbox_df = pd.read_csv('/kaggle/working/dataset/train_bounding_boxes.csv')\n\n# Class distribution using the correct column\nclasses = bbox_df['LabelName'].value_counts()  # Changed from 'ClassName' to 'LabelName'\nplt.figure(figsize=(10,6))\nclasses.head(10).plot(kind='bar')\nplt.title('Top 10 Classes')\nplt.savefig('/kaggle/working/eda_class_distribution.png')\nplt.show()\n\n# Image and bounding box counts\nprint(f\"Total images: {len(bbox_df['ImageID'].unique())}\")\nprint(f\"Total bounding boxes: {len(bbox_df)}\")\n\n# Visualize a sample image (from the preprocessed path)\nsample_img_path = '/kaggle/working/yolo_data/images/train_final/sample.jpg'  # Replace with an actual file if needed\nif os.path.exists(sample_img_path):\n    img = cv2.imread(sample_img_path)\n    sub_df = bbox_df[bbox_df['ImageID'] == 'sample']  # Replace with actual ID\n    for _, row in sub_df.iterrows():\n        x_min, x_max, y_min, y_max = int(row['XMin']*img.shape[1]), int(row['XMax']*img.shape[1]), int(row['YMin']*img.shape[0]), int(row['YMax']*img.shape[0])\n        cv2.rectangle(img, (x_min, y_min), (x_max, y_max), (0,255,0), 2)\n    cv2.imwrite('/kaggle/working/eda_sample_image.png', img)\n    print(\"Sample image with bounding boxes saved.\")\n\n!ls /kaggle/working/  # Check output files\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T17:52:34.056335Z","iopub.execute_input":"2025-10-16T17:52:34.057385Z","iopub.status.idle":"2025-10-16T17:52:55.865923Z","shell.execute_reply.started":"2025-10-16T17:52:34.057351Z","shell.execute_reply":"2025-10-16T17:52:55.864968Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install wandb\nimport wandb\nwandb.login(key='cd140180dbb1272f6728b870e9ab35b3bc609ea6')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T18:04:17.796677Z","iopub.execute_input":"2025-10-16T18:04:17.79703Z","iopub.status.idle":"2025-10-16T18:04:21.808154Z","shell.execute_reply.started":"2025-10-16T18:04:17.797002Z","shell.execute_reply":"2025-10-16T18:04:21.807346Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nfrom ultralytics import YOLO\n\n# Load class descriptions to get the actual class names\nlabels_df = pd.read_csv('/kaggle/working/dataset/class-descriptions.csv')\n\n# Use 'description' column for class names\nif 'description' in labels_df.columns:\n    class_names = labels_df['description'].tolist()  # Get the list from 'description'\n    nc = len(class_names)  # Number of classes\nelse:\n    raise ValueError(\"Column 'description' not found in class-descriptions.csv. Check your CSV columns.\")\n\n# Now create the data.yaml file with the correct values\ndata_yaml_content = f\"\"\"\ntrain: /kaggle/working/yolo_data/images/train_final\nval: /kaggle/working/yolo_data/images/val\nnc: {nc}  # Automatically set to the number of classes\nnames: {class_names}  # Automatically set to the list of class names from 'description'\n\"\"\"\nwith open('/kaggle/working/data.yaml', 'w') as f:\n    f.write(data_yaml_content)\n\nprint(f\"data.yaml created with {nc} classes: {class_names[:5]}...\")  # Print the first few for verification\n\n# Train the model\n!yolo detect train data=/kaggle/working/data.yaml model=yolov8n.pt epochs=50 imgsz=416 device=0  # device=0 uses GPU","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T18:05:03.396268Z","iopub.execute_input":"2025-10-16T18:05:03.396641Z","iopub.status.idle":"2025-10-16T18:05:32.205257Z","shell.execute_reply.started":"2025-10-16T18:05:03.396613Z","shell.execute_reply":"2025-10-16T18:05:32.204181Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"","metadata":{"execution":{"iopub.status.busy":"2025-10-16T17:54:42.582403Z","iopub.execute_input":"2025-10-16T17:54:42.582726Z"}}},{"cell_type":"code","source":"!pip install ultralytics pandas matplotlib opencv-python-headless\nimport pandas as pd\nimport os\nimport cv2\nimport matplotlib.pyplot as plt\nfrom ultralytics import YOLO\n\nprint(\"Libraries installed successfully!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T19:49:12.762198Z","iopub.execute_input":"2025-10-16T19:49:12.762482Z","iopub.status.idle":"2025-10-16T19:50:52.768175Z","shell.execute_reply.started":"2025-10-16T19:49:12.762457Z","shell.execute_reply":"2025-10-16T19:50:52.767444Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\n# Specify your dataset name (e.g., 'inclusive-images-challenge')\ndataset_name = 'inclusive-images-challenge'  # Replace with your actual dataset name\nbase_path = f'/kaggle/input/{dataset_name}/'\n\nprint(\"Contents of your dataset directory:\")\n!ls {base_path}\n\n# Verify key files\nkey_files = ['train_bounding_boxes.csv', 'class-descriptions.csv']  # Add others if needed\nfor file in key_files:\n    if os.path.exists(os.path.join(base_path, file)):\n        print(f\"Found: {file}\")\n    else:\n        print(f\"Warning: {file} not found. Check your dataset upload.\")\n\n# Scan for images in subfolders\nimage_subfolders = ['stage_1_test_images', 'images']  # Adjust based on your dataset structure\nfor subfolder in image_subfolders:\n    subpath = os.path.join(base_path, subfolder)\n    if os.path.exists(subpath):\n        print(f\"Contents of {subpath}:\")\n        !ls {subpath}\n    else:\n        print(f\"Subfolder {subfolder} not found.\")\n\nprint(\"Dataset verification complete. Use this output to adjust paths in later cells if needed.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T19:50:55.171878Z","iopub.execute_input":"2025-10-16T19:50:55.172615Z","iopub.status.idle":"2025-10-16T19:50:56.071957Z","shell.execute_reply.started":"2025-10-16T19:50:55.172588Z","shell.execute_reply":"2025-10-16T19:50:56.071037Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\ndataset_name = 'inclusive-images-challenge'  # Replace with your actual dataset name, e.g., 'inclusive-images-challenge'\nbase_path = f'/kaggle/input/{dataset_name}/'\nprint(\"Full contents of your dataset directory:\")\n!ls {base_path}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T19:59:53.339744Z","iopub.execute_input":"2025-10-16T19:59:53.340443Z","iopub.status.idle":"2025-10-16T19:59:53.474328Z","shell.execute_reply.started":"2025-10-16T19:59:53.340403Z","shell.execute_reply":"2025-10-16T19:59:53.473591Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport os\nimport cv2\nimport random\nimport numpy as np\n\ndataset_name = 'inclusive-images-challenge'  # Replace with your actual dataset name\nbbox_path = f'/kaggle/input/{dataset_name}/train_bounding_boxes.csv'\nlabels_path = f'/kaggle/input/{dataset_name}/class-descriptions.csv'\nimage_directory = f'/kaggle/input/{dataset_name}/stage_1_test_images/'  # From your output\n\ntry:\n    bbox_df = pd.read_csv(bbox_path, engine='python', on_bad_lines='skip')\nexcept Exception as e:\n    print(f\"Error reading CSV: {e}\")\n    raise\n\nfor col in ['XMin', 'XMax', 'YMin', 'YMax']:\n    if col in bbox_df.columns:\n        bbox_df[col] = pd.to_numeric(bbox_df[col], errors='coerce')\nbbox_df = bbox_df.dropna(subset=['XMin', 'XMax', 'YMin', 'YMax'])\n\nlabels_df = pd.read_csv(labels_path, engine='python', on_bad_lines='skip')\n\nprint(f\"Loaded bbox_df with columns: {bbox_df.columns.tolist()}\")\nprint(f\"Loaded labels_df with columns: {labels_df.columns.tolist()}\")\n\nif 'description' in labels_df.columns:\n    class_mapping = {row['description']: i for i, row in labels_df.iterrows()}\nelse:\n    raise ValueError(\"Column 'description' not found in class-descriptions.csv.\")\n\ndef convert_to_yolo_bbox(row, img_width, img_height):\n    try:\n        x_min = float(row['XMin'])\n        x_max = float(row['XMax'])\n        y_min = float(row['YMin'])\n        y_max = float(row['YMax'])\n        x_center = (x_min + x_max) / 2.0\n        y_center = (y_min + y_max) / 2.0\n        width = x_max - x_min\n        height = y_max - y_min\n        return np.float64([x_center, y_center, width, height])\n    except ValueError as e:\n        print(f\"Error in row: {row}. {e}\")\n        return None\n\nunique_image_ids = bbox_df['ImageID'].unique()[:100]\nprocessed_images = 0\n\nfor image_id in unique_image_ids:\n    img_path = os.path.join(image_directory, f\"{image_id}.jpg\")\n    if os.path.exists(img_path):\n        img = cv2.imread(img_path)\n        if img is not None and isinstance(img, np.ndarray):\n            processed_images += 1\n            img_height, img_width = img.shape[:2]\n            sub_df = bbox_df[bbox_df['ImageID'] == image_id]\n            yolo_labels = []\n            for _, row in sub_df.iterrows():\n                if 'LabelName' in row and pd.notna(row['XMin']):\n                    class_id = class_mapping.get(row['LabelName'], -1)\n                    if class_id != -1:\n                        bbox = convert_to_yolo_bbox(row, img_width, img_height)\n                        if bbox is not None:\n                            x_center, y_center, width, height = bbox\n                            x_center /= img_width\n                            y_center /= img_height\n                            width /= img_width\n                            height /= img_height\n                            yolo_labels.append(f\"{class_id} {x_center} {y_center} {width} {height}\")\n            if yolo_labels:\n                label_path = f'/kaggle/working/yolo_data/labels/{image_id}.txt'\n                os.makedirs(os.path.dirname(label_path), exist_ok=True)\n                with open(label_path, 'w') as f:\n                    for label in yolo_labels:\n                        f.write(label + '\\n')\n    else:\n        print(f\"Warning: Image {image_id}.jpg not found at {img_path}\")\n\nprint(f\"Processed {processed_images} images.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T20:41:18.61709Z","iopub.execute_input":"2025-10-16T20:41:18.617368Z","iopub.status.idle":"2025-10-16T20:42:52.67838Z","shell.execute_reply.started":"2025-10-16T20:41:18.617349Z","shell.execute_reply":"2025-10-16T20:42:52.677361Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install ultralytics pandas opencv-python-headless","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T20:46:18.714814Z","iopub.execute_input":"2025-10-16T20:46:18.715299Z","iopub.status.idle":"2025-10-16T20:46:23.115023Z","shell.execute_reply.started":"2025-10-16T20:46:18.715275Z","shell.execute_reply":"2025-10-16T20:46:23.114255Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport os\n\ndataset_name = 'inclusive-images-challenge'  # Replace with your dataset name\nbbox_path = f'/kaggle/input/{dataset_name}/train_bounding_boxes.csv'\nlabels_path = f'/kaggle/input/{dataset_name}/class-descriptions.csv'\nimage_directory = f'/kaggle/input/{dataset_name}/stage_1_test_images/'\n\nlabels_df = pd.read_csv(labels_path, engine='python', on_bad_lines='skip')\nif 'description' in labels_df.columns:\n    class_names = labels_df['description'].tolist()\n    nc = len(class_names)\nelse:\n    raise ValueError(\"Column 'description' not found.\")\n\ndata_yaml = f\"\"\"\ntrain: {image_directory}\nval: {image_directory}\nnc: {nc}\nnames: {class_names}\n\"\"\"\nwith open('/kaggle/working/data.yaml', 'w') as f:\n    f.write(data_yaml)\n\nprint(\"Data prepared. Starting training next.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T20:47:21.02992Z","iopub.execute_input":"2025-10-16T20:47:21.03018Z","iopub.status.idle":"2025-10-16T20:47:21.09783Z","shell.execute_reply.started":"2025-10-16T20:47:21.030161Z","shell.execute_reply":"2025-10-16T20:47:21.096758Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"with open('/kaggle/working/data.yaml', 'w') as f:\n    f.write(\"\"\"\ntrain: /kaggle/input/your_dataset_name/stage_1_test_images/\nval: /kaggle/input/your_dataset_name/stage_1_test_images/\nnc: 80  # Replace with your actual number of classes (e.g., from class-descriptions.csv)\nnames: ['class1', 'class2', 'class3']  # Replace with your actual class names\n    \"\"\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T20:49:15.677011Z","iopub.execute_input":"2025-10-16T20:49:15.677562Z","iopub.status.idle":"2025-10-16T20:49:15.68167Z","shell.execute_reply.started":"2025-10-16T20:49:15.677538Z","shell.execute_reply":"2025-10-16T20:49:15.680838Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 3: Train the Model\n!yolo detect train data=/kaggle/working/data.yaml model=yolov8n.pt epochs=50 imgsz=416 device=0","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T20:49:51.524307Z","iopub.execute_input":"2025-10-16T20:49:51.52495Z","iopub.status.idle":"2025-10-16T20:49:57.161305Z","shell.execute_reply.started":"2025-10-16T20:49:51.524925Z","shell.execute_reply":"2025-10-16T20:49:57.160575Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Import necessary libraries\nimport os\nimport pandas as pd\nimport cv2  # For image processing\nimport numpy as np\nfrom ultralytics import YOLO  # For YOLOv8 model\nimport matplotlib.pyplot as plt  # For EDA visualizations\nimport seaborn as sns  # For EDA plots\nfrom sklearn.model_selection import train_test_split  # For splitting data\nimport shutil  # For file operations\n\nprint(\"Libraries imported successfully. You're ready to go in Kaggle!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T20:58:36.009471Z","iopub.execute_input":"2025-10-16T20:58:36.00979Z","iopub.status.idle":"2025-10-16T20:58:36.64909Z","shell.execute_reply.started":"2025-10-16T20:58:36.009766Z","shell.execute_reply":"2025-10-16T20:58:36.647868Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport cv2\nfrom tqdm import tqdm\n\n# Define base path\ndataset_path = '/kaggle/input/inclusive-images-challenge/'  # Adjust if needed\n\n# Load CSVs\nbounding_boxes_df = pd.read_csv(os.path.join(dataset_path, 'train_bounding_boxes.csv'))\nhuman_labels_df = pd.read_csv(os.path.join(dataset_path, 'train_human_labels.csv'))\nmachine_labels_df = pd.read_csv(os.path.join(dataset_path, 'train_machine_labels.csv'))\nclass_descriptions_df = pd.read_csv(os.path.join(dataset_path, 'class-descriptions.csv'), \n                                    header=None, names=['LabelName', 'Description'])\n\n# Preview\nprint(\"Bounding Boxes DataFrame Preview:\")\nprint(bounding_boxes_df.head())\nprint(f\"Total bounding boxes: {len(bounding_boxes_df):,}\")\n\n# Create class mapping\nclass_descriptions = dict(zip(class_descriptions_df['LabelName'], class_descriptions_df['Description']))\nlabel_to_id = {label: idx for idx, label in enumerate(class_descriptions_df['LabelName'])}\n\n# Define function to convert to YOLO format\ndef convert_to_yolo_format(df, images_dir, output_dir):\n    os.makedirs(output_dir, exist_ok=True)\n    missing_images = 0\n    total_saved = 0\n    \n    # Loop over unique image IDs\n    for image_id in tqdm(df['ImageID'].unique(), desc=\"Converting to YOLO format\"):\n        image_df = df[df['ImageID'] == image_id]\n        \n        image_path = os.path.join(images_dir, f\"{image_id}.jpg\")\n        if not os.path.exists(image_path):\n            missing_images += 1\n            continue\n        \n        # Try to read the image\n        image = cv2.imread(image_path)\n        if image is None:\n            missing_images += 1\n            continue\n        \n        img_height, img_width = image.shape[:2]\n        label_file_path = os.path.join(output_dir, f\"{image_id}.txt\")\n        \n        with open(label_file_path, 'w') as f:\n            for _, row in image_df.iterrows():\n                label = row['LabelName']\n                class_id = label_to_id[label]\n                \n                # The coordinates in OpenImages are already normalized [0,1]\n                x_min, x_max, y_min, y_max = row['XMin'], row['XMax'], row['YMin'], row['YMax']\n                \n                # Convert to YOLO format\n                x_center = (x_min + x_max) / 2\n                y_center = (y_min + y_max) / 2\n                width = x_max - x_min\n                height = y_max - y_min\n                \n                # Save YOLO annotation\n                f.write(f\"{class_id} {x_center:.6f} {y_center:.6f} {width:.6f} {height:.6f}\\n\")\n        \n        total_saved += 1\n\n    print(f\"\\n✅ Conversion complete! {total_saved} label files created.\")\n    print(f\"⚠️ Missing or unreadable images: {missing_images}\")\n\n# Unzip images if needed\nimages_dir = os.path.join(dataset_path, 'images/')\nif not os.path.exists(images_dir):\n    !unzip -q /kaggle/input/inclusive-images-public/stage_1_test_images.zip -d /kaggle/working/images/\n    images_dir = '/kaggle/working/images/'\n\n# Output YOLO labels directory\nyolo_labels_dir = '/kaggle/working/yolo_labels/'\n\n# Convert to YOLO\nconvert_to_yolo_format(bounding_boxes_df, images_dir, yolo_labels_dir)\nprint(\"\\nDataset preprocessing finished successfully ✅\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T21:01:04.750306Z","iopub.execute_input":"2025-10-16T21:01:04.750604Z","iopub.status.idle":"2025-10-16T21:01:20.346625Z","shell.execute_reply.started":"2025-10-16T21:01:04.750582Z","shell.execute_reply":"2025-10-16T21:01:20.345495Z"}},"outputs":[],"execution_count":null}]}