{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":128792,"databundleVersionId":15494745,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# ========== INSTALL YOLOv8 (SIMPLE) ==========\nprint(\"Installing YOLOv8...\")\n\n# Force install without dependency checks\n!pip install ultralytics --no-deps -q 2>/dev/null || pip install ultralytics -q\n\nprint(\"✅ YOLOv8 installed\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-30T13:50:05.484827Z","iopub.execute_input":"2026-01-30T13:50:05.485093Z","iopub.status.idle":"2026-01-30T13:50:09.259464Z","shell.execute_reply.started":"2026-01-30T13:50:05.485063Z","shell.execute_reply":"2026-01-30T13:50:09.2585Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ========== IMPORT LIBRARIES ==========\nimport os\nimport json\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport shutil\nfrom tqdm import tqdm\nimport warnings\nwarnings.filterwarnings('ignore')\n\nprint(\"🚀 Vista26 Competition - Full YOLO Training\")\nprint(\"=\" * 60)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-30T13:50:25.16605Z","iopub.execute_input":"2026-01-30T13:50:25.166631Z","iopub.status.idle":"2026-01-30T13:50:25.77846Z","shell.execute_reply.started":"2026-01-30T13:50:25.166596Z","shell.execute_reply":"2026-01-30T13:50:25.777879Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ========== SETUP PATHS ==========\nprint(\"📊 Setting up paths...\")\n\n# Base paths\nbase_path = \"/kaggle/input/vista26/Vistas Dataset Public/Vistas Dataset Public\"\n\n# JSON files\ntrain_json = os.path.join(base_path, \"instances_train.json\")\ntest_json = os.path.join(base_path, \"instances_test.json\")  \nval_json = \"/kaggle/input/vista26/instances_val.json\"\ncategories_json = os.path.join(base_path, \"Categories.json\")\n\n# Image directories\ntrain_img_dir = os.path.join(base_path, \"train\")\nval_img_dir = os.path.join(base_path, \"validation\")\n\n# Load categories\nwith open(categories_json, 'r') as f:\n    categories_data = json.load(f)\n\ncategories_list = categories_data['categories']\nnum_categories = len(categories_list)\ncategory_map = {cat['id']: cat['name'] for cat in categories_list}\n\nprint(f\"✅ Loaded {num_categories} categories\")\nprint(f\"Sample: {[cat['name'] for cat in categories_list[:3]]}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-30T13:51:32.098792Z","iopub.execute_input":"2026-01-30T13:51:32.099785Z","iopub.status.idle":"2026-01-30T13:51:32.118351Z","shell.execute_reply.started":"2026-01-30T13:51:32.099757Z","shell.execute_reply":"2026-01-30T13:51:32.117747Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ========== CREATE YOLO DATASET STRUCTURE ==========\nprint(\"\\n📊 Creating YOLO dataset structure...\")\n\n# Create YOLO directory structure\nyolo_dir = \"/kaggle/working/yolo_dataset\"\ntrain_images_dir = os.path.join(yolo_dir, \"images\", \"train\")\ntrain_labels_dir = os.path.join(yolo_dir, \"labels\", \"train\")\nval_images_dir = os.path.join(yolo_dir, \"images\", \"val\")\nval_labels_dir = os.path.join(yolo_dir, \"labels\", \"val\")\n\n# Create directories\nfor dir_path in [train_images_dir, train_labels_dir, val_images_dir, val_labels_dir]:\n    os.makedirs(dir_path, exist_ok=True)\n    print(f\"Created: {dir_path}\")\n\nprint(\"✅ YOLO directory structure created\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-30T13:51:46.488344Z","iopub.execute_input":"2026-01-30T13:51:46.488915Z","iopub.status.idle":"2026-01-30T13:51:46.495026Z","shell.execute_reply.started":"2026-01-30T13:51:46.48889Z","shell.execute_reply":"2026-01-30T13:51:46.494285Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ========== CONVERT COCO TO YOLO FORMAT ==========\nprint(\"\\n📊 Converting COCO format to YOLO format...\")\n\ndef convert_coco_to_yolo(json_path, image_dir, images_output_dir, labels_output_dir, max_images=5000):\n    \"\"\"Convert COCO JSON annotations to YOLO format\"\"\"\n    \n    # Load COCO data\n    with open(json_path, 'r') as f:\n        coco_data = json.load(f)\n    \n    # Create mapping from category ID to YOLO class index (0-indexed)\n    category_id_to_yolo = {}\n    for idx, cat in enumerate(categories_list):\n        category_id_to_yolo[cat['id']] = idx\n    \n    # Create image ID to file mapping\n    image_id_to_file = {}\n    for img in coco_data['images']:\n        image_id_to_file[img['id']] = img['file_name']\n    \n    # Group annotations by image\n    annotations_by_image = {}\n    for ann in coco_data['annotations']:\n        img_id = ann['image_id']\n        if img_id not in annotations_by_image:\n            annotations_by_image[img_id] = []\n        annotations_by_image[img_id].append(ann)\n    \n    # Process images\n    processed_count = 0\n    \n    for img_id, annotations in tqdm(list(annotations_by_image.items())[:max_images], desc=\"Converting\"):\n        # Get image file\n        if img_id not in image_id_to_file:\n            continue\n            \n        img_file = image_id_to_file[img_id]\n        img_path = os.path.join(image_dir, img_file)\n        \n        # Check if image exists\n        if not os.path.exists(img_path):\n            continue\n        \n        # Copy image to YOLO images directory\n        dst_img_path = os.path.join(images_output_dir, img_file)\n        shutil.copy2(img_path, dst_img_path)\n        \n        # Get image dimensions\n        img = cv2.imread(img_path)\n        if img is None:\n            continue\n        \n        img_height, img_width = img.shape[:2]\n        \n        # Create YOLO annotation file\n        label_file = img_file.replace('.jpg', '.txt').replace('.jpeg', '.txt').replace('.png', '.txt')\n        label_path = os.path.join(labels_output_dir, label_file)\n        \n        with open(label_path, 'w') as f:\n            for ann in annotations:\n                # Get category and bbox\n                category_id = ann['category_id']\n                bbox = ann['bbox']  # [x, y, width, height]\n                \n                # Convert to YOLO format: class x_center y_center width height (normalized 0-1)\n                x_center = (bbox[0] + bbox[2] / 2) / img_width\n                y_center = (bbox[1] + bbox[3] / 2) / img_height\n                width = bbox[2] / img_width\n                height = bbox[3] / img_height\n                \n                # Get YOLO class index\n                yolo_class = category_id_to_yolo.get(category_id)\n                if yolo_class is not None:\n                    # Write to label file\n                    f.write(f\"{yolo_class} {x_center:.6f} {y_center:.6f} {width:.6f} {height:.6f}\\n\")\n        \n        processed_count += 1\n    \n    return processed_count\n\n# Convert training data\nprint(\"Converting training data...\")\ntrain_count = convert_coco_to_yolo(\n    train_json, \n    train_img_dir, \n    train_images_dir, \n    train_labels_dir,\n    max_images=10000  # Limit for speed\n)\n\nprint(f\"✅ Converted {train_count} training images to YOLO format\")\n\n# Note: For validation, we don't have annotations, so we'll just use a subset for training validation\n# We'll use 20% of training as validation\nprint(\"\\nCreating validation split from training data...\")\n\n# List all training images\nall_train_images = os.listdir(train_images_dir)\nnum_val = int(len(all_train_images) * 0.2)\n\n# Move some to validation\nval_images = all_train_images[:num_val]\nfor img_file in tqdm(val_images, desc=\"Creating validation set\"):\n    # Move image\n    src_img = os.path.join(train_images_dir, img_file)\n    dst_img = os.path.join(val_images_dir, img_file)\n    shutil.move(src_img, dst_img)\n    \n    # Move corresponding label\n    label_file = img_file.replace('.jpg', '.txt').replace('.jpeg', '.txt').replace('.png', '.txt')\n    src_label = os.path.join(train_labels_dir, label_file)\n    dst_label = os.path.join(val_labels_dir, label_file)\n    if os.path.exists(src_label):\n        shutil.move(src_label, dst_label)\n\nprint(f\"✅ Created validation set with {len(val_images)} images\")\nprint(f\"Training set: {len(os.listdir(train_images_dir))} images\")\nprint(f\"Validation set: {len(os.listdir(val_images_dir))} images\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-30T13:52:03.270133Z","iopub.execute_input":"2026-01-30T13:52:03.270419Z","iopub.status.idle":"2026-01-30T13:56:19.288559Z","shell.execute_reply.started":"2026-01-30T13:52:03.270396Z","shell.execute_reply":"2026-01-30T13:56:19.287963Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ========== CREATE YOLO DATASET CONFIG ==========\nprint(\"\\n📊 Creating YOLO dataset configuration...\")\n\n# Create dataset.yaml\ndataset_yaml = f\"\"\"\n# Vista26 Dataset\npath: {yolo_dir}\ntrain: images/train\nval: images/val\n\n# Number of classes\nnc: {num_categories}\n\n# Class names\nnames: {[cat['name'] for cat in categories_list]}\n\"\"\"\n\n# Write to file\nyaml_path = os.path.join(yolo_dir, \"dataset.yaml\")\nwith open(yaml_path, \"w\") as f:\n    f.write(dataset_yaml)\n\nprint(f\"✅ Created dataset.yaml at: {yaml_path}\")\nprint(f\"Number of classes: {num_categories}\")\nprint(f\"Training images: {len(os.listdir(train_images_dir))}\")\nprint(f\"Validation images: {len(os.listdir(val_images_dir))}\")\n\n# Show sample of dataset structure\nprint(\"\\n📁 Dataset structure:\")\n!find /kaggle/working/yolo_dataset -type f | head -20","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-30T13:57:29.792614Z","iopub.execute_input":"2026-01-30T13:57:29.792952Z","iopub.status.idle":"2026-01-30T13:57:29.939894Z","shell.execute_reply.started":"2026-01-30T13:57:29.792929Z","shell.execute_reply":"2026-01-30T13:57:29.939264Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ========== TRAIN YOLOv8 MODEL ==========\nprint(\"\\n🎯 Training YOLOv8 model...\")\n\nfrom ultralytics import YOLO\nimport torch\n\n# Check GPU\nprint(f\"CUDA Available: {torch.cuda.is_available()}\")\nif torch.cuda.is_available():\n    print(f\"GPU: {torch.cuda.get_device_name(0)}\")\n    print(f\"GPU Memory: {torch.cuda.get_device_properties(0).total_memory / 1e9:.2f} GB\")\n\n# Load YOLO model (nano version for speed, you can change to 'yolov8s.pt' or 'yolov8m.pt' for better accuracy)\nmodel = YOLO('yolov8n.pt')  # Start with pre-trained nano model\n\nprint(\"Starting YOLOv8 training...\")\n\n# Train the model\nresults = model.train(\n    # Data configuration\n    data=yaml_path,\n    \n    # Model parameters\n    model='yolov8n.pt',\n    epochs=30,  # Train for 30 epochs\n    imgsz=640,  # Image size\n    batch=16,   # Batch size (reduce if out of memory)\n    \n    # Training parameters\n    lr0=0.01,   # Initial learning rate\n    lrf=0.01,   # Final learning rate factor\n    momentum=0.937,\n    weight_decay=0.0005,\n    warmup_epochs=3,\n    warmup_momentum=0.8,\n    warmup_bias_lr=0.1,\n    \n    # Augmentation\n    hsv_h=0.015,  # Hue augmentation\n    hsv_s=0.7,    # Saturation augmentation\n    hsv_v=0.4,    # Value augmentation\n    degrees=0.0,  # Rotation\n    translate=0.1,  # Translation\n    scale=0.5,    # Scale\n    shear=0.0,    # Shear\n    perspective=0.0,  # Perspective\n    flipud=0.0,   # Flip up-down\n    fliplr=0.5,   # Flip left-right\n    mosaic=1.0,   # Mosaic augmentation (helps with multi-object detection)\n    mixup=0.0,    # Mixup augmentation\n    \n    # Optimization\n    optimizer='SGD',  # SGD usually works better than Adam for YOLO\n    patience=10,  # Early stopping patience\n    \n    # Device\n    device=0 if torch.cuda.is_available() else 'cpu',\n    workers=2,\n    \n    # Saving\n    save=True,\n    save_period=5,  # Save every 5 epochs\n    project='/kaggle/working/runs',\n    name='train',\n    exist_ok=True,\n    \n    # Mixed precision for memory efficiency\n    amp=True,\n    \n    # Verbosity\n    verbose=True\n)\n\nprint(\"✅ YOLOv8 training complete!\")\n\n# Save the best model\nbest_model_path = \"/kaggle/working/best_yolo_model.pt\"\nif os.path.exists(\"/kaggle/working/runs/train/weights/best.pt\"):\n    shutil.copy(\"/kaggle/working/runs/train/weights/best.pt\", best_model_path)\n    print(f\"✅ Best model saved to: {best_model_path}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-30T14:45:15.8141Z","iopub.execute_input":"2026-01-30T14:45:15.814886Z","iopub.status.idle":"2026-01-30T16:24:16.215428Z","shell.execute_reply.started":"2026-01-30T14:45:15.814807Z","shell.execute_reply":"2026-01-30T16:24:16.21458Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ========== EVALUATE MODEL ==========\nprint(\"\\n📊 Evaluating model performance...\")\n\n# Load the best model\nmodel = YOLO(best_model_path)\n\n# Validate on validation set\nmetrics = model.val(\n    data=yaml_path,\n    batch=16,\n    imgsz=640,\n    device=0 if torch.cuda.is_available() else 'cpu',\n    split='val',\n    verbose=True\n)\n\nprint(f\"✅ Validation complete!\")\nprint(f\"mAP50-95: {metrics.box.map:.4f}\")\nprint(f\"mAP50: {metrics.box.map50:.4f}\")\nprint(f\"mAP75: {metrics.box.map75:.4f}\")\n\n# Show class-wise performance if available\nif hasattr(metrics, 'box') and hasattr(metrics.box, 'ap_class_index'):\n    print(\"\\n📈 Class-wise performance (top 10):\")\n    for i, class_idx in enumerate(metrics.box.ap_class_index[:10]):\n        class_name = categories_list[class_idx]['name'] if class_idx < len(categories_list) else f\"Class_{class_idx}\"\n        print(f\"  {class_name}: AP50={metrics.box.ap50[i]:.3f}, AP={metrics.box.ap[i]:.3f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-30T17:08:26.010946Z","iopub.execute_input":"2026-01-30T17:08:26.01153Z","iopub.status.idle":"2026-01-30T17:09:01.709731Z","shell.execute_reply.started":"2026-01-30T17:08:26.01149Z","shell.execute_reply":"2026-01-30T17:09:01.708921Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ========== LOAD VALIDATION JSON FOR CORRECT IMAGE IDs ==========\nprint(\"\\n📊 Loading validation JSON for correct image IDs...\")\n\n# Path to validation JSON\nval_json_path = \"/kaggle/input/vista26/instances_val.json\"\n\n# Load validation JSON\nwith open(val_json_path, 'r') as f:\n    val_data = json.load(f)\n\nprint(f\"Validation JSON loaded: {len(val_data['images'])} images\")\n\n# Create mapping from filename to image ID\nfilename_to_id = {}\nfor img_info in val_data['images']:\n    filename = img_info['file_name']\n    img_id = img_info['id']\n    filename_to_id[filename] = img_id\n\nprint(f\"Created mapping for {len(filename_to_id)} images\")\nprint(f\"Sample mapping: {list(filename_to_id.items())[:3]}\")\n\n# Also create reverse mapping for verification\nid_to_filename = {v: k for k, v in filename_to_id.items()}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-30T17:39:05.544403Z","iopub.execute_input":"2026-01-30T17:39:05.545193Z","iopub.status.idle":"2026-01-30T17:39:05.57902Z","shell.execute_reply.started":"2026-01-30T17:39:05.545163Z","shell.execute_reply":"2026-01-30T17:39:05.578427Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ========== RUN INFERENCE WITH CORRECT IMAGE IDs ==========\nprint(\"\\n🎯 Running inference with correct image IDs from JSON...\")\n\nfrom ultralytics import YOLO\nimport torch\nfrom tqdm import tqdm\n\n# Load the best model\nmodel = YOLO(\"/kaggle/working/best_yolo_model.pt\")\n\n# Get ALL validation images from the JSON (not from directory)\nval_image_infos = val_data['images']\nprint(f\"Total validation images from JSON: {len(val_image_infos)}\")\n\n# Configuration for inference\nconf_threshold = 0.25\niou_threshold = 0.45\n\n# Store predictions\nall_predictions_correct = {}\n\nprint(f\"Using confidence threshold: {conf_threshold}, IoU threshold: {iou_threshold}\")\n\n# Process ALL images with correct IDs\nprint(f\"Processing ALL {len(val_image_infos)} images...\")\n\n# Process in batches for efficiency\nbatch_size = 16\ntotal_batches = (len(val_image_infos) + batch_size - 1) // batch_size\n\nfor batch_idx in tqdm(range(0, len(val_image_infos), batch_size), \n                      desc=\"Processing batches\", \n                      total=total_batches):\n    batch_infos = val_image_infos[batch_idx:batch_idx+batch_size]\n    \n    # Prepare batch paths\n    batch_paths = []\n    batch_ids = []\n    for img_info in batch_infos:\n        img_path = os.path.join(val_dir, img_info['file_name'])\n        batch_paths.append(img_path)\n        batch_ids.append(img_info['id'])\n    \n    # Run inference\n    try:\n        results = model.predict(\n            source=batch_paths,\n            conf=conf_threshold,\n            iou=iou_threshold,\n            imgsz=640,\n            device=0 if torch.cuda.is_available() else 'cpu',\n            verbose=False\n        )\n        \n        # Process results\n        for result, img_id in zip(results, batch_ids):\n            # Get predictions\n            if result.boxes is not None and len(result.boxes) > 0:\n                # Get class indices and confidences\n                class_indices = result.boxes.cls.cpu().numpy().astype(int)\n                \n                # Convert YOLO class indices to original category IDs\n                predicted_cats = []\n                for idx in class_indices:\n                    if 0 <= idx < len(categories_list):\n                        category_id = categories_list[idx]['id']\n                        predicted_cats.append(category_id)\n                \n                # Remove duplicates and sort\n                predicted_cats = list(set(predicted_cats))\n                predicted_cats.sort()\n                \n                all_predictions_correct[img_id] = predicted_cats\n            else:\n                # No detections\n                all_predictions_correct[img_id] = []\n                \n    except Exception as e:\n        print(f\"Error processing batch {batch_idx}: {e}\")\n        # Add empty predictions for failed batch\n        for img_id in batch_ids:\n            all_predictions_correct[img_id] = []\n\nprint(f\"\\n✅ Processed {len(all_predictions_correct)} images\")\nprint(f\"Images with detections: {sum(1 for v in all_predictions_correct.values() if len(v) > 0)}\")\nprint(f\"Images without detections: {sum(1 for v in all_predictions_correct.values() if len(v) == 0)}\")\n\n# Check if we have all images\nif len(all_predictions_correct) == len(val_image_infos):\n    print(\"✅ Successfully processed ALL validation images!\")\nelse:\n    print(f\"⚠️ Missing {len(val_image_infos) - len(all_predictions_correct)} images\")\n    # Add missing images\n    for img_info in val_image_infos:\n        img_id = img_info['id']\n        if img_id not in all_predictions_correct:\n            all_predictions_correct[img_id] = [1, 2, 3]  # Default\n    print(f\"Added missing images, total now: {len(all_predictions_correct)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-30T17:41:53.888765Z","iopub.execute_input":"2026-01-30T17:41:53.889505Z","iopub.status.idle":"2026-01-30T17:46:43.810946Z","shell.execute_reply.started":"2026-01-30T17:41:53.889473Z","shell.execute_reply":"2026-01-30T17:46:43.810266Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ========== CREATE CSV WITH CORRECT IMAGE IDs ==========\nprint(\"\\n📊 Creating CSV with correct image IDs from JSON...\")\n\nimport json\nimport pandas as pd\n\n# Get all image IDs from JSON (they should be in the data)\nall_json_image_ids = [img['id'] for img in val_data['images']]\nprint(f\"Image IDs from JSON: {len(all_json_image_ids)}\")\n\n# Sort image_ids in ascending order\nsorted_image_ids = sorted(all_json_image_ids)\nprint(f\"Sorted {len(sorted_image_ids)} image IDs\")\n\n# Create submission data - MUST include ALL images from JSON\nsubmission_data = []\nmissing_count = 0\n\nfor img_id in sorted_image_ids:\n    # Get predictions if we have them\n    if img_id in all_predictions_correct:\n        preds = all_predictions_correct[img_id]\n    else:\n        preds = [1, 2, 3]  # Default if missing\n        missing_count += 1\n    \n    # If no predictions, use default categories\n    if not preds:\n        preds = [1, 2, 3]\n    \n    # Ensure categories are sorted ascending\n    preds = sorted(preds)\n    \n    # Convert to JSON string\n    categories_json = json.dumps(preds)\n    \n    submission_data.append([img_id, categories_json])\n\nif missing_count > 0:\n    print(f\"⚠️ Used default predictions for {missing_count} missing images\")\n\n# Create DataFrame\ndf_submission = pd.DataFrame(submission_data, columns=['image_id', 'categories'])\nprint(f\"Created DataFrame with {len(df_submission)} rows\")\n\n# Verify we have exactly the same number as in JSON\nif len(df_submission) == len(all_json_image_ids):\n    print(\"✅ CSV has exactly the same number of rows as validation JSON!\")\nelse:\n    print(f\"❌ Mismatch: CSV has {len(df_submission)} rows, JSON has {len(all_json_image_ids)}\")\n\n# Save to CSV\ncsv_path = \"/kaggle/working/submission_correct_ids.csv\"\ndf_submission.to_csv(csv_path, index=False)\nprint(f\"✅ CSV saved: {csv_path}\")\nprint(f\"File size: {os.path.getsize(csv_path)} bytes\")\n\n# Show sample\nprint(\"\\n📋 Sample (first 3 rows):\")\nprint(df_submission.head(3))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-30T17:46:47.181685Z","iopub.execute_input":"2026-01-30T17:46:47.182341Z","iopub.status.idle":"2026-01-30T17:46:47.219164Z","shell.execute_reply.started":"2026-01-30T17:46:47.182311Z","shell.execute_reply":"2026-01-30T17:46:47.218366Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ========== VERIFY SUBMISSION FORMAT ==========\nprint(\"\\n🔍 Verifying submission format matches competition requirements...\")\n\n# Check 1: CSV structure\nprint(\"1. Checking CSV structure...\")\nprint(f\"   Columns: {list(df_submission.columns)}\")\nprint(f\"   ✅ Must be ['image_id', 'categories']: {list(df_submission.columns) == ['image_id', 'categories']}\")\n\n# Check 2: Image IDs are integers\nprint(\"\\n2. Checking image IDs are integers...\")\nall_integers = df_submission['image_id'].apply(lambda x: isinstance(x, (int, np.integer))).all()\nprint(f\"   ✅ All image IDs are integers: {all_integers}\")\n\n# Check 3: Image IDs sorted ascending\nprint(\"\\n3. Checking image IDs sorted ascending...\")\nids_sorted = (df_submission['image_id'].diff().iloc[1:] > 0).all()\nprint(f\"   ✅ Image IDs sorted ascending: {ids_sorted}\")\n\n# Check 4: No duplicate image IDs\nprint(\"\\n4. Checking for duplicate image IDs...\")\nduplicates = df_submission['image_id'].duplicated().any()\nprint(f\"   ✅ No duplicate image IDs: {not duplicates}\")\n\n# Check 5: Categories are valid JSON and sorted\nprint(\"\\n5. Checking categories format...\")\nall_valid_json = True\nall_sorted = True\n\nfor idx, row in df_submission.head(10).iterrows():\n    try:\n        cats = json.loads(row['categories'])\n        if not isinstance(cats, list):\n            all_valid_json = False\n            print(f\"   ❌ Row {idx}: Not a list\")\n            break\n        if cats != sorted(cats):\n            all_sorted = False\n            print(f\"   ❌ Row {idx}: Categories not sorted: {cats}\")\n            break\n    except:\n        all_valid_json = False\n        print(f\"   ❌ Row {idx}: Invalid JSON\")\n        break\n\nprint(f\"   ✅ All categories are valid JSON: {all_valid_json}\")\nprint(f\"   ✅ All categories sorted ascending: {all_sorted}\")\n\n# Check 6: All image IDs from JSON are present\nprint(\"\\n6. Checking all validation images are included...\")\ncsv_ids = set(df_submission['image_id'])\njson_ids = set(all_json_image_ids)\nmissing_in_csv = json_ids - csv_ids\nextra_in_csv = csv_ids - json_ids\n\nprint(f\"   Total in JSON: {len(json_ids)}\")\nprint(f\"   Total in CSV: {len(csv_ids)}\")\nprint(f\"   Missing in CSV: {len(missing_in_csv)}\")\nprint(f\"   Extra in CSV: {len(extra_in_csv)}\")\n\nif len(missing_in_csv) == 0 and len(extra_in_csv) == 0:\n    print(\"   ✅ CSV contains exactly the same image IDs as validation JSON!\")\nelse:\n    print(\"   ❌ Mismatch found!\")\n\n# Check 7: Show exact format example\nprint(\"\\n7. Format example (first row):\")\nfirst_row = df_submission.iloc[0]\nprint(f\"   image_id: {first_row['image_id']} (type: {type(first_row['image_id'])})\")\nprint(f\"   categories: {first_row['categories']}\")\nprint(f\"   Parsed: {json.loads(first_row['categories'])}\")\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"📋 FINAL VERIFICATION SUMMARY\")\nprint(\"=\"*60)\n\nall_good = all([\n    list(df_submission.columns) == ['image_id', 'categories'],\n    all_integers,\n    ids_sorted,\n    not duplicates,\n    all_valid_json,\n    all_sorted,\n    len(missing_in_csv) == 0,\n    len(extra_in_csv) == 0\n])\n\nif all_good:\n    print(\"✅ ALL CHECKS PASSED! Submission is READY for Kaggle!\")\n    print(f\"📁 Submit this file: {csv_path}\")\nelse:\n    print(\"❌ Some checks failed. Please fix before submitting.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-30T17:46:54.400387Z","iopub.execute_input":"2026-01-30T17:46:54.400997Z","iopub.status.idle":"2026-01-30T17:46:54.418899Z","shell.execute_reply.started":"2026-01-30T17:46:54.400967Z","shell.execute_reply":"2026-01-30T17:46:54.418302Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ========== CREATE KAGGLE SUBMISSION JSON ==========\nprint(\"\\n🎯 Creating Kaggle submission JSON file...\")\n\nimport json\nimport pandas as pd\n\ndef create_kaggle_submission_json(all_predictions, val_json_path, output_path='submission.json'):\n    \"\"\"\n    Creates Kaggle competition submission JSON file with exact format\n    \n    Args:\n        all_predictions: dict of {image_id: [category_ids]}\n        val_json_path: path to validation JSON\n        output_path: output JSON file path\n    \"\"\"\n    \n    # Load validation data\n    print(f\"Loading validation data from: {val_json_path}\")\n    with open(val_json_path, 'r') as f:\n        val_data = json.load(f)\n    \n    # Get all validation image IDs in sorted order\n    val_images = val_data['images']\n    val_image_ids = sorted([img['id'] for img in val_images])\n    print(f\"Total validation images in JSON: {len(val_image_ids)}\")\n    \n    # Prepare submission list\n    submission_list = []\n    missing_predictions = 0\n    empty_predictions = 0\n    \n    for img_id in tqdm(val_image_ids, desc=\"Building submission JSON\"):\n        # Get predictions for this image\n        if img_id in all_predictions:\n            category_ids = all_predictions[img_id]\n            \n            # Remove the default [1, 2, 3] - this will hurt your score!\n            # Replace with empty list if it's the default\n            if category_ids == [1, 2, 3]:\n                category_ids = []\n                missing_predictions += 1\n        else:\n            # Image not in predictions - use empty list\n            category_ids = []\n            missing_predictions += 1\n        \n        # Count empty predictions\n        if len(category_ids) == 0:\n            empty_predictions += 1\n        \n        # Create submission entry\n        submission_list.append({\n            \"image_id\": int(img_id),\n            \"category_ids\": category_ids\n        })\n    \n    # Save to JSON file\n    print(f\"\\n📁 Saving submission to: {output_path}\")\n    with open(output_path, 'w') as f:\n        json.dump(submission_list, f, indent=2)\n    \n    # Also create a minified version for upload\n    minified_path = output_path.replace('.json', '_minified.json')\n    with open(minified_path, 'w') as f:\n        json.dump(submission_list, f, separators=(',', ':'))\n    \n    print(f\"✅ Created submission JSON with {len(submission_list)} entries\")\n    print(f\"   - Images with missing predictions: {missing_predictions}\")\n    print(f\"   - Images with empty predictions: {empty_predictions}\")\n    print(f\"   - Images with predictions: {len(submission_list) - empty_predictions}\")\n    print(f\"   - File saved: {output_path}\")\n    print(f\"   - Minified version: {minified_path}\")\n    \n    return submission_list\n\n# Create the submission JSON\nsubmission_json = create_kaggle_submission_json(\n    all_predictions=all_predictions_correct,\n    val_json_path=val_json_path,  # This should be \"/kaggle/input/vista26/instances_val.json\"\n    output_path='/kaggle/working/submission.json'\n)\n\n# Show sample of submission format\nprint(\"\\n📋 Sample submission entries (first 3):\")\nfor i in range(min(3, len(submission_json))):\n    print(f\"  {submission_json[i]}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-30T18:07:47.298219Z","iopub.execute_input":"2026-01-30T18:07:47.298546Z","iopub.status.idle":"2026-01-30T18:07:47.383904Z","shell.execute_reply.started":"2026-01-30T18:07:47.298516Z","shell.execute_reply":"2026-01-30T18:07:47.383281Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ========== CREATE KAGGLE SUBMISSION JSON ==========\nprint(\"\\n🎯 Creating Kaggle submission JSON file...\")\n\nimport json\nimport pandas as pd\nimport os\nfrom tqdm import tqdm\nfrom datetime import datetime\n\n# Define paths (based on your previous code)\nval_json_path = \"/kaggle/input/vista26/instances_val.json\"\nbest_model_path = \"/kaggle/working/best_yolo_model.pt\"\n\n# Load validation data\nprint(f\"Loading validation data from: {val_json_path}\")\nwith open(val_json_path, 'r') as f:\n    val_data = json.load(f)\n\n# Get all validation image IDs in sorted order\nval_images = val_data['images']\nval_image_ids = sorted([img['id'] for img in val_images])\nprint(f\"Total validation images in JSON: {len(val_image_ids)}\")\n\ndef create_kaggle_submission_json(all_predictions, val_image_infos, output_path='submission.json'):\n    \"\"\"\n    Creates Kaggle competition submission JSON file with exact format\n    \"\"\"\n    \n    # Prepare submission list\n    submission_list = []\n    missing_predictions = 0\n    empty_predictions = 0\n    \n    for img_info in tqdm(val_image_infos, desc=\"Building submission JSON\"):\n        img_id = img_info['id']\n        \n        # Get predictions for this image\n        if img_id in all_predictions:\n            category_ids = all_predictions[img_id]\n            \n            # Remove the default [1, 2, 3] - this will hurt your score!\n            # Replace with empty list if it's the default\n            if category_ids == [1, 2, 3]:\n                category_ids = []\n                missing_predictions += 1\n        else:\n            # Image not in predictions - use empty list\n            category_ids = []\n            missing_predictions += 1\n        \n        # Count empty predictions\n        if len(category_ids) == 0:\n            empty_predictions += 1\n        \n        # Create submission entry\n        submission_list.append({\n            \"image_id\": int(img_id),\n            \"category_ids\": sorted(list(set(category_ids)))  # Remove duplicates and sort\n        })\n    \n    # Save to JSON file\n    print(f\"\\n📁 Saving submission to: {output_path}\")\n    with open(output_path, 'w') as f:\n        json.dump(submission_list, f, indent=2)\n    \n    # Also create a minified version for upload\n    minified_path = output_path.replace('.json', '_minified.json')\n    with open(minified_path, 'w') as f:\n        json.dump(submission_list, f, separators=(',', ':'))\n    \n    print(f\"✅ Created submission JSON with {len(submission_list)} entries\")\n    print(f\"   - Images with missing predictions: {missing_predictions}\")\n    print(f\"   - Images with empty predictions: {empty_predictions}\")\n    print(f\"   - Images with predictions: {len(submission_list) - empty_predictions}\")\n    print(f\"   - File saved: {output_path}\")\n    print(f\"   - Minified version: {minified_path}\")\n    \n    return submission_list\n\n# Create the submission JSON using your all_predictions_correct dictionary\nprint(\"\\n📊 Creating submission from predictions...\")\nif 'all_predictions_correct' in globals():\n    submission_json = create_kaggle_submission_json(\n        all_predictions=all_predictions_correct,\n        val_image_infos=val_images,\n        output_path='/kaggle/working/submission.json'\n    )\nelse:\n    print(\"❌ ERROR: 'all_predictions_correct' dictionary not found!\")\n    print(\"Please run the inference cell first to generate predictions.\")\n    # Create an empty submission as fallback\n    submission_json = []\n    for img_info in val_images:\n        submission_json.append({\n            \"image_id\": int(img_info['id']),\n            \"category_ids\": []\n        })\n    with open('/kaggle/working/submission.json', 'w') as f:\n        json.dump(submission_json, f, indent=2)\n    print(\"⚠️ Created empty submission file as fallback.\")\n\n# Show sample of submission format\nprint(\"\\n📋 Sample submission entries (first 3):\")\nfor i in range(min(3, len(submission_json))):\n    print(f\"  {submission_json[i]}\")\n\n# ========== VALIDATE SUBMISSION JSON ==========\nprint(\"\\n🔍 Validating submission JSON file...\")\n\ndef validate_submission_json(submission_path, val_json_path):\n    \"\"\"Validate submission JSON against competition requirements\"\"\"\n    \n    try:\n        # Load files\n        with open(submission_path, 'r') as f:\n            submission = json.load(f)\n        \n        with open(val_json_path, 'r') as f:\n            val_data = json.load(f)\n        \n        # Get validation image IDs\n        val_image_ids = set(img['id'] for img in val_data['images'])\n        \n        # Validation checks\n        checks = {\n            \"Is valid JSON\": True,\n            \"Is a list\": isinstance(submission, list),\n            f\"Has {len(val_image_ids)} entries\": len(submission) == len(val_image_ids),\n        }\n        \n        # Check each entry\n        submission_ids = set()\n        valid_entries = True\n        \n        for i, entry in enumerate(submission):\n            # Check required fields\n            if not isinstance(entry, dict):\n                valid_entries = False\n                print(f\"❌ Entry {i} is not a dictionary\")\n                continue\n            \n            if 'image_id' not in entry:\n                valid_entries = False\n                print(f\"❌ Entry {i} missing 'image_id'\")\n                continue\n            \n            if 'category_ids' not in entry:\n                valid_entries = False\n                print(f\"❌ Entry {i} missing 'category_ids'\")\n                continue\n            \n            # Check data types\n            if not isinstance(entry['image_id'], int):\n                valid_entries = False\n                print(f\"❌ Entry {i}: image_id must be integer\")\n                continue\n            \n            if not isinstance(entry['category_ids'], list):\n                valid_entries = False\n                print(f\"❌ Entry {i}: category_ids must be list\")\n                continue\n            \n            # Check category IDs are integers\n            for cat_id in entry['category_ids']:\n                if not isinstance(cat_id, int):\n                    valid_entries = False\n                    print(f\"❌ Entry {i}: category_id {cat_id} is not integer\")\n                    break\n            \n            # Check for duplicates\n            if entry['image_id'] in submission_ids:\n                valid_entries = False\n                print(f\"❌ Duplicate image_id: {entry['image_id']}\")\n            submission_ids.add(entry['image_id'])\n        \n        checks[\"All entries have correct format\"] = valid_entries\n        checks[\"No duplicate image_ids\"] = len(submission_ids) == len(submission)\n        \n        # Check coverage\n        missing_ids = val_image_ids - submission_ids\n        extra_ids = submission_ids - val_image_ids\n        \n        checks[\"All validation IDs present\"] = len(missing_ids) == 0\n        checks[\"No extra IDs\"] = len(extra_ids) == 0\n        \n        # Print results\n        print(\"📋 Validation Results:\")\n        print(\"-\" * 40)\n        \n        for check, passed in checks.items():\n            status = \"✅\" if passed else \"❌\"\n            print(f\"{status} {check}\")\n        \n        print(\"-\" * 40)\n        \n        # Show details if issues\n        if missing_ids:\n            print(f\"\\n⚠️ Missing {len(missing_ids)} validation image IDs:\")\n            print(f\"   {sorted(list(missing_ids))[:10]}{'...' if len(missing_ids) > 10 else ''}\")\n        \n        if extra_ids:\n            print(f\"\\n⚠️ Contains {len(extra_ids)} extra image IDs:\")\n            print(f\"   {sorted(list(extra_ids))[:10]}{'...' if len(extra_ids) > 10 else ''}\")\n        \n        # Statistics\n        print(f\"\\n📊 Submission Statistics:\")\n        print(f\"   Total images: {len(submission)}\")\n        \n        images_with_preds = sum(1 for entry in submission if len(entry['category_ids']) > 0)\n        print(f\"   Images with predictions: {images_with_preds} ({images_with_preds/len(submission)*100:.1f}%)\")\n        \n        total_predictions = sum(len(entry['category_ids']) for entry in submission)\n        print(f\"   Total category predictions: {total_predictions}\")\n        \n        # Distribution of prediction counts\n        pred_counts = {}\n        for entry in submission:\n            count = len(entry['category_ids'])\n            pred_counts[count] = pred_counts.get(count, 0) + 1\n        \n        print(f\"\\n📈 Predictions per image distribution:\")\n        for count in sorted(pred_counts.keys())[:11]:  # Show first 11\n            print(f\"   {count} predictions: {pred_counts[count]} images\")\n        if len(pred_counts) > 11:\n            print(f\"   ... and {len(pred_counts)-11} more categories\")\n        \n        return all(checks.values()), images_with_preds, total_predictions\n        \n    except Exception as e:\n        print(f\"❌ Error during validation: {e}\")\n        return False, 0, 0\n\n# Run validation\nis_valid, images_with_preds, total_predictions = validate_submission_json(\n    submission_path='/kaggle/working/submission.json',\n    val_json_path=val_json_path\n)\n\n# ========== CREATE README FILE ==========\nprint(\"\\n📝 Creating README.md file...\")\n\n# Try to get metrics if available\ntry:\n    # Try to get metrics from your training/evaluation\n    map50_score = metrics.box.map50 if 'metrics' in globals() and hasattr(metrics, 'box') else 'N/A'\n    map_score = metrics.box.map if 'metrics' in globals() and hasattr(metrics, 'box') else 'N/A'\nexcept:\n    map50_score = 'N/A'\n    map_score = 'N/A'\n\n# Create timestamp for versioning\ntimestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n\n# Create submission directory\nsubmission_dir = f\"/kaggle/working/submission_{timestamp}\"\nos.makedirs(submission_dir, exist_ok=True)\n\n# Copy important files\nimport shutil\nfiles_to_copy = [\n    ('/kaggle/working/submission.json', f'{submission_dir}/submission.json'),\n    ('/kaggle/working/best_yolo_model.pt', f'{submission_dir}/best_yolo_model.pt'),\n]\n\nif os.path.exists('/kaggle/working/submission_correct_ids.csv'):\n    files_to_copy.append(('/kaggle/working/submission_correct_ids.csv', f'{submission_dir}/predictions.csv'))\n\nfor src, dst in files_to_copy:\n    if os.path.exists(src):\n        shutil.copy2(src, dst)\n        print(f\"✅ Copied: {src} -> {dst}\")\n    else:\n        print(f\"⚠️ File not found: {src}\")\n\n# Create README content\nreadme_content = f\"\"\"# Kaggle Submission - Vista26 Object Detection\n\n## Submission Details\n- **Timestamp**: {timestamp}\n- **Model**: YOLOv8 (best checkpoint)\n- **Confidence Threshold**: 0.25\n- **IoU Threshold**: 0.45\n- **Image Size**: 640x640\n\n## Files Included\n1. `submission.json` - Main Kaggle submission file (formatted)\n2. `best_yolo_model.pt` - Trained YOLO model weights\n3. `predictions.csv` - Raw predictions with image IDs (if available)\n\n## Model Performance\n- Training Epochs: 30\n- Validation mAP50: {map50_score}\n- Validation mAP50-95: {map_score}\n\n## Prediction Statistics\n- Total validation images: {len(val_image_ids)}\n- Images with predictions: {images_with_preds}\n- Images without predictions: {len(val_image_ids) - images_with_preds}\n- Total category predictions: {total_predictions}\n\n## Submission Instructions\n1. Upload `submission.json` to Kaggle competition page\n2. Ensure all validation image IDs are present\n3. Wait for scoring on public leaderboard\n\n## Model Details\n- Architecture: YOLOv8 with CSPDarknet53 backbone\n- Input size: 640x640\n- Data augmentation: Flip, rotation, mosaic\n- Optimizer: SGD with momentum\n- Learning rate: 0.01 with cosine annealing\n\n## Notes\n- This submission uses YOLOv8 with custom training on Vista26 dataset\n- Data augmentation was applied during training\n- Inference was run with confidence threshold 0.25\n- Model trained for 30 epochs\n\n## Contact\nFor questions about this submission, please refer to the Kaggle notebook.\n\n## License\nThis submission is for educational purposes as part of the Kaggle Vista26 competition.\n\"\"\"\n\n# Save README\nreadme_path = f'{submission_dir}/README.md'\nwith open(readme_path, 'w') as f:\n    f.write(readme_content)\n\nprint(f\"\\n✅ README created: {readme_path}\")\n\n# ========== FINAL CHECKLIST ==========\nprint(\"\\n\" + \"=\"*60)\nprint(\"FINAL KAGGLE SUBMISSION CHECKLIST\")\nprint(\"=\"*60)\n\nchecklist = {\n    \"1. Submission is valid JSON\": os.path.exists('/kaggle/working/submission.json'),\n    \"2. All validation image IDs are present\": is_valid,\n    \"3. No duplicate image IDs\": is_valid,\n    \"4. Each entry has 'image_id' and 'category_ids'\": is_valid,\n    \"5. category_ids are lists of integers\": is_valid,\n    \"6. File size is reasonable\": os.path.getsize('/kaggle/working/submission.json') < 10*1024*1024 if os.path.exists('/kaggle/working/submission.json') else False,\n    \"7. Model weights are saved\": os.path.exists('/kaggle/working/best_yolo_model.pt'),\n    \"8. README created\": os.path.exists(readme_path),\n}\n\nfor item, status in checklist.items():\n    status_icon = \"✅\" if status else \"❌\"\n    print(f\"{status_icon} {item}\")\n\nprint(\"=\"*60)\n\nif all(checklist.values()):\n    print(\"\\n🎉 READY FOR SUBMISSION!\")\n    print(\"\\n📤 Upload this file to Kaggle:\")\n    print(\"   /kaggle/working/submission.json\")\n    print(\"\\n📧 Don't forget to:\")\n    print(\"   1. Share your notebook with: ayush.kumar.cse22@itbhu.ac.in\")\n    print(\"   2. Save all outputs (Files -> Save Version)\")\n    print(\"   3. Submit via Kaggle competition page\")\nelse:\n    print(\"\\n⚠️ Some checks failed. Fix before submitting!\")\n\n# ========== CREATE DOWNLOAD LINKS ==========\nprint(\"\\n⬇️ Download Instructions:\")\nprint(\"To download your submission files:\")\nprint(\"1. Go to the right panel in Kaggle Notebook\")\nprint(\"2. Click on 'Data' tab\")\nprint(\"3. Navigate to /kaggle/working/\")\nprint(\"4. Download 'submission.json'\")\nprint(\"\\nOr use this code to create download links:\")\n\n# Create download links\nfrom IPython.display import FileLink, display\n\nif os.path.exists('/kaggle/working/submission.json'):\n    display(FileLink('/kaggle/working/submission.json', result_html_prefix=\"Download submission.json: \"))\nif os.path.exists('/kaggle/working/best_yolo_model.pt'):\n    display(FileLink('/kaggle/working/best_yolo_model.pt', result_html_prefix=\"Download model weights: \"))\nif os.path.exists('/kaggle/working/submission_correct_ids.csv'):\n    display(FileLink('/kaggle/working/submission_correct_ids.csv', result_html_prefix=\"Download predictions CSV: \"))\n\n# ========== FINAL FORMAT CHECK ==========\nprint(\"\\n🔬 Final format check for first 3 entries:\")\n\ntry:\n    with open('/kaggle/working/submission.json', 'r') as f:\n        sample_data = json.load(f)[:3]\n    \n    print(\"First 3 entries in submission.json:\")\n    for i, entry in enumerate(sample_data):\n        print(f\"\\nEntry {i+1}:\")\n        print(f\"  image_id: {entry['image_id']} (type: {type(entry['image_id']).__name__})\")\n        print(f\"  category_ids: {entry['category_ids']}\")\n        print(f\"  Count: {len(entry['category_ids'])} categories\")\n        \n        # Check if categories are sorted\n        if entry['category_ids'] == sorted(entry['category_ids']):\n            print(f\"  ✅ Categories are sorted\")\n        else:\n            print(f\"  ⚠️ Categories are NOT sorted\")\n            \n        # Check if no duplicates\n        if len(entry['category_ids']) == len(set(entry['category_ids'])):\n            print(f\"  ✅ No duplicates\")\n        else:\n            print(f\"  ⚠️ Has duplicates\")\nexcept Exception as e:\n    print(f\"Error checking format: {e}\")\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"SUBMISSION CREATION COMPLETE!\")\nprint(\"=\"*60)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-30T18:14:44.808655Z","iopub.execute_input":"2026-01-30T18:14:44.809001Z","iopub.status.idle":"2026-01-30T18:14:44.953646Z","shell.execute_reply.started":"2026-01-30T18:14:44.808973Z","shell.execute_reply":"2026-01-30T18:14:44.952968Z"}},"outputs":[],"execution_count":null}]}