{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":103103,"databundleVersionId":13042974,"sourceType":"competition"}],"dockerImageVersionId":31090,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import glob\nimport argparse\nimport shutil\n\n\nif __name__ == '__main__':\n    inp_folder = '/kaggle/input/alpha-dent/AlphaDent' + '/'\n    out_folder = '/kaggle/output/alpha-dent/AlphaDent_4_classes' + '/'\n\n    shutil.copytree(inp_folder, out_folder)\n\n    # Replace txt files\n    txt_paths = glob.glob(out_folder + '**/*.txt', recursive=True)\n    for txt_path in txt_paths:\n        lines = open(txt_path).readlines()\n        out = open(txt_path, 'w')\n        for line in lines:\n            if line[0] == '4' or line[0] == '5' or line[0] == '6' or line[0] == '7' or line[0] == '8':\n                out.write('3' + line[1:])\n            else:\n                out.write(line)\n        out.close()\n\n    id_to_classes = {\n        1: 'Abrasion',\n        2: 'Filling',\n        3: 'Crown',\n        4: 'Caries',\n    }\n\n    # Create .yaml file\n    out = open(out_folder + 'yolo_seg_train.yaml', 'w')\n    out.write('path: {}\\n'.format(out_folder))\n    out.write('train: images/train\\n')\n    out.write('val: images/valid\\n')\n    out.write('names:\\n')\n    out.write('  0: Abrasion\\n')\n    out.write('  1: Filling\\n')\n    out.write('  2: Crown\\n')\n    out.write('  3: Caries\\n')\n    out.close()\n\n\nprint(\"Done\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-06T17:47:46.233486Z","iopub.execute_input":"2025-09-06T17:47:46.233734Z","iopub.status.idle":"2025-09-06T17:49:05.578362Z","shell.execute_reply.started":"2025-09-06T17:47:46.233717Z","shell.execute_reply":"2025-09-06T17:49:05.577575Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport sys\nimport time\nimport glob\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom pathlib import Path\nfrom tqdm.auto import tqdm\nimport yaml\nimport random\nfrom PIL import Image\nimport warnings","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-06T17:49:29.832470Z","iopub.execute_input":"2025-09-06T17:49:29.833033Z","iopub.status.idle":"2025-09-06T17:49:31.147830Z","shell.execute_reply.started":"2025-09-06T17:49:29.833010Z","shell.execute_reply":"2025-09-06T17:49:31.147291Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport sys\nimport time\nimport glob\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom pathlib import Path\nfrom tqdm.auto import tqdm\nimport yaml\nimport random\nfrom PIL import Image\nimport shutil\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Disable wandb\nos.environ['WANDB_DISABLED'] = 'true'\n\n# Set random seeds for reproducibility\nrandom.seed(42)\nnp.random.seed(42)\n\n# Install required packages\nprint(\"Installing required packages...\")\nos.system('pip install -q ultralytics')\n\nimport torch\nfrom ultralytics import YOLO\n\n# Set deterministic behavior for PyTorch\ntorch.manual_seed(42)\nif torch.cuda.is_available():\n    torch.cuda.manual_seed(42)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\nprint(f'\\nPyTorch Version: {torch.__version__}')\nprint(f'CUDA Available: {torch.cuda.is_available()}')\nif torch.cuda.is_available():\n    print(f'CUDA Device: {torch.cuda.get_device_name(0)}')\n\n# Define original paths\nBASE_PATH = '/kaggle/output/alpha-dent/AlphaDent_4_classes'\nORIGINAL_TRAIN_IMAGES_PATH = f'{BASE_PATH}/images/train'\nVALID_IMAGES_PATH = f'{BASE_PATH}/images/valid'\nTEST_IMAGES_PATH = f'{BASE_PATH}/images/test'\nORIGINAL_TRAIN_LABELS_PATH = f'{BASE_PATH}/labels/train'\nVALID_LABELS_PATH = f'{BASE_PATH}/labels/valid'\n\n# Output paths\nOUTPUT_DIR = '/kaggle/working/'\nWEIGHTS_DIR = f'{OUTPUT_DIR}/weights'\nos.makedirs(WEIGHTS_DIR, exist_ok=True)\n\n# Create new dataset structure with 90/10 split\nNEW_DATASET_PATH = f'{OUTPUT_DIR}/alphadent_90_10_split'\nNEW_TRAIN_IMAGES_PATH = f'{NEW_DATASET_PATH}/images/train'\nNEW_EVAL_IMAGES_PATH = f'{NEW_DATASET_PATH}/images/eval'\nNEW_VALID_IMAGES_PATH = f'{NEW_DATASET_PATH}/images/valid'\nNEW_TEST_IMAGES_PATH = f'{NEW_DATASET_PATH}/images/test'\nNEW_TRAIN_LABELS_PATH = f'{NEW_DATASET_PATH}/labels/train'\nNEW_EVAL_LABELS_PATH = f'{NEW_DATASET_PATH}/labels/eval'\nNEW_VALID_LABELS_PATH = f'{NEW_DATASET_PATH}/labels/valid'\n\n# Create directories\nos.makedirs(NEW_TRAIN_IMAGES_PATH, exist_ok=True)\nos.makedirs(NEW_EVAL_IMAGES_PATH, exist_ok=True)\nos.makedirs(NEW_VALID_IMAGES_PATH, exist_ok=True)\nos.makedirs(NEW_TEST_IMAGES_PATH, exist_ok=True)\nos.makedirs(NEW_TRAIN_LABELS_PATH, exist_ok=True)\nos.makedirs(NEW_EVAL_LABELS_PATH, exist_ok=True)\nos.makedirs(NEW_VALID_LABELS_PATH, exist_ok=True)\n\nprint(\"\\n=== Creating 90/10 Train/Eval Split ===\")\n\n# Get all training images\noriginal_train_images = sorted(glob.glob(f'{ORIGINAL_TRAIN_IMAGES_PATH}/*.jpg'))\nprint(f\"Total original training images: {len(original_train_images)}\")\n\n# Shuffle and split into 90% train, 10% eval\nrandom.shuffle(original_train_images)\nsplit_idx = int(0.9 * len(original_train_images))\ntrain_90_images = original_train_images[:split_idx]\neval_10_images = original_train_images[split_idx:]\n\nprint(f\"90% for training: {len(train_90_images)}\")\nprint(f\"10% for evaluation: {len(eval_10_images)}\")\n\ndef copy_files(image_list, dest_images_dir, dest_labels_dir, source_labels_dir, desc):\n    \"\"\"Copy images and corresponding labels to destination directories.\"\"\"\n    for img_path in tqdm(image_list, desc=desc):\n        # Copy image\n        img_filename = os.path.basename(img_path)\n        shutil.copy2(img_path, os.path.join(dest_images_dir, img_filename))\n        \n        # Copy corresponding label\n        label_filename = img_filename.replace('.jpg', '.txt')\n        source_label_path = os.path.join(source_labels_dir, label_filename)\n        dest_label_path = os.path.join(dest_labels_dir, label_filename)\n        \n        if os.path.exists(source_label_path):\n            shutil.copy2(source_label_path, dest_label_path)\n\n# Copy 90% training data\ncopy_files(train_90_images, NEW_TRAIN_IMAGES_PATH, NEW_TRAIN_LABELS_PATH, \n          ORIGINAL_TRAIN_LABELS_PATH, \"Copying 90% training data\")\n\n# Copy 10% evaluation data\ncopy_files(eval_10_images, NEW_EVAL_IMAGES_PATH, NEW_EVAL_LABELS_PATH, \n          ORIGINAL_TRAIN_LABELS_PATH, \"Copying 10% evaluation data\")\n\n# Copy original validation data (unchanged)\noriginal_valid_images = glob.glob(f'{VALID_IMAGES_PATH}/*.jpg')\ncopy_files(original_valid_images, NEW_VALID_IMAGES_PATH, NEW_VALID_LABELS_PATH, \n          VALID_LABELS_PATH, \"Copying validation data\")\n\n# Copy test data (images only, no labels)\nprint(\"Copying test images...\")\ntest_images = glob.glob(f'{TEST_IMAGES_PATH}/*.jpg')\nfor img_path in tqdm(test_images, desc=\"Copying test images\"):\n    img_filename = os.path.basename(img_path)\n    shutil.copy2(img_path, os.path.join(NEW_TEST_IMAGES_PATH, img_filename))\n\n# Define class information\nCLASS_INFO = {\n    0: {'name': 'Abrasion', 'description': 'Teeth with mechanical wear of hard tissues'},\n    1: {'name': 'Filling', 'description': 'Dental fillings of various types'},\n    2: {'name': 'Crown', 'description': 'Dental crown (restoration)'},\n    3: {'name': 'Caries', 'description': 'Caries in fissures and pits'}\n}\n\n# Create YAML configuration for YOLO\nprint(\"\\n=== Creating YOLO Configuration ===\")\nyolo_config = {\n    'path': NEW_DATASET_PATH,\n    'train': 'images/train',\n    'val': 'images/valid',\n    'test': 'images/test',\n    'nc': 4,\n    'names': [CLASS_INFO[i]['name'] for i in range(4)]\n}\n\n# Save the configuration\nCUSTOM_YAML_PATH = f'{OUTPUT_DIR}/alphadent_config_90_10.yaml'\nwith open(CUSTOM_YAML_PATH, 'w') as f:\n    yaml.dump(yolo_config, f, default_flow_style=False)\nprint(f\"Created custom YAML config at: {CUSTOM_YAML_PATH}\")\n\n# Count images in new dataset\nnew_train_images = sorted(glob.glob(f'{NEW_TRAIN_IMAGES_PATH}/*.jpg'))\nnew_eval_images = sorted(glob.glob(f'{NEW_EVAL_IMAGES_PATH}/*.jpg'))\nnew_valid_images = sorted(glob.glob(f'{NEW_VALID_IMAGES_PATH}/*.jpg'))\nnew_test_images = sorted(glob.glob(f'{NEW_TEST_IMAGES_PATH}/*.jpg'))\n\nprint(f\"\\n=== New Dataset Statistics ===\")\nprint(f\"Training images (90%): {len(new_train_images)}\")\nprint(f\"Evaluation images (10%): {len(new_eval_images)}\")\nprint(f\"Validation images: {len(new_valid_images)}\")\nprint(f\"Test images: {len(new_test_images)}\")\n\n# Analyze class distribution\ndef analyze_class_distribution(labels_path, dataset_name):\n    \"\"\"Analyze class distribution in dataset.\"\"\"\n    class_counts = {i: 0 for i in range(4)}\n    total_annotations = 0\n    \n    label_files = glob.glob(f'{labels_path}/*.txt')\n    \n    for label_file in tqdm(label_files, desc=f\"Analyzing {dataset_name} labels\", leave=False):\n        if os.path.exists(label_file) and os.path.getsize(label_file) > 0:\n            try:\n                with open(label_file, 'r') as f:\n                    lines = f.readlines()\n                    for line in lines:\n                        if line.strip():\n                            parts = line.strip().split()\n                            if parts:\n                                class_id = int(parts[0])\n                                if 0 <= class_id < 4:\n                                    class_counts[class_id] += 1\n                                    total_annotations += 1\n            except Exception:\n                continue\n    \n    return class_counts, total_annotations\n\nprint(\"\\n=== Analyzing Class Distribution ===\")\ntrain_class_counts, train_total = analyze_class_distribution(NEW_TRAIN_LABELS_PATH, \"training\")\neval_class_counts, eval_total = analyze_class_distribution(NEW_EVAL_LABELS_PATH, \"evaluation\")\nvalid_class_counts, valid_total = analyze_class_distribution(NEW_VALID_LABELS_PATH, \"validation\")\n\nprint(f\"Training set (90%): {train_total} total annotations\")\nfor i, (class_id, count) in enumerate(train_class_counts.items()):\n    print(f\"  {CLASS_INFO[class_id]['name']}: {count}\")\n\nprint(f\"Evaluation set (10%): {eval_total} total annotations\")\nfor i, (class_id, count) in enumerate(eval_class_counts.items()):\n    print(f\"  {CLASS_INFO[class_id]['name']}: {count}\")\n\nprint(f\"Validation set: {valid_total} total annotations\")\n\n# Training configuration\nprint(\"\\n=== Model Training Configuration ===\")\nEPOCHS = 30\nIMAGE_SIZE = 640\nBATCH_SIZE = 8 if torch.cuda.is_available() else 4\nPATIENCE = 5\n\nprint(f\"Epochs: {EPOCHS}\")\nprint(f\"Image Size: {IMAGE_SIZE}\")\nprint(f\"Batch Size: {BATCH_SIZE}\")\nprint(f\"Early Stopping Patience: {PATIENCE}\")\n\n# Initialize and train model\nprint(\"\\n=== Starting Model Training ===\")\nmodel = YOLO('yolov8x-seg.pt')\n\n# Train the model with optimized parameters\nresults = model.train(\n    data=CUSTOM_YAML_PATH,\n    epochs=EPOCHS,\n    imgsz=IMAGE_SIZE,\n    batch=BATCH_SIZE,\n    patience=PATIENCE,\n    save=True,\n    save_period=10,\n    project=OUTPUT_DIR,\n    name='alphadent_yolov8x_90_10',\n    exist_ok=True,\n    pretrained=True,\n    optimizer='AdamW',\n    lr0=0.001,\n    lrf=0.01,\n    momentum=0.937,\n    weight_decay=0.0005,\n    warmup_epochs=3.0,\n    warmup_momentum=0.8,\n    warmup_bias_lr=0.1,\n    box=7.5,\n    cls=0.5,\n    dfl=1.5,\n    hsv_h=0.015,\n    hsv_s=0.7,\n    hsv_v=0.4,\n    degrees=0.0,\n    translate=0.1,\n    scale=0.5,\n    shear=0.0,\n    perspective=0.0,\n    flipud=0.0,\n    fliplr=0.5,\n    mosaic=1.0,\n    mixup=0.0,\n    copy_paste=0.0,\n    plots=True,\n    device=0 if torch.cuda.is_available() else 'cpu',\n    workers=2,\n    verbose=True,\n    amp=True,\n    val=True\n)\n\nprint(\"\\nTraining completed!\")\n\n# Load best model\nprint(\"\\n=== Loading Best Model ===\")\nbest_model_path = f'{OUTPUT_DIR}/alphadent_yolov8x_90_10/weights/best.pt'\nif os.path.exists(best_model_path):\n    model = YOLO(best_model_path)\n    print(f\"Loaded best model from: {best_model_path}\")\nelse:\n    last_model_path = f'{OUTPUT_DIR}/alphadent_yolov8x_90_10/weights/last.pt'\n    if os.path.exists(last_model_path):\n        model = YOLO(last_model_path)\n        print(f\"Loaded last model from: {last_model_path}\")\n    else:\n        print(\"Warning: No trained model found, using pretrained model\")\n        model = YOLO('yolov8x-seg.pt')\n\n# Validate model on original validation set\nprint(\"\\n=== Model Validation on Original Validation Set ===\")\ntry:\n    metrics = model.val(\n        data=CUSTOM_YAML_PATH,\n        imgsz=IMAGE_SIZE,\n        batch=1,\n        conf=0.001,\n        iou=0.5,\n        max_det=300,\n        device=0 if torch.cuda.is_available() else 'cpu',\n        plots=False,\n        save_json=False,\n    )\n    \n    print(f\"\\nValidation Results on Original Validation Set:\")\n    print(f\"mAP@50: {metrics.seg.map50:.4f}\")\n    print(f\"mAP@50-95: {metrics.seg.map:.4f}\")\nexcept Exception as e:\n    print(f\"Validation error (non-critical): {e}\")\n\n# Create custom YAML for evaluation on the 10% held-out data\neval_config = {\n    'path': NEW_DATASET_PATH,\n    'train': 'images/train',\n    'val': 'images/eval',  # Point to eval set for validation\n    'test': 'images/test',\n    'nc': 4,\n    'names': [CLASS_INFO[i]['name'] for i in range(4)]\n}\n\nEVAL_YAML_PATH = f'{OUTPUT_DIR}/alphadent_eval_config.yaml'\nwith open(EVAL_YAML_PATH, 'w') as f:\n    yaml.dump(eval_config, f, default_flow_style=False)\n\n# Evaluate model on the 10% held-out labeled data\nprint(\"\\n=== Model Evaluation on 10% Held-out Labeled Data ===\")\ntry:\n    eval_metrics = model.val(\n        data=EVAL_YAML_PATH,\n        imgsz=IMAGE_SIZE,\n        batch=1,\n        conf=0.001,\n        iou=0.5,\n        max_det=300,\n        device=0 if torch.cuda.is_available() else 'cpu',\n        plots=True,\n        save_json=True,\n        name='eval_10_percent'\n    )\n    \n    print(f\"\\nEvaluation Results on 10% Held-out Data:\")\n    print(f\"mAP@50: {eval_metrics.seg.map50:.4f}\")\n    print(f\"mAP@50-95: {eval_metrics.seg.map:.4f}\")\n    print(f\"Per-class mAP@50:\")\n    for i, map_val in enumerate(eval_metrics.seg.maps):\n        print(f\"  {CLASS_INFO[i]['name']}: {map_val:.4f}\")\n        \nexcept Exception as e:\n    print(f\"Evaluation error: {e}\")\n\nprint(\"\\n=== Summary ===\")\nprint(f\"✓ Successfully created 90/10 train/eval split\")\nprint(f\"✓ Trained model on 90% of original training data ({len(new_train_images)} images)\")\nprint(f\"✓ Evaluated model on 10% held-out labeled data ({len(new_eval_images)} images)\")\nprint(f\"✓ Also validated on original validation set ({len(new_valid_images)} images)\")\nprint(f\"✓ Model weights saved to: {OUTPUT_DIR}/alphadent_yolov8x_90_10/weights/\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-06T19:19:05.769183Z","iopub.execute_input":"2025-09-06T19:19:05.769541Z","iopub.status.idle":"2025-09-06T20:51:06.086719Z","shell.execute_reply.started":"2025-09-06T19:19:05.769503Z","shell.execute_reply":"2025-09-06T20:51:06.085917Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\ntorch.cuda.empty_cache()\n# Inference on test set\nprint(\"\\n=== Running Inference on Test Set ===\")\n\ndef convert_to_submission_format(results, image_paths):\n    submission_rows = []\n    \n    for idx, result in enumerate(results):\n        # Get image ID (filename without extension)\n        image_id = os.path.basename(image_paths[idx]).replace('.jpg', '')\n        \n        if result.masks is not None and len(result.masks) > 0:\n            try:\n                # Get masks, classes, and confidences\n                masks = result.masks.xy\n                classes = result.boxes.cls.cpu().numpy().astype(int)\n                confidences = result.boxes.conf.cpu().numpy()\n                h, w = result.orig_shape\n                \n                # Process each detection\n                for mask_idx in range(len(masks)):\n                    if mask_idx < len(classes) and mask_idx < len(confidences):\n                        polygon = masks[mask_idx]\n                        \n                        if len(polygon) >= 3:  # Valid polygon (at least 3 points)\n                            # Normalize coordinates to [0, 1]\n                            normalized_coords = []\n                            for point in polygon:\n                                x_norm = float(point[0]) / w\n                                y_norm = float(point[1]) / h\n                                # Ensure coordinates are within [0, 1]\n                                x_norm = max(0.0, min(1.0, x_norm))\n                                y_norm = max(0.0, min(1.0, y_norm))\n                                normalized_coords.extend([x_norm, y_norm])\n                            \n                            # Format polygon string\n                            poly_str = ' '.join([f'{coord:.6f}' for coord in normalized_coords])\n                            \n                            submission_rows.append({\n                                'patient_id': image_id,\n                                'class_id': int(classes[mask_idx]),\n                                'confidence': float(confidences[mask_idx]),\n                                'poly': poly_str\n                            })\n            except Exception as e:\n                print(f\"Error processing result for image {idx}: {e}\")\n                continue\n    \n    return submission_rows\n\n# Process test images\ntest_images = sorted(glob.glob(f'{TEST_IMAGES_PATH}/*.jpg'))\nall_submission_rows = []\nINFERENCE_BATCH_SIZE = 2 if torch.cuda.is_available() else 4\n\nprint(f\"Processing {len(test_images)} test images...\")\n\n# Process in batches\nfor i in tqdm(range(0, len(test_images), INFERENCE_BATCH_SIZE)):\n    torch.cuda.empty_cache()\n    batch_images = test_images[i:i + INFERENCE_BATCH_SIZE]\n    \n    try:\n        # Run inference\n        results = model.predict(\n            batch_images,\n            imgsz=IMAGE_SIZE,\n            conf=0.25,  # Confidence threshold\n            iou=0.45,   # NMS IoU threshold\n            max_det=300,\n            device=0 if torch.cuda.is_available() else 'cpu',\n            verbose=False,\n            agnostic_nms=True,\n            retina_masks=True,\n        )\n        \n        # Convert results to submission format\n        batch_rows = convert_to_submission_format(results, batch_images)\n        all_submission_rows.extend(batch_rows)\n        \n    except Exception as e:\n        print(f\"Error in batch {i//INFERENCE_BATCH_SIZE}: {e}\")\n        continue\n\nprint(f\"\\nGenerated {len(all_submission_rows)} predictions\")\n\n# Create submission DataFrame\nprint(\"\\n=== Creating Submission File ===\")\nsubmission_df = pd.DataFrame(all_submission_rows)\n\n# Ensure all test images have at least one prediction\nall_test_ids = [os.path.basename(img).replace('.jpg', '') for img in test_images]\nif len(submission_df) > 0:\n    predicted_ids = submission_df['patient_id'].unique()\n    missing_ids = set(all_test_ids) - set(predicted_ids)\nelse:\n    missing_ids = set(all_test_ids)\n\n# Add dummy predictions for images without detections\nif missing_ids:\n    print(f\"Adding dummy predictions for {len(missing_ids)} images without detections\")\n    dummy_rows = []\n    for img_id in missing_ids:\n        # Create a small dummy polygon\n        dummy_rows.append({\n            'patient_id': img_id,\n            'class_id': 0,  # Default to class 0 (Abrasion)\n            'confidence': 0.01,  # Very low confidence\n            'poly': '0.1 0.1 0.1 0.2 0.2 0.2 0.2 0.1'  # Small square polygon\n        })\n    \n    submission_df = pd.concat([submission_df, pd.DataFrame(dummy_rows)], ignore_index=True)\n\n# Sort by patient_id and then by confidence (descending)\nsubmission_df = submission_df.sort_values(['patient_id', 'confidence'], ascending=[True, False])\n\n# Ensure correct column order\nsubmission_df = submission_df[['patient_id', 'class_id', 'confidence', 'poly']]\n\n# Save the main submission file\nsubmission_df.to_csv('submission.csv', index=False)\nprint(\"Main submission file created: submission.csv\")\n\n# Verify submission format\nprint(\"\\n=== Verifying Submission Format ===\")\nprint(f\"Total predictions: {len(submission_df)}\")\nprint(f\"Unique images: {submission_df['patient_id'].nunique()}\")\nprint(f\"All test images included: {submission_df['patient_id'].nunique() == len(test_images)}\")\n\n# Display first few rows\nprint(\"\\nFirst 5 rows of submission:\")\nprint(submission_df.head())\n\n# Check for any potential issues\nprint(\"\\n=== Checking for Potential Issues ===\")\n\n# Check for missing test images\nmissing_in_submission = set(all_test_ids) - set(submission_df['patient_id'].unique())\nif missing_in_submission:\n    print(f\"WARNING: Missing images in submission: {missing_in_submission}\")\nelse:\n    print(\"✓ All test images have predictions\")\n\n# Check class distribution\nprint(\"\\nPredictions per class:\")\nclass_dist = submission_df['class_id'].value_counts().sort_index()\nfor class_id, count in class_dist.items():\n    if 0 <= class_id < 9:\n        print(f\"  Class {class_id} ({CLASS_INFO[class_id]['name']}): {count}\")\n\n# Check confidence distribution\nprint(f\"\\nConfidence statistics:\")\nprint(f\"  Min: {submission_df['confidence'].min():.4f}\")\nprint(f\"  Max: {submission_df['confidence'].max():.4f}\")\nprint(f\"  Mean: {submission_df['confidence'].mean():.4f}\")\nprint(f\"  Median: {submission_df['confidence'].median():.4f}\")\n\n# Create alternative submission with higher confidence threshold\nprint(\"\\n=== Creating Alternative Submission (Higher Confidence) ===\")\nhigh_conf_df = submission_df[submission_df['confidence'] >= 0.3].copy()\n\n# Ensure all images still have at least one prediction\nhigh_conf_ids = high_conf_df['patient_id'].unique()\nmissing_high_conf = set(all_test_ids) - set(high_conf_ids)\n\nif missing_high_conf:\n    # Add the highest confidence prediction for each missing image\n    for img_id in missing_high_conf:\n        img_preds = submission_df[submission_df['patient_id'] == img_id]\n        if len(img_preds) > 0:\n            # Add the highest confidence prediction\n            high_conf_df = pd.concat([high_conf_df, img_preds.head(1)], ignore_index=True)\n        else:\n            # Add dummy prediction\n            dummy_row = pd.DataFrame([{\n                'patient_id': img_id,\n                'class_id': 0,\n                'confidence': 0.01,\n                'poly': '0.1 0.1 0.1 0.2 0.2 0.2 0.2 0.1'\n            }])\n            high_conf_df = pd.concat([high_conf_df, dummy_row], ignore_index=True)\n\nhigh_conf_df = high_conf_df.sort_values(['patient_id', 'confidence'], ascending=[True, False])\nhigh_conf_df.to_csv('submission_high_conf.csv', index=False)\nprint(f\"Created high confidence submission with {len(high_conf_df)} predictions\")\n\nprint(\"\\n=== Pipeline Completed Successfully! ===\")\nprint(\"Submission files created:\")\nprint(\"  - submission.csv (main submission)\")\nprint(\"  - submission_high_conf.csv (alternative with higher confidence threshold)\")\nprint(\"\\nReady to submit to Kaggle!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-06T20:53:58.709702Z","iopub.execute_input":"2025-09-06T20:53:58.710583Z","iopub.status.idle":"2025-09-06T20:57:01.778508Z","shell.execute_reply.started":"2025-09-06T20:53:58.710550Z","shell.execute_reply":"2025-09-06T20:57:01.777528Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import cv2\nimport matplotlib.pyplot as plt\nfrom random import sample\nimport torch\n\n# Function to visualize predictions on images\ndef visualize_predictions(model, image_paths, class_names, num_samples=5, imgsz=640):\n    # Randomly pick some images\n    sample_paths = sample(image_paths, min(num_samples, len(image_paths)))\n\n    for img_path in sample_paths:\n        # Run prediction on the single image\n        results = model.predict(img_path, imgsz=imgsz, conf=0.25, iou=0.45, retina_masks=True, verbose=False)\n        result = results[0]\n\n        # Read image\n        img = cv2.imread(img_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n\n        # Draw boxes\n        if result.boxes is not None and len(result.boxes) > 0:\n            boxes = result.boxes.xyxy.cpu().numpy()  # x1, y1, x2, y2\n            classes = result.boxes.cls.cpu().numpy().astype(int)\n            confidences = result.boxes.conf.cpu().numpy()\n\n            for i, (box, cls, conf) in enumerate(zip(boxes, classes, confidences)):\n                x1, y1, x2, y2 = box.astype(int)\n                color = (0, 255, 0)  # green box\n                cv2.rectangle(img, (x1, y1), (x2, y2), color, 2)\n                label = f\"{class_names[cls]} {conf:.2f}\"\n                # Increase font size\n                cv2.putText(\n                    img,\n                    label,\n                    (x1, y1-5),\n                    cv2.FONT_HERSHEY_SIMPLEX,\n                    fontScale=5.0,  # increase from 0.5\n                    color=color,\n                    thickness=10\n                )\n\n\n        if result.masks is not None and len(result.masks) > 0:\n            masks = result.masks.data.cpu().numpy()  # HxW mask\n            for mask in masks:\n                # Convert mask to binary and apply color overlay\n                mask = (mask > 0.5).astype(np.uint8)\n                color_mask = np.zeros_like(img)\n                color_mask[:, :, 0] = mask * 255  # Red channel\n                img = cv2.addWeighted(img, 1.0, color_mask, 0.5, 0)\n\n        plt.figure(figsize=(8, 8))\n        plt.imshow(img)\n        plt.axis('off')\n        plt.title(f\"Prediction for {os.path.basename(img_path)}\")\n        plt.show()\n\n\nvisualize_predictions(model, test_images, [CLASS_INFO[i]['name'] for i in range(4)], num_samples=5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-06T18:55:01.665628Z","iopub.execute_input":"2025-09-06T18:55:01.665862Z","iopub.status.idle":"2025-09-06T18:55:30.695075Z","shell.execute_reply.started":"2025-09-06T18:55:01.665845Z","shell.execute_reply":"2025-09-06T18:55:30.694272Z"}},"outputs":[],"execution_count":null}]}