{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":103103,"databundleVersionId":13042974,"sourceType":"competition"}],"dockerImageVersionId":31090,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-07-29T18:28:14.640853Z","iopub.execute_input":"2025-07-29T18:28:14.641206Z","iopub.status.idle":"2025-07-29T18:28:18.626280Z","shell.execute_reply.started":"2025-07-29T18:28:14.641179Z","shell.execute_reply":"2025-07-29T18:28:18.625194Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport sys\nimport time\nimport glob\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom pathlib import Path\nfrom tqdm.auto import tqdm\nimport yaml\nimport random\nfrom PIL import Image\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Disable wandb\nos.environ['WANDB_DISABLED'] = 'true'\n\n# Set random seeds for reproducibility\nrandom.seed(42)\nnp.random.seed(42)\n\n# Install required packages\nprint(\"Installing required packages...\")\nos.system('pip install -q ultralytics')\n\nimport torch\nfrom ultralytics import YOLO\n\n# Set deterministic behavior for PyTorch\ntorch.manual_seed(42)\nif torch.cuda.is_available():\n    torch.cuda.manual_seed(42)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\nprint(f'\\nPyTorch Version: {torch.__version__}')\nprint(f'CUDA Available: {torch.cuda.is_available()}')\nif torch.cuda.is_available():\n    print(f'CUDA Device: {torch.cuda.get_device_name(0)}')\n\n# Define paths\nBASE_PATH = '/kaggle/input/alpha-dent/AlphaDent'\nTRAIN_IMAGES_PATH = f'{BASE_PATH}/images/train'\nVALID_IMAGES_PATH = f'{BASE_PATH}/images/valid'\nTEST_IMAGES_PATH = f'{BASE_PATH}/images/test'\nTRAIN_LABELS_PATH = f'{BASE_PATH}/labels/train'\nVALID_LABELS_PATH = f'{BASE_PATH}/labels/valid'\n\n# Output paths\nOUTPUT_DIR = '/kaggle/working/'\nWEIGHTS_DIR = f'{OUTPUT_DIR}/weights'\nos.makedirs(WEIGHTS_DIR, exist_ok=True)\n\n# Define class information\nCLASS_INFO = {\n    0: {'name': 'Abrasion', 'description': 'Teeth with mechanical wear of hard tissues'},\n    1: {'name': 'Filling', 'description': 'Dental fillings of various types'},\n    2: {'name': 'Crown', 'description': 'Dental crown (restoration)'},\n    3: {'name': 'Caries Class 1', 'description': 'Caries in fissures and pits'},\n    4: {'name': 'Caries Class 2', 'description': 'Caries on proximal surfaces of molars/premolars'},\n    5: {'name': 'Caries Class 3', 'description': 'Caries on proximal surfaces of incisors/canines without incisal edge'},\n    6: {'name': 'Caries Class 4', 'description': 'Caries on proximal surfaces of incisors/canines with incisal edge'},\n    7: {'name': 'Caries Class 5', 'description': 'Cervical caries (buccal/lingual surfaces)'},\n    8: {'name': 'Caries Class 6', 'description': 'Caries on incisal edges or cusps'}\n}\n\n# Create YAML configuration for YOLO\nprint(\"\\n=== Creating YOLO Configuration ===\")\nyolo_config = {\n    'path': BASE_PATH,\n    'train': 'images/train',\n    'val': 'images/valid',\n    'test': 'images/test',\n    'nc': 9,\n    'names': [CLASS_INFO[i]['name'] for i in range(9)]\n}\n\n# Save the configuration\nCUSTOM_YAML_PATH = f'{OUTPUT_DIR}/alphadent_config.yaml'\nwith open(CUSTOM_YAML_PATH, 'w') as f:\n    yaml.dump(yolo_config, f, default_flow_style=False)\nprint(f\"Created custom YAML config at: {CUSTOM_YAML_PATH}\")\n\n# Count images\ntrain_images = sorted(glob.glob(f'{TRAIN_IMAGES_PATH}/*.jpg'))\nvalid_images = sorted(glob.glob(f'{VALID_IMAGES_PATH}/*.jpg'))\ntest_images = sorted(glob.glob(f'{TEST_IMAGES_PATH}/*.jpg'))\n\nprint(f\"\\n=== Dataset Statistics ===\")\nprint(f\"Training images: {len(train_images)}\")\nprint(f\"Validation images: {len(valid_images)}\")\nprint(f\"Test images: {len(test_images)}\")\n\n# Analyze class distribution\ndef analyze_class_distribution(labels_path):\n    \"\"\"Analyze class distribution in dataset.\"\"\"\n    class_counts = {i: 0 for i in range(9)}\n    total_annotations = 0\n    \n    label_files = glob.glob(f'{labels_path}/*.txt')\n    \n    for label_file in tqdm(label_files, desc=\"Analyzing labels\", leave=False):\n        if os.path.exists(label_file) and os.path.getsize(label_file) > 0:\n            try:\n                with open(label_file, 'r') as f:\n                    lines = f.readlines()\n                    for line in lines:\n                        if line.strip():\n                            parts = line.strip().split()\n                            if parts:\n                                class_id = int(parts[0])\n                                if 0 <= class_id < 9:\n                                    class_counts[class_id] += 1\n                                    total_annotations += 1\n            except Exception:\n                continue\n    \n    return class_counts, total_annotations\n\nprint(\"\\n=== Analyzing Class Distribution ===\")\ntrain_class_counts, train_total = analyze_class_distribution(TRAIN_LABELS_PATH)\nvalid_class_counts, valid_total = analyze_class_distribution(VALID_LABELS_PATH)\n\nprint(f\"Training set: {train_total} total annotations\")\nprint(f\"Validation set: {valid_total} total annotations\")\n\n# Training configuration\nprint(\"\\n=== Model Training Configuration ===\")\nEPOCHS = 30\nIMAGE_SIZE = 640\nBATCH_SIZE = 8 if torch.cuda.is_available() else 4\nPATIENCE = 5\n\nprint(f\"Epochs: {EPOCHS}\")\nprint(f\"Image Size: {IMAGE_SIZE}\")\nprint(f\"Batch Size: {BATCH_SIZE}\")\nprint(f\"Early Stopping Patience: {PATIENCE}\")\n\n# Initialize and train model\nprint(\"\\n=== Starting Model Training ===\")\nmodel = YOLO('yolov8x-seg.pt')\n\n# Train the model with optimized parameters\nresults = model.train(\n    data=CUSTOM_YAML_PATH,\n    epochs=EPOCHS,\n    imgsz=IMAGE_SIZE,\n    batch=BATCH_SIZE,\n    patience=PATIENCE,\n    save=True,\n    save_period=10,\n    project=OUTPUT_DIR,\n    name='alphadent_yolov8x',\n    exist_ok=True,\n    pretrained=True,\n    optimizer='AdamW',\n    lr0=0.001,\n    lrf=0.01,\n    momentum=0.937,\n    weight_decay=0.0005,\n    warmup_epochs=3.0,\n    warmup_momentum=0.8,\n    warmup_bias_lr=0.1,\n    box=7.5,\n    cls=0.5,\n    dfl=1.5,\n    hsv_h=0.015,\n    hsv_s=0.7,\n    hsv_v=0.4,\n    degrees=0.0,\n    translate=0.1,\n    scale=0.5,\n    shear=0.0,\n    perspective=0.0,\n    flipud=0.0,\n    fliplr=0.5,\n    mosaic=1.0,\n    mixup=0.0,\n    copy_paste=0.0,\n    plots=True,\n    device=0 if torch.cuda.is_available() else 'cpu',\n    workers=2,\n    verbose=True,\n    amp=True,\n    val=True\n)\n\nprint(\"\\nTraining completed!\")\n\n# Load best model\nprint(\"\\n=== Loading Best Model ===\")\nbest_model_path = f'{OUTPUT_DIR}/alphadent_yolov8x/weights/best.pt'\nif os.path.exists(best_model_path):\n    model = YOLO(best_model_path)\n    print(f\"Loaded best model from: {best_model_path}\")\nelse:\n    last_model_path = f'{OUTPUT_DIR}/alphadent_yolov8x/weights/last.pt'\n    if os.path.exists(last_model_path):\n        model = YOLO(last_model_path)\n        print(f\"Loaded last model from: {last_model_path}\")\n    else:\n        print(\"Warning: No trained model found, using pretrained model\")\n        model = YOLO('yolov8x-seg.pt')\n\n# Validate model\nprint(\"\\n=== Model Validation ===\")\ntry:\n    metrics = model.val(\n        data=CUSTOM_YAML_PATH,\n        imgsz=IMAGE_SIZE,\n        batch=1,\n        conf=0.001,\n        iou=0.5,\n        max_det=300,\n        device=0 if torch.cuda.is_available() else 'cpu',\n        plots=False,\n        save_json=False,\n    )\n    \n    print(f\"\\nValidation Results:\")\n    print(f\"mAP@50: {metrics.seg.map50:.4f}\")\n    print(f\"mAP@50-95: {metrics.seg.map:.4f}\")\nexcept Exception as e:\n    print(f\"Validation error (non-critical): {e}\")\n\n# Inference on test set\nprint(\"\\n=== Running Inference on Test Set ===\")\n\ndef convert_to_submission_format(results, image_paths):\n    \"\"\"Convert YOLO results to competition submission format.\"\"\"\n    submission_rows = []\n    \n    for idx, result in enumerate(results):\n        # Get image ID (filename without extension)\n        image_id = os.path.basename(image_paths[idx]).replace('.jpg', '')\n        \n        if result.masks is not None and len(result.masks) > 0:\n            try:\n                # Get masks, classes, and confidences\n                masks = result.masks.xy\n                classes = result.boxes.cls.cpu().numpy().astype(int)\n                confidences = result.boxes.conf.cpu().numpy()\n                h, w = result.orig_shape\n                \n                # Process each detection\n                for mask_idx in range(len(masks)):\n                    if mask_idx < len(classes) and mask_idx < len(confidences):\n                        polygon = masks[mask_idx]\n                        \n                        if len(polygon) >= 3:  # Valid polygon (at least 3 points)\n                            # Normalize coordinates to [0, 1]\n                            normalized_coords = []\n                            for point in polygon:\n                                x_norm = float(point[0]) / w\n                                y_norm = float(point[1]) / h\n                                # Ensure coordinates are within [0, 1]\n                                x_norm = max(0.0, min(1.0, x_norm))\n                                y_norm = max(0.0, min(1.0, y_norm))\n                                normalized_coords.extend([x_norm, y_norm])\n                            \n                            # Format polygon string\n                            poly_str = ' '.join([f'{coord:.6f}' for coord in normalized_coords])\n                            \n                            submission_rows.append({\n                                'patient_id': image_id,\n                                'class_id': int(classes[mask_idx]),\n                                'confidence': float(confidences[mask_idx]),\n                                'poly': poly_str\n                            })\n            except Exception as e:\n                print(f\"Error processing result for image {idx}: {e}\")\n                continue\n    \n    return submission_rows\n\n# Process test images\ntest_images = sorted(glob.glob(f'{TEST_IMAGES_PATH}/*.jpg'))\nall_submission_rows = []\nINFERENCE_BATCH_SIZE = 8 if torch.cuda.is_available() else 4\n\nprint(f\"Processing {len(test_images)} test images...\")\n\n# Process in batches\nfor i in tqdm(range(0, len(test_images), INFERENCE_BATCH_SIZE)):\n    batch_images = test_images[i:i + INFERENCE_BATCH_SIZE]\n    \n    try:\n        # Run inference\n        results = model.predict(\n            batch_images,\n            imgsz=IMAGE_SIZE,\n            conf=0.25,  # Confidence threshold\n            iou=0.45,   # NMS IoU threshold\n            max_det=300,\n            device=0 if torch.cuda.is_available() else 'cpu',\n            verbose=False,\n            agnostic_nms=True,\n            retina_masks=True,\n        )\n        \n        # Convert results to submission format\n        batch_rows = convert_to_submission_format(results, batch_images)\n        all_submission_rows.extend(batch_rows)\n        \n    except Exception as e:\n        print(f\"Error in batch {i//INFERENCE_BATCH_SIZE}: {e}\")\n        continue\n\nprint(f\"\\nGenerated {len(all_submission_rows)} predictions\")\n\n# Create submission DataFrame\nprint(\"\\n=== Creating Submission File ===\")\nsubmission_df = pd.DataFrame(all_submission_rows)\n\n# Ensure all test images have at least one prediction\nall_test_ids = [os.path.basename(img).replace('.jpg', '') for img in test_images]\nif len(submission_df) > 0:\n    predicted_ids = submission_df['patient_id'].unique()\n    missing_ids = set(all_test_ids) - set(predicted_ids)\nelse:\n    missing_ids = set(all_test_ids)\n\n# Add dummy predictions for images without detections\nif missing_ids:\n    print(f\"Adding dummy predictions for {len(missing_ids)} images without detections\")\n    dummy_rows = []\n    for img_id in missing_ids:\n        # Create a small dummy polygon\n        dummy_rows.append({\n            'patient_id': img_id,\n            'class_id': 0,  # Default to class 0 (Abrasion)\n            'confidence': 0.01,  # Very low confidence\n            'poly': '0.1 0.1 0.1 0.2 0.2 0.2 0.2 0.1'  # Small square polygon\n        })\n    \n    submission_df = pd.concat([submission_df, pd.DataFrame(dummy_rows)], ignore_index=True)\n\n# Sort by patient_id and then by confidence (descending)\nsubmission_df = submission_df.sort_values(['patient_id', 'confidence'], ascending=[True, False])\n\n# Ensure correct column order\nsubmission_df = submission_df[['patient_id', 'class_id', 'confidence', 'poly']]\n\n# Save the main submission file\nsubmission_df.to_csv('submission.csv', index=False)\nprint(\"Main submission file created: submission.csv\")\n\n# Verify submission format\nprint(\"\\n=== Verifying Submission Format ===\")\nprint(f\"Total predictions: {len(submission_df)}\")\nprint(f\"Unique images: {submission_df['patient_id'].nunique()}\")\nprint(f\"All test images included: {submission_df['patient_id'].nunique() == len(test_images)}\")\n\n# Display first few rows\nprint(\"\\nFirst 5 rows of submission:\")\nprint(submission_df.head())\n\n# Check for any potential issues\nprint(\"\\n=== Checking for Potential Issues ===\")\n\n# Check for missing test images\nmissing_in_submission = set(all_test_ids) - set(submission_df['patient_id'].unique())\nif missing_in_submission:\n    print(f\"WARNING: Missing images in submission: {missing_in_submission}\")\nelse:\n    print(\"✓ All test images have predictions\")\n\n# Check class distribution\nprint(\"\\nPredictions per class:\")\nclass_dist = submission_df['class_id'].value_counts().sort_index()\nfor class_id, count in class_dist.items():\n    if 0 <= class_id < 9:\n        print(f\"  Class {class_id} ({CLASS_INFO[class_id]['name']}): {count}\")\n\n# Check confidence distribution\nprint(f\"\\nConfidence statistics:\")\nprint(f\"  Min: {submission_df['confidence'].min():.4f}\")\nprint(f\"  Max: {submission_df['confidence'].max():.4f}\")\nprint(f\"  Mean: {submission_df['confidence'].mean():.4f}\")\nprint(f\"  Median: {submission_df['confidence'].median():.4f}\")\n\n# Create alternative submission with higher confidence threshold\nprint(\"\\n=== Creating Alternative Submission (Higher Confidence) ===\")\nhigh_conf_df = submission_df[submission_df['confidence'] >= 0.3].copy()\n\n# Ensure all images still have at least one prediction\nhigh_conf_ids = high_conf_df['patient_id'].unique()\nmissing_high_conf = set(all_test_ids) - set(high_conf_ids)\n\nif missing_high_conf:\n    # Add the highest confidence prediction for each missing image\n    for img_id in missing_high_conf:\n        img_preds = submission_df[submission_df['patient_id'] == img_id]\n        if len(img_preds) > 0:\n            # Add the highest confidence prediction\n            high_conf_df = pd.concat([high_conf_df, img_preds.head(1)], ignore_index=True)\n        else:\n            # Add dummy prediction\n            dummy_row = pd.DataFrame([{\n                'patient_id': img_id,\n                'class_id': 0,\n                'confidence': 0.01,\n                'poly': '0.1 0.1 0.1 0.2 0.2 0.2 0.2 0.1'\n            }])\n            high_conf_df = pd.concat([high_conf_df, dummy_row], ignore_index=True)\n\nhigh_conf_df = high_conf_df.sort_values(['patient_id', 'confidence'], ascending=[True, False])\nhigh_conf_df.to_csv('submission_high_conf.csv', index=False)\nprint(f\"Created high confidence submission with {len(high_conf_df)} predictions\")\n\nprint(\"\\n=== Pipeline Completed Successfully! ===\")\nprint(\"Submission files created:\")\nprint(\"  - submission.csv (main submission)\")\nprint(\"  - submission_high_conf.csv (alternative with higher confidence threshold)\")\nprint(\"\\nReady to submit to Kaggle!\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}