{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":103103,"databundleVersionId":13042974,"sourceType":"competition"}],"dockerImageVersionId":31090,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# AlphaDent YOLO Training and Inference Pipeline\n# Dental condition detection using YOLOv8 with segmentation\n\nimport os\nimport sys\nimport time\nimport glob\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom pathlib import Path\nfrom tqdm.auto import tqdm\nimport yaml\nimport random\nfrom PIL import Image\nimport warnings\n\nwarnings.filterwarnings('ignore')\n\n# Disable wandb\nos.environ['WANDB_DISABLED'] = 'true'\n\n# Set random seeds for reproducibility\nrandom.seed(42)\nnp.random.seed(42)\n\n# Install required packages\nprint(\"Installing required packages...\")\nos.system('pip install -q ultralytics')\n\nimport torch\nfrom ultralytics import YOLO\n\n# Set deterministic behavior for PyTorch\ntorch.manual_seed(42)\nif torch.cuda.is_available():\n    torch.cuda.manual_seed(42)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\nprint(f'\\nPyTorch Version: {torch.__version__}')\nprint(f'CUDA Available: {torch.cuda.is_available()}')\nif torch.cuda.is_available():\n    print(f'CUDA Device: {torch.cuda.get_device_name(0)}')\n\n# =============================================================================\n# PATH CONFIGURATION\n# =============================================================================\n\n# Define paths\nBASE_PATH = '/kaggle/input/alpha-dent/AlphaDent'\nTRAIN_IMAGES_PATH = f'{BASE_PATH}/images/train'\nVALID_IMAGES_PATH = f'{BASE_PATH}/images/valid'\nTEST_IMAGES_PATH = f'{BASE_PATH}/images/test'\nTRAIN_LABELS_PATH = f'{BASE_PATH}/labels/train'\nVALID_LABELS_PATH = f'{BASE_PATH}/labels/valid'\n\n# Output paths\nOUTPUT_DIR = '/kaggle/working/'\nWEIGHTS_DIR = f'{OUTPUT_DIR}/weights'\nos.makedirs(WEIGHTS_DIR, exist_ok=True)\n\n# =============================================================================\n# CLASS DEFINITIONS\n# =============================================================================\n\n# Define class information\nCLASS_INFO = {\n    0: {'name': 'Abrasion', 'description': 'Teeth with mechanical wear of hard tissues'},\n    1: {'name': 'Filling', 'description': 'Dental fillings of various types'},\n    2: {'name': 'Crown', 'description': 'Dental crown (restoration)'},\n    3: {'name': 'Caries Class 1', 'description': 'Caries in fissures and pits'},\n    4: {'name': 'Caries Class 2', 'description': 'Caries on proximal surfaces of molars/premolars'},\n    5: {'name': 'Caries Class 3', 'description': 'Caries on proximal surfaces of incisors/canines without incisal edge'},\n    6: {'name': 'Caries Class 4', 'description': 'Caries on proximal surfaces of incisors/canines with incisal edge'},\n    7: {'name': 'Caries Class 5', 'description': 'Cervical caries (buccal/lingual surfaces)'},\n    8: {'name': 'Caries Class 6', 'description': 'Caries on incisal edges or cusps'}\n}\n\n# =============================================================================\n# YOLO CONFIGURATION\n# =============================================================================\n\nprint(\"\\n=== Creating YOLO Configuration ===\")\nyolo_config = {\n    'path': BASE_PATH,\n    'train': 'images/train',\n    'val': 'images/valid',\n    'test': 'images/test',\n    'nc': 9,\n    'names': [CLASS_INFO[i]['name'] for i in range(9)]\n}\n\n# Save the configuration\nCUSTOM_YAML_PATH = f'{OUTPUT_DIR}/alphadent_config.yaml'\nwith open(CUSTOM_YAML_PATH, 'w') as f:\n    yaml.dump(yolo_config, f, default_flow_style=False)\n\nprint(f\"Created custom YAML config at: {CUSTOM_YAML_PATH}\")\n\n# =============================================================================\n# DATASET ANALYSIS\n# =============================================================================\n\n# Count images\ntrain_images = sorted(glob.glob(f'{TRAIN_IMAGES_PATH}/*.jpg'))\nvalid_images = sorted(glob.glob(f'{VALID_IMAGES_PATH}/*.jpg'))\ntest_images = sorted(glob.glob(f'{TEST_IMAGES_PATH}/*.jpg'))\n\nprint(f\"\\n=== Dataset Statistics ===\")\nprint(f\"Training images: {len(train_images)}\")\nprint(f\"Validation images: {len(valid_images)}\")\nprint(f\"Test images: {len(test_images)}\")\n\ndef analyze_class_distribution(labels_path):\n    \"\"\"Analyze class distribution in dataset.\"\"\"\n    class_counts = {i: 0 for i in range(9)}\n    total_annotations = 0\n    label_files = glob.glob(f'{labels_path}/*.txt')\n    \n    for label_file in tqdm(label_files, desc=\"Analyzing labels\", leave=False):\n        if os.path.exists(label_file) and os.path.getsize(label_file) > 0:\n            try:\n                with open(label_file, 'r') as f:\n                    lines = f.readlines()\n                for line in lines:\n                    if line.strip():\n                        parts = line.strip().split()\n                        if parts:\n                            class_id = int(parts[0])\n                            if 0 <= class_id < 9:\n                                class_counts[class_id] += 1\n                                total_annotations += 1\n            except Exception:\n                continue\n    \n    return class_counts, total_annotations\n\nprint(\"\\n=== Analyzing Class Distribution ===\")\ntrain_class_counts, train_total = analyze_class_distribution(TRAIN_LABELS_PATH)\nvalid_class_counts, valid_total = analyze_class_distribution(VALID_LABELS_PATH)\n\nprint(f\"Training set: {train_total} total annotations\")\nprint(f\"Validation set: {valid_total} total annotations\")\n\n# =============================================================================\n# TRAINING CONFIGURATION\n# =============================================================================\n\nprint(\"\\n=== Model Training Configuration ===\")\nEPOCHS = 30\nIMAGE_SIZE = 640\nBATCH_SIZE = 8 if torch.cuda.is_available() else 4\nPATIENCE = 5\n\nprint(f\"Epochs: {EPOCHS}\")\nprint(f\"Image Size: {IMAGE_SIZE}\")\nprint(f\"Batch Size: {BATCH_SIZE}\")\nprint(f\"Early Stopping Patience: {PATIENCE}\")\n\n# =============================================================================\n# MODEL TRAINING\n# =============================================================================\n\nprint(\"\\n=== Starting Model Training ===\")\nmodel = YOLO('yolov8x-seg.pt')\n\n# Train the model with optimized parameters\nresults = model.train(\n    data=CUSTOM_YAML_PATH,\n    epochs=EPOCHS,\n    imgsz=IMAGE_SIZE,\n    batch=BATCH_SIZE,\n    patience=PATIENCE,\n    save=True,\n    save_period=10,\n    project=OUTPUT_DIR,\n    name='alphadent_yolov8x',\n    exist_ok=True,\n    pretrained=True,\n    optimizer='AdamW',\n    lr0=0.001,\n    lrf=0.01,\n    momentum=0.937,\n    weight_decay=0.0005,\n    warmup_epochs=3.0,\n    warmup_momentum=0.8,\n    warmup_bias_lr=0.1,\n    box=7.5,\n    cls=0.5,\n    dfl=1.5,\n    hsv_h=0.015,\n    hsv_s=0.7,\n    hsv_v=0.4,\n    degrees=0.0,\n    translate=0.1,\n    scale=0.5,\n    shear=0.0,\n    perspective=0.0,\n    flipud=0.0,\n    fliplr=0.5,\n    mosaic=1.0,\n    mixup=0.0,\n    copy_paste=0.0,\n    plots=True,\n    device=0 if torch.cuda.is_available() else 'cpu',\n    workers=2,\n    verbose=True,\n    amp=True,\n    val=True\n)\n\nprint(\"\\nTraining completed!\")\n\n# =============================================================================\n# MODEL LOADING\n# =============================================================================\n\nprint(\"\\n=== Loading Best Model ===\")\nbest_model_path = f'{OUTPUT_DIR}/alphadent_yolov8x/weights/best.pt'\n\nif os.path.exists(best_model_path):\n    model = YOLO(best_model_path)\n    print(f\"Loaded best model from: {best_model_path}\")\nelse:\n    last_model_path = f'{OUTPUT_DIR}/alphadent_yolov8x/weights/last.pt'\n    if os.path.exists(last_model_path):\n        model = YOLO(last_model_path)\n        print(f\"Loaded last model from: {last_model_path}\")\n    else:\n        print(\"Warning: No trained model found, using pretrained model\")\n        model = YOLO('yolov8x-seg.pt')\n\n# =============================================================================\n# MODEL VALIDATION\n# =============================================================================\n\nprint(\"\\n=== Model Validation ===\")\ntry:\n    metrics = model.val(\n        data=CUSTOM_YAML_PATH,\n        imgsz=IMAGE_SIZE,\n        batch=1,\n        conf=0.001,\n        iou=0.5,\n        max_det=300,\n        device=0 if torch.cuda.is_available() else 'cpu',\n        plots=False,\n        save_json=False,\n    )\n    \n    print(f\"\\nValidation Results:\")\n    print(f\"mAP@50: {metrics.seg.map50:.4f}\")\n    print(f\"mAP@50-95: {metrics.seg.map:.4f}\")\n    \nexcept Exception as e:\n    print(f\"Validation error (non-critical): {e}\")\n\n# =============================================================================\n# INFERENCE FUNCTIONS\n# =============================================================================\n\ndef convert_to_submission_format(results, image_paths):\n    \"\"\"Convert YOLO results to competition submission format.\"\"\"\n    submission_rows = []\n    \n    for idx, result in enumerate(results):\n        # Get image ID (filename without extension)\n        image_id = os.path.basename(image_paths[idx]).replace('.jpg', '')\n        \n        if result.masks is not None and len(result.masks) > 0:\n            try:\n                # Get masks, classes, and confidences\n                masks = result.masks.xy\n                classes = result.boxes.cls.cpu().numpy().astype(int)\n                confidences = result.boxes.conf.cpu().numpy()\n                h, w = result.orig_shape\n                \n                # Process each detection\n                for mask_idx in range(len(masks)):\n                    if mask_idx < len(classes) and mask_idx < len(confidences):\n                        polygon = masks[mask_idx]\n                        \n                        if len(polygon) >= 3:  # Valid polygon (at least 3 points)\n                            # Normalize coordinates to [0, 1]\n                            normalized_coords = []\n                            for point in polygon:\n                                x_norm = float(point[0]) / w\n                                y_norm = float(point[1]) / h\n                                \n                                # Ensure coordinates are within [0, 1]\n                                x_norm = max(0.0, min(1.0, x_norm))\n                                y_norm = max(0.0, min(1.0, y_norm))\n                                normalized_coords.extend([x_norm, y_norm])\n                            \n                            # Format polygon string\n                            poly_str = ' '.join([f'{coord:.6f}' for coord in normalized_coords])\n                            \n                            submission_rows.append({\n                                'patient_id': image_id,\n                                'class_id': int(classes[mask_idx]),\n                                'confidence': float(confidences[mask_idx]),\n                                'poly': poly_str\n                            })\n                            \n            except Exception as e:\n                print(f\"Error processing result for image {idx}: {e}\")\n                continue\n    \n    return submission_rows\n\n# =============================================================================\n# TEST SET INFERENCE\n# =============================================================================\n\nprint(\"\\n=== Running Inference on Test Set ===\")\n\n# Process test images\ntest_images = sorted(glob.glob(f'{TEST_IMAGES_PATH}/*.jpg'))\nall_submission_rows = []\nINFERENCE_BATCH_SIZE = 8 if torch.cuda.is_available() else 4\n\nprint(f\"Processing {len(test_images)} test images...\")\n\n# Process in batches\nfor i in tqdm(range(0, len(test_images), INFERENCE_BATCH_SIZE)):\n    batch_images = test_images[i:i + INFERENCE_BATCH_SIZE]\n    \n    try:\n        # Run inference\n        results = model.predict(\n            batch_images,\n            imgsz=IMAGE_SIZE,\n            conf=0.25,  # Confidence threshold\n            iou=0.45,   # NMS IoU threshold\n            max_det=300,\n            device=0 if torch.cuda.is_available() else 'cpu',\n            verbose=False,\n            agnostic_nms=True,\n            retina_masks=True,\n        )\n        \n        # Convert results to submission format\n        batch_rows = convert_to_submission_format(results, batch_images)\n        all_submission_rows.extend(batch_rows)\n        \n    except Exception as e:\n        print(f\"Error in batch {i//INFERENCE_BATCH_SIZE}: {e}\")\n        continue\n\nprint(f\"\\nGenerated {len(all_submission_rows)} predictions\")\n\n# =============================================================================\n# SUBMISSION FILE CREATION\n# =============================================================================\n\nprint(\"\\n=== Creating Submission File ===\")\nsubmission_df = pd.DataFrame(all_submission_rows)\n\n# Ensure all test images have at least one prediction\nall_test_ids = [os.path.basename(img).replace('.jpg', '') for img in test_images]\n\nif len(submission_df) > 0:\n    predicted_ids = submission_df['patient_id'].unique()\n    missing_ids = set(all_test_ids) - set(predicted_ids)\nelse:\n    missing_ids = set(all_test_ids)\n\n# Add dummy predictions for images without detections\nif missing_ids:\n    print(f\"Adding dummy predictions for {len(missing_ids)} images without detections\")\n    dummy_rows = []\n    for img_id in missing_ids:\n        # Create a small dummy polygon\n        dummy_rows.append({\n            'patient_id': img_id,\n            'class_id': 0,  # Default to class 0 (Abrasion)\n            'confidence': 0.01,  # Very low confidence\n            'poly': '0.1 0.1 0.1 0.2 0.2 0.2 0.2 0.1'  # Small square polygon\n        })\n    \n    submission_df = pd.concat([submission_df, pd.DataFrame(dummy_rows)], ignore_index=True)\n\n# Sort by patient_id and then by confidence (descending)\nsubmission_df = submission_df.sort_values(['patient_id', 'confidence'], ascending=[True, False])\n\n# Ensure correct column order\nsubmission_df = submission_df[['patient_id', 'class_id', 'confidence', 'poly']]\n\n# Save the main submission file\nsubmission_df.to_csv('submission.csv', index=False)\nprint(\"Main submission file created: submission.csv\")\n\n# =============================================================================\n# SUBMISSION VERIFICATION\n# =============================================================================\n\nprint(\"\\n=== Verifying Submission Format ===\")\nprint(f\"Total predictions: {len(submission_df)}\")\nprint(f\"Unique images: {submission_df['patient_id'].nunique()}\")\nprint(f\"All test images included: {submission_df['patient_id'].nunique() == len(test_images)}\")\n\n# Display first few rows\nprint(\"\\nFirst 5 rows of submission:\")\nprint(submission_df.head())\n\nprint(\"\\n=== Checking for Potential Issues ===\")\n\n# Check for missing test images\nmissing_in_submission = set(all_test_ids) - set(submission_df['patient_id'].unique())\nif missing_in_submission:\n    print(f\"WARNING: Missing images in submission: {missing_in_submission}\")\nelse:\n    print(\"✓ All test images have predictions\")\n\n# Check class distribution\nprint(\"\\nPredictions per class:\")\nclass_dist = submission_df['class_id'].value_counts().sort_index()\nfor class_id, count in class_dist.items():\n    if 0 <= class_id < 9:\n        print(f\"  Class {class_id} ({CLASS_INFO[class_id]['name']}): {count}\")\n\n# Check confidence distribution\nprint(f\"\\nConfidence statistics:\")\nprint(f\"  Min: {submission_df['confidence'].min():.4f}\")\nprint(f\"  Max: {submission_df['confidence'].max():.4f}\")\nprint(f\"  Mean: {submission_df['confidence'].mean():.4f}\")\nprint(f\"  Median: {submission_df['confidence'].median():.4f}\")\n\n# =============================================================================\n# ALTERNATIVE SUBMISSION\n# =============================================================================\n\nprint(\"\\n=== Creating Alternative Submission (Higher Confidence) ===\")\nhigh_conf_df = submission_df[submission_df['confidence'] >= 0.3].copy()\n\n# Ensure all images still have at least one prediction\nhigh_conf_ids = high_conf_df['patient_id'].unique()\nmissing_high_conf = set(all_test_ids) - set(high_conf_ids)\n\nif missing_high_conf:\n    # Add the highest confidence prediction for each missing image\n    for img_id in missing_high_conf:\n        img_preds = submission_df[submission_df['patient_id'] == img_id]\n        if len(img_preds) > 0:\n            # Add the highest confidence prediction\n            high_conf_df = pd.concat([high_conf_df, img_preds.head(1)], ignore_index=True)\n        else:\n            # Add dummy prediction\n            dummy_row = pd.DataFrame([{\n                'patient_id': img_id,\n                'class_id': 0,\n                'confidence': 0.01,\n                'poly': '0.1 0.1 0.1 0.2 0.2 0.2 0.2 0.1'\n            }])\n            high_conf_df = pd.concat([high_conf_df, dummy_row], ignore_index=True)\n\nhigh_conf_df = high_conf_df.sort_values(['patient_id', 'confidence'], ascending=[True, False])\nhigh_conf_df.to_csv('submission_high_conf.csv', index=False)\nprint(f\"Created high confidence submission with {len(high_conf_df)} predictions\")\n\n# =============================================================================\n# COMPLETION\n# =============================================================================\n\nprint(\"\\n=== Pipeline Completed Successfully! ===\")\nprint(\"Submission files created:\")\nprint(\"  - submission.csv (main submission)\")\nprint(\"  - submission_high_conf.csv (alternative with higher confidence threshold)\")\nprint(\"\\nReady to submit to Kaggle!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-06T09:56:38.360223Z","iopub.execute_input":"2025-09-06T09:56:38.360908Z","iopub.status.idle":"2025-09-06T11:16:03.098515Z","shell.execute_reply.started":"2025-09-06T09:56:38.360884Z","shell.execute_reply":"2025-09-06T11:16:03.097712Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from IPython.display import FileLink, FileLinks\n\n# Create clickable download links\ndisplay(FileLink(\"submission.csv\"))\ndisplay(FileLink(\"submission_high_conf.csv\"))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-06T11:18:56.239291Z","iopub.execute_input":"2025-09-06T11:18:56.239786Z","iopub.status.idle":"2025-09-06T11:18:56.247550Z","shell.execute_reply.started":"2025-09-06T11:18:56.239751Z","shell.execute_reply":"2025-09-06T11:18:56.246770Z"}},"outputs":[],"execution_count":null}]}