{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":33679,"databundleVersionId":3212216,"sourceType":"competition"}],"dockerImageVersionId":31193,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"###############################################################\n# HERBARIUM 2022 FGVC9 — FULL BASELINE NOTEBOOK (FIXED)\n# EfficientNet-B3 | 512x512 | AdamW | CrossEntropy\n# Single GPU (P100) SAFE\n###############################################################\nimport os, gc, json, random\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nfrom sklearn.model_selection import train_test_split\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nfrom timm import create_model\n\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nprint(\"Device:\", DEVICE)\n\n###############################################################\n# 1. LOAD METADATA (FIXED - COCO FORMAT)\n###############################################################\nBASE = \"/kaggle/input/herbarium-2022-fgvc9\"\n\n# Load metadata JSON (COCO format)\nwith open(os.path.join(BASE, \"train_metadata.json\"), \"r\") as f:\n    meta = json.load(f)\n\n# The JSON has COCO structure with separate sections\nprint(\"JSON keys:\", meta.keys())\n\n# Extract images info (id -> filename mapping)\nimages_dict = {img['image_id']: img['file_name'] for img in meta['images']}\n\n# Extract annotations (image_id -> category_id mapping)\nrows = []\nfor ann in meta['annotations']:\n    image_id = ann['image_id']\n    category_id = ann['category_id']\n    file_name = images_dict.get(image_id)\n    \n    if file_name:\n        rows.append({\n            \"image_id\": image_id,\n            \"file_name\": file_name,\n            \"category_id\": category_id\n        })\n\ndf = pd.DataFrame(rows)\nprint(\"\\nDataFrame head:\")\nprint(df.head())\nprint(f\"\\nTotal training samples: {len(df)}\")\nprint(f\"Number of unique categories: {df['category_id'].nunique()}\")\n\n# Map image paths - use file_name from metadata directly\nimage_map = {}\nroot = os.path.join(BASE, \"train_images\")\n\n# Option 1: Map using the relative path from metadata\nfor _, row in df.iterrows():\n    file_name = row['file_name']\n    full_path = os.path.join(root, file_name)\n    if os.path.exists(full_path):\n        image_map[file_name] = full_path\n\nprint(f\"\\nMapped {len(image_map)} images\")\n\n# Verify all images are found\nmissing = df[~df['file_name'].isin(image_map.keys())]\nif len(missing) > 0:\n    print(f\"WARNING: {len(missing)} images not found in filesystem\")\n\n###############################################################\n# 2. TRAIN/VAL SPLIT (FIXED)\n###############################################################\ntrain_df, val_df = train_test_split(\n    df,\n    test_size=0.1,\n    stratify=df[\"category_id\"],\n    random_state=42\n)\nprint(f\"\\nTrain samples: {len(train_df)}, Val samples: {len(val_df)}\")\n\n###############################################################\n# 3. DATASET CLASS\n###############################################################\nclass HerbariumDataset(Dataset):\n    def __init__(self, df, image_map, transform=None):\n        self.df = df.reset_index(drop=True)\n        self.image_map = image_map\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        img_path = self.image_map[row['file_name']]\n        image = Image.open(img_path).convert('RGB')\n        \n        if self.transform:\n            image = self.transform(image)\n            \n        label = int(row['category_id'])\n        return image, label\n\n###############################################################\n# 4. TRANSFORMS\n###############################################################\ntrain_transform = transforms.Compose([\n    transforms.Resize((512, 512)),\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomRotation(15),\n    transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2),\n    transforms.ToTensor(),\n    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n])\n\nval_transform = transforms.Compose([\n    transforms.Resize((512, 512)),\n    transforms.ToTensor(),\n    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n])\n\n###############################################################\n# 5. DATALOADERS\n###############################################################\ntrain_dataset = HerbariumDataset(train_df, image_map, train_transform)\nval_dataset = HerbariumDataset(val_df, image_map, val_transform)\n\ntrain_loader = DataLoader(train_dataset, batch_size=32, shuffle=True, num_workers=2, pin_memory=True)\nval_loader = DataLoader(val_dataset, batch_size=32, shuffle=False, num_workers=2, pin_memory=True)\n\nprint(f\"\\nDataset created: {len(train_dataset)} train, {len(val_dataset)} val\")\n\n###############################################################\n# 6. MODEL\n###############################################################\nnum_classes = df['category_id'].nunique()\nprint(f\"Building model with {num_classes} classes...\")\n\nmodel = create_model('efficientnet_b3', pretrained=True, num_classes=num_classes)\nmodel = model.to(DEVICE)\n\n###############################################################\n# 7. TRAINING SETUP\n###############################################################\ncriterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.AdamW(model.parameters(), lr=1e-4, weight_decay=1e-5)\nscheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=10)\n\n###############################################################\n# 8. TRAINING LOOP\n###############################################################\ndef train_epoch(model, loader, criterion, optimizer):\n    model.train()\n    running_loss = 0.0\n    correct = 0\n    total = 0\n    \n    for batch_idx, (images, labels) in enumerate(loader):\n        images, labels = images.to(DEVICE), labels.to(DEVICE)\n        \n        optimizer.zero_grad()\n        outputs = model(images)\n        loss = criterion(outputs, labels)\n        loss.backward()\n        optimizer.step()\n        \n        running_loss += loss.item()\n        _, predicted = outputs.max(1)\n        total += labels.size(0)\n        correct += predicted.eq(labels).sum().item()\n        \n        if (batch_idx + 1) % 100 == 0:\n            print(f\"  Batch {batch_idx+1}/{len(loader)} - Loss: {loss.item():.4f}\")\n    \n    return running_loss / len(loader), 100. * correct / total\n\ndef validate(model, loader, criterion):\n    model.eval()\n    running_loss = 0.0\n    correct = 0\n    total = 0\n    \n    with torch.no_grad():\n        for images, labels in loader:\n            images, labels = images.to(DEVICE), labels.to(DEVICE)\n            outputs = model(images)\n            loss = criterion(outputs, labels)\n            \n            running_loss += loss.item()\n            _, predicted = outputs.max(1)\n            total += labels.size(0)\n            correct += predicted.eq(labels).sum().item()\n    \n    return running_loss / len(loader), 100. * correct / total\n\n###############################################################\n# 9. TRAIN\n###############################################################\nEPOCHS = 10\nbest_acc = 0\n\nprint(\"\\nStarting training...\")\nfor epoch in range(EPOCHS):\n    print(f\"\\n{'='*60}\")\n    print(f\"Epoch {epoch+1}/{EPOCHS}\")\n    print('='*60)\n    \n    train_loss, train_acc = train_epoch(model, train_loader, criterion, optimizer)\n    val_loss, val_acc = validate(model, val_loader, criterion)\n    scheduler.step()\n    \n    print(f\"\\nEpoch {epoch+1} Summary:\")\n    print(f\"  Train Loss: {train_loss:.4f} | Train Acc: {train_acc:.2f}%\")\n    print(f\"  Val Loss: {val_loss:.4f} | Val Acc: {val_acc:.2f}%\")\n    \n    if val_acc > best_acc:\n        best_acc = val_acc\n        torch.save(model.state_dict(), 'best_model.pth')\n        print(f\"  ✓ Saved best model with accuracy: {best_acc:.2f}%\")\n    \n    gc.collect()\n    if torch.cuda.is_available():\n        torch.cuda.empty_cache()\n\nprint(f\"\\n{'='*60}\")\nprint(f\"Training complete!\")\nprint(f\"Best validation accuracy: {best_acc:.2f}%\")\nprint('='*60)\n\n###############################################################\n# 10. LOAD BEST MODEL\n###############################################################\nmodel.load_state_dict(torch.load(\"best_model.pth\", map_location=DEVICE))\nmodel.eval()\nprint(\"Loaded best model.\")\n\n\n###############################################################\n# 11. LOAD TEST METADATA + MAP PATHS\n###############################################################\nwith open(os.path.join(BASE, \"test_metadata.json\"), \"r\") as f:\n    meta_test = json.load(f)\n\ntest_rows = []\nfor fname, data in meta_test.items():\n    test_rows.append({\"image_id\": fname})\n\ntest_df = pd.DataFrame(test_rows)\nprint(\"Test samples:\", len(test_df))\n\n# map test images recursively\ntest_image_map = {}\nroot_test = os.path.join(BASE, \"test_images\")\n\nfor dirpath, _, filenames in os.walk(root_test):\n    for f in filenames:\n        test_image_map[f] = os.path.join(dirpath, f)\n\nprint(\"\\nMapped test images:\", len(test_image_map))\nprint(list(test_image_map.items())[:5])\n\n\nclass HerbariumTestDataset(Dataset):\n    def __init__(self, df, image_map, transform=None):\n        self.df = df.reset_index(drop=True)\n        self.image_map = image_map\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        fname = self.df.iloc[idx]['image_id']\n        img = Image.open(self.image_map[fname]).convert('RGB')\n        \n        if self.transform:\n            img = self.transform(img)\n            \n        return img, fname\n\n\ntest_dataset = HerbariumTestDataset(test_df, test_image_map, val_transform)\ntest_loader = DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=2)\n\n\n###############################################################\n# 14. TEST INFERENCE\n###############################################################\npredictions = []\n\nwith torch.no_grad():\n    for images, fnames in test_loader:\n        images = images.to(DEVICE)\n        outputs = model(images)\n        preds = outputs.argmax(1).cpu().numpy()\n        \n        for f, p in zip(fnames, preds):\n            predictions.append([f, p])\n\n\n###############################################################\n# 15. CREATE SUBMISSION CSV\n###############################################################\nsub = pd.DataFrame(predictions, columns=[\"image_id\", \"category_id\"])\nsub.to_csv(\"submission.csv\", index=False)\nsub.head()\nprint(\"\\nSaved submission.csv!\")\n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}