{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":13836,"databundleVersionId":1718836,"sourceType":"competition"}],"dockerImageVersionId":31041,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-06-23T09:19:47.740823Z","iopub.execute_input":"2025-06-23T09:19:47.741072Z","iopub.status.idle":"2025-06-23T09:20:20.230322Z","shell.execute_reply.started":"2025-06-23T09:19:47.741049Z","shell.execute_reply":"2025-06-23T09:20:20.229213Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Imports ---\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, Dataset\nfrom torchvision import transforms, models\n# Note: ImageFolder is imported but CustomCassavaDataset is preferred for this setup.\nfrom torchvision.datasets import ImageFolder \nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score, f1_score, confusion_matrix\nimport pandas as pd\nimport numpy as np\nimport os\nfrom PIL import Image\nfrom tqdm import tqdm\nimport json \nimport shutil \n\n# --- Configuration ---\nclass Config:\n    # Adjust paths based on Kaggle Notebook environment\n    DATA_ROOT = '/kaggle/input/cassava-leaf-disease-classification' \n    TRAIN_CSV = os.path.join(DATA_ROOT, 'train.csv')\n    TRAIN_IMAGES_DIR = os.path.join(DATA_ROOT, 'train_images') \n    PROCESSED_TRAIN_DIR = os.path.join('/kaggle/working', 'processed_train_images') \n\n    IMAGE_SIZE = 384 \n    BATCH_SIZE = 32\n    NUM_EPOCHS = 10\n    LEARNING_RATE = 1e-4\n    NUM_CLASSES = 5 \n    DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    RANDOM_SEED = 42\n\n# Set random seeds for reproducibility\ntorch.manual_seed(Config.RANDOM_SEED)\nnp.random.seed(Config.RANDOM_SEED)\nif torch.cuda.is_available():\n    torch.cuda.manual_seed_all(Config.RANDOM_SEED)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False \n\nprint(f\"Using device: {Config.DEVICE}\")\n\n\n# --- Data Preparation Utility (Optional) ---\n# This function is used if you want to physically move/copy files for ImageFolder structure.\n# For CustomCassavaDataset, it's not strictly necessary.\ndef prepare_data_for_imagefolder(train_csv_path, train_images_dir, output_dir):\n    \"\"\"\n    Reads the train.csv and organizes images into class-specific subfolders\n    for torchvision.datasets.ImageFolder.\n    \"\"\"\n    if os.path.exists(output_dir) and len(os.listdir(output_dir)) > 0:\n        print(f\"'{output_dir}' already exists and is not empty. Skipping data preparation.\")\n        return\n\n    print(f\"Preparing data for ImageFolder structure in '{output_dir}'...\")\n    df = pd.read_csv(train_csv_path)\n\n    for class_id in range(Config.NUM_CLASSES):\n        os.makedirs(os.path.join(output_dir, str(class_id)), exist_ok=True)\n\n    for index, row in tqdm(df.iterrows(), total=len(df), desc=\"Organizing images\"):\n        img_filename = row['image_id']\n        label = str(row['label'])\n        \n        src_path = os.path.join(train_images_dir, img_filename)\n        dst_path = os.path.join(output_dir, label, img_filename)\n\n        if not os.path.exists(src_path):\n            print(f\"Warning: Image {src_path} not found. Skipping.\")\n            continue\n        \n        try:\n            shutil.copy2(src_path, dst_path)\n        except Exception as e:\n            print(f\"Error copying {src_path} to {dst_path}: {e}\")\n            \n    print(\"Data preparation complete.\")\n\n# --- Custom Dataset Definition ---\nclass CustomCassavaDataset(Dataset):\n    def __init__(self, image_ids, labels, img_dir, transform=None):\n        self.image_ids = image_ids.tolist()\n        self.labels = labels.tolist()\n        self.img_dir = img_dir\n        self.transform = transform\n        \n        self.label_map = {label: i for i, label in enumerate(sorted(list(set(self.labels))))}\n        print(f\"Dataset initialized with {len(self.image_ids)} samples.\")\n        print(f\"Label map: {self.label_map}\")\n\n    def __len__(self):\n        return len(self.image_ids)\n\n    def __getitem__(self, idx):\n        img_name = self.image_ids[idx]\n        label = self.labels[idx]\n        img_path = os.path.join(self.img_dir, img_name)\n        \n        img = Image.open(img_path).convert('RGB')\n        \n        if self.transform:\n            img = self.transform(img)\n        \n        return img, self.label_map[label]\n\n# --- Data Transforms ---\ntrain_transforms = transforms.Compose([\n    transforms.Resize((Config.IMAGE_SIZE, Config.IMAGE_SIZE)),\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomVerticalFlip(),\n    transforms.RandomRotation(15),\n    transforms.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1, hue=0.1),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), # ImageNet means and stds\n])\n\nval_transforms = transforms.Compose([\n    transforms.Resize((Config.IMAGE_SIZE, Config.IMAGE_SIZE)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n])\n\n\n# --- Model Definition (ResNet50 Baseline) ---\ndef get_resnet50_baseline_model(num_classes=Config.NUM_CLASSES):\n    \"\"\"\n    Loads a pre-trained ResNet50 model and modifies its final classification layer.\n    \"\"\"\n    print(\"11\")\n    model = models.resnet50(weights=None) \n    num_ftrs = model.fc.in_features\n    model.fc = nn.Linear(num_ftrs, num_classes)\n    return model\n\n# --- Training Function ---\ndef train_model(model, train_loader, val_loader, criterion, optimizer, num_epochs, device):\n    best_val_accuracy = 0.0\n    \n    model.to(device) \n    best_model_save_path = os.path.join('/kaggle/working', \"resnet50_baseline_best_model.pth\")\n\n    for epoch in range(num_epochs):\n        model.train() \n        running_loss = 0.0\n        correct_predictions = 0\n        total_samples = 0\n\n        for inputs, labels in tqdm(train_loader, desc=f\"Epoch {epoch+1}/{num_epochs} [Train]\"):\n            inputs, labels = inputs.to(device), labels.to(device)\n\n            optimizer.zero_grad() \n\n            outputs = model(inputs)\n            loss = criterion(outputs, labels)\n            loss.backward() \n            optimizer.step() \n\n            running_loss += loss.item() * inputs.size(0)\n            _, predicted = torch.max(outputs.data, 1)\n            total_samples += labels.size(0)\n            correct_predictions += (predicted == labels).sum().item()\n\n        epoch_loss = running_loss / total_samples\n        epoch_accuracy = correct_predictions / total_samples\n        print(f\"Epoch {epoch+1} Train Loss: {epoch_loss:.4f} Acc: {epoch_accuracy:.4f}\")\n\n        # Corrected unpacking here\n        val_loss, val_accuracy, val_f1, _, _, _ = evaluate_model(model, val_loader, criterion, device)\n        print(f\"Epoch {epoch+1} Val Loss: {val_loss:.4f} Acc: {val_accuracy:.4f} F1: {val_f1:.4f}\")\n\n        if val_accuracy > best_val_accuracy:\n            best_val_accuracy = val_accuracy\n            torch.save(model.state_dict(), best_model_save_path)\n            print(f\"Saved best model with accuracy: {best_val_accuracy:.4f}\")\n\n    print(\"Training complete!\")\n\n# --- Evaluation Function ---\ndef evaluate_model(model, data_loader, criterion, device):\n    model.eval() \n    running_loss = 0.0\n    correct_predictions = 0\n    total_samples = 0\n    all_labels = []\n    all_predictions = []\n\n    with torch.no_grad(): \n        for inputs, labels in tqdm(data_loader, desc=\"Evaluating\"):\n            inputs, labels = inputs.to(device), labels.to(device)\n\n            outputs = model(inputs)\n            loss = criterion(outputs, labels)\n\n            running_loss += loss.item() * inputs.size(0)\n            _, predicted = torch.max(outputs.data, 1)\n\n            total_samples += labels.size(0)\n            correct_predictions += (predicted == labels).sum().item()\n\n            all_labels.extend(labels.cpu().numpy())\n            all_predictions.extend(predicted.cpu().numpy())\n\n    avg_loss = running_loss / total_samples\n    accuracy = correct_predictions / total_samples\n    f1 = f1_score(all_labels, all_predictions, average='macro') \n    cm = confusion_matrix(all_labels, all_predictions)\n\n    return avg_loss, accuracy, f1, all_labels, all_predictions, cm\n\n\n# --- Main Execution - Step 1: Create Dataset and DataLoader ---\ndf_train = pd.read_csv(Config.TRAIN_CSV)\ntrain_img_ids, val_img_ids, train_labels, val_labels = train_test_split(\n    df_train['image_id'], df_train['label'],\n    test_size=0.2, stratify=df_train['label'], random_state=Config.RANDOM_SEED\n)\n\ntrain_dataset = CustomCassavaDataset(\n    image_ids=train_img_ids,\n    labels=train_labels,\n    img_dir=Config.TRAIN_IMAGES_DIR, \n    transform=train_transforms\n)\nval_dataset = CustomCassavaDataset(\n    image_ids=val_img_ids,\n    labels=val_labels,\n    img_dir=Config.TRAIN_IMAGES_DIR,\n    transform=val_transforms\n)\n\ntrain_loader = DataLoader(\n    train_dataset,\n    batch_size=Config.BATCH_SIZE,\n    shuffle=True,\n    num_workers=os.cpu_count() // 2, \n    pin_memory=True\n)\nval_loader = DataLoader(\n    val_dataset,\n    batch_size=Config.BATCH_SIZE * 2, \n    shuffle=False,\n    num_workers=os.cpu_count() // 2,\n    pin_memory=True\n)\nprint(f\"Train samples: {len(train_dataset)}, Val samples: {len(val_dataset)}\")\nprint(f\"Train batches: {len(train_loader)}, Val batches: {len(val_loader)}\")\n\n# --- Main Execution - Step 2: Initialize Model, Loss Function, and Optimizer ---\nmodel = get_resnet50_baseline_model()\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(model.parameters(), lr=Config.LEARNING_RATE)\n\n# --- Main Execution - Step 3: Train the model ---\nprint(\"--- Starting Model Training ---\")\ntrain_model(model, train_loader, val_loader, criterion, optimizer, Config.NUM_EPOCHS, Config.DEVICE)\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T09:36:42.285375Z","iopub.execute_input":"2025-06-23T09:36:42.285919Z","iopub.status.idle":"2025-06-23T09:38:17.679745Z","shell.execute_reply.started":"2025-06-23T09:36:42.285894Z","shell.execute_reply":"2025-06-23T09:38:17.678657Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Main Execution - Step 4: Load the best model and evaluate on the validation set one last time ---\nprint(\"\\n--- Final Evaluation of Best Model ---\")\nbest_model = get_resnet50_baseline_model()\nbest_model_path = os.path.join('/kaggle/working', \"resnet50_baseline_best_model.pth\")\n\nif os.path.exists(best_model_path):\n    best_model.load_state_dict(torch.load(best_model_path))\n    best_model.to(Config.DEVICE) \n\n    val_loss, val_accuracy, val_f1, all_labels_final, all_predictions_final, cm_final = evaluate_model(\n        best_model, val_loader, criterion, Config.DEVICE\n    )\n    print(f\"Best Model Val Loss: {val_loss:.4f}\")\n    print(f\"Best Model Val Accuracy: {val_accuracy:.4f}\")\n    print(f\"Best Model Val F1-Score (Macro): {val_f1:.4f}\")\n    print(\"Confusion Matrix:\\n\", cm_final)\nelse:\n    print(f\"Best model not found at {best_model_path}. Please ensure training completed successfully.\")\n\nprint(\"\\nResNet50 Baseline experiment complete.\")\n\n\n# --- 1. Imports for this independent module ---\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import DataLoader, Dataset\nfrom torchvision import transforms, models\nimport pandas as pd\nimport os\nfrom PIL import Image\nfrom tqdm import tqdm\n\n# --- 2. Define ALL Paths and Configurations for this Test Module (Independent) ---\n# IMPORTANT: Adjust these paths if your dataset location differs in your Kaggle environment.\n# These paths are assumed to be consistent with standard Kaggle competition data mounting.\nDATA_ROOT_FOR_INFERENCE = '/kaggle/input/cassava-leaf-disease-classification'\nTEST_IMAGES_DIR_FOR_INFERENCE = os.path.join(DATA_ROOT_FOR_INFERENCE, 'test_images')\nSAMPLE_SUBMISSION_CSV_FOR_INFERENCE = os.path.join(DATA_ROOT_FOR_INFERENCE, 'sample_submission.csv')\nBEST_MODEL_PATH_FOR_INFERENCE = os.path.join('/kaggle/working', 'resnet50_baseline_best_model.pth') # Explicit model path\n\nIMAGE_SIZE_FOR_INFERENCE = 384\nBATCH_SIZE_INFERENCE = 64 # Use a larger batch size for inference\nNUM_CLASSES_FOR_INFERENCE = 5\nDEVICE_FOR_INFERENCE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nprint(f\"Using device for inference: {DEVICE_FOR_INFERENCE}\")\nprint(f\"Loading test images from: {TEST_IMAGES_DIR_FOR_INFERENCE}\")\nprint(f\"Loading sample submission from: {SAMPLE_SUBMISSION_CSV_FOR_INFERENCE}\")\nprint(f\"Loading best model from: {BEST_MODEL_PATH_FOR_INFERENCE}\")\n\n\n# --- 3. Define the Model Architecture (Must exactly match the trained model) ---\n# This definition is self-contained within this block.\ndef get_resnet50_model_for_inference(num_classes=NUM_CLASSES_FOR_INFERENCE):\n    model = models.resnet50(weights=None) # No ImageNet weights here; we'll load custom ones\n    num_ftrs = model.fc.in_features\n    model.fc = nn.Linear(num_ftrs, num_classes)\n    return model\n\n# --- 4. Define Test Dataset Class ---\n# This definition is self-contained within this block.\nclass TestCassavaDataset(Dataset):\n    def __init__(self, image_ids, img_dir, transform=None):\n        self.image_ids = image_ids\n        self.img_dir = img_dir\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.image_ids)\n\n    def __getitem__(self, idx):\n        img_name = self.image_ids[idx]\n        img_path = os.path.join(self.img_dir, img_name)\n        img = Image.open(img_path).convert('RGB')\n        if self.transform:\n            img = self.transform(img)\n        return img, img_name # Return image tensor and its ID\n\n# --- 5. Define Test Transforms ---\n# These transforms are self-contained within this block.\ntest_transforms = transforms.Compose([\n    transforms.Resize((IMAGE_SIZE_FOR_INFERENCE, IMAGE_SIZE_FOR_INFERENCE)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), # ImageNet means and stds\n])\n\n# --- 6. Main Inference Logic ---\nprint(\"\\n--- Starting Test Inference ---\")\n\n# Load the model structure\nmodel = get_resnet50_model_for_inference()\n\n# Load the trained weights from the explicitly defined path\nif os.path.exists(BEST_MODEL_PATH_FOR_INFERENCE):\n    # map_location ensures it loads correctly whether on CPU or GPU\n    model.load_state_dict(torch.load(BEST_MODEL_PATH_FOR_INFERENCE, map_location=DEVICE_FOR_INFERENCE))\n    model.to(DEVICE_FOR_INFERENCE)\n    model.eval() # Set model to evaluation mode\n    print(f\"Model successfully loaded from {BEST_MODEL_PATH_FOR_INFERENCE}\")\nelse:\n    print(f\"Error: Best model weights not found at {BEST_MODEL_PATH_FOR_INFERENCE}.\")\n    print(\"Please ensure your training run successfully saved the model in /kaggle/working/.\")\n    # Exit the script if the model is not found, as inference cannot proceed.\n    import sys\n    sys.exit(\"Model file not found. Exiting.\")\n\n\n# Load test image IDs from the sample submission file\nsubmission_df_template = pd.read_csv(SAMPLE_SUBMISSION_CSV_FOR_INFERENCE)\ntest_image_ids = submission_df_template['image_id'].tolist()\n\n# Create test dataset and DataLoader\ntest_dataset = TestCassavaDataset(\n    image_ids=test_image_ids,\n    img_dir=TEST_IMAGES_DIR_FOR_INFERENCE, # Using the explicitly defined test images directory\n    transform=test_transforms\n)\n\ntest_loader = DataLoader(\n    test_dataset,\n    batch_size=BATCH_SIZE_INFERENCE,\n    shuffle=False,\n    num_workers=os.cpu_count() // 2, # Use half of CPU cores for data loading\n    pin_memory=True\n)\n\nall_test_predictions = []\n\nwith torch.no_grad(): # Disable gradient calculation for inference\n    for inputs, _ in tqdm(test_loader, desc=\"Predicting on test set\"): # _ for image_ids; we just need inputs here\n        inputs = inputs.to(DEVICE_FOR_INFERENCE)\n        outputs = model(inputs)\n        _, predicted = torch.max(outputs.data, 1)\n        all_test_predictions.extend(predicted.cpu().numpy())\n\n# Create the final submission DataFrame\n# Kaggle expects the submission.csv to have image_ids in the same order as sample_submission.csv.\n# Since we loaded test_image_ids from sample_submission.csv and process them sequentially,\n# the predictions will already be in the correct order corresponding to these image_ids.\nsubmission_df = pd.DataFrame({\n    'image_id': test_image_ids, # Use the original order of image_ids from the template\n    'label': all_test_predictions\n})\n\n# Save the submission file to /kaggle/working/\nsubmission_file_path = os.path.join('/kaggle/working', 'submission.csv')\nsubmission_df.to_csv(submission_file_path, index=False)\n\nprint(f\"\\nSubmission file saved to: {submission_file_path}\")\nprint(\"First 5 rows of generated submission.csv:\")\nprint(submission_df.head())\n\nprint(\"\\nFully Independent Test Module execution complete.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T09:27:29.835088Z","iopub.execute_input":"2025-06-23T09:27:29.835601Z","iopub.status.idle":"2025-06-23T09:28:02.250242Z","shell.execute_reply.started":"2025-06-23T09:27:29.835580Z","shell.execute_reply":"2025-06-23T09:28:02.249263Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}