{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":14774,"databundleVersionId":875431,"sourceType":"competition"},{"sourceId":9641206,"sourceType":"datasetVersion","datasetId":5887412},{"sourceId":9885587,"sourceType":"datasetVersion","datasetId":6070567},{"sourceId":9938278,"sourceType":"datasetVersion","datasetId":6109939}],"dockerImageVersionId":30787,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nimport timm\nimport pandas as pd\nimport numpy as np\nimport cv2\nimport albumentations as A\nimport random\nfrom albumentations.pytorch import ToTensorV2\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import cohen_kappa_score\nfrom torch.cuda.amp import autocast, GradScaler\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import accuracy_score, cohen_kappa_score\nfrom sklearn.metrics import f1_score, cohen_kappa_score\nimport warnings\nimport torch.optim as optim\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-11-19T08:40:44.542276Z","iopub.execute_input":"2024-11-19T08:40:44.542672Z","iopub.status.idle":"2024-11-19T08:41:27.723604Z","shell.execute_reply.started":"2024-11-19T08:40:44.542639Z","shell.execute_reply":"2024-11-19T08:41:27.722884Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def set_seed(seed=42):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed_all(seed)\n        torch.backends.cudnn.deterministic = True\n        torch.backends.cudnn.benchmark = False\n\nset_seed(42)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Custom Preprocessing Class\nclass AdvancedPreprocessing:\n    def __init__(self, image_size=512):\n        self.image_size = image_size\n        self.cache = {}\n\n    @torch.no_grad()\n    def preprocess_image(self, image_path):\n        if image_path in self.cache:\n            return self.cache[image_path]\n        \n        img = cv2.imread(image_path)\n        if img is None:\n            raise ValueError(f\"Could not read image at {image_path}\")\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        \n        gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n        _, mask = cv2.threshold(gray, 5, 255, cv2.THRESH_BINARY)\n        contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n        \n        if contours:\n            largest_contour = max(contours, key=cv2.contourArea)\n            mask = np.zeros_like(gray)\n            cv2.drawContours(mask, [largest_contour], -1, 255, -1)\n        \n        img = cv2.bitwise_and(img, img, mask=mask)\n        img = cv2.resize(img, (self.image_size, self.image_size))\n        \n        self.cache[image_path] = img\n        return img\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-19T08:41:27.725572Z","iopub.execute_input":"2024-11-19T08:41:27.726451Z","iopub.status.idle":"2024-11-19T08:41:27.733399Z","shell.execute_reply.started":"2024-11-19T08:41:27.726409Z","shell.execute_reply":"2024-11-19T08:41:27.732502Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# EfficientNet Model with Pre-trained Weights and Additional Layers\nclass EfficientNetWithMixup(nn.Module):\n    def __init__(self, num_classes=5):\n        super().__init__()\n        # Load EfficientNet base model without final classification head\n        self.model = timm.create_model('efficientnet_b0', pretrained=False)\n        self.model.load_state_dict(torch.load('/kaggle/input/efficientnet-b0/efficientnet_b0_ra-3dd342df.pth'))\n        \n        # Get the input features of the original classifier\n        in_features = self.model.classifier.in_features\n        \n        # Define a new classifier with additional layers\n        self.model.classifier = nn.Sequential(\n            nn.Linear(in_features, 1024),  # First layer with more units\n            nn.ReLU(),\n            nn.Dropout(0.3),              # Dropout for regularization\n            nn.Linear(1024, 512),         # Intermediate layer\n            nn.ReLU(),\n            nn.Dropout(0.3),\n            nn.Linear(512, 256),          # Another intermediate layer\n            nn.ReLU(),\n            nn.Dropout(0.2),\n            nn.Linear(256, num_classes)   # Final output layer\n        )\n        \n    def forward(self, x):\n        return self.model(x)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-19T08:41:27.734450Z","iopub.execute_input":"2024-11-19T08:41:27.734812Z","iopub.status.idle":"2024-11-19T08:41:27.750342Z","shell.execute_reply.started":"2024-11-19T08:41:27.734775Z","shell.execute_reply":"2024-11-19T08:41:27.749767Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Dataset Class\nclass RetinopathyDataset(Dataset):\n    def __init__(self, df, image_dir, transform=None, preprocessing=None):\n        self.df = df\n        self.image_dir = image_dir\n        self.transform = transform\n        self.preprocessing = preprocessing\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        image_path = f\"{self.image_dir}/{self.df.iloc[idx]['id_code']}.png\"\n        image_id = self.df.iloc[idx]['id_code']  # Retrieve the image ID\n        image = self.preprocessing.preprocess_image(image_path)\n        \n        if self.transform:\n            image = self.transform(image=image)['image']\n        \n        label = self.df.iloc[idx]['diagnosis'] if 'diagnosis' in self.df.columns else -1\n        return image, label, image_id  # Return the image ID along with the image and label\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-19T08:41:27.751234Z","iopub.execute_input":"2024-11-19T08:41:27.751477Z","iopub.status.idle":"2024-11-19T08:41:27.767671Z","shell.execute_reply.started":"2024-11-19T08:41:27.751454Z","shell.execute_reply":"2024-11-19T08:41:27.767028Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def mixup_data(x, y, alpha=1.0, device='cuda'):\n    if alpha > 0.0:\n        lam = np.random.beta(alpha, alpha)\n    else:\n        lam = 1.0\n    batch_size = x.size(0)\n    index = torch.randperm(batch_size).to(device)\n    mixed_x = lam * x + (1 - lam) * x[index, :]\n    y_a, y_b = y, y[index]\n    return mixed_x, y_a, y_b, lam\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-19T08:41:27.769275Z","iopub.execute_input":"2024-11-19T08:41:27.769587Z","iopub.status.idle":"2024-11-19T08:41:27.780057Z","shell.execute_reply.started":"2024-11-19T08:41:27.769548Z","shell.execute_reply":"2024-11-19T08:41:27.779384Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def mixup_criterion(criterion, pred, y_a, y_b, lam):\n    return lam * criterion(pred, y_a) + (1 - lam) * criterion(pred, y_b)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-19T08:41:27.780930Z","iopub.execute_input":"2024-11-19T08:41:27.781193Z","iopub.status.idle":"2024-11-19T08:41:27.789598Z","shell.execute_reply.started":"2024-11-19T08:41:27.781169Z","shell.execute_reply":"2024-11-19T08:41:27.789013Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Prediction CSV Generation\ndef predict_and_generate_csv(model, test_loader, device, output_csv_path):\n    model.eval()\n    predictions = []\n    image_ids = []\n    with torch.no_grad():\n        for inputs, _, ids in test_loader:  # Adjusted to receive image IDs\n            inputs = inputs.to(device)\n            outputs = model(inputs)\n            preds = torch.argmax(outputs, dim=1)\n            predictions.extend(preds.cpu().numpy())\n            image_ids.extend(ids)  # Collect image IDs\n    \n    # Create the submission DataFrame\n    output_df = pd.DataFrame({'id_code': image_ids, 'diagnosis': predictions})\n    output_df.to_csv(output_csv_path, index=False)\n    print(f\"Predictions saved to {output_csv_path}\")\n    print(output_df.head())\n    print(output_df.columns)\n    print(\"Unique diagnoses:\", output_df['diagnosis'].unique())\n    print(\"Number of rows:\", len(output_df))\n    print(\"Missing values:\", output_df.isnull().sum())\n    print(f\"Predictions saved to {output_csv_path}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-19T08:41:27.790436Z","iopub.execute_input":"2024-11-19T08:41:27.790674Z","iopub.status.idle":"2024-11-19T08:41:27.802363Z","shell.execute_reply.started":"2024-11-19T08:41:27.790651Z","shell.execute_reply":"2024-11-19T08:41:27.801716Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_fold(fold, model, train_loader, valid_loader, device, criterion, optimizer, scheduler, num_epochs=15):\n    scaler = GradScaler()\n    best_kappa = 0\n    metrics_history = {'kappa': [], 'accuracy': []}\n\n    for epoch in range(num_epochs):\n        model.train()\n        train_loss = 0\n        for inputs, targets, _ in train_loader:\n            inputs, targets = inputs.to(device), targets.to(device)\n            inputs_mixed, targets_a, targets_b, lam = mixup_data(inputs, targets, alpha=0.2, device=device)\n\n            optimizer.zero_grad()\n            with autocast():\n                outputs = model(inputs_mixed)\n                loss = mixup_criterion(criterion, outputs, targets_a, targets_b, lam)\n\n            scaler.scale(loss).backward()\n            scaler.step(optimizer)\n            scaler.update()\n            train_loss += loss.item()\n\n        model.eval()\n        valid_preds = []\n        valid_targets = []\n        with torch.no_grad(), autocast():\n            for inputs, targets, _ in valid_loader:\n                inputs, targets = inputs.to(device), targets.to(device)\n                outputs = model(inputs)\n                preds = torch.argmax(outputs, dim=1)\n                valid_preds.extend(preds.cpu().numpy())\n                valid_targets.extend(targets.cpu().numpy())\n\n        # Calculate metrics\n        kappa = cohen_kappa_score(valid_targets, valid_preds, weights='quadratic')\n        accuracy = accuracy_score(valid_targets, valid_preds)\n        metrics_history['kappa'].append(kappa)\n        metrics_history['accuracy'].append(accuracy)\n\n        if kappa > best_kappa:\n            best_kappa = kappa\n            torch.save(model.state_dict(), f'/kaggle/working/best_model_fold_{fold}.pth')\n\n        scheduler.step(kappa)\n        print(f'Epoch {epoch+1}/{num_epochs}, Valid Kappa: {kappa:.4f}, Valid Accuracy: {accuracy:.4f}, Best Kappa: {best_kappa:.4f}')\n\n    return best_kappa, metrics_history","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-19T08:41:27.803585Z","iopub.execute_input":"2024-11-19T08:41:27.803996Z","iopub.status.idle":"2024-11-19T08:41:27.820987Z","shell.execute_reply.started":"2024-11-19T08:41:27.803959Z","shell.execute_reply":"2024-11-19T08:41:27.820387Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Plot metrics for each fold\ndef plot_metrics(metrics_history, fold):\n    epochs = len(metrics_history['kappa'])\n    plt.figure(figsize=(12, 6))\n    plt.plot(range(1, epochs + 1), metrics_history['kappa'], label='Kappa')\n    plt.plot(range(1, epochs + 1), metrics_history['accuracy'], label='Accuracy')\n    plt.title(f'Fold {fold} Metrics')\n    plt.xlabel('Epoch')\n    plt.ylabel('Score')\n    plt.legend()\n    plt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-19T08:41:27.821801Z","iopub.execute_input":"2024-11-19T08:41:27.822023Z","iopub.status.idle":"2024-11-19T08:41:27.838668Z","shell.execute_reply.started":"2024-11-19T08:41:27.822000Z","shell.execute_reply":"2024-11-19T08:41:27.838013Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def main():\n    config = {\n        'SEED': 42,\n        'IMAGE_SIZE': 384,\n        'BATCH_SIZE': 32,\n        'NUM_EPOCHS': 15,\n        'N_FOLDS': 5,\n        'LEARNING_RATE': 2e-4,\n        'DATA_PATH': '/kaggle/input/aptos2019-blindness-detection',\n    }\n\n    torch.manual_seed(config['SEED'])\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    train_df = pd.read_csv(f\"{config['DATA_PATH']}/train.csv\")\n    test_df = pd.read_csv(f\"{config['DATA_PATH']}/test.csv\")\n    preprocessing = AdvancedPreprocessing(image_size=config['IMAGE_SIZE'])\n\n    transforms = A.Compose([\n        A.RandomRotate90(),\n        A.Flip(),\n        A.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.1, rotate_limit=45),\n        A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),\n        ToTensorV2()\n    ])\n\n    skf = StratifiedKFold(n_splits=config['N_FOLDS'], shuffle=True, random_state=config['SEED'])\n    best_scores = []\n    all_metrics = []\n\n    for fold, (train_idx, valid_idx) in enumerate(skf.split(train_df, train_df['diagnosis']), 1):\n        print(f\"Starting Fold {fold}/{config['N_FOLDS']}\")\n\n        train_dataset = RetinopathyDataset(train_df.iloc[train_idx], f\"{config['DATA_PATH']}/train_images\", transforms, preprocessing)\n        valid_dataset = RetinopathyDataset(train_df.iloc[valid_idx], f\"{config['DATA_PATH']}/train_images\", transforms, preprocessing)\n        train_loader = DataLoader(train_dataset, batch_size=config['BATCH_SIZE'], shuffle=True, num_workers=4)\n        valid_loader = DataLoader(valid_dataset, batch_size=config['BATCH_SIZE'], shuffle=False, num_workers=4)\n\n        model = EfficientNetWithMixup(num_classes=5).to(device)\n        criterion = nn.CrossEntropyLoss()\n        optimizer = torch.optim.AdamW(model.parameters(), lr=config['LEARNING_RATE'])\n        scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max')\n\n        best_kappa, metrics_history = train_fold(fold, model, train_loader, valid_loader, device, criterion, optimizer, scheduler, config['NUM_EPOCHS'])\n        best_scores.append(best_kappa)\n        all_metrics.append(metrics_history)\n\n        plot_metrics(metrics_history, fold)\n\n    # Load the best model for test predictions\n    best_fold = np.argmax(best_scores) + 1\n    model.load_state_dict(torch.load(f'/kaggle/working/best_model_fold_{best_fold}.pth'))\n    test_dataset = RetinopathyDataset(test_df, f\"{config['DATA_PATH']}/test_images\", transforms, preprocessing)\n    test_loader = DataLoader(test_dataset, batch_size=config['BATCH_SIZE'], shuffle=False)\n    predict_and_generate_csv(model, test_loader, device, \"/kaggle/working/submission.csv\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if __name__ == \"__main__\":\n    main()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-19T08:41:27.858074Z","iopub.execute_input":"2024-11-19T08:41:27.858404Z","iopub.status.idle":"2024-11-19T14:03:00.379996Z","shell.execute_reply.started":"2024-11-19T08:41:27.858368Z","shell.execute_reply":"2024-11-19T14:03:00.378951Z"}},"outputs":[],"execution_count":null}]}