{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":14774,"databundleVersionId":875431,"sourceType":"competition"}],"dockerImageVersionId":30787,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install prefetch_generator","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Fragment 1: Imports and Setup\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nimport timm\nimport pandas as pd\nimport numpy as np\nimport cv2\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom torch.cuda.amp import autocast, GradScaler\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import cohen_kappa_score\nimport warnings\nfrom torch.nn.parallel import DistributedDataParallel as DDP\nimport torch.multiprocessing as mp\nimport torch.distributed as dist\nfrom prefetch_generator import BackgroundGenerator\nwarnings.filterwarnings('ignore')\n\n# Fragment 2: Custom DataLoader for Background Loading\nclass DataLoaderX(DataLoader):\n    def __iter__(self):\n        return BackgroundGenerator(super().__iter__())\n\n# Fragment 3: Advanced Image Preprocessing\nclass AdvancedPreprocessing:\n    def __init__(self, image_size=512):\n        self.image_size = image_size\n        self.cache = {}\n        \n    @torch.no_grad()\n    def preprocess_image(self, image_path):\n        if image_path in self.cache:\n            return self.cache[image_path]\n            \n        img = cv2.imread(image_path)\n        if img is None:\n            raise ValueError(f\"Could not read image at {image_path}\")\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        \n        # Optimize mask creation\n        gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n        _, mask = cv2.threshold(gray, 5, 255, cv2.THRESH_BINARY)\n        contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n        \n        if contours:\n            largest_contour = max(contours, key=cv2.contourArea)\n            mask = np.zeros_like(gray)\n            cv2.drawContours(mask, [largest_contour], -1, 255, -1)\n        \n        img = cv2.bitwise_and(img, img, mask=mask)\n        img = cv2.resize(img, (self.image_size, self.image_size))\n        \n        # Cache the result\n        self.cache[image_path] = img\n        return img\n\n# Fragment 4: Dataset Class\nclass RetinopathyDataset(Dataset):\n    def __init__(self, df, img_dir, transform=None, preprocessing=None, is_test=False):\n        self.df = df\n        self.img_dir = img_dir\n        self.transform = transform\n        self.preprocessing = preprocessing\n        self.is_test = is_test\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        img_name = self.df.iloc[idx]['id_code']\n        img_path = f\"{self.img_dir}/{img_name}.png\"\n        \n        # Preprocess image\n        image = self.preprocessing.preprocess_image(img_path)\n        \n        # Apply transforms\n        if self.transform:\n            transformed = self.transform(image=image)\n            image = transformed[\"image\"]\n        \n        if self.is_test:\n            return image\n        else:\n            label = self.df.iloc[idx]['diagnosis']\n            return image, label\n\n# Fragment 5: Model Architecture\nclass EfficientNetWithMixup(nn.Module):\n    def __init__(self, num_classes=5):\n        super().__init__()\n        self.model = timm.create_model('tf_efficientnetv2_s', pretrained=True)\n        in_features = self.model.classifier.in_features\n        \n        self.model.classifier = nn.Sequential(\n            nn.Linear(in_features, 512),\n            nn.ReLU(),\n            nn.Dropout(0.3),\n            nn.Linear(512, num_classes)\n        )\n        \n    def forward(self, x):\n        return self.model(x)\n\n# Fragment 6: Mixup Implementation\ndef mixup_data(x, y, alpha=0.2, device='cuda'):\n    if alpha > 0:\n        lam = np.random.beta(alpha, alpha)\n    else:\n        lam = 1\n\n    batch_size = x.size()[0]\n    index = torch.randperm(batch_size).to(device)\n\n    mixed_x = lam * x + (1 - lam) * x[index]\n    y_a, y_b = y, y[index]\n    return mixed_x, y_a, y_b, lam\n\ndef mixup_criterion(criterion, pred, y_a, y_b, lam):\n    return lam * criterion(pred, y_a) + (1 - lam) * criterion(pred, y_b)\n\n# Fragment 7: Training Functions\ndef train_fold(fold, model, train_loader, valid_loader, device, criterion, optimizer, scheduler, num_epochs=15):\n    scaler = GradScaler()\n    best_score = 0\n    \n    for epoch in range(num_epochs):\n        model.train()\n        train_loss = 0\n        \n        for batch_idx, (inputs, targets) in enumerate(train_loader):\n            inputs, targets = inputs.to(device), targets.to(device)\n            \n            # Apply mixup to training data\n            inputs_mixed, targets_a, targets_b, lam = mixup_data(inputs, targets, alpha=0.2, device=device)\n            \n            optimizer.zero_grad()\n            \n            with autocast():\n                outputs = model(inputs_mixed)\n                loss = mixup_criterion(criterion, outputs, targets_a, targets_b, lam)\n            \n            scaler.scale(loss).backward()\n            scaler.step(optimizer)\n            scaler.update()\n            \n            train_loss += loss.item()\n            \n            if batch_idx % 100 == 0:\n                print(f'Epoch: {epoch}, Batch: {batch_idx}, Loss: {loss.item():.4f}')\n        \n        # Validation\n        model.eval()\n        valid_preds = []\n        valid_targets = []\n        valid_loss = 0\n        \n        with torch.no_grad(), autocast():\n            for inputs, targets in valid_loader:\n                inputs, targets = inputs.to(device), targets.to(device)\n                outputs = model(inputs)\n                loss = criterion(outputs, targets)\n                valid_loss += loss.item()\n                \n                preds = torch.argmax(outputs, dim=1)\n                valid_preds.extend(preds.cpu().numpy())\n                valid_targets.extend(targets.cpu().numpy())\n        \n        valid_score = cohen_kappa_score(valid_targets, valid_preds, weights='quadratic')\n        \n        if valid_score > best_score:\n            best_score = valid_score\n            torch.save(model.state_dict(), f'best_model_fold_{fold}.pth')\n            \n        scheduler.step(valid_score)\n        \n        print(f'Epoch {epoch+1}/{num_epochs}')\n        print(f'Train Loss: {train_loss/len(train_loader):.4f}')\n        print(f'Valid Loss: {valid_loss/len(valid_loader):.4f}')\n        print(f'Valid Kappa Score: {valid_score:.4f}')\n        print(f'Best Kappa Score: {best_score:.4f}\\n')\n    \n    return best_score\n\n# Fragment 8: Testing and Prediction Functions\ndef prepare_test_data(test_df, preprocessing, config, transforms):\n    \"\"\"Prepare test dataset and dataloader\"\"\"\n    test_dataset = RetinopathyDataset(\n        test_df,\n        f\"{config['DATA_PATH']}/test_images\",\n        transform=transforms,\n        preprocessing=preprocessing,\n        is_test=True\n    )\n    \n    test_loader = DataLoaderX(\n        test_dataset,\n        batch_size=config['BATCH_SIZE'] * 2,\n        shuffle=False,\n        num_workers=config['NUM_WORKERS'],\n        pin_memory=config['PIN_MEMORY']\n    )\n    \n    return test_loader\n\ndef make_predictions(model, test_loader, device):\n    \"\"\"Make predictions on test data using trained model\"\"\"\n    model.eval()\n    predictions = []\n    \n    with torch.no_grad(), autocast():\n        for inputs in test_loader:\n            inputs = inputs.to(device)\n            outputs = model(inputs)\n            preds = torch.argmax(outputs, dim=1)\n            predictions.extend(preds.cpu().numpy())\n    \n    return predictions\n\ndef ensemble_predictions(config, test_df, preprocessing, transforms, device):\n    \"\"\"Ensemble predictions from all trained model folds\"\"\"\n    all_predictions = []\n    \n    for fold in range(1, config['N_FOLDS'] + 1):\n        try:\n            # Load model for current fold\n            model = EfficientNetWithMixup(num_classes=5).to(device)\n            if torch.cuda.device_count() > 1:\n                model = nn.DataParallel(model)\n            \n            model.load_state_dict(torch.load(f'best_model_fold_{fold}.pth'))\n            \n            # Prepare test data\n            test_loader = prepare_test_data(test_df, preprocessing, config, transforms)\n            \n            # Make predictions\n            fold_predictions = make_predictions(model, test_loader, device)\n            all_predictions.append(fold_predictions)\n            \n            print(f\"Completed predictions for fold {fold}\")\n            \n        except Exception as e:\n            print(f\"Error making predictions for fold {fold}: {str(e)}\")\n            continue\n    \n    # Average predictions from all folds\n    if all_predictions:\n        final_predictions = np.mean(all_predictions, axis=0)\n        return np.round(final_predictions).astype(int)\n    else:\n        raise ValueError(\"No successful predictions made\")\n\ndef generate_submission(config):\n    \"\"\"Generate submission file with test predictions\"\"\"\n    try:\n        # Read test data\n        test_df = pd.read_csv(f\"{config['DATA_PATH']}/test.csv\")\n        \n        # Initialize preprocessing and transforms\n        preprocessing = AdvancedPreprocessing(image_size=config['IMAGE_SIZE'])\n        transforms = A.Compose([\n            A.Normalize(),\n            ToTensorV2()\n        ])\n        \n        # Set device\n        device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n        \n        # Get ensemble predictions\n        predictions = ensemble_predictions(config, test_df, preprocessing, transforms, device)\n        \n        # Create submission DataFrame\n        submission_df = pd.DataFrame({\n            'id_code': test_df['id_code'],\n            'diagnosis': predictions\n        })\n        \n        # Save submission file\n        submission_df.to_csv('submission.csv', index=False)\n        print(\"Submission file generated successfully!\")\n        \n    except Exception as e:\n        print(f\"Error generating submission: {str(e)}\")\n        import traceback\n        traceback.print_exc()\n\n# Fragment 9: Main Function\ndef main():\n    config = {\n        'SEED': 42,\n        'IMAGE_SIZE': 384,\n        'BATCH_SIZE': 32,\n        'NUM_EPOCHS': 15,\n        'N_FOLDS': 5,\n        'LEARNING_RATE': 2e-4,\n        'WEIGHT_DECAY': 1e-5,\n        'NUM_WORKERS': 4,\n        'PIN_MEMORY': True,\n        'DATA_PATH': '/kaggle/input/aptos2019-blindness-detection',\n    }\n    \n    # Enable cuDNN autotuner\n    torch.backends.cudnn.benchmark = True\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    print(f\"Using device: {device}\")\n    \n    # Set random seeds\n    torch.manual_seed(config['SEED'])\n    np.random.seed(config['SEED'])\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed(config['SEED'])\n    \n    try:\n        train_df = pd.read_csv(f\"{config['DATA_PATH']}/train.csv\")\n    except FileNotFoundError as e:\n        print(f\"Error: Could not find train.csv at {config['DATA_PATH']}\")\n        return\n    \n    preprocessing = AdvancedPreprocessing(image_size=config['IMAGE_SIZE'])\n    \n    # Training transforms\n    transforms = A.Compose([\n        A.RandomRotate90(),\n        A.Flip(),\n        A.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.1, rotate_limit=45),\n        A.OneOf([\n            A.OpticalDistortion(p=0.3),\n            A.GridDistortion(p=.1),\n        ], p=0.2),\n        A.Normalize(),\n        ToTensorV2()\n    ])\n    \n    skf = StratifiedKFold(n_splits=config['N_FOLDS'], shuffle=True, random_state=config['SEED'])\n    scores = []\n    \n    for fold, (train_idx, valid_idx) in enumerate(skf.split(train_df, train_df['diagnosis']), 1):\n        print(f'\\nTraining Fold {fold}/{config[\"N_FOLDS\"]}')\n        \n        try:\n            train_fold_df = train_df.iloc[train_idx].reset_index(drop=True)\n            valid_fold_df = train_df.iloc[valid_idx].reset_index(drop=True)\n            \n            train_dataset = RetinopathyDataset(\n                train_fold_df,\n                f\"{config['DATA_PATH']}/train_images\",\n                transform=transforms,\n                preprocessing=preprocessing\n            )\n            \n            valid_dataset = RetinopathyDataset(\n                valid_fold_df,\n                f\"{config['DATA_PATH']}/train_images\",\n                transform=transforms,\n                preprocessing=preprocessing\n            )\n            \n            train_loader = DataLoaderX(\n                train_dataset,\n                batch_size=config['BATCH_SIZE'],\n                shuffle=True,\n                num_workers=config['NUM_WORKERS'],\n                pin_memory=config['PIN_MEMORY']\n            )\n            \n            valid_loader = DataLoaderX(\n                valid_dataset,\n                batch_size=config['BATCH_SIZE'] * 2,\n                shuffle=False,\n                num_workers=config['NUM_WORKERS'],\n                pin_memory=config['PIN_MEMORY']\n            )\n            \n            model = EfficientNetWithMixup(num_classes=5).to(device)\n            if torch.cuda.device_count() > 1:\n                print(f\"Using {torch.cuda.device_count()} GPUs!\")\n                model = nn.DataParallel(model)\n                \n            criterion = nn.CrossEntropyLoss()\n            optimizer = torch.optim.AdamW(\n                model.parameters(),\n                lr=config['LEARNING_RATE'],\n                weight_decay=config['WEIGHT_DECAY']\n            )\n            scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n                optimizer, mode='max', factor=0.5, patience=2, verbose=True\n            )\n            \n            score = train_fold(\n                fold, model, train_loader, valid_loader,\n                device, criterion, optimizer, scheduler,\n                config['NUM_EPOCHS']\n            )\n            scores.append(score)\n            \n        except Exception as e:\n            print(f\"Error in fold {fold}: {str(e)}\")\n            import traceback\n            traceback.print_exc()\n            continue\n    \n    if scores:\n        print(\"\\nCross-validation scores:\", scores)\n        print(f\"Mean score: {np.mean(scores):.4f}\")\n        print(f\"Std score: {np.std(scores):.4f}\")\n    \n    # Generate submission file\n    try:\n        generate_submission(config)\n    except Exception as e:\n        print(f\"Error generating submission: {str(e)}\")\n        traceback.print_exc()\n\nif __name__ == \"__main__\":\n    main()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null}]}