{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# ১. পুরনো ডেটাসেট ক্যাশ মুছুন (যাতে এবার ৫০০০+ ছবি লোড হয়)\n#!rm -f /kaggle/working/preprocessed/manifest_384.csv\n\n# ২. পুরনো RESUME ফাইল মুছুন (এটি খুব জরুরি, যাতে ইপোক ১ থেকে ফ্রেশ লার্নিং রেট নিয়ে শুরু হয়)\n#!rm -f /kaggle/working/checkpoints/VimDR_fold1_RESUME.pth","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-05T19:12:47.779125Z","iopub.execute_input":"2026-06-05T19:12:47.779488Z","iopub.status.idle":"2026-06-05T19:12:48.010823Z","shell.execute_reply.started":"2026-06-05T19:12:47.779462Z","shell.execute_reply":"2026-06-05T19:12:48.009787Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%writefile vimdr_model_kaggle_v2.py\nimport math\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom copy import deepcopy\n\n# =============================================================================\n# 0. EfficientNet-B4 Backbone Helper\n# =============================================================================\ndef _build_backbone(pretrained: bool = True):\n    \"\"\"\n    Returns (backbone, feature_dim).\n    Uses timm if available, otherwise falls back to torchvision.\n    \"\"\"\n    try:\n        import timm\n        backbone = timm.create_model(\n            'efficientnet_b4',\n            pretrained=pretrained,\n            num_classes=0,\n            global_pool='avg',\n        )\n        feat_dim = backbone.num_features\n        return backbone, feat_dim\n    except ImportError:\n        from torchvision.models import efficientnet_b4, EfficientNet_B4_Weights\n        weights = EfficientNet_B4_Weights.IMAGENET1K_V1 if pretrained else None\n        m = efficientnet_b4(weights=weights)\n        feat_dim = m.classifier[1].in_features\n        m.classifier = nn.Identity()\n        m.avgpool = nn.AdaptiveAvgPool2d(1)\n        \n        class _Wrapper(nn.Module):\n            def __init__(self, base, dim):\n                super().__init__()\n                self.base = base\n                self.num_features = dim\n            def forward(self, x):\n                x = self.base.features(x)\n                x = self.base.avgpool(x)\n                return x.flatten(1)\n                \n        return _Wrapper(m, feat_dim), feat_dim\n\n# =============================================================================\n# 1. CBAM (Channel & Spatial Attention)\n# =============================================================================\nclass ChannelAttention(nn.Module):\n    def __init__(self, dim: int, ratio: int = 16):\n        super().__init__()\n        self.avg_pool = nn.AdaptiveAvgPool2d(1)\n        self.max_pool = nn.AdaptiveMaxPool2d(1)\n        self.fc = nn.Sequential(\n            nn.Conv2d(dim, dim // ratio, 1, bias=False),\n            nn.ReLU(),\n            nn.Conv2d(dim // ratio, dim, 1, bias=False),\n        )\n        self.sigmoid = nn.Sigmoid()\n\n    def forward(self, x):\n        a = self.fc(self.avg_pool(x))\n        m = self.fc(self.max_pool(x))\n        return x * self.sigmoid(a + m)\n\nclass SpatialAttention(nn.Module):\n    def __init__(self, kernel_size: int = 7):\n        super().__init__()\n        p = kernel_size // 2\n        self.conv = nn.Conv2d(2, 1, kernel_size, padding=p, bias=False)\n        self.sigmoid = nn.Sigmoid()\n\n    def forward(self, x):\n        avg = x.mean(1, keepdim=True)\n        mx, _ = x.max(1, keepdim=True)\n        return x * self.sigmoid(self.conv(torch.cat([avg, mx], 1)))\n\nclass CBAM(nn.Module):\n    def __init__(self, dim: int, ratio: int = 16, kernel_size: int = 7):\n        super().__init__()\n        self.ca = ChannelAttention(dim, ratio)\n        self.sa = SpatialAttention(kernel_size)\n\n    def forward(self, x):\n        return self.sa(self.ca(x))\n\n# =============================================================================\n# 2. Ordinal Uncertainty Head\n# =============================================================================\nclass OrdinalUncertaintyHead(nn.Module):\n    def __init__(self, in_dim: int, num_classes: int = 5, dropout: float = 0.3):\n        super().__init__()\n        self.num_classes = num_classes\n        hidden = max(256, in_dim // 4)\n        self.proj = nn.Sequential(\n            nn.Linear(in_dim, hidden * 2), nn.GELU(), nn.Dropout(dropout),\n            nn.Linear(hidden * 2, hidden), nn.GELU(), nn.Dropout(dropout),\n        )\n        self.ordinal_fc = nn.Linear(hidden, num_classes - 1)\n        self.reg_fc = nn.Linear(hidden, 1)\n        self.temperature = nn.Parameter(torch.ones(1))\n\n    def forward(self, x: torch.Tensor) -> dict:\n        feat = self.proj(x)\n        return {\n            'grade_logits': self.ordinal_fc(feat) / (self.temperature.abs() + 1e-6),\n            'reg_score': self.reg_fc(feat),\n            'features': feat,\n        }\n\n    def predict_grade(self, logits: torch.Tensor) -> torch.Tensor:\n        return torch.clamp(\n            torch.round(torch.sigmoid(logits).sum(dim=-1)),\n            0, self.num_classes - 1\n        ).long()\n\n    @staticmethod\n    def logits_to_probs(logits: torch.Tensor) -> torch.Tensor:\n        p = torch.sigmoid(logits)\n        p0 = 1.0 - p[:, :1]\n        pmid = p[:, :-1] - p[:, 1:]\n        pK = p[:, -1:]\n        return torch.cat([p0, pmid, pK], dim=-1).clamp(min=1e-7)\n\n# =============================================================================\n# 3. VimDRPlus v2 Model\n# =============================================================================\nclass VimDRPlus(nn.Module):\n    def __init__(self, num_classes: int = 5, dropout: float = 0.3, pretrained: bool = True):\n        super().__init__()\n        \n        # Backbone implementation\n        try:\n            import timm\n            self.backbone = timm.create_model(\n                'efficientnet_b4',\n                pretrained=pretrained,\n                num_classes=0,\n                global_pool='', \n            )\n            self.feat_dim = self.backbone.num_features\n        except ImportError:\n            from torchvision.models import efficientnet_b4, EfficientNet_B4_Weights\n            weights = EfficientNet_B4_Weights.IMAGENET1K_V1 if pretrained else None\n            _m = efficientnet_b4(weights=weights)\n            self.backbone = _m.features\n            self.feat_dim = 1792\n\n        self.cbam = CBAM(self.feat_dim)\n        self.pool = nn.AdaptiveAvgPool2d(1)\n        self.head = OrdinalUncertaintyHead(self.feat_dim, num_classes, dropout)\n\n        self._freeze_backbone_partial(freeze_blocks=2)\n\n    def _freeze_backbone_partial(self, freeze_blocks: int = 2):\n        try:\n            blocks = list(self.backbone.blocks.children())\n        except AttributeError:\n            blocks = list(self.backbone.children())\n            \n        for i, block in enumerate(blocks):\n            requires = (i >= freeze_blocks)\n            for p in block.parameters():\n                p.requires_grad = requires\n\n    def unfreeze_all(self):\n        for p in self.backbone.parameters():\n            p.requires_grad = True\n\n    def forward(self, x: torch.Tensor) -> dict:\n        feats = self.backbone(x)\n        feats = self.cbam(feats)\n        pooled = self.pool(feats).flatten(1)\n        return self.head(pooled)\n\n    @torch.inference_mode()\n    def predict_with_uncertainty(self, x: torch.Tensor, n_samples: int = 10) -> dict:\n        def _set_modes(m):\n            if isinstance(m, (nn.BatchNorm1d, nn.BatchNorm2d)):\n                m.eval()\n            elif isinstance(m, (nn.Dropout, nn.Dropout2d)):\n                m.train()\n                \n        self.apply(_set_modes)\n        preds = torch.stack(\n            [self.head.predict_grade(self.forward(x)['grade_logits']).float()\n             for _ in range(n_samples)], dim=0\n        )\n        self.eval()\n        \n        return {\n            'mean_grade': torch.clamp(\n                torch.round(preds.mean(0)), 0, self.head.num_classes - 1\n            ).long(),\n            'uncertainty': preds.std(0),\n        }\n\n    def param_count(self) -> int:\n        return sum(p.numel() for p in self.parameters() if p.requires_grad)\n\n# =============================================================================\n# 4. Model EMA Wrapper\n# =============================================================================\nclass ModelEMA:\n    def __init__(self, model: nn.Module, decay: float = 0.9998):\n        self.decay = decay\n        self.shadow = deepcopy(model).eval()\n        for p in self.shadow.parameters():\n            p.requires_grad_(False)\n\n    @torch.no_grad()\n    def update(self, model: nn.Module):\n        sd_model = model.state_dict()\n        sd_shadow = self.shadow.state_dict()\n        for k in sd_shadow:\n            sd_shadow[k].copy_(\n                self.decay * sd_shadow[k] + (1.0 - self.decay) * sd_model[k].float()\n            )\n\n    def apply_shadow(self):\n        pass\n\n    @property\n    def module(self):\n        return self.shadow\n\n# =============================================================================\n# 5. Focal Ordinal Loss\n# =============================================================================\nclass FocalOrdinalLoss(nn.Module):\n    def __init__(\n        self,\n        num_classes: int = 5,\n        lambda_kl: float = 0.3,\n        lambda_qwk: float = 0.5,\n        lambda_mse: float = 0.3,\n        gamma: float = 2.0,\n        label_smooth: float = 0.1,\n    ):\n        super().__init__()\n        self.K = num_classes\n        self.lk = lambda_kl\n        self.lq = lambda_qwk\n        self.lm = lambda_mse\n        self.gamma = gamma\n        self.ls = label_smooth\n        self.register_buffer('_k_range', torch.arange(num_classes, dtype=torch.float))\n        \n        i_idx = torch.arange(num_classes).float()\n        j_idx = torch.arange(num_classes).float()\n        W = ((i_idx.unsqueeze(1) - j_idx.unsqueeze(0)) ** 2) / ((num_classes - 1) ** 2)\n        self.register_buffer('_W', W)\n\n    def _smooth_dist(self, target: torch.Tensor) -> torch.Tensor:\n        B, K, ls = target.shape[0], self.K, self.ls\n        dist = torch.zeros(B, K, device=target.device)\n        g = target.view(B, 1)\n        dist.scatter_(1, g, 1.0 - ls)\n        \n        left = (g - 1).clamp(min=0)\n        left_mask = (g > 0).float()\n        dist.scatter_add_(1, left, left_mask * (ls / 2.0))\n        \n        right = (g + 1).clamp(max=K - 1)\n        right_mask = (g < K - 1).float()\n        dist.scatter_add_(1, right, right_mask * (ls / 2.0))\n        \n        return dist\n\n    def _qwk_loss(self, logits: torch.Tensor, target: torch.Tensor) -> torch.Tensor:\n        B = target.size(0)\n        probs = OrdinalUncertaintyHead.logits_to_probs(logits)\n        target_oh = F.one_hot(target, self.K).float()\n\n        O = torch.matmul(target_oh.T, probs) / B\n        true_hist = target_oh.sum(0)\n        pred_hist = probs.sum(0)\n        E = torch.outer(true_hist, pred_hist) / (B * B)\n\n        num = (self._W * O).sum()\n        denom = (self._W * E).sum() + 1e-7\n        return num / denom\n\n    def forward(self, out: dict, target: torch.Tensor):\n        logit = out['grade_logits']\n        reg = out['reg_score'].squeeze(-1)\n        tf = target.float()\n\n        mse = (reg - tf) ** 2\n        focal_mse = ((1 - torch.exp(-mse)) ** self.gamma * mse).mean()\n\n        ordinal_tgt = torch.stack(\n            [(target > k).float() for k in range(self.K - 1)], dim=1\n        )\n        bce_loss = F.binary_cross_entropy_with_logits(logit, ordinal_tgt)\n\n        log_pred = F.log_softmax(\n            -torch.abs(reg.unsqueeze(1) - self._k_range), dim=-1\n        )\n        dist = self._smooth_dist(target)\n        kl_loss = F.kl_div(log_pred, dist, reduction='batchmean')\n\n        qwk_loss = self._qwk_loss(logit, target)\n\n        total = (bce_loss\n                 + self.lm * focal_mse\n                 + self.lk * kl_loss\n                 + self.lq * qwk_loss)\n\n        return total, {\n            'focal_mse': focal_mse.item(),\n            'bce': bce_loss.item(),\n            'kl': kl_loss.item(),\n            'qwk_loss': qwk_loss.item(),\n        }\n\n# =============================================================================\n# 6. Sanity Check Execution\n# =============================================================================\nif __name__ == '__main__':\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n    print(\"Building VimDRPlus v2 (EfficientNet-B4 backbone)...\")\n    model = VimDRPlus(num_classes=5, dropout=0.3, pretrained=False).to(device)\n    ema = ModelEMA(model, decay=0.9998)\n\n    x = torch.randn(2, 3, 384, 384, device=device)\n    with torch.no_grad():\n        out = model(x)\n\n    logits = out['grade_logits']\n    probs = OrdinalUncertaintyHead.logits_to_probs(logits)\n    \n    print(f\"Parameters  : {model.param_count():,}\")\n    print(f\"Logits shape: {logits.shape}\")\n    print(f\"Probs sum   : {probs.sum(-1)}\")\n    print(f\"Grades      : {model.head.predict_grade(logits)}\")\n\n    criterion = FocalOrdinalLoss(num_classes=5)\n    target = torch.tensor([0, 2], device=device)\n    loss, info = criterion(out, target)\n    \n    print(f\"Loss: {loss.item():.4f} | {info}\")\n    print(\"v2 sanity check passed.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport cv2\nimport json\nimport random\nimport hashlib\nimport math\nimport warnings\nimport gc\nimport multiprocessing as mp\n\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.cuda.amp import GradScaler, autocast\nfrom torch.utils.data import Dataset, DataLoader, WeightedRandomSampler\nfrom sklearn.model_selection import StratifiedKFold, train_test_split\nfrom sklearn.metrics import cohen_kappa_score, accuracy_score, roc_auc_score\nfrom scipy.optimize import minimize\nfrom tqdm import tqdm\n\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nfrom vimdr_model_kaggle_v2 import (\n    VimDRPlus, FocalOrdinalLoss, OrdinalUncertaintyHead, ModelEMA\n)\n\nwarnings.filterwarnings('ignore')\n\n# =============================================================================\n# 0. Global Config\n# =============================================================================\nSEED = 42\nrandom.seed(SEED)\nnp.random.seed(SEED)\ntorch.manual_seed(SEED)\ntorch.cuda.manual_seed_all(SEED)\ntorch.backends.cudnn.deterministic = False\ntorch.backends.cudnn.benchmark = True\n\nCONFIG = dict(\n    # Paths\n    data_dir=\"/kaggle/working/preprocessed\",\n    ckpt_dir=\"/kaggle/working/checkpoints\",\n    output_dir=\"/kaggle/working/outputs\",\n\n    # Model Parameters\n    num_classes=5,\n    dropout_rate=0.3,\n    pretrained=True,\n\n    # Training Phases\n    phase1_epochs=5,\n    phase1_lr=2e-4,\n    phase2_epochs=30,\n    phase2_lr=5e-5,\n    min_lr=1e-7,\n    weight_decay=1e-4,\n    warmup_epochs=3,\n\n    # Training Hyper-parameters\n    batch_size=12,\n    grad_accum=2,\n    ema_decay=0.9998,\n\n    # Loss Weights\n    lambda_kl=0.3,\n    lambda_qwk=0.5,\n    lambda_mse=0.3,\n    label_smooth=0.1,\n\n    # Early Stopping\n    early_stop_pat=12,\n    val_every=1,\n\n    # Data Settings\n    resolution=384,\n    dr2015_frac=0.30,\n    mixup_alpha=0.2,\n    mixup_prob=0.15,\n    num_workers=2,\n\n    # Cross Validation\n    n_folds=5,\n    fold_to_run=1,  # 0-indexed fold 1\n    mc_samples=10,\n    seed=SEED,\n)\n\nGRADE_NAMES = ['No DR', 'Mild', 'Moderate', 'Severe', 'Proliferative']\nAPTOS_PATH = \"/kaggle/input/competitions/aptos2019-blindness-detection\"\nDR2015_PATH = \"/kaggle/input/datasets/tanlikesmath/diabetic-retinopathy-resized\"\n\nfor d in [CONFIG['data_dir'], CONFIG['ckpt_dir'], CONFIG['output_dir']]:\n    os.makedirs(d, exist_ok=True)\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n\n# =============================================================================\n# 1. Preprocessing\n# =============================================================================\ndef _preprocess_one(args):\n    path, size, cache_dir = args\n    uid = hashlib.md5(f\"{path}{size}\".encode()).hexdigest()[:12]\n    npath = os.path.join(cache_dir, f\"{uid}.npy\")\n    \n    if os.path.exists(npath):\n        return npath\n        \n    img = cv2.imread(path)\n    if img is None:\n        return None\n        \n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    img = cv2.resize(img, (size, size))\n    \n    if img.mean() < 0.10 * 255:\n        return None  \n\n    blur = cv2.GaussianBlur(img, (0, 0), size // 30)\n    enh = cv2.addWeighted(img, 4, blur, -4, 128)\n\n    clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8))\n    g_ch = clahe.apply(img[:, :, 1])\n    enh[:, :, 1] = np.clip(\n        enh[:, :, 1].astype(np.float32) * 0.7 + g_ch.astype(np.float32) * 0.3,\n        0, 255\n    ).astype(np.uint8)\n\n    mask = np.zeros(enh.shape[:2], dtype=np.uint8)\n    cv2.circle(mask, (size // 2, size // 2), int(size * 0.475), 1, -1)\n    out = (enh * mask[:, :, np.newaxis]).astype(np.uint8)\n\n    np.save(npath, out)\n    return npath\n\ndef preprocess_and_cache(df: pd.DataFrame, res: int) -> pd.DataFrame:\n    cache_dir = os.path.join(CONFIG['data_dir'], f'cache_{res}')\n    os.makedirs(cache_dir, exist_ok=True)\n    args = [(row['path'], res, cache_dir) for _, row in df.iterrows()]\n    n_workers = min(CONFIG['num_workers'], mp.cpu_count())\n    npaths = []\n    \n    try:\n        with mp.Pool(n_workers) as pool:\n            npaths = list(tqdm(pool.imap(_preprocess_one, args, chunksize=16),\n                               total=len(args), desc=f'Cache {res}px'))\n    except Exception:\n        npaths = [_preprocess_one(a) for a in tqdm(args, desc=f'Cache {res}px (serial)')]\n        \n    valid_rows = []\n    for (_, row), npath in zip(df.iterrows(), npaths):\n        if npath is not None:\n            valid_rows.append({**row.to_dict(), 'npy_path': npath})\n            \n    return pd.DataFrame(valid_rows).reset_index(drop=True)\n\n# =============================================================================\n# 2. Data Loading Helpers\n# =============================================================================\ndef load_aptos(base: str) -> pd.DataFrame:\n    df = pd.read_csv(f'{base}/train.csv')\n    df['path'] = df['id_code'].apply(lambda x: f'{base}/train_images/{x}.png')\n    df['source'] = 'APTOS2019'\n    return df[['path', 'diagnosis', 'source']].rename(columns={'diagnosis': 'grade'})\n\ndef load_dr2015(base: str, sample_frac: float = 0.30) -> pd.DataFrame:\n    csv = f'{base}/trainLabels.csv'\n    if not os.path.exists(csv):\n        return pd.DataFrame(columns=['path', 'grade', 'source'])\n        \n    df = pd.read_csv(csv)\n    df['path'] = df['image'].apply(lambda x: f'{base}/resized_train/resized_train/{x}.jpeg')\n    df['source'] = 'DR2015'\n    df = df.rename(columns={'level': 'grade'})[['path', 'grade', 'source']]\n    df = df[df['path'].apply(os.path.exists)]\n    \n    return (df.groupby('grade', group_keys=False)\n              .apply(lambda g: g.sample(frac=sample_frac, random_state=SEED))\n              .reset_index(drop=True))\n\ndef build_dataset() -> pd.DataFrame:\n    df = pd.concat([\n        load_aptos(APTOS_PATH),\n        load_dr2015(DR2015_PATH, sample_frac=CONFIG['dr2015_frac'])\n    ], ignore_index=True)\n    df = df[df['path'].apply(os.path.exists)].reset_index(drop=True)\n    df = df[df['path'].apply(lambda p: os.path.getsize(p) > 5000)].reset_index(drop=True)\n    return df\n\ndef create_splits(df: pd.DataFrame) -> pd.DataFrame:\n    df = df.copy()\n    df['split'] = 'train'\n    _, test_idx = train_test_split(\n        df.index, test_size=0.15, stratify=df['grade'], random_state=SEED\n    )\n    df.loc[test_idx, 'split'] = 'test'\n    return df\n\n# =============================================================================\n# 3. Augmentation Pipeline\n# =============================================================================\n_MEAN = [0.485, 0.456, 0.406]\n_STD = [0.229, 0.224, 0.225]\n\ndef get_train_transform(size: int = 384) -> A.Compose:\n    return A.Compose([\n        A.RandomResizedCrop(size=(size, size), scale=(0.85, 1.0), ratio=(0.95, 1.05)),\n        A.HorizontalFlip(p=0.5),\n        A.VerticalFlip(p=0.5),\n        A.RandomRotate90(p=0.3),\n        A.Rotate(limit=15, p=0.4),\n        A.ColorJitter(brightness=0.3, contrast=0.3, saturation=0.2, hue=0.05, p=0.5),\n        A.GridDistortion(num_steps=5, distort_limit=0.05, p=0.2),\n        A.ElasticTransform(alpha=1, sigma=50, alpha_affine=10, p=0.2),\n        A.CoarseDropout(max_holes=8, max_height=32, max_width=32,\n                        min_holes=1, min_height=8, min_width=8,\n                        fill_value=0, p=0.2),\n        A.GaussNoise(var_limit=(5.0, 30.0), p=0.2),\n        A.Normalize(mean=_MEAN, std=_STD),\n        ToTensorV2(),\n    ])\n\ndef get_eval_transform(size: int = 384) -> A.Compose:\n    return A.Compose([\n        # Fix: explicitly using height and width\n        A.Resize(height=size, width=size),\n        A.Normalize(mean=_MEAN, std=_STD),\n        ToTensorV2(),\n    ])\n\n# =============================================================================\n# 4. Dataset\n# =============================================================================\nclass DRDataset(Dataset):\n    def __init__(self, df: pd.DataFrame, split: str, resolution: int = 384):\n        self.df = df[df['split'] == split].reset_index(drop=True)\n        self.res = resolution\n        self.tfm = (get_train_transform(resolution)\n                    if split == 'train'\n                    else get_eval_transform(resolution))\n\n    def __len__(self): \n        return len(self.df)\n\n    def __getitem__(self, i: int):\n        row = self.df.iloc[i]\n        img = np.load(row['npy_path']).astype(np.uint8)\n        if img.shape[0] != self.res:\n            img = cv2.resize(img, (self.res, self.res))\n        img = self.tfm(image=img)['image']\n        return img, int(row['grade'])\n\n# =============================================================================\n# 5. DataLoaders\n# =============================================================================\ndef build_loaders(df: pd.DataFrame, batch_size: int):\n    res = CONFIG['resolution']\n    train_ds = DRDataset(df, 'train', res)\n    val_ds = DRDataset(df, 'val', res)\n    test_ds = DRDataset(df, 'test', res)\n\n    grades = train_ds.df['grade'].values.astype(int)\n    counts = np.bincount(grades, minlength=CONFIG['num_classes'])\n    weights = 1.0 / (counts[grades] + 1)\n    sampler = WeightedRandomSampler(\n        torch.tensor(weights, dtype=torch.float),\n        num_samples=len(grades),\n        replacement=True,\n    )\n\n    kw = dict(\n        num_workers=CONFIG['num_workers'],\n        pin_memory=True,\n        persistent_workers=CONFIG['num_workers'] > 0,\n    )\n    \n    return (\n        DataLoader(train_ds, batch_size=batch_size, sampler=sampler, **kw),\n        DataLoader(val_ds, batch_size=batch_size * 2, shuffle=False, **kw),\n        DataLoader(test_ds, batch_size=batch_size * 2, shuffle=False, **kw),\n    )\n\n# =============================================================================\n# 6. LR Scheduler\n# =============================================================================\nclass WarmupCosineScheduler:\n    def __init__(self, opt, warmup, total, base_lr, min_lr):\n        self.opt = opt\n        self.warmup = warmup\n        self.total = total\n        self.base_lr = base_lr\n        self.min_lr = min_lr\n        self._step = 0\n\n    def step(self) -> float:\n        self._step += 1\n        s = self._step\n        if s <= self.warmup:\n            lr = self.base_lr * s / self.warmup\n        else:\n            prog = (s - self.warmup) / max(1, self.total - self.warmup)\n            lr = self.min_lr + 0.5 * (self.base_lr - self.min_lr) * (1 + math.cos(math.pi * prog))\n            \n        for pg in self.opt.param_groups:\n            pg['lr'] = lr\n        return lr\n\n    def state_dict(self): \n        return {'step': self._step}\n        \n    def load_state_dict(self, d): \n        self._step = d['step']\n\n# =============================================================================\n# 7. MixUp\n# =============================================================================\nclass MixUp:\n    def __init__(self, alpha=0.4): \n        self.alpha = alpha\n\n    def __call__(self, x, y):\n        lam = float(np.random.beta(self.alpha, self.alpha))\n        idx = torch.randperm(x.size(0), device=x.device)\n        return lam * x + (1 - lam) * x[idx], y, y[idx], lam\n\ndef mixup_criterion(criterion, out, ya, yb, lam):\n    la, _ = criterion(out, ya)\n    lb, _ = criterion(out, yb)\n    return lam * la + (1 - lam) * lb\n\n# =============================================================================\n# 8. Evaluation & TTA\n# =============================================================================\n@torch.no_grad()\ndef _forward_tta(model, imgs: torch.Tensor) -> torch.Tensor:\n    out0 = model(imgs)['grade_logits']\n    out1 = model(torch.flip(imgs, dims=[-1]))['grade_logits']\n    out2 = model(torch.flip(imgs, dims=[-2]))['grade_logits']\n    return (out0 + out1 + out2) / 3.0\n\n@torch.no_grad()\ndef evaluate(model, loader, thresholds=None, use_tta: bool = True) -> dict:\n    model.eval()\n    all_labels, all_cont, all_logits = [], [], []\n\n    for imgs, labels in loader:\n        imgs = imgs.to(device)\n        logits = _forward_tta(model, imgs) if use_tta else model(imgs)['grade_logits']\n        cont = torch.sigmoid(logits).sum(dim=-1)\n        all_cont.extend(cont.cpu().tolist())\n        all_labels.extend(labels.tolist())\n        all_logits.append(logits.cpu())\n\n    y_true = np.array(all_labels)\n    y_cont = np.array(all_cont)\n    logits_cat = torch.cat(all_logits, dim=0)\n    probs = OrdinalUncertaintyHead.logits_to_probs(logits_cat).numpy()\n\n    if thresholds is None:\n        def _neg_qwk(th):\n            preds = np.clip(np.digitize(y_cont, bins=np.sort(th)), 0, CONFIG['num_classes'] - 1)\n            return -cohen_kappa_score(y_true, preds, weights='quadratic')\n\n        res = minimize(_neg_qwk, x0=[0.5, 1.5, 2.5, 3.5], method='nelder-mead',\n                       options={'xatol': 1e-4, 'fatol': 1e-4, 'maxiter': 1000})\n        thresholds = np.sort(res.x)\n\n    y_pred = np.clip(np.digitize(y_cont, bins=thresholds), 0, CONFIG['num_classes'] - 1)\n    qwk = cohen_kappa_score(y_true, y_pred, weights='quadratic')\n    acc = accuracy_score(y_true, y_pred)\n\n    aucs = []\n    for k in range(CONFIG['num_classes']):\n        pos = (y_true == k).astype(int)\n        if pos.sum() > 0 and (1 - pos).sum() > 0:\n            aucs.append(roc_auc_score(pos, probs[:, k]))\n    mean_auc = float(np.mean(aucs)) if aucs else 0.0\n\n    return {\n        'qwk': qwk,\n        'accuracy': acc,\n        'mean_auc': mean_auc,\n        'thresholds': thresholds.tolist(),\n    }\n\n# =============================================================================\n# 9. Training Loop Operations\n# =============================================================================\ndef train_one_epoch(model, loader, optimizer, criterion, scaler, mixup, grad_accum, epoch, ema=None) -> dict:\n    model.train()\n    losses, step = [], 0\n    optimizer.zero_grad(set_to_none=True)\n\n    pbar = tqdm(loader, desc=f'Ep{epoch:03d}', leave=False, mininterval=5.0)\n    for imgs, labels in pbar:\n        imgs = imgs.to(device, non_blocking=True)\n        labels = labels.to(device, non_blocking=True)\n\n        use_mix = (np.random.rand() < CONFIG['mixup_prob'])\n        with autocast():\n            if use_mix:\n                imgs_m, ya, yb, lam = mixup(imgs, labels)\n                loss = mixup_criterion(criterion, model(imgs_m), ya, yb, lam)\n            else:\n                loss, _ = criterion(model(imgs), labels)\n\n        scaler.scale(loss / grad_accum).backward()\n        step += 1\n\n        if step % grad_accum == 0:\n            scaler.unscale_(optimizer)\n            nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n            scaler.step(optimizer)\n            scaler.update()\n            optimizer.zero_grad(set_to_none=True)\n            if ema is not None:\n                ema.update(model)\n\n        losses.append(loss.item())\n        pbar.set_postfix({'loss': f\"{np.mean(losses[-30:]):.4f}\"})\n\n    if step % grad_accum != 0:\n        scaler.unscale_(optimizer)\n        nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n        scaler.step(optimizer)\n        scaler.update()\n        optimizer.zero_grad(set_to_none=True)\n        if ema is not None:\n            ema.update(model)\n\n    return {'loss': float(np.mean(losses))}\n\n# =============================================================================\n# 10. Checkpoint Management\n# =============================================================================\ndef save_checkpoint(path, model, ema, optimizer, scheduler, scaler, epoch, best_qwk, patience, history):\n    m_state = model.module.state_dict() if hasattr(model, 'module') else model.state_dict()\n    ema_state = ema.shadow.state_dict() if ema is not None else None\n    \n    torch.save({\n        'model': m_state,\n        'ema': ema_state,\n        'optimizer': optimizer.state_dict(),\n        'scheduler': scheduler.state_dict(),\n        'scaler': scaler.state_dict(),\n        'epoch': epoch,\n        'best_qwk': best_qwk,\n        'patience': patience,\n        'history': history,\n    }, path)\n\ndef load_checkpoint(path, model, ema, optimizer, scheduler, scaler):\n    ckpt = torch.load(path, map_location=device, weights_only=False)\n    m_inner = model.module if hasattr(model, 'module') else model\n    m_inner.load_state_dict(ckpt['model'])\n    \n    if ema is not None and ckpt.get('ema') is not None:\n        ema.shadow.load_state_dict(ckpt['ema'])\n        \n    optimizer.load_state_dict(ckpt['optimizer'])\n    scheduler.load_state_dict(ckpt['scheduler'])\n    scaler.load_state_dict(ckpt['scaler'])\n    \n    return ckpt['epoch'], ckpt['best_qwk'], ckpt['patience'], ckpt['history']\n\n# =============================================================================\n# 11. Main Fold Training\n# =============================================================================\ndef train_single_fold(df_train: pd.DataFrame, df_test: pd.DataFrame, fold: int) -> dict:\n    print(f\"\\n{'='*60}\\n  FOLD {fold+1}/{CONFIG['n_folds']}\\n{'='*60}\")\n\n    skf = StratifiedKFold(n_splits=CONFIG['n_folds'], shuffle=True, random_state=SEED)\n    folds = list(skf.split(df_train.index, df_train['grade']))\n    tr_idx, val_idx = folds[fold]\n\n    df_fold = df_train.copy()\n    train_paths = set(df_train.iloc[tr_idx]['path'])\n    val_paths = set(df_train.iloc[val_idx]['path'])\n    \n    df_fold['split'] = df_fold['path'].apply(\n        lambda p: 'train' if p in train_paths else ('val' if p in val_paths else 'skip')\n    )\n    df_fold = df_fold[df_fold['split'] != 'skip'].reset_index(drop=True)\n\n    df_test_copy = df_test.copy()\n    df_test_copy['split'] = 'test'\n    df_fold = pd.concat([df_fold, df_test_copy], ignore_index=True)\n\n    train_loader, val_loader, test_loader = build_loaders(df_fold, CONFIG['batch_size'])\n    print(f\"  Train: {len(train_loader.dataset):,}  Val: {len(val_loader.dataset):,}  Test: {len(test_loader.dataset):,}\")\n\n    model = VimDRPlus(\n        num_classes=CONFIG['num_classes'],\n        dropout=CONFIG['dropout_rate'],\n        pretrained=CONFIG['pretrained'],\n    ).to(device)\n\n    ema = ModelEMA(model, decay=CONFIG['ema_decay'])\n    n_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\n    print(f\"  Trainable params: {n_params:,}\")\n\n    criterion = FocalOrdinalLoss(\n        num_classes=CONFIG['num_classes'],\n        lambda_kl=CONFIG['lambda_kl'],\n        lambda_qwk=CONFIG['lambda_qwk'],\n        lambda_mse=CONFIG['lambda_mse'],\n        label_smooth=CONFIG['label_smooth'],\n    ).to(device)\n\n    try:\n        backbone_params = list(model.backbone.parameters())\n    except AttributeError:\n        backbone_params = []\n    head_params = [p for p in model.parameters() if not any(p is bp for bp in backbone_params)]\n\n    scaler = GradScaler()\n    mixup = MixUp(alpha=CONFIG['mixup_alpha'])\n\n    total_epochs = CONFIG['phase1_epochs'] + CONFIG['phase2_epochs']\n    resume_path = f\"{CONFIG['ckpt_dir']}/VimDR_fold{fold+1}_RESUME.pth\"\n    best_path = f\"{CONFIG['ckpt_dir']}/VimDR_fold{fold+1}_BEST.pth\"\n    \n    start_epoch = 1\n    best_qwk = -1.0\n    patience = 0\n    best_thresholds = [0.5, 1.5, 2.5, 3.5]\n    history = {'train_loss': [], 'val_qwk': [], 'val_acc': [], 'val_auc': [], 'lr': []}\n\n    print(f\"\\n  [Phase 1] Backbone partially frozen, {CONFIG['phase1_epochs']} epochs\")\n    optimizer = torch.optim.AdamW([\n        {'params': backbone_params, 'lr': CONFIG['phase1_lr'] * 0.1},\n        {'params': head_params, 'lr': CONFIG['phase1_lr']},\n    ], weight_decay=CONFIG['weight_decay'])\n    \n    scheduler = WarmupCosineScheduler(\n        optimizer, warmup=CONFIG['warmup_epochs'], total=CONFIG['phase1_epochs'],\n        base_lr=CONFIG['phase1_lr'], min_lr=CONFIG['min_lr'],\n    )\n\n    if os.path.exists(resume_path):\n        print(f\"  ▶ Resuming from {resume_path}\")\n        start_epoch, best_qwk, patience, history = load_checkpoint(\n            resume_path, model, ema, optimizer, scheduler, scaler\n        )\n        start_epoch += 1\n        print(f\"  Resumed epoch {start_epoch}, best QWK={best_qwk:.4f}\")\n\n    for epoch in range(start_epoch, CONFIG['phase1_epochs'] + 1):\n        stats = train_one_epoch(model, train_loader, optimizer, criterion, scaler, mixup, CONFIG['grad_accum'], epoch, ema)\n        lr = scheduler.step()\n        \n        history['train_loss'].append(stats['loss'])\n        history['lr'].append(lr)\n\n        val_m = evaluate(ema.shadow, val_loader, thresholds=None, use_tta=False)\n        history['val_qwk'].append(val_m['qwk'])\n        history['val_acc'].append(val_m['accuracy'])\n        history['val_auc'].append(val_m['mean_auc'])\n        \n        print(f\"  [P1] Ep{epoch:03d} | loss={stats['loss']:.4f} | QWK={val_m['qwk']:.4f} | lr={lr:.2e}\")\n\n        if val_m['qwk'] > best_qwk:\n            best_qwk = val_m['qwk']\n            best_thresholds = val_m['thresholds']\n            patience = 0\n            m_state = model.module.state_dict() if hasattr(model, 'module') else model.state_dict()\n            torch.save({'state_dict': m_state, 'ema_state': ema.shadow.state_dict(),\n                        'metrics': val_m, 'fold': fold, 'epoch': epoch}, best_path)\n        else:\n            patience += 1\n\n        save_checkpoint(resume_path, model, ema, optimizer, scheduler, scaler, epoch, best_qwk, patience, history)\n        torch.cuda.empty_cache()\n        gc.collect()\n\n    print(f\"\\n  [Phase 2] Full unfreeze, {CONFIG['phase2_epochs']} epochs\")\n    model.unfreeze_all()\n    optimizer = torch.optim.AdamW(model.parameters(), lr=CONFIG['phase2_lr'], weight_decay=CONFIG['weight_decay'])\n    scheduler = WarmupCosineScheduler(\n        optimizer, warmup=2, total=CONFIG['phase2_epochs'], base_lr=CONFIG['phase2_lr'], min_lr=CONFIG['min_lr']\n    )\n    patience = 0\n\n    for epoch in range(CONFIG['phase1_epochs'] + 1, total_epochs + 1):\n        stats = train_one_epoch(model, train_loader, optimizer, criterion, scaler, mixup, CONFIG['grad_accum'], epoch, ema)\n        lr = scheduler.step()\n        \n        history['train_loss'].append(stats['loss'])\n        history['lr'].append(lr)\n\n        val_m = evaluate(ema.shadow, val_loader, thresholds=None, use_tta=True)\n        history['val_qwk'].append(val_m['qwk'])\n        history['val_acc'].append(val_m['accuracy'])\n        history['val_auc'].append(val_m['mean_auc'])\n        \n        print(f\"  [P2] Ep{epoch:03d} | loss={stats['loss']:.4f} | QWK={val_m['qwk']:.4f} | AUC={val_m['mean_auc']:.4f} | lr={lr:.2e}\")\n\n        if val_m['qwk'] > best_qwk:\n            best_qwk = val_m['qwk']\n            best_thresholds = val_m['thresholds']\n            patience = 0\n            m_state = model.module.state_dict() if hasattr(model, 'module') else model.state_dict()\n            torch.save({'state_dict': m_state, 'ema_state': ema.shadow.state_dict(),\n                        'metrics': val_m, 'fold': fold, 'epoch': epoch}, best_path)\n        else:\n            patience += 1\n            if patience >= CONFIG['early_stop_pat']:\n                print(f\"  Early stop at epoch {epoch}.\")\n                break\n\n        save_checkpoint(resume_path, model, ema, optimizer, scheduler, scaler, epoch, best_qwk, patience, history)\n        torch.cuda.empty_cache()\n        gc.collect()\n\n    print(f\"\\n  Loading best weights for test evaluation...\")\n    ckpt = torch.load(best_path, map_location=device, weights_only=False)\n    m_inner = model.module if hasattr(model, 'module') else model\n    m_inner.load_state_dict(ckpt['state_dict'])\n    ema.shadow.load_state_dict(ckpt['ema_state'])\n    final_th = ckpt['metrics']['thresholds']\n\n    test_m = evaluate(ema.shadow, test_loader, thresholds=final_th, use_tta=True)\n    print(f\"\\n  Fold {fold+1} TEST | QWK={test_m['qwk']:.4f} | ACC={test_m['accuracy']:.4f} | AUC={test_m['mean_auc']:.4f}\")\n\n    result = {\n        'fold': fold + 1,\n        'best_val_qwk': best_qwk,\n        'test_qwk': test_m['qwk'],\n        'test_acc': test_m['accuracy'],\n        'test_auc': test_m['mean_auc'],\n        'history': history,\n    }\n    \n    with open(f\"{CONFIG['output_dir']}/fold{fold+1}_result.json\", 'w') as f:\n        json.dump({k: v for k, v in result.items() if k != 'history'}, f, indent=2)\n\n    return result\n\n# =============================================================================\n# 12. Cross Validation Aggregation\n# =============================================================================\ndef aggregate_cv_results():\n    results = []\n    for f in range(1, CONFIG['n_folds'] + 1):\n        path = f\"{CONFIG['output_dir']}/fold{f}_result.json\"\n        if os.path.exists(path):\n            with open(path) as fp:\n                results.append(json.load(fp))\n        else:\n            print(f\"  WARNING: fold {f} result not found.\")\n            \n    if not results:\n        return\n\n    qwks = [r['test_qwk'] for r in results]\n    aucs = [r['test_auc'] for r in results]\n    accs = [r['test_acc'] for r in results]\n\n    summary = {\n        'n_folds_complete': len(results),\n        'qwk_mean': round(float(np.mean(qwks)), 4),\n        'qwk_std': round(float(np.std(qwks)), 4),\n        'auc_mean': round(float(np.mean(aucs)), 4),\n        'auc_std': round(float(np.std(aucs)), 4),\n        'acc_mean': round(float(np.mean(accs)), 4),\n        'per_fold': results,\n    }\n    \n    with open(f\"{CONFIG['output_dir']}/cv_summary.json\", 'w') as f:\n        json.dump(summary, f, indent=2)\n\n    print(f\"\\n{'='*60}\\n  5-Fold CV Results ({len(results)} folds complete)\")\n    print(f\"  QWK : {summary['qwk_mean']:.4f} ± {summary['qwk_std']:.4f}\")\n    print(f\"  AUC : {summary['auc_mean']:.4f} ± {summary['auc_std']:.4f}\")\n    print(f\"{'='*60}\\n\")\n    return summary\n\n# =============================================================================\n# 13. Application Entry Point\n# =============================================================================\ndef main():\n    print(f\"Device : {device}\")\n    if torch.cuda.is_available():\n        print(f\"GPU    : {torch.cuda.get_device_name(0)}\")\n        print(f\"VRAM   : {torch.cuda.get_device_properties(0).total_memory / 1e9:.1f} GB\")\n\n    manifest_path = os.path.join(CONFIG['data_dir'], f\"manifest_{CONFIG['resolution']}.csv\")\n\n    if os.path.exists(manifest_path):\n        print(f\"Loading existing manifest: {manifest_path}\")\n        df_cached = pd.read_csv(manifest_path)\n    else:\n        print(\"Building dataset from scratch...\")\n        df_raw = build_dataset()\n        df_raw = create_splits(df_raw)\n        df_cached = preprocess_and_cache(df_raw, res=CONFIG['resolution'])\n        df_cached.to_csv(manifest_path, index=False)\n        print(f\"Manifest saved: {len(df_cached):,} images\")\n\n    fold = CONFIG['fold_to_run']\n    print(f\"\\nRunning fold {fold+1}/{CONFIG['n_folds']} ...\")\n\n    df_train = df_cached[df_cached['split'] != 'test'].reset_index(drop=True)\n    df_test = df_cached[df_cached['split'] == 'test'].reset_index(drop=True)\n\n    result = train_single_fold(df_train, df_test, fold=fold)\n    print(f\"\\nFold {fold+1} complete: QWK={result['test_qwk']:.4f}\")\n\n    n_done = sum(1 for f in range(1, CONFIG['n_folds'] + 1)\n                 if os.path.exists(f\"{CONFIG['output_dir']}/fold{f}_result.json\"))\n                 \n    print(f\"\\n{n_done}/{CONFIG['n_folds']} folds complete.\")\n    if n_done == CONFIG['n_folds']:\n        aggregate_cv_results()\n\nif __name__ == '__main__':\n    main()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#import gc, torch\n#torch.cuda.empty_cache(); gc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-05T19:16:10.180943Z","iopub.execute_input":"2026-06-05T19:16:10.181754Z","iopub.status.idle":"2026-06-05T19:16:10.267191Z","shell.execute_reply.started":"2026-06-05T19:16:10.181724Z","shell.execute_reply":"2026-06-05T19:16:10.266409Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ১. চেকপয়েন্ট রাখার জন্য ফোল্ডার তৈরি\n#!mkdir -p /kaggle/working/checkpoints\n\n# ২. আপনার বের করা সঠিক পাথ থেকে RESUME এবং BEST ফাইল দুটি ওয়ার্কিং ফোল্ডারে কপি করা\n#!cp /kaggle/input/datasets/arafat120/vimdr-checkpoints/checkpoints/VimDR_fold2_RESUME.pth /kaggle/working/checkpoints/\n#!cp /kaggle/input/datasets/arafat120/vimdr-checkpoints/checkpoints/VimDR_fold2_BEST.pth /kaggle/working/checkpoints/\n\n# ৩. ট্রেইনিং শুরু করা (এবার এটি ঠিক ৭ নম্বর ইপোক থেকে শুরু হবে)\n#!python vimdr_train_kaggle.py","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-04T04:48:04.710184Z","iopub.execute_input":"2026-06-04T04:48:04.710612Z","iopub.status.idle":"2026-06-04T04:48:05.099404Z","shell.execute_reply.started":"2026-06-04T04:48:04.710577Z","shell.execute_reply":"2026-06-04T04:48:05.098661Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#!zip -r /kaggle/working/vimdr_fold1_backup.zip /kaggle/working/preprocessed /kaggle/working/checkpoints /kaggle/working/outputs","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}