{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# ১. পুরনো ডেটাসেট ক্যাশ মুছুন (যাতে এবার ৫০০০+ ছবি লোড হয়)\n#!rm -f /kaggle/working/preprocessed/manifest_384.csv\n\n# ২. পুরনো RESUME ফাইল মুছুন (এটি খুব জরুরি, যাতে ইপোক ১ থেকে ফ্রেশ লার্নিং রেট নিয়ে শুরু হয়)\n#!rm -f /kaggle/working/checkpoints/VimDR_fold1_RESUME.pth","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-05T19:12:47.779125Z","iopub.execute_input":"2026-06-05T19:12:47.779488Z","iopub.status.idle":"2026-06-05T19:12:48.010823Z","shell.execute_reply.started":"2026-06-05T19:12:47.779462Z","shell.execute_reply":"2026-06-05T19:12:48.009787Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%writefile vimdr_model_kaggle.py\nimport math, functools\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.checkpoint import checkpoint\n\ndef _maybe_compile(module: nn.Module) -> nn.Module:\n    try:\n        return torch.compile(module, mode='reduce-overhead', fullgraph=False)\n    except Exception:\n        return module\n\nclass SelectiveScan1D(nn.Module):\n    def __init__(self, dim: int, d_state: int = 16, expand: int = 2):\n        super().__init__()\n        self.d_inner = dim * expand\n        self.d_state = d_state\n        self.in_proj  = nn.Linear(dim, self.d_inner * 2, bias=False)\n        self.conv1d   = nn.Conv1d(self.d_inner, self.d_inner, kernel_size=4,\n                                  padding=3, groups=self.d_inner, bias=True)\n        self.dt_proj  = nn.Linear(self.d_inner, self.d_inner, bias=True)\n        self.B_proj   = nn.Linear(self.d_inner, d_state, bias=False)\n        self.C_proj   = nn.Linear(self.d_inner, d_state, bias=False)\n        self.A_log    = nn.Parameter(\n            torch.log(torch.arange(1, d_state + 1).float())\n                .unsqueeze(0).repeat(self.d_inner, 1))\n        self.D        = nn.Parameter(torch.ones(self.d_inner))\n        self.out_proj = nn.Linear(self.d_inner, dim, bias=False)\n        self.norm     = nn.LayerNorm(self.d_inner)\n        nn.init.constant_(self.dt_proj.bias, math.log(math.expm1(1.0)))\n\n    @staticmethod\n    def _parallel_scan(dA: torch.Tensor, dBu: torch.Tensor) -> torch.Tensor:\n        B, L, D, S = dA.shape\n        L_pad = 1 << (L - 1).bit_length()\n        pad   = L_pad - L\n        if pad:\n            dA  = F.pad(dA,  (0, 0, 0, 0, 0, pad))\n            dBu = F.pad(dBu, (0, 0, 0, 0, 0, pad))\n\n        a, b = dA.clone(), dBu.clone()\n        stride = 1\n        while stride < L_pad:\n            a_next = a.clone()\n            b_next = b.clone()\n            left_a  = a[:, stride - 1::2 * stride]\n            left_b  = b[:, stride - 1::2 * stride]\n            right_a = a[:, 2 * stride - 1::2 * stride]\n            right_b = b[:, 2 * stride - 1::2 * stride]\n            \n            b_next[:, 2 * stride - 1::2 * stride] = right_a * left_b + right_b\n            a_next[:, 2 * stride - 1::2 * stride] = right_a * left_a\n            \n            a, b = a_next, b_next\n            stride *= 2\n\n        b_next = b.clone()\n        b_next[:, L_pad - 1] = 0\n        b = b_next\n        \n        stride = L_pad // 2\n        while stride >= 1:\n            b_next = b.clone()\n            tmp_a = a[:, stride - 1::2 * stride]\n            tmp_b = b[:, stride - 1::2 * stride]\n            \n            b_next[:, stride - 1::2 * stride] = b[:, 2 * stride - 1::2 * stride]\n            b_next[:, 2 * stride - 1::2 * stride] = tmp_a * b[:, 2 * stride - 1::2 * stride] + tmp_b\n            \n            b = b_next\n            stride //= 2\n\n        h = dA * b + dBu\n        return h[:, :L]\n\n    def forward(self, x: torch.Tensor) -> torch.Tensor:\n        B, L, _ = x.shape\n        xr, z   = self.in_proj(x).chunk(2, dim=-1)\n        xr = F.silu(self.conv1d(xr.transpose(1, 2))[:, :, :L].transpose(1, 2))\n        dt  = F.softplus(self.dt_proj(xr))\n        Bm  = self.B_proj(xr)\n        Cm  = self.C_proj(xr)\n        A   = -torch.exp(self.A_log.float())\n        dA  = torch.exp(torch.einsum('bld,ds->blds', dt, A))\n        dBu = torch.einsum('bld,bls->blds', dt, Bm)\n        h   = self._parallel_scan(dA, dBu)\n        y   = (h * Cm.unsqueeze(2)).sum(-1) + xr * self.D\n        return self.out_proj(self.norm(y) * F.silu(z))\n\nclass CrossDirectionalSSM(nn.Module):\n    def __init__(self, dim: int, d_state: int = 16):\n        super().__init__()\n        assert dim % 4 == 0\n        c = dim // 4\n        self.ssm_h  = SelectiveScan1D(c, d_state)\n        self.ssm_v  = SelectiveScan1D(c, d_state)\n        self.ssm_d  = SelectiveScan1D(c, d_state)\n        self.ssm_a  = SelectiveScan1D(c, d_state)\n        self.fusion = nn.Sequential(nn.Linear(dim, dim, bias=False), nn.LayerNorm(dim))\n        self.norm   = nn.LayerNorm(dim)\n        self._idx_cache: dict = {}\n\n    def _get_diag_index(self, H: int, W: int, anti: bool, device: torch.device):\n        key = (H, W, anti, device)\n        if key not in self._idx_cache:\n            indices = []\n            for s in range(H + W - 1):\n                for i in range(max(0, s - W + 1), min(H, s + 1)):\n                    j = s - i\n                    if anti: j = W - 1 - j\n                    indices.append(i * W + j)\n            idx     = torch.tensor(indices, dtype=torch.long, device=device)\n            inv_idx = torch.argsort(idx)\n            self._idx_cache[key] = (idx, inv_idx)\n        return self._idx_cache[key]\n\n    def _diag_scan(self, x: torch.Tensor, H: int, W: int, anti: bool = False) -> torch.Tensor:\n        idx, inv_idx = self._get_diag_index(H, W, anti, x.device)\n        ssm = self.ssm_a if anti else self.ssm_d\n        return ssm(x[:, idx, :])[:, inv_idx, :]\n\n    def forward(self, x: torch.Tensor, H: int, W: int) -> torch.Tensor:\n        x  = self.norm(x)\n        c   = x.shape[-1] // 4\n        x1, x2, x3, x4 = (x[..., i * c:(i + 1) * c] for i in range(4))\n        out_h = self.ssm_h(x1)\n        B, L, Dc = x2.shape\n        out_v = (self.ssm_v(x2.view(B, H, W, Dc).permute(0, 2, 1, 3).reshape(B, L, Dc))\n                    .view(B, W, H, Dc).permute(0, 2, 1, 3).reshape(B, L, Dc))\n        out_d = self._diag_scan(x3, H, W, anti=False)\n        out_a = self._diag_scan(x4, H, W, anti=True)\n        return x + self.fusion(torch.cat([out_h, out_v, out_d, out_a], dim=-1))\n\nclass VesselMorphologicalBlock(nn.Module):\n    def __init__(self, dim: int, d_state: int = 16, dropout: float = 0.1):\n        super().__init__()\n        self.vessel_prior = nn.ModuleList([\n            nn.Sequential(\n                nn.Conv2d(dim, dim // 4, 1, bias=False),\n                nn.Conv2d(dim // 4, dim // 4, 3, padding=r, dilation=r,\n                          groups=dim // 4, bias=False),\n                nn.BatchNorm2d(dim // 4), nn.GELU(),\n            ) for r in [1, 2, 4, 8]\n        ])\n        self.prior_proj = nn.Conv2d(dim, dim, 1, bias=False)\n        self.prior_gate = nn.Sigmoid()\n        self.ssm   = CrossDirectionalSSM(dim, d_state)\n        self.norm  = nn.LayerNorm(dim)\n        self.norm2 = nn.LayerNorm(dim)\n        self.ffn   = nn.Sequential(\n            nn.Linear(dim, dim * 4, bias=False), nn.GELU(),\n            nn.Dropout(dropout), nn.Linear(dim * 4, dim, bias=False),\n        )\n\n    def forward(self, x: torch.Tensor, H: int, W: int) -> torch.Tensor:\n        B, L, D = x.shape\n        feat  = x.transpose(1, 2).view(B, D, H, W)\n        prior = self.prior_gate(\n            self.prior_proj(torch.cat([b(feat) for b in self.vessel_prior], dim=1))\n        ).flatten(2).transpose(1, 2)\n        x = x + self.ssm(self.norm(x), H, W) * (1.0 + prior)\n        return x + self.ffn(self.norm2(x))\n\nclass AdaptivePathologyScaleDetector(nn.Module):\n    def __init__(self, dim: int):\n        super().__init__()\n        hd = dim // 4\n        self.scale_convs = nn.ModuleList([\n            nn.Sequential(\n                nn.Conv2d(dim, hd, k, padding=k // 2, groups=hd, bias=False),\n                nn.BatchNorm2d(hd), nn.GELU(),\n            ) for k in [1, 3, 5, 7]\n        ])\n        self.fusion     = nn.Conv2d(dim, dim, 1, bias=False)\n        self.norm_bn    = nn.BatchNorm2d(dim)\n        self.scale_attn = nn.Sequential(\n            nn.AdaptiveAvgPool2d(1), nn.Flatten(),\n            nn.Linear(dim, dim // 4), nn.ReLU(),\n            nn.Linear(dim // 4, 4), nn.Softmax(dim=-1),\n        )\n\n    def forward(self, x: torch.Tensor, H: int, W: int) -> torch.Tensor:\n        B, L, D = x.shape\n        feat = x.transpose(1, 2).view(B, D, H, W)\n        outs = [conv(feat) for conv in self.scale_convs]\n        out  = self.norm_bn(self.fusion(torch.cat(outs, dim=1)))\n        return x + out.flatten(2).transpose(1, 2)\n\nclass OrdinalUncertaintyHead(nn.Module):\n    def __init__(self, in_dim: int, num_classes: int = 5, dropout: float = 0.3):\n        super().__init__()\n        self.num_classes = num_classes\n        self.proj = nn.Sequential(\n            nn.Linear(in_dim, in_dim // 2), nn.GELU(), nn.Dropout(dropout),\n            nn.Linear(in_dim // 2, in_dim // 4), nn.GELU(), nn.Dropout(dropout),\n        )\n        self.ordinal_fc  = nn.Linear(in_dim // 4, num_classes - 1)\n        self.reg_fc      = nn.Linear(in_dim // 4, 1)\n        self.temperature = nn.Parameter(torch.ones(1))\n\n    def forward(self, x: torch.Tensor) -> dict:\n        feat = self.proj(x)\n        return {\n            'grade_logits': self.ordinal_fc(feat) / (self.temperature.abs() + 1e-6),\n            'reg_score':    self.reg_fc(feat),\n            'features':     feat,\n        }\n\n    def predict_grade(self, logits: torch.Tensor) -> torch.Tensor:\n        # ✅ FIXED: Changed to robust thresholding instead of sigmoid summation\n        return torch.clamp((logits > 0.0).sum(dim=-1), 0, self.num_classes - 1).long()\n\n    @staticmethod\n    def logits_to_probs(logits: torch.Tensor) -> torch.Tensor:\n        p    = torch.sigmoid(logits)\n        p0   = 1 - p[:, :1]\n        pmid = p[:, :-1] - p[:, 1:]\n        pK   = p[:, -1:]\n        return torch.cat([p0, pmid, pK], dim=-1).clamp(min=1e-7)\n\nclass VimDRPlus(nn.Module):\n    def __init__(\n        self,\n        embed_dim:   int   = 192,       \n        depth:       int   = 8,         \n        num_classes: int   = 5,\n        d_state:     int   = 16,        \n        dropout:     float = 0.3,\n        use_compile: bool  = True,\n    ):\n        super().__init__()\n        assert depth % 2 == 0\n        self.patch_embed = nn.Sequential(\n            nn.Conv2d(3, embed_dim // 4, 4, stride=4, bias=False),\n            nn.BatchNorm2d(embed_dim // 4), nn.GELU(),\n            nn.Conv2d(embed_dim // 4, embed_dim // 2, 2, stride=2, bias=False),\n            nn.BatchNorm2d(embed_dim // 2), nn.GELU(),\n            nn.Conv2d(embed_dim // 2, embed_dim, 2, stride=2, bias=False),\n            nn.BatchNorm2d(embed_dim),\n        )\n        self.pos_drop = nn.Dropout(0.1)\n        self.blocks   = nn.ModuleList([\n            m for _ in range(depth // 2)\n            for m in (\n                AdaptivePathologyScaleDetector(embed_dim),\n                VesselMorphologicalBlock(embed_dim, d_state, dropout=0.1),\n            )\n        ])\n        self.norm = nn.LayerNorm(embed_dim)\n        self.pool = nn.AdaptiveAvgPool1d(1)\n        self.head = OrdinalUncertaintyHead(embed_dim, num_classes, dropout)\n        self.apply(self._init_weights)\n        if use_compile:\n            self.patch_embed = _maybe_compile(self.patch_embed)\n\n    @staticmethod\n    def _init_weights(m):\n        if isinstance(m, nn.Linear):\n            nn.init.trunc_normal_(m.weight, std=0.02)\n            if m.bias is not None: nn.init.zeros_(m.bias)\n        elif isinstance(m, (nn.LayerNorm, nn.BatchNorm2d)):\n            nn.init.ones_(m.weight)\n            nn.init.zeros_(m.bias)\n        elif isinstance(m, nn.Conv2d):\n            nn.init.kaiming_normal_(m.weight, mode='fan_out')\n            if m.bias is not None: nn.init.zeros_(m.bias)\n\n    @staticmethod\n    def _pos_embed(H: int, W: int, D: int, device) -> torch.Tensor:\n        pe  = torch.zeros(1, H * W, D, device=device)\n        div = torch.exp(\n            torch.arange(0, D, 2, device=device, dtype=torch.float)\n            * (-math.log(10000.0) / D)\n        )\n        y  = torch.arange(H, device=device).float().unsqueeze(1)\n        xc = torch.arange(W, device=device).float().unsqueeze(1)\n        pe[0, :, 0::2] = (torch.sin(y * div).unsqueeze(1)\n                           + torch.sin(xc * div).unsqueeze(0)).reshape(H * W, -1)\n        pe[0, :, 1::2] = (torch.cos(y * div).unsqueeze(1)\n                           + torch.cos(xc * div).unsqueeze(0)).reshape(H * W, -1)\n        return pe\n\n    def forward(self, x: torch.Tensor) -> dict:\n        x        = self.patch_embed(x)\n        B, D, H, W = x.shape\n        tokens   = x.flatten(2).transpose(1, 2)\n        tokens   = self.pos_drop(tokens + self._pos_embed(H, W, D, x.device))\n        \n        for block in self.blocks:\n            tokens = checkpoint(block, tokens, H, W, use_reentrant=False)\n            \n        pooled   = self.pool(self.norm(tokens).transpose(1, 2)).flatten(1)\n        return self.head(pooled)\n\n    @torch.inference_mode()\n    def predict_with_uncertainty(self, x: torch.Tensor, n_samples: int = 10) -> dict:\n        def _set_modes(m):\n            if isinstance(m, (nn.BatchNorm1d, nn.BatchNorm2d, nn.LayerNorm)):\n                m.eval()\n            elif isinstance(m, (nn.Dropout, nn.Dropout2d)):\n                m.train()\n        self.apply(_set_modes)\n        preds = torch.stack(\n            [self.head.predict_grade(self.forward(x)['grade_logits']).float()\n             for _ in range(n_samples)], dim=0\n        )\n        self.eval()\n        return {\n            'mean_grade':  torch.clamp(torch.round(preds.mean(0)), 0,\n                                        self.head.num_classes - 1).long(),\n            'uncertainty': preds.std(0),\n        }\n\n    def param_count(self) -> int:\n        return sum(p.numel() for p in self.parameters() if p.requires_grad)\n\nclass FocalOrdinalLoss(nn.Module):\n    def __init__(self, num_classes=5, lambda_kl=0.5, gamma=2.0, label_smooth=0.1):\n        super().__init__()\n        self.K, self.lk, self.gamma, self.ls = num_classes, lambda_kl, gamma, label_smooth\n        self.register_buffer('_k_range', torch.arange(num_classes, dtype=torch.float))\n\n    def _smooth_dist(self, target: torch.Tensor) -> torch.Tensor:\n        B, K, ls = target.shape[0], self.K, self.ls\n        dist = torch.zeros(B, K, device=target.device)\n        g    = target.view(B, 1)                \n        dist.scatter_(1, g, 1.0 - ls)\n        left = (g - 1).clamp(min=0)\n        left_mask = (g > 0).float()\n        dist.scatter_add_(1, left, left_mask * (ls / 2.0))\n        right = (g + 1).clamp(max=K - 1)\n        right_mask = (g < K - 1).float()\n        dist.scatter_add_(1, right, right_mask * (ls / 2.0))\n        return dist\n\n    def forward(self, out: dict, target: torch.Tensor):\n        logit = out['grade_logits']\n        reg   = out['reg_score'].squeeze(-1)\n        tf    = target.float()\n        \n        mse       = (reg - tf) ** 2\n        focal_mse = ((1 - torch.exp(-mse)) ** self.gamma * mse).mean()\n        ordinal_tgt = torch.stack([(target > k).float() for k in range(self.K - 1)], dim=1)\n        bce_loss = F.binary_cross_entropy_with_logits(logit, ordinal_tgt)\n        log_pred = F.log_softmax(-torch.abs(reg.unsqueeze(1) - self._k_range), dim=-1)\n        dist    = self._smooth_dist(target)\n        kl_loss = F.kl_div(log_pred, dist, reduction='batchmean')\n        \n        probs = OrdinalUncertaintyHead.logits_to_probs(logit)\n        target_onehot = F.one_hot(target, self.K).float()\n        W = torch.arange(self.K).unsqueeze(1) - torch.arange(self.K).unsqueeze(0)\n        W = (W ** 2).float().to(probs.device)\n        O = torch.matmul(target_onehot.T, probs)\n        E = torch.matmul(target_onehot.sum(0).unsqueeze(1), probs.sum(0).unsqueeze(0)) / target.size(0)\n        qwk_loss = (W * O).sum() / ((W * E).sum() + 1e-6)\n        \n        total = focal_mse + bce_loss + self.lk * kl_loss + 0.5 * qwk_loss\n        \n        return total, {'focal_mse': focal_mse.item(), 'bce': bce_loss.item(), 'kl': kl_loss.item(), 'qwk_loss': qwk_loss.item()}\n\nif __name__ == '__main__':\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    model  = VimDRPlus(embed_dim=192, depth=8, num_classes=5, d_state=16).to(device)\n    x      = torch.randn(2, 3, 384, 384, device=device)   \n    with torch.no_grad():\n        out = model(x)\n    logits = out['grade_logits']\n    probs  = OrdinalUncertaintyHead.logits_to_probs(logits)\n    print(f\"Parameters  : {model.param_count():,}\")\n    print(\"Kaggle model sanity check passed.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%writefile vimdr_train_kaggle.py\n# -*- coding: utf-8 -*-\nimport os, cv2, json, random, hashlib, math, warnings, gc\nimport multiprocessing as mp\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.cuda.amp import GradScaler, autocast\nfrom torch.utils.data import Dataset, DataLoader, WeightedRandomSampler\nfrom torchvision import transforms\nfrom sklearn.model_selection import StratifiedKFold, train_test_split\nfrom sklearn.metrics import cohen_kappa_score, accuracy_score, roc_auc_score\nfrom scipy.optimize import minimize\nfrom tqdm import tqdm\nfrom vimdr_model_kaggle import VimDRPlus, FocalOrdinalLoss, OrdinalUncertaintyHead\n\nwarnings.filterwarnings('ignore')\n\nSEED = 42\nrandom.seed(SEED)\nnp.random.seed(SEED)\ntorch.manual_seed(SEED); torch.cuda.manual_seed_all(SEED)\ntorch.backends.cudnn.deterministic = False\ntorch.backends.cudnn.benchmark     = True\n\nCONFIG = dict(\n    data_dir   = \"/kaggle/working/preprocessed\",\n    ckpt_dir   = \"/kaggle/working/checkpoints\",\n    output_dir = \"/kaggle/working/outputs\",\n    embed_dim     = 192,\n    depth         = 8,\n    num_classes   = 5,\n    d_state       = 16,\n    dropout_rate  = 0.2,\n    use_compile   = False,\n    batch_size    = 16,              \n    grad_accum    = 2,               \n    total_epochs  = 35,  \n    warmup_epochs = 3,          \n    lr            = 3e-4, # ✅ FIXED: Increased LR for better convergence\n    min_lr        = 5e-7, \n    weight_decay  = 1e-4,\n    early_stop_pat= 15,              \n    lambda_kl     = 0.5, \n    label_smooth  = 0.1, \n    val_every     = 1,    \n    resolution    = 384,             \n    dr2015_frac   = 0.25, \n    mixup_alpha   = 0.1,  \n    mixup_prob    = 0.05,             \n    mc_samples    = 10,\n    num_workers   = 2,    # ✅ FIXED: Changed to 2 for faster IO without freezing\n    prefetch      = None,              \n    n_folds       = 5,\n    fold_to_run   = 1,    # Mapping internally       \n    seed          = SEED,\n)\n\nGRADE_NAMES = ['No DR', 'Mild', 'Moderate', 'Severe', 'Proliferative']\n\nAPTOS_PATH  = \"/kaggle/input/competitions/aptos2019-blindness-detection\"\nDR2015_PATH = \"/kaggle/input/datasets/tanlikesmath/diabetic-retinopathy-resized\"\n\nfor d in [CONFIG['data_dir'], CONFIG['ckpt_dir'], CONFIG['output_dir']]:\n    os.makedirs(d, exist_ok=True)\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\ndef _preprocess_one(args):\n    path, size, cache_dir = args\n    uid   = hashlib.md5(f\"{path}{size}\".encode()).hexdigest()[:12]\n    npath = os.path.join(cache_dir, f\"{uid}.npy\")\n    if os.path.exists(npath): return npath\n\n    img = cv2.imread(path)\n    if img is None: return None\n    \n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    img = cv2.resize(img, (size, size))\n    if img.mean() < 0.15 * 255: return None\n\n    clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8))\n    g_ch  = clahe.apply(img[:, :, 1])\n    blur  = cv2.GaussianBlur(img, (0, 0), size // 30)\n    enh   = cv2.addWeighted(img, 4, blur, -4, 128)\n    enh[:, :, 1] = np.clip(\n        enh[:, :, 1].astype(np.float32) * 0.7 + g_ch.astype(np.float32) * 0.3,\n        0, 255\n    ).astype(np.uint8)\n    \n    mask = np.zeros(enh.shape[:2], dtype=np.uint8)\n    cv2.circle(mask, (size // 2, size // 2), int(size * 0.475), 1, -1)\n    out = (enh * mask[:, :, np.newaxis]).astype(np.uint8)\n    np.save(npath, out)\n    return npath\n\ndef preprocess_and_cache(df: pd.DataFrame, res: int) -> pd.DataFrame:\n    cache_dir = os.path.join(CONFIG['data_dir'], f'cache_{res}')\n    os.makedirs(cache_dir, exist_ok=True)\n    args = [(row['path'], res, cache_dir) for _, row in df.iterrows()]\n    n_workers = min(CONFIG['num_workers'], mp.cpu_count())\n    npaths = []\n    try:\n        with mp.Pool(n_workers) as pool:\n            npaths = list(tqdm(pool.imap(_preprocess_one, args, chunksize=16), total=len(args), desc=f'Cache {res}px (parallel)'))\n    except Exception:\n        npaths = [_preprocess_one(a) for a in tqdm(args, desc=f'Cache {res}px')]\n    valid_rows = []\n    for (_, row), npath in zip(df.iterrows(), npaths):\n        if npath is not None:\n            valid_rows.append({**row.to_dict(), 'npy_path': npath})\n    return pd.DataFrame(valid_rows).reset_index(drop=True)\n\ndef load_aptos(base: str) -> pd.DataFrame:\n    df         = pd.read_csv(f'{base}/train.csv')\n    df['path'] = df['id_code'].apply(lambda x: f'{base}/train_images/{x}.png')\n    df['source'] = 'APTOS2019'\n    return df[['path', 'diagnosis', 'source']].rename(columns={'diagnosis': 'grade'})\n\ndef load_dr2015(base: str, sample_frac: float = 0.15) -> pd.DataFrame:\n    csv = f'{base}/trainLabels.csv'\n    if not os.path.exists(csv): \n        return pd.DataFrame(columns=['path', 'grade', 'source'])\n    df         = pd.read_csv(csv)\n    df['path'] = df['image'].apply(lambda x: f'{base}/resized_train/resized_train/{x}.jpeg')\n    df['source'] = 'DR2015'\n    df         = df.rename(columns={'level': 'grade'})[['path', 'grade', 'source']]\n    df         = df[df['path'].apply(os.path.exists)]\n    return (df.groupby('grade', group_keys=False).apply(lambda g: g.sample(frac=sample_frac, random_state=SEED)).reset_index(drop=True))\n\ndef build_dataset() -> pd.DataFrame:\n    df = pd.concat([load_aptos(APTOS_PATH), load_dr2015(DR2015_PATH, sample_frac=CONFIG['dr2015_frac'])], ignore_index=True)\n    df = df[df['path'].apply(os.path.exists)].reset_index(drop=True)\n    df = df[df['path'].apply(lambda p: os.path.getsize(p) > 5000)].reset_index(drop=True)\n    return df\n\ndef create_splits(df: pd.DataFrame) -> pd.DataFrame:\n    df = df.copy()\n    df['split'] = 'train'\n    _, test_idx = train_test_split(df.index, test_size=0.15, stratify=df['grade'], random_state=SEED)\n    df.loc[test_idx, 'split'] = 'test'\n    return df\n\nclass DRDataset(Dataset):\n    _MEAN = [0.485, 0.456, 0.406]\n    _STD  = [0.229, 0.224, 0.225]\n    TRAIN_AUG = transforms.Compose([\n        transforms.ToPILImage(), transforms.RandomHorizontalFlip(0.5), transforms.RandomVerticalFlip(0.5),\n        transforms.RandomRotation(15), transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.1, hue=0.05),\n        transforms.RandomAffine(degrees=0, shear=8, scale=(0.92, 1.08)), transforms.ToTensor(),\n        transforms.Normalize(_MEAN, _STD), transforms.RandomErasing(p=0.15, scale=(0.02, 0.08)),\n    ])\n    EVAL_AUG = transforms.Compose([\n        transforms.ToPILImage(), transforms.ToTensor(), transforms.Normalize(_MEAN, _STD),\n    ])\n\n    def __init__(self, df: pd.DataFrame, split: str, resolution: int = 384):\n        self.df  = df[df['split'] == split].reset_index(drop=True)\n        self.res = resolution\n        self.aug = self.TRAIN_AUG if split == 'train' else self.EVAL_AUG\n\n    def __len__(self):  return len(self.df)\n    def __getitem__(self, i: int):\n        row = self.df.iloc[i]\n        img = np.load(row['npy_path']).astype(np.uint8)\n        if img.shape[0] != self.res: img = cv2.resize(img, (self.res, self.res))\n        return self.aug(img), int(row['grade'])\n\ndef build_loaders(df: pd.DataFrame, batch_size: int):\n    res = CONFIG['resolution']\n    train_ds = DRDataset(df, 'train', res)\n    val_ds   = DRDataset(df, 'val',   res)\n    test_ds  = DRDataset(df, 'test',  res)\n    grades  = train_ds.df['grade'].values.astype(int)\n    counts  = np.bincount(grades, minlength=CONFIG['num_classes'])\n    weights = 1.0 / (counts + 1)\n    sampler = WeightedRandomSampler(torch.tensor(weights[grades], dtype=torch.float), num_samples=len(grades), replacement=True)\n    kw = dict(num_workers=CONFIG['num_workers'], pin_memory=True, persistent_workers=False, prefetch_factor=CONFIG['prefetch'])\n    return (\n        DataLoader(train_ds, batch_size=batch_size, sampler=sampler, **kw),\n        DataLoader(val_ds,   batch_size=batch_size * 2, shuffle=False, **kw),\n        DataLoader(test_ds,  batch_size=batch_size * 2, shuffle=False, **kw),\n    )\n\nclass WarmupCosineScheduler:\n    def __init__(self, opt, warmup, total, base_lr, min_lr):\n        self.opt, self.warmup, self.total = opt, warmup, total\n        self.base_lr, self.min_lr = base_lr, min_lr\n        self._step = 0\n    def step(self) -> float:\n        self._step += 1\n        s = self._step\n        if s <= self.warmup: lr = self.base_lr * s / self.warmup\n        else:\n            prog = (s - self.warmup) / max(1, self.total - self.warmup)\n            lr   = self.min_lr + 0.5 * (self.base_lr - self.min_lr) * (1 + math.cos(math.pi * prog))\n        for pg in self.opt.param_groups: pg['lr'] = lr\n        return lr\n    def state_dict(self): return {'step': self._step}\n    def load_state_dict(self, d): self._step = d['step']\n\nclass MixUp:\n    def __init__(self, alpha=0.4): self.alpha = alpha\n    def __call__(self, x, y):\n        lam = float(np.random.beta(self.alpha, self.alpha))\n        idx = torch.randperm(x.size(0), device=x.device)\n        return lam * x + (1 - lam) * x[idx], y, y[idx], lam\n\ndef mixup_criterion(criterion, out, ya, yb, lam):\n    la, _ = criterion(out, ya)\n    lb, _ = criterion(out, yb)\n    return lam * la + (1 - lam) * lb\n\n@torch.no_grad()\ndef evaluate(model, loader, thresholds=None) -> dict:\n    model.eval()\n    all_labels, all_preds_cont, all_logits = [], [], []\n    \n    for imgs, labels in loader:\n        imgs = imgs.to(device)\n        \n        out_orig = model(imgs)\n        out_flip = model(torch.flip(imgs, dims=[-1]))\n        \n        logits = (out_orig['grade_logits'] + out_flip['grade_logits']) / 2.0\n        cont_pred = torch.sigmoid(logits).sum(dim=-1)\n        \n        all_preds_cont.extend(cont_pred.cpu().tolist())\n        all_labels.extend(labels.tolist())\n        all_logits.append(logits.cpu())\n        \n    y_true = np.array(all_labels)\n    y_pred_cont = np.array(all_preds_cont)\n    logits_cat = torch.cat(all_logits, dim=0)\n    probs = OrdinalUncertaintyHead.logits_to_probs(logits_cat).numpy()\n    \n    if thresholds is None:\n        thresholds = [0.5, 1.5, 2.5, 3.5]\n        \n    y_pred_classes = np.clip(np.digitize(y_pred_cont, bins=thresholds), 0, CONFIG['num_classes'] - 1)\n    \n    qwk = cohen_kappa_score(y_true, y_pred_classes, weights='quadratic')\n    acc = accuracy_score(y_true, y_pred_classes)\n    \n    aucs = []\n    for k in range(CONFIG['num_classes']):\n        pos = (y_true == k).astype(int)\n        if pos.sum() > 0 and (1 - pos).sum() > 0: \n            aucs.append(roc_auc_score(pos, probs[:, k]))\n    mean_auc = float(np.mean(aucs)) if aucs else 0.0\n    \n    return {'qwk': qwk, 'accuracy': acc, 'mean_auc': mean_auc, 'thresholds': list(thresholds)}\n\ndef train_one_epoch(model, loader, optimizer, criterion, scaler, mixup, grad_accum, epoch) -> dict:\n    model.train()\n    losses, step = [], 0\n    optimizer.zero_grad(set_to_none=True)\n    pbar = tqdm(loader, desc=f'Ep{epoch:03d}', leave=False, mininterval=5.0)\n    for imgs, labels in pbar:\n        imgs, labels = imgs.to(device, non_blocking=True), labels.to(device, non_blocking=True)\n        use_mix = (np.random.rand() < CONFIG['mixup_prob'])\n        with autocast():\n            if use_mix:\n                imgs_m, ya, yb, lam = mixup(imgs, labels)\n                loss = mixup_criterion(criterion, model(imgs_m), ya, yb, lam)\n            else:\n                loss, _ = criterion(model(imgs), labels)\n        scaler.scale(loss / grad_accum).backward()\n        step += 1\n        if step % grad_accum == 0:\n            scaler.unscale_(optimizer)\n            nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n            scaler.step(optimizer)\n            scaler.update()\n            optimizer.zero_grad(set_to_none=True)\n        losses.append(loss.item())\n        pbar.set_postfix({'loss': f\"{np.mean(losses[-30:]):.4f}\"})\n    if step % grad_accum != 0:\n        scaler.unscale_(optimizer)\n        nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n        scaler.step(optimizer)\n        scaler.update()\n        optimizer.zero_grad(set_to_none=True)\n    return {'loss': float(np.mean(losses))}\n\ndef save_checkpoint(path, model, optimizer, scheduler, scaler, epoch, best_qwk, patience, history):\n    m_state = (model.module.state_dict() if hasattr(model, 'module') else model.state_dict())\n    torch.save({\n        'model': m_state, 'optimizer': optimizer.state_dict(), 'scheduler': scheduler.state_dict(),\n        'scaler': scaler.state_dict(), 'epoch': epoch, 'best_qwk': best_qwk, 'patience': patience, 'history': history,\n    }, path)\n\ndef load_checkpoint(path, model, optimizer, scheduler, scaler):\n    ckpt = torch.load(path, map_location=device, weights_only=False)\n    m_inner = model.module if hasattr(model, 'module') else model\n    m_inner.load_state_dict(ckpt['model'])\n    optimizer.load_state_dict(ckpt['optimizer'])\n    scheduler.load_state_dict(ckpt['scheduler'])\n    scaler.load_state_dict(ckpt['scaler'])\n    return (ckpt['epoch'], ckpt['best_qwk'], ckpt['patience'], ckpt['history'])\n\ndef train_single_fold(df_train: pd.DataFrame, df_test: pd.DataFrame, fold: int) -> dict:\n    print(f\"\\n{'='*60}\\n  FOLD {fold+1}/{CONFIG['n_folds']}\\n{'='*60}\")\n    skf = StratifiedKFold(n_splits=CONFIG['n_folds'], shuffle=True, random_state=SEED)\n    folds = list(skf.split(df_train.index, df_train['grade']))\n    tr_idx, val_idx = folds[fold]\n    df_fold = df_train.copy()\n    train_paths = set(df_train.iloc[tr_idx]['path'])\n    val_paths   = set(df_train.iloc[val_idx]['path'])\n    df_fold['split'] = df_fold['path'].apply(lambda p: 'train' if p in train_paths else ('val' if p in val_paths else 'skip'))\n    df_fold = df_fold[df_fold['split'] != 'skip'].reset_index(drop=True)\n    df_test_copy = df_test.copy()\n    df_test_copy['split'] = 'test'\n    df_fold = pd.concat([df_fold, df_test_copy], ignore_index=True)\n    train_loader, val_loader, test_loader = build_loaders(df_fold, CONFIG['batch_size'])\n    \n    print(f\"  Train: {len(train_loader.dataset):,}  Val: {len(val_loader.dataset):,}  Test: {len(test_loader.dataset):,}\")\n    model = VimDRPlus(embed_dim=CONFIG['embed_dim'], depth=CONFIG['depth'], num_classes=CONFIG['num_classes'], d_state=CONFIG['d_state'], dropout=CONFIG['dropout_rate'], use_compile=CONFIG['use_compile']).to(device)\n    \n    if torch.cuda.device_count() > 1:\n        model = nn.DataParallel(model)\n        \n    print(f\"  Params: {model.param_count() if not hasattr(model, 'module') else sum(p.numel() for p in model.parameters() if p.requires_grad):,}\")\n    \n    optimizer = torch.optim.AdamW(model.parameters(), lr=CONFIG['lr'], weight_decay=CONFIG['weight_decay'])\n    scheduler = WarmupCosineScheduler(optimizer, warmup=CONFIG['warmup_epochs'], total=CONFIG['total_epochs'], base_lr=CONFIG['lr'], min_lr=CONFIG['min_lr'])\n\n    pretrained_path = f\"{CONFIG['ckpt_dir']}/VimDR_fold{fold+1}_BEST.pth\"\n    if os.path.exists(pretrained_path):\n        print(f\"  [+] Loading Pre-trained Weights from {pretrained_path}...\")\n        ckpt = torch.load(pretrained_path, map_location=device, weights_only=False)\n        m_inner = model.module if hasattr(model, 'module') else model\n        m_inner.load_state_dict(ckpt['state_dict'])\n    \n    criterion = FocalOrdinalLoss(num_classes=CONFIG['num_classes'], lambda_kl=CONFIG['lambda_kl'], label_smooth=CONFIG['label_smooth']).to(device)\n    scaler = GradScaler()\n    mixup  = MixUp(alpha=CONFIG['mixup_alpha'])\n\n    resume_path = f\"{CONFIG['ckpt_dir']}/VimDR_fold{fold+1}_RESUME.pth\"\n    best_path   = f\"{CONFIG['ckpt_dir']}/VimDR_fold{fold+1}_BEST.pth\"\n    start_epoch, best_qwk, patience = 1, -1.0, 0\n    best_thresholds = [0.5, 1.5, 2.5, 3.5]\n    history = {'train_loss': [], 'val_qwk': [], 'val_acc': [], 'val_auc': [], 'lr': []}\n\n    if os.path.exists(resume_path):\n        print(f\"  ▶ Resuming from {resume_path}\")\n        start_epoch, best_qwk, patience, history = load_checkpoint(resume_path, model, optimizer, scheduler, scaler)\n        start_epoch += 1\n        print(f\"  Resumed at epoch {start_epoch}, best QWK={best_qwk:.4f}\")\n\n    current_epoch = start_epoch\n    try:\n        for epoch in range(start_epoch, CONFIG['total_epochs'] + 1):\n            current_epoch = epoch\n            train_stats = train_one_epoch(model, train_loader, optimizer, criterion, scaler, mixup, CONFIG['grad_accum'], epoch)\n            lr = scheduler.step()\n            history['train_loss'].append(train_stats['loss'])\n            history['lr'].append(lr)\n\n            if epoch % CONFIG['val_every'] == 0 or epoch == CONFIG['total_epochs']:\n                val_m = evaluate(model, val_loader, thresholds=None) \n                history['val_qwk'].append(val_m['qwk'])\n                history['val_acc'].append(val_m['accuracy'])\n                history['val_auc'].append(val_m['mean_auc'])\n                print(f\"  Ep{epoch:03d} | loss={train_stats['loss']:.4f} | QWK={val_m['qwk']:.4f} | AUC={val_m['mean_auc']:.4f} | lr={lr:.2e}\")\n                \n                if val_m['qwk'] > best_qwk:\n                    best_qwk = val_m['qwk']\n                    best_thresholds = val_m['thresholds']\n                    patience = 0\n                    m_state  = (model.module.state_dict() if hasattr(model, 'module') else model.state_dict())\n                    torch.save({'state_dict': m_state, 'metrics': val_m, 'fold': fold, 'epoch': epoch}, best_path)\n                else:\n                    patience += 1\n                    if patience >= CONFIG['early_stop_pat']:\n                        print(f\"  Early stop at epoch {epoch}.\")\n                        break\n            else:\n                if history['val_qwk']:\n                    history['val_qwk'].append(history['val_qwk'][-1])\n                    history['val_acc'].append(history['val_acc'][-1])\n                    history['val_auc'].append(history['val_auc'][-1])\n\n            save_checkpoint(resume_path, model, optimizer, scheduler, scaler, epoch, best_qwk, patience, history)\n            torch.cuda.empty_cache()\n            gc.collect()\n\n    except (KeyboardInterrupt, SystemExit):\n        print(f\"\\n[!] Training stopped unexpectedly at epoch {current_epoch}. Saving progress... 💾\")\n        save_checkpoint(resume_path, model, optimizer, scheduler, scaler, current_epoch, best_qwk, patience, history)\n        print(\"[-] Progress saved successfully. You can resume later! ✅\")\n        return None \n\n    ckpt = torch.load(best_path, map_location=device, weights_only=False)\n    m_inner = model.module if hasattr(model, 'module') else model\n    m_inner.load_state_dict(ckpt['state_dict'])\n    final_th = ckpt['metrics']['thresholds']\n    test_m = evaluate(model, test_loader, thresholds=final_th)\n    print(f\"\\n  Fold {fold+1} TEST | QWK={test_m['qwk']:.4f} | AUC={test_m['mean_auc']:.4f}\")\n\n    result = {\n        'fold': fold + 1, 'best_val_qwk': best_qwk, 'test_qwk': test_m['qwk'], 'test_acc': test_m['accuracy'],\n        'test_auc': test_m['mean_auc'], 'history': {k: v for k, v in history.items() if k != 'history'},\n    }\n    with open(f\"{CONFIG['output_dir']}/fold{fold+1}_result.json\", 'w') as f:\n        json.dump({k: v for k, v in result.items() if k != 'history'}, f, indent=2)\n    \n    return result\n\ndef aggregate_cv_results():\n    results = []\n    for f in range(1, CONFIG['n_folds'] + 1):\n        path = f\"{CONFIG['output_dir']}/fold{f}_result.json\"\n        if os.path.exists(path):\n            with open(path) as fp:\n                results.append(json.load(fp))\n        else:\n            print(f\"  WARNING: fold {f} result not found — run that fold first.\")\n    if not results: return\n    qwks = [r['test_qwk'] for r in results]\n    aucs = [r['test_auc'] for r in results]\n    accs = [r['test_acc'] for r in results]\n    summary = {\n        'n_folds_complete': len(results),\n        'qwk_mean':  round(float(np.mean(qwks)),  4),\n        'qwk_std':   round(float(np.std(qwks)),   4),\n        'auc_mean':  round(float(np.mean(aucs)),  4),\n        'auc_std':   round(float(np.std(aucs)),   4),\n        'acc_mean':  round(float(np.mean(accs)),  4),\n        'per_fold':  results,\n    }\n    with open(f\"{CONFIG['output_dir']}/cv_summary.json\", 'w') as f:\n        json.dump(summary, f, indent=2)\n    print(f\"\\n{'='*60}\\n  5-Fold CV Results ({len(results)} folds)\")\n    print(f\"  QWK : {summary['qwk_mean']:.4f} ± {summary['qwk_std']:.4f}\")\n    print(f\"  AUC : {summary['auc_mean']:.4f} ± {summary['auc_std']:.4f}\\n{'='*60}\\n\")\n    return summary\n\ndef main():\n    print(f\"Device : {device}\")\n    if torch.cuda.is_available():\n        print(f\"GPU    : {torch.cuda.get_device_name(0)}\")\n        print(f\"VRAM   : {torch.cuda.get_device_properties(0).total_memory / 1e9:.1f} GB\")\n\n    manifest_path = os.path.join(CONFIG['data_dir'], f\"manifest_{CONFIG['resolution']}.csv\")\n    if os.path.exists(manifest_path):\n        print(f\"Loading existing manifest: {manifest_path}\")\n        df_cached = pd.read_csv(manifest_path)\n    else:\n        print(\"Building dataset from scratch...\")\n        df_raw    = build_dataset()\n        df_raw    = create_splits(df_raw)\n        df_cached = preprocess_and_cache(df_raw, res=CONFIG['resolution'])\n        df_cached.to_csv(manifest_path, index=False)\n        print(f\"Manifest saved: {len(df_cached):,} images\")\n\n    fold = CONFIG['fold_to_run']\n    print(f\"\\nRunning fold {fold+1}/{CONFIG['n_folds']} ...\")\n    df_train = df_cached[df_cached['split'] != 'test'].reset_index(drop=True)\n    df_test  = df_cached[df_cached['split'] == 'test'].reset_index(drop=True)\n    result = train_single_fold(df_train, df_test, fold=fold)\n    \n    if result is not None:\n        print(f\"\\nFold {fold+1} complete: QWK={result['test_qwk']:.4f}\")\n\n    n_done = sum(1 for f in range(1, CONFIG['n_folds'] + 1) if os.path.exists(f\"{CONFIG['output_dir']}/fold{f}_result.json\"))\n    print(f\"\\n{n_done}/{CONFIG['n_folds']} folds complete.\")\n    if n_done == CONFIG['n_folds']: aggregate_cv_results()\n\nif __name__ == '__main__':\n    main()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#import gc, torch\n#torch.cuda.empty_cache(); gc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-05T19:16:10.180943Z","iopub.execute_input":"2026-06-05T19:16:10.181754Z","iopub.status.idle":"2026-06-05T19:16:10.267191Z","shell.execute_reply.started":"2026-06-05T19:16:10.181724Z","shell.execute_reply":"2026-06-05T19:16:10.266409Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ১. চেকপয়েন্ট রাখার জন্য ফোল্ডার তৈরি\n!mkdir -p /kaggle/working/checkpoints\n\n# ২. আপনার বের করা সঠিক পাথ থেকে RESUME এবং BEST ফাইল দুটি ওয়ার্কিং ফোল্ডারে কপি করা\n!cp /kaggle/input/datasets/arafat120/vimdr-checkpoints/checkpoints/VimDR_fold2_RESUME.pth /kaggle/working/checkpoints/\n!cp /kaggle/input/datasets/arafat120/vimdr-checkpoints/checkpoints/VimDR_fold2_BEST.pth /kaggle/working/checkpoints/\n\n# ৩. ট্রেইনিং শুরু করা (এবার এটি ঠিক ৭ নম্বর ইপোক থেকে শুরু হবে)\n!python vimdr_train_kaggle.py","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-04T04:48:04.710184Z","iopub.execute_input":"2026-06-04T04:48:04.710612Z","iopub.status.idle":"2026-06-04T04:48:05.099404Z","shell.execute_reply.started":"2026-06-04T04:48:04.710577Z","shell.execute_reply":"2026-06-04T04:48:05.098661Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#!zip -r /kaggle/working/vimdr_fold1_backup.zip /kaggle/working/preprocessed /kaggle/working/checkpoints /kaggle/working/outputs","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}