{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":14774,"databundleVersionId":875431,"isSourceIdPinned":false,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## APTOS2019 train kernel (PyTorch)","metadata":{}},{"cell_type":"markdown","source":"### Flags for training","metadata":{}},{"cell_type":"code","source":"# !git clone \"https://github.com/YasufumiNakama/kaggle-APTOS-2019.git\"\n# %cd kaggle-APTOS-2019/proto/code/","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-02T05:00:47.402662Z","iopub.execute_input":"2025-08-02T05:00:47.403128Z","iopub.status.idle":"2025-08-02T05:00:49.033955Z","shell.execute_reply.started":"2025-08-02T05:00:47.403105Z","shell.execute_reply":"2025-08-02T05:00:49.033226Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install efficientnet_pytorch albumentations opencv-python pandas scikit-learn","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-02T05:00:52.763148Z","iopub.execute_input":"2025-08-02T05:00:52.763803Z","iopub.status.idle":"2025-08-02T05:02:15.388584Z","shell.execute_reply.started":"2025-08-02T05:00:52.763772Z","shell.execute_reply":"2025-08-02T05:02:15.387724Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport random\nimport numpy as np\nimport pandas as pd\nimport cv2\nfrom tqdm import tqdm\nfrom sklearn.model_selection import StratifiedKFold, train_test_split\nfrom sklearn.metrics import cohen_kappa_score\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom efficientnet_pytorch import EfficientNet\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nimport torch.nn.functional as F\n\n# ------------------ CONFIG ------------------\nSEED = 3\nMODEL_NAME = 'efficientnet-b5'\nIMG_SIZE = 256\nBATCH_SIZE = 16\nNUM_CLASSES = 5\nEPOCHS = 10\nDATA_PATH = \"/kaggle/input/aptos2019-blindness-detection\"\nSAVE_DIR = \"./saved_models\"\nos.makedirs(SAVE_DIR, exist_ok=True)\n# --------------------------------------------\n\n# ------------- SET SEED ---------------------\ndef set_seed(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\nset_seed(SEED)\n\n# ------------- DATASET ----------------------\nclass DRDataset(Dataset):\n    def __init__(self, df, img_dir, transform=None):\n        self.df = df\n        self.img_dir = img_dir\n        self.transform = transform\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        img_path = os.path.join(self.img_dir, row['id_code'] + \".png\")\n        image = cv2.imread(img_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        if self.transform:\n            image = self.transform(image=image)['image']\n        label = row['diagnosis']\n        return image, label\n\n    def __len__(self):\n        return len(self.df)\n\ntransform_train = A.Compose([\n    A.Resize(IMG_SIZE, IMG_SIZE),\n    A.HorizontalFlip(),\n    A.RandomBrightnessContrast(),\n    A.Normalize(),\n    ToTensorV2(),\n])\ntransform_val = A.Compose([\n    A.Resize(IMG_SIZE, IMG_SIZE),\n    A.Normalize(),\n    ToTensorV2(),\n])\n\n# ------------- LOAD + SPLIT ------------------\ndf = pd.read_csv(f\"{DATA_PATH}/train.csv\")\ndf.columns = ['id_code', 'diagnosis']\ntrainval_df, test_df = train_test_split(\n    df, test_size=0.2, stratify=df['diagnosis'], random_state=SEED\n)\ntest_df = test_df.reset_index(drop=True)\ntest_dataset = DRDataset(test_df, f\"{DATA_PATH}/train_images\", transform_val)\ntest_loader = DataLoader(test_dataset, batch_size=1, shuffle=False)\n\n# ------------- MODEL -------------------------\nclass DRModel(nn.Module):\n    def __init__(self, model_name='efficientnet-b5', num_classes=5):\n        super(DRModel, self).__init__()\n        self.base = EfficientNet.from_pretrained(model_name)\n        in_features = self.base._fc.in_features\n        self.base._fc = nn.Linear(in_features, num_classes)\n    def forward(self, x):\n        return self.base(x)\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n# ---------- TRAIN & SAVE TEST PREDICTIONS -----\nskf = StratifiedKFold(n_splits=5, shuffle=True, random_state=SEED)\n\nfor fold, (train_idx, val_idx) in enumerate(skf.split(trainval_df, trainval_df['diagnosis'])):\n    print(f\"\\n📦 Fold {fold+1}\")\n    train_df = trainval_df.iloc[train_idx].reset_index(drop=True)\n    val_df = trainval_df.iloc[val_idx].reset_index(drop=True)\n\n    train_ds = DRDataset(train_df, f\"{DATA_PATH}/train_images\", transform_train)\n    val_ds = DRDataset(val_df, f\"{DATA_PATH}/train_images\", transform_val)\n    train_loader = DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle=True, num_workers=2)\n    val_loader = DataLoader(val_ds, batch_size=BATCH_SIZE, shuffle=False, num_workers=2)\n\n    model = DRModel(model_name=MODEL_NAME, num_classes=NUM_CLASSES).to(device)\n    optimizer = optim.Adam(model.parameters(), lr=1e-4)\n    criterion = nn.CrossEntropyLoss()\n\n    best_qwk = -1\n    for epoch in range(EPOCHS):\n        model.train()\n        total_loss = 0\n        for imgs, labels in tqdm(train_loader, desc=f\"Fold {fold} Epoch {epoch+1}\"):\n            imgs, labels = imgs.to(device), labels.to(device)\n            optimizer.zero_grad()\n            outputs = model(imgs)\n            loss = criterion(outputs, labels)\n            loss.backward()\n            optimizer.step()\n            total_loss += loss.item()\n\n        print(f\"Train Loss: {total_loss / len(train_loader):.4f}\")\n\n        # Validation\n        model.eval()\n        preds, targets = [], []\n        with torch.no_grad():\n            for imgs, labels in val_loader:\n                imgs = imgs.to(device)\n                output = model(imgs)\n                probs = F.softmax(output, dim=1)\n                pred = torch.argmax(probs, dim=1).cpu().numpy()\n                preds.extend(pred)\n                targets.extend(labels.numpy())\n\n        qwk = cohen_kappa_score(targets, preds, weights='quadratic')\n        print(f\"QWK: {qwk:.4f}\")\n\n        if qwk > best_qwk:\n            best_qwk = qwk\n            model_path = os.path.join(SAVE_DIR, f'ensemble_model_fold{fold}.pth')\n            torch.save(model.state_dict(), model_path)\n            print(f\"✅ Saved best model: {model_path}\")\n\n    # 🔍 Test on held-out test set\n    print(f\"\\n🧪 Testing best model of Fold {fold} on held-out test set...\")\n    model.load_state_dict(torch.load(model_path, map_location=device))\n    model.eval()\n    softmax_preds = []\n\n    with torch.no_grad():\n        for imgs, _ in tqdm(test_loader, desc=f\"Fold {fold} Test\"):\n            imgs = imgs.to(device)\n            outputs = model(imgs)\n            probs = F.softmax(outputs, dim=1).cpu().numpy().squeeze()\n            softmax_preds.append(probs.astype(np.float32))\n\n    softmax_preds = np.stack(softmax_preds)\n    np.save(os.path.join(SAVE_DIR, f'test_predictions_fold{fold}.npy'), softmax_preds)\n    print(f\"📁 Saved: test_predictions_fold{fold}.npy\")\n\n# 📝 Also save test_df labels for final evaluation\ntest_df[['id_code', 'diagnosis']].to_csv(os.path.join(SAVE_DIR, 'test_ground_truth.csv'), index=False)\nprint(\"✅ Saved test ground truth CSV\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-02T06:46:11.113651Z","iopub.execute_input":"2025-08-02T06:46:11.114282Z","iopub.status.idle":"2025-08-02T09:05:44.876217Z","shell.execute_reply.started":"2025-08-02T06:46:11.11425Z","shell.execute_reply":"2025-08-02T09:05:44.875331Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom sklearn.metrics import accuracy_score, cohen_kappa_score, brier_score_loss\nimport matplotlib.pyplot as plt\n\nNUM_CLASSES = 5\nSAVE_DIR = \"./saved_models\"\n\n# ---------- Load Test Labels ----------\ngt_df = pd.read_csv(f\"{SAVE_DIR}/test_ground_truth.csv\")\ntrue_labels = gt_df['diagnosis'].values\n\n# ---------- Load All Predictions ----------\nall_preds = []\nfor fold in range(5):\n    path = f\"{SAVE_DIR}/test_predictions_fold{fold}.npy\"\n    preds = np.load(path)  # shape: [n_samples, num_classes]\n    all_preds.append(preds)\n\nall_preds = np.stack(all_preds)  # shape: [n_folds, n_samples, num_classes]\nensemble_mean = np.mean(all_preds, axis=0)  # shape: [n_samples, num_classes]\nensemble_var = np.var(all_preds, axis=0).mean(axis=1)  # variance across folds, mean over classes\n\n# ---------- Uncertainty Metrics ----------\nentropy = -np.sum(ensemble_mean * np.log(ensemble_mean + 1e-8), axis=1)\nbrier = [brier_score_loss(np.eye(NUM_CLASSES)[true], prob) for true, prob in zip(true_labels, ensemble_mean)]\npreds = np.argmax(ensemble_mean, axis=1)\n\n# ---------- Accuracy Metrics ----------\nacc = accuracy_score(true_labels, preds)\nqwk = cohen_kappa_score(true_labels, preds, weights='quadratic')\n\n# ---------- Results DataFrame ----------\nresults_df = pd.DataFrame({\n    'Ground Truth': true_labels,\n    'Prediction': preds,\n    'Entropy': entropy,\n    'Variance': ensemble_var,\n    'Brier Score': brier\n})\n\n# ---------- Summary ----------\nprint(f\"✅ Accuracy: {acc:.4f}\")\nprint(f\"✅ QWK:      {qwk:.4f}\")\nprint(f\"✅ Avg Entropy: {np.mean(entropy):.4f}\")\nprint(f\"✅ Avg Variance: {np.mean(ensemble_var):.6f}\")\nprint(f\"✅ Avg Brier Score: {np.mean(brier):.6f}\")\n\n# ---------- Optional: Visualize ----------\nplt.hist(entropy, bins=30)\nplt.title(\"Prediction Entropy (Uncertainty)\")\nplt.xlabel(\"Entropy\")\nplt.ylabel(\"Count\")\nplt.show()\n\nplt.hist(ensemble_var, bins=30)\nplt.title(\"Prediction Variance Across Models\")\nplt.xlabel(\"Variance\")\nplt.ylabel(\"Count\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-02T09:05:57.611727Z","iopub.execute_input":"2025-08-02T09:05:57.612533Z","iopub.status.idle":"2025-08-02T09:05:58.26011Z","shell.execute_reply.started":"2025-08-02T09:05:57.612496Z","shell.execute_reply":"2025-08-02T09:05:58.25935Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# From previous script\nensemble_mean  # shape: [n_samples, num_classes]\ntrue_labels    # shape: [n_samples]\nentropy        # shape: [n_samples]\npreds          = np.argmax(ensemble_mean, axis=1)\n\ndef compute_ece(probs, labels, n_bins=15):\n    confidences = np.max(probs, axis=1)\n    predictions = np.argmax(probs, axis=1)\n    accuracies = predictions == labels\n\n    bin_bounds = np.linspace(0, 1, n_bins + 1)\n    ece = 0.0\n\n    for i in range(n_bins):\n        start, end = bin_bounds[i], bin_bounds[i + 1]\n        mask = (confidences > start) & (confidences <= end)\n        if np.sum(mask) > 0:\n            bin_acc = np.mean(accuracies[mask])\n            bin_conf = np.mean(confidences[mask])\n            ece += np.abs(bin_acc - bin_conf) * np.sum(mask) / len(labels)\n    return ece\n\nece_value = compute_ece(ensemble_mean, true_labels)\nprint(f\"📏 Expected Calibration Error (ECE): {ece_value:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-02T09:06:37.594521Z","iopub.execute_input":"2025-08-02T09:06:37.595389Z","iopub.status.idle":"2025-08-02T09:06:37.603677Z","shell.execute_reply.started":"2025-08-02T09:06:37.59533Z","shell.execute_reply":"2025-08-02T09:06:37.603096Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\ndef referral_accuracy_curve(y_true, y_pred, uncertainty, bins=10):\n    idx_sorted = np.argsort(-uncertainty)  # high to low\n    step = len(y_true) // bins\n    accs = []\n\n    for i in range(bins + 1):\n        keep_idx = idx_sorted[i * step:] if i * step < len(y_true) else []\n        if len(keep_idx) == 0:\n            accs.append(np.nan)\n        else:\n            accs.append(accuracy_score(np.array(y_true)[keep_idx], np.array(y_pred)[keep_idx]))\n\n    referral_rates = [i / bins for i in range(bins + 1)]\n    plt.plot(referral_rates, accs, marker='o')\n    plt.title(\"Accuracy vs. Referral Rate\")\n    plt.xlabel(\"Referral Rate (Proportion Deferred)\")\n    plt.ylabel(\"Accuracy on Remaining\")\n    plt.grid(True)\n    plt.show()\n\nreferral_accuracy_curve(true_labels, preds, entropy)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-02T09:06:42.254573Z","iopub.execute_input":"2025-08-02T09:06:42.255275Z","iopub.status.idle":"2025-08-02T09:06:42.407539Z","shell.execute_reply.started":"2025-08-02T09:06:42.255247Z","shell.execute_reply":"2025-08-02T09:06:42.406785Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import cohen_kappa_score\n\ndef qwk_risk_curve(y_true, y_pred, uncertainty, bins=10):\n    idx_sorted = np.argsort(-uncertainty)\n    step = len(y_true) // bins\n    qwks = []\n\n    for i in range(bins + 1):\n        keep_idx = idx_sorted[i * step:] if i * step < len(y_true) else []\n        if len(keep_idx) < 2:  # QWK needs at least 2 samples\n            qwks.append(np.nan)\n        else:\n            kept_y = np.array(y_true)[keep_idx]\n            kept_p = np.array(y_pred)[keep_idx]\n            qwks.append(cohen_kappa_score(kept_y, kept_p, weights='quadratic'))\n\n    referral_rates = [i / bins for i in range(bins + 1)]\n    plt.plot(referral_rates, qwks, marker='s', color='green')\n    plt.title(\"QWK vs. Referral Rate\")\n    plt.xlabel(\"Referral Rate (Proportion Deferred)\")\n    plt.ylabel(\"QWK on Remaining\")\n    plt.grid(True)\n    plt.show()\n\nqwk_risk_curve(true_labels, preds, entropy)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-02T09:06:49.575098Z","iopub.execute_input":"2025-08-02T09:06:49.575642Z","iopub.status.idle":"2025-08-02T09:06:49.732824Z","shell.execute_reply.started":"2025-08-02T09:06:49.575617Z","shell.execute_reply":"2025-08-02T09:06:49.732101Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport numpy as np\n\ndef plot_reliability_diagram(probs, labels, n_bins=15):\n    confidences = np.max(probs, axis=1)\n    predictions = np.argmax(probs, axis=1)\n    accuracies = predictions == labels\n\n    bins = np.linspace(0, 1, n_bins + 1)\n    bin_centers = (bins[:-1] + bins[1:]) / 2\n    bin_accuracies = np.zeros(n_bins)\n    bin_confidences = np.zeros(n_bins)\n    bin_counts = np.zeros(n_bins)\n\n    for i in range(n_bins):\n        mask = (confidences > bins[i]) & (confidences <= bins[i+1])\n        if np.any(mask):\n            bin_accuracies[i] = np.mean(accuracies[mask])\n            bin_confidences[i] = np.mean(confidences[mask])\n            bin_counts[i] = np.sum(mask)\n\n    plt.figure(figsize=(6, 6))\n    plt.plot([0, 1], [0, 1], linestyle='--', color='black', label='Perfect Calibration')\n    plt.bar(bin_centers, bin_accuracies, width=1/n_bins, alpha=0.6, edgecolor='black', label='Model Accuracy')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-02T09:39:21.713982Z","iopub.execute_input":"2025-08-02T09:39:21.714726Z","iopub.status.idle":"2025-08-02T09:39:21.72155Z","shell.execute_reply.started":"2025-08-02T09:39:21.714694Z","shell.execute_reply":"2025-08-02T09:39:21.720936Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ensemble_mean  # shape: [n_samples, num_classes]\ntrue_labels    # shape: [n_samples]\nplot_reliability_diagram(ensemble_mean, true_labels)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-02T09:39:24.238042Z","iopub.execute_input":"2025-08-02T09:39:24.238644Z","iopub.status.idle":"2025-08-02T09:39:24.387339Z","shell.execute_reply.started":"2025-08-02T09:39:24.238617Z","shell.execute_reply":"2025-08-02T09:39:24.386678Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport numpy as np\n\n# probs = softmax output from your model (shape: [n_samples, num_classes])\npreds = np.argmax(ensemble_mean, axis=1)               # predicted class\nconfidences = np.max(ensemble_mean, axis=1)            # confidence (probability of predicted class)\ntrue_labels = np.array(true_labels)\n\nimport matplotlib.pyplot as plt\n\nplt.hist(confidences, bins=20, color='skyblue', edgecolor='black')\nplt.title(\"Model Confidence Distribution\")\nplt.xlabel(\"Confidence (Max Softmax Probability)\")\nplt.ylabel(\"Number of Predictions\")\nplt.grid(True)\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-02T09:46:14.388442Z","iopub.execute_input":"2025-08-02T09:46:14.388821Z","iopub.status.idle":"2025-08-02T09:46:14.570874Z","shell.execute_reply.started":"2025-08-02T09:46:14.388796Z","shell.execute_reply":"2025-08-02T09:46:14.570136Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score\n\ndef confidence_vs_accuracy_plot(confidences, preds, labels, bins=10):\n    bin_bounds = np.linspace(0, 1, bins + 1)\n    bin_centers = (bin_bounds[:-1] + bin_bounds[1:]) / 2\n    bin_accs = []\n\n    for i in range(bins):\n        mask = (confidences > bin_bounds[i]) & (confidences <= bin_bounds[i+1])\n        if np.sum(mask) == 0:\n            bin_accs.append(np.nan)\n        else:\n            acc = accuracy_score(labels[mask], preds[mask])\n            bin_accs.append(acc)\n\n    plt.plot(bin_centers, bin_accs, marker='o', label='Accuracy per confidence bin')\n    plt.xlabel(\"Confidence\")\n    plt.ylabel(\"Accuracy\")\n    plt.title(\"Accuracy vs. Confidence\")\n    plt.grid(True)\n    plt.ylim(0, 1)\n    plt.axhline(y=np.mean(preds == labels), linestyle='--', color='gray', label='Overall Accuracy')\n    plt.legend()\n    plt.show()\n\nconfidence_vs_accuracy_plot(confidences, preds, true_labels)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-02T09:46:44.903273Z","iopub.execute_input":"2025-08-02T09:46:44.903589Z","iopub.status.idle":"2025-08-02T09:46:45.062758Z","shell.execute_reply.started":"2025-08-02T09:46:44.903568Z","shell.execute_reply":"2025-08-02T09:46:45.062028Z"}},"outputs":[],"execution_count":null}]}