{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.10","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":4104,"databundleVersionId":46661,"sourceType":"competition"},{"sourceId":418031,"sourceType":"datasetVersion","datasetId":131128}],"dockerImageVersionId":30097,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## COMPUTER VISION LAB 10\n### Name: Jay Deep Singh\n### 211020421 DSAI\n\n\nTopic Chosen: **Retinal Image Analysis**\n\nReference Paper: S. Prabha, S. S. R. Kumar, G. G. Reddy and K. S. Sankaran, \"Retinal Image Analysis Using Machine Learning,\" 2020 International Conference on Communication and Signal Processing (ICCSP), Chennai, India, 2020, pp. 1410-1413, doi: 10.1109/ICCSP48568.2020.9182227.\n\nPaper link: https://ieeexplore.ieee.org/document/9182227\n\nDataset: https://www.kaggle.com/competitions/diabetic-retinopathy-detection/data\n\nExisting Methodology: SVM using SAUCE and Tyler Core Algorithm\n\nProposed Methodology: Using a pretrained inception v3 as a base and retraining some modified final layers with attention","metadata":{}},{"cell_type":"markdown","source":"Lets work with sample data.","metadata":{}},{"cell_type":"code","source":"from torch.utils.data import Dataset, DataLoader\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport torch\nimport os\nimport numpy as np\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import StratifiedShuffleSplit","metadata":{"execution":{"iopub.status.busy":"2023-11-23T05:55:19.989493Z","iopub.execute_input":"2023-11-23T05:55:19.989865Z","iopub.status.idle":"2023-11-23T05:55:21.813676Z","shell.execute_reply.started":"2023-11-23T05:55:19.989785Z","shell.execute_reply":"2023-11-23T05:55:21.812931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!unzip ../input/diabetic-retinopathy-detection/sample.zip\n!unzip ../input/diabetic-retinopathy-detection/sampleSubmission.csv.zip\n! dir sample\n!unzip ../input/diabetic-retinopathy-detection/trainLabels.csv.zipyY","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-11-23T05:55:21.814927Z","iopub.execute_input":"2023-11-23T05:55:21.815190Z","iopub.status.idle":"2023-11-23T05:55:25.823115Z","shell.execute_reply.started":"2023-11-23T05:55:21.815165Z","shell.execute_reply":"2023-11-23T05:55:25.822247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Check Images","metadata":{}},{"cell_type":"code","source":"f, axarr = plt.subplots(2,2, figsize=(10, 10))\naxarr[0,0].imshow(Image.open(\"./sample/10_right.jpeg\"))\naxarr[0,1].imshow(Image.open(\"./sample/13_right.jpeg\"))\naxarr[1,0].imshow(Image.open(\"./sample/15_right.jpeg\"))\naxarr[1,1].imshow(Image.open(\"./sample/17_right.jpeg\"))","metadata":{"execution":{"iopub.status.busy":"2023-11-23T05:55:25.825149Z","iopub.execute_input":"2023-11-23T05:55:25.825447Z","iopub.status.idle":"2023-11-23T05:55:30.770273Z","shell.execute_reply.started":"2023-11-23T05:55:25.825411Z","shell.execute_reply":"2023-11-23T05:55:30.769437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Check data distribution","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(\"../input/diabetic-retinopathy-resized/trainLabels.csv\")[:5000]\ndf['image'] = df['image'].apply(lambda x: \"../input/diabetic-retinopathy-resized/resized_train_cropped/resized_train_cropped/\" + x + \".jpeg\")\ndf.head()\nnum_classes = len(np.unique(df['level']))\nclass_dist = [len(df.loc[df['level'] == x]) for x in range(num_classes)]\n\nprint(f\"NUM DATA : {len(df)}\")\nprint(f\"NUM CLASSES : {num_classes}\")\nprint(f\"CLASS DIST : {class_dist}\")\nplt.bar(np.unique(df['level']), class_dist)\n\n# unlabeled ","metadata":{"execution":{"iopub.status.busy":"2023-11-23T05:55:30.771872Z","iopub.execute_input":"2023-11-23T05:55:30.772165Z","iopub.status.idle":"2023-11-23T05:55:30.995078Z","shell.execute_reply.started":"2023-11-23T05:55:30.772136Z","shell.execute_reply":"2023-11-23T05:55:30.994218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Lets get the first train file","metadata":{}},{"cell_type":"markdown","source":"# Visualization","metadata":{}},{"cell_type":"code","source":"# visualize\ndef visualize_image_per_level(level=0):\n    level_images = df.loc[df['level'] == level]['image'].values\n    IMG_DIR = \"../input/diabetic-retinopathy-resized/resized_train/resized_train\"\n    images = []\n    for i in range(10):\n        img = Image.open(level_images[i])\n        img = img.resize((512, 512))\n        images.append(np.array(img))\n    images = np.concatenate(images, axis=1)\n    plt.figure(figsize=(20, 40))\n    plt.imshow(images)\n    \nfor c in range(num_classes):\n    visualize_image_per_level(c)","metadata":{"execution":{"iopub.status.busy":"2023-11-23T05:55:30.996226Z","iopub.execute_input":"2023-11-23T05:55:30.996528Z","iopub.status.idle":"2023-11-23T05:55:35.092788Z","shell.execute_reply.started":"2023-11-23T05:55:30.996499Z","shell.execute_reply":"2023-11-23T05:55:35.091669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Construct dataset","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def examine_files(files, labels):\n    confirmed_files = []\n    confirmed_labels = []\n    for i in range(len(files)):\n        if os.path.isfile(files[i]):\n            confirmed_files.append(files[i])\n            confirmed_labels.append(labels[i])\n    return confirmed_files, np.array(confirmed_labels)\n\n# train / val /test split\ntotal_files = df['image'].values\ntotal_labels = df['level'].values\n\n# data scarcity setting\ntotal_files = total_files[:1000]\ntotal_labels = total_labels[:1000]\n\nunlabeled_files = total_files[1000:]\nunlabeled_labels = total_labels[1000:]\n\nsss = StratifiedShuffleSplit(n_splits=1, test_size=0.2)\nremained_idx, test_idx = next(sss.split(total_files, total_labels))\nremained_files = total_files[remained_idx]\nremained_labels = total_labels[remained_idx]\n\ntest_files = total_files[test_idx]\ntest_labels = total_labels[test_idx]\n\nsss = StratifiedShuffleSplit(n_splits=1, test_size=0.2)\ntrain_idx, val_idx = next(sss.split(remained_files, remained_labels))\ntrain_files = remained_files[train_idx]\ntrain_labels = remained_labels[train_idx]\n\nval_files = remained_files[val_idx]\nval_labels = remained_labels[val_idx]\n\ntrain_files, train_labels = examine_files(train_files, train_labels)\nval_files, val_labels = examine_files(val_files, val_labels)\ntest_files, test_labels = examine_files(test_files, test_labels)\n\ntrain_df = pd.DataFrame(data={'image': train_files, 'level': train_labels})\nval_df = pd.DataFrame(data={'image': val_files, 'level': val_labels})\ntest_df = pd.DataFrame(data={'image': test_files, 'level': test_labels})\n\nprint(f\"NUM TRAIN / VAL / TEST : {len(train_files)} / {len(val_files)} / {len(test_files)}\")\nfor c in range(num_classes):\n    print(f\"NUM CLASS {c} TRAIN / VAL / TEST : {len(np.where(train_labels==c)[0])} / {len(np.where(val_labels==c)[0])} / {len(np.where(test_labels==c)[0])}\")","metadata":{"execution":{"iopub.status.busy":"2023-11-23T05:55:35.094224Z","iopub.execute_input":"2023-11-23T05:55:35.094583Z","iopub.status.idle":"2023-11-23T05:55:39.375778Z","shell.execute_reply.started":"2023-11-23T05:55:35.094548Z","shell.execute_reply":"2023-11-23T05:55:39.374889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torchvision.transforms as transforms\nmy_transform = transforms.Compose([\n    transforms.Resize((256,256)),\n    transforms.ToTensor(),\n])\n\nclass retinaDataset(Dataset):\n    def __init__(self, df,transform=my_transform):\n        self.df = df\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        img = Image.open(self.df.iloc[index].image)\n        \n        if(self.transform):\n            img = self.transform(img)\n        \n        return img, torch.tensor(self.df.iloc[index].level)","metadata":{"execution":{"iopub.status.busy":"2023-11-23T05:55:39.377195Z","iopub.execute_input":"2023-11-23T05:55:39.377601Z","iopub.status.idle":"2023-11-23T05:55:39.495622Z","shell.execute_reply.started":"2023-11-23T05:55:39.377560Z","shell.execute_reply":"2023-11-23T05:55:39.494725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 16\n\ntrain_dataset = retinaDataset(train_df)\nval_dataset = retinaDataset(val_df)\ntest_dataset = retinaDataset(test_df)\ntrain_dataloader = DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True, drop_last=True)\nval_dataloader = DataLoader(dataset=val_dataset, batch_size=batch_size, shuffle=False)\ntest_dataloader = DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2023-11-23T05:55:39.498179Z","iopub.execute_input":"2023-11-23T05:55:39.498484Z","iopub.status.idle":"2023-11-23T05:55:39.504014Z","shell.execute_reply.started":"2023-11-23T05:55:39.498455Z","shell.execute_reply":"2023-11-23T05:55:39.503058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Build","metadata":{}},{"cell_type":"code","source":"from torchvision.models import inception_v3\n\nlearning_rate = 1e-4\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\nmodel = inception_v3(pretrained=True)  \nmodel.fc = torch.nn.Linear(in_features=2048, out_features=num_classes, bias=True)\nmodel.aux_logits = False\nmodel = model.to(device=device)\n\noptimizer = torch.optim.Adam(model.parameters(), lr = learning_rate)\nloss_criterion = torch.nn.CrossEntropyLoss()","metadata":{"execution":{"iopub.status.busy":"2023-11-23T05:55:39.505346Z","iopub.execute_input":"2023-11-23T05:55:39.505617Z","iopub.status.idle":"2023-11-23T05:55:44.726751Z","shell.execute_reply.started":"2023-11-23T05:55:39.505591Z","shell.execute_reply":"2023-11-23T05:55:44.725946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device","metadata":{"execution":{"iopub.status.busy":"2023-11-23T05:55:44.727841Z","iopub.execute_input":"2023-11-23T05:55:44.728127Z","iopub.status.idle":"2023-11-23T05:55:44.733386Z","shell.execute_reply.started":"2023-11-23T05:55:44.728099Z","shell.execute_reply":"2023-11-23T05:55:44.732537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{}},{"cell_type":"code","source":"from tqdm import tqdm\n\ndef check_accuracy(model, loader):\n    model.eval()\n    \n    correct_output = 0\n    total_output = 0\n    \n    with torch.no_grad():\n        for x, y in tqdm(loader):\n            x = x.to(device=device)\n            y = y.to(device=device)\n            \n            score = model(x)\n            _,predictions = score.max(1)\n            \n            correct_output += (y==predictions).sum()\n            total_output += predictions.shape[0]\n    model.train()\n    print(f\"out of {total_output} , total correct: {correct_output} with an accuracy of {float(correct_output/total_output)*100}\")\n    return float(correct_output/total_output)*100\n\nnum_epochs = 10\nbest_score = 0\nfor epoch in range(num_epochs):\n    for data, target in tqdm(train_dataloader):\n        data = data.to(device=device)\n        target = target.to(device=device)\n        \n        score = model(data)\n        optimizer.zero_grad()\n        \n        loss = loss_criterion(score, target)\n        loss.backward()\n        \n        optimizer.step()\n    \n    print(f\"for epoch {epoch}, loss : {loss}\")\n    val_acc = check_accuracy(model, val_dataloader)\n    if val_acc > best_score:\n        best_score = val_acc\n        torch.save(model.state_dict(), \"/kaggle/working/best_model.bin\")\n    ","metadata":{"execution":{"iopub.status.busy":"2023-11-23T05:55:44.734519Z","iopub.execute_input":"2023-11-23T05:55:44.734829Z","iopub.status.idle":"2023-11-23T05:59:25.314157Z","shell.execute_reply.started":"2023-11-23T05:55:44.734799Z","shell.execute_reply":"2023-11-23T05:59:25.313279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluate","metadata":{}},{"cell_type":"code","source":"model.load_state_dict(torch.load(\"/kaggle/working/best_model.bin\"))","metadata":{"execution":{"iopub.status.busy":"2023-11-23T05:59:25.315260Z","iopub.execute_input":"2023-11-23T05:59:25.315585Z","iopub.status.idle":"2023-11-23T05:59:25.476777Z","shell.execute_reply.started":"2023-11-23T05:59:25.315556Z","shell.execute_reply":"2023-11-23T05:59:25.475912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_preds = []\ntest_labels = []\nwith torch.no_grad():\n    for data, target in tqdm(test_dataloader):\n        data = data.to(device=device)\n        target = target.to(device=device)\n        logit = model(data)\n        pred = torch.softmax(logit, dim=-1)\n        pred = pred.cpu().numpy()\n        label = target.cpu().numpy()\n        test_preds.append(pred)\n        test_labels.append(label)\n        \ntest_preds = np.concatenate(test_preds, axis=0)\ntest_labels = np.concatenate(test_labels, axis=0)        ","metadata":{"execution":{"iopub.status.busy":"2023-11-23T05:59:25.477754Z","iopub.execute_input":"2023-11-23T05:59:25.478002Z","iopub.status.idle":"2023-11-23T05:59:31.153734Z","shell.execute_reply.started":"2023-11-23T05:59:25.477978Z","shell.execute_reply":"2023-11-23T05:59:31.152788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import roc_auc_score\n\ndef calc_ece(softmax, label, bins=15):\n    softmax = torch.FloatTensor(softmax)\n    label = torch.FloatTensor(label)\n    bin_boundaries = torch.linspace(0, 1, bins + 1)\n    bin_lowers = bin_boundaries[:-1]\n    bin_uppers = bin_boundaries[1:]\n\n    softmax = torch.tensor(softmax)\n    labels = torch.tensor(label)\n\n    softmax_max, predictions = torch.max(softmax, 1)\n    correctness = predictions.eq(labels)\n\n    ece = torch.zeros(1)\n    mce = 0\n\n    for bin_lower, bin_upper in zip(bin_lowers, bin_uppers):\n        in_bin = softmax_max.gt(bin_lower.item()) * softmax_max.le(bin_upper.item())\n        prop_in_bin = in_bin.float().mean()\n\n        if prop_in_bin.item() > 0.0:\n            accuracy_in_bin = correctness[in_bin].float().mean()\n            avg_confidence_in_bin = softmax_max[in_bin].mean()\n            diff = torch.abs(avg_confidence_in_bin - accuracy_in_bin)\n            ece += diff * prop_in_bin\n            if diff.item() > mce:\n                mce = diff.item()\n\n    return ece.item(), mce\n\n# auroc\nfor c in range(num_classes):\n    if c == 0:\n        continue\n    c_preds = test_preds[:, c]\n    c_labels = np.uint8(test_labels == c)\n    test_auc = roc_auc_score(c_labels, c_preds)\n    print(f\"CLASS {c} AUC : {test_auc}\")\nece_score = calc_ece(test_preds, test_labels)\nprint(f\"ECE : {ece_score[0] * 100}, MCE : {ece_score[1] * 100}\")","metadata":{"execution":{"iopub.status.busy":"2023-11-23T05:59:31.155158Z","iopub.execute_input":"2023-11-23T05:59:31.155551Z","iopub.status.idle":"2023-11-23T05:59:31.191279Z","shell.execute_reply.started":"2023-11-23T05:59:31.155511Z","shell.execute_reply":"2023-11-23T05:59:31.190403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# OOD Detection","metadata":{}},{"cell_type":"code","source":"from tqdm import trange\nnum_ood = 100\nood_preds = []\nwith torch.no_grad():\n    for i in trange(num_ood):\n        img = np.array(Image.open(test_files[i]))\n        np.random.shuffle(img)\n        img = np.transpose(img, [2, 0, 1])\n        img_tensor = torch.FloatTensor(img).to(device).unsqueeze(0)\n        logit = model(img_tensor)\n        pred = torch.softmax(logit, dim=-1)\n        pred = pred.cpu().numpy()\n        ood_preds.append(pred)\nood_preds = np.concatenate(ood_preds, axis=0)","metadata":{"execution":{"iopub.status.busy":"2023-11-23T05:59:31.192393Z","iopub.execute_input":"2023-11-23T05:59:31.192646Z","iopub.status.idle":"2023-11-23T05:59:38.008273Z","shell.execute_reply.started":"2023-11-23T05:59:31.192622Z","shell.execute_reply":"2023-11-23T05:59:38.007410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_labels = [1] * len(test_labels) + [0] * len(ood_preds)\ntotal_preds = np.concatenate([test_preds, ood_preds], axis=0)\nmsp = np.max(total_preds, axis=-1)\nood_auc = roc_auc_score(total_labels, msp)\nprint(f\"OOD Detection performance (AUC) : {ood_auc}\")","metadata":{"execution":{"iopub.status.busy":"2023-11-23T05:59:38.009543Z","iopub.execute_input":"2023-11-23T05:59:38.009825Z","iopub.status.idle":"2023-11-23T05:59:38.018016Z","shell.execute_reply.started":"2023-11-23T05:59:38.009797Z","shell.execute_reply":"2023-11-23T05:59:38.017095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}