{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Lets work with sample data.","metadata":{}},{"cell_type":"code","source":"from torch.utils.data import Dataset, DataLoader\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport torch\nimport os\nimport numpy as np\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import StratifiedShuffleSplit","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:23:44.103318Z","iopub.execute_input":"2022-07-26T06:23:44.103736Z","iopub.status.idle":"2022-07-26T06:23:46.093118Z","shell.execute_reply.started":"2022-07-26T06:23:44.103654Z","shell.execute_reply":"2022-07-26T06:23:46.092117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!unzip ../input/diabetic-retinopathy-detection/sample.zip\n!unzip ../input/diabetic-retinopathy-detection/sampleSubmission.csv.zip\n! dir sample\n!unzip ../input/diabetic-retinopathy-detection/trainLabels.csv.zipyY","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-07-26T06:23:46.094598Z","iopub.execute_input":"2022-07-26T06:23:46.094995Z","iopub.status.idle":"2022-07-26T06:23:49.216070Z","shell.execute_reply.started":"2022-07-26T06:23:46.094953Z","shell.execute_reply":"2022-07-26T06:23:49.214964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Check Images","metadata":{}},{"cell_type":"code","source":"f, axarr = plt.subplots(2,2, figsize=(10, 10))\naxarr[0,0].imshow(Image.open(\"./sample/10_right.jpeg\"))\naxarr[0,1].imshow(Image.open(\"./sample/13_right.jpeg\"))\naxarr[1,0].imshow(Image.open(\"./sample/15_right.jpeg\"))\naxarr[1,1].imshow(Image.open(\"./sample/17_right.jpeg\"))","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:23:49.219095Z","iopub.execute_input":"2022-07-26T06:23:49.219408Z","iopub.status.idle":"2022-07-26T06:23:53.890998Z","shell.execute_reply.started":"2022-07-26T06:23:49.219366Z","shell.execute_reply":"2022-07-26T06:23:53.890139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Check data distribution","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(\"../input/diabetic-retinopathy-resized/trainLabels.csv\")[:5000]\ndf['image'] = df['image'].apply(lambda x: \"../input/diabetic-retinopathy-resized/resized_train_cropped/resized_train_cropped/\" + x + \".jpeg\")\ndf.head()\nnum_classes = len(np.unique(df['level']))\nclass_dist = [len(df.loc[df['level'] == x]) for x in range(num_classes)]\n\nprint(f\"NUM DATA : {len(df)}\")\nprint(f\"NUM CLASSES : {num_classes}\")\nprint(f\"CLASS DIST : {class_dist}\")\nplt.bar(np.unique(df['level']), class_dist)\n\n# unlabeled ","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:23:53.892574Z","iopub.execute_input":"2022-07-26T06:23:53.892894Z","iopub.status.idle":"2022-07-26T06:23:54.077255Z","shell.execute_reply.started":"2022-07-26T06:23:53.892863Z","shell.execute_reply":"2022-07-26T06:23:54.076466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Lets get the first train file","metadata":{}},{"cell_type":"markdown","source":"# Visualization","metadata":{}},{"cell_type":"code","source":"# visualize\ndef visualize_image_per_level(level=0):\n    level_images = df.loc[df['level'] == level]['image'].values\n    IMG_DIR = \"../input/diabetic-retinopathy-resized/resized_train/resized_train\"\n    images = []\n    for i in range(10):\n        img = Image.open(level_images[i])\n        img = img.resize((512, 512))\n        images.append(np.array(img))\n    images = np.concatenate(images, axis=1)\n    plt.figure(figsize=(20, 40))\n    plt.imshow(images)\n    \nfor c in range(num_classes):\n    visualize_image_per_level(c)","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:23:54.078449Z","iopub.execute_input":"2022-07-26T06:23:54.078793Z","iopub.status.idle":"2022-07-26T06:23:58.161588Z","shell.execute_reply.started":"2022-07-26T06:23:54.078765Z","shell.execute_reply":"2022-07-26T06:23:58.160520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Construct dataset","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def examine_files(files, labels):\n    confirmed_files = []\n    confirmed_labels = []\n    for i in range(len(files)):\n        if os.path.isfile(files[i]):\n            confirmed_files.append(files[i])\n            confirmed_labels.append(labels[i])\n    return confirmed_files, np.array(confirmed_labels)\n\n# train / val /test split\ntotal_files = df['image'].values\ntotal_labels = df['level'].values\n\n# data scarcity setting\ntotal_files = total_files[:1000]\ntotal_labels = total_labels[:1000]\n\nunlabeled_files = total_files[1000:]\nunlabeled_labels = total_labels[1000:]\n\nsss = StratifiedShuffleSplit(n_splits=1, test_size=0.2)\nremained_idx, test_idx = next(sss.split(total_files, total_labels))\nremained_files = total_files[remained_idx]\nremained_labels = total_labels[remained_idx]\n\ntest_files = total_files[test_idx]\ntest_labels = total_labels[test_idx]\n\nsss = StratifiedShuffleSplit(n_splits=1, test_size=0.2)\ntrain_idx, val_idx = next(sss.split(remained_files, remained_labels))\ntrain_files = remained_files[train_idx]\ntrain_labels = remained_labels[train_idx]\n\nval_files = remained_files[val_idx]\nval_labels = remained_labels[val_idx]\n\ntrain_files, train_labels = examine_files(train_files, train_labels)\nval_files, val_labels = examine_files(val_files, val_labels)\ntest_files, test_labels = examine_files(test_files, test_labels)\n\ntrain_df = pd.DataFrame(data={'image': train_files, 'level': train_labels})\nval_df = pd.DataFrame(data={'image': val_files, 'level': val_labels})\ntest_df = pd.DataFrame(data={'image': test_files, 'level': test_labels})\n\nprint(f\"NUM TRAIN / VAL / TEST : {len(train_files)} / {len(val_files)} / {len(test_files)}\")\nfor c in range(num_classes):\n    print(f\"NUM CLASS {c} TRAIN / VAL / TEST : {len(np.where(train_labels==c)[0])} / {len(np.where(val_labels==c)[0])} / {len(np.where(test_labels==c)[0])}\")","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:23:58.163121Z","iopub.execute_input":"2022-07-26T06:23:58.163780Z","iopub.status.idle":"2022-07-26T06:24:01.837800Z","shell.execute_reply.started":"2022-07-26T06:23:58.163734Z","shell.execute_reply":"2022-07-26T06:24:01.836951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torchvision.transforms as transforms\nmy_transform = transforms.Compose([\n    transforms.Resize((256,256)),\n    transforms.ToTensor(),\n])\n\nclass retinaDataset(Dataset):\n    def __init__(self, df,transform=my_transform):\n        self.df = df\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        img = Image.open(self.df.iloc[index].image)\n        \n        if(self.transform):\n            img = self.transform(img)\n        \n        return img, torch.tensor(self.df.iloc[index].level)","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:24:01.839387Z","iopub.execute_input":"2022-07-26T06:24:01.840150Z","iopub.status.idle":"2022-07-26T06:24:01.966118Z","shell.execute_reply.started":"2022-07-26T06:24:01.840108Z","shell.execute_reply":"2022-07-26T06:24:01.964924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 16\n\ntrain_dataset = retinaDataset(train_df)\nval_dataset = retinaDataset(val_df)\ntest_dataset = retinaDataset(test_df)\ntrain_dataloader = DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True, drop_last=True)\nval_dataloader = DataLoader(dataset=val_dataset, batch_size=batch_size, shuffle=False)\ntest_dataloader = DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:24:01.968386Z","iopub.execute_input":"2022-07-26T06:24:01.968653Z","iopub.status.idle":"2022-07-26T06:24:01.974248Z","shell.execute_reply.started":"2022-07-26T06:24:01.968627Z","shell.execute_reply":"2022-07-26T06:24:01.973289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Build","metadata":{}},{"cell_type":"code","source":"from torchvision.models import inception_v3\n\nlearning_rate = 1e-4\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\nmodel = inception_v3(pretrained=True)  \nmodel.fc = torch.nn.Linear(in_features=2048, out_features=num_classes, bias=True)\nmodel.aux_logits = False\nmodel = model.to(device=device)\n\noptimizer = torch.optim.Adam(model.parameters(), lr = learning_rate)\nloss_criterion = torch.nn.CrossEntropyLoss()","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:24:01.975831Z","iopub.execute_input":"2022-07-26T06:24:01.976334Z","iopub.status.idle":"2022-07-26T06:24:15.428252Z","shell.execute_reply.started":"2022-07-26T06:24:01.976292Z","shell.execute_reply":"2022-07-26T06:24:15.427353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:24:15.429501Z","iopub.execute_input":"2022-07-26T06:24:15.429826Z","iopub.status.idle":"2022-07-26T06:24:15.434682Z","shell.execute_reply.started":"2022-07-26T06:24:15.429793Z","shell.execute_reply":"2022-07-26T06:24:15.433800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{}},{"cell_type":"code","source":"from tqdm import tqdm\n\ndef check_accuracy(model, loader):\n    model.eval()\n    \n    correct_output = 0\n    total_output = 0\n    \n    with torch.no_grad():\n        for x, y in tqdm(loader):\n            x = x.to(device=device)\n            y = y.to(device=device)\n            \n            score = model(x)\n            _,predictions = score.max(1)\n            \n            correct_output += (y==predictions).sum()\n            total_output += predictions.shape[0]\n    model.train()\n    print(f\"out of {total_output} , total correct: {correct_output} with an accuracy of {float(correct_output/total_output)*100}\")\n    return float(correct_output/total_output)*100\n\nnum_epochs = 10\nbest_score = 0\nfor epoch in range(num_epochs):\n    for data, target in tqdm(train_dataloader):\n        data = data.to(device=device)\n        target = target.to(device=device)\n        \n        score = model(data)\n        optimizer.zero_grad()\n        \n        loss = loss_criterion(score, target)\n        loss.backward()\n        \n        optimizer.step()\n    \n    print(f\"for epoch {epoch}, loss : {loss}\")\n    val_acc = check_accuracy(model, val_dataloader)\n    if val_acc > best_score:\n        best_score = val_acc\n        torch.save(model.state_dict(), \"/kaggle/working/best_model.bin\")\n    ","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:24:15.435925Z","iopub.execute_input":"2022-07-26T06:24:15.436410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluate","metadata":{}},{"cell_type":"code","source":"model.load_state_dict(torch.load(\"/kaggle/working/best_model.bin\"))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_preds = []\ntest_labels = []\nwith torch.no_grad():\n    for data, target in tqdm(test_dataloader):\n        data = data.to(device=device)\n        target = target.to(device=device)\n        logit = model(data)\n        pred = torch.softmax(logit, dim=-1)\n        pred = pred.cpu().numpy()\n        label = target.cpu().numpy()\n        test_preds.append(pred)\n        test_labels.append(label)\n        \ntest_preds = np.concatenate(test_preds, axis=0)\ntest_labels = np.concatenate(test_labels, axis=0)        ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import roc_auc_score\n\ndef calc_ece(softmax, label, bins=15):\n    softmax = torch.FloatTensor(softmax)\n    label = torch.FloatTensor(label)\n    bin_boundaries = torch.linspace(0, 1, bins + 1)\n    bin_lowers = bin_boundaries[:-1]\n    bin_uppers = bin_boundaries[1:]\n\n    softmax = torch.tensor(softmax)\n    labels = torch.tensor(label)\n\n    softmax_max, predictions = torch.max(softmax, 1)\n    correctness = predictions.eq(labels)\n\n    ece = torch.zeros(1)\n    mce = 0\n\n    for bin_lower, bin_upper in zip(bin_lowers, bin_uppers):\n        in_bin = softmax_max.gt(bin_lower.item()) * softmax_max.le(bin_upper.item())\n        prop_in_bin = in_bin.float().mean()\n\n        if prop_in_bin.item() > 0.0:\n            accuracy_in_bin = correctness[in_bin].float().mean()\n            avg_confidence_in_bin = softmax_max[in_bin].mean()\n            diff = torch.abs(avg_confidence_in_bin - accuracy_in_bin)\n            ece += diff * prop_in_bin\n            if diff.item() > mce:\n                mce = diff.item()\n\n    return ece.item(), mce\n\n# auroc\nfor c in range(num_classes):\n    if c == 0:\n        continue\n    c_preds = test_preds[:, c]\n    c_labels = np.uint8(test_labels == c)\n    test_auc = roc_auc_score(c_labels, c_preds)\n    print(f\"CLASS {c} AUC : {test_auc}\")\nece_score = calc_ece(test_preds, test_labels)\nprint(f\"ECE : {ece_score[0] * 100}, MCE : {ece_score[1] * 100}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# OOD Detection","metadata":{}},{"cell_type":"code","source":"from tqdm import trange\nnum_ood = 100\nood_preds = []\nwith torch.no_grad():\n    for i in trange(num_ood):\n        img = np.array(Image.open(test_files[i]))\n        np.random.shuffle(img)\n        img = np.transpose(img, [2, 0, 1])\n        img_tensor = torch.FloatTensor(img).to(device).unsqueeze(0)\n        logit = model(img_tensor)\n        pred = torch.softmax(logit, dim=-1)\n        pred = pred.cpu().numpy()\n        ood_preds.append(pred)\nood_preds = np.concatenate(ood_preds, axis=0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_labels = [1] * len(test_labels) + [0] * len(ood_preds)\ntotal_preds = np.concatenate([test_preds, ood_preds], axis=0)\nmsp = np.max(total_preds, axis=-1)\nood_auc = roc_auc_score(total_labels, msp)\nprint(f\"OOD Detection performance (AUC) : {ood_auc}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}