{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# HuBMAP Prediction Visualization","metadata":{}},{"cell_type":"markdown","source":"**Here is a notebook showing how one can visualize predictions made by your model in relation to validation data. Hope you find it useful** 😃","metadata":{}},{"cell_type":"markdown","source":"**Inspired by** :
\nhttps://www.kaggle.com/code/leonidkulyk/eda-hubmap-hhv-interactive-annotations
\nhttps://www.kaggle.com/code/itsuki9180/hubmap-inference","metadata":{}},{"cell_type":"markdown","source":"# Install packages","metadata":{}},{"cell_type":"code","source":"import os, glob\nimport sys\nimport json\nfrom PIL import Image\nfrom collections import Counter\n\nimport numpy as np\nimport pandas as pd\nimport plotly.express as px\nimport plotly.graph_objects as go\nimport tifffile as tiff\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nimport torch\nimport cv2\nfrom skimage.morphology import binary_dilation\n\nimport pandas as pd\n\nfrom sklearn.model_selection import KFold\n\nsys.path.append(\"/kaggle/input/detection-wheel\")","metadata":{"execution":{"iopub.status.busy":"2023-06-29T06:43:05.283183Z","iopub.execute_input":"2023-06-29T06:43:05.283806Z","iopub.status.idle":"2023-06-29T06:43:10.658866Z","shell.execute_reply.started":"2023-06-29T06:43:05.283743Z","shell.execute_reply":"2023-06-29T06:43:10.65789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Install pycocotools package\nimport os\n!mkdir /kaggle/working/packages\n!cp -r /kaggle/input/pycocotools/* /kaggle/working/packages\nos.chdir(\"/kaggle/working/packages/pycocotools-2.0.6/\")\n!python setup.py install -q\n!pip install . --no-index --find-links /kaggle/working/packages/ -q\nos.chdir(\"/kaggle/working\")","metadata":{"execution":{"iopub.status.busy":"2023-06-29T06:43:10.660723Z","iopub.execute_input":"2023-06-29T06:43:10.661425Z","iopub.status.idle":"2023-06-29T06:44:01.414482Z","shell.execute_reply.started":"2023-06-29T06:43:10.661392Z","shell.execute_reply":"2023-06-29T06:44:01.413255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define model","metadata":{}},{"cell_type":"code","source":"import base64\nimport numpy as np\nfrom pycocotools import _mask as coco_mask\nimport typing as t\nimport zlib\n\nimport torchvision\nimport torchvision\nfrom torchvision.models.detection.faster_rcnn import FastRCNNPredictor\nfrom torchvision.models.detection.mask_rcnn import MaskRCNNPredictor\nfrom torchvision.models.resnet import ResNet50_Weights\n\ndef get_model_instance_segmentation(num_classes):\n # load an instance segmentation model pre-trained on COCO\n model = torchvision.models.detection.maskrcnn_resnet50_fpn_v2(weights=\"DEFAULT\", weights_backbone=ResNet50_Weights.IMAGENET1K_V2)\n\n # get number of input features for the classifier\n in_features = model.roi_heads.box_predictor.cls_score.in_features\n # replace the pre-trained head with a new one\n model.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)\n\n # now get the number of input features for the mask classifier\n in_features_mask = model.roi_heads.mask_predictor.conv5_mask.in_channels\n hidden_layer = 256\n # and replace the mask predictor with a new one\n model.roi_heads.mask_predictor = MaskRCNNPredictor(in_features_mask,\n hidden_layer,\n num_classes)\n\n return model","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_path = '/kaggle/input/hubmap-train/fold_0_epoch4.pth'","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = get_model_instance_segmentation(num_classes=2)\nmodel.to(device)\nmodel.load_state_dict(torch.load(model_path))\nmodel.eval()\nprint()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data","metadata":{}},{"cell_type":"code","source":"n_imgs = len(glob.glob('/kaggle/input/hubmap-making-dataset/train/image/*'))\nn_imgs","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kf = KFold(n_splits=5, shuffle=True, random_state=43) # Since model was trained where data was partition with random_state=43\nval_img = []\nval_mask = []\nval_indexes = []\nfor i, (train_index, test_index) in enumerate(kf.split(range(n_imgs))):\n if i!=0: continue\n all_imgs = sorted(glob.glob('/kaggle/input/hubmap-making-dataset/train/image/*.png'))\n all_masks = sorted(glob.glob('/kaggle/input/hubmap-making-dataset/train/mask/*.png'))\n all_imgs = np.array(all_imgs)\n all_masks = np.array(all_masks)\n train_img = all_imgs[train_index]\n train_mask = all_masks[train_index]\n val_img = all_imgs[test_index]\n val_mask = all_masks[test_index]\n val_indexes = test_index\n break","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualize","metadata":{}},{"cell_type":"code","source":"def get_cartesian_coords(coords, img_height):\n coords_array = np.array(coords).squeeze()\n xs = coords_array[:, 0]\n ys = -coords_array[:, 1] + img_height\n \n return xs, ys\n\ndef plot_annotated_image(image_dict, image_path, scale_factor: int = 1.0) -> None:\n \n for i, tldc in enumerate(tqdm(image_dict)):\n if tldc[\"id\"] == os.path.splitext(os.path.basename(image_path))[0]:\n #array = tiff.imread(CFG.img_path_template.format(image_dict[\"id\"]))\n array = tiff.imread(f'/kaggle/input/hubmap-hacking-the-human-vasculature/train/{tldc[\"id\"]}.tif')\n #image_path = [f'/kaggle/input/hubmap-making-dataset/train/image/{image_dict[\"id\"]}.png']\n #array = mpimg.imread(image_path[0])\n\n img_example = Image.fromarray(array)\n annotations = tldc[\"annotations\"]\n\n # create figure\n fig = go.Figure()\n\n # constants\n img_width = img_example.size[0]\n img_height = img_example.size[1]\n\n\n # add invisible scatter trace\n fig.add_trace(\n go.Scatter(\n x=[0, img_width],\n y=[0, img_height],\n mode=\"markers\",\n marker_opacity=0\n )\n )\n\n # configure axes\n fig.update_xaxes(\n visible=False,\n range=[0, img_width]\n )\n\n fig.update_yaxes(\n visible=False,\n range=[0, img_height],\n # the scaleanchor attribute ensures that the aspect ratio stays constant\n scaleanchor=\"x\"\n )\n\n # add image\n fig.add_layout_image(dict(\n x=0,\n sizex=img_width,\n y=img_height,\n sizey=img_height,\n xref=\"x\", yref=\"y\",\n opacity=1.0,\n layer=\"below\",\n sizing=\"stretch\",\n source=img_example\n ))\n\n # add polygons\n for annotation in annotations:\n name = annotation[\"type\"]\n xs, ys = get_cartesian_coords(annotation[\"coordinates\"], img_height)\n fig.add_trace(go.Scatter(\n x=xs, y=ys, fill=\"toself\",\n name=name,\n hovertemplate=\"%{name}\",\n mode='lines'\n ))\n\n # configure other layout\n fig.update_layout(\n width=img_width * scale_factor,\n height=img_height * scale_factor,\n margin={\"l\": 0, \"r\": 0, \"t\": 0, \"b\": 0},\n showlegend=False\n )\n\n # disable the autosize on double click because it adds unwanted margins around the image\n # and finally show figure\n fig.show(config={'doubleClick': 'reset'})\n\n break","metadata":{"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nimport matplotlib.patches as mpatches\n\ndef plot_prediction_mask(image_path, masks, scores):\n # Load the original image\n image = mpimg.imread(image_path)\n #image = tiff.imread(f'/kaggle/input/hubmap-hacking-the-human-vasculature/train/{image_path[\"id\"]}.tif')\n\n fig, ax = plt.subplots(figsize=(7, 7))\n\n # Plot the original image\n ax.imshow(image)\n \n for num, mask in enumerate(masks):\n\n masked_image = np.ma.masked_where(mask == 0, mask)\n\n # Overlay the masked image on the original image\n im = ax.imshow(masked_image, 'Set1', interpolation='none', alpha=0.5)\n \n # Calculate the centroid of the mask\n mask_indices = np.where(mask != 0)\n y_center = np.mean(mask_indices[0])\n x_center = np.mean(mask_indices[1])\n\n # Add text label next to the mask\n ax.text(x_center, y_center, f\"{(num+1)}\", color='yellow', backgroundcolor='none', fontsize=7,\n verticalalignment='center', horizontalalignment='center', weight='bold')\n\n\n # Set the plot limits and aspect ratio\n ax.set_xlim(0, image.shape[1])\n ax.set_ylim(image.shape[0], 0)\n ax.set_aspect('equal')\n \n # Add the custom legend\n # Create a legend patch for the current mask\n legend_patch = mpatches.Patch(color='red', alpha=1.0, label='Blood vessels')\n ax.legend(handles=[legend_patch], bbox_to_anchor=(1.35, 1), loc='upper right')\n\n # Hide the axis ticks and labels\n ax.axis('off')\n\n # Show the plot\n plt.show()","metadata":{"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_pred_and_truth(img_index, score_limit):\n\n # Make prediction\n previous_masks = []\n previous_scores = []\n previous_boxes = []\n with torch.no_grad():\n\n img_path = val_img[img_index]\n array = Image.open(img_path).convert(\"RGB\")\n img = Image.fromarray(np.array(array))\n\n # Convert the PIL image to a tensor\n transform = torchvision.transforms.ToTensor()\n img_tensor = transform(img)\n\n # Add an extra dimension to the tensor\n img = img_tensor.unsqueeze(0)\n\n img = img.to(device)\n pred = model(img)\n\n for m in range(len(pred[0]['masks'])):\n score = pred[0]['scores'][m].detach().cpu().numpy()\n box = pred[0]['boxes'][m].detach().cpu().numpy()\n\n mask = pred[0]['masks'][m].detach().permute(1,2,0).cpu().numpy()\n mask = np.where(mask>0.5, 1, 0).astype(np.bool)\n mask = binary_dilation(mask)\n mask = mask.astype(np.uint8)\n mask = mask.squeeze()\n\n if score >= score_limit:\n previous_masks.append(mask)\n previous_scores.append(score)\n previous_boxes.append(box)\n \n # Display predictions\n print(f\"Number of predictions: {len(previous_masks)}\")\n all_scores = {}\n for i in range(len(previous_scores)):\n all_scores[f\"{i+1}\"] = float(previous_scores[i])\n print(f\"Scores of predictions: {all_scores}\")\n \n print()\n print(\"Prediction\")\n image_path = val_img[img_index] # tiles_dicts[val_indexes[img_index]]\n plot_prediction_mask(image_path, previous_masks, previous_scores)\n\n # Display true values\n with open('/kaggle/input/hubmap-hacking-the-human-vasculature/polygons.jsonl', 'r') as json_file:\n json_list = list(json_file)\n tiles_dicts = []\n for json_str in json_list:\n tiles_dicts.append(json.loads(json_str))\n\n print()\n print(\"True labels\")\n plot_annotated_image(tiles_dicts, image_path)","metadata":{"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"img_index range: 0 - {len(val_indexes)-1}\")","metadata":{"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_pred_and_truth(img_index=0, score_limit=0.7)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_pred_and_truth(img_index=137, score_limit=0.7)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_pred_and_truth(img_index=245, score_limit=0.7)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n\n
","metadata":{}}]}