{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# EfficientDet Pytorch Inference Starter","metadata":{}},{"cell_type":"markdown","source":"Hi Again Everyone,\n\nFirst of all - Go [here](https://www.kaggle.com/mikeb127/efficientdet-pytorch-training-starter) to get training starter. Submitting this notebook as is without properly training the starter will probably result in a pretty low score as the starter has only been trained for an epoch or two. You will also have to add your classifier model.","metadata":{}},{"cell_type":"markdown","source":"First of all, lets load all our dependencies:","metadata":{}},{"cell_type":"code","source":"!conda install '/kaggle/input/pydicom-conda-helper/libjpeg-turbo-2.1.0-h7f98852_0.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/libgcc-ng-9.3.0-h2828fa1_19.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/gdcm-2.8.9-py37h500ead1_1.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/conda-4.10.1-py37h89c1867_0.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/certifi-2020.12.5-py37h89c1867_1.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/openssl-1.1.1k-h7f98852_0.tar.bz2' -c conda-forge -y\n!pip install --no-deps '../input/timm-package/timm-0.1.26-py3-none-any.whl' > /dev/null\n!pip install --no-deps '../input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null","metadata":{"_uuid":"5506c8ee-389e-45ab-b6ce-c86e57cd20ee","_cell_guid":"23fc7d70-382b-4201-9ced-8b9f1fcc5df1","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-07-07T06:09:29.593848Z","iopub.execute_input":"2021-07-07T06:09:29.594210Z","iopub.status.idle":"2021-07-07T06:11:21.483553Z","shell.execute_reply.started":"2021-07-07T06:09:29.594125Z","shell.execute_reply":"2021-07-07T06:11:21.482532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Next, lets just get the imports:","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)  \nimport torch\nimport torch.nn.functional as F\nimport torchvision\nimport sys\nsys.path.insert(0, \"../input/timmefficienctdetpytorchstable/archive\")\nsys.path.insert(0, \"../input/omegaconf\")\nimport traceback\nfrom tqdm import tqdm\nfrom torch.utils.data import Dataset\nimport torchvision.models as models\nfrom PIL import Image\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut #Apply a VOI lookup table or windowing operation to arr.\nfrom torch import optim\nfrom effdet import *\nfrom effdet.efficientdet import HeadNet\nfrom effdet.anchors import Anchors, AnchorLabeler, generate_detections, MAX_DETECTION_POINTS\nfrom effdet.loss import DetectionLoss\n\nimport os\nfiles = []\nfor dirname, _, filenames in os.walk('/kaggle/input/siim-covid19-detection/test/'):\n    for filename in filenames:\n        files.append(os.path.join(dirname, filename))\n\nNormalizer = torchvision.transforms.Compose([torchvision.transforms.ToTensor(),\n                                             torchvision.transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                             std=[0.229, 0.224, 0.225])])","metadata":{"execution":{"iopub.status.busy":"2021-07-07T06:12:00.097406Z","iopub.execute_input":"2021-07-07T06:12:00.097749Z","iopub.status.idle":"2021-07-07T06:12:05.025983Z","shell.execute_reply.started":"2021-07-07T06:12:00.097718Z","shell.execute_reply":"2021-07-07T06:12:05.025111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Ok, below is all the code I had to modify to get things to work. This is not a tidy solution but works for now. If there is interest I will incorporate these changes into the referenced package so it can be used going forward","metadata":{}},{"cell_type":"code","source":"def _post_process(config, cls_outputs, box_outputs):\n    \"\"\"Selects top-k predictions.\n\n    Post-proc code adapted from Tensorflow version at: https://github.com/google/automl/tree/master/efficientdet\n    and optimized for PyTorch.\n\n    Args:\n        config: a parameter dictionary that includes `min_level`, `max_level`,  `batch_size`, and `num_classes`.\n\n        cls_outputs: an OrderDict with keys representing levels and values\n            representing logits in [batch_size, height, width, num_anchors].\n\n        box_outputs: an OrderDict with keys representing levels and values\n            representing box regression targets in [batch_size, height, width, num_anchors * 4].\n    \"\"\"\n    batch_size = cls_outputs[0].shape[0]\n    cls_outputs_all = torch.cat([\n        cls_outputs[level].permute(0, 2, 3, 1).reshape([batch_size, -1, config.num_classes])\n        for level in range(config.num_levels)], 1)\n\n    box_outputs_all = torch.cat([\n        box_outputs[level].permute(0, 2, 3, 1).reshape([batch_size, -1, 4])\n        for level in range(config.num_levels)], 1)\n\n    _, cls_topk_indices_all = torch.topk(cls_outputs_all.reshape(batch_size, -1), dim=1, k=MAX_DETECTION_POINTS)\n    indices_all = cls_topk_indices_all / config.num_classes\n    classes_all = cls_topk_indices_all % config.num_classes\n    \n    indices_all = indices_all.type(torch.long)\n\n    box_outputs_all_after_topk = torch.gather(\n        box_outputs_all, 1, indices_all.unsqueeze(2).expand(-1, -1, 4))\n\n    cls_outputs_all_after_topk = torch.gather(\n        cls_outputs_all, 1, indices_all.unsqueeze(2).expand(-1, -1, config.num_classes))\n    cls_outputs_all_after_topk = torch.gather(\n        cls_outputs_all_after_topk, 2, classes_all.unsqueeze(2))\n\n    return cls_outputs_all_after_topk, box_outputs_all_after_topk, indices_all, classes_all\n\n\nclass DetBenchEvalMB(torch.nn.Module):\n    \n    def __init__(self, model, config, device):\n        super(DetBenchEvalMB, self).__init__()\n        self.config = config\n        self.model = model\n        self.anchors = Anchors(\n            config.min_level, config.max_level,\n            config.num_scales, config.aspect_ratios,\n            config.anchor_scale, config.image_size,device)\n\n    def forward(self, x, image_scales):\n        class_out, box_out = self.model(x)\n        class_out, box_out, indices, classes = _post_process(self.config, class_out, box_out)\n\n        batch_detections = []\n        for i in range(x.shape[0]):\n            detections = generate_detections(\n                class_out[i], box_out[i], self.anchors.boxes, indices[i], classes[i], image_scales[i])\n            batch_detections.append(detections)\n        return torch.stack(batch_detections, dim=0)\n\n        \ndef get_efficientDet():\n    \n    config = get_efficientdet_config('tf_efficientdet_d1')\n    net = EfficientDet(config, pretrained_backbone=False)\n    config.num_classes = 1\n    config.image_size = 512\n    net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))\n    return net, config\n\nmodel, config = get_efficientDet()\n\n        \nclass out_class(torch.nn.Module):\n    \n    def __init__(self,backbone):\n        super(out_class, self).__init__()\n        self.backbone = backbone\n        self.sigmoid = torch.nn.Sigmoid()\n        \n    def forward(self,x):        \n        x = self.backbone(x)\n        x = self.sigmoid(x)\n        return x\n        \n        \ndef post_process_outputs(output, width, height):\n    for z in range(0,len(output)):\n        #Returns Values as xywh. Sort this out\n        output[z][:,0] = output[z][:,0] * width[z]/512\n        output[z][:,1] = output[z][:,1] * height[z]/512 \n        output[z][:,2] = output[z][:,2] * width[z]/512 \n        output[z][:,3] = output[z][:,3] * height[z]/512\n        output[z,:,3] = output[z,:,3] + output[z,:,1]\n        output[z,:,2] = output[z,:,2] + output[z,:,0]\n    return output","metadata":{"execution":{"iopub.status.busy":"2021-07-07T06:12:46.359397Z","iopub.execute_input":"2021-07-07T06:12:46.359752Z","iopub.status.idle":"2021-07-07T06:12:46.661526Z","shell.execute_reply.started":"2021-07-07T06:12:46.359721Z","shell.execute_reply":"2021-07-07T06:12:46.660773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's do our dataloader. Thanks here goes to users tensorchoko and raddar for the DICOM processing methods\n\nhttps://www.kaggle.com/tensorchoko/siim-yolov5-predict\n\nhttps://www.kaggle.com/raddar/convert-dicom-to-np-array-the-correct-way\n\n\n","metadata":{}},{"cell_type":"code","source":"def read_xray(path, voi_lut = True, fix_monochrome = True):\n    # Original from: https://www.kaggle.com/raddar/convert-dicom-to-np-array-the-correct-way\n    # I pinched it from https://www.kaggle.com/tensorchoko/siim-yolov5-predict\n    dicom = pydicom.dcmread(path)\n    \n    # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to \n    # \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n               \n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n        \n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n        \n    return data\n\n\nBATCH_SIZE = 2\n\ndef collate_fn(batch):\n    return tuple(zip(*batch))\n\n\nclass COVIDXRay_Dataset(Dataset):\n\n    def __init__(self, dcm_file_list, transform):\n\n        self.dcm_file_list = dcm_file_list\n        self.transform = transform\n\n    def __len__(self):\n        \n        return len(self.dcm_file_list)\n\n    def __getitem__(self, idx):\n        \n        #First of all, we load the image\n        image = read_xray(self.dcm_file_list[idx])\n        img = Image.fromarray(image).convert('RGB')\n        width = img.size[0]\n        height = img.size[1]\n        img = img.resize((512,512))\n        img = self.transform(img)\n        \n        #Lets get our desired output at the study level\n        file = self.dcm_file_list[idx].split('/')\n        \n        #First of all we get our study level ground truth\n        study_file = file[5] + '_study'\n        \n        #Now our image level ground truth\n        image_file = file[7].split('.')[0] + '_image'\n        dummy = torch.Tensor(np.ones(1))\n        return img, image_file, study_file, width, height, dummy\n    \n    \ntrain_dataset = COVIDXRay_Dataset(\n        files, Normalizer\n)\n\ndata_loader_test = torch.utils.data.DataLoader(\n    train_dataset,\n    batch_size=BATCH_SIZE,\n    num_workers=1,\n    shuffle=False,\n    collate_fn=collate_fn\n)","metadata":{"execution":{"iopub.status.busy":"2021-07-07T06:12:54.384478Z","iopub.execute_input":"2021-07-07T06:12:54.384828Z","iopub.status.idle":"2021-07-07T06:12:54.399709Z","shell.execute_reply.started":"2021-07-07T06:12:54.384799Z","shell.execute_reply":"2021-07-07T06:12:54.398802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Finally update the code below to load the model we trained previously and create a submission file","metadata":{}},{"cell_type":"code","source":"device = torch.device(\"cuda:0\")\n#@TODO: Replace this with the link to your copy of the training notebook!!\na = torch.load(###Your notebook here###)\nmodel.load_state_dict(a)\nmodel = model.to(device)\nmodel = DetBenchEvalMB(model,config,device)\nmodel.eval()\n\ntk0 = tqdm(data_loader_test, desc=\"Iteration\")\nbox_detections = []\n\nfor batch_idx, (input, image_file, study_file, width, height,dummy) in enumerate(tk0):\n    input = [i.to(device) for i in input]\n    dummy = [d.to(device) for d in dummy]\n    input = torch.stack(input)\n    output = model(input,dummy)\n    output = post_process_outputs(output,width,height)\n    output = output.cpu().detach().numpy()\n    for q in range(0,output.shape[0]):\n        boxes_string = ''\n        for b in range(0,output.shape[1]):\n            if output[q,b,4] > 0.1:          \n                boxes_string = boxes_string + 'opacity ' + str output[q,b,4]) + ' ' + str(int(output[q,b,0])) + ' ' + str(int(output[q,b,1])) + ' ' + str(int(output[q,b,2])) + ' ' + str(int(output[q,b,3])) + ' '\n        if len(boxes_string) == 0:\n            boxes_string = 'none 1 0 0 1 1 '\n        boxes_string = boxes_string[:len(boxes_string)-1]\n        box_string_rec = [image_file[q], boxes_string]\n        box_detections.append(box_string_rec)\n  \ndf_box_detections = pd.DataFrame(box_detections,columns=['id', 'PredictionString'])\ndf_box_detections.to_csv('/kaggle/working/submission.csv',index = False)","metadata":{"execution":{"iopub.status.busy":"2021-07-07T06:13:01.262668Z","iopub.execute_input":"2021-07-07T06:13:01.263545Z","iopub.status.idle":"2021-07-07T06:21:32.649844Z","shell.execute_reply.started":"2021-07-07T06:13:01.263505Z","shell.execute_reply":"2021-07-07T06:21:32.648987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"And that should be it! You can now combine this with your classification model.\n\nThanks for reading","metadata":{}}]}