{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **EfficientDet Inference**","metadata":{}},{"cell_type":"markdown","source":"* [Dependencies and imports](#section-one)\n* [Overide functions](#section-two)\n* [Basic configurations](#section-three)\n* [Prepare Dataset](#section-four)\n    * [DICOM files](#section-four-one)\n    * [Augmentation for resizing image](#section-four-two)\n* [Create custom dataset](#section-five)\n* [Object detection and multi-class classification](#section-seven)\n    * [Load pre-trained models](#section-seven-one)\n    * [Ensemble models](#section-seven-two)\n    * [Create test dataset and dataloader](#section-seven-three)\n    * [Get predictions](#section-seven-four)\n* [Create submission file](#section-eight)","metadata":{}},{"cell_type":"markdown","source":"<a id=\"section-one\"></a>\n## **Dependencies and imports**","metadata":{}},{"cell_type":"code","source":"!pip install --no-deps '../input/timm-package/timm-0.1.26-py3-none-any.whl' > /dev/null\n!pip install --no-deps '../input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2021-08-10T21:06:11.658851Z","iopub.execute_input":"2021-08-10T21:06:11.659316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!conda install '../input/dicom/libjpeg-turbo-2.1.0-h7f98852_0.tar.bz2' -c conda-forge -y\n!conda install '../input/dicom/libgcc-ng-9.3.0-h2828fa1_19.tar.bz2' -c conda-forge -y\n!conda install '../input/dicom/gdcm-2.8.9-py37h500ead1_1.tar.bz2' -c conda-forge -y\n!conda install '../input/dicom/conda-4.10.1-py37h89c1867_0.tar.bz2' -c conda-forge -y\n!conda install '../input/dicom/certifi-2020.12.5-py37h89c1867_1.tar.bz2' -c conda-forge -y\n!conda install '../input/dicom/openssl-1.1.1k-h7f98852_0.tar.bz2' -c conda-forge -y","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append('../input/timm-efficientdet-pytorch')\nsys.path.append(\"../input/omegaconf\")\nsys.path.append(\"../input/weightedboxesfusion\")\nsys.path.append('../input/imagemodels/pytorch-image-models-master')\n\nimport torch\nfrom torch import nn\nimport timm\nimport numpy as np\nimport pandas as pd\nimport gc\nimport os\nimport ast\nfrom tqdm.notebook import tqdm\nfrom matplotlib import pyplot as plt\n# --- effdet ---\nfrom effdet import get_efficientdet_config, EfficientDet, DetBenchEval\nfrom effdet.efficientdet import HeadNet\n# --- ensemble boxes ---\nimport ensemble_boxes\nfrom ensemble_boxes import *\n# --- data ---\nfrom torch.utils.data import Dataset,DataLoader\n# --- images ---\nimport albumentations as A\nimport cv2\n# --- wandb ---\nimport wandb\nfrom kaggle_secrets import UserSecretsClient\n# --- dicom ---\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"OFFLINE = True\n\nif not OFFLINE:\n    user_secrets = UserSecretsClient()\n    wandb_key = user_secrets.get_secret(\"wandb-key\")\n    wandb.login(key=wandb_key)\n\n    run = wandb.init(project=\"siim-covid19-detection\", name=\"inference\", mode='online')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"section-two\"></a>\n## **Overide functions**\n_post_process in effdet/bench.py raised error due division for calculating indices returned float (using '/') instead of an int (using '//')","metadata":{}},{"cell_type":"code","source":"import effdet.bench as bench\nfrom effdet.anchors import MAX_DETECTION_POINTS\n\ndef new_post_process(config, cls_outputs, box_outputs):\n    \"\"\"Selects top-k predictions.\n\n    Post-proc code adapted from Tensorflow version at: https://github.com/google/automl/tree/master/efficientdet\n    and optimized for PyTorch.\n\n    Args:\n        config: a parameter dictionary that includes `min_level`, `max_level`,  `batch_size`, and `num_classes`.\n\n        cls_outputs: an OrderDict with keys representing levels and values\n            representing logits in [batch_size, height, width, num_anchors].\n\n        box_outputs: an OrderDict with keys representing levels and values\n            representing box regression targets in [batch_size, height, width, num_anchors * 4].\n    \"\"\"\n    batch_size = cls_outputs[0].shape[0]\n    cls_outputs_all = torch.cat([\n        cls_outputs[level].permute(0, 2, 3, 1).reshape([batch_size, -1, config.num_classes])\n        for level in range(config.num_levels)], 1)\n\n    box_outputs_all = torch.cat([\n        box_outputs[level].permute(0, 2, 3, 1).reshape([batch_size, -1, 4])\n        for level in range(config.num_levels)], 1)\n\n    _, cls_topk_indices_all = torch.topk(cls_outputs_all.reshape(batch_size, -1), dim=1, k=MAX_DETECTION_POINTS)\n    ############################# changed / to // as indices should be int64 #############################\n    indices_all = cls_topk_indices_all // config.num_classes \n    ######################################################################################################\n    classes_all = cls_topk_indices_all % config.num_classes\n\n    box_outputs_all_after_topk = torch.gather(\n        box_outputs_all, 1, indices_all.unsqueeze(2).expand(-1, -1, 4))\n\n    cls_outputs_all_after_topk = torch.gather(\n        cls_outputs_all, 1, indices_all.unsqueeze(2).expand(-1, -1, config.num_classes))\n    cls_outputs_all_after_topk = torch.gather(\n        cls_outputs_all_after_topk, 2, classes_all.unsqueeze(2))\n\n    return cls_outputs_all_after_topk, box_outputs_all_after_topk, indices_all, classes_all\n\nbench._post_process  = new_post_process","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"section-three\"></a>\n## **Basic configurations**","metadata":{}},{"cell_type":"code","source":"NONE = 'none'\nOPACITY = 'opacity'\n\nNEGATIVE = 'negative'\nTYPICAL = 'typical'\nINDERTEMINATE = 'indeterminate'\nATYPICAL = 'atypical'\n\nclass Configs:\n    n_folds = 5\n    img_size = 512\n    device = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")\n    batch_size = 4\n    num_workers = 8\n    study_level = {1:TYPICAL, 2:INDERTEMINATE, 3:ATYPICAL}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"section-four\"></a>\n## **Prepare Dataset**","metadata":{}},{"cell_type":"markdown","source":"<a id=\"section-four-one\"></a>\n#### **DICOM files**","metadata":{}},{"cell_type":"markdown","source":"**Get DICOM files paths**","metadata":{}},{"cell_type":"code","source":"paths = []\nfor dirname, _, filenames in os.walk('../input/siim-covid19-detection/test'):\n    for filename in filenames:\n        paths.append(os.path.join(dirname, filename))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_img_id(path):\n    # dicom path of format '../input/siim-covid19-detection/test/study_id/dir/image_id.dcm'\n    return path.split('/')[-1].split('.')[0] # extract img_id from path\n\ndef get_study_id(path):\n    # dicom path of format '../input/siim-covid19-detection/test/study_id/dir/dicom_image'\n    return path.split('/')[-3]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Get dicom image**","metadata":{}},{"cell_type":"code","source":"def get_dicom_img(path):\n    data_file = pydicom.dcmread(path)\n    img = apply_voi_lut(data_file.pixel_array, data_file)\n\n    if data_file.PhotometricInterpretation == \"MONOCHROME1\":\n        img = np.amax(img) - img\n    \n    # Rescaling grey scale between 0-255 and convert to uint\n    img = img - np.min(img)\n    img = img / np.max(img)\n    img = (img * 255).astype(np.uint8)\n\n    return img","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"section-four-two\"></a>\n#### **Augmentation for resizing image**","metadata":{}},{"cell_type":"code","source":"def get_test_transforms():\n    return A.Compose([A.Resize(height=Configs.img_size, width=Configs.img_size, p=1.0),], p=1.0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"section-five\"></a>\n## **Create custom dataset**","metadata":{}},{"cell_type":"code","source":"class Covid19TestDataset(Dataset):\n    def __init__(self, dicom_paths, transform=None):\n        super().__init__()\n        self.paths = dicom_paths\n        self.transform = transform\n\n    def __getitem__(self, idx):\n        img_id = get_img_id(self.paths[idx])\n        study_id = get_study_id(self.paths[idx])\n        img = get_dicom_img(self.paths[idx])\n        #img = cv2.imread(img_path, cv2.IMREAD_COLOR)\n        \n        if self.transform:\n            transformed = self.transform(image=img)\n            img = transformed['image']\n           \n        # normalize img\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB).astype(np.float32)\n        img /= 255.0\n        \n        # convert image into a torch.Tensor\n        img = torch.as_tensor(img, dtype=torch.float32)\n        #idx = torch.tensor([idx])\n        \n        # permute image to [C,H,W] from [H,W,C] and normalize\n        img = img.permute(2, 0, 1)\n        \n        return img, img_id, study_id\n    \n    def __len__(self):\n        return len(self.paths)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"section-seven\"></a>\n## **Object detection and multi-class classification**","metadata":{}},{"cell_type":"markdown","source":"<a id=\"section-seven-one\"></a>\n#### **Load pre-trained models**","metadata":{}},{"cell_type":"code","source":"# get efficientdet model\ndef get_model(model_name):\n    config = get_efficientdet_config(model_name)\n    model = EfficientDet(config, pretrained_backbone=False)\n\n    config.num_classes = len(Configs.study_level)\n    config.image_size = Configs.img_size\n\n    checkpoint = torch.load('../input/efficientdet/efficientdet_d7-f05bf714.pth')\n    model.load_state_dict(checkpoint)\n    \n    model.class_net = HeadNet(config, num_outputs=config.num_classes)\n\n    model = DetBenchEval(model, config)\n    model.eval();\n    return model\n\n# load checkpoint\ndef load_obj_detection_model(model_name, fold, checkpoint_path):\n    model = get_model(model_name)\n    checkpoint = torch.load(checkpoint_path)['model_state_dict']\n    checkpoint['anchors.boxes'] = checkpoint.pop('anchor_labeler.anchors.boxes')\n    model.load_state_dict(checkpoint)\n    \n    return model.to(Configs.device)\n\nmodels = []\nmodel_name = 'tf_efficientdet_d7'\nfor fold in range(Configs.n_folds): \n    models.append(load_obj_detection_model(model_name, fold, f'../input/models/object_detection_models/{model_name}/{model_name}_fold{fold}/best-checkpoint.bin'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def log_prediction(image, image_id, boxes, labels):\n    new_image = image.permute(1, 2, 0).numpy().copy() \n    new_image = (new_image*255).astype(np.uint8)\n    image_size = max(image.shape)\n    caption = []\n    \n    for label in labels:\n        caption.append(Configs.study_level[label])\n    for box in boxes:\n        x, y, w, h = box\n        cv2.rectangle(new_image, (int(x), (int(y))), (int(x+w),  int(y+h)), (255,0,0), image_size//200)\n        \n    wandb.log({'predictions/'+image_id: [wandb.Image(new_image, caption=', '.join(caption))]})","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"section-seven-two\"></a>\n#### **Ensemble models**","metadata":{}},{"cell_type":"code","source":"# wbf = weighted boxes fusion for ensembling boxes from object detection models\ndef run_wbf(predictions, img_idx, iou_thr=0.65, skip_box_thr=0.4, weights=None):\n    # prediction[img_idx] is the prediction of each model for the same image (the image in img_idx)\n    # weighted_boxes_fusion function received boxes with values in range [0-1]\n    # normalize by dividing over Configs.img_size-1 (includes 0, therefore -1)\n    boxes = [(prediction[img_idx]['boxes']/(Configs.img_size-1)).tolist() for prediction in predictions]\n    scores = [prediction[img_idx]['scores'].tolist() for prediction in predictions]\n    labels = [prediction[img_idx]['labels'].astype(int).tolist() for prediction in predictions]\n    boxes, scores, labels = ensemble_boxes.ensemble_boxes_wbf.weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)\n    # scale back boxes by multiplying with Configs.img_size-1\n    boxes = boxes*(Configs.img_size-1)\n    return boxes, scores, labels\n\n# prediction for a single batch\ndef batch_predictions(images, image_ids, score_threshold=0.4):\n    with torch.no_grad():\n        predictions = []\n        images = images.float().cuda()\n        \n        # for each model get predictions for each image in batch\n        for model in models:  \n            model_predictions = []\n            outputs = model(images, torch.tensor([1]*images.shape[0]).float().cuda())\n        \n            for i, (image, image_id) in enumerate(zip(images, image_ids)):\n                boxes = outputs[i].detach().cpu().numpy()[:,:4]    \n                scores = outputs[i].detach().cpu().numpy()[:,4]\n                labels = outputs[i].detach().cpu().numpy()[:, 5]\n                indexes = np.where(scores > score_threshold)[0]\n                \n                model_predictions.append({\n                'boxes': boxes[indexes],\n                'scores': scores[indexes],\n                'labels': labels[indexes]\n                })\n            # append model prediction to all predictions\n            predictions.append(model_predictions)     \n            \n    return predictions\n\ndef get_boxes(image, boxes):\n    # get ratio to scale boxes\n    new_image = image.permute(1, 2, 0).numpy().copy() \n    height, width = new_image.shape[0], new_image.shape[1]\n    h_ratio = height / Configs.img_size\n    w_ratio = width / Configs.img_size\n    \n    # scale and convert from xywh to xyxy\n    new_boxes = []\n    for box in boxes:\n        x,y,w,h = box\n        x1=x*w_ratio\n        x2=(x+w)*w_ratio\n        y1=y*h_ratio\n        y2=(y+h)*h_ratio\n        new_boxes.append(np.array([int(x1),int(y1),int(x2),int(y2)]))\n        \n    return np.array(new_boxes)    \n            \ndef ensemble_predictions(test_loader):\n    # returns a df with img_id, study_id and predicted values (labels, scores and boxes) for each image\n    img_ids = []\n    stdy_ids = []\n    pred_boxes =[]\n    pred_scores = []\n    pred_labels = []\n    \n    for images, image_ids, study_ids in tqdm(test_loader):\n        # get batch predictions of all models\n        # predictions[0] = batch predictions for model0, predictions[1] = batch predictions for model1 and etc...\n        predictions = batch_predictions(images, image_ids)\n        for i, (image, image_id, study_id) in enumerate(zip(images, image_ids, study_ids)):\n            # for each image get ensembled prediction by using wbf\n            boxes, scores, labels = run_wbf(predictions, img_idx=i)\n            if not OFFLINE:\n                log_prediction(image, image_id, boxes, labels)\n            new_boxes = get_boxes(image, boxes)\n            \n            img_ids.append(image_id)\n            stdy_ids.append(study_id)\n            pred_boxes.append(new_boxes)\n            pred_scores.append(scores)\n            pred_labels.append(labels.astype(int))\n            \n    return pd.DataFrame({'img_id':img_ids, 'study_id':stdy_ids, 'labels':pred_labels, 'scores':pred_scores, 'boxes':pred_boxes})","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"section-seven-three\"></a>\n#### **Create test dataset and dataloader**","metadata":{}},{"cell_type":"code","source":"test_dataset = Covid19TestDataset(paths, transform=get_test_transforms())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data.sampler import SequentialSampler\n\ntest_loader = torch.utils.data.DataLoader(\n    test_dataset, \n    batch_size=Configs.batch_size,\n    num_workers=Configs.num_workers,\n    shuffle=False,\n    pin_memory=False,\n    drop_last=False,\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"section-seven-four\"></a>\n#### **Get predictions**","metadata":{}},{"cell_type":"code","source":"preds_df = ensemble_predictions(test_loader)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"section-eight\"></a>\n## **Create submission file**","metadata":{}},{"cell_type":"code","source":"def create_image_prediction_string(row):\n    pred_strings = []\n    if len(row['labels'].values[0]) > 0:\n        # if there are predicted labels\n        for score,box in zip(row['scores'].values[0], row['boxes'].values[0]):\n            x1,y1,x2,y2 = box\n            pred_strings.append(\"opacity {:.1f} {} {} {} {}\".format(score, x1, y1, x2, y2))\n        return ' '.join(pred_strings)\n    else:\n        return 'none 1 0 0 1 1'\n\ndef create_study_prediction_string(rows):\n    pred_strings = []\n    for index, row in rows.iterrows():\n        if len(row['labels'])==0:\n            pred_strings.append('negative 1 0 0 1 1')\n        else:\n            labels = []\n            for label in row['labels']:\n                if label not in labels:\n                    labels.append(label)\n            for label in labels:\n                pred_strings.append(\"{} 1 0 0 1 1\".format(Configs.study_level[label]))\n    return ' '.join(pred_strings)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_prediction_string(sample_submission):\n    ids = []\n    preds = []\n    \n    for index, row in sample_submission.iterrows():\n        id = row['id']\n        ids.append(id)\n        if id.endswith('_image'):\n            id = id.split('_')[0]\n            preds.append(create_image_prediction_string(preds_df[preds_df['img_id'] == id]))\n        else:\n            id = id.split('_')[0]\n            preds.append(create_study_prediction_string(preds_df[preds_df['study_id'] == id]))\n    \n    return pd.DataFrame({'id':ids, 'PredictionString':preds})\n\nsubmission = create_prediction_string(pd.read_csv('../input/siim-covid19-detection/sample_submission.csv'))\nsubmission","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('./submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}