{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!conda install gdcm -c conda-forge -y\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport pydicom\nimport glob\nfrom tqdm.notebook import tqdm\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nimport matplotlib.pyplot as plt\nfrom skimage import exposure\n#import gdcm\nimport cv2\nimport warnings\nfrom fastai.vision.all import *\nfrom fastai.medical.imaging import *\nwarnings.filterwarnings('ignore')\ndataset_path = Path('../input/siim-covid19-detection')\nimport vtk\n# numba\nimport numba\nfrom numba import jit\nfrom vtk.util import numpy_support\n\nreader = vtk.vtkDICOMImageReader()","metadata":{"execution":{"iopub.status.busy":"2021-09-27T19:51:53.142858Z","iopub.execute_input":"2021-09-27T19:51:53.14344Z","iopub.status.idle":"2021-09-27T19:52:51.89207Z","shell.execute_reply.started":"2021-09-27T19:51:53.143353Z","shell.execute_reply":"2021-09-27T19:52:51.891328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np \nimport pandas as pd \nfrom datetime import datetime\nimport time\nimport random\nfrom tqdm import tqdm_notebook as tqdm # progress bar\nimport matplotlib.pyplot as plt\n\n# torch\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\n\n# torchvision\nimport torchvision\nfrom torchvision.models.detection.faster_rcnn import FastRCNNPredictor, FasterRCNN\nfrom torchvision.models.detection.backbone_utils import resnet_fpn_backbone\n\n# sklearn\nfrom sklearn.model_selection import StratifiedKFold\n\n# CV\nimport cv2\n\n# Albumenatations\nimport albumentations as A\nfrom albumentations.pytorch.transforms import ToTensorV2\n\n#from pycocotools.coco import COCO\nfrom sklearn.model_selection import StratifiedKFold\n\n# glob\nfrom glob import glob\n\n# numba\nimport numba\nfrom numba import jit\n\nimport warnings\nwarnings.filterwarnings('ignore') #Ignore \"future\" warnings and Data-Frame-Slicing warnings.","metadata":{"execution":{"iopub.status.busy":"2021-09-27T19:52:51.893905Z","iopub.execute_input":"2021-09-27T19:52:51.894162Z","iopub.status.idle":"2021-09-27T19:52:52.649716Z","shell.execute_reply.started":"2021-09-27T19:52:51.894122Z","shell.execute_reply":"2021-09-27T19:52:52.648932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DefaultConfig:\n    n_folds: int = 5\n    seed: int = 2021\n    num_classes: int = 4 # \"negative\", \"typical\", \"indeterminate\", \"atypical\"\n    img_size: int = 256\n    fold_num: int = 0\n    device: str = 'cuda:0'","metadata":{"execution":{"iopub.status.busy":"2021-09-27T19:52:52.651024Z","iopub.execute_input":"2021-09-27T19:52:52.651321Z","iopub.status.idle":"2021-09-27T19:52:52.657509Z","shell.execute_reply.started":"2021-09-27T19:52:52.651285Z","shell.execute_reply":"2021-09-27T19:52:52.655435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ce = torch.device(DefaultConfig.device) if torch.cuda.is_available() else torch.device('cpu')","metadata":{"execution":{"iopub.status.busy":"2021-09-27T19:52:52.659849Z","iopub.execute_input":"2021-09-27T19:52:52.660329Z","iopub.status.idle":"2021-09-27T19:52:52.665064Z","shell.execute_reply.started":"2021-09-27T19:52:52.660291Z","shell.execute_reply":"2021-09-27T19:52:52.664254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Choose your optimizers:\nAdam = False\nif Adam: \n    Adam_config = {\"lr\" : 0.001, \"betas\" : (0.9, 0.999), \"eps\" : 1e-08}\nelse:\n    SGD_config = {\"lr\" : 0.001, \"momentum\" : 0.9, \"weight_decay\" : 0.001}","metadata":{"execution":{"iopub.status.busy":"2021-09-27T19:52:52.666448Z","iopub.execute_input":"2021-09-27T19:52:52.666766Z","iopub.status.idle":"2021-09-27T19:52:52.673762Z","shell.execute_reply.started":"2021-09-27T19:52:52.666733Z","shell.execute_reply":"2021-09-27T19:52:52.673114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n    \nseed_everything(DefaultConfig.seed)","metadata":{"execution":{"iopub.status.busy":"2021-09-27T19:52:52.675161Z","iopub.execute_input":"2021-09-27T19:52:52.675461Z","iopub.status.idle":"2021-09-27T19:52:52.68423Z","shell.execute_reply.started":"2021-09-27T19:52:52.675427Z","shell.execute_reply":"2021-09-27T19:52:52.683489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Thanks https://www.kaggle.com/tanlikesmath/siim-covid-19-detection-a-simple-eda\ndef dicom2array(path, voi_lut=True, fix_monochrome=True):\n    # Original from: https://www.kaggle.com/raddar/convert-dicom-to-np-array-the-correct-way\n    dicom = pydicom.read_file(path)\n    \n    # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to \n    # \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n               \n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n        \n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n        \n    return data\n        \n    \ndef plot_img(img, size=(7, 7), is_rgb=True, title=\"\", cmap='gray'):\n    plt.figure(figsize=size)\n    plt.imshow(img, cmap=cmap)\n    plt.suptitle(title)\n    plt.show()\n\n\ndef plot_imgs(imgs, cols=4, size=7, is_rgb=True, title=\"\", cmap='gray', img_size=(500,500)):\n    rows = len(imgs)//cols + 1\n    fig = plt.figure(figsize=(cols*size, rows*size))\n    for i, img in enumerate(imgs):\n        if img_size is not None:\n            img = cv2.resize(img, img_size)\n        fig.add_subplot(rows, cols, i+1)\n        plt.imshow(img, cmap=cmap)\n    plt.suptitle(title)\n    plt.show()\n    \n\n\ndef image_path(row):\n    study_path = dataset_path/'train'/row.StudyInstanceUID\n    for i in get_dicom_files(study_path):\n        if row.id.split('_')[0] == i.stem: return i \n        \n\n\n\nclass Config:\n    n_folds: int = 5\n    seed: int = 2021\n    num_classes: int = 2 \n    img_size: int = 512\n    fold_num: int = 0\n    device: str = 'cuda:0'\n\n\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n    \nseed_everything(Config.seed)","metadata":{"execution":{"iopub.status.busy":"2021-09-27T19:52:52.685451Z","iopub.execute_input":"2021-09-27T19:52:52.685704Z","iopub.status.idle":"2021-09-27T19:52:52.702718Z","shell.execute_reply.started":"2021-09-27T19:52:52.685672Z","shell.execute_reply":"2021-09-27T19:52:52.702007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dicom_paths = get_dicom_files(dataset_path/'train')\nimgs = [dicom2array(path) for path in dicom_paths[:4]]\nplot_imgs(imgs)","metadata":{"execution":{"iopub.status.busy":"2021-09-27T19:52:52.703818Z","iopub.execute_input":"2021-09-27T19:52:52.704076Z","iopub.status.idle":"2021-09-27T19:53:31.717072Z","shell.execute_reply.started":"2021-09-27T19:52:52.704044Z","shell.execute_reply":"2021-09-27T19:53:31.716394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_image_df = pd.read_csv(dataset_path/'train_image_level.csv')\n\n\n# Thanks https://www.kaggle.com/tanlikesmath/siim-covid-19-detection-a-simple-eda\ntrain_image_df['class'] = train_image_df.label.apply(lambda x: x.split()[0])\n\n\ntrain_image_df['x_min'] = train_image_df.label.apply(lambda x: float(x.split()[2]))\ntrain_image_df['y_min'] = train_image_df.label.apply(lambda x: float(x.split()[3]))\ntrain_image_df['x_max'] = train_image_df.label.apply(lambda x: float(x.split()[4]))\ntrain_image_df['y_max'] = train_image_df.label.apply(lambda x: float(x.split()[5]))\n\n\n\ndef get_bbox_area(row):\n    return (row['x_max']-row['x_min'])*(row['y_max']-row['y_min'])\n\n\ntrain_image_df['bbox_area'] = train_image_df.apply(get_bbox_area, axis=1)\n\ntrain_image_df['image_path'] = train_image_df.apply(image_path, axis=1)\n\ntrain_image_df['bbox_area'].hist()\n","metadata":{"execution":{"iopub.status.busy":"2021-09-27T19:53:31.718011Z","iopub.execute_input":"2021-09-27T19:53:31.718251Z","iopub.status.idle":"2021-09-27T19:53:38.221737Z","shell.execute_reply.started":"2021-09-27T19:53:31.718201Z","shell.execute_reply":"2021-09-27T19:53:38.221081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_image_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-27T19:53:38.224701Z","iopub.execute_input":"2021-09-27T19:53:38.224907Z","iopub.status.idle":"2021-09-27T19:53:38.247268Z","shell.execute_reply.started":"2021-09-27T19:53:38.224883Z","shell.execute_reply":"2021-09-27T19:53:38.246493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imgs = []\nimage_paths = train_image_df['image_path'].values\nclass_ids = train_image_df['class']\n\n# map label_id to specify color\nlabel2color = {class_id:[random.randint(0,255) for i in range(3)] for class_id in class_ids}\nthickness = 3\nscale = 5\n\n\nfor i in range(8):\n    image_path = random.choice(image_paths)\n    print(image_path)\n    img = dicom2array(str(image_path))\n    img = cv2.resize(img, None, fx=1/scale, fy=1/scale)\n    img = np.stack([img, img, img], axis=-1)\n    \n    box = train_image_df.loc[train_image_df['image_path'] == image_path, ['x_min', 'y_min', 'x_max', 'y_max']].values[0]/scale\n    label = train_image_df.loc[train_image_df['image_path'] == image_path, ['class']].values[0][0]\n    \n    color = label2color[label]\n    img = cv2.rectangle(\n        img,\n        (int(box[0]), int(box[1])),\n        (int(box[2]), int(box[3])),\n        color, thickness)\n    \n    img = cv2.resize(img, (500,500))\n    imgs.append(img)\n    \nplot_imgs(imgs, cmap=None)","metadata":{"execution":{"iopub.status.busy":"2021-09-27T19:53:38.248512Z","iopub.execute_input":"2021-09-27T19:53:38.24883Z","iopub.status.idle":"2021-09-27T19:53:44.188909Z","shell.execute_reply.started":"2021-09-27T19:53:38.248795Z","shell.execute_reply":"2021-09-27T19:53:44.188113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"6334*0.8\nprint","metadata":{"execution":{"iopub.status.busy":"2021-09-27T19:53:44.190074Z","iopub.execute_input":"2021-09-27T19:53:44.190388Z","iopub.status.idle":"2021-09-27T19:53:44.19619Z","shell.execute_reply.started":"2021-09-27T19:53:44.190355Z","shell.execute_reply":"2021-09-27T19:53:44.195393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_ids = train_image_df['id'].unique()\nvalid_ids = image_ids[1401:2000]# Tran and Validation Split \ntrain_ids = image_ids[:1400]\n\n\nvalid_df = train_image_df[train_image_df['id'].isin(valid_ids)]\ntrain_df = train_image_df[train_image_df['id'].isin(train_ids)]\n\ntrain_df[\"class_id\"] = [1]*len(train_df)\nvalid_df[\"class_id\"] = [1]*len(valid_df)\nprint(len(train_image_df))\nprint(len(train_ids) + len(valid_ids))\nprint(len(train_ids))\nprint(len(valid_ids))\nprint(train_df.shape)\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-27T19:53:44.197682Z","iopub.execute_input":"2021-09-27T19:53:44.197959Z","iopub.status.idle":"2021-09-27T19:53:44.235304Z","shell.execute_reply.started":"2021-09-27T19:53:44.197922Z","shell.execute_reply":"2021-09-27T19:53:44.234505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class COVIDTrainDataLoader(Dataset): #Class to load Training Data\n    \n    def __init__(self, dataframe, transforms=None,stat = 'Train'):\n        super().__init__()\n        \n        self.image_ids = dataframe[\"id\"].unique()\n        \n        self.df = dataframe\n        self.transforms = transforms\n        self.stat = stat\n        \n    def __getitem__(self, index):\n        if self.stat == 'Train':\n            \n            image_id = self.image_ids[index]\n            \n            records = self.df[(self.df['id'] == image_id)]\n            records = records.reset_index(drop=True)\n            image = dicom2array(self.df[(self.df['id'] == image_id)]['image_path'].values[0])#dcmread\n\n            \n           \n            '''if \"PhotometricInterpretation\" in dicom:\n                if dicom.PhotometricInterpretation == \"MONOCHROME1\":\n                    image = np.amax(image) - image'''\n\n            intercept =  0.0\n            slope =1.0\n\n            if slope != 1:\n                image = slope * image.astype(np.float64)\n                image = image.astype(np.int16)\n\n            \n            image += np.int16(intercept)        \n\n            image = np.stack([image, image, image])\n            image = image.astype('float32')\n            image = image - image.min()\n            image = image / image.max()\n            image = image * 255.0\n            image = image.transpose(1,2,0)\n\n            if records.loc[0, \"class_id\"] == 0:\n                records = records.loc[[0], :]\n\n            boxes = records[['x_min', 'y_min', 'x_max', 'y_max']].values\n            area = (boxes[:, 3] - boxes[:, 1]) * (boxes[:, 2] - boxes[:, 0])\n            area = torch.as_tensor(area, dtype=torch.float32)\n            labels = torch.tensor(records[\"class_id\"].values, dtype=torch.int64)\n\n            # suppose all instances are not crowd\n            iscrowd = torch.zeros((records.shape[0],), dtype=torch.int64)\n\n            target = {}\n            target['boxes'] = boxes\n            target['labels'] = labels\n            target['id'] = torch.tensor([index])\n            target['area'] = area\n            target['iscrowd'] = iscrowd\n\n            if self.transforms:\n                sample = {\n                    'image': image,\n                    'bboxes': target['boxes'],\n                    'labels': labels\n                }\n                sample = self.transforms(**sample)\n                image = sample['image']\n\n                target['boxes'] = torch.tensor(sample['bboxes'])\n\n            if target[\"boxes\"].shape[0] == 0:\n                # Albumentation cuts the target (class 14, 1x1px in the corner)\n                target[\"boxes\"] = torch.from_numpy(np.array([[0.0, 0.0, 1.0, 1.0]]))\n                target[\"area\"] = torch.tensor([1.0], dtype=torch.float32)\n                target[\"labels\"] = torch.tensor([0], dtype=torch.int64)\n            \n            return image, target, image_ids\n        \n        else:\n                   \n            image_id = self.image_ids[index]\n            records = self.df[(self.df['id'] == image_id)]\n            records = records.reset_index(drop=True)\n\n            image = dicom2array(self.df[(self.df['id'] == image_id)]['image_path'].values[0])#dcmread\n\n            #image = ds.pixel_array\n\n            intercept =  0.0\n            slope = 1.0\n\n            if slope != 1:\n                image = slope * image.astype(np.float64)\n                image = image.astype(np.int16)\n\n            image += np.int16(intercept)        \n\n            image = np.stack([image, image, image])\n            image = image.astype('float32')\n            image = image - image.min()\n            image = image / image.max()\n            image = image * 255.0\n            image = image.transpose(1,2,0)\n\n            if self.transforms:\n                sample = {\n                    'image': image,\n                }\n                sample = self.transforms(**sample)\n                image = sample['image']\n\n            return image, image_id\n    \n    def __len__(self):\n        return self.image_ids.shape[0]","metadata":{"execution":{"iopub.status.busy":"2021-09-27T19:53:44.236696Z","iopub.execute_input":"2021-09-27T19:53:44.237128Z","iopub.status.idle":"2021-09-27T19:53:44.259188Z","shell.execute_reply.started":"2021-09-27T19:53:44.237092Z","shell.execute_reply":"2021-09-27T19:53:44.258419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Albumentations\ndef get_train_transform():\n    return A.Compose([\n        A.Resize(256,256),\n        A.ShiftScaleRotate(scale_limit=0.1, rotate_limit=45, p=0.25),\n        A.LongestMaxSize(max_size=800, p=1.0),\n        A.Normalize(mean=(0, 0, 0), std=(1, 1, 1), max_pixel_value=255.0, p=1.0),\n        ToTensorV2(p=1.0)\n    ], bbox_params={'format': 'pascal_voc', 'label_fields': ['labels']})\n\ndef get_valid_transform():\n    return A.Compose([\n        A.Resize(256,256),\n        A.Normalize(mean=(0, 0, 0), std=(1, 1, 1), max_pixel_value=255.0, p=1.0),\n        ToTensorV2(p=1.0)\n    ], bbox_params={'format': 'pascal_voc', 'label_fields': ['labels']})\n\ndef get_test_transform():\n    return A.Compose([\n        A.Resize(256,256),\n        A.Normalize(mean=(0, 0, 0), std=(1, 1, 1), max_pixel_value=255.0, p=1.0),\n        ToTensorV2(p=1.0)\n    ])","metadata":{"execution":{"iopub.status.busy":"2021-09-27T19:53:44.260122Z","iopub.execute_input":"2021-09-27T19:53:44.260342Z","iopub.status.idle":"2021-09-27T19:53:44.272643Z","shell.execute_reply.started":"2021-09-27T19:53:44.260312Z","shell.execute_reply":"2021-09-27T19:53:44.271895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Thanks https://www.kaggle.com/pestipeti/competition-metric-details-script\n\n\n@jit(nopython=True)\ndef calculate_iou(gt, pr, form='pascal_voc') -> float:\n    \"\"\"Calculates the Intersection over Union.\n\n    Args:\n        gt: (np.ndarray[Union[int, float]]) coordinates of the ground-truth box\n        pr: (np.ndarray[Union[int, float]]) coordinates of the prdected box\n        form: (str) gt/pred coordinates format\n            - pascal_voc: [xmin, ymin, xmax, ymax]\n            - coco: [xmin, ymin, w, h]\n    Returns:\n        (float) Intersection over union (0.0 <= iou <= 1.0)\n    \"\"\"\n    if form == 'coco':\n        gt = gt.copy()\n        pr = pr.copy()\n\n        gt[2] = gt[0] + gt[2]\n        gt[3] = gt[1] + gt[3]\n        pr[2] = pr[0] + pr[2]\n        pr[3] = pr[1] + pr[3]\n\n    # Calculate overlap area\n    dx = min(gt[2], pr[2]) - max(gt[0], pr[0]) + 1\n    \n    if dx < 0:\n        return 0.0\n    \n    dy = min(gt[3], pr[3]) - max(gt[1], pr[1]) + 1\n\n    if dy < 0:\n        return 0.0\n\n    overlap_area = dx * dy\n\n    # Calculate union area\n    union_area = (\n            (gt[2] - gt[0] + 1) * (gt[3] - gt[1] + 1) +\n            (pr[2] - pr[0] + 1) * (pr[3] - pr[1] + 1) -\n            overlap_area\n    )\n\n    return overlap_area / union_area\n\n@jit(nopython=True)\ndef find_best_match(gts, pred, pred_idx, threshold = 0.5, form = 'pascal_voc', ious=None) -> int:\n    \"\"\"Returns the index of the 'best match' between the\n    ground-truth boxes and the prediction. The 'best match'\n    is the highest IoU. (0.0 IoUs are ignored).\n\n    Args:\n        gts: (List[List[Union[int, float]]]) Coordinates of the available ground-truth boxes\n        pred: (List[Union[int, float]]) Coordinates of the predicted box\n        pred_idx: (int) Index of the current predicted box\n        threshold: (float) Threshold\n        form: (str) Format of the coordinates\n        ious: (np.ndarray) len(gts) x len(preds) matrix for storing calculated ious.\n\n    Return:\n        (int) Index of the best match GT box (-1 if no match above threshold)\n    \"\"\"\n    best_match_iou = -np.inf\n    best_match_idx = -1\n\n    for gt_idx in range(len(gts)):\n        \n        if gts[gt_idx][0] < 0:\n            # Already matched GT-box\n            continue\n        \n        iou = -1 if ious is None else ious[gt_idx][pred_idx]\n\n        if iou < 0:\n            iou = calculate_iou(gts[gt_idx], pred, form=form)\n            \n            if ious is not None:\n                ious[gt_idx][pred_idx] = iou\n\n        if iou < threshold:\n            continue\n\n        if iou > best_match_iou:\n            best_match_iou = iou\n            best_match_idx = gt_idx\n\n    return best_match_idx\n\n@jit(nopython=True)\ndef calculate_precision(gts, preds, threshold = 0.5, form = 'coco', ious=None) -> float:\n    \"\"\"Calculates precision for GT - prediction pairs at one threshold.\n\n    Args:\n        gts: (List[List[Union[int, float]]]) Coordinates of the available ground-truth boxes\n        preds: (List[List[Union[int, float]]]) Coordinates of the predicted boxes,\n               sorted by confidence value (descending)\n        threshold: (float) Threshold\n        form: (str) Format of the coordinates\n        ious: (np.ndarray) len(gts) x len(preds) matrix for storing calculated ious.\n\n    Return:\n        (float) Precision\n    \"\"\"\n    n = len(preds)\n    tp = 0\n    fp = 0\n    \n    # for pred_idx, pred in enumerate(preds_sorted):\n    for pred_idx in range(n):\n\n        best_match_gt_idx = find_best_match(gts, preds[pred_idx], pred_idx,\n                                            threshold=threshold, form=form, ious=ious)\n\n        if best_match_gt_idx >= 0:\n            # True positive: The predicted box matches a gt box with an IoU above the threshold.\n            tp += 1\n            # Remove the matched GT box\n            gts[best_match_gt_idx] = -1\n\n        else:\n            # No match\n            # False positive: indicates a predicted box had no associated gt box.\n            fp += 1\n\n    # False negative: indicates a gt box had no associated predicted box.\n    fn = (gts.sum(axis=1) > 0).sum()\n\n    return tp / (tp + fp + fn)\n\n\n@jit(nopython=True)\ndef calculate_image_precision(gts, preds, thresholds = (0.5, ), form = 'coco') -> float:\n    \"\"\"Calculates image precision.\n       The mean average precision at different intersection over union (IoU) thresholds.\n\n    Args:\n        gts: (List[List[Union[int, float]]]) Coordinates of the available ground-truth boxes\n        preds: (List[List[Union[int, float]]]) Coordinates of the predicted boxes,\n               sorted by confidence value (descending)\n        thresholds: (float) Different thresholds\n        form: (str) Format of the coordinates\n\n    Return:\n        (float) Precision\n    \"\"\"\n    n_threshold = len(thresholds)\n    image_precision = 0.0\n    \n    ious = np.ones((len(gts), len(preds))) * -1\n    # ious = None\n\n    for threshold in thresholds:\n        precision_at_threshold = calculate_precision(gts.copy(), preds, threshold=threshold,\n                                                     form=form, ious=ious)\n        image_precision += precision_at_threshold / n_threshold\n\n    return image_precision\n\niou_thresholds = [0.5]\n\nclass EvalMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.image_precision = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, gt_boxes, pred_boxes, n=1):       \n        \"\"\" pred_boxes : need to be sorted.\"\"\"\n        \n        self.image_precision = calculate_image_precision(pred_boxes,\n                                                         gt_boxes,\n                                                         thresholds=iou_thresholds,\n                                                         form='pascal_voc')\n        self.count += n\n        self.sum += self.image_precision * n\n        self.avg = self.sum / self.count","metadata":{"execution":{"iopub.status.busy":"2021-09-27T19:53:44.275254Z","iopub.execute_input":"2021-09-27T19:53:44.27552Z","iopub.status.idle":"2021-09-27T19:53:44.300602Z","shell.execute_reply.started":"2021-09-27T19:53:44.275489Z","shell.execute_reply":"2021-09-27T19:53:44.299881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Fitter:\n    def __init__(self, model, device, config):\n        self.config = config\n        self.epoch = 0\n\n        self.base_dir = f'/kaggle/working'\n        if not os.path.exists(self.base_dir):\n            os.makedirs(self.base_dir)\n        \n        self.log_path = f'{self.base_dir}/log.txt'\n        self.best_summary_loss = 10**5\n        self.best_score = 0\n\n        self.model = model\n        self.device = device\n\n        param_optimizer = list(self.model.named_parameters())\n        no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n        optimizer_grouped_parameters = [\n            {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},\n            {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}\n        ]\n        \n        # get the configured optimizer\n        if Adam:\n            self.optimizer = torch.optim.Adam(self.model.parameters(), **Adam_config)\n        else:\n            self.optimizer = torch.optim.SGD(self.model.parameters(), **SGD_config)\n\n        self.scheduler = config.SchedulerClass(self.optimizer, **config.scheduler_params)\n        self.log(f'Fitter prepared. Device is {self.device}')\n        self.log(f'Fold num is {DefaultConfig.fold_num}')\n\n    def fit(self, train_loader, validation_loader):\n        for e in range(self.config.n_epochs):\n            if self.config.verbose:\n                lr = self.optimizer.param_groups[0]['lr']\n                timestamp = datetime.utcnow().isoformat()\n                self.log(f'\\n{timestamp}\\nLR: {lr}')\n\n            t = time.time()\n            summary_loss = self.train_one_epoch(train_loader)\n            \n            if e == 0:\n                self.best_summary_loss = summary_loss.avg\n\n            self.log(f'[RESULT]: Train. Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, time: {(time.time() - t):.5f}')\n            self.save(f'{self.base_dir}/last-checkpoint.bin')\n\n            t = time.time()\n            _, eval_scores  = self.validation(validation_loader)\n\n            self.log(f'[RESULT]: Val. Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, image_precision: {eval_scores.avg:.5f}, time: {(time.time() - t):.5f}')\n            \n            #if summary_loss.avg < self.best_summary_loss:\n            if eval_scores.avg > self.best_score:\n                self.best_summary_loss = summary_loss.avg\n                self.best_score = eval_scores.avg\n                self.model.eval()\n                self.save(f'{self.base_dir}/best-checkpoint.bin')\n                for path in sorted(glob(f'{self.base_dir}/best-checkpoint.bin'))[:-3]:\n                    os.remove(path)\n\n            if self.config.validation_scheduler:\n                self.scheduler.step(metrics=eval_scores.avg)\n                #self.scheduler.step(metrics=summary_loss.avg)\n\n            self.epoch += 1\n\n    def validation(self, val_loader):\n        self.model.eval()\n        \n        # model.eval() mode --> it will return boxes and scores.\n        # in this part, just print train_loss\n        summary_loss = AverageMeter()\n        summary_loss.update(self.best_summary_loss, self.config.batch_size)\n        \n        eval_scores = EvalMeter()\n        validation_image_precisions = []\n        \n        t = time.time()\n        for step, (images, targets, image_ids) in enumerate(val_loader):\n            if self.config.verbose:\n                if step % self.config.verbose_step == 0:\n                    print(\n                        f'Val Step {step}/{len(val_loader)}, ' + \\\n                        f'summary_loss: {summary_loss.avg:.5f}, ' + \\\n                        f'val_precision: {eval_scores.avg:.5f}, ' + \\\n                        f'time: {(time.time() - t):.5f}', end='\\r'\n                    )\n            with torch.no_grad():\n                images = torch.stack(images)\n                batch_size = images.shape[0]\n                images = images.to(self.device).float()\n                labels = [target['labels'].float() for target in targets]\n\n                \"\"\"\n                In model.train() mode, model(images)  is returning losses.\n                We are using model.eval() mode --> it will return boxes and scores. \n                \"\"\"\n                outputs = self.model(images)               \n                \n                for i, image in enumerate(images):               \n                    gt_boxes = targets[i]['boxes'].data.cpu().numpy()\n                    boxes = outputs[i]['boxes'].data.cpu().numpy()\n                    scores = outputs[i]['scores'].detach().cpu().numpy()\n                    \n                    preds_sorted_idx = np.argsort(scores)[::-1]\n                    preds_sorted_boxes = boxes[preds_sorted_idx]\n\n                    eval_scores.update(pred_boxes=preds_sorted_boxes, gt_boxes=gt_boxes)\n\n        return summary_loss, eval_scores\n\n    def train_one_epoch(self, train_loader):\n        self.model.train()\n        summary_loss = AverageMeter()\n\n        t = time.time()\n        for step, (images, targets, image_ids) in enumerate(train_loader):\n            if self.config.verbose:\n                if step % self.config.verbose_step == 0:\n                    print(\n                        f'Train Step {step}/{len(train_loader)}, ' + \\\n                        f'summary_loss: {summary_loss.avg:.5f}, ' + \\\n                        f'time: {(time.time() - t):.5f}', end='\\r'\n                    )\n            \n            images = torch.stack(images)\n            images = images.to(self.device).float()\n            batch_size = images.shape[0]\n            boxes = [target['boxes'].to(self.device).float() for target in targets]\n            labels = [target['labels'].to(self.device).float() for target in targets]\n            targets = [{k: v.to(device) for k, v in t.items()} for t in targets] \n\n            self.optimizer.zero_grad()\n            \n            outputs = self.model(images, targets)\n            \n            loss = sum(loss for loss in outputs.values())\n            \n            loss.backward()\n\n            summary_loss.update(loss.detach().item(), batch_size)\n\n            self.optimizer.step()\n\n            if self.config.step_scheduler:\n                self.scheduler.step()\n\n        return summary_loss\n    \n    def save(self, path):\n        self.model.eval()\n        torch.save({\n            'model_state_dict': self.model.state_dict(), #'model_state_dict': self.model.model.state_dict(),\n            'optimizer_state_dict': self.optimizer.state_dict(),\n            'scheduler_state_dict': self.scheduler.state_dict(),\n            'best_summary_loss': self.best_summary_loss,\n            'epoch': self.epoch,\n        }, path)\n\n    def load(self, path):\n        checkpoint = torch.load(path)\n        self.model.load_state_dict(checkpoint['model_state_dict'])\n        self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])\n        self.scheduler.load_state_dict(checkpoint['scheduler_state_dict'])\n        self.best_summary_loss = checkpoint['best_summary_loss']\n        self.epoch = checkpoint['epoch'] + 1\n        \n    def log(self, message):\n        if self.config.verbose:\n            print(message)\n        with open(self.log_path, 'a+') as logger:\n            logger.write(f'{message}\\n')","metadata":{"execution":{"iopub.status.busy":"2021-09-27T19:53:44.301952Z","iopub.execute_input":"2021-09-27T19:53:44.302448Z","iopub.status.idle":"2021-09-27T19:53:44.337134Z","shell.execute_reply.started":"2021-09-27T19:53:44.302413Z","shell.execute_reply":"2021-09-27T19:53:44.336467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TrainGlobalConfig:\n    num_workers: int = 4\n    batch_size: int = 16\n    n_epochs: int = 10 #40\n    lr: float = 0.0002\n\n    img_size = DefaultConfig.img_size\n        \n    folder = '/kaggle/working/' #folder_name \n\n    # -------------------\n    verbose = True\n    verbose_step = 1\n    # -------------------\n\n    # --------------------\n    step_scheduler = False  # do scheduler.step after optimizer.step\n    validation_scheduler = False  # do scheduler.step after validation stage loss\n\n#     SchedulerClass = torch.optim.lr_scheduler.OneCycleLR\n#     scheduler_params = dict(\n#         max_lr=0.001,\n#         epochs=n_epochs,\n#         steps_per_epoch=int(len(train_dataset) / batch_size),\n#         pct_start=0.1,\n#         anneal_strategy='cos', \n#         final_div_factor=10**5\n#     )\n    \n    SchedulerClass = torch.optim.lr_scheduler.ReduceLROnPlateau\n    scheduler_params = dict(\n        mode='min',\n        factor=0.5,\n        patience=1,\n        verbose=False, \n        threshold=0.0001,\n        threshold_mode='abs',\n        cooldown=0, \n        min_lr=1e-8,\n        eps=1e-08\n    )\n    # --------------------","metadata":{"execution":{"iopub.status.busy":"2021-09-27T20:18:26.112107Z","iopub.execute_input":"2021-09-27T20:18:26.112853Z","iopub.status.idle":"2021-09-27T20:18:26.119538Z","shell.execute_reply.started":"2021-09-27T20:18:26.112814Z","shell.execute_reply":"2021-09-27T20:18:26.118789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class FasterRCNNDetector(torch.nn.Module):\n    def __init__(self, pretrained=False, **kwargs):\n        super(FasterRCNNDetector, self).__init__()\n        # load pre-trained model incl. head\n        self.model = torchvision.models.detection.fasterrcnn_resnet50_fpn(pretrained=pretrained, pretrained_backbone=pretrained)\n        \n        # get number of input features for the classifier custom head\n        in_features = self.model.roi_heads.box_predictor.cls_score.in_features\n        \n        # replace the pre-trained head with a new one\n        self.model.roi_heads.box_predictor = FastRCNNPredictor(in_features, DefaultConfig.num_classes)\n        \n    def forward(self, images, targets=None):\n        return self.model(images, targets)","metadata":{"execution":{"iopub.status.busy":"2021-09-27T19:53:44.350861Z","iopub.execute_input":"2021-09-27T19:53:44.351129Z","iopub.status.idle":"2021-09-27T19:53:44.361349Z","shell.execute_reply.started":"2021-09-27T19:53:44.351096Z","shell.execute_reply":"2021-09-27T19:53:44.360624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\n\ndef get_model(checkpoint_path=None, pretrained=False):\n    model = FasterRCNNDetector(pretrained=pretrained)\n    \n    # Load the trained weights\n    if checkpoint_path is not None:\n        checkpoint = torch.load(checkpoint_path)\n        model.load_state_dict(checkpoint['model_state_dict'])\n\n        del checkpoint\n        gc.collect()\n        \n    return model.cuda()\n\nnet = get_model(pretrained=True)","metadata":{"execution":{"iopub.status.busy":"2021-09-27T19:53:44.362433Z","iopub.execute_input":"2021-09-27T19:53:44.362833Z","iopub.status.idle":"2021-09-27T19:53:54.136796Z","shell.execute_reply.started":"2021-09-27T19:53:44.362798Z","shell.execute_reply":"2021-09-27T19:53:54.135992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count","metadata":{"execution":{"iopub.status.busy":"2021-09-27T19:53:54.138197Z","iopub.execute_input":"2021-09-27T19:53:54.138484Z","iopub.status.idle":"2021-09-27T19:53:54.144903Z","shell.execute_reply.started":"2021-09-27T19:53:54.13845Z","shell.execute_reply":"2021-09-27T19:53:54.144103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n\ndef collate_fn(batch):\n    return tuple(zip(*batch))\n\ntrain_dataset = COVIDTrainDataLoader(train_df, get_train_transform())\nvalid_dataset = COVIDTrainDataLoader(valid_df, get_valid_transform())\n\n\n# split the dataset in train and test set\nindices = torch.randperm(len(train_dataset)).tolist()\n# Create train and validate data loader\ntrain_data_loader = DataLoader(\n    train_dataset,\n    batch_size=16,\n    shuffle=True,\n    num_workers=4,\n    collate_fn=collate_fn\n)\n\nvalid_data_loader = DataLoader(\n    valid_dataset,\n    batch_size=16,\n    shuffle=False,\n    num_workers=4,\n    collate_fn=collate_fn\n)","metadata":{"execution":{"iopub.status.busy":"2021-09-27T20:18:30.982552Z","iopub.execute_input":"2021-09-27T20:18:30.982834Z","iopub.status.idle":"2021-09-27T20:18:30.997086Z","shell.execute_reply.started":"2021-09-27T20:18:30.982805Z","shell.execute_reply":"2021-09-27T20:18:30.996247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nimage, target, image_id = train_dataset[2]\nboxes = target['boxes'].cpu().numpy().astype(np.int32)\n\nnumpy_image = image.permute(1,2,0).cpu().numpy()\n\nfig, ax = plt.subplots(1, 1, figsize=(16, 8))\n\nfor box in boxes:\n    cv2.rectangle(numpy_image, (box[0], box[1]), (box[2],  box[3]), (0, 255, 0), 2)\n    \nax.set_axis_off()\nax.imshow(numpy_image);","metadata":{"execution":{"iopub.status.busy":"2021-09-27T19:53:54.16158Z","iopub.execute_input":"2021-09-27T19:53:54.161913Z","iopub.status.idle":"2021-09-27T19:53:55.333427Z","shell.execute_reply.started":"2021-09-27T19:53:54.161878Z","shell.execute_reply":"2021-09-27T19:53:55.331118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target","metadata":{"execution":{"iopub.status.busy":"2021-09-27T19:53:55.33461Z","iopub.execute_input":"2021-09-27T19:53:55.334919Z","iopub.status.idle":"2021-09-27T19:53:55.369619Z","shell.execute_reply.started":"2021-09-27T19:53:55.334885Z","shell.execute_reply":"2021-09-27T19:53:55.368824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_rows=4\nn_cols=4\n\nimages, targets, image_ids = next(iter(train_data_loader))\n\nimages = list(image.to(device) for image in images)\ntargets = [{k: v.to(device) for k, v in t.items()} for t in targets]\n\n# plot some augmentations!\nfig, ax = plt.subplots(figsize=(20, 20),  nrows=n_rows, ncols=n_cols)\nfor i in range (n_rows*n_cols):    \n    boxes = targets[i]['boxes'].cpu().numpy().astype(np.int32)\n    sample = images[i].permute(1,2,0).cpu().numpy()\n    for box in boxes:\n        cv2.rectangle(sample,\n                      (box[0], box[1]),\n                      (box[2], box[3]),\n                      (255, 0, 0), 3)\n    \n    ax[i // n_rows][i % n_cols].imshow(sample)","metadata":{"execution":{"iopub.status.busy":"2021-09-27T19:53:55.371074Z","iopub.execute_input":"2021-09-27T19:53:55.371344Z","iopub.status.idle":"2021-09-27T19:54:31.00202Z","shell.execute_reply.started":"2021-09-27T19:53:55.37131Z","shell.execute_reply":"2021-09-27T19:54:31.001216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef run_training():\n    try: \n        net.to(device)\n    \n        train_dataset = COVIDTrainDataLoader(train_df, get_train_transform())\n        train_data_loader = DataLoader(\n            train_dataset,\n            batch_size=16,\n            shuffle=True,\n            num_workers=4,\n            collate_fn=collate_fn\n        )\n    \n        valid_dataset = COVIDTrainDataLoader(valid_df, get_valid_transform())\n        validation_data_loader = DataLoader(\n            valid_dataset,\n            batch_size=16,\n            shuffle=False,\n            num_workers=4,\n            collate_fn=collate_fn\n        )\n\n        fitter = Fitter(model=net, device=device, config=TrainGlobalConfig)\n        fitter.fit(train_data_loader, validation_data_loader)\n    except ValueError: \n        pass \n    ","metadata":{"execution":{"iopub.status.busy":"2021-09-27T20:18:38.992533Z","iopub.execute_input":"2021-09-27T20:18:38.993451Z","iopub.status.idle":"2021-09-27T20:18:39.001634Z","shell.execute_reply.started":"2021-09-27T20:18:38.993412Z","shell.execute_reply":"2021-09-27T20:18:38.999347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"run_training()","metadata":{"execution":{"iopub.status.busy":"2021-09-27T20:18:41.161186Z","iopub.execute_input":"2021-09-27T20:18:41.161465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file = open('/kaggle/working/log.txt', 'r')\nfor line in file.readlines():\n    print(line[:-1])\nfile.close()","metadata":{"execution":{"iopub.status.busy":"2021-09-27T20:15:11.618605Z","iopub.execute_input":"2021-09-27T20:15:11.618841Z","iopub.status.idle":"2021-09-27T20:15:11.627935Z","shell.execute_reply.started":"2021-09-27T20:15:11.618811Z","shell.execute_reply":"2021-09-27T20:15:11.627159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#os.makedirs('/kaggle/working/tmp/', exist_ok=True)\n#%ls","metadata":{"execution":{"iopub.status.busy":"2021-09-27T20:15:11.630804Z","iopub.execute_input":"2021-09-27T20:15:11.631049Z","iopub.status.idle":"2021-09-27T20:15:11.635829Z","shell.execute_reply.started":"2021-09-27T20:15:11.631015Z","shell.execute_reply":"2021-09-27T20:15:11.634799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_dataset = COVIDTrainDataLoader(valid_df, get_valid_transform())\nvalidation_data_loader = DataLoader(\n        valid_dataset,\n        batch_size=16,\n        shuffle=False,\n        num_workers=4,\n        collate_fn=collate_fn\n    )","metadata":{"execution":{"iopub.status.busy":"2021-09-27T20:15:11.63736Z","iopub.execute_input":"2021-09-27T20:15:11.63763Z","iopub.status.idle":"2021-09-27T20:15:11.645509Z","shell.execute_reply.started":"2021-09-27T20:15:11.637597Z","shell.execute_reply":"2021-09-27T20:15:11.64472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images, targets, image_id = next(iter(validation_data_loader))\n\nimages = list(img.to(device) for img in images)\ntargets = [{k: v.to(device) for k, v in t.items()} for t in targets]\n\nboxes = targets[1]['boxes'].cpu().numpy().astype(np.int32)\nsample = images[1].permute(1,2,0).cpu().numpy()","metadata":{"execution":{"iopub.status.busy":"2021-09-27T20:15:11.646547Z","iopub.execute_input":"2021-09-27T20:15:11.646925Z","iopub.status.idle":"2021-09-27T20:15:43.216515Z","shell.execute_reply.started":"2021-09-27T20:15:11.64689Z","shell.execute_reply":"2021-09-27T20:15:43.215437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd /kaggle/working\n%ls \nnet = get_model('/kaggle/working/best-checkpoint.bin')\n","metadata":{"execution":{"iopub.status.busy":"2021-09-27T20:17:04.630609Z","iopub.execute_input":"2021-09-27T20:17:04.630876Z","iopub.status.idle":"2021-09-27T20:17:05.932223Z","shell.execute_reply.started":"2021-09-27T20:17:04.630848Z","shell.execute_reply":"2021-09-27T20:17:05.931199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"net.eval()\ncpu_device = torch.device(\"cpu\")\n\noutputs = net(images)\noutputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs]","metadata":{"execution":{"iopub.status.busy":"2021-09-27T20:15:44.273267Z","iopub.status.idle":"2021-09-27T20:15:44.273575Z","shell.execute_reply.started":"2021-09-27T20:15:44.273415Z","shell.execute_reply":"2021-09-27T20:15:44.273434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize=(16, 8))\n\nfor box in boxes:\n    cv2.rectangle(sample,\n                  (box[0], box[1]),\n                  (box[2], box[3]),\n                  (220, 0, 0), 2)\n    \nax.set_axis_off()\nax.imshow(sample)","metadata":{"execution":{"iopub.status.busy":"2021-09-27T20:15:44.275353Z","iopub.status.idle":"2021-09-27T20:15:44.275931Z","shell.execute_reply.started":"2021-09-27T20:15:44.27554Z","shell.execute_reply":"2021-09-27T20:15:44.275561Z"},"trusted":true},"execution_count":null,"outputs":[]}]}