{"cells":[{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"markdown","source":"I want to analyze model results on part of the training set (usually this will be validation set)\n\nLet's verify and inspect its performance in ways that are criticall in terms of [evaluation metric](https://www.kaggle.com/c/rsna-pneumonia-detection-challenge#evaluation):\n\n# 1. Classification errors\n Classification errors contribute to the error the most.\n False positive and false negative errors gives one an additional 1 in the metric denominator.\n True negatives are just ignored while true positives can have a precision from 0 to 1 depending on the IoU error.\n In this part I will assume that I only have a classification model and look at the model results as if the bounding boxes for correctly classified images were perfect.\n \n# 2. bounding box errors\nHere I want to calculate what I believe is the real competition metric and visualize it for various confidence thresholds.\nFurthermore I want to visualize those that are the most incorrect, hoping it will give me some insight on model weak points.\n\n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import confusion_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1ffd9c46d0e82678affddce1e5301b12d5d492d2"},"cell_type":"code","source":"INPUT_DIR = '/kaggle/input'\nDATA_DIR = os.path.join(INPUT_DIR,\"rsna-pneumonia-detection-challenge\")\nMODEL_RESULTS = os.path.join(INPUT_DIR,\"predict-on-validation\",\"validation_predictions.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b9ddccee927760f1d910b09a571f75ead0ef2e60"},"cell_type":"code","source":"anns = pd.read_csv(os.path.join(DATA_DIR,'stage_1_train_labels.csv'))\nanns.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f9636828c43c35fc5bf6bba8939ca55391687a32"},"cell_type":"code","source":"anns.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c5c57d33785245a97b61929e407c5c256a92245a","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"# I need to group by patient id so that it is easy to join with predictions:\n\ndef gather_gt(patient_df):\n    if np.sum(patient_df['x'].isna()) > 0:\n        return []\n    else:\n        gts = []\n        for index, row in patient_df.iterrows():\n            gts.append({\n                'x':row['x'],\n                'y':row['y'],\n                'width':row['width'],\n                'height':row['height']\n            })\n        return gts\n\ngt_patient = anns.groupby('patientId').apply(gather_gt)\ngt_patient = gt_patient.to_frame(\"gt\").reset_index()\ngt_patient['Target'] = gt_patient['gt'].apply(lambda x: 1 * (x != []))\nlen(gt_patient)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6de02e425019f566b507d79b4a527a7f709d281f","_kg_hide-input":true},"cell_type":"code","source":"results = pd.read_csv(MODEL_RESULTS, header=None, names=['patientId','prediction'])\nresults = results[~results['prediction'].isna()]\n# we will join ground truth only for patients in the results df\ndf = results.merge(gt_patient, on='patientId', how='left')\nprint(df.shape)\ndf.head(n=10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"04692dd7bac58f5dcc44e869620cad393adacee7","_kg_hide-input":true},"cell_type":"code","source":"def filter_predictions(min_confidence):\n    def mapper(prediction):\n        exploded = prediction.strip().split(\" \")\n        predictions = [exploded[x:x+5] for x in range(0, len(exploded),5)]\n        filtered_predictions = [p for p in predictions if float(p[0]) >= min_confidence]\n        return \" \".join([\" \".join(p) for p in filtered_predictions])\n    return mapper\ndef minimal_confidence_predictions(bbox_predictions, min_confidence):\n    return bbox_predictions.apply(filter_predictions(min_confidence))\n\ndef rsna_precision(tp, fp, fn):\n    return tp/(tp+fp+fn)\n\ndef rsna_precision_for(min_conf):\n    # In my results there is no dedicated column for classification prediction -\n    # I will be inferring it from bounding box prediction for various confidence thresholds\n    y_pred = 1 * (minimal_confidence_predictions(df['prediction'], min_conf) != '')\n    tn, fp, fn, tp = confusion_matrix(df['Target'], y_pred).ravel()\n    return rs\n\ndef metrics_for_confidences():\n    for min_conf in [x/100.0 for x in range(70,100)]:\n        y_pred = 1 * (minimal_confidence_predictions(df['prediction'], min_conf) != '')\n        tn, fp, fn, tp = confusion_matrix(df['Target'], y_pred).ravel()\n        prec = rsna_precision(tp,fp,fn)\n        cnt = tn + fp + fn + tp\n        yield {\"confidence\":min_conf,\"tn\":tn/cnt,\"fp\":fp/cnt,\"fn\":fn/cnt,\"tp\":tp/cnt,\"prec\":prec}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4e7484c3393a1d91d6319020c608bac61183ab9b"},"cell_type":"markdown","source":"# Let's plot Precision and other confusion matrix errors as a function of confidence threshold"},{"metadata":{"trusted":true,"_uuid":"c072e440e673a4ae25546ec22a84f7066a6188e5"},"cell_type":"code","source":"metrics = pd.DataFrame(list(metrics_for_confidences()))\nmelted = metrics.melt(id_vars='confidence',value_vars=['tn','fp','fn','tp','prec'])\n\nplt.figure(figsize=(20,10))\n\nsns.lineplot(x='confidence', y='value', hue='variable',data=melted)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e1af9c2acc45a05af491ca9285cfd8a9911557d8"},"cell_type":"code","source":"metrics.iloc[np.argmax(metrics['prec'])]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"857be7f749e008fdab012e63bf8f09a720a4404d"},"cell_type":"markdown","source":"### in my case the best performance in terns of precision is achieved with confidence >= 0.96.\n\n# Now let's calculate the real metric including bounding box IoU for every confidence threshold"},{"metadata":{"trusted":true,"_uuid":"36a683677aafed60dda369a3dc36e851f0b72683"},"cell_type":"code","source":"def to_structure(prediction):\n    exploded = prediction.strip().split(\" \")\n    predictions = [exploded[x:x+5] for x in range(0, len(exploded),5)]    \n    return [{'x':float(p[1]), 'y':float(p[2]), 'width': float(p[3]), 'height':float(p[4]), 'confidence':float(p[0])} for p in predictions]\n        \ndf['all_predictions'] = df['prediction'].apply(to_structure)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e894e989941b953aa9dd44f86fb8918f386ee5b4","_kg_hide-input":true},"cell_type":"code","source":"# source: https://www.kaggle.com/chenyc15/mean-average-precision-metric\n\n# extended version of metrics per patient giving more information:\n\niouthresholds = np.linspace(0.4,0.75,num=8)\n\n# helper function to calculate IoU\ndef iou(box1, box2):\n    x11, y11, w1, h1 = box1\n    x21, y21, w2, h2 = box2\n    assert w1 * h1 > 0\n    assert w2 * h2 > 0\n    x12, y12 = x11 + w1, y11 + h1\n    x22, y22 = x21 + w2, y21 + h2\n\n    area1, area2 = w1 * h1, w2 * h2\n    xi1, yi1, xi2, yi2 = max([x11, x21]), max([y11, y21]), min([x12, x22]), min([y12, y22])\n    \n    if xi2 <= xi1 or yi2 <= yi1:\n        return 0\n    else:\n        intersect = (xi2-xi1) * (yi2-yi1)\n        union = area1 + area2 - intersect\n        return intersect / union\n\ndef map_iou(boxes_true, boxes_pred, scores, thresholds = iouthresholds):\n    \"\"\"\n    Mean average precision at differnet intersection over union (IoU) threshold\n    \n    input:\n        boxes_true: Mx4 numpy array of ground true bounding boxes of one image. \n                    bbox format: (x1, y1, w, h)\n        boxes_pred: Nx4 numpy array of predicted bounding boxes of one image. \n                    bbox format: (x1, y1, w, h)\n        scores:     length N numpy array of scores associated with predicted bboxes\n        thresholds: IoU shresholds to evaluate mean average precision on\n    output: \n        map: mean average precision of the image\n    \"\"\"\n    \n    # According to the introduction, images with no ground truth bboxes will not be \n    # included in the map score unless there is a false positive detection (?)\n    result= {\n        'tp':0,\n        'tn':0,\n        'fp':0,\n        'fn':0,\n        'skipped':0,\n        'predicted_cnt':len(boxes_pred),\n        'gt_cnt':len(boxes_true),\n        'ious':{}\n    }    \n    # return None if both are empty, don't count the image in final evaluation (?)\n    if len(boxes_true) == 0 and len(boxes_pred) == 0:\n        result['skipped'] = 1\n        result['tn'] = 1\n        return result\n    if len(boxes_true) > 0 and len(boxes_pred) == 0:\n        result['prec'] = 0\n        result['fn'] = len(iouthresholds) * (len(boxes_true) - len(boxes_pred))\n        return result\n    if len(boxes_true) == 0 and len(boxes_pred) > 0:\n        result['prec'] = 0\n        result['fp'] = len(iouthresholds) * (len(boxes_pred) - len(boxes_true))\n        return result\n    \n    assert boxes_true.shape[1] == 4 or boxes_pred.shape[1] == 4, \"boxes should be 2D arrays with shape[1]=4\"\n    \n    # I am not doing any sorting just assume that predictions are sorted according to confidence, since I cannot find a way to so\n    if len(boxes_pred):\n        assert len(scores) == len(boxes_pred), \"boxes_pred and scores should be same length\"\n        # sort boxes_pred by scores in decreasing order\n        boxes_pred = boxes_pred[np.argsort(-1 * scores, kind='mergesort'), :]\n    \n    map_total = 0\n    \n    # loop over thresholds\n    total_tp = 0\n    total_fp = 0\n    total_fn = 0\n    for t in thresholds:\n        matched_bt = set()\n        tp, fn = 0, 0\n        for i, bt in enumerate(boxes_true):\n            matched = False\n            for j, bp in enumerate(boxes_pred):\n                miou = iou(bt, bp)\n                result['ious'][(i,j)] = miou\n                if miou >= t and not matched and j not in matched_bt:\n                    matched = True\n                    tp += 1 # bt is matched for the first time, count as TP\n                    matched_bt.add(j)                    \n            if not matched:\n                fn += 1 # bt has no match, count as FN\n                \n        fp = len(boxes_pred) - len(matched_bt) # FP is the bp that not matched to any bt\n        m = tp / (tp + fn + fp)\n        map_total += m\n        total_tp += tp\n        total_fp += fp\n        total_fn += fn\n    \n    result['prec'] = map_total / len(thresholds)\n    result['tp'] = total_tp\n    result['fn'] = total_fn\n    result['fp'] = total_fp\n    \n    return result\n\ndef bbox_to_array(bbox_dict):\n    return [\n                    bbox_dict['x'],\n                    bbox_dict['y'],\n                    bbox_dict['width'],\n                    bbox_dict['height'],\n                ]\n\ndef patient_metrics_off(row):\n    gtboxes = np.array([bbox_to_array(b) for b in row['gt']])\n    predboxes = np.array([bbox_to_array(b) for b in row['predictions']])\n    confidences = np.array([b['confidence'] for b in row['predictions']])\n    return map_iou(gtboxes,predboxes, confidences)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0fc9d2d0e021794f5c42eedf687c9326023edd82","_kg_hide-input":true},"cell_type":"code","source":"def filter_by_min_confidence(min_conf):\n    def f(predictions):\n        return [p for p in predictions if p['confidence'] > min_conf]\n    return f\n    \ndef metrics_for_confidences_bbox(df):\n    for min_conf in [x/100.0 for x in range(70,100)]:\n        df['predictions'] = df['all_predictions'].apply(filter_by_min_confidence(min_conf))\n        patient_metrics_df = df.apply(patient_metrics_off, axis=1,  result_type='expand')\n        tp = np.sum(patient_metrics_df.tp)\n        tn = np.sum(patient_metrics_df.tn)\n        fp = np.sum(patient_metrics_df.fp)\n        fn = np.sum(patient_metrics_df.fn)\n        not_skipped = patient_metrics_df[patient_metrics_df.skipped != 1]\n        prec = np.mean(not_skipped.prec)        \n        cnt = tn + fp + fn + tp\n        yield {\"confidence\":min_conf,\"tn\":tn/cnt,\"fp\":fp/cnt,\"fn\":fn/cnt,\"tp\":tp/cnt,\"prec\":prec}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fe11d1e462d36e7803b50113a2fbc44667096224"},"cell_type":"code","source":"plt.figure(figsize=(20,10))\n\nmetrics_bbox = pd.DataFrame(list(metrics_for_confidences_bbox(df)))\nmelted_bbox = metrics_bbox.melt(id_vars='confidence',value_vars=['tn','fp','fn','tp','prec'])\n\nsns.lineplot(x='confidence', y='value', hue='variable',data=melted_bbox)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8830c093428b693dd6ddf88d219085fcd24f3ef0"},"cell_type":"code","source":"metrics_bbox.iloc[np.argmax(metrics_bbox['prec'])]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0c9896e33d53f0bd41c059954c356928913ea710","_kg_hide-input":true},"cell_type":"code","source":"df['predictions'] = df['all_predictions']\npatient_metrics_df = df.join(df.apply(patient_metrics_off, axis=1,  result_type='expand'))","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"scrolled":false,"_uuid":"2ef20ff223d02b35a8ca3047885881d3548f3b4c"},"cell_type":"code","source":"# source: https://www.kaggle.com/meaninglesslives/dataset-visualization-using-opencv\ndef overlay_box(im, box, rgb, stroke=1):\n    \"\"\"\n    Method to overlay single box on image\n\n    \"\"\"\n    # --- Convert coordinates to integers\n    box = [int(b) for b in box]\n    \n    # --- Extract coordinates\n    y1, x1, height, width = box\n    y2 = y1 + height\n    x2 = x1 + width\n\n    im[y1:y1 + stroke, x1:x2] = rgb\n    im[y2:y2 + stroke, x1:x2] = rgb\n    im[y1:y2, x1:x1 + stroke] = rgb\n    im[y1:y2, x2:x2 + stroke] = rgb\n\n    return im\n\n\nimport cv2\nimport pydicom\nfrom IPython.display import display, Image\n\n\ndef cvshow(image, format='.png', rate=255 ):\n    decoded_bytes = cv2.imencode(format, image*rate)[1].tobytes()\n    display(Image(data=decoded_bytes))\n    return\n\ndef visualize(df, patientId):\n    dcm_file = '../input/rsna-pneumonia-detection-challenge/stage_1_train_images/%s.dcm' % patientId\n    dcm_data = pydicom.read_file(dcm_file)\n    img = dcm_data.pixel_array\n    img = np.stack([img] * 3, axis=2)\n    \n    def show_boxes(img,boxes,rgb):\n        for box in boxes:\n            #y,x,h,w\n            box = [box['y'],box['x'],box['height'],box['width']]                \n            overlay_box(img, box, rgb)\n            \n    gt_boxes = df[df['patientId'] == patientId]['gt'].values[0]\n    gt_col = np.array([1,250,1])  \n    pred_boxes = df[df['patientId'] == patientId]['predictions'].values[0]    \n    pred_col = np.array([1,1,250])  \n    \n    show_boxes(img,gt_boxes,gt_col)\n    show_boxes(img,pred_boxes,pred_col)\n    \n    ious = df[df['patientId'] == patientId]['ious'].values[0]\n    offsets = dict([(i,2) for i in range(len(pred_boxes))])\n    for box in pred_boxes:\n        confidence = box['confidence']        \n        img = cv2.putText(img=np.copy(img), text=\"conf = {:.2f}\".format(confidence), org=(int(box['x'])+5,int(box['y']) + 15),\n                          fontFace=1, fontScale=1, color=(0,0,255), thickness=1)\n    for k,iou in ious.items():     \n        if iou > 0.0:            \n            box = pred_boxes[k[1]]            \n            img = cv2.putText(img=np.copy(img), text=\"iou = {:.2f}\".format(iou), org=(int(box['x'])+5,int(box['y']) + 15 * offsets[k[1]]),\n                          fontFace=1, fontScale=1, color=(0,0,255), thickness=1)\n            offsets[k[1]]+=1\n    prec = df[df['patientId'] == patientId]['prec'].values[0]\n    img = cv2.putText(img=np.copy(img), text=\"precision = {:.2f}\".format(prec), org=(10,300),\n                          fontFace=1, fontScale=1, color=(0,0,255), thickness=1)\n    cvshow(img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e36d0895d0c58f93bacb8ae69edb0ff818314887"},"cell_type":"markdown","source":"# Let's investigate the biggest errors:\n\n# 1. purple boxes are ground true.\n# 2. light blue are predictions with iou presented\n\n### False Negatives - not detecting bounding box at all"},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"b5854adac051c5cff5eaf4122fa309705c58fd58"},"cell_type":"code","source":"patient_metrics_df_3 = patient_metrics_df[patient_metrics_df['gt_cnt'] == 3]\npatient_metrics_df_2 = patient_metrics_df[patient_metrics_df['gt_cnt'] == 2]\npatient_metrics_df_1 = patient_metrics_df[patient_metrics_df['gt_cnt'] == 1]\npatient_metrics_df_0 = patient_metrics_df[patient_metrics_df['gt_cnt'] == 0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"1634c382d3316e77b5703c6370bf6d3616f171ac"},"cell_type":"code","source":"fns = np.hstack([patient_metrics_df_3.sort_values('fn', ascending=False).patientId.values[:2],\npatient_metrics_df_2.sort_values('fn', ascending=False).patientId.values[:2],\npatient_metrics_df_1.sort_values('fn', ascending=False).patientId.values[:2]])\n\nfor pid in fns:\n    visualize(patient_metrics_df, pid)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d40d7aa0532bb3958637946c9762693c7290d6ac"},"cell_type":"markdown","source":"# 2. False Positives - detecting bounding boxes for no real GT:"},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"7462289a3000e7baf4118c4539837d959407c7ab"},"cell_type":"code","source":"fps = np.hstack([patient_metrics_df_3.sort_values('fp', ascending=False).patientId.values[:2],\npatient_metrics_df_2.sort_values('fp', ascending=False).patientId.values[:2],\npatient_metrics_df_1.sort_values('fp', ascending=False).patientId.values[:2]])\n\nfor pid in fps:\n    visualize(patient_metrics_df, pid)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5088ea6acdb7a7dc053adc513e9be60ed78c33be"},"cell_type":"markdown","source":"# Examples with largest precision:\n"},{"metadata":{"trusted":true,"_uuid":"2e525ea2a4ea6c154582e6ecbc2c629497577e48","scrolled":false},"cell_type":"code","source":"best_prec = np.hstack([patient_metrics_df_3.sort_values('prec', ascending=False).patientId.values[:2],\npatient_metrics_df_2.sort_values('prec', ascending=False).patientId.values[:2],\npatient_metrics_df_1.sort_values('prec', ascending=False).patientId.values[:2]])\n\nfor pid in best_prec:\n    visualize(patient_metrics_df, pid)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7cf9b3eee9c72068ddaa872de0934714d5b8cf33"},"cell_type":"markdown","source":"## explore the worst examples at high confidence"},{"metadata":{"trusted":true,"_uuid":"ed0e3b82f39a386e92b23ad45554c43d9a232239"},"cell_type":"code","source":"high_conf_df = df.copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9dee738c6a8a4490da7038c6a8dd7875bb510455"},"cell_type":"code","source":"high_conf_df['predictions'] = df['all_predictions'].apply(filter_by_min_confidence(0.98))\nhigh_conf = high_conf_df.join(high_conf_df.apply(patient_metrics_off, axis=1, result_type='expand'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"9dfb6b766b32c13992bf0be82a5679d2bb4f5867"},"cell_type":"code","source":"high_conf_low_prec = high_conf.sort_values('fp', ascending=False).patientId.values[:6]\n# high_conf.sort_values('fp', ascending=False).head()\nfor pid in high_conf_low_prec:\n    visualize(high_conf, pid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"55120c5d6d8bbc804311d2873d80781051e0c13d"},"cell_type":"markdown","source":"## explore the smallest/largest gt/predicted boxes "},{"metadata":{"trusted":true,"_uuid":"e342705ba0973bcd0cde54915409d59e3ea5be49","_kg_hide-input":true},"cell_type":"code","source":"def area(bbox):\n    return bbox['width'] * bbox['height']    \n    \ndef min_bbox_size(bboxes):\n    if len(bboxes)==0:\n        return 0\n    else:\n        return np.min([area(bbox) for bbox in bboxes])\n\ndef max_bbox_size(bboxes):\n    if len(bboxes)==0:\n        return 0\n    else:\n        return np.max([area(bbox) for bbox in bboxes])\n\npatient_metrics_df['min_pred_size'] = patient_metrics_df['all_predictions'].apply(min_bbox_size)\npatient_metrics_df['max_pred_size'] = patient_metrics_df['all_predictions'].apply(max_bbox_size)\npatient_metrics_df['min_gt_size'] = patient_metrics_df['gt'].apply(min_bbox_size)\npatient_metrics_df['max_gt_size'] = patient_metrics_df['gt'].apply(max_bbox_size)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"adfbe49ece51b65ac7cdaff58ae03ac79ade0642"},"cell_type":"markdown","source":"## smallest predicted"},{"metadata":{"trusted":true,"_uuid":"fdd782ba8784ff5462387c799ddcab9eadd1f768","scrolled":false},"cell_type":"code","source":"smallest_pred = patient_metrics_df[patient_metrics_df['min_pred_size'] > 0].sort_values('min_pred_size', ascending=True).patientId.values[:4]\n\nfor pid in smallest_pred:\n    visualize(patient_metrics_df, pid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9f3097f5e7ae9c33cad7bc7c42e0fa48039a6cda"},"cell_type":"markdown","source":"## smallest gt"},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"03c1eb95dda1389ba3b84000bb58973ab543398c"},"cell_type":"code","source":"smallest_gt = patient_metrics_df[patient_metrics_df['min_gt_size'] > 0].sort_values('min_gt_size', ascending=True).patientId.values[:4]\n\nfor pid in smallest_gt:\n    visualize(patient_metrics_df, pid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6a15a20d97b9b2f98157406ea6af0f06cdf50177"},"cell_type":"markdown","source":"## largest predicted"},{"metadata":{"trusted":true,"_uuid":"65b3e844ea2ef3daf6c68057020ecd83bf5c6281","scrolled":false},"cell_type":"code","source":"largest_pred = patient_metrics_df.sort_values('max_pred_size', ascending=False).patientId.values[:4]\n\nfor pid in largest_pred:\n    visualize(patient_metrics_df, pid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cdfc27d531286c2ce522c9fe935c845b39d48426"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"31e02f3b650290e957978ffa6a5759c5eea50b76"},"cell_type":"code","source":"largest_gt = patient_metrics_df.sort_values('max_gt_size', ascending=False).patientId.values[:4]\n\nfor pid in largest_gt:\n    visualize(patient_metrics_df, pid)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c40b8e4b68a98e45c8c5e912fd00821d9b7c5205"},"cell_type":"markdown","source":"# In this case I didn't find any significant patterns by looking at those images. For me it is hard to say whether this is indeed a model failure or an annotation misalignment"},{"metadata":{"trusted":true,"_uuid":"7006fd109f43c4d192c25b70c9be44fe279775d7"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}