{"cells":[{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"markdown","source":"# If you, like me, started from training a object detection model without looking at the classification - you might think 'how could are my bounding boxes predictions, provided I would have a perfect classifier'. \n## Having a perfect classifier would set the false positives down a lot and could force a detector to give a prediction even if the confidence is small -> to minimize false negatives.\n\n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import confusion_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1ffd9c46d0e82678affddce1e5301b12d5d492d2"},"cell_type":"code","source":"INPUT_DIR = '/kaggle/input'\nDATA_DIR = os.path.join(INPUT_DIR,\"rsna-pneumonia-detection-challenge\")\nMODEL_RESULTS = os.path.join(INPUT_DIR,\"predict-on-validation\",\"validation_predictions.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b9ddccee927760f1d910b09a571f75ead0ef2e60"},"cell_type":"code","source":"anns = pd.read_csv(os.path.join(DATA_DIR,'stage_1_train_labels.csv'))\nanns.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c5c57d33785245a97b61929e407c5c256a92245a","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"# I need to group by patient id so that it is easy to join with predictions:\n\ndef gather_gt(patient_df):\n    if np.sum(patient_df['x'].isna()) > 0:\n        return []\n    else:\n        gts = []\n        for index, row in patient_df.iterrows():\n            gts.append({\n                'x':row['x'],\n                'y':row['y'],\n                'width':row['width'],\n                'height':row['height']\n            })\n        return gts\n\ngt_patient = anns.groupby('patientId').apply(gather_gt)\ngt_patient = gt_patient.to_frame(\"gt\").reset_index()\ngt_patient['Target'] = gt_patient['gt'].apply(lambda x: 1 * (x != []))\nlen(gt_patient)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6de02e425019f566b507d79b4a527a7f709d281f","_kg_hide-input":false,"_kg_hide-output":false},"cell_type":"code","source":"results = pd.read_csv(MODEL_RESULTS, header=None, names=['patientId','prediction'])\nresults = results[~results['prediction'].isna()]\n# we will join ground truth only for patients in the results df\ndf = results.merge(gt_patient, on='patientId', how='left')\nprint(df.shape)\ndf.head(n=10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"36a683677aafed60dda369a3dc36e851f0b72683","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"def to_structure(prediction):\n    exploded = prediction.strip().split(\" \")\n    predictions = [exploded[x:x+5] for x in range(0, len(exploded),5)]    \n    return [{'x':float(p[1]), 'y':float(p[2]), 'width': float(p[3]), 'height':float(p[4]), 'confidence':float(p[0])} for p in predictions]\n        \ndf['all_predictions'] = df['prediction'].apply(to_structure)\ndf = df.drop('prediction',axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"de049aa55c4d5c2cacc1298a2b2279c9922ea5e5"},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e74731778151c5429254da193fb6ce3c46cc008b"},"cell_type":"markdown","source":"## now for every class = 0 I will get rid of predictions to mimic the 'ideal classifier' scenario"},{"metadata":{"trusted":true,"_uuid":"2861017ef9242613c040b7995eda408a79722a8e"},"cell_type":"code","source":"df_no_hard_fps = df.copy()\ndf_no_hard_fps['all_predictions'] = np.where(df_no_hard_fps.Target ==0, df_no_hard_fps['gt'], df_no_hard_fps['all_predictions'])\ndf_no_hard_fps.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e894e989941b953aa9dd44f86fb8918f386ee5b4","_kg_hide-input":true},"cell_type":"code","source":"# source: https://www.kaggle.com/chenyc15/mean-average-precision-metric\n\n# extended version of metrics per patient giving more information:\n\niouthresholds = np.linspace(0.4,0.75,num=8)\n\n# helper function to calculate IoU\ndef iou(box1, box2):\n    x11, y11, w1, h1 = box1\n    x21, y21, w2, h2 = box2\n    assert w1 * h1 > 0\n    assert w2 * h2 > 0\n    x12, y12 = x11 + w1, y11 + h1\n    x22, y22 = x21 + w2, y21 + h2\n\n    area1, area2 = w1 * h1, w2 * h2\n    xi1, yi1, xi2, yi2 = max([x11, x21]), max([y11, y21]), min([x12, x22]), min([y12, y22])\n    \n    if xi2 <= xi1 or yi2 <= yi1:\n        return 0\n    else:\n        intersect = (xi2-xi1) * (yi2-yi1)\n        union = area1 + area2 - intersect\n        return intersect / union\n\ndef map_iou(boxes_true, boxes_pred, scores, thresholds = iouthresholds):\n    \"\"\"\n    Mean average precision at differnet intersection over union (IoU) threshold\n    \n    input:\n        boxes_true: Mx4 numpy array of ground true bounding boxes of one image. \n                    bbox format: (x1, y1, w, h)\n        boxes_pred: Nx4 numpy array of predicted bounding boxes of one image. \n                    bbox format: (x1, y1, w, h)\n        scores:     length N numpy array of scores associated with predicted bboxes\n        thresholds: IoU shresholds to evaluate mean average precision on\n    output: \n        map: mean average precision of the image\n    \"\"\"\n    \n    # According to the introduction, images with no ground truth bboxes will not be \n    # included in the map score unless there is a false positive detection (?)\n    result= {\n        'tp':0,\n        'tn':0,\n        'fp':0,\n        'fn':0,\n        'skipped':0,\n        'predicted_cnt':len(boxes_pred),\n        'gt_cnt':len(boxes_true),\n        'ious':{}\n    }    \n    # return None if both are empty, don't count the image in final evaluation (?)\n    if len(boxes_true) == 0 and len(boxes_pred) == 0:\n        result['skipped'] = 1\n        result['tn'] = 1\n        return result\n    if len(boxes_true) > 0 and len(boxes_pred) == 0:\n        result['prec'] = 0\n        result['fn'] = len(iouthresholds) * (len(boxes_true) - len(boxes_pred))\n        return result\n    if len(boxes_true) == 0 and len(boxes_pred) > 0:\n        result['prec'] = 0\n        result['fp'] = len(iouthresholds) * (len(boxes_pred) - len(boxes_true))\n        return result\n    \n    assert boxes_true.shape[1] == 4 or boxes_pred.shape[1] == 4, \"boxes should be 2D arrays with shape[1]=4\"\n    \n    # I am not doing any sorting just assume that predictions are sorted according to confidence, since I cannot find a way to so\n    if len(boxes_pred):\n        assert len(scores) == len(boxes_pred), \"boxes_pred and scores should be same length\"\n        # sort boxes_pred by scores in decreasing order\n        boxes_pred = boxes_pred[np.argsort(-1 * scores, kind='mergesort'), :]\n    \n    map_total = 0\n    \n    # loop over thresholds\n    total_tp = 0\n    total_fp = 0\n    total_fn = 0\n    for t in thresholds:\n        matched_bt = set()\n        tp, fn = 0, 0\n        for i, bt in enumerate(boxes_true):\n            matched = False\n            for j, bp in enumerate(boxes_pred):\n                miou = iou(bt, bp)\n                result['ious'][(i,j)] = miou\n                if miou >= t and not matched and j not in matched_bt:\n                    matched = True\n                    tp += 1 # bt is matched for the first time, count as TP\n                    matched_bt.add(j)                    \n            if not matched:\n                fn += 1 # bt has no match, count as FN\n                \n        fp = len(boxes_pred) - len(matched_bt) # FP is the bp that not matched to any bt\n        m = tp / (tp + fn + fp)\n        map_total += m\n        total_tp += tp\n        total_fp += fp\n        total_fn += fn\n    \n    result['prec'] = map_total / len(thresholds)\n    result['tp'] = total_tp\n    result['fn'] = total_fn\n    result['fp'] = total_fp\n    \n    return result\n\ndef bbox_to_array(bbox_dict):\n    return [\n                    bbox_dict['x'],\n                    bbox_dict['y'],\n                    bbox_dict['width'],\n                    bbox_dict['height'],\n                ]\n\ndef patient_metrics_off(row):\n    gtboxes = np.array([bbox_to_array(b) for b in row['gt']])\n    predboxes = np.array([bbox_to_array(b) for b in row['predictions']])\n    confidences = np.array([b['confidence'] for b in row['predictions']])\n    return map_iou(gtboxes,predboxes, confidences)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0fc9d2d0e021794f5c42eedf687c9326023edd82","_kg_hide-input":true},"cell_type":"code","source":"def filter_by_min_confidence(min_conf):\n    def f(predictions):\n        return [p for p in predictions if p['confidence'] > min_conf]\n    return f\n    \ndef metrics_for_confidences_bbox(df):\n    for min_conf in [x/100.0 for x in range(70,100)]:\n        df['predictions'] = df['all_predictions'].apply(filter_by_min_confidence(min_conf))\n        patient_metrics_df = df.apply(patient_metrics_off, axis=1,  result_type='expand')\n        tp = np.sum(patient_metrics_df.tp)\n        tn = np.sum(patient_metrics_df.tn)\n        fp = np.sum(patient_metrics_df.fp)\n        fn = np.sum(patient_metrics_df.fn)\n        not_skipped = patient_metrics_df[patient_metrics_df.skipped != 1]\n        prec = np.mean(not_skipped.prec)        \n        cnt = tn + fp + fn + tp\n        yield {\"confidence\":min_conf,\"tn\":tn/cnt,\"fp\":fp/cnt,\"fn\":fn/cnt,\"tp\":tp/cnt,\"prec\":prec}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fe11d1e462d36e7803b50113a2fbc44667096224"},"cell_type":"code","source":"plt.figure(figsize=(20,10))\n\nmetrics_original = pd.DataFrame(list(metrics_for_confidences_bbox(df))).rename(columns = {'prec':'original'})\nmetrics_perfect = pd.DataFrame(list(metrics_for_confidences_bbox(df_no_hard_fps))).rename(columns = {'prec':'with_perfect_class'})\nj = metrics_original.merge(metrics_perfect,on='confidence').melt(\nid_vars='confidence',value_vars=['original','with_perfect_class'])\n\nsns.lineplot(x='confidence', y='value', hue='variable',data=j)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8b48dddf321397fc8d74c0aa39900cd90600a605"},"cell_type":"markdown","source":"# Even with perfect classifier my model would get just ~0.35 mAP"},{"metadata":{"trusted":true,"_uuid":"a60c5f9b0c9144c1d276a21ab0f88ed731a07e88"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}