{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook explores the AP scores for a 90% accurate prediction.\n\nIt shows that the AP score is significantly dependent on the confidence scores attributed to false positives and true positives.\n\nFurther, it shows that this effect is greater for rarely ocurring classes than for commonly occurring classes.\n\nIf the confidence is successfully allocated such that false positives are less confident than true positives, then AP will correspond to the intuitive notion of the accuracy of the test (here 90%).\n\nHowever, if the confidence is not successfully allocated, then particularly for rare labels, the AP score may be heavily penalised.\n","metadata":{}},{"cell_type":"markdown","source":"Acknowledgments\n\nThanks to @Tito and @ZFTurbo for ZFTurbo's mAP code and Tito's explanation of it in:\n\nhttps://www.kaggle.com/c/hpa-single-cell-image-classification/discussion/217158","metadata":{}},{"cell_type":"code","source":"\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\n\n\n!pip install map-boxes\n\nfrom map_boxes import mean_average_precision_for_boxes\n\n","metadata":{"execution":{"iopub.status.busy":"2021-06-19T22:08:17.848544Z","iopub.execute_input":"2021-06-19T22:08:17.848866Z","iopub.status.idle":"2021-06-19T22:08:24.444035Z","shell.execute_reply.started":"2021-06-19T22:08:17.848835Z","shell.execute_reply":"2021-06-19T22:08:24.44281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Start with something simple:\n\nThere are 10 images and one label\n\nEach of the ground truth (ann) image has exactly one instance of the label\n\n9 of the images are detected (det)\n\nBounding boxes are always the same values (like having one cell per image in the HPA competition)","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n\nann_data = {'ImageID':    ['0','1','2','3','4','5','6','7','8','9'],\n            'LabelName': ['1','1','1','1','1','1','1','1','1','1'],\n            'XMin':[1,1,1,1,1,1,1,1,1,1],\n            'XMax':[10,10,10,10,10,10,10,10,10,10],\n            'YMin':[1,1,1,1,1,1,1,1,1,1],\n            'YMax':[10,10,10,10,10,10,10,10,10,10]\n        }\n\ndet_data = {'ImageID':    ['0','1','2','3','4','5','6','7','8'],\n            'LabelName': ['1','1','1','1','1','1','1','1','1'],\n            'Conf':[1,1,1,1,1,1,1,1,1],\n            'XMin':[1,1,1,1,1,1,1,1,1],\n            'XMax':[10,10,10,10,10,10,10,10,10],\n            'YMin':[1,1,1,1,1,1,1,1,1],\n            'YMax':[10,10,10,10,10,10,10,10,10]\n        }\n\nann_df = pd.DataFrame (ann_data, columns = ['ImageID', 'LabelName', 'XMin', 'XMax', 'YMin', 'YMax'])\ndet_df = pd.DataFrame (det_data, columns = ['ImageID', 'LabelName','Conf', 'XMin', 'XMax', 'YMin', 'YMax'])\n\nprint (ann_df)\nprint (det_df)\n\n\nmean_ap, average_precisions = mean_average_precision_for_boxes(ann_df, det_df)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-19T22:08:24.447679Z","iopub.execute_input":"2021-06-19T22:08:24.448129Z","iopub.status.idle":"2021-06-19T22:08:24.480212Z","shell.execute_reply.started":"2021-06-19T22:08:24.448078Z","shell.execute_reply":"2021-06-19T22:08:24.47919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So we found nine out of 10 and got 90% score - pretty intuitive so far","metadata":{}},{"cell_type":"markdown","source":"Now, lets get 9 right and also detect a false positive (by using a different bounding box)\n\nWe give it a lower confidence, (plausible since it is a wrong detection)","metadata":{}},{"cell_type":"code","source":"\nann_data = {'ImageID':    ['0','1','2','3','4','5','6','7','8','9'],\n            'LabelName': ['1','1','1','1','1','1','1','1','1','1'],\n            'XMin':[1,1,1,1,1,1,1,1,1,1],\n            'XMax':[10,10,10,10,10,10,10,10,10,10],\n            'YMin':[1,1,1,1,1,1,1,1,1,1],\n            'YMax':[10,10,10,10,10,10,10,10,10,10]\n        }\n\ndet_data = {'ImageID':    ['0','1','2','3','4','5','6','7','8','9'],\n            'LabelName': ['1','1','1','1','1','1','1','1','1','1'],\n            'Conf':[1,1,1,1,1,1,1,1,1,0.9],\n            'XMin':[1,1,1,1,1,1,1,1,1,11],\n            'XMax':[10,10,10,10,10,10,10,10,10,20],\n            'YMin':[1,1,1,1,1,1,1,1,1,11],\n            'YMax':[10,10,10,10,10,10,10,10,10,20]\n        }\n\nann_df = pd.DataFrame (ann_data, columns = ['ImageID', 'LabelName', 'XMin', 'XMax', 'YMin', 'YMax'])\ndet_df = pd.DataFrame (det_data, columns = ['ImageID', 'LabelName','Conf', 'XMin', 'XMax', 'YMin', 'YMax'])\n\nprint (ann_df)\nprint (det_df)\n\n\nmean_ap, average_precisions = mean_average_precision_for_boxes(ann_df, det_df)","metadata":{"execution":{"iopub.status.busy":"2021-06-19T22:08:24.482172Z","iopub.execute_input":"2021-06-19T22:08:24.482556Z","iopub.status.idle":"2021-06-19T22:08:24.513181Z","shell.execute_reply.started":"2021-06-19T22:08:24.482522Z","shell.execute_reply":"2021-06-19T22:08:24.511587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"That still got 90% still intuitive","metadata":{}},{"cell_type":"markdown","source":"Now let's make some of the true positives less confident than the false positive by changeing 'Conf'","metadata":{}},{"cell_type":"code","source":"\nann_data = {'ImageID':    ['0','1','2','3','4','5','6','7','8','9'],\n            'LabelName': ['1','1','1','1','1','1','1','1','1','1'],\n            'XMin':[1,1,1,1,1,1,1,1,1,1],\n            'XMax':[10,10,10,10,10,10,10,10,10,10],\n            'YMin':[1,1,1,1,1,1,1,1,1,1],\n            'YMax':[10,10,10,10,10,10,10,10,10,10]\n        }\n\ndet_data = {'ImageID':    ['0','1','2','3','4','5','6','7','8','9'],\n            'LabelName': ['1','1','1','1','1','1','1','1','1','1'],\n            'Conf':[0.8,0.8,0.8,0.8,1,1,1,1,1,0.9],\n            'XMin':[1,1,1,1,1,1,1,1,1,11],\n            'XMax':[10,10,10,10,10,10,10,10,10,20],\n            'YMin':[1,1,1,1,1,1,1,1,1,11],\n            'YMax':[10,10,10,10,10,10,10,10,10,20]\n        }\n\nann_df = pd.DataFrame (ann_data, columns = ['ImageID', 'LabelName', 'XMin', 'XMax', 'YMin', 'YMax'])\ndet_df = pd.DataFrame (det_data, columns = ['ImageID', 'LabelName','Conf', 'XMin', 'XMax', 'YMin', 'YMax'])\n\nprint (ann_df)\nprint (det_df)\n\n\nmean_ap, average_precisions = mean_average_precision_for_boxes(ann_df, det_df)","metadata":{"execution":{"iopub.status.busy":"2021-06-19T22:08:24.515191Z","iopub.execute_input":"2021-06-19T22:08:24.51578Z","iopub.status.idle":"2021-06-19T22:08:24.553388Z","shell.execute_reply.started":"2021-06-19T22:08:24.515635Z","shell.execute_reply":"2021-06-19T22:08:24.552154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Aha: lost some score on that, now only 0.86","metadata":{}},{"cell_type":"markdown","source":"and if we make all our true positives less confident than our false negative:","metadata":{}},{"cell_type":"code","source":"ann_data = {'ImageID':    ['0','1','2','3','4','5','6','7','8','9'],\n            'LabelName': ['1','1','1','1','1','1','1','1','1','1'],\n            'XMin':[1,1,1,1,1,1,1,1,1,1],\n            'XMax':[10,10,10,10,10,10,10,10,10,10],\n            'YMin':[1,1,1,1,1,1,1,1,1,1],\n            'YMax':[10,10,10,10,10,10,10,10,10,10]\n        }\n\ndet_data = {'ImageID':    ['0','1','2','3','4','5','6','7','8','9'],\n            'LabelName': ['1','1','1','1','1','1','1','1','1','1'],\n            'Conf':[0.8,0.8,0.8,0.8,0.8,0.8,0.8,0.8,0.8,0.9],\n            'XMin':[1,1,1,1,1,1,1,1,1,11],\n            'XMax':[10,10,10,10,10,10,10,10,10,20],\n            'YMin':[1,1,1,1,1,1,1,1,1,11],\n            'YMax':[10,10,10,10,10,10,10,10,10,20]\n        }\n\nann_df = pd.DataFrame (ann_data, columns = ['ImageID', 'LabelName', 'XMin', 'XMax', 'YMin', 'YMax'])\ndet_df = pd.DataFrame (det_data, columns = ['ImageID', 'LabelName','Conf', 'XMin', 'XMax', 'YMin', 'YMax'])\n\nprint (ann_df)\nprint (det_df)\n\n\nmean_ap, average_precisions = mean_average_precision_for_boxes(ann_df, det_df)","metadata":{"execution":{"iopub.status.busy":"2021-06-19T22:08:24.554554Z","iopub.execute_input":"2021-06-19T22:08:24.554904Z","iopub.status.idle":"2021-06-19T22:08:24.587089Z","shell.execute_reply.started":"2021-06-19T22:08:24.55487Z","shell.execute_reply":"2021-06-19T22:08:24.585951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"now only 0.81, although we still got 90% right !","metadata":{}},{"cell_type":"markdown","source":"ok, so lets model a working detection solution, with the following characteristics:\n\nGiven a ground truth positive, detects positive with 90% probability\n\nGiven a ground truth negative (using label 18 as in HPA competition for 'Negative' label), detects positive with 10% probability","metadata":{}},{"cell_type":"code","source":"from random import *\n\nann_data = {'ImageID':    ['0','1','2','3','4','5','6','7','8','9'],\n            'LabelName': ['1','1','1','1','1','1','1','1','1','1'],\n            'XMin':[1,1,1,1,1,1,1,1,1,1],\n            'XMax':[10,10,10,10,10,10,10,10,10,10],\n            'YMin':[1,1,1,1,1,1,1,1,1,1],\n            'YMax':[10,10,10,10,10,10,10,10,10,10]\n        }\n\nann_df = pd.DataFrame (ann_data, columns = ['ImageID', 'LabelName', 'XMin', 'XMax', 'YMin', 'YMax'])\n\nimage_id_list = []\nlabel_name_list = []\nconf_list = []\nxmin_list = []\nxmax_list = []\nymin_list = []\nymax_list = []\n\nimage_id=0\n\nfor i, ann in ann_df.iterrows():\n    image_id_list.append(str(i))\n    ground_truth_label=ann['LabelName']\n    rnd=random()\n    if (ground_truth_label == '1'):\n        if rnd  < 0.9:\n            label_name_list.append('1')\n        else:\n            label_name_list.append('18')\n    else:\n        if rnd  < 0.9:\n            label_name_list.append('18')\n        else:\n            label_name_list.append('1')\n    conf_list.append(random())\n    xmin_list.append(1)\n    xmax_list.append(10)\n    ymin_list.append(1)\n    ymax_list.append(10) \n\n\ndet_data = {'ImageID':   image_id_list,\n            'LabelName': label_name_list,\n            'Conf':conf_list,\n            'XMin':xmin_list,\n            'XMax':xmax_list,\n            'YMin':ymin_list,\n            'YMax':ymax_list\n        }\n\n\ndet_df = pd.DataFrame (det_data, columns = ['ImageID', 'LabelName','Conf', 'XMin', 'XMax', 'YMin', 'YMax'])\n\nprint (ann_df)\nprint (det_df)\n\n\nmean_ap, average_precisions = mean_average_precision_for_boxes(ann_df, det_df)","metadata":{"execution":{"iopub.status.busy":"2021-06-19T22:08:24.589358Z","iopub.execute_input":"2021-06-19T22:08:24.589783Z","iopub.status.idle":"2021-06-19T22:08:24.625942Z","shell.execute_reply.started":"2021-06-19T22:08:24.589723Z","shell.execute_reply":"2021-06-19T22:08:24.624887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Running this a few times you will see scores of 1, 0.9, 0.8, 0.7 ... depending on how the random numbers turns out\n\n(note that there are no ground truth negatives here and no differing bounding boxes, so it is like the HPA case of one cell per image, and all cells classified with the label '1' - In this case the AP directly corresponds to the intuition of percentage correct identifications)\n","metadata":{}},{"cell_type":"markdown","source":"Lets add some further ground truth negatives:","metadata":{}},{"cell_type":"code","source":"ann_data = {'ImageID':    ['0','1','2','3','4','5','6','7','8','9','10','11','12','13','14','15','16','17','18','19'],\n            'LabelName': ['1','1','1','1','1','1','1','1','1','1','18','18','18','18','18','18','18','18','18','18'],\n            'XMin':[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1],\n            'XMax':[10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10],\n            'YMin':[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1],\n            'YMax':[10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10]\n        }\n\nann_df = pd.DataFrame (ann_data, columns = ['ImageID', 'LabelName', 'XMin', 'XMax', 'YMin', 'YMax'])\n\nimage_id_list = []\nlabel_name_list = []\nconf_list = []\nxmin_list = []\nxmax_list = []\nymin_list = []\nymax_list = []\n\nimage_id=0\n\nfor i, ann in ann_df.iterrows():\n    image_id_list.append(str(i))\n    ground_truth_label=ann['LabelName']\n    rnd=random()\n    if (ground_truth_label == '1'):\n        if rnd  < 0.9:\n            label_name_list.append('1')\n        else:\n            label_name_list.append('18')\n    else:\n        if rnd  < 0.9:\n            label_name_list.append('18')\n        else:\n            label_name_list.append('1')\n    conf_list.append(random())\n    xmin_list.append(1)\n    xmax_list.append(10)\n    ymin_list.append(1)\n    ymax_list.append(10) \n\n\ndet_data = {'ImageID':   image_id_list,\n            'LabelName': label_name_list,\n            'Conf':conf_list,\n            'XMin':xmin_list,\n            'XMax':xmax_list,\n            'YMin':ymin_list,\n            'YMax':ymax_list\n        }\n\n\ndet_df = pd.DataFrame (det_data, columns = ['ImageID', 'LabelName','Conf', 'XMin', 'XMax', 'YMin', 'YMax'])\n\nprint (ann_df)\nprint (det_df)\n\n\nmean_ap, average_precisions = mean_average_precision_for_boxes(ann_df, det_df)","metadata":{"execution":{"iopub.status.busy":"2021-06-19T22:08:24.627477Z","iopub.execute_input":"2021-06-19T22:08:24.627829Z","iopub.status.idle":"2021-06-19T22:08:24.669803Z","shell.execute_reply.started":"2021-06-19T22:08:24.627796Z","shell.execute_reply":"2021-06-19T22:08:24.668767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"If you exercise this a few times, you will see that, depending on the random numbers, the score is often high, but can also get quite low\n\nFor instance, my first execution of this version gave:\n\n1                              | 0.466667 |      10\n\n18                             | 0.626190 |      10\n\nmAP: 0.546429\n\nSo although our detection is designed to behave with 90% accuracy, the AP for label 1 was less that 0.5 and the overall mAP was 0.54 !\n\nOk, that was just bad luck, so we need to bring probabilities into it.\n\nThis is reminiscent of the much cited case of a test for a rare disease where the probability of actually having the disease given that a fairly reliable test was positive is intuitively expected to be higher than it is.  The reason is Bayes theorem and the low probability of actually having the disease and therefore relatively high probability of a false positive test rather than a true positive test.\n\nSo now, lets do just that: make label 1 rare, only one in 20:\n","metadata":{}},{"cell_type":"code","source":"ann_data = {'ImageID':    ['0','1','2','3','4','5','6','7','8','9','10','11','12','13','14','15','16','17','18','19'],\n            'LabelName': ['1','18','18','18','18','18','18','18','18','18','18','18','18','18','18','18','18','18','18','18'],\n            'XMin':[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1],\n            'XMax':[10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10],\n            'YMin':[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1],\n            'YMax':[10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10]\n        }\n\nann_df = pd.DataFrame (ann_data, columns = ['ImageID', 'LabelName', 'XMin', 'XMax', 'YMin', 'YMax'])\n\nimage_id_list = []\nlabel_name_list = []\nconf_list = []\nxmin_list = []\nxmax_list = []\nymin_list = []\nymax_list = []\n\nimage_id=0\n\nfor i, ann in ann_df.iterrows():\n    image_id_list.append(str(i))\n    ground_truth_label=ann['LabelName']\n    rnd=random()\n    if (ground_truth_label == '1'):\n        if rnd  < 0.9:\n            label_name_list.append('1')\n        else:\n            label_name_list.append('18')\n    else:\n        if rnd  < 0.9:\n            label_name_list.append('18')\n        else:\n            label_name_list.append('1')\n    conf_list.append(random())\n    xmin_list.append(1)\n    xmax_list.append(10)\n    ymin_list.append(1)\n    ymax_list.append(10) \n\n\ndet_data = {'ImageID':   image_id_list,\n            'LabelName': label_name_list,\n            'Conf':conf_list,\n            'XMin':xmin_list,\n            'XMax':xmax_list,\n            'YMin':ymin_list,\n            'YMax':ymax_list\n        }\n\n\ndet_df = pd.DataFrame (det_data, columns = ['ImageID', 'LabelName','Conf', 'XMin', 'XMax', 'YMin', 'YMax'])\n\nprint (ann_df)\nprint (det_df)\n\n\nmean_ap, average_precisions = mean_average_precision_for_boxes(ann_df, det_df)","metadata":{"execution":{"iopub.status.busy":"2021-06-19T22:08:24.67124Z","iopub.execute_input":"2021-06-19T22:08:24.671588Z","iopub.status.idle":"2021-06-19T22:08:24.712505Z","shell.execute_reply.started":"2021-06-19T22:08:24.671554Z","shell.execute_reply":"2021-06-19T22:08:24.711544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"well, I had to run it quite a few times to get one with a false negative\n\nand then the score was:\n\nAnnotations length: 20\n\n1                              | 0.000000 |       1\n\n18                             | 0.814241 |      19\n\nmAP: 0.407121\n\nand when I had a true positive, the high \"label 1 score\" for detecting label 1 correctly was often dragged down by several false positives that had greater Conf than the true positive\n","metadata":{}},{"cell_type":"markdown","source":"Now let:\n\np = number of cells having ground truth label 1\n\nn  = number of cells in the test set\n\nthen:\n\nexpected number of label 1 true positives: 0.9 * p\n\nexpected number of label 1 false positives: 0.1 * (n-p)\n\nlooking at a specific case where these expected results occur, If all the false positives have a lower \nconfidence than all the true positives then the false positives are ignored (end of the curve)\nbecause the precision stays at 1 over the whole range of recall from 0 to 0.9, giving area under the curve of 1 * 0.9 = 0.9\n\nHowever, if the false positives all have higher confidence than the true positives, then when the first\ntrue positive is found, the accuracy is:\n\nnumber of correct answers / number of predictions\n\n= 1 / (number of false positives + 1)\n\nand as each further true positive is found (increasing recall), this grows to:\n\n= (number of true positives) / (number of false positives + number of true positives)\n\n= 0.9 * p / (( 0.1 * (n-p)) * (0.9 * p))\n\n= 0.9 * p / ( (0.1 * n) + (0.8 * p) )\n\n= 0.9 / ( 0.1 * (n/p) ) + 0.8 )                  [assuming p != 0 , in which case the accuracy would be zero]\n\nso as p approaches n the score approaches 0.9 / (0.1+0.8) = 1\n\nbut as p approaches one, n/p will approch n and the score will become small if n is large\n\nSo to conclude: \nThe relative confidence allocated to true positives and false positives is important.\nIf the confidence is successfully allocated such that false positives are less confident than true positives, then AP will correspond to the intuitive notion of the accuracy of the test (here 90%).\n\nHowever, if the confidence is not successfully allocated, then particularly for rare labels, the AP score may be heavily penalised.\n\n\nThe next two code cells run an experiment to check this in practice, since the above argument doesn't treat the different distributions of conf scores.\n\nIn both cells, the function make_det makes 90% correct predictions as above and is called 1000 times to get a representative result over the randomly created predictions and Conf scores.\n\nIn both cells n=20, in the first experiment, p=1 (label 1 is rare), and in the seconde p=15 (label 1 is common)\n\nResult:\n\nWith p=1 AP is around 0.60 and with p=15 AP is around 0.88 so we see that with the same 90% accuracy of test the AP score of the rare label is lower.\n","metadata":{}},{"cell_type":"code","source":"ann_data = {'ImageID':    ['0','1','2','3','4','5','6','7','8','9','10','11','12','13','14','15','16','17','18','19'],\n            'LabelName': ['1','18','18','18','18','18','18','18','18','18','18','18','18','18','18','18','18','18','18','18'],\n            'XMin':[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1],\n            'XMax':[10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10],\n            'YMin':[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1],\n            'YMax':[10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10]\n        }\n\nann_df = pd.DataFrame (ann_data, columns = ['ImageID', 'LabelName', 'XMin', 'XMax', 'YMin', 'YMax'])\n\ndef make_det(adf):\n# produce a set of predictions for the elements of adf with 90% probability of correctness\n    image_id_list = []\n    label_name_list = []\n    conf_list = []\n    xmin_list = []\n    xmax_list = []\n    ymin_list = []\n    ymax_list = []\n\n    image_id=0\n\n    for i, ann in adf.iterrows():\n        image_id_list.append(str(i))\n        ground_truth_label=ann['LabelName']\n        rnd=random()\n        if (ground_truth_label == '1'):\n            if rnd  < 0.9:\n                label_name_list.append('1')\n            else:\n                label_name_list.append('18')\n        else:\n            if rnd  < 0.9:\n                label_name_list.append('18')\n            else:\n                label_name_list.append('1')\n        conf_list.append(random())\n        xmin_list.append(1)\n        xmax_list.append(10)\n        ymin_list.append(1)\n        ymax_list.append(10) \n\n\n    det_data = {'ImageID':   image_id_list,\n                'LabelName': label_name_list,\n                'Conf':conf_list,\n                'XMin':xmin_list,\n                'XMax':xmax_list,\n                'YMin':ymin_list,\n                'YMax':ymax_list\n            }\n\n\n    ddf = pd.DataFrame (det_data, columns = ['ImageID', 'LabelName','Conf', 'XMin', 'XMax', 'YMin', 'YMax'])\n    return ddf\n\nsum_ap = 0\nfor i in range(1000):\n    det_df = make_det(ann_df)\n    mean_ap, average_precisions = mean_average_precision_for_boxes(ann_df, det_df,verbose = False)\n    sum_ap = sum_ap + average_precisions['1'][0]\n    \n\nprint(sum_ap)    \n    \n    \n    ","metadata":{"execution":{"iopub.status.busy":"2021-06-19T22:08:24.714236Z","iopub.execute_input":"2021-06-19T22:08:24.714778Z","iopub.status.idle":"2021-06-19T22:08:31.982989Z","shell.execute_reply.started":"2021-06-19T22:08:24.714707Z","shell.execute_reply":"2021-06-19T22:08:31.981616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ann_data = {'ImageID':    ['0','1','2','3','4','5','6','7','8','9','10','11','12','13','14','15','16','17','18','19'],\n            'LabelName': ['1','1','1','1','1','1','1','1','1','1','1','1','1','1','1','18','18','18','18','18'],\n            'XMin':[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1],\n            'XMax':[10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10],\n            'YMin':[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1],\n            'YMax':[10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10]\n        }\n\nann_df = pd.DataFrame (ann_data, columns = ['ImageID', 'LabelName', 'XMin', 'XMax', 'YMin', 'YMax'])\n\nsum_ap = 0\nfor i in range(1000):\n    det_df = make_det(ann_df)\n    mean_ap, average_precisions = mean_average_precision_for_boxes(ann_df, det_df,verbose = False)\n    sum_ap = sum_ap + average_precisions['1'][0]\n    \n\nprint(sum_ap)    ","metadata":{"execution":{"iopub.status.busy":"2021-06-19T22:08:31.984326Z","iopub.execute_input":"2021-06-19T22:08:31.984618Z","iopub.status.idle":"2021-06-19T22:08:39.29658Z","shell.execute_reply.started":"2021-06-19T22:08:31.984586Z","shell.execute_reply":"2021-06-19T22:08:39.29544Z"},"trusted":true},"execution_count":null,"outputs":[]}]}