{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":67356,"databundleVersionId":8006601,"sourceType":"competition"}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#https://stats.stackexchange.com/questions/394494/calculating-sklearns-average-precision-by-hand\n\nfrom sklearn.metrics import average_precision_score, precision_recall_curve\nimport sklearn\nimport numpy as np\n\n#create some dummy ground truth\n\nnum_all = 1000\nratio_share = 0.8\nnum_share = int(round(ratio_share*num_all))\nnum_nonshare = num_all-num_share\nratio_share = num_share/num_all  #no rounding error\n\nratio_share_pos = 0.011\nnum_share_pos = int(round(ratio_share_pos*num_share))\nnum_share_neg = num_share-num_share_pos\nratio_share_pos = num_share_pos/num_share  #no rounding error\n\nratio_nonshare_pos = 0.015\nnum_nonshare_pos = int(round(ratio_nonshare_pos*num_nonshare))\nnum_nonshare_neg = num_nonshare-num_nonshare_pos\nratio_nonshare_pos = num_nonshare_pos/num_nonshare  #no rounding error\n\nnum_pos = num_share_pos+num_nonshare_pos\nratio_pos = num_pos/num_all\n\nprint('we want to recover:')\nprint('\\tratio_share_pos:',ratio_share_pos)\nprint('\\tratio_nonshare_pos:',ratio_nonshare_pos)\nprint('\\tratio_share:', ratio_share)\nprint('\\tratio_pos:', ratio_pos)\nprint('')\n\n\nshare_truth = [0]*num_share_neg +[1]*num_share_pos\nnonshare_truth = [0]*num_nonshare_neg +[1]*num_nonshare_pos\ntruth = share_truth+nonshare_truth\nprint('setup truth ok')\n\n# we want to recover the ratio above\n# by making submission to server with magic numbers","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-05-11T01:19:22.424069Z","iopub.execute_input":"2024-05-11T01:19:22.424462Z","iopub.status.idle":"2024-05-11T01:19:22.443810Z","shell.execute_reply.started":"2024-05-11T01:19:22.424433Z","shell.execute_reply":"2024-05-11T01:19:22.442434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#making null submission\nshare_p = [0]*num_share\nnonshare_p = [0]*num_nonshare\np = share_p+nonshare_p\ns = average_precision_score(truth, p)\n\n#the ratio num_pos/num_all is recovered\nprint('s recovers ratio_pos')\nprint('null submit s:',s)\nprint('ratio_pos', ratio_pos)","metadata":{"execution":{"iopub.status.busy":"2024-05-11T01:04:53.847008Z","iopub.execute_input":"2024-05-11T01:04:53.847426Z","iopub.status.idle":"2024-05-11T01:04:53.857207Z","shell.execute_reply.started":"2024-05-11T01:04:53.847378Z","shell.execute_reply":"2024-05-11T01:04:53.856218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#make nonshare submit\nshare_p = [0]*num_share\nnonshare_p = [1]*num_nonshare\np = share_p+nonshare_p\ns1 = average_precision_score(truth, p)\nprint('nonshare=1 submit:', s1)\nprint('')\n\n#-----------------------------------------------------------\nprecision = [0,ratio_nonshare_pos, s] \nrecall = [0, ratio_nonshare_pos*(1-ratio_share)/s, 1]\nthreshold = [1,0] \nprint('precision', precision)\nprint('recall', recall)\n\nprint('check if our precision,recall is correct?\\n', sklearn.metrics.precision_recall_curve(truth, p))\nprint('')\n\nap=(\n     precision[2]*(recall[2]-recall[1])\n    +precision[1]*(recall[1]-recall[0])\n)\nprint('hand computed ap:', ap)\nprint('s1',s1)\nprint('this is same as submission score!!!')\nprint('\"nonshare=1 submit\" gives an equation of \"ratio_nonshare_pos\" and \"ratio_share\" ')\nprint('')","metadata":{"execution":{"iopub.status.busy":"2024-05-11T01:04:57.292197Z","iopub.execute_input":"2024-05-11T01:04:57.293174Z","iopub.status.idle":"2024-05-11T01:04:57.308347Z","shell.execute_reply.started":"2024-05-11T01:04:57.293136Z","shell.execute_reply":"2024-05-11T01:04:57.307201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# smiliary we make share submit \nshare_p = [1]*num_share\nnonshare_p = [0]*num_nonshare\np = share_p+nonshare_p\ns2 = average_precision_score(truth, p)\nprint('share=1 submit:', s2)\n\n\nprecision = [0, ratio_share_pos, s] \nrecall = [0, ratio_share_pos*ratio_share/s, 1]\nap=(\n     precision[2]*(recall[2]-recall[1])\n    +precision[1]*(recall[1]-recall[0])\n)\nprint('hand computed ap:', ap)\nprint('')","metadata":{"execution":{"iopub.status.busy":"2024-05-11T01:05:03.458706Z","iopub.execute_input":"2024-05-11T01:05:03.459152Z","iopub.status.idle":"2024-05-11T01:05:03.471534Z","shell.execute_reply.started":"2024-05-11T01:05:03.459118Z","shell.execute_reply":"2024-05-11T01:05:03.470109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nwe make some submssion\n\nnull submit     s  = 0.008\nnonshare submit s2 = 0.017\nshare submit    s2 = 0.007\n\ntest data statistics\nshare\t\t0.420307236\nnonshare\t\t0.579692764\n\ntrain data statistics\npost label 0.005385007\n\n'''\n\n#let's try some brute force search\nratio_pos = s = 0.0085\nfor ratio_share in np.linspace(1,0.5,50):#[0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9]:\n    #print('ratio_share',ratio_share,'===================')\n    for ratio_share_pos in np.linspace(0,0.1,50):# [0.005, 0.010, 0.015, 0.020]:\n        for ratio_nonshare_pos in  np.linspace(0,0.1,50):#[0.005, 0.010, 0.015, 0.020]:\n            #print('ratio_share_pos',ratio_share_pos,'ratio_nonshare_pos',ratio_nonshare_pos)\n            precision = [0, ratio_nonshare_pos, s]\n            recall = [0, ratio_nonshare_pos * (1 - ratio_share) / s, 1]\n            threshold = [1, 0]\n            ap1 =   precision[2] * (recall[2] - recall[1]) + precision[1] * (recall[1] - recall[0])\n            #print('nonshare=1 submit s1', ap1)\n\n            precision = [0, ratio_share_pos, s]\n            recall = [0, ratio_share_pos * ratio_share / s, 1]\n            threshold = [1, 0]\n            ap2 =   precision[2] * (recall[2] - recall[1]) + precision[1] * (recall[1] - recall[0])\n            #print('share=1 submit s2', ap2)\n\n\n            if (ap1>=0.017) & (ap1<0.018) &  (ap2>=0.007) & (ap2<0.008):\n                print('ratio_share', ratio_share, '===================')\n                print('ratio_share_pos', ratio_share_pos, 'ratio_nonshare_pos', ratio_nonshare_pos)\n                print('nonshare=1 submit s1', ap1)\n                print('share=1 submit s2', ap2)\n                print('')\n\nprint('ok')","metadata":{"execution":{"iopub.status.busy":"2024-05-11T01:20:38.747712Z","iopub.execute_input":"2024-05-11T01:20:38.748129Z","iopub.status.idle":"2024-05-11T01:20:39.520748Z","shell.execute_reply.started":"2024-05-11T01:20:38.748101Z","shell.execute_reply":"2024-05-11T01:20:39.519420Z"},"trusted":true},"execution_count":null,"outputs":[]}]}