{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":67356,"databundleVersionId":8006601,"sourceType":"competition"}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# BELKA: graphical understanding of metric AP\n\nsimple demonstration of how average precision (AP) is calculated and some experiments.\n\nHere, list_all (True binary labels) consists of two parts, list_a and list_b (analogue to each protein or BB combination in BELKA competition).\n\n## Notebooks on the metric and probing\n\nhttps://www.kaggle.com/code/junkoda/unknowns-in-public-test-set\n\nhttps://www.kaggle.com/code/hengck23/how-to-probe","metadata":{}},{"cell_type":"code","source":"import numpy as np\n\n\"\"\"\nlist_a: length=300, pos_ratio=0.05, n_pos=15, n_neg = 285\nlist_b: length=600, pos_ratio=0.2, n_pos=120, n_neg = 480\ntotal: length=900, pos_ratio=0.15, n_pos=135, n_neg = 765\n\"\"\"\nlength_a = 300\nratio_a = 0.05\nnum_ones_a = int(length_a * ratio_a)\nnum_zeros_a = length_a - num_ones_a\nlist_a = np.append(np.ones(num_ones_a), np.zeros(num_zeros_a))\nnp.random.shuffle(list_a)\n\nlength_b = 600\nratio_b = 0.20\nnum_ones_b = int(length_b * ratio_b)\nnum_zeros_b = length_b - num_ones_b\nlist_b = np.append(np.ones(num_ones_b), np.zeros(num_zeros_b))\nnp.random.shuffle(list_b)\n\nprint(\"List a:\", list_a)\nprint(\"positive ratio a:\", sum(list_a)/len(list_a))\nprint(\"List b:\", list_b)\nprint(\"positive ratio b:\", sum(list_b)/len(list_b))\n\nlist_all = np.append(list_a, list_b)","metadata":{"execution":{"iopub.status.busy":"2024-05-01T13:03:07.625910Z","iopub.execute_input":"2024-05-01T13:03:07.626493Z","iopub.status.idle":"2024-05-01T13:03:07.656910Z","shell.execute_reply.started":"2024-05-01T13:03:07.626442Z","shell.execute_reply":"2024-05-01T13:03:07.655540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import average_precision_score\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import precision_recall_curve\n\ndef PR_experiment(predictions, list_all):\n    avg_precision = average_precision_score(list_all, predictions)\n\n    print(\"Average Precision:\", avg_precision)\n\n    precision, recall, _ = precision_recall_curve(list_all, predictions)\n\n    plt.figure()\n    plt.plot(recall, precision, label='PR Curve')\n    plt.step(recall, precision, color='b', alpha=0.2, where='post')\n    plt.fill_between(recall, precision, step='post', alpha=0.2, color='b')\n\n    plt.xlabel('Recall')\n    plt.ylabel('Precision')\n    plt.ylim([0.0, 1.05])\n    plt.xlim([0.0, 1.0])\n    plt.title('2-class Precision-Recall curve: AP={0:0.2f}'.format(avg_precision))\n    plt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-05-01T13:03:08.740671Z","iopub.execute_input":"2024-05-01T13:03:08.741530Z","iopub.status.idle":"2024-05-01T13:03:08.753034Z","shell.execute_reply.started":"2024-05-01T13:03:08.741494Z","shell.execute_reply":"2024-05-01T13:03:08.751696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The colored area is the score in the following plots; note that it does not necessarily correspond to the area under the PR curve.","metadata":{}},{"cell_type":"code","source":"predictions = np.zeros(len(list_all))\nPR_experiment(predictions, list_all)\n\npredictions = np.ones(len(list_all))\nPR_experiment(predictions, list_all)","metadata":{"execution":{"iopub.status.busy":"2024-05-01T13:03:09.507069Z","iopub.execute_input":"2024-05-01T13:03:09.507454Z","iopub.status.idle":"2024-05-01T13:03:10.054313Z","shell.execute_reply.started":"2024-05-01T13:03:09.507425Z","shell.execute_reply":"2024-05-01T13:03:10.053316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = np.append(np.zeros(len(list_a)), np.ones(len(list_b)))\nPR_experiment(predictions, list_all)\n# recall = 120 / 135, precision = 120 / 600\n\npredictions = np.append(np.ones(len(list_a)), np.zeros(len(list_b)))\nPR_experiment(predictions, list_all)\n# recall = 15 / 135, precision = 15 / 300","metadata":{"execution":{"iopub.status.busy":"2024-05-01T13:03:10.181008Z","iopub.execute_input":"2024-05-01T13:03:10.181414Z","iopub.status.idle":"2024-05-01T13:03:10.726930Z","shell.execute_reply.started":"2024-05-01T13:03:10.181383Z","shell.execute_reply":"2024-05-01T13:03:10.725797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Demonstraion is done! By submitting specific predictions, you might understand some statistics such as pos-neg ratio for public test dataset... ","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}