{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":67356,"databundleVersionId":8006601,"sourceType":"competition"},{"sourceId":9085054,"sourceType":"datasetVersion","datasetId":5481644},{"sourceId":9098966,"sourceType":"datasetVersion","datasetId":5491298},{"sourceId":178141383,"sourceType":"kernelVersion"}],"dockerImageVersionId":30746,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# AP score by split group","metadata":{}},{"cell_type":"markdown","source":"- This notebook is forked from https://www.kaggle.com/code/kirkdco/precision-by-protein-and-group\n- The ground truth of the test set (public and private) is from [the dataset posted by the host](https://www.kaggle.com/datasets/andrewdblevins/belka-solution-key/data) and [the discussion](https://www.kaggle.com/competitions/leash-BELKA/discussion/523644); it also contains the split group of kin0 (non-triazine), share (triazine with shared BBs), and non-share (triazine with non-shared BBs)\n- To calculate Average Precision, please upload the submission file (make sure the filename on line 1 is correct).\n- Please post your result in discussion: https://www.kaggle.com/competitions/leash-BELKA/discussion/523779","metadata":{}},{"cell_type":"code","source":"import numpy as np, pandas as pd\nfrom tqdm import tqdm\nfrom sklearn.metrics import average_precision_score\nfrom collections import defaultdict","metadata":{"execution":{"iopub.status.busy":"2024-08-03T23:45:45.795285Z","iopub.execute_input":"2024-08-03T23:45:45.795723Z","iopub.status.idle":"2024-08-03T23:45:45.802146Z","shell.execute_reply.started":"2024-08-03T23:45:45.795687Z","shell.execute_reply":"2024-08-03T23:45:45.800679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Your submission file","metadata":{}},{"cell_type":"markdown","source":"Take one for example","metadata":{}},{"cell_type":"code","source":"sub = pd.read_csv('/kaggle/input/belka1dcnn-0-310-private-based-on-ah-s-notebook/submission.csv')","metadata":{"execution":{"iopub.status.busy":"2024-08-03T20:27:03.861808Z","iopub.execute_input":"2024-08-03T20:27:03.862348Z","iopub.status.idle":"2024-08-03T20:27:04.582128Z","shell.execute_reply.started":"2024-08-03T20:27:03.862304Z","shell.execute_reply":"2024-08-03T20:27:04.581029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub","metadata":{"execution":{"iopub.status.busy":"2024-08-03T20:23:58.141236Z","iopub.execute_input":"2024-08-03T20:23:58.141712Z","iopub.status.idle":"2024-08-03T20:23:58.158681Z","shell.execute_reply.started":"2024-08-03T20:23:58.141678Z","shell.execute_reply":"2024-08-03T20:23:58.157335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Ground truth data with split group","metadata":{}},{"cell_type":"code","source":"# ground truth\ntarget = pd.read_csv('/kaggle/input/belka-solution-key/solution_with_groups.csv')","metadata":{"execution":{"iopub.status.busy":"2024-08-03T20:27:17.889651Z","iopub.execute_input":"2024-08-03T20:27:17.890061Z","iopub.status.idle":"2024-08-03T20:27:19.029599Z","shell.execute_reply.started":"2024-08-03T20:27:17.890030Z","shell.execute_reply":"2024-08-03T20:27:19.028272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target.shape","metadata":{"execution":{"iopub.status.busy":"2024-08-03T20:27:20.339447Z","iopub.execute_input":"2024-08-03T20:27:20.339886Z","iopub.status.idle":"2024-08-03T20:27:20.347628Z","shell.execute_reply.started":"2024-08-03T20:27:20.339851Z","shell.execute_reply":"2024-08-03T20:27:20.346201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target.split_group.value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-08-03T20:27:20.833373Z","iopub.execute_input":"2024-08-03T20:27:20.833811Z","iopub.status.idle":"2024-08-03T20:27:21.143254Z","shell.execute_reply.started":"2024-08-03T20:27:20.833777Z","shell.execute_reply":"2024-08-03T20:27:21.141882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target.split_group = target.split_group.str.replace('kin0','non-triazine')","metadata":{"execution":{"iopub.status.busy":"2024-08-03T21:09:45.337765Z","iopub.execute_input":"2024-08-03T21:09:45.338209Z","iopub.status.idle":"2024-08-03T21:09:46.080906Z","shell.execute_reply.started":"2024-08-03T21:09:45.338150Z","shell.execute_reply":"2024-08-03T21:09:46.079650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target.head()","metadata":{"execution":{"iopub.status.busy":"2024-08-03T20:27:22.045372Z","iopub.execute_input":"2024-08-03T20:27:22.045821Z","iopub.status.idle":"2024-08-03T20:27:22.061260Z","shell.execute_reply.started":"2024-08-03T20:27:22.045788Z","shell.execute_reply":"2024-08-03T20:27:22.059961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Merge the two","metadata":{}},{"cell_type":"code","source":"# merge the two on id\ndf = pd.merge(sub, target, on = 'id')","metadata":{"execution":{"iopub.status.busy":"2024-08-03T21:10:11.145721Z","iopub.execute_input":"2024-08-03T21:10:11.146510Z","iopub.status.idle":"2024-08-03T21:10:11.537658Z","shell.execute_reply.started":"2024-08-03T21:10:11.146475Z","shell.execute_reply":"2024-08-03T21:10:11.536327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2024-08-03T21:10:12.141245Z","iopub.execute_input":"2024-08-03T21:10:12.141681Z","iopub.status.idle":"2024-08-03T21:10:12.162082Z","shell.execute_reply.started":"2024-08-03T21:10:12.141651Z","shell.execute_reply":"2024-08-03T21:10:12.160522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Calculate average precision by split group","metadata":{}},{"cell_type":"code","source":"defaultdict?","metadata":{"execution":{"iopub.status.busy":"2024-08-03T21:10:13.277156Z","iopub.execute_input":"2024-08-03T21:10:13.277576Z","iopub.status.idle":"2024-08-03T21:10:13.285368Z","shell.execute_reply.started":"2024-08-03T21:10:13.277545Z","shell.execute_reply":"2024-08-03T21:10:13.284098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# generate an empty dictionary that can be append\nprecision_dict = defaultdict(list)","metadata":{"execution":{"iopub.status.busy":"2024-08-03T21:10:13.693284Z","iopub.execute_input":"2024-08-03T21:10:13.693701Z","iopub.status.idle":"2024-08-03T21:10:13.699070Z","shell.execute_reply.started":"2024-08-03T21:10:13.693667Z","shell.execute_reply":"2024-08-03T21:10:13.697767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"When you access or modify entries in `precision_dict`, it behaves mostly like a regular dictionary, but with one key difference. If you try to access a key that doesn't exist, instead of raising an error, it creates a new entry with the default type","metadata":{}},{"cell_type":"code","source":"# generate an empty list that can be append\nscores = []\ngroups=[]","metadata":{"execution":{"iopub.status.busy":"2024-08-03T21:10:15.174636Z","iopub.execute_input":"2024-08-03T21:10:15.175057Z","iopub.status.idle":"2024-08-03T21:10:15.180166Z","shell.execute_reply.started":"2024-08-03T21:10:15.175017Z","shell.execute_reply":"2024-08-03T21:10:15.178839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# iterate through target protein\nfor protein in df['protein_name'].unique():\n    \n    # iterate through split_group\n    for group in df['split_group'].unique():\n        \n        # obtain the idx of target protein and split group\n        condition = (df['protein_name'] == protein) & (df['split_group'] == group)\n        \n        # query true and pred\n        y_true = df['binds_y'][condition]\n        y_pred = df['binds_x'][condition]\n        \n        # calculate AP\n        precision = average_precision_score(y_true, y_pred)\n        \n        # append to the dictionary\n        precision_dict[protein].append(precision)\n        \n        # append to the list\n        scores.append(precision)\n        \n        groups.append(f'{protein}_{group}')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-08-03T21:10:16.208491Z","iopub.execute_input":"2024-08-03T21:10:16.208915Z","iopub.status.idle":"2024-08-03T21:10:22.846434Z","shell.execute_reply.started":"2024-08-03T21:10:16.208882Z","shell.execute_reply":"2024-08-03T21:10:22.845217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores","metadata":{"execution":{"iopub.status.busy":"2024-08-03T21:10:22.848679Z","iopub.execute_input":"2024-08-03T21:10:22.849160Z","iopub.status.idle":"2024-08-03T21:10:22.857404Z","shell.execute_reply.started":"2024-08-03T21:10:22.849117Z","shell.execute_reply":"2024-08-03T21:10:22.856162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Mean AP by split group\nnp.mean(scores)","metadata":{"execution":{"iopub.status.busy":"2024-08-03T21:10:24.965107Z","iopub.execute_input":"2024-08-03T21:10:24.965568Z","iopub.status.idle":"2024-08-03T21:10:24.975003Z","shell.execute_reply.started":"2024-08-03T21:10:24.965534Z","shell.execute_reply":"2024-08-03T21:10:24.973522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visualize dictionary in df\nout = pd.DataFrame(precision_dict,index=df.split_group.unique()).T\nout.style.background_gradient(cmap='Reds',axis=None)","metadata":{"execution":{"iopub.status.busy":"2024-08-03T21:10:25.428777Z","iopub.execute_input":"2024-08-03T21:10:25.429234Z","iopub.status.idle":"2024-08-03T21:10:25.581887Z","shell.execute_reply.started":"2024-08-03T21:10:25.429198Z","shell.execute_reply":"2024-08-03T21:10:25.580532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Overall, `share` group has the highest score, and `non-triazine` has the lowest score.","metadata":{}},{"cell_type":"code","source":"pd.DataFrame(scores,index=groups,columns=['310_1DCNN'])","metadata":{"execution":{"iopub.status.busy":"2024-08-03T21:11:32.876722Z","iopub.execute_input":"2024-08-03T21:11:32.877135Z","iopub.status.idle":"2024-08-03T21:11:32.890913Z","shell.execute_reply.started":"2024-08-03T21:11:32.877100Z","shell.execute_reply":"2024-08-03T21:11:32.889448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Function","metadata":{}},{"cell_type":"code","source":"target = pd.read_csv('/kaggle/input/belka-solution-key/solution_with_groups.csv')\ntarget.split_group = target.split_group.str.replace('kin0','non-triazine')","metadata":{"execution":{"iopub.status.busy":"2024-08-03T23:45:53.480183Z","iopub.execute_input":"2024-08-03T23:45:53.480606Z","iopub.status.idle":"2024-08-03T23:45:55.312470Z","shell.execute_reply.started":"2024-08-03T23:45:53.480571Z","shell.execute_reply":"2024-08-03T23:45:55.311267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calculate_AP(sub_pth,# path of a submission file\n              name,# name that appears in the out df\n             ):\n    \n    \"Calculate AP score split by protein and group, returns an unstacked df with AP scores\"\n    \n    sub = pd.read_csv(sub_pth)\n        \n    df = pd.merge(sub, target, on = 'id')\n    \n\n    scores = []\n    groups=[]\n    # iterate through target protein\n    for protein in df['protein_name'].unique():\n\n        # iterate through split_group\n        for group in df['split_group'].unique():\n\n            # obtain the idx of target protein and split group\n            condition = (df['protein_name'] == protein) & (df['split_group'] == group)\n\n            # query true and pred\n            y_true = df['binds_y'][condition]\n            y_pred = df['binds_x'][condition]\n\n            # calculate AP\n            precision = average_precision_score(y_true, y_pred)\n\n\n            # append to the list\n            scores.append(precision)\n\n            groups.append(f'{protein}_{group}')\n\n    mean_AP = np.mean(scores)\n    scores.append(mean_AP)\n    groups.append('mean_AP')\n    \n    out = pd.DataFrame(scores,index=groups,columns=[name])\n\n    return out","metadata":{"execution":{"iopub.status.busy":"2024-08-03T23:45:58.183965Z","iopub.execute_input":"2024-08-03T23:45:58.184393Z","iopub.status.idle":"2024-08-03T23:45:58.195971Z","shell.execute_reply.started":"2024-08-03T23:45:58.184360Z","shell.execute_reply":"2024-08-03T23:45:58.194574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Solutions from Hengck23 [github](https://github.com/hengck23/solution-leash-BELKA/tree/main) and [google drive](https://drive.google.com/drive/folders/1bEBGtTJrQlYc_MQRYceBp0Kb9zGYue9H?usp=drive_link)","metadata":{}},{"cell_type":"code","source":"submissions = {\n    'mamba_fold3_tx2a':'/kaggle/input/hengck23-submissions-leash/final-3fold-tx2a-mamba-fix.submit.csv',\n    'cnn1d_fold0':'/kaggle/input/hengck23-submissions-leash/final-cnn1d-ly5-bn-mean-pool-fold0-00400000.submit.csv',\n    'cnn1d_fold1':'/kaggle/input/hengck23-submissions-leash/final-cnn1d-ly5-bn-mean-pool-fold1-00550000.submit.csv',\n    'cnn1d_fold3':'/kaggle/input/hengck23-submissions-leash/final-cnn1d-ly5-bn-mean-pool-fold3-00415000.submit.csv',\n    'mamba_fold0':'/kaggle/input/hengck23-submissions-leash/final-mamba-03-fold0-00255000.submit.csv',\n    'transformer_fold2':'/kaggle/input/hengck23-submissions-leash/final-transfomer-fa-03-fold2-00264000.submit.csv',\n    'transformer_fold4':'/kaggle/input/hengck23-submissions-leash/final-transfomer-fa-03-fold4-00264000.submit.csv',\n}","metadata":{"execution":{"iopub.status.busy":"2024-08-03T23:46:00.445050Z","iopub.execute_input":"2024-08-03T23:46:00.445607Z","iopub.status.idle":"2024-08-03T23:46:00.452649Z","shell.execute_reply.started":"2024-08-03T23:46:00.445566Z","shell.execute_reply":"2024-08-03T23:46:00.451251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"outs=[]\nfor k,v in tqdm(submissions.items()):\n    out = calculate_AP(v,k)\n    outs.append(out)","metadata":{"execution":{"iopub.status.busy":"2024-08-03T23:46:02.691949Z","iopub.execute_input":"2024-08-03T23:46:02.693285Z","iopub.status.idle":"2024-08-03T23:46:57.797259Z","shell.execute_reply.started":"2024-08-03T23:46:02.693207Z","shell.execute_reply":"2024-08-03T23:46:57.795975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"For CNN1D scored 310","metadata":{}},{"cell_type":"code","source":"cnn1d_310 = calculate_AP('/kaggle/input/belka1dcnn-0-310-private-based-on-ah-s-notebook/submission.csv','1DCNN_310')","metadata":{"execution":{"iopub.status.busy":"2024-08-03T23:46:57.799692Z","iopub.execute_input":"2024-08-03T23:46:57.800158Z","iopub.status.idle":"2024-08-03T23:47:05.808597Z","shell.execute_reply.started":"2024-08-03T23:46:57.800117Z","shell.execute_reply":"2024-08-03T23:47:05.807293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"outs.append(cnn1d_310)","metadata":{"execution":{"iopub.status.busy":"2024-08-03T23:47:05.810380Z","iopub.execute_input":"2024-08-03T23:47:05.810740Z","iopub.status.idle":"2024-08-03T23:47:05.816250Z","shell.execute_reply.started":"2024-08-03T23:47:05.810711Z","shell.execute_reply":"2024-08-03T23:47:05.814924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"out_df = pd.concat(outs,axis=1).T","metadata":{"execution":{"iopub.status.busy":"2024-08-03T23:47:05.819582Z","iopub.execute_input":"2024-08-03T23:47:05.820014Z","iopub.status.idle":"2024-08-03T23:47:05.833594Z","shell.execute_reply.started":"2024-08-03T23:47:05.819980Z","shell.execute_reply":"2024-08-03T23:47:05.832359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualize the sorted","metadata":{}},{"cell_type":"code","source":"sorted_df = out_df.sort_values('mean_AP',ascending=False)","metadata":{"execution":{"iopub.status.busy":"2024-08-03T23:47:05.834856Z","iopub.execute_input":"2024-08-03T23:47:05.835261Z","iopub.status.idle":"2024-08-03T23:47:05.848347Z","shell.execute_reply.started":"2024-08-03T23:47:05.835226Z","shell.execute_reply":"2024-08-03T23:47:05.846853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sorted_df.style.background_gradient(cmap='Reds',axis=None)","metadata":{"execution":{"iopub.status.busy":"2024-08-03T23:47:05.849996Z","iopub.execute_input":"2024-08-03T23:47:05.850430Z","iopub.status.idle":"2024-08-03T23:47:05.882098Z","shell.execute_reply.started":"2024-08-03T23:47:05.850396Z","shell.execute_reply":"2024-08-03T23:47:05.880392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def sort_specific(cols):\n    data = out_df[cols].copy()\n    data['average'] = data.mean(1)\n    return data.sort_values('average',ascending=False).style.background_gradient(cmap='Reds',axis=None)","metadata":{"execution":{"iopub.status.busy":"2024-08-03T23:54:41.888647Z","iopub.execute_input":"2024-08-03T23:54:41.889229Z","iopub.status.idle":"2024-08-03T23:54:41.898123Z","shell.execute_reply.started":"2024-08-03T23:54:41.889169Z","shell.execute_reply":"2024-08-03T23:54:41.896205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# non share\nsort_specific(['BRD4_non_share','HSA_non_share','sEH_non_share'])","metadata":{"execution":{"iopub.status.busy":"2024-08-03T23:55:01.087882Z","iopub.execute_input":"2024-08-03T23:55:01.088347Z","iopub.status.idle":"2024-08-03T23:55:01.110881Z","shell.execute_reply.started":"2024-08-03T23:55:01.088314Z","shell.execute_reply":"2024-08-03T23:55:01.109501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# share\nsort_specific(['BRD4_share','HSA_share','sEH_share'])","metadata":{"execution":{"iopub.status.busy":"2024-08-03T23:55:25.996413Z","iopub.execute_input":"2024-08-03T23:55:25.996823Z","iopub.status.idle":"2024-08-03T23:55:26.018859Z","shell.execute_reply.started":"2024-08-03T23:55:25.996791Z","shell.execute_reply":"2024-08-03T23:55:26.017360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# non-triazine\nsort_specific(['BRD4_non-triazine','HSA_non-triazine','sEH_non-triazine'])","metadata":{"execution":{"iopub.status.busy":"2024-08-03T23:56:14.284392Z","iopub.execute_input":"2024-08-03T23:56:14.284827Z","iopub.status.idle":"2024-08-03T23:56:14.308817Z","shell.execute_reply.started":"2024-08-03T23:56:14.284791Z","shell.execute_reply":"2024-08-03T23:56:14.307412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def url2path(url):\n#     # Extract the part of the URL after the last '/'\n#     base_name = url.rsplit('/', 1)[-1]\n    \n#     # Construct the new path by appending the extracted name to '/kaggle/input/' and adding '/submission.csv'\n#     new_path = f\"/kaggle/input/{base_name}/submission.csv\"\n    \n#     return new_path\n\n# subs = pd.read_csv(sdfjsdklf)\n# subs['pth'] = subs.url.apply(url2path)\n# outs=[]\n# for r in subs.iterrows():\n#     out = get_score(r['pth'],r['name'])\n#     outs.append(out)","metadata":{"execution":{"iopub.status.busy":"2024-08-03T23:07:49.732657Z","iopub.execute_input":"2024-08-03T23:07:49.733250Z","iopub.status.idle":"2024-08-03T23:07:49.739784Z","shell.execute_reply.started":"2024-08-03T23:07:49.733205Z","shell.execute_reply":"2024-08-03T23:07:49.738153Z"},"trusted":true},"execution_count":null,"outputs":[]}]}