{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Version details: compared to OTTO [Covisitation CPU | Test]\n- V6 ~ V2 Test: CV: 0.56428 | LB: 0.574\n- V7: CV 0.56439, add top common aids for each type\n- V12: first attempt 4covi\n- V14: same as LB 0.575\n- V15: ~ V12 with top_output=50\n- V17: ~ V12 with delta_ts_buy = 14days\n- V18: ~ V17, top_output=50\n- V23,24,25: ~ V12 with top_output=100\n- V26: new pipeline for generating candidates\n- V28,V29: ~ V26, the order of aids2,aids3 -> V30 merged\n- V31: ~V30, top_output=150\n- V32: ~V30, top_output=200\n- V36: ~V30, add top common aid in test","metadata":{}},{"cell_type":"code","source":"!pip install pyarrow fastparquet","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEBUG = False\n\n# --- OTTO [Build metadata] notebook params\nTOP_K = 20\nTOP_OUTPUT = 100\nLEAK_DATA = True\nDELTA_TS = 24 # hours\nDISK_PIECES = 4 \nSIZE = 1.86e6/DISK_PIECES # total sessions\n# ---\n\nmetadata_path = \"/kaggle/input/otto-covisitation-validation\"\ntype_labels = {'clicks':0, 'carts':1, 'orders':2}","metadata":{"execution":{"iopub.status.busy":"2023-01-25T13:21:29.811223Z","iopub.execute_input":"2023-01-25T13:21:29.812196Z","iopub.status.idle":"2023-01-25T13:21:29.851383Z","shell.execute_reply.started":"2023-01-25T13:21:29.812081Z","shell.execute_reply":"2023-01-25T13:21:29.850097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd, numpy as np\nimport copy\nfrom tqdm.notebook import tqdm\nimport os, sys, pickle, glob, gc\nfrom collections import Counter\nimport itertools\n\nfrom multiprocessing import Pool\nimport psutil\nN_CPU = psutil.cpu_count()\nprint(\"Number of cpu:\", N_CPU)","metadata":{"execution":{"iopub.status.busy":"2023-01-25T13:21:29.853303Z","iopub.execute_input":"2023-01-25T13:21:29.854019Z","iopub.status.idle":"2023-01-25T13:21:29.982207Z","shell.execute_reply.started":"2023-01-25T13:21:29.853976Z","shell.execute_reply":"2023-01-25T13:21:29.981005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def df_parallelize_run(func, t_split):\n    \n    num_cores = np.min([N_CPU, len(t_split)])\n    pool = Pool(num_cores)\n    df = pool.map(func, t_split)\n    pool.close()\n    pool.join()\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2023-01-25T13:21:29.984906Z","iopub.execute_input":"2023-01-25T13:21:29.985945Z","iopub.status.idle":"2023-01-25T13:21:29.992174Z","shell.execute_reply.started":"2023-01-25T13:21:29.985895Z","shell.execute_reply":"2023-01-25T13:21:29.990854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pqt_to_dict(df):\n#     if USE_FREQUENCY_SCORE:\n#         return df.groupby('aid_x').apply(lambda df: Counter(dict(zip(df.aid_y, df.wgt))))\n    return df.groupby('aid_x').aid_y.apply(list).to_dict()","metadata":{"execution":{"iopub.status.busy":"2023-01-25T13:21:29.995297Z","iopub.execute_input":"2023-01-25T13:21:29.995706Z","iopub.status.idle":"2023-01-25T13:21:30.005446Z","shell.execute_reply.started":"2023-01-25T13:21:29.995672Z","shell.execute_reply":"2023-01-25T13:21:30.004383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data","metadata":{}},{"cell_type":"code","source":"def load_test(path):    \n    dfs = []\n    for e, chunk_file in sorted(enumerate(glob.glob(path))):\n        chunk = pd.read_parquet(chunk_file)\n        chunk.ts = (chunk.ts/1000).astype('int32')\n        chunk['type'] = chunk['type'].map(type_labels).astype('int8')\n        dfs.append(chunk)\n    return pd.concat(dfs).reset_index(drop=True) #.astype({\"ts\": \"datetime64[ms]\"})\n\ntest_df = load_test('../input/otto-validation/test_parquet/*')\nprint('Test data has shape',test_df.shape)\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-25T13:21:30.007036Z","iopub.execute_input":"2023-01-25T13:21:30.007724Z","iopub.status.idle":"2023-01-25T13:21:33.073534Z","shell.execute_reply.started":"2023-01-25T13:21:30.007669Z","shell.execute_reply":"2023-01-25T13:21:33.071702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nPIECES = 5\nVER = 1\ntest_bysession_list = []\nfor PART in range(PIECES):\n    with open(f'../input/otto-valid-test-list/valid_group_tolist_{PART}_{VER}.pkl', 'rb') as f:\n        test_bysession_list.extend(pickle.load(f))\nprint(len(test_bysession_list))","metadata":{"execution":{"iopub.status.busy":"2023-01-25T13:21:33.075696Z","iopub.execute_input":"2023-01-25T13:21:33.077820Z","iopub.status.idle":"2023-01-25T13:21:44.530962Z","shell.execute_reply.started":"2023-01-25T13:21:33.077738Z","shell.execute_reply":"2023-01-25T13:21:44.529360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nTYPE_COO = \"fulltype\"\nDELTA_TS = 24\nOUTPUT_NAME = f\"top_{TOP_K}_{TYPE_COO}_{DELTA_TS}hours\"\ntry:\n    assert USE_PICKLE_COVI == True\n    with open(f'/kaggle/input/otto-covisitation-validation/{OUTPUT_NAME}_full.pickle', 'rb') as file:\n        top_fulltype = pickle.load(file)\n    print(f\"Load {TYPE_COO} covisitaion | dict format\")\nexcept:\n    print(f\"Load {TYPE_COO} covisitaion | dataframe format\")\n    top_fulltype = pqt_to_dict(pd.read_parquet(f'/kaggle/input/otto-covisitation-validation/{OUTPUT_NAME}_full.pqt'))","metadata":{"execution":{"iopub.status.busy":"2023-01-25T13:21:44.532705Z","iopub.execute_input":"2023-01-25T13:21:44.533243Z","iopub.status.idle":"2023-01-25T13:22:29.140305Z","shell.execute_reply.started":"2023-01-25T13:21:44.533187Z","shell.execute_reply":"2023-01-25T13:22:29.138507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"top_clicks = list(test_df.loc[test_df['type']== 0,'aid'].value_counts().index.values[:TOP_OUTPUT]) \ntop_carts = list(test_df.loc[test_df['type']== 1,'aid'].value_counts().index.values[:TOP_OUTPUT])\ntop_orders = list(test_df.loc[test_df['type']== 2,'aid'].value_counts().index.values[:TOP_OUTPUT])\ndel test_df\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-01-25T13:22:29.142251Z","iopub.execute_input":"2023-01-25T13:22:29.142710Z","iopub.status.idle":"2023-01-25T13:22:31.893107Z","shell.execute_reply.started":"2023-01-25T13:22:29.142672Z","shell.execute_reply":"2023-01-25T13:22:31.891856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#type_weight_multipliers = {'clicks': 1, 'carts': 6, 'orders': 3}\ntype_weight_multipliers = {0: 0.5, 1: 9, 2: 0.5}\n\ndef suggest_clicks(df):\n    session = df[0]\n    aids = df[1]\n    types = df[2]\n    unique_aids = list(dict.fromkeys(aids[::-1]))\n    \n    # history candidates\n    weights=np.logspace(0.1,1,len(aids),base=2, endpoint=True)-1\n    aids_temp = Counter() \n    # RERANK BASED ON REPEAT ITEMS AND TYPE OF ITEMS\n    for aid,w,t in zip(aids,weights,types): \n        aids_temp[aid] += w * type_weight_multipliers[t]\n\n    sorted_aids = []\n    weights = []\n    type_cands = []\n    \n    if len(unique_aids) >= TOP_OUTPUT:\n        for aid, cnt in aids_temp.most_common(TOP_OUTPUT):\n            sorted_aids.append(aid)\n            weights.append(cnt)\n            type_cands.append(1)\n        return session, sorted_aids, weights, type_cands\n    else:\n        for aid, cnt in aids_temp.most_common(len(unique_aids)):\n            sorted_aids.append(aid)\n            weights.append(cnt)\n            type_cands.append(1)\n            \n    # potential candidates\n    aids2 = list(itertools.chain(*[top_click[aid] for aid in unique_aids if aid in top_click]))\n    aids3 = list(itertools.chain(*[top_fulltype[aid] for aid in unique_aids if aid in top_fulltype]))\n    \n    top_aids2 = Counter(aids2+aids3)\n    top_aids2 = [(aid2,cnt) for aid2, cnt in top_aids2.most_common(TOP_OUTPUT) if aid2 not in unique_aids]\n    for aid, cnt in top_aids2[:TOP_OUTPUT - len(unique_aids)]:\n        sorted_aids.append(aid)\n        weights.append(cnt)\n        type_cands.append(0)\n        \n    if len(sorted_aids) < TOP_OUTPUT:\n        top_common_aid = [aid for aid in top_clicks if aid not in sorted_aids]\n        for aid in top_common_aid[:TOP_OUTPUT - len(sorted_aids)]:\n            sorted_aids.append(aid)\n            weights.append(0)\n            type_cands.append(2)\n        \n    return session, sorted_aids, weights, type_cands\n\ndef suggest_carts(df):\n    # USE USER HISTORY AIDS AND TYPES\n    session = df[0]\n    aids = df[1]\n    types = df[2]\n    unique_aids = list(dict.fromkeys(aids[::-1]))\n\n    # history candidates\n    weights=np.logspace(0.5,1,len(aids),base=2, endpoint=True)-1\n    aids_temp = Counter() \n    # RERANK BASED ON REPEAT ITEMS AND TYPE OF ITEMS\n    for aid,w,t in zip(aids,weights,types): \n        aids_temp[aid] += w * type_weight_multipliers[t]\n\n    sorted_aids = []\n    weights = []\n    type_cands = []\n    \n    if len(unique_aids) >= TOP_OUTPUT:\n        for aid, cnt in aids_temp.most_common(TOP_OUTPUT):\n            sorted_aids.append(aid)\n            weights.append(cnt)\n            type_cands.append(1)\n        return session, sorted_aids, weights, type_cands\n    else:\n        for aid, cnt in aids_temp.most_common(len(unique_aids)):\n            sorted_aids.append(aid)\n            weights.append(cnt)\n            type_cands.append(1)\n            \n    # potential candidates\n    aids2 = list(itertools.chain(*[top_cart[aid] for aid in unique_aids if aid in top_cart]))\n    aids3 = list(itertools.chain(*[top_fulltype[aid] for aid in unique_aids if aid in top_fulltype]))\n    \n    top_aids2 = Counter(aids3+aids2)\n    top_aids2 = [(aid2,cnt) for aid2, cnt in top_aids2.most_common(TOP_OUTPUT) if aid2 not in unique_aids]\n    for aid, cnt in top_aids2[:TOP_OUTPUT - len(unique_aids)]:\n        sorted_aids.append(aid)\n        weights.append(cnt)\n        type_cands.append(0)\n        \n    if len(sorted_aids) < TOP_OUTPUT:\n        top_common_aid = [aid for aid in top_carts if aid not in sorted_aids]\n        for aid in top_common_aid[:TOP_OUTPUT - len(sorted_aids)]:\n            sorted_aids.append(aid)\n            weights.append(0)\n            type_cands.append(2)\n            \n    return session, sorted_aids, weights, type_cands\n\n\ndef suggest_orders(df):\n    # USE USER HISTORY AIDS AND TYPES\n    session = df[0]\n    aids = df[1]\n    types = df[2]\n    unique_aids = list(dict.fromkeys(aids[::-1] ))\n    \n    # history candidates\n    weights=np.logspace(0.5,1,len(aids),base=2, endpoint=True)-1\n    aids_temp = Counter() \n    # RERANK BASED ON REPEAT ITEMS AND TYPE OF ITEMS\n    for aid,w,t in zip(aids,weights,types): \n        aids_temp[aid] += w * type_weight_multipliers[t]\n\n    sorted_aids = []\n    weights = []\n    type_cands = []\n    \n    if len(unique_aids) >= TOP_OUTPUT:\n        for aid, cnt in aids_temp.most_common(TOP_OUTPUT):\n            sorted_aids.append(aid)\n            weights.append(cnt)\n            type_cands.append(1)\n        return session, sorted_aids, weights, type_cands\n    else:\n        for aid, cnt in aids_temp.most_common(len(unique_aids)):\n            sorted_aids.append(aid)\n            weights.append(cnt)\n            type_cands.append(1)\n            \n    # potential candidates\n    aids2 = list(itertools.chain(*[top_purchase[aid] for aid in unique_aids if aid in top_purchase]))\n    aids3 = list(itertools.chain(*[top_fulltype[aid] for aid in unique_aids if aid in top_fulltype]))\n    \n    top_aids2 = Counter(aids3+aids2)\n    top_aids2 = [(aid2,cnt) for aid2, cnt in top_aids2.most_common(TOP_OUTPUT) if aid2 not in unique_aids]\n    for aid, cnt in top_aids2[:TOP_OUTPUT - len(unique_aids)]:\n        sorted_aids.append(aid)\n        weights.append(cnt)\n        type_cands.append(0)\n    \n    if len(sorted_aids) < TOP_OUTPUT:\n        top_common_aid = [aid for aid in top_orders if aid not in sorted_aids]\n        for aid in top_common_aid[:TOP_OUTPUT - len(sorted_aids)]:\n            sorted_aids.append(aid)\n            weights.append(0)\n            type_cands.append(2)\n            \n    return session, sorted_aids, weights, type_cands\n","metadata":{"execution":{"iopub.status.busy":"2023-01-25T13:22:31.895173Z","iopub.execute_input":"2023-01-25T13:22:31.895654Z","iopub.status.idle":"2023-01-25T13:22:31.929595Z","shell.execute_reply.started":"2023-01-25T13:22:31.895607Z","shell.execute_reply":"2023-01-25T13:22:31.928269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predic click","metadata":{}},{"cell_type":"code","source":"%%time\nTYPE_COO = \"click\"\nDELTA_TS = 24\nOUTPUT_NAME = f\"top_{TOP_K}_{TYPE_COO}_{DELTA_TS}hours\"\ntry:\n    assert USE_PICKLE_COVI == True\n    with open(f'/kaggle/input/otto-covisitation-validation/{OUTPUT_NAME}_full.pickle', 'rb') as file:\n        top_click = pickle.load(file)\n    print(f\"Load {TYPE_COO} covisitaion | dict format\")\nexcept:\n    print(f\"Load {TYPE_COO} covisitaion | dataframe format\")\n    top_click = pqt_to_dict(pd.read_parquet(f'/kaggle/input/otto-covisitation-validation/{OUTPUT_NAME}_full.pqt'))","metadata":{"execution":{"iopub.status.busy":"2023-01-25T13:22:31.933631Z","iopub.execute_input":"2023-01-25T13:22:31.934998Z","iopub.status.idle":"2023-01-25T13:22:53.684322Z","shell.execute_reply.started":"2023-01-25T13:22:31.934951Z","shell.execute_reply":"2023-01-25T13:22:53.683045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\n# # Predict on all sessions in parallel\ntemp = df_parallelize_run(suggest_clicks, test_bysession_list)\nsession, aids, weights, type_cands = [], [], [], []\nfor _session, _aids, _weights, _type in temp:\n    session.extend([_session]*len(_aids))\n    aids.extend(_aids)\n    weights.extend(_weights)\n    type_cands.extend(_type)\n    \ncandidates_df = pd.DataFrame({'session':session,'aid':aids,'score':weights,'type_candidate':type_cands})\ncandidates_df.to_parquet('/kaggle/working/clicks_candidates.pqt')","metadata":{"execution":{"iopub.status.busy":"2023-01-25T13:22:53.685633Z","iopub.execute_input":"2023-01-25T13:22:53.686023Z","iopub.status.idle":"2023-01-25T13:27:26.940371Z","shell.execute_reply.started":"2023-01-25T13:22:53.685988Z","shell.execute_reply":"2023-01-25T13:27:26.937855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del top_click, temp, top_clicks, candidates_df\ndel session, aids, weights, type_cands\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-01-25T13:27:26.943740Z","iopub.execute_input":"2023-01-25T13:27:26.944233Z","iopub.status.idle":"2023-01-25T13:27:38.677667Z","shell.execute_reply.started":"2023-01-25T13:27:26.944152Z","shell.execute_reply":"2023-01-25T13:27:38.676435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predict cart","metadata":{}},{"cell_type":"code","source":"%%time\nTYPE_COO = \"cart\"\nDELTA_TS = 24*7\nOUTPUT_NAME = f\"top_{TOP_K}_{TYPE_COO}_{DELTA_TS}hours\"\ntry:\n    assert USE_PICKLE_COVI == True\n    with open(f'/kaggle/input/otto-covisitation-validation/{OUTPUT_NAME}_full.pickle', 'rb') as file:\n        top_cart = pickle.load(file)\n    print(f\"Load {TYPE_COO} covisitaion | dict format\")\nexcept:\n    print(f\"Load {TYPE_COO} covisitaion | dataframe format\")\n    top_cart = pqt_to_dict(pd.read_parquet(f'/kaggle/input/otto-covisitation-validation/{OUTPUT_NAME}_full.pqt'))","metadata":{"execution":{"iopub.status.busy":"2023-01-25T13:27:38.679625Z","iopub.execute_input":"2023-01-25T13:27:38.680046Z","iopub.status.idle":"2023-01-25T13:28:21.389093Z","shell.execute_reply.started":"2023-01-25T13:27:38.680007Z","shell.execute_reply":"2023-01-25T13:28:21.387980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\n# # Predict on all sessions in parallel\ntemp = df_parallelize_run(suggest_carts, test_bysession_list)\nsession, aids, weights, type_cands = [], [], [], []\nfor _session, _aids, _weights, _type in temp:\n    session.extend([_session]*len(_aids))\n    aids.extend(_aids)\n    weights.extend(_weights)\n    type_cands.extend(_type)\n    \ncandidates_df = pd.DataFrame({'session':session,'aid':aids,'score':weights,'type_candidate':type_cands})\ncandidates_df.to_parquet('/kaggle/working/carts_candidates.pqt')","metadata":{"execution":{"iopub.status.busy":"2023-01-25T13:28:21.390522Z","iopub.execute_input":"2023-01-25T13:28:21.390931Z","iopub.status.idle":"2023-01-25T13:30:52.562408Z","shell.execute_reply.started":"2023-01-25T13:28:21.390886Z","shell.execute_reply":"2023-01-25T13:30:52.560225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del top_cart, temp, top_carts, candidates_df\ndel session, aids, weights, type_cands\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-01-25T13:30:52.565888Z","iopub.execute_input":"2023-01-25T13:30:52.566572Z","iopub.status.idle":"2023-01-25T13:30:55.113978Z","shell.execute_reply.started":"2023-01-25T13:30:52.566510Z","shell.execute_reply":"2023-01-25T13:30:55.111418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predict order","metadata":{}},{"cell_type":"code","source":"%%time\nTYPE_COO = \"purchase\"\nDELTA_TS = 24*7\nOUTPUT_NAME = f\"top_{TOP_K}_{TYPE_COO}_{DELTA_TS}hours\"\ntry:\n    assert USE_PICKLE_COVI == True\n    with open(f'/kaggle/input/otto-covisitation-validation/{OUTPUT_NAME}_full.pickle', 'rb') as file:\n        top_purchase = pickle.load(file)\n    print(f\"Load {TYPE_COO} covisitaion | dict format\")\nexcept:\n    print(f\"Load {TYPE_COO} covisitaion | dataframe format\")\n    top_purchase = pqt_to_dict(pd.read_parquet(f'/kaggle/input/otto-covisitation-validation/{OUTPUT_NAME}_full.pqt'))","metadata":{"execution":{"iopub.status.busy":"2023-01-25T13:30:55.115500Z","iopub.status.idle":"2023-01-25T13:30:55.116305Z","shell.execute_reply.started":"2023-01-25T13:30:55.116039Z","shell.execute_reply":"2023-01-25T13:30:55.116068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\n# # Predict on all sessions in parallel\ntemp = df_parallelize_run(suggest_orders, test_bysession_list)\nsession, aids, weights, type_cands = [], [], [], []\nfor _session, _aids, _weights, _type in temp:\n    session.extend([_session]*len(_aids))\n    aids.extend(_aids)\n    weights.extend(_weights)\n    type_cands.extend(_type)\n    \ncandidates_df = pd.DataFrame({'session':session,'aid':aids,'score':weights,'type_candidate':type_cands})\ncandidates_df.to_parquet('/kaggle/working/orders_candidates.pqt')","metadata":{"execution":{"iopub.status.busy":"2023-01-25T13:30:55.118557Z","iopub.status.idle":"2023-01-25T13:30:55.119493Z","shell.execute_reply.started":"2023-01-25T13:30:55.119219Z","shell.execute_reply":"2023-01-25T13:30:55.119250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del top_purchase, temp, top_orders, candidates_df\ndel session, aids, weights, type_cands\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-01-25T13:30:55.121219Z","iopub.status.idle":"2023-01-25T13:30:55.121699Z","shell.execute_reply.started":"2023-01-25T13:30:55.121484Z","shell.execute_reply":"2023-01-25T13:30:55.121508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Compute metric","metadata":{}},{"cell_type":"code","source":"# %%time\n# COMPUTE METRIC\nfull_test_labels = pd.read_parquet('/kaggle/input/otto-validation/test_labels.parquet')\n\ntypes = ['clicks','carts','orders']\ncandidates = dict()\nfor _type in types:\n    cans = pd.read_parquet(f\"/kaggle/working/{_type}_candidates.pqt\")\n    candidates[_type] = cans[['session','aid']].groupby('session').agg({'aid': lambda x: list(x)}).reset_index().rename(columns={'aid':'labels'})\n    del cans\n    gc.collect()\n    \nscore = 0\nweights = {'clicks': 0.10, 'carts': 0.30, 'orders': 0.60}\n\nfor t in types:\n    sub = candidates[t]\n    test_labels = full_test_labels.loc[full_test_labels['type']==t]\n    test_labels = test_labels.merge(sub, how='left', on=['session'])\n    test_labels['hits'] = test_labels.apply(lambda df: len(set(df.ground_truth).intersection(set(df.labels))), axis=1)\n    test_labels['gt_count'] = test_labels.ground_truth.str.len().clip(0,20)\n\n    recall = test_labels['hits'].sum() / test_labels['gt_count'].sum()\n    score += weights[t]*recall\n    print(f'{t} recall =',recall)\n\nprint('=============')\nprint('Overall Recall =',score)\nprint('=============')","metadata":{"execution":{"iopub.status.busy":"2023-01-25T13:30:55.124119Z","iopub.status.idle":"2023-01-25T13:30:55.124600Z","shell.execute_reply.started":"2023-01-25T13:30:55.124377Z","shell.execute_reply":"2023-01-25T13:30:55.124407Z"},"trusted":true},"execution_count":null,"outputs":[]}]}