{"cells":[{"metadata":{},"cell_type":"markdown","source":"This notebook is a sample code to calculate metrics.\nSince There are many ambiguous parts in [official evaluation page](https://www.kaggle.com/c/pku-autonomous-driving/overview/evaluation), this is just my understanding of metrics."},{"metadata":{},"cell_type":"markdown","source":"## helper funcs from some kernels"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom math import sqrt, acos, pi, sin, cos\nfrom scipy.spatial.transform import Rotation as R\nfrom sklearn.metrics import average_precision_score\nfrom sklearn.metrics import precision_recall_curve\nfrom inspect import signature\nimport matplotlib.pyplot as plt\nfrom multiprocessing import Pool\n\ndef expand_df(df, PredictionStringCols):\n    df = df.dropna().copy()\n    df['NumCars'] = [int((x.count(' ')+1)/7) for x in df['PredictionString']]\n\n    image_id_expanded = [item for item, count in zip(df['ImageId'], df['NumCars']) for i in range(count)]\n    prediction_strings_expanded = df['PredictionString'].str.split(' ',expand = True).values.reshape(-1,7).astype(float)\n    prediction_strings_expanded = prediction_strings_expanded[~np.isnan(prediction_strings_expanded).all(axis=1)]\n    df = pd.DataFrame(\n        {\n            'ImageId': image_id_expanded,\n            PredictionStringCols[0]:prediction_strings_expanded[:,0],\n            PredictionStringCols[1]:prediction_strings_expanded[:,1],\n            PredictionStringCols[2]:prediction_strings_expanded[:,2],\n            PredictionStringCols[3]:prediction_strings_expanded[:,3],\n            PredictionStringCols[4]:prediction_strings_expanded[:,4],\n            PredictionStringCols[5]:prediction_strings_expanded[:,5],\n            PredictionStringCols[6]:prediction_strings_expanded[:,6]\n        })\n    return df\n\ndef str2coords(s, names):\n    coords = []\n    for l in np.array(s.split()).reshape([-1, 7]):\n        coords.append(dict(zip(names, l.astype('float'))))\n    return coords\n\ndef TranslationDistance(p,g, abs_dist = False):\n    dx = p['x'] - g['x']\n    dy = p['y'] - g['y']\n    dz = p['z'] - g['z']\n    diff0 = (g['x']**2 + g['y']**2 + g['z']**2)**0.5\n    diff1 = (dx**2 + dy**2 + dz**2)**0.5\n    if abs_dist:\n        diff = diff1\n    else:\n        diff = diff1/diff0\n    return diff\n\ndef RotationDistance(p, g):\n    true=[ g['pitch'] ,g['yaw'] ,g['roll'] ]\n    pred=[ p['pitch'] ,p['yaw'] ,p['roll'] ]\n    q1 = R.from_euler('xyz', true)\n    q2 = R.from_euler('xyz', pred)\n    diff = R.inv(q2) * q1\n    W = np.clip(diff.as_quat()[-1], -1., 1.)\n    \n    # in the official metrics code:\n    # https://www.kaggle.com/c/pku-autonomous-driving/overview/evaluation\n    #   return Object3D.RadianToDegree( Math.Acos(diff.W) )\n    # this code treat θ and θ+2π differntly.\n    # So this should be fixed as follows.\n    W = (acos(W)*360)/pi\n    if W > 180:\n        W = 360 - W\n    return W\n\ndef print_pr_curve(result_flg, scores, recall_total=1):\n    average_precision = average_precision_score(result_flg, scores)\n    precision, recall, _ = precision_recall_curve(result_flg, scores)\n    recall *= recall_total\n    plt.step(recall, precision, color='b', alpha=0.2, where='post')\n    plt.fill_between(recall, precision, alpha=0.2, color='b')\n    plt.xlabel('Recall')\n    plt.ylabel('Precision')\n    plt.ylim([0.0, 1.05])\n    plt.xlim([0.0, 1.0])\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## TP, FP calculation\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"thres_tr_list = [0.1, 0.09, 0.08, 0.07, 0.06, 0.05, 0.04, 0.03, 0.02, 0.01]\nthres_ro_list = [50, 45, 40, 35, 30, 25, 20, 15, 10, 5]\n\ndef check_match(idx):\n    keep_gt=False\n    thre_tr_dist = thres_tr_list[idx]\n    thre_ro_dist = thres_ro_list[idx]\n    train_dict = {imgID:str2coords(s, names=['carid_or_score', 'pitch', 'yaw', 'roll', 'x', 'y', 'z']) for imgID,s in zip(train_df['ImageId'],train_df['PredictionString'])}\n    valid_dict = {imgID:str2coords(s, names=['pitch', 'yaw', 'roll', 'x', 'y', 'z', 'carid_or_score']) for imgID,s in zip(valid_df['ImageId'],valid_df['PredictionString'])}\n    result_flg = [] # 1 for TP, 0 for FP\n    scores = []\n    MAX_VAL = 10**10\n    for img_id in valid_dict:\n        for pcar in sorted(valid_dict[img_id], key=lambda x: -x['carid_or_score']):\n            # find nearest GT\n            min_tr_dist = MAX_VAL\n            min_idx = -1\n            for idx, gcar in enumerate(train_dict[img_id]):\n                tr_dist = TranslationDistance(pcar,gcar)\n                if tr_dist < min_tr_dist:\n                    min_tr_dist = tr_dist\n                    min_ro_dist = RotationDistance(pcar,gcar)\n                    min_idx = idx\n                    \n            # set the result\n            if min_tr_dist < thre_tr_dist and min_ro_dist < thre_ro_dist:\n                if not keep_gt:\n                    train_dict[img_id].pop(min_idx)\n                result_flg.append(1)\n            else:\n                result_flg.append(0)\n            scores.append(pcar['carid_or_score'])\n    \n    return result_flg, scores","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## map calculation"},{"metadata":{"trusted":true},"cell_type":"code","source":"validation_prediction = '../input/autonomous-driving-validation-data/prediction_for_validation_data.csv'\n!head -2 $validation_prediction","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This file is a prediction file for validaion data with same format as submision file.\nI used this [kernel](https://www.kaggle.com/hocop1/centernet-baseline) for prediction."},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_df = pd.read_csv(validation_prediction)\nexpanded_valid_df = expand_df(valid_df, ['pitch','yaw','roll','x','y','z','Score'])\nvalid_df = valid_df.fillna('')\n\ntrain_df = pd.read_csv('../input/pku-autonomous-driving/train.csv')\ntrain_df = train_df[train_df.ImageId.isin(valid_df.ImageId.unique())]\n# data description page says, The pose information is formatted as\n# model type, yaw, pitch, roll, x, y, z\n# but it doesn't, and it should be\n# model type, pitch, yaw, roll, x, y, z\nexpanded_train_df = expand_df(train_df, ['model_type','pitch','yaw','roll','x','y','z'])\n\nmax_workers = 10\nn_gt = len(expanded_train_df)\nap_list = []\np = Pool(processes=max_workers)\nfor result_flg, scores in p.imap(check_match, range(10)):\n    if np.sum(result_flg) > 0:\n        n_tp = np.sum(result_flg)\n        recall = n_tp/n_gt\n        ap = average_precision_score(result_flg, scores)*recall\n        print_pr_curve(result_flg, scores, recall)\n    else:\n        ap = 0\n    ap_list.append(ap)\nmap = np.mean(ap_list)\nprint('map:', map)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Average Precision is area under the curve of precision-recall curve (blue area ratio) and mAP is average of these 10 APs."},{"metadata":{},"cell_type":"markdown","source":"Let's see the result without taking FN into account. I guess the previous LB was similar to this."},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_df = pd.read_csv(validation_prediction)\nexpanded_valid_df = expand_df(valid_df, ['pitch','yaw','roll','x','y','z','Score'])\nvalid_df = valid_df.fillna('')\n\ntrain_df = pd.read_csv('../input/pku-autonomous-driving/train.csv')\ntrain_df = train_df[train_df.ImageId.isin(valid_df.ImageId.unique())]\n# data description page says, The pose information is formatted as\n# model type, yaw, pitch, roll, x, y, z\n# but it doesn't, and it should be\n# model type, pitch, yaw, roll, x, y, z\nexpanded_train_df = expand_df(train_df, ['model_type','pitch','yaw','roll','x','y','z'])\n\nmax_workers = 10\nn_gt = len(expanded_train_df)\nap_list = []\np = Pool(processes=max_workers)\nfor result_flg, scores in p.imap(check_match, range(10)):\n    if np.sum(result_flg) > 0:\n        n_tp = np.sum(result_flg)\n        ap = average_precision_score(result_flg, scores)\n        print_pr_curve(result_flg, scores)\n    else:\n        ap = 0\n    ap_list.append(ap)\nmap = np.mean(ap_list)\nprint('map:', map)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# note\n\nThere are some issues for metrics.\n\n - [~~Recall is not considered in Current LB~~](https://www.kaggle.com/c/pku-autonomous-driving/discussion/116661#latest-673344)\n \n - [Confidence score doesn't seem to work](https://www.kaggle.com/c/pku-autonomous-driving/discussion/120580)\n \n - [Is translation thresholds meter or percentage?](https://www.kaggle.com/c/pku-autonomous-driving/discussion/115630#latest-665163)\n\n - Official rotation metrics function has issue.(treating θ and θ+2π differntly)\n \n\nI hope these issues will be cleard ASAP."}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}