{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!pip install lyft-dataset-sdk","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install squaternion","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"from pathlib import Path\nfrom PIL import Image, ImageDraw, ImageFont\nfrom matplotlib import pyplot as plt\nimport matplotlib.patches as patches\nimport pandas as pd\nimport numpy as np\n# Load the SDK\nfrom lyft_dataset_sdk.lyftdataset import LyftDataset, LyftDatasetExplorer\nfrom lyft_dataset_sdk.utils.data_classes import Box, LidarPointCloud, RadarPointCloud \nfrom lyft_dataset_sdk.utils.geometry_utils import BoxVisibility, box_in_image, view_points\nfrom lyft_dataset_sdk.eval.detection.mAP_evaluation import Box3D, recall_precision, get_class_names, get_average_precisions\nimport math\n# from source.utilities import print_progress\nfrom pyquaternion import Quaternion\nimport os\nimport time\nfrom multiprocessing import Process\nfrom squaternion import euler2quat, quat2euler\nfrom tqdm import tqdm_notebook as tqdm\nfrom shapely.geometry import Point\nfrom shapely.geometry.polygon import Polygon","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# gotta do this for LyftDataset SDK, it expects folders to be named as `images`, `maps`, `lidar`\n\n!ln -s /kaggle/input/3d-object-detection-for-autonomous-vehicles/train_images images\n!ln -s /kaggle/input/3d-object-detection-for-autonomous-vehicles/train_maps maps\n!ln -s /kaggle/input/3d-object-detection-for-autonomous-vehicles/train_lidar lidar\n!ln -s /kaggle/input/3d-object-detection-for-autonomous-vehicles/train_data data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"level5data = LyftDataset(data_path='.', json_path='data/', verbose=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/lyft3d-inference-kernel-train-dataset/lyft3d_pred_train.csv')\ntrain_num = len(train)\nprint(train_num)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_animal = pd.read_csv('../input/animal-evaluation/lyft3d_train_pred_animal.csv')\npred_animal.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_motorcycle = pd.read_csv('../input/motorcycle-train-evaluation/lyft3d_train_pred_motorcycle.csv')\npred_motorcycle.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_boxes_list_3d(pred_boxes):\n    if not pd.isna(pred_boxes):\n        pred_boxes = pred_boxes.split(' ')\n        pred_boxes = pred_boxes[:-1]\n        pred_boxes_list = np.reshape(pred_boxes, (len(pred_boxes)//9, 9))\n    else:\n        pred_boxes_list = []\n    return pred_boxes_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_tokens = []\nfor idx in range(train_num):\n    sample_tokens.append(pred_animal.iloc[idx]['Id'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"columns = ['Id', 'PredictionString']\ndf_pred = pd.DataFrame(columns=columns)   ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i, sample_token in tqdm(enumerate(sample_tokens)):\n    row = [] \n    result = ''\n    token = sample_token\n    row.append(token)\n    df = train.loc[train['Id'] == token]\n    pred_str = str(np.array(df['PredictionString'])[0])\n    pred_str_animal = pred_animal.iloc[i]['PredictionString']\n    df = pred_motorcycle.loc[pred_motorcycle['Id'] == token]\n    pred_str_motorcycle = str(np.array(df['PredictionString'])[0])\n    if pd.isna(pred_str_animal):\n        pred_str_animal = ''\n    if pd.isna(pred_str_motorcycle):\n        pred_str_motorcycle = ''   \n    result = pred_str + pred_str_animal + pred_str_motorcycle\n    row.append(result)  \n    df_row = pd.DataFrame([row], columns=columns)\n    df_pred = df_pred.append(df_row)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_pred.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_pred.tail()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_pred.to_csv('train_correct.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_groundtruth_boxes(nuscenes, sample_tokens):\n    gt_box3ds = []\n\n    # Load annotations and filter predictions and annotations.\n    for sample_token in tqdm(sample_tokens):\n\n        sample = nuscenes.get('sample', sample_token)\n        sample_annotation_tokens = sample['anns']\n        \n        for sample_annotation_token in sample_annotation_tokens:\n            sample_annotation = nuscenes.get('sample_annotation', sample_annotation_token)\n            sample_annotation_translation = sample_annotation['translation']\n            \n            class_name = sample_annotation['category_name']\n            \n            box3d = Box3D(\n                sample_token=sample_token,\n                translation=sample_annotation_translation,\n                size=sample_annotation['size'],\n                rotation=sample_annotation['rotation'],\n                name=class_name\n            )\n            gt_box3ds.append(box3d)\n            \n    return gt_box3ds\n\ngt_box3ds = load_groundtruth_boxes(level5data, sample_tokens)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(gt_box3ds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_box3ds = []\npred_box3ds_correct = []\nfor i, sample_token in tqdm(enumerate(sample_tokens)):\n    row = [] \n    token = sample_token\n    df = train.loc[train['Id'] == token]\n    pred_boxes = str(np.array(df['PredictionString'])[0])\n    pred_boxes_list_3d = get_boxes_list_3d(pred_boxes)\n    for j, pred_box in enumerate(pred_boxes_list_3d):\n        cls_conf, x, y, z, w, l, h, yaw, cls_pred = pred_box\n        cls_conf = float(cls_conf)\n        x = float(x)\n        y = float(y)\n        z = float(z)\n        w = float(w)\n        h = float(h)\n        l = float(l)\n        yaw = float(yaw)\n        q = euler2quat(0, 0, yaw)\n        box3d = Box3D(\n                    sample_token=token,\n                    translation=[x, y, z],\n                    size=[w, l, h],\n                    rotation=[q[0],q[1],q[2],q[3]],\n                    name=cls_pred,\n                    score=cls_conf\n                    )\n        pred_box3ds.append(box3d)\n    pred_boxes_correct = df_pred.iloc[i]['PredictionString']\n    pred_boxes_list_3d_correct = get_boxes_list_3d(pred_boxes_correct)  \n    for j, pred_box in enumerate(pred_boxes_list_3d_correct):\n        cls_conf, x, y, z, w, l, h, yaw, cls_pred = pred_box\n        cls_conf = float(cls_conf)\n        x = float(x)\n        y = float(y)\n        z = float(z)\n        w = float(w)\n        h = float(h)\n        l = float(l)\n        yaw = float(yaw)\n        q = euler2quat(0, 0, yaw)\n        box3d_correct = Box3D(\n                    sample_token=token,\n                    translation=[x, y, z],\n                    size=[w, l, h],\n                    rotation=[q[0],q[1],q[2],q[3]],\n                    name=cls_pred,\n                    score=cls_conf\n                    )\n        pred_box3ds_correct.append(box3d_correct)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(pred_box3ds), len(pred_box3ds_correct))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(pred_box3ds[0], pred_box3ds_correct[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import json\nARTIFACTS_FOLDER = \"./artifacts\"\nos.makedirs(ARTIFACTS_FOLDER, exist_ok=True)\ngt = [b.serialize() for b in gt_box3ds[:10000]]\npr = [b.serialize() for b in pred_box3ds[:10000]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gt[0], pr[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"iou_th_range = np.linspace(0.5, 0.95, 10)\nmetric = {}\nprocesses = []\noutput_dir = 'tmp/'\noutput_dir = Path(output_dir)\noutput_dir.mkdir(parents=True, exist_ok=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class_names = ['animal', 'bicycle', 'bus', 'car', 'emergency_vehicle',\n                    'motorcycle', 'other_vehicle', 'pedestrian', 'truck']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def save_AP(gt, predictions, class_names, iou_threshold, output_dir):\n    #computes average precisions (AP) for a given threshold, and saves the metrics in a temp file \n    # use lyft's provided function to compute AP\n    AP = get_average_precisions(gt, predictions, class_names, iou_threshold)\n    # create a dict with keys as class names and values as their respective APs\n    metric = {c:AP[idx] for idx, c in enumerate(class_names)}\n\n    # save the dict in a temp file\n    summary_path = str(output_dir) + f'metric_summary_{iou_threshold}.json'\n    with open(str(summary_path), 'w') as f:\n        json.dump(metric, f)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_metric_overall_AP(iou_th_range, output_dir, class_names):\n    ''' reads temp files and calculates overall per class APs.\n    returns:\n        `metric`: a dict with key as iou thresholds and value as dicts of class and their respective APs,\n        `overall_AP`: overall AP of each class\n    '''\n\n    metric = {}\n    overall_AP = np.zeros(len(class_names))\n    for iou_threshold in iou_th_range:\n        summary_path = str(output_dir) + f'metric_summary_{iou_threshold}.json'\n        with open(str(summary_path), 'r') as f:\n            data = json.load(f) # type(data): dict\n            metric[iou_threshold] = data\n            overall_AP += np.array([data[c] for c in class_names])\n    overall_AP /= len(iou_th_range)\n    return metric, overall_AP","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"time_start = time.time()\nfor iou_threshold in iou_th_range:\n    process = Process(target=save_AP, args=(gt, pr, class_names, iou_threshold, output_dir))\n    process.start()\n    processes.append(process)\n\nfor process in processes:\n    process.join()\nprint(\"Time to evaluate = \", time.time() - time_start)      ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get overall metrics\nmetric, overall_AP = get_metric_overall_AP(iou_th_range, output_dir, class_names)\nmetric['overall'] = {c: overall_AP[idx] for idx, c in enumerate(class_names)}\nmetric['mAP'] = np.mean(overall_AP)\nfor th in iou_th_range:\n    print(\"IOU threshold = \", th)\n    average_precisions = list(metric[th].values())\n    mAP = np.mean(average_precisions)\n    print(\"Average per class mean average precision = \", mAP)\n    for class_id in sorted(list(zip(class_names, average_precisions))):\n        print(class_id)\n    print(\"_______________________________________________\")\nprint(\"Overall mean average precision = \", metric['mAP'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"time_start = time.time()\naverage_precisions = get_average_precisions(gt, pr, class_names, 0.01)\nmAP = np.mean(average_precisions)\nprint(\"Average per class mean average precision = \", mAP)\nfor class_id in sorted(list(zip(class_names, average_precisions.flatten().tolist()))):\n    print(class_id)\nprint(\"Time to evaluate = \", time.time() - time_start)  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pr_corr = [b.serialize() for b in pred_box3ds_correct[:10000]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"time_start = time.time()\nfor iou_threshold in iou_th_range:\n    process = Process(target=save_AP, args=(gt, pr_corr, class_names, iou_threshold, output_dir))\n    process.start()\n    processes.append(process)\n\nfor process in processes:\n    process.join()\nprint(\"Time to evaluate = \", time.time() - time_start)      ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get overall metrics\nmetric, overall_AP = get_metric_overall_AP(iou_th_range, output_dir, class_names)\nmetric['overall'] = {c: overall_AP[idx] for idx, c in enumerate(class_names)}\nmetric['mAP'] = np.mean(overall_AP)\nfor th in iou_th_range:\n    print(\"IOU threshold = \", th)\n    average_precisions = list(metric[th].values())\n    mAP = np.mean(average_precisions)\n    print(\"Average per class mean average precision = \", mAP)\n    for class_id in sorted(list(zip(class_names, average_precisions))):\n        print(class_id)\n    print(\"_______________________________________________\")\nprint(\"Overall mean average precision = \", metric['mAP'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"time_start = time.time()\naverage_precisions = get_average_precisions(gt, pr_corr, class_names, 0.01)\nmAP = np.mean(average_precisions)\nprint(\"Average per class mean average precision = \", mAP)\nfor class_id in sorted(list(zip(class_names, average_precisions.flatten().tolist()))):\n    print(class_id)\nprint(\"Time to evaluate = \", time.time() - time_start)  ","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}