{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# About\nTo understand Competition Metric \"DFL Event Detection AP\", I prepared some simple label data and some predicted results.<br>\nI hope this will help for you! <br>\n\n## references\nhttps://www.kaggle.com/code/hidebu/exploration-of-competition-metric-dfl-ap-jp-en","metadata":{}},{"cell_type":"markdown","source":"## Competition metric functions","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom pandas.testing import assert_index_equal\nfrom typing import Dict, Tuple\n\ntolerances = {\n    \"challenge\": [0.3, 0.4, 0.5, 0.6, 0.7],\n    \"play\": [0.15, 0.20, 0.25, 0.30, 0.35],\n    \"throwin\": [0.15, 0.20, 0.25, 0.30, 0.35],\n}\n            \n\ndef filter_detections(\n        detections: pd.DataFrame, intervals: pd.DataFrame\n) -> pd.DataFrame:\n    \"\"\"Drop detections not inside a scoring interval.\"\"\"\n    detection_time = detections.loc[:, 'time'].sort_values().to_numpy()\n    intervals = intervals.to_numpy()\n    is_scored = np.full_like(detection_time, False, dtype=bool)\n\n    i, j = 0, 0\n    while i < len(detection_time) and j < len(intervals):\n        time = detection_time[i]\n        int_ = intervals[j]\n\n        # If the detection is prior in time to the interval, go to the next detection.\n        if time < int_.left:\n            i += 1\n        # If the detection is inside the interval, keep it and go to the next detection.        \n        elif time in int_:\n            is_scored[i] = True\n            i += 1\n        # If the detection is later in time, go to the next interval.\n        else:\n            j += 1\n\n    return detections.loc[is_scored].reset_index(drop=True)\n\n\ndef match_detections(\n        tolerance: float, ground_truths: pd.DataFrame, detections: pd.DataFrame\n) -> pd.DataFrame:\n    \"\"\"Match detections to ground truth events. Arguments are taken from a common event x tolerance x video evaluation group.\"\"\"\n    detections_sorted = detections.sort_values('score', ascending=False).dropna()\n\n    is_matched = np.full_like(detections_sorted['event'], False, dtype=bool)\n    gts_matched = set()\n    for i, det in enumerate(detections_sorted.itertuples(index=False)):\n        best_error = tolerance\n        best_gt = None\n\n        for gt in ground_truths.itertuples(index=False):\n            error = abs(det.time - gt.time)\n            if error < best_error and not gt in gts_matched:\n                best_gt = gt\n                best_error = error\n            \n        if best_gt is not None:\n            is_matched[i] = True\n            gts_matched.add(best_gt)\n\n    detections_sorted['matched'] = is_matched\n\n    return detections_sorted\n\n\ndef precision_recall_curve(\n        matches: np.ndarray, scores: np.ndarray, p: int\n) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:\n    if len(matches) == 0:\n        return [1], [0], []\n\n    # Sort matches by decreasing confidence\n    idxs = np.argsort(scores, kind='stable')[::-1]\n    scores = scores[idxs]\n    matches = matches[idxs]\n    \n    distinct_value_indices = np.where(np.diff(scores))[0]\n    threshold_idxs = np.r_[distinct_value_indices, matches.size - 1]\n    thresholds = scores[threshold_idxs]\n    \n    # Matches become TPs and non-matches FPs as confidence threshold decreases\n    tps = np.cumsum(matches)[threshold_idxs]\n    fps = np.cumsum(~matches)[threshold_idxs]\n    \n    precision = tps / (tps + fps)\n    precision[np.isnan(precision)] = 0\n    recall = tps / p  # total number of ground truths might be different than total number of matches\n    \n    # Stop when full recall attained and reverse the outputs so recall is non-increasing.\n    last_ind = tps.searchsorted(tps[-1])\n    sl = slice(last_ind, None, -1)\n\n    # Final precision is 1 and final recall is 0\n    return np.r_[precision[sl], 1], np.r_[recall[sl], 0], thresholds[sl]\n\n\ndef average_precision_score(matches: np.ndarray, scores: np.ndarray, p: int) -> float:\n    precision, recall, thresholds = precision_recall_curve(matches, scores, p)\n    # Compute step integral\n    return -np.sum(np.diff(recall) * np.array(precision)[:-1])\n\n\ndef event_detection_ap(\n        solution: pd.DataFrame,\n        submission: pd.DataFrame,\n        tolerances: Dict[str, float],\n) -> float:\n\n    assert_index_equal(solution.columns, pd.Index(['video_id', 'time', 'event']))\n    assert_index_equal(submission.columns, pd.Index(['video_id', 'time', 'event', 'score']))\n\n    # Extract scoring intervals.\n    intervals = (\n        solution\n        .query(\"event in ['start', 'end']\")\n        .assign(interval=lambda x: x.groupby(['video_id', 'event']).cumcount())\n        .pivot(index='interval', columns=['video_id', 'event'], values='time')\n        .stack('video_id')\n        .swaplevel()\n        .sort_index()\n        .loc[:, ['start', 'end']]\n        .apply(lambda x: pd.Interval(*x, closed='both'), axis=1)\n    )\n\n    # Extract ground-truth events.\n    ground_truths = (\n        solution\n        .query(\"event not in ['start', 'end']\")\n        .reset_index(drop=True)\n    )\n\n    # Map each event class to its prevalence (needed for recall calculation)\n    class_counts = ground_truths.value_counts('event').to_dict()\n\n    # Create table for detections with a column indicating a match to a ground-truth event\n    detections = submission.assign(matched = False)\n\n    # Remove detections outside of scoring intervals\n    detections_filtered = []\n    for (det_group, dets), (int_group, ints) in zip(\n        detections.groupby('video_id'), intervals.groupby('video_id')\n    ):\n        assert det_group == int_group\n        detections_filtered.append(filter_detections(dets, ints))\n    detections_filtered = pd.concat(detections_filtered, ignore_index=True)\n\n    # Create table of event-class x tolerance x video_id values\n    aggregation_keys = pd.DataFrame(\n        [(ev, tol, vid)\n         for ev in tolerances.keys()\n         for tol in tolerances[ev]\n         for vid in ground_truths['video_id'].unique()],\n        columns=['event', 'tolerance', 'video_id'],\n    )\n\n    # Create match evaluation groups: event-class x tolerance x video_id\n    detections_grouped = (\n        aggregation_keys\n        .merge(detections_filtered, on=['event', 'video_id'], how='left')\n        .groupby(['event', 'tolerance', 'video_id'])\n    )\n    ground_truths_grouped = (\n        aggregation_keys\n        .merge(ground_truths, on=['event', 'video_id'], how='left')\n        .groupby(['event', 'tolerance', 'video_id'])\n    )\n    \n    # Match detections to ground truth events by evaluation group\n    detections_matched = []\n    for key in aggregation_keys.itertuples(index=False):\n        dets = detections_grouped.get_group(key)\n        gts = ground_truths_grouped.get_group(key)\n        detections_matched.append(\n            match_detections(dets['tolerance'].iloc[0], gts, dets)\n        )\n    detections_matched = pd.concat(detections_matched)\n    print(f\"detection_matched: \\n {detections_matched}\")\n    \n    # Compute AP per event x tolerance group\n    event_classes = ground_truths['event'].unique()\n    ap_table = (\n        detections_matched\n        .query(\"event in @event_classes\")\n        .groupby(['event', 'tolerance']).apply(\n        lambda group: average_precision_score(\n        group['matched'].to_numpy(),\n                group['score'].to_numpy(),\n                class_counts[group['event'].iat[0]],\n            )\n        )\n    )\n    print(f\"ap_table: \\n {ap_table}\")\n\n    # Average over tolerances, then over event classes\n    mean_ap = ap_table.groupby('event').mean().mean()\n\n    return mean_ap","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-03T10:07:18.098540Z","iopub.execute_input":"2022-08-03T10:07:18.098854Z","iopub.status.idle":"2022-08-03T10:07:18.126530Z","shell.execute_reply.started":"2022-08-03T10:07:18.098822Z","shell.execute_reply":"2022-08-03T10:07:18.125761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Easy Examples","metadata":{}},{"cell_type":"code","source":"# Label data: play(time=1-3), throwin(time=11-13), challenge(time=21-23)\ndf_case1 = pd.DataFrame({\n    \"video_id\": [\"case1\"] * 9, \n    \"time\": [1, 2, 3] + [11, 12, 13] + [21, 22, 23],\n    \"event\": [\"start\", \"play\", \"end\"] + [\"start\", \"challenge\", \"end\"] + [\"start\", \"throwin\", \"end\"]\n})\ndf_case1","metadata":{"execution":{"iopub.status.busy":"2022-08-03T10:07:18.623447Z","iopub.execute_input":"2022-08-03T10:07:18.624262Z","iopub.status.idle":"2022-08-03T10:07:18.644586Z","shell.execute_reply.started":"2022-08-03T10:07:18.624236Z","shell.execute_reply":"2022-08-03T10:07:18.643814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Prediction test case","metadata":{}},{"cell_type":"markdown","source":"## case1. perfect submission","metadata":{"execution":{"iopub.status.busy":"2022-08-02T12:09:02.200613Z","iopub.execute_input":"2022-08-02T12:09:02.201074Z","iopub.status.idle":"2022-08-02T12:09:02.206169Z","shell.execute_reply.started":"2022-08-02T12:09:02.201038Z","shell.execute_reply":"2022-08-02T12:09:02.205003Z"}}},{"cell_type":"code","source":"df_pred1 = pd.DataFrame({\n    \"video_id\": [\"case1\"] * 3,\n    \"time\": [2, 12, 22],\n    \"event\": [\"play\", \"challenge\", \"throwin\"],\n    \"score\": [1, 1, 1]\n})\nevent_detection_ap(solution=df_case1, submission=df_pred1, tolerances=tolerances)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T10:07:18.734184Z","iopub.execute_input":"2022-08-03T10:07:18.734921Z","iopub.status.idle":"2022-08-03T10:07:18.840096Z","shell.execute_reply.started":"2022-08-03T10:07:18.734887Z","shell.execute_reply":"2022-08-03T10:07:18.839250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## case2. predict play -0.3sec","metadata":{"execution":{"iopub.status.busy":"2022-08-02T12:09:17.444433Z","iopub.execute_input":"2022-08-02T12:09:17.444832Z","iopub.status.idle":"2022-08-02T12:09:17.450222Z","shell.execute_reply.started":"2022-08-02T12:09:17.444799Z","shell.execute_reply":"2022-08-02T12:09:17.449314Z"}}},{"cell_type":"code","source":"# Average Precision = (1 + 1 + 0.2) / 3 = 0.7333\n#   play: 0.2 (n_prediction=5, n_correct=1(failed to predict torelances [0.15, 0.20, 0.25, 0.30]))\n#   challenge: 1\n#   throwin: 1\n\n# [Example] In case of event \"play\" or \"throwin\", if answer is time=2:\n#   get 1.0(5/5) score if you predict play time 1.85 ~ 2.15\n#   get 0.8(4/5) score if you predict play time 1.80 ~ 1.85 or 2.15 ~ 2.20\n#   get 0.6(3/5) score if you predict play time 1.75 ~ 1.80 or 2.20 ~ 2.25\n#   get 0.4(2/5) score if you predict play time 1.70 ~ 1.75 or 2.25 ~ 2.30\n#   get 0.2(1/5) score if you predict play time 1.65 ~ 1.70 or 2.30 ~ 2.35\n#   get 0.0(0/5) score if you predict play time ~ 1.65 or 2.35 ~\n\ndf_pred1 = pd.DataFrame({\n    \"video_id\": [\"case1\"] * 3,\n    \"time\": [1.7, 12, 22],\n    \"event\": [\"play\", \"challenge\", \"throwin\"],\n    \"score\": [1, 1, 1]\n})\nevent_detection_ap(solution=df_case1, submission=df_pred1, tolerances=tolerances)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T10:07:18.841670Z","iopub.execute_input":"2022-08-03T10:07:18.842012Z","iopub.status.idle":"2022-08-03T10:07:18.931435Z","shell.execute_reply.started":"2022-08-03T10:07:18.841982Z","shell.execute_reply":"2022-08-03T10:07:18.930525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## case3. predict challenge +0.4sec","metadata":{}},{"cell_type":"code","source":"# Average Precision = (1 + 1 + 0.6) / 3 = 0.8667\n#   play: 1 \n#   challenge: 0.6 (n_prediction=5, n_correct=3(failed to predict torelances [0.3, 0.4]))\n#   throwin: 1\n\n# [Example] In case of event \"challenge\", if answer is time=12:\n#   get 1.0(5/5) score if you predict play time 11.7 ~ 12.3\n#   get 0.8(4/5) score if you predict play time 11.6 ~ 11.7 or 12.3 ~ 12.4\n#   get 0.6(3/5) score if you predict play time 11.5 ~ 11.6 or 12.4 ~ 12.5\n#   get 0.4(2/5) score if you predict play time 11.4 ~ 11.5 or 12.5 ~ 12.6\n#   get 0.2(1/5) score if you predict play time 11.3 ~ 11.4 or 12.6 ~ 12.7\n#   get 0.0(0/5) score if you predict play time ~ 11.3 or 12.7 ~\n\ndf_pred1 = pd.DataFrame({\n    \"video_id\": [\"case1\"] * 3,\n    \"time\": [2, 12.4, 22],\n    \"event\": [\"play\", \"challenge\", \"throwin\"],\n    \"score\": [1, 1, 1]\n})\nevent_detection_ap(solution=df_case1, submission=df_pred1, tolerances=tolerances)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T10:07:18.933229Z","iopub.execute_input":"2022-08-03T10:07:18.933545Z","iopub.status.idle":"2022-08-03T10:07:19.025573Z","shell.execute_reply.started":"2022-08-03T10:07:18.933515Z","shell.execute_reply":"2022-08-03T10:07:19.024665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## case4. perfect prediction + out of [start, end] range (time=25, event=play)","metadata":{}},{"cell_type":"code","source":"# Only prediction between start and end is evaluated, so this prediction does not affect the score.\n\ndf_pred1 = pd.DataFrame({\n    \"video_id\": [\"case1\"] * 4,\n    \"time\": [2, 12, 22, 25],\n    \"event\": [\"play\", \"challenge\", \"throwin\", \"play\"],\n    \"score\": [1, 1, 1, 1]\n})\nevent_detection_ap(solution=df_case1, submission=df_pred1, tolerances=tolerances)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T10:07:19.026768Z","iopub.execute_input":"2022-08-03T10:07:19.027405Z","iopub.status.idle":"2022-08-03T10:07:19.114406Z","shell.execute_reply.started":"2022-08-03T10:07:19.027373Z","shell.execute_reply":"2022-08-03T10:07:19.113541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## case5. perfect prediction + miss prediction(event=play, time=12)","metadata":{}},{"cell_type":"code","source":"#  Average Precision = (1 + 1 + 0.5) / 3 = 0.8333\n#    play: 0.5 (n_prediction=10, n_correct=5(event=play, time=12 is wrong prediction))\n#    challenge: 1 \n#    throwin: 1\n\ndf_pred1 = pd.DataFrame({\n    \"video_id\": [\"case1\"] * 4,\n    \"time\": [2, 12, 22, 12],\n    \"event\": [\"play\", \"challenge\", \"throwin\", \"play\"],\n    \"score\": [1, 1, 1, 1]\n})\nevent_detection_ap(solution=df_case1, submission=df_pred1, tolerances=tolerances)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T10:07:19.117113Z","iopub.execute_input":"2022-08-03T10:07:19.117344Z","iopub.status.idle":"2022-08-03T10:07:19.213229Z","shell.execute_reply.started":"2022-08-03T10:07:19.117322Z","shell.execute_reply":"2022-08-03T10:07:19.212405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## case6. perfect submisson, score = 0.5 for play","metadata":{}},{"cell_type":"code","source":"# TODO: add explanation\ndf_pred1 = pd.DataFrame({\n    \"video_id\": [\"case1\"] * 3,\n    \"time\": [2, 12, 22],\n    \"event\": [\"play\", \"challenge\", \"throwin\"],\n    \"score\": [0.5, 1, 1]\n})\nevent_detection_ap(solution=df_case1, submission=df_pred1, tolerances=tolerances)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T10:07:19.214144Z","iopub.execute_input":"2022-08-03T10:07:19.214442Z","iopub.status.idle":"2022-08-03T10:07:19.309342Z","shell.execute_reply.started":"2022-08-03T10:07:19.214407Z","shell.execute_reply":"2022-08-03T10:07:19.308631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import precision_recall_curve as prcurve_sklearn\nfrom sklearn.metrics import auc","metadata":{"execution":{"iopub.status.busy":"2022-08-03T10:07:19.310776Z","iopub.execute_input":"2022-08-03T10:07:19.311201Z","iopub.status.idle":"2022-08-03T10:07:19.698265Z","shell.execute_reply.started":"2022-08-03T10:07:19.311170Z","shell.execute_reply":"2022-08-03T10:07:19.697536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## case7. perfect submisson(score = 0.5 for play) + miss prediction(score=0.25)","metadata":{}},{"cell_type":"code","source":"def prauc(true, pred):\n    # for this competition metric\n    p = 1  # count of play labels. in this testcase, always 1. This variable is what makes it different from the sklearn precision_recall_curve functions.\n    precision, recall, _ = precision_recall_curve(true, pred, p)\n    \n    # for general PRAUC\n    # precision, recall, _ = precision_recall_curve(true, pred)\n    prauc = auc(recall, precision)\n    return prauc","metadata":{"execution":{"iopub.status.busy":"2022-08-03T10:07:19.700391Z","iopub.execute_input":"2022-08-03T10:07:19.700607Z","iopub.status.idle":"2022-08-03T10:07:19.704967Z","shell.execute_reply.started":"2022-08-03T10:07:19.700586Z","shell.execute_reply":"2022-08-03T10:07:19.704137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"play_matched = np.array([\n    True,  # time = 2\n    False  # time = 12\n])\nplay_score = np.array([\n    0.5,   # time = 2\n    0.25   # time = 12\n])\nap_for_play = prauc(play_matched, play_score)\nprint(f\"play_ap = {ap_for_play}, total_ap = {(1 + 1 + ap_for_play) / 3}\")","metadata":{"execution":{"iopub.status.busy":"2022-08-03T10:07:19.705672Z","iopub.execute_input":"2022-08-03T10:07:19.705889Z","iopub.status.idle":"2022-08-03T10:07:19.717878Z","shell.execute_reply.started":"2022-08-03T10:07:19.705869Z","shell.execute_reply":"2022-08-03T10:07:19.716432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_pred1 = pd.DataFrame({\n    \"video_id\": [\"case1\"] * 4,\n    \"time\": [2, 12, 22, 12],\n    \"event\": [\"play\", \"challenge\", \"throwin\", \"play\"],\n    \"score\": [0.5, 1, 1, 0.25]\n})\nevent_detection_ap(solution=df_case1, submission=df_pred1, tolerances=tolerances)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T10:07:19.718960Z","iopub.execute_input":"2022-08-03T10:07:19.719690Z","iopub.status.idle":"2022-08-03T10:07:19.815336Z","shell.execute_reply.started":"2022-08-03T10:07:19.719665Z","shell.execute_reply":"2022-08-03T10:07:19.814444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## case8 perfect submisson(score = 0.5 for play) + miss prediction(score=0.75)","metadata":{}},{"cell_type":"code","source":"play_matched = np.array([\n    True,  # time = 2\n    False  # time = 12\n])\nplay_score = np.array([\n    0.5,   # time = 2\n    0.75   # time = 12\n])\nap_for_play = prauc(play_matched, play_score)\nprint(f\"play_ap = {ap_for_play}, total_ap = {(1 + 1 + ap_for_play) / 3}\")","metadata":{"execution":{"iopub.status.busy":"2022-08-03T10:07:19.816391Z","iopub.execute_input":"2022-08-03T10:07:19.816612Z","iopub.status.idle":"2022-08-03T10:07:19.823053Z","shell.execute_reply.started":"2022-08-03T10:07:19.816591Z","shell.execute_reply":"2022-08-03T10:07:19.822195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# I expect AP=0.75, but actual AP=0.833. This difference seems to stem from a different way of thinking about the auc function.\ndf_pred1 = pd.DataFrame({\n    \"video_id\": [\"case1\"] * 4,\n    \"time\": [2, 12, 22, 12],\n    \"event\": [\"play\", \"challenge\", \"throwin\", \"play\"],\n    \"score\": [0.5, 1, 1, 0.75]\n})\nevent_detection_ap(solution=df_case1, submission=df_pred1, tolerances=tolerances)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T10:07:19.824314Z","iopub.execute_input":"2022-08-03T10:07:19.824561Z","iopub.status.idle":"2022-08-03T10:07:19.916207Z","shell.execute_reply.started":"2022-08-03T10:07:19.824539Z","shell.execute_reply":"2022-08-03T10:07:19.915148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Explanation between DFL (Average Precision) and Sklearn PRAUC","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport matplotlib.patches as pat","metadata":{"execution":{"iopub.status.busy":"2022-08-03T10:07:19.917161Z","iopub.execute_input":"2022-08-03T10:07:19.917375Z","iopub.status.idle":"2022-08-03T10:07:19.921509Z","shell.execute_reply.started":"2022-08-03T10:07:19.917355Z","shell.execute_reply":"2022-08-03T10:07:19.920599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def auc_dfl(recall, precision):\n    return -np.sum(np.diff(recall) * np.array(precision)[:-1])\n\ndef auc_sklearn(recall, precision):\n    return -np.sum(np.diff(recall) * (precision[:-1] + precision[1:]) / 2)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T10:07:19.922801Z","iopub.execute_input":"2022-08-03T10:07:19.923117Z","iopub.status.idle":"2022-08-03T10:07:19.931762Z","shell.execute_reply.started":"2022-08-03T10:07:19.923087Z","shell.execute_reply":"2022-08-03T10:07:19.930949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's see difference between dfl auc and sklearn auc\nrecall = np.array([1, 0.5, 0])\nprecision = np.array([1, 0.75, 0])","metadata":{"execution":{"iopub.status.busy":"2022-08-03T10:07:19.932964Z","iopub.execute_input":"2022-08-03T10:07:19.933207Z","iopub.status.idle":"2022-08-03T10:07:19.943021Z","shell.execute_reply.started":"2022-08-03T10:07:19.933187Z","shell.execute_reply":"2022-08-03T10:07:19.942158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"dfl_auc: {auc_dfl(recall, precision)}, sklearn_auc: {auc_sklearn(recall, precision)}\")","metadata":{"execution":{"iopub.status.busy":"2022-08-03T10:07:19.947000Z","iopub.execute_input":"2022-08-03T10:07:19.947258Z","iopub.status.idle":"2022-08-03T10:07:19.953795Z","shell.execute_reply.started":"2022-08-03T10:07:19.947237Z","shell.execute_reply":"2022-08-03T10:07:19.953090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def draw_pr_curve_auc(xy, title, xy_point):\n    # https://python.atelierkobato.com/polygon/\n    fig = plt.figure(figsize=(7, 7))\n    ax = fig.add_subplot(111)\n    p = pat.Polygon(xy=xy, fc=\"pink\", ec=\"black\")\n    ax.add_patch(p)\n    plt.scatter(xy_point[:, 0], xy_point[:, 1], s=24, color=\"red\", label=\"pr_point\")\n    plt.xlabel(\"recall\", fontsize=16)\n    plt.ylabel(\"precision\", fontsize=16)\n    plt.title(title, fontsize=20)\n    plt.legend(fontsize=12)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-03T10:07:19.956400Z","iopub.execute_input":"2022-08-03T10:07:19.956662Z","iopub.status.idle":"2022-08-03T10:07:19.964767Z","shell.execute_reply.started":"2022-08-03T10:07:19.956641Z","shell.execute_reply":"2022-08-03T10:07:19.964167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"draw_pr_curve_auc(xy=[(1, 1), (0.5, 0.75), (0, 0), (1, 0)], title=f\"sklearn: PRAUC(area={auc_sklearn(recall, precision)})\", xy_point=np.array([[1, 1], [0.5, 0.75], [0, 0]]))","metadata":{"execution":{"iopub.status.busy":"2022-08-03T10:07:19.965790Z","iopub.execute_input":"2022-08-03T10:07:19.965996Z","iopub.status.idle":"2022-08-03T10:07:20.169249Z","shell.execute_reply.started":"2022-08-03T10:07:19.965977Z","shell.execute_reply":"2022-08-03T10:07:20.168541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"draw_pr_curve_auc(xy=[(1, 1), (0.5, 1), (0.5, 0.75), (0, 0.75), (0, 0), (1, 0)], title=f\"dfl: average_precision(area={auc_dfl(recall, precision)})\", xy_point=np.array([[1, 1], [0.5, 0.75], [0, 0]]))","metadata":{"execution":{"iopub.status.busy":"2022-08-03T10:07:20.170231Z","iopub.execute_input":"2022-08-03T10:07:20.170515Z","iopub.status.idle":"2022-08-03T10:07:20.326671Z","shell.execute_reply.started":"2022-08-03T10:07:20.170484Z","shell.execute_reply":"2022-08-03T10:07:20.325875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### If the amount of data is large, maybe the error between the two is slight.","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## case9 perfect submisson(score = 0.5 for play) + miss prediction(score=0.25, score=0.75)","metadata":{}},{"cell_type":"code","source":"play_matched = np.array([\n    True,  # time = 2\n    False, # time = 12\n    False  # time = 22\n])\nplay_score = np.array([\n    0.5,   # time = 2\n    0.75,  # time = 12\n    0.25,  # time = 22\n])\nap_for_play = prauc(play_matched, play_score)\nprint(f\"play_ap = {ap_for_play}, total_ap = {(1 + 1 + ap_for_play) / 3}\")","metadata":{"execution":{"iopub.status.busy":"2022-08-03T10:07:24.124056Z","iopub.execute_input":"2022-08-03T10:07:24.125100Z","iopub.status.idle":"2022-08-03T10:07:24.132684Z","shell.execute_reply.started":"2022-08-03T10:07:24.124994Z","shell.execute_reply":"2022-08-03T10:07:24.131798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_pred1 = pd.DataFrame({\n    \"video_id\": [\"case1\"] * 5,\n    \"time\": [2, 12, 22, 12, 22],\n    \"event\": [\"play\", \"challenge\", \"throwin\", \"play\", \"play\"],\n    \"score\": [0.5, 1, 1, 0.75, 0.25]\n})\nevent_detection_ap(solution=df_case1, submission=df_pred1, tolerances=tolerances)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T10:07:24.733118Z","iopub.execute_input":"2022-08-03T10:07:24.733418Z","iopub.status.idle":"2022-08-03T10:07:24.829096Z","shell.execute_reply.started":"2022-08-03T10:07:24.733395Z","shell.execute_reply":"2022-08-03T10:07:24.827592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## case10 perfect submisson(score = 0.5 for play) + miss prediction(score=0.65, score=0.75)","metadata":{}},{"cell_type":"code","source":"df_pred1 = pd.DataFrame({\n    \"video_id\": [\"case1\"] * 5,\n    \"time\": [2, 12, 22, 12, 22],\n    \"event\": [\"play\", \"challenge\", \"throwin\", \"play\", \"play\"],\n    \"score\": [0.5, 1, 1, 0.75, 0.65]\n})\nevent_detection_ap(solution=df_case1, submission=df_pred1, tolerances=tolerances)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T10:07:25.044514Z","iopub.execute_input":"2022-08-03T10:07:25.044833Z","iopub.status.idle":"2022-08-03T10:07:25.139394Z","shell.execute_reply.started":"2022-08-03T10:07:25.044809Z","shell.execute_reply":"2022-08-03T10:07:25.138800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## case11 perfect submisson(score=0.5, score=0.55 for play) + miss prediction(score=0.25, score=0.75) ","metadata":{}},{"cell_type":"code","source":"df_pred1 = pd.DataFrame({\n    \"video_id\": [\"case1\"] * 6,\n    \"time\": [2, 12, 22, 12, 22, 2.02],\n    \"event\": [\"play\", \"challenge\", \"throwin\", \"play\", \"play\", \"play\"],\n    \"score\": [0.5, 1, 1, 0.75, 0.25, 0.55]\n})\nevent_detection_ap(solution=df_case1, submission=df_pred1, tolerances=tolerances)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T10:09:41.517461Z","iopub.execute_input":"2022-08-03T10:09:41.518511Z","iopub.status.idle":"2022-08-03T10:09:41.613329Z","shell.execute_reply.started":"2022-08-03T10:09:41.518461Z","shell.execute_reply":"2022-08-03T10:09:41.612184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## case12 1 correct and 1 wrong prediction exist in one terms(from start to end)","metadata":{}},{"cell_type":"code","source":"df_pred1 = pd.DataFrame({\n    \"video_id\": [\"case1\"] * 4,\n    \"time\": [2, 12, 22, 2.5],\n    \"event\": [\"play\", \"challenge\", \"throwin\", \"play\"],\n    \"score\": [1, 1, 1, 1]\n})\nevent_detection_ap(solution=df_case1, submission=df_pred1, tolerances=tolerances)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T10:09:42.466402Z","iopub.execute_input":"2022-08-03T10:09:42.467134Z","iopub.status.idle":"2022-08-03T10:09:42.563967Z","shell.execute_reply.started":"2022-08-03T10:09:42.467106Z","shell.execute_reply":"2022-08-03T10:09:42.563027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## case13 1 correct and 2 wrong predictions exist in one terms(from start to end)","metadata":{}},{"cell_type":"code","source":"df_pred1 = pd.DataFrame({\n    \"video_id\": [\"case1\"] * 5,\n    \"time\": [2, 12, 22, 2.5, 2.6],\n    \"event\": [\"play\", \"challenge\", \"throwin\", \"play\", \"play\"],\n    \"score\": [1, 1, 1, 1, 1]\n})\nevent_detection_ap(solution=df_case1, submission=df_pred1, tolerances=tolerances)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T10:09:43.775427Z","iopub.execute_input":"2022-08-03T10:09:43.775770Z","iopub.status.idle":"2022-08-03T10:09:43.872028Z","shell.execute_reply.started":"2022-08-03T10:09:43.775725Z","shell.execute_reply":"2022-08-03T10:09:43.871115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"## case14 perfect predictions 2 times.","metadata":{}},{"cell_type":"code","source":"# Only the first predictions is treated as \"correct\". After the second predicitions, it is treated as \"wrong\". \ndf_pred1 = pd.DataFrame({\n    \"video_id\": [\"case1\"] * 3 * 2,\n    \"time\": [2, 12, 22] * 2,\n    \"event\": [\"play\", \"challenge\", \"throwin\"] * 2,\n    \"score\": [1, 1, 1] * 2\n})\nevent_detection_ap(solution=df_case1, submission=df_pred1, tolerances=tolerances)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T10:43:35.615603Z","iopub.execute_input":"2022-08-03T10:43:35.615953Z","iopub.status.idle":"2022-08-03T10:43:35.719462Z","shell.execute_reply.started":"2022-08-03T10:43:35.615930Z","shell.execute_reply":"2022-08-03T10:43:35.718442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## case15 perfect submission only challenge, throwin\nAP achieve 1.0!?<br> ","metadata":{}},{"cell_type":"code","source":"df_pred1 = pd.DataFrame({\n    \"video_id\": [\"case1\"] * 2,\n    \"time\": [12, 22],\n    \"event\": [\"challenge\", \"throwin\"],\n    \"score\": [1, 1]\n})\nevent_detection_ap(solution=df_case1, submission=df_pred1, tolerances=tolerances)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T10:43:36.407510Z","iopub.execute_input":"2022-08-03T10:43:36.407823Z","iopub.status.idle":"2022-08-03T10:43:36.498505Z","shell.execute_reply.started":"2022-08-03T10:43:36.407798Z","shell.execute_reply":"2022-08-03T10:43:36.497629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# you can add example here!","metadata":{"execution":{"iopub.status.busy":"2022-08-02T12:43:03.373693Z","iopub.execute_input":"2022-08-02T12:43:03.374143Z","iopub.status.idle":"2022-08-02T12:43:03.378976Z","shell.execute_reply.started":"2022-08-02T12:43:03.374109Z","shell.execute_reply":"2022-08-02T12:43:03.377794Z"},"trusted":true},"execution_count":null,"outputs":[]}]}