{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84969,"databundleVersionId":10033515,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# With no change","metadata":{}},{"cell_type":"code","source":"\"\"\"\nDerived from:\nhttps://github.com/cellcanvas/album-catalog/blob/main/solutions/copick/compare-picks/solution.py\n\"\"\"\n\nimport numpy as np\nimport pandas as pd\n\nfrom scipy.spatial import KDTree\n\n\nclass ParticipantVisibleError(Exception):\n    pass\n\n\ndef compute_metrics(reference_points, reference_radius, candidate_points):\n    num_reference_particles = len(reference_points)\n    num_candidate_particles = len(candidate_points)\n\n    if len(reference_points) == 0:\n        return 0, num_candidate_particles, 0\n\n    if len(candidate_points) == 0:\n        return 0, 0, num_reference_particles\n\n    ref_tree = KDTree(reference_points)\n    candidate_tree = KDTree(candidate_points)\n    raw_matches = ref_tree.query_ball_tree(candidate_tree, r=reference_radius)\n    matches_within_threshold = []\n    for match in raw_matches:\n        matches_within_threshold.extend(match)\n    # Prevent submitting multiple matches per particle.\n    # This won't be be strictly correct in the (extremely rare) case where true particles\n    # are very close to each other.\n    matches_within_threshold = set(matches_within_threshold)\n    tp = int(len(matches_within_threshold))\n    fp = int(num_candidate_particles - tp)\n    fn = int(num_reference_particles - tp)\n    return tp, fp, fn\n\n\ndef score(\n        solution: pd.DataFrame,\n        submission: pd.DataFrame,\n        row_id_column_name: str,\n        distance_multiplier: float,\n        beta: int) -> float:\n    '''\n    F_beta\n      - a true positive occurs when\n         - (a) the predicted location is within a threshold of the particle radius, and\n         - (b) the correct `particle_type` is specified\n      - raw results (TP, FP, FN) are aggregated across all experiments for each particle type\n      - f_beta is calculated for each particle type\n      - individual f_beta scores are weighted by particle type for final score\n    '''\n\n    particle_radius = {\n        'apo-ferritin': 60,\n        'beta-amylase': 65,\n        'beta-galactosidase': 90,\n        'ribosome': 150,\n        'thyroglobulin': 130,\n        'virus-like-particle': 135,\n    }\n\n    weights = {\n        'apo-ferritin': 1,\n        'beta-amylase': 0,\n        'beta-galactosidase': 2,\n        'ribosome': 1,\n        'thyroglobulin': 2,\n        'virus-like-particle': 1,\n    }\n\n    particle_radius = {k: v * distance_multiplier for k, v in particle_radius.items()}\n\n    # Filter submission to only contain experiments found in the solution split\n    split_experiments = set(solution['experiment'].unique())\n    submission = submission.loc[submission['experiment'].isin(split_experiments)]\n\n    # Only allow known particle types\n    if not set(submission['particle_type'].unique()).issubset(set(weights.keys())):\n        raise ParticipantVisibleError('Unrecognized `particle_type`.')\n\n    assert solution.duplicated(subset=['experiment', 'x', 'y', 'z']).sum() == 0\n    assert particle_radius.keys() == weights.keys()\n\n    results = {}\n    for particle_type in solution['particle_type'].unique():\n        results[particle_type] = {\n            'total_tp': 0,\n            'total_fp': 0,\n            'total_fn': 0,\n        }\n\n    for experiment in split_experiments:\n        for particle_type in solution['particle_type'].unique():\n            reference_radius = particle_radius[particle_type]\n            select = (solution['experiment'] == experiment) & (solution['particle_type'] == particle_type)\n            reference_points = solution.loc[select, ['x', 'y', 'z']].values\n\n            select = (submission['experiment'] == experiment) & (submission['particle_type'] == particle_type)\n            candidate_points = submission.loc[select, ['x', 'y', 'z']].values\n\n            if len(reference_points) == 0:\n                reference_points = np.array([])\n                reference_radius = 1\n\n            if len(candidate_points) == 0:\n                candidate_points = np.array([])\n\n            tp, fp, fn = compute_metrics(reference_points, reference_radius, candidate_points)\n\n            results[particle_type]['total_tp'] += tp\n            results[particle_type]['total_fp'] += fp\n            results[particle_type]['total_fn'] += fn\n    display(results)\n    aggregate_fbeta = 0.0\n    for particle_type, totals in results.items():\n        tp = totals['total_tp']\n        fp = totals['total_fp']\n        fn = totals['total_fn']\n\n        precision = tp / (tp + fp) if tp + fp > 0 else 0\n        recall = tp / (tp + fn) if tp + fn > 0 else 0\n        fbeta = (1 + beta**2) * (precision * recall) / (beta**2 * precision + recall) if (precision + recall) > 0 else 0.0\n        aggregate_fbeta += fbeta * weights.get(particle_type, 1.0)\n\n    if weights:\n        aggregate_fbeta = aggregate_fbeta / sum(weights.values())\n    else:\n        aggregate_fbeta = aggregate_fbeta / len(results)\n    return aggregate_fbeta\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-11-16T20:08:32.110387Z","iopub.execute_input":"2024-11-16T20:08:32.110845Z","iopub.status.idle":"2024-11-16T20:08:32.755919Z","shell.execute_reply.started":"2024-11-16T20:08:32.110786Z","shell.execute_reply":"2024-11-16T20:08:32.754852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reference = pd.DataFrame([['TS_5_4','apo-ferritin',2983.596,3154.13,764.124],\n                          ['TS_5_4','beta-galactosidase',2883.596,3054.13,1764.124],\n                          ['TS_5_4','thyroglobulin',883.596,3054.13,1764.124]],\n                        columns=['experiment','particle_type','x','y','z'])\n\ncandidate = pd.DataFrame([['TS_5_4','apo-ferritin',2983.596,3154.13,764.124],\n                          ['TS_5_4','beta-galactosidase',2883.596,3054.13,1764.124],\n                          ['TS_5_4','thyroglobulin',883.596,3054.13,1764.124],\n                          ['TS_5_4','apo-ferritin',2983.596,3164.13,764.124],\n                          ['TS_5_4','beta-galactosidase',2883.596,3064.13,1764.124],\n                          ['TS_5_4','thyroglobulin',883.596,3064.13,1764.124]],\n                        columns=['experiment','particle_type','x','y','z'])","metadata":{"execution":{"iopub.status.busy":"2024-11-16T20:08:39.332562Z","iopub.execute_input":"2024-11-16T20:08:39.333216Z","iopub.status.idle":"2024-11-16T20:08:39.346329Z","shell.execute_reply.started":"2024-11-16T20:08:39.333170Z","shell.execute_reply":"2024-11-16T20:08:39.344675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Score =', score(reference, candidate, 'row_id', 0.5, 4))","metadata":{"execution":{"iopub.status.busy":"2024-11-16T20:08:45.059911Z","iopub.execute_input":"2024-11-16T20:08:45.060408Z","iopub.status.idle":"2024-11-16T20:08:45.106746Z","shell.execute_reply.started":"2024-11-16T20:08:45.060366Z","shell.execute_reply":"2024-11-16T20:08:45.105370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# After correction","metadata":{}},{"cell_type":"code","source":"def compute_metrics(reference_points, reference_radius, candidate_points):\n    num_reference_particles = len(reference_points)\n    num_candidate_particles = len(candidate_points)\n\n    if len(reference_points) == 0:\n        return 0, num_candidate_particles, 0\n\n    if len(candidate_points) == 0:\n        return 0, 0, num_reference_particles\n\n    ref_tree = KDTree(reference_points)\n    candidate_tree = KDTree(candidate_points)\n    # raw_matches = ref_tree.query_ball_tree(candidate_tree, r=reference_radius)\n    ## CORRECTION HERE\n    raw_matches = candidate_tree.query_ball_tree(ref_tree, r=reference_radius)\n    matches_within_threshold = []\n    for match in raw_matches:\n        matches_within_threshold.extend(match)\n    # Prevent submitting multiple matches per particle.\n    # This won't be be strictly correct in the (extremely rare) case where true particles\n    # are very close to each other.\n    matches_within_threshold = set(matches_within_threshold)\n    tp = int(len(matches_within_threshold))\n    fp = int(num_candidate_particles - tp)\n    fn = int(num_reference_particles - tp)\n    return tp, fp, fn","metadata":{"execution":{"iopub.status.busy":"2024-11-16T20:09:35.779318Z","iopub.execute_input":"2024-11-16T20:09:35.779760Z","iopub.status.idle":"2024-11-16T20:09:35.789967Z","shell.execute_reply.started":"2024-11-16T20:09:35.779718Z","shell.execute_reply":"2024-11-16T20:09:35.788474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Score =', score(reference, candidate, 'row_id', 0.5, 4))","metadata":{"execution":{"iopub.status.busy":"2024-11-16T20:09:41.443830Z","iopub.execute_input":"2024-11-16T20:09:41.444284Z","iopub.status.idle":"2024-11-16T20:09:41.468221Z","shell.execute_reply.started":"2024-11-16T20:09:41.444241Z","shell.execute_reply":"2024-11-16T20:09:41.466768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}