{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":91249,"databundleVersionId":11294684,"sourceType":"competition"}],"dockerImageVersionId":30918,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# BYU METRICS GUIDE\nThis notebook will help you better understand simple metrics.  \n\nOriginal Official Notebook is here:  https://www.kaggle.com/code/metric/byu-biophysics-91249","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport sklearn.metrics\n\n\nclass ParticipantVisibleError(Exception):\n    # If you want an error message to be shown to participants, you must raise the error as a ParticipantVisibleError\n    # All other errors will only be shown to the competition host. This helps prevent unintentional leakage of solution data.\n    pass\n\n\ndef distance_metric(\n    solution: pd.DataFrame,\n    submission: pd.DataFrame,\n    thresh_ratio: float,\n    min_radius: float,\n):\n    coordinate_cols = ['Motor axis 0', 'Motor axis 1', 'Motor axis 2']\n    label_tensor = solution[coordinate_cols].values.reshape(len(solution), -1, len(coordinate_cols))\n    predicted_tensor = submission[coordinate_cols].values.reshape(len(submission), -1, len(coordinate_cols))\n    \n    # Find the minimum euclidean distances between the true and predicted points\n    solution['distance'] = np.linalg.norm(label_tensor - predicted_tensor, axis=2).min(axis=1)\n    \n    # Convert thresholds from angstroms to voxels\n    solution['thresholds'] = solution['Voxel spacing'].apply(lambda x: (min_radius * thresh_ratio) / x)\n    solution['predictions'] = submission['Has motor'].values\n    solution.loc[(solution['distance'] > solution['thresholds']) & (solution['Has motor'] == 1) & (submission['Has motor'] == 1), 'predictions'] = 0\n    return solution['predictions'].values\n\n\ndef score(solution: pd.DataFrame, submission: pd.DataFrame, min_radius: float, beta: float) -> float:\n    \"\"\"\n    Parameters:\n    solution (pd.DataFrame): DataFrame containing ground truth motor positions.\n    submission (pd.DataFrame): DataFrame containing predicted motor positions.\n\n    Returns:\n    float: FBeta score.\n\n    Example\n    --------\n    >>> solution = pd.DataFrame({\n    ...     'tomo_id': [0, 1, 2, 3],\n    ...     'Motor axis 0': [-1, 250, 100, 200],\n    ...     'Motor axis 1': [-1, 250, 100, 200],\n    ...     'Motor axis 2': [-1, 250, 100, 200],\n    ...     'Voxel spacing': [10, 10, 10, 10],\n    ...     'Has motor': [0, 1, 1, 1]\n    ... })\n    >>> submission = pd.DataFrame({\n    ...     'tomo_id': [0, 1, 2, 3],\n    ...     'Motor axis 0': [100, 251, 600, -1],\n    ...     'Motor axis 1': [100, 251, 600, -1],\n    ...     'Motor axis 2': [100, 251, 600, -1]\n    ... })\n    >>> score(solution, submission, 1000, 2)\n    0.3571428571428571\n    \"\"\"\n\n    solution = solution.sort_values('tomo_id').reset_index(drop=True)\n    submission = submission.sort_values('tomo_id').reset_index(drop=True)\n\n    filename_equiv_array = solution['tomo_id'].eq(submission['tomo_id'], fill_value=0).values\n\n    if np.sum(filename_equiv_array) != len(solution['tomo_id']):\n        raise ValueError('Submitted tomo_id values do not match the sample_submission file')\n\n    submission['Has motor'] = 1\n    # If any columns are missing an axis, it's marked with no motor\n    select = (submission[['Motor axis 0', 'Motor axis 1', 'Motor axis 2']] == -1).any(axis='columns')\n    submission.loc[select, 'Has motor'] = 0\n\n    cols = ['Has motor', 'Motor axis 0', 'Motor axis 1', 'Motor axis 2']\n    assert all(col in submission.columns for col in cols)\n\n    # Calculate a label of 0 or 1 using the 'has motor', and 'motor axis' values\n    predictions = distance_metric(\n        solution,\n        submission,\n        thresh_ratio=1.0,\n        min_radius=min_radius,\n    )\n\n    return sklearn.metrics.fbeta_score(solution['Has motor'].values, predictions, beta=beta)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-17T15:17:08.532133Z","iopub.execute_input":"2025-03-17T15:17:08.532391Z","iopub.status.idle":"2025-03-17T15:17:10.275520Z","shell.execute_reply.started":"2025-03-17T15:17:08.532368Z","shell.execute_reply":"2025-03-17T15:17:10.274533Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## SAMPLE SCORE\n・We must keep the misalignment of the flagellar motor within 1000 angstroms.  \n・Look out for tomo_id 2! Label is (100, 100, 100), prediction (200, 200, 200) is failure, (150, 150, 150) is correct.  \n・Note, however, that this depends on Voxel spacing.  ","metadata":{}},{"cell_type":"code","source":"true_label = pd.DataFrame({\n    'tomo_id': [0, 1, 2, 3],\n    'Motor axis 0': [-1, 250, 100, 200],\n    'Motor axis 1': [-1, 250, 100, 200],\n    'Motor axis 2': [-1, 250, 100, 200],\n    'Voxel spacing': [10, 10, 10, 10],\n    'Has motor': [0, 1, 1, 1]\n    })\nprediction = pd.DataFrame({\n    'tomo_id': [0, 1, 2, 3],\n    'Motor axis 0': [100, 251, 200, -1],\n    'Motor axis 1': [100, 251, 200, -1],\n    'Motor axis 2': [100, 251, 200, -1]\n})\nscore(true_label, prediction, 1000, 2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-17T15:43:37.318516Z","iopub.execute_input":"2025-03-17T15:43:37.318866Z","iopub.status.idle":"2025-03-17T15:43:37.338666Z","shell.execute_reply.started":"2025-03-17T15:43:37.318836Z","shell.execute_reply":"2025-03-17T15:43:37.337755Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"true_label = pd.DataFrame({\n    'tomo_id': [0, 1, 2, 3],\n    'Motor axis 0': [-1, 250, 100, 200],\n    'Motor axis 1': [-1, 250, 100, 200],\n    'Motor axis 2': [-1, 250, 100, 200],\n    'Voxel spacing': [10, 10, 10, 10],\n    'Has motor': [0, 1, 1, 1]\n    })\nprediction = pd.DataFrame({\n    'tomo_id': [0, 1, 2, 3],\n    'Motor axis 0': [100, 251, 150, -1],\n    'Motor axis 1': [100, 251, 150, -1],\n    'Motor axis 2': [100, 251, 150, -1]\n})\nscore(true_label, prediction, 1000, 2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-17T15:47:28.518220Z","iopub.execute_input":"2025-03-17T15:47:28.518603Z","iopub.status.idle":"2025-03-17T15:47:28.538367Z","shell.execute_reply.started":"2025-03-17T15:47:28.518569Z","shell.execute_reply":"2025-03-17T15:47:28.537329Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## ALL NEGATIVE\n・All negative prediction score is 0.  \n・We need to guess a positive sample because of Fβ-score metrics.  ","metadata":{"execution":{"iopub.status.busy":"2025-03-17T15:29:03.767583Z","iopub.execute_input":"2025-03-17T15:29:03.767908Z","iopub.status.idle":"2025-03-17T15:29:03.772057Z","shell.execute_reply.started":"2025-03-17T15:29:03.767883Z","shell.execute_reply":"2025-03-17T15:29:03.770911Z"}}},{"cell_type":"code","source":"true_label = pd.DataFrame({\n    'tomo_id': [0, 1, 2, 3],\n    'Motor axis 0': [-1, -1, -1, -1],\n    'Motor axis 1': [-1, -1, -1, -1],\n    'Motor axis 2': [-1, -1, -1, -1],\n    'Voxel spacing': [10, 10, 10, 10],\n    'Has motor': [0, 0, 0, 0]\n    })\nprediction = pd.DataFrame({\n    'tomo_id': [0, 1, 2, 3],\n    'Motor axis 0': [-1, -1, -1, -1],\n    'Motor axis 1': [-1, -1, -1, -1],\n    'Motor axis 2': [-1, -1, -1, -1],\n})\nscore(true_label, prediction, 1000, 2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-17T15:29:59.206449Z","iopub.execute_input":"2025-03-17T15:29:59.206888Z","iopub.status.idle":"2025-03-17T15:29:59.229496Z","shell.execute_reply.started":"2025-03-17T15:29:59.206844Z","shell.execute_reply":"2025-03-17T15:29:59.228175Z"}},"outputs":[],"execution_count":null}]}