{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"}],"dockerImageVersionId":30732,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"In this notebook we are going to implemented the scoring method for the competition mentioned here [RSNA Lumbar Metric 71549](https://www.kaggle.com/code/metric/rsna-lumbar-metric-71549) to test the validation set from the given train set.","metadata":{}},{"cell_type":"code","source":"import os\nimport sys\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport random\nimport pandas as pd\nfrom glob import glob\nfrom tqdm import tqdm\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import KFold\nimport polars as pl\npl.Config.set_tbl_rows(40)\npl.Config.set_fmt_str_lengths(n=40)\nimport seaborn as sns\nimport pydicom\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms","metadata":{"execution":{"iopub.status.busy":"2024-07-05T06:28:35.556960Z","iopub.execute_input":"2024-07-05T06:28:35.557805Z","iopub.status.idle":"2024-07-05T06:28:44.506131Z","shell.execute_reply.started":"2024-07-05T06:28:35.557765Z","shell.execute_reply":"2024-07-05T06:28:44.505049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"root_dir = \"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification\"\ntrain_main = pd.read_csv(os.path.join(root_dir, \"train.csv\"))\ntrain_labels = pd.read_csv(os.path.join(root_dir, \"train_label_coordinates.csv\"))\ntrain_des = pd.read_csv(os.path.join(root_dir, \"train_series_descriptions.csv\"))\ntest_des = pd.read_csv(os.path.join(root_dir, \"test_series_descriptions.csv\"))\nsample_sub = pd.read_csv(os.path.join(root_dir, \"sample_submission.csv\"))","metadata":{"execution":{"iopub.status.busy":"2024-07-05T06:28:44.507912Z","iopub.execute_input":"2024-07-05T06:28:44.508462Z","iopub.status.idle":"2024-07-05T06:28:44.679519Z","shell.execute_reply.started":"2024-07-05T06:28:44.508427Z","shell.execute_reply":"2024-07-05T06:28:44.678542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Provided Metric Script ","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport pandas.api.types\nimport sklearn.metrics\n\n\nclass ParticipantVisibleError(Exception):\n    pass\n\n\ndef get_condition(full_location: str) -> str:\n    # Given an input like spinal_canal_stenosis_l1_l2 extracts 'spinal'\n    for injury_condition in ['spinal', 'foraminal', 'subarticular']:\n        if injury_condition in full_location:\n            return injury_condition\n    raise ValueError(f'condition not found in {full_location}')\n\n\ndef score(\n        solution: pd.DataFrame,\n        submission: pd.DataFrame,\n        row_id_column_name: str,\n        any_severe_scalar: float\n    ) -> float:\n    '''\n    Pseudocode:\n    1. Calculate the sample weighted log loss for each medical condition:\n    2. Derive a new any_severe label.\n    3. Calculate the sample weighted log loss for the new any_severe label.\n    4. Return the average of all of the label group log losses as the final score, normalized for the number of columns in each group.\n       This mitigates the impact of spinal stenosis having only half as many columns as the other two conditions.\n    '''\n\n    target_levels = ['normal_mild', 'moderate', 'severe']\n\n    # Run basic QC checks on the inputs\n    if not pandas.api.types.is_numeric_dtype(submission[target_levels].values):\n        raise ParticipantVisibleError('All submission values must be numeric')\n\n    if not np.isfinite(submission[target_levels].values).all():\n        raise ParticipantVisibleError('All submission values must be finite')\n\n    if solution[target_levels].min().min() < 0:\n        raise ParticipantVisibleError('All labels must be at least zero')\n    if submission[target_levels].min().min() < 0:\n        raise ParticipantVisibleError('All predictions must be at least zero')\n\n    solution['study_id'] = solution['row_id'].apply(lambda x: x.split('_')[0])\n    solution['location'] = solution['row_id'].apply(lambda x: '_'.join(x.split('_')[1:]))\n    solution['condition'] = solution['row_id'].apply(get_condition)\n\n    del solution[row_id_column_name]\n    del submission[row_id_column_name]\n    assert sorted(submission.columns) == sorted(target_levels)\n\n    submission['study_id'] = solution['study_id']\n    submission['location'] = solution['location']\n    submission['condition'] = solution['condition']\n\n    condition_losses = []\n    condition_weights = []\n    for condition in ['spinal', 'foraminal', 'subarticular']:\n        condition_indices = solution.loc[solution['condition'] == condition].index.values\n        condition_loss = sklearn.metrics.log_loss(\n            y_true=solution.loc[condition_indices, target_levels].values,\n            y_pred=submission.loc[condition_indices, target_levels].values,\n            sample_weight=solution.loc[condition_indices, 'sample_weight'].values\n        )\n        condition_losses.append(condition_loss)\n        condition_weights.append(1)\n\n    any_severe_spinal_labels = pd.Series(solution.loc[solution['condition'] == 'spinal'].groupby('study_id')['severe'].max())\n    any_severe_spinal_weights = pd.Series(solution.loc[solution['condition'] == 'spinal'].groupby('study_id')['sample_weight'].max())\n    any_severe_spinal_predictions = pd.Series(submission.loc[submission['condition'] == 'spinal'].groupby('study_id')['severe'].max())\n    any_severe_spinal_loss = sklearn.metrics.log_loss(\n        y_true=any_severe_spinal_labels,\n        y_pred=any_severe_spinal_predictions,\n        sample_weight=any_severe_spinal_weights\n    )\n    condition_losses.append(any_severe_spinal_loss)\n    condition_weights.append(any_severe_scalar)\n    return np.average(condition_losses, weights=condition_weights)","metadata":{"execution":{"iopub.status.busy":"2024-07-05T06:28:44.680758Z","iopub.execute_input":"2024-07-05T06:28:44.681074Z","iopub.status.idle":"2024-07-05T06:28:44.698804Z","shell.execute_reply.started":"2024-07-05T06:28:44.681047Z","shell.execute_reply":"2024-07-05T06:28:44.697283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating ground truth solution from the train set","metadata":{}},{"cell_type":"code","source":"solution = train_main.melt(id_vars=[\"study_id\"], var_name=\"full_label\", value_name=\"severity\")\nsolution[\"row_id\"] = solution.apply(lambda row: str(row.study_id) + \"_\" + row.full_label, axis=1)\nsolution.severity = solution.severity.fillna(\"Normal/Mild\")\nsolution.loc[solution.severity == \"Normal/Mild\", \"normal_mild\"] = 1\nsolution.loc[solution.severity == \"Moderate\", \"moderate\"] = 1\nsolution.loc[solution.severity == \"Severe\", \"severe\"] = 1\n\nsolution.loc[solution.severity == \"Normal/Mild\", \"sample_weight\"] = 1\nsolution.loc[solution.severity == \"Moderate\", \"sample_weight\"] = 2\nsolution.loc[solution.severity == \"Severe\", \"sample_weight\"] = 3\n\nsolution = solution[[\"study_id\", \"row_id\", \"normal_mild\", \"moderate\", \"severe\", \"sample_weight\"]]\nsolution = solution.fillna(0)\nsolution = solution.sort_values(by=[\"row_id\"])\nsolution.to_csv(\"temp_train_solution.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2024-07-05T06:28:44.701385Z","iopub.execute_input":"2024-07-05T06:28:44.701757Z","iopub.status.idle":"2024-07-05T06:28:45.994514Z","shell.execute_reply.started":"2024-07-05T06:28:44.701725Z","shell.execute_reply":"2024-07-05T06:28:45.993250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"solution.head()","metadata":{"execution":{"iopub.status.busy":"2024-07-05T06:28:45.995962Z","iopub.execute_input":"2024-07-05T06:28:45.996447Z","iopub.status.idle":"2024-07-05T06:28:46.018761Z","shell.execute_reply.started":"2024-07-05T06:28:45.996405Z","shell.execute_reply":"2024-07-05T06:28:46.017618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"solution.shape","metadata":{"execution":{"iopub.status.busy":"2024-07-05T06:28:46.020162Z","iopub.execute_input":"2024-07-05T06:28:46.020593Z","iopub.status.idle":"2024-07-05T06:28:46.027637Z","shell.execute_reply.started":"2024-07-05T06:28:46.020555Z","shell.execute_reply":"2024-07-05T06:28:46.026488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Generating prediction","metadata":{}},{"cell_type":"code","source":"\nrandom_pred = np.ones((solution.shape[0], 3)) / 3.0\nrandom_pred","metadata":{"execution":{"iopub.status.busy":"2024-07-05T06:28:46.029108Z","iopub.execute_input":"2024-07-05T06:28:46.030040Z","iopub.status.idle":"2024-07-05T06:28:46.041541Z","shell.execute_reply.started":"2024-07-05T06:28:46.029997Z","shell.execute_reply":"2024-07-05T06:28:46.040435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = solution.copy()[[\"row_id\", \"normal_mild\", \"moderate\", \"severe\"]]\nsubmission[[\"normal_mild\", \"moderate\", \"severe\"]] = random_pred\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2024-07-05T06:28:46.042987Z","iopub.execute_input":"2024-07-05T06:28:46.043607Z","iopub.status.idle":"2024-07-05T06:28:46.069551Z","shell.execute_reply.started":"2024-07-05T06:28:46.043569Z","shell.execute_reply":"2024-07-05T06:28:46.068462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"solution = solution[[\"row_id\", \"normal_mild\", \"moderate\", \"severe\", \"sample_weight\"]]\nsolution.head()","metadata":{"execution":{"iopub.status.busy":"2024-07-05T06:28:46.070746Z","iopub.execute_input":"2024-07-05T06:28:46.071081Z","iopub.status.idle":"2024-07-05T06:28:46.090954Z","shell.execute_reply.started":"2024-07-05T06:28:46.071052Z","shell.execute_reply":"2024-07-05T06:28:46.089837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Scoring and testing different orders of instances/rows","metadata":{}},{"cell_type":"code","source":"s = score(solution.copy(), submission.copy(), \"row_id\", 1)\nf\"Score {s}\"","metadata":{"execution":{"iopub.status.busy":"2024-07-05T06:28:46.093549Z","iopub.execute_input":"2024-07-05T06:28:46.093896Z","iopub.status.idle":"2024-07-05T06:28:46.315401Z","shell.execute_reply.started":"2024-07-05T06:28:46.093868Z","shell.execute_reply":"2024-07-05T06:28:46.314355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"s = score(solution.sort_values(\"row_id\").copy(), submission.sort_values(\"row_id\").copy(), \"row_id\", 1)\nf\"Score {s}\"","metadata":{"execution":{"iopub.status.busy":"2024-07-05T06:28:46.316541Z","iopub.execute_input":"2024-07-05T06:28:46.316868Z","iopub.status.idle":"2024-07-05T06:28:46.577027Z","shell.execute_reply.started":"2024-07-05T06:28:46.316840Z","shell.execute_reply":"2024-07-05T06:28:46.575881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"s = score(solution.sort_values(\"row_id\", ascending=True).copy(), submission.sort_values(\"row_id\", ascending=False).copy(), \"row_id\", 1)\nf\"Score {s}\"","metadata":{"execution":{"iopub.status.busy":"2024-07-05T06:28:46.578584Z","iopub.execute_input":"2024-07-05T06:28:46.579411Z","iopub.status.idle":"2024-07-05T06:28:46.865240Z","shell.execute_reply.started":"2024-07-05T06:28:46.579368Z","shell.execute_reply":"2024-07-05T06:28:46.863967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"s = score(solution.sort_values(\"row_id\", ascending=False).copy(), submission.sort_values(\"row_id\", ascending=True).copy(), \"row_id\", 1)\nf\"Score {s}\"","metadata":{"execution":{"iopub.status.busy":"2024-07-05T06:28:46.866847Z","iopub.execute_input":"2024-07-05T06:28:46.867325Z","iopub.status.idle":"2024-07-05T06:28:47.165914Z","shell.execute_reply.started":"2024-07-05T06:28:46.867282Z","shell.execute_reply":"2024-07-05T06:28:47.164739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}