{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"},{"sourceId":183798121,"sourceType":"kernelVersion"},{"sourceId":186726771,"sourceType":"kernelVersion"},{"sourceId":189037487,"sourceType":"kernelVersion"}],"dockerImageVersionId":30732,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nimport numpy as np \nimport pandas as pd\nimport os\nimport pydicom\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport PIL.Image\nfrom io import BytesIO\nimport cv2\n\nfrom IPython.display import display\nfrom ipywidgets import interact_manual, interact\nfrom ipywidgets import GridBox, Box, Layout\nfrom ipywidgets import Dropdown, Label, Text, Image, Button, HTML, ToggleButton, IntSlider, FloatLogSlider, Checkbox\n\ntrain_root = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images'\n","metadata":{"execution":{"iopub.status.busy":"2024-08-20T11:25:31.747967Z","iopub.execute_input":"2024-08-20T11:25:31.748418Z","iopub.status.idle":"2024-08-20T11:25:31.756656Z","shell.execute_reply.started":"2024-08-20T11:25:31.748384Z","shell.execute_reply":"2024-08-20T11:25:31.755274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\ndef clip_extreme_values(image, low_percentile=1, high_percentile=99):\n    lower_bound = np.percentile(image, low_percentile).astype(image.dtype)\n    upper_bound = np.percentile(image, high_percentile).astype(image.dtype)\n    clipped_image = np.clip(image, lower_bound, upper_bound)\n    return clipped_image\n\ndef histogram_equalization(img, precision=2048): \n    img = (img.astype(np.float32) - float(img.min())) / float(img.max() - float(img.min()))\n    hist, bins = np.histogram(img.flatten(), precision, [0,1])\n    hist[0] = 0\n    cdf = hist.cumsum()\n    cdf_m = np.ma.masked_equal(cdf, 0)\n    cdf_m = (cdf_m - cdf_m.min()) / (cdf_m.max() - cdf_m.min())\n    cdf = np.ma.filled(cdf_m, 0)\n    img = cdf[((precision - 1) * img).astype(np.int64)]\n    return img\n\ndef dicom_preprocess(dicom_obj, apply_he=True):\n    img = dicom_obj.pixel_array\n    val_max, val_min = img.max(), img.min()\n    if dicom_obj.PhotometricInterpretation == 'MONOCHROME1':\n        img = val_max + val_min - img # invert\n    img = img - img.min()\n    img = img.astype(np.uint16)\n    img = clip_extreme_values(img)\n    if apply_he:\n        img = cv2.createCLAHE().apply(img)\n#         img = histogram_equalization(img)\n    img = (img.astype(np.float32) - float(img.min())) / float(img.max() - float(img.min()))\n    img = (255 * img).round().astype(np.uint8)\n    return img\n\n","metadata":{"execution":{"iopub.status.busy":"2024-08-20T10:54:48.751540Z","iopub.execute_input":"2024-08-20T10:54:48.752341Z","iopub.status.idle":"2024-08-20T10:54:48.766011Z","shell.execute_reply.started":"2024-08-20T10:54:48.752306Z","shell.execute_reply":"2024-08-20T10:54:48.764951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_df = pd.read_csv('/kaggle/input/rsna2024-file-meta-to-dataframe/dicom_meta.csv')\ntrain_df = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train.csv')\nlabel_coordinates_df = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_label_coordinates.csv')\nseries_descriptions_df = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_series_descriptions.csv')\n\n","metadata":{"execution":{"iopub.status.busy":"2024-08-20T10:54:51.011196Z","iopub.execute_input":"2024-08-20T10:54:51.012154Z","iopub.status.idle":"2024-08-20T10:54:52.912630Z","shell.execute_reply.started":"2024-08-20T10:54:51.012118Z","shell.execute_reply":"2024-08-20T10:54:52.911499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_description_dict = {str(row['series_id']):row['series_description'] \n                           for _, row in series_descriptions_df.iterrows()}\nseries_id2study_id = {str(row['series_id']):str(row['study_id']) \n                      for _, row in series_descriptions_df.iterrows()}\nseries_id2coords_index = dict()\nfor i, row in tqdm(label_coordinates_df.iterrows(), total=len(label_coordinates_df)):\n    key = str(row['series_id'])\n    item = series_id2coords_index.get(key, [])\n    series_id2coords_index[key] = item\n    item.append(i)\n    \n# to_meta_dict = dict()\n# for i, row in tqdm(meta_df.iterrows(), total=len(meta_df)):\n#     study_id, series_id, fn = row[[\"study_id\", \"series_id\", \"fn\"]]\n#     study_id, series_id, instance_id = str(study_id), str(series_id), os.path.splitext(fn)[0]\n#     study_item = to_meta_dict.get(study_id, dict())\n#     series_item = study_item.get(series_id, dict())\n#     to_meta_dict[study_id] = study_item\n#     study_item[series_id] = series_item\n#     series_item[instance_id] = row\n","metadata":{"execution":{"iopub.status.busy":"2024-08-20T11:28:42.765452Z","iopub.execute_input":"2024-08-20T11:28:42.765876Z","iopub.status.idle":"2024-08-20T11:29:55.623305Z","shell.execute_reply.started":"2024-08-20T11:28:42.765847Z","shell.execute_reply":"2024-08-20T11:29:55.622215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# results = {study_id:{\n#     'Sagittal T2/STIR':0,\n#     'Sagittal T1':0,\n#     'Axial T2':0,\n# }\n#  for study_id in np.unique(series_descriptions_df['study_id'])}\n\n# for i, row in series_descriptions_df.iterrows():\n#     results[row['study_id']][row['series_description']] += 1\n# result2 = []\n# counts = 0\n# for study_id, items in results.items():\n#     for des, c in items.items():\n#         if c >= 1 and des == 'Sagittal T1':\n#             rows = series_descriptions_df.loc[series_descriptions_df['study_id'] == study_id]\n#             series_ids = rows['series_id'].tolist()\n#             series_ids = [str(series_id) for series_id in series_ids \n#                           if series_description_dict[str(series_id)] == 'Sagittal T1']\n#             if any([series_id not in series_id2coords_index for series_id in series_ids]):\n#                 continue\n#             indices_list = [series_id2coords_index[series_id] for series_id in series_ids]\n#             indices = [i for indices in indices_list for i in indices]\n#             level_counts = [0, 0, 0, 0, 0]\n#             for i in indices:\n#                 level_counts[\n#                     {\n#                         'L1/L2':0,\n#                         'L2/L3':1,\n#                         'L3/L4':2,\n#                         'L4/L5':3,\n#                         'L5/S1':4,\n#                     }[label_coordinates_df['level'].iloc[i]]\n#                 ] += 1 \n#             result2.append(level_counts)\n#             if any([c < 2 for c in level_counts]):\n#                 print(study_id, level_counts)\n#                 counts += 1\n    \n# counts","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# label_coordinates_df","metadata":{"execution":{"iopub.status.busy":"2024-07-20T07:35:56.684324Z","iopub.execute_input":"2024-07-20T07:35:56.684759Z","iopub.status.idle":"2024-07-20T07:35:56.691649Z","shell.execute_reply.started":"2024-07-20T07:35:56.684726Z","shell.execute_reply":"2024-07-20T07:35:56.690315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sag_t1_missing = [\n    [4127200281, 'L3/L4', 13, 149, 175, 'Right Neural Foraminal Narrowing'], # Right Neural Foraminal Narrowing\n]\n\nsag_t2_missing = [\n    [346177997, 'L1/L2', 8, 263, 138],\n    [346177997, 'L2/L3', 8, 249, 190],\n    [1629655671, 'L1/L2', 7, 251, 112],\n    [687903591, 'L1/L2', 8, 314, 183],\n    [354867316, 'L1/L2', 7, 350, 144],\n    [2498767947, 'L1/L2', 8, 331, 170],\n    [2498767947, 'L2/L3', 8, 327, 230],\n    [3154521770, 'L1/L2', 8, 281, 143],\n    [3142410579, 'L1/L2', 12, 481, 248],\n    [3046865229, 'L1/L2', 9, 227, 109],\n    [3640628768, 'L1/L2', 8, 365, 170],\n    [833184386, 'L1/L2', 9, 488, 314],\n    [833184386, 'L2/L3', 9, 466, 400],\n    [691961783, 'L1/L2', 7, 313, 169],\n    [691961783, 'L2/L3', 7, 297, 222],\n    [521193458, 'L1/L2', 8, 472, 265],\n    [521193458, 'L2/L3', 8, 461, 358],\n    [121254627, 'L1/L2', 8, 319, 179],\n    [121254627, 'L2/L3', 8, 298, 243],\n    [987058749, 'L1/L2', 8, 341, 317],\n    [987058749, 'L2/L3', 8, 325, 398],\n    [1607547232, 'L2/L3', 8, 370, 308],\n    [4119560068, 'L1/L2', 8, 479, 338],\n    [2836494846, 'L1/L2', 8, 301, 177],\n    [4147986695, 'L1/L2', 11, 277, 221],\n    [4147986695, 'L2/L3', 11, 264, 280],\n    [4147986695, 'L3/L4', 11, 256, 347],\n    [4147986695, 'L4/L5', 12, 247, 416],\n    [4147986695, 'L5/S1', 12, 244, 468],\n    [475861931, 'L1/L2', 10, 272, 47],\n    [3196351526, 'L1/L2', 8, 262, 187],\n    [3196351526, 'L2/L3', 8, 256, 231],\n    [3231229349, 'L1/L2', 12, 217, 121],\n    [1992065903, 'L1/L2', 8, 407, 201],\n    [1992065903, 'L2/L3', 8, 381, 293],\n    [1848330865, 'L1/L2', 8, 335, 158],\n    [1616142642, 'L1/L2', 8, 259, 191],\n    [1616142642, 'L2/L3', 8, 259, 191],\n    [3157189845, 'L5/S1', 8, 167, 209],\n    [721342892, 'L1/L2', 10, 250, 160],\n    [1317312291, 'L1/L2', 8, 439, 171],\n    [1317312291, 'L2/L3', 8, 424, 260],\n    [1396432972, 'L1/L2', 11, 263, 169],\n    [1358207531, 'L1/L2', 8, 458, 237],\n    [1358207531, 'L2/L3', 8, 452, 334],\n    [1832528536, 'L1/L2', 7, 324, 137],\n    [1832528536, 'L2/L3', 7, 321, 182],\n    [3522304723, 'L1/L2', 9, 270, 171],\n    [3522304723, 'L2/L3', 9, 270, 224],\n    [605159348, 'L1/L2', 10, 261, 175],\n    [50386656, 'L1/L2', 8, 456, 255],\n    [50386656, 'L2/L3', 8, 450, 342],\n    [2376943158, 'L1/L2', 7, 309, 240],\n    [2376943158, 'L2/L3', 7, 294, 282],\n    [831870936, 'L1/L2', 7, 552, 260],\n    [831870936, 'L2/L3', 7, 554, 365],\n    [3636907775, 'L1/L2', 8, 284, 138],\n    [3636907775, 'L2/L3', 8, 276, 188],\n    [564062395, 'L1/L2', 9, 292, 132],\n    [564062395, 'L2/L3', 9, 280, 186],\n    [2844011887, 'L1/L2', 7, 276, 172],\n    [2844011887, 'L2/L3', 7, 267, 226],\n    [1208240347, 'L1/L2', 9, 184, 97],\n    [1208240347, 'L2/L3', 9, 176, 138],\n    [3913032764, 'L1/L2', 9, 210, 93],\n    [3913032764, 'L2/L3', 9, 200, 145],\n    [1304954941, 'L1/L2', 11, 432, 227],\n    [1304954941, 'L2/L3', 11, 397, 318],\n    [619645067, 'L1/L2', 8, 496, 121],\n    [1852465903, 'L1/L2', 7, 302, 173],\n    [1852465903, 'L2/L3', 7, 285, 237],\n    [647735603, 'L1/L2', 7, 326, 149],\n    [379008764, 'L1/L2', 8, 304, 136],\n    [379008764, 'L2/L3', 8, 297, 186],\n    [3433809812, 'L1/L2', 10, 232, 158],\n    [3433809812, 'L2/L3', 10, 229, 203],\n    [2143604834, 'L1/L2', 8, 205, 75],\n    [2143604834, 'L2/L3', 8, 192, 120],\n    [4025662051, 'L1/L2', 9, 282, 149],\n    [3551899591, 'L1/L2', 8, 276, 188],\n    [3551899591, 'L2/L3', 8, 269, 232],\n    [3939069145, 'L1/L2', 6, 338, 183],\n    [4174470410, 'L1/L2', 7, 278, 165],\n    [3359428067, 'L1/L2', 14, 351, 145],\n    [3359428067, 'L2/L3', 14, 342, 212],\n    [1277039062, 'L1/L2', 7, 289, 182],\n    [1277039062, 'L2/L3', 7, 282, 232],\n    [1915807167, 'L1/L2', 12, 320, 182],\n    [992741205, 'L1/L2', 9, 284, 174],\n    [992741205, 'L2/L3', 9, 266, 232],\n    [2327425347, 'L1/L2', 9, 473, 216],\n    [2327425347, 'L2/L3', 9, 448, 320],\n    [4135322219, 'L1/L2', 9, 286, 165],\n    [339473966, 'L1/L2', 10, 465, 305],\n    [2933836082, 'L1/L2', 8, 382, 206],\n    [490266903, 'L1/L2', 8, 462, 207],\n    [490266903, 'L2/L3', 8, 448, 282],\n    [3562294103, 'L1/L2', 7, 508, 179],\n    [3562294103, 'L2/L3', 7, 477, 277],\n    [1314846886, 'L1/L2', 8, 483, 265],\n    [3308149973, 'L1/L2', 7, 328, 230],\n    [152760150, 'L1/L2', 8, 276, 138],\n    [152760150, 'L2/L3', 8, 271, 187],\n    [4051330926, 'L1/L2', 7, 289, 161],\n    [4051330926, 'L2/L3', 7, 272, 223],\n    [9334563, 'L1/L2', 8, 409, 243],\n    [9334563, 'L2/L3', 8, 404, 334],\n    [2116827468, 'L1/L2', 14, 470, 248],\n    [2421706412, 'L1/L2', 11, 405, 154],\n    [578994413, 'L1/L2', 8, 480, 252],\n    [3451027225, 'L1/L2', 8, 462, 208],\n    [3451027225, 'L2/L3', 8, 428, 322],\n    [4089185953, 'L1/L2', 8, 314, 94],\n    [2575571960, 'L1/L2', 8, 373, 215],\n    [2676982925, 'L1/L2', 12, 461, 316],\n    [2231471633, 'L4/L5', 9, 550, 565],\n    [2231471633, 'L5/S1', 9, 560, 645],\n]\nsag_t2_incorrect = [\n    [880361156, 'L5/S1', 183, 218],\n    [1921917205, 'L5/S1', 327, 428],\n    [221289021, 'L5/S1', 111, 173],\n    [2231471633, 'L1/L2', 600, 250],\n    [737753815, 'L1/L2', 240, 150],\n    [1488857550, 'L4/L5', 175, 200],\n    [3736941525, 'L5/S1', 270, 410],\n    [1490272456, 'L3/L4', 330, 220],\n    [3086719329, 'L3/L4', 290, 268],\n    [3072714332, 'L4/L5', 300, 350],\n    [1485193299, 'L1/L2', 205, 120],\n    [3521409198, 'L5/S1', 208, 310],\n    [816381378, 'L1/L2', 208, 130], \n    [131094096, 'L2/L3', 275, 175], \n    [1735851779, 'L5/S1', 265, 340], \n]\n# series_id, instance_number, level, x, y\n# no coordinate: 3008676218\n# problems: study 3225351618\n# delete series 4147986695 instance 1\n# series 3892989905 not waist\n\nsag_t1_missing = pd.DataFrame(sag_t1_missing, \n                              columns=['series_id', 'level', 'instance_number', 'x', 'y', 'condition'])\nsag_t2_missing = pd.DataFrame(sag_t2_missing, \n                              columns=['series_id', 'level', 'instance_number', 'x', 'y'])\nsag_t2_incorrect = pd.DataFrame(sag_t2_incorrect, \n                                columns=['series_id', 'level', 'x', 'y'])\n\n\ndef rebuild_label_coordinates_df(label_coordinates_df):\n    series_ids = list(set(series_descriptions_df['series_id']))\n    results = []\n    for series_id in tqdm(series_ids):\n        if str(series_id) not in series_id2coords_index:\n            continue\n        indices = series_id2coords_index[str(series_id)]\n        rows = label_coordinates_df.iloc[indices]\n        study_id = rows.iloc[0]['study_id']\n        sag_t1_missing_rows = sag_t1_missing.loc[sag_t1_missing['series_id'] == series_id]\n        sag_t2_missing_rows = sag_t2_missing.loc[sag_t2_missing['series_id'] == series_id]\n        sag_t2_incorrect_rows = sag_t2_incorrect.loc[sag_t2_incorrect['series_id'] == series_id]\n        \n        row_results = []\n        for _, row in rows.iterrows():\n            if row['level'] in sag_t2_incorrect_rows['level']:\n                correct_row = sag_t2_incorrect_rows.loc[sag_t2_incorrect_rows['level'] == row['level']]\n                row_results.append([row['study_id'], row['series_id'], row['instance_number'], row['condition'], row['level'], correct_row['x'], correct_row['y']])\n            else:\n                row_results.append(row.tolist())\n        for _, row in sag_t1_missing_rows.iterrows():\n            row_results.append([study_id, row['series_id'], row['instance_number'], row['condition'], row['level'], row['x'], row['y']])\n        for _, row in sag_t2_missing_rows.iterrows():\n            row_results.append([study_id, row['series_id'], row['instance_number'], 'Spinal Canal Stenosis', row['level'], row['x'], row['y']])\n        \n        results.extend(row_results)\n    results = pd.DataFrame(results, \n                           columns=['study_id', 'series_id', 'instance_number', 'condition', 'level', 'x', 'y'])\n    results = results.iloc[results['study_id'].argsort()]\n    return results\n\n# label_coordinates_df = rebuild_label_coordinates_df(label_coordinates_df)\n# label_coordinates_df.to_csv('train_label_coordinates.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-07-20T08:06:42.434426Z","iopub.execute_input":"2024-07-20T08:06:42.434945Z","iopub.status.idle":"2024-07-20T08:06:42.960876Z","shell.execute_reply.started":"2024-07-20T08:06:42.434904Z","shell.execute_reply":"2024-07-20T08:06:42.959720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def image_to_realword_coordinate(coords, positions, orientations, pixel_spacing):\n    mat = np.array([\n        [orientations[0] * pixel_spacing[0], orientations[3] * pixel_spacing[1], 0, positions[0]],\n        [orientations[1] * pixel_spacing[0], orientations[4] * pixel_spacing[1], 0, positions[1]],\n        [orientations[2] * pixel_spacing[0], orientations[5] * pixel_spacing[1], 0, positions[2]],\n        [0, 0, 0, 1],\n    ])\n    coords = np.array([coords[0], coords[1], 0, 1]).reshape(4, 1)\n    new_coords = mat @ coords\n    return new_coords.ravel()[:3]\n\ndef realword_to_image_coordinate(coords, positions, orientations, pixel_spacing):\n    mat = np.array([\n        [orientations[0] / pixel_spacing[0], orientations[3] / pixel_spacing[1]],\n        [orientations[1] / pixel_spacing[0], orientations[4] / pixel_spacing[1]],\n        [orientations[2] / pixel_spacing[0], orientations[5] / pixel_spacing[1]],\n    ]).T\n    offset = np.array(positions).reshape(3, 1)\n    coords = np.array([coords[0], coords[1], coords[2]]).reshape(3, 1)\n    new_coords = mat @ (coords - offset)\n    return new_coords.ravel()\n\ndef build_condition_level_to_coordinates(rows, to_realword=False):\n    levels = ['L1/L2', 'L2/L3', 'L3/L4', 'L4/L5', 'L5/S1']\n    results = {\n        'Spinal Canal Stenosis': {level: None for level in levels},\n        'Left Neural Foraminal Narrowing': {level: None for level in levels},\n        'Right Neural Foraminal Narrowing': {level: None for level in levels},\n        'Left Subarticular Stenosis': {level: None for level in levels},\n        'Right Subarticular Stenosis': {level: None for level in levels},\n    }\n    for _, row in rows.iterrows():\n        meta = to_meta_dict[str(row['study_id'])][str(row['series_id'])][str(row['instance_number'])]\n        image_coord = np.array(row[['x', 'y']].tolist())\n        if to_realword:\n            real_coord = image_to_realword_coordinate(\n                image_coord, \n                meta[['ImagePositionPatient_0', 'ImagePositionPatient_1', 'ImagePositionPatient_2']].tolist(),\n                meta[[f'ImageOrientationPatient_{i}' for i in range(6)]].tolist(),\n                meta[['PixelSpacing_0', 'PixelSpacing_1']].tolist()\n            )\n            results[row['condition']][row['level']] = real_coord\n        else:\n            results[row['condition']][row['level']] = image_coord\n    return results\n        \ndef check_sagittal_t1_match_sagittal_t2(err_thresh=15):\n    results = {\n        'missing_id': [],\n        'missing_coords': [],\n        'unmatch': [],\n    }\n    study_ids = list(set(train_df['study_id']))\n    study_ids = [str(study_id) for study_id in study_ids]\n    for study_id in tqdm(study_ids):\n        series_ids = os.listdir(os.path.join(train_root, study_id))\n        descriptions = {series_description_dict[series_id]:series_id \n                        for series_id in series_ids}\n        sagittal_t1_series_id = descriptions.get('Sagittal T1', None)\n        sagittal_t2_series_id = descriptions.get('Sagittal T2/STIR', None)\n        t1_indices = series_id2coords_index.get(sagittal_t1_series_id, None)\n        t2_indices = series_id2coords_index.get(sagittal_t2_series_id, None)\n        if t1_indices is None:\n            results['missing_id'].append((study_id, 'Sagittal T1'))\n        if t2_indices is None:\n            results['missing_id'].append((study_id, 'Sagittal T2/STIR'))\n        if t1_indices is None or t2_indices is None:\n            continue\n        \n        t1_rows = label_coordinates_df.iloc[t1_indices]\n        t2_rows = label_coordinates_df.iloc[t2_indices]\n        t1_coords_dict = build_condition_level_to_coordinates(t1_rows, to_realword=False)\n        t2_coords_dict = build_condition_level_to_coordinates(t2_rows, to_realword=True)\n        t1_meta = to_meta_dict[str(t1_rows.iloc[0]['study_id'])][str(t1_rows.iloc[0]['series_id'])][str(t1_rows.iloc[0]['instance_number'])]\n        \n        for level in ['L1/L2', 'L2/L3', 'L3/L4', 'L4/L5', 'L5/S1']:\n            t1_coord1 = t1_coords_dict['Left Neural Foraminal Narrowing'][level]\n            t1_coord2 = t1_coords_dict['Right Neural Foraminal Narrowing'][level]\n            real_t2_coord = t2_coords_dict['Spinal Canal Stenosis'][level]\n            if t1_coord1 is None or t1_coord2 is None or real_t2_coord is None:\n                results['missing_coords'].append((study_id, level))\n                continue\n            t2_coord = realword_to_image_coordinate( # project to coordinates on sag t1\n                real_t2_coord, \n                t1_meta[['ImagePositionPatient_0', 'ImagePositionPatient_1', 'ImagePositionPatient_2']].tolist(),\n                t1_meta[[f'ImageOrientationPatient_{i}' for i in range(6)]].tolist(),\n                t1_meta[['PixelSpacing_0', 'PixelSpacing_1']].tolist()\n            )\n#             offset = t1_coord2 - t1_coord1\n#             vec = real_t2_coord - t1_coord1\n#             expected_t2_coord = t1_coord1 + offset * np.dot(offset, vec) / np.sum(offset ** 2)\n            err = max((np.sum(t1_coord1 - t2_coord)**2)**0.5, (np.sum(t1_coord2 - t2_coord)**2)**0.5)\n            err /= (t1_meta['Columns'] * t1_meta['Rows']) **0.5 * 320\n            if err > err_thresh:\n                results['unmatch'].append((study_id, level, err))\n    return results\n\ndef check_sagittal_t1_itself(err_thresh=15):\n    results = {\n        'missing_id': [],\n        'missing_coords': [],\n        'unmatch': [],\n    }\n    study_ids = list(set(train_df['study_id']))\n    study_ids = [str(study_id) for study_id in study_ids]\n    for study_id in tqdm(study_ids):\n        series_ids = os.listdir(os.path.join(train_root, study_id))\n        descriptions = {series_description_dict[series_id]:series_id \n                        for series_id in series_ids}\n        sagittal_t1_series_id = descriptions.get('Sagittal T1', None)\n        t1_indices = series_id2coords_index.get(sagittal_t1_series_id, None)\n        if t1_indices is None:\n            results['missing_id'].append((study_id, 'Sagittal T1'))\n        if t1_indices is None:\n            continue\n        t1_rows = label_coordinates_df.iloc[t1_indices]\n        t1_coords_dict = build_condition_level_to_coordinates(t1_rows, to_realword=False)\n        meta = to_meta_dict[str(t1_rows.iloc[0]['study_id'])][str(t1_rows.iloc[0]['series_id'])][str(t1_rows.iloc[0]['instance_number'])]\n        for level in ['L1/L2', 'L2/L3', 'L3/L4', 'L4/L5', 'L5/S1']:\n            t1_coord1 = t1_coords_dict['Left Neural Foraminal Narrowing'][level]\n            t1_coord2 = t1_coords_dict['Right Neural Foraminal Narrowing'][level]\n            if t1_coord1 is None or t1_coord2 is None:\n                results['missing_coords'].append((study_id, level))\n                continue\n            \n            err = (np.sum(t1_coord1 - t1_coord2)**2)**0.5 / (meta['Columns'] * meta['Rows']) **0.5 *  320\n            if err > err_thresh:\n                results['unmatch'].append((study_id, level, err))\n    return results\n\ndef check_sagittal_t1_match_axial_t2(err_thresh=40):\n    results = {\n        'missing_id': [],\n        'missing_coords': [],\n        'unmatch': [],\n    }\n    study_ids = list(set(train_df['study_id']))\n    study_ids = [str(study_id) for study_id in study_ids]\n    for study_id in tqdm(study_ids):\n        series_ids = os.listdir(os.path.join(train_root, study_id))\n        descriptions = {series_description_dict[series_id]:series_id \n                        for series_id in series_ids}\n        sag_series_id = descriptions.get('Sagittal T1', None)\n        axi_series_id = descriptions.get('Axial T2', None)\n        sag_indices = series_id2coords_index.get(sag_series_id, None)\n        axi_indices = series_id2coords_index.get(axi_series_id, None)\n        if sag_indices is None:\n            results['missing_id'].append((study_id, 'Sagittal T1'))\n        if axi_indices is None:\n            results['missing_id'].append((study_id, 'Axial T2'))\n        if sag_indices is None or axi_indices is None:\n            continue\n        \n        sag_rows = label_coordinates_df.iloc[sag_indices]\n        axi_rows = label_coordinates_df.iloc[axi_indices]\n        sag_coords_dict = build_condition_level_to_coordinates(sag_rows, to_realword=True)\n        axi_coords_dict = build_condition_level_to_coordinates(axi_rows, to_realword=True)\n        for level in ['L1/L2', 'L2/L3', 'L3/L4', 'L4/L5', 'L5/S1']:\n            left_sag_coord = sag_coords_dict['Left Neural Foraminal Narrowing'][level]\n            right_sag_coord = sag_coords_dict['Right Neural Foraminal Narrowing'][level]\n            left_axi_coord = axi_coords_dict['Right Subarticular Stenosis'][level]\n            right_axi_coord = axi_coords_dict['Right Subarticular Stenosis'][level]\n            if left_sag_coord is None or right_sag_coord is None or left_axi_coord is None or left_axi_coord is None:\n                results['missing_coords'].append((study_id, level))\n                continue\n            left_err = np.sum((left_sag_coord - left_axi_coord)**2)**0.5\n            right_err = np.sum((right_sag_coord - right_axi_coord)**2)**0.5\n            err = max(left_err, right_err)\n            if err > err_thresh:\n                results['unmatch'].append((study_id, level, err))\n    return results\n\nresult1 = check_sagittal_t1_match_sagittal_t2(0)\nresult2 = check_sagittal_t1_itself(0)\nresult3 = check_sagittal_t1_match_axial_t2(0)\n\nid1, level1, err1 = list(zip(*result1['unmatch']))\nid2, level2, err2 = list(zip(*result2['unmatch']))\nid3, level3, err3 = list(zip(*result3['unmatch']))","metadata":{"execution":{"iopub.status.busy":"2024-08-20T10:55:13.433089Z","iopub.status.idle":"2024-08-20T10:55:13.433642Z","shell.execute_reply.started":"2024-08-20T10:55:13.433372Z","shell.execute_reply":"2024-08-20T10:55:13.433396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # plt.plot(sorted(err2))\n# plt.plot([np.sum(np.array(err2) > i) for i in range(260, 800)])\n\n# np.array(id2)[err2 > 260]","metadata":{"execution":{"iopub.status.busy":"2024-07-20T08:04:18.603639Z","iopub.execute_input":"2024-07-20T08:04:18.604103Z","iopub.status.idle":"2024-07-20T08:04:18.610805Z","shell.execute_reply.started":"2024-07-20T08:04:18.604069Z","shell.execute_reply":"2024-07-20T08:04:18.609023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# study_id, series_id, instance_number, description\nproblematic_df = [\n    [490052995, None, None, 'Two Sagittal T1, but w/o Left/Right Neural Foraminal Narrowing, respectively.'],\n    [1261271580, None, None, 'Two Sagittal T1, but w/o Left/Right Neural Foraminal Narrowing, respectively.'],\n    [2507107985, None, None, 'Two Sagittal T1, but w/o Left/Right Neural Foraminal Narrowing, respectively.'],\n    [2626030939, None, None, 'Two Sagittal T1, but w/o Left/Right Neural Foraminal Narrowing, respectively.'],\n    [2773343225, None, None, 'Two Sagittal T1, but w/o Left/Right Neural Foraminal Narrowing, respectively.'],\n    [3109648055, None, None, 'Two Sagittal T1, but w/o Left/Right Neural Foraminal Narrowing, respectively.'],\n    [3387993595, None, None, 'Two Sagittal T1, but w/o Left/Right Neural Foraminal Narrowing, respectively.'],\n    \n    [3008676218, None, None, 'No coordinates, no Sagittal T2/STIR.'],\n    [3225351618, None, None, 'Coordinates of Left/Right Neural Foraminal Narrowing in Sagittal T1 are inconsistent. Coordinates of Sagittal T2/STIR may incorrect.'],\n#     [3294654272, 4147986695, 1, 'Coordinate of Spinal Canal Stenosis in Sagittal T2/STIR is incorrect.'],\n    [3637444890, None, None, 'L1/L2-L5/S1 are not in Sagittal T2/STIR. Coordinates of Spinal Canal Stenosis are given in Sagittal T1.']\n]\n\ndelete_study_ids = [\n    490052995, 1261271580, 2507107985, 2626030939, 2773343225, 3109648055, 3387993595, 3008676218,\n    3225351618, 3637444890\n]\n","metadata":{"execution":{"iopub.status.busy":"2024-07-20T08:05:53.724556Z","iopub.execute_input":"2024-07-20T08:05:53.725050Z","iopub.status.idle":"2024-07-20T08:05:53.735579Z","shell.execute_reply.started":"2024-07-20T08:05:53.725013Z","shell.execute_reply":"2024-07-20T08:05:53.734005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from PIL import Image\n\n\n# def convert_to_png(train_root, target_root):\n#     study_ids = os.listdir(train_root)\n#     for study_id in tqdm(study_ids):\n#         series_ids = os.listdir(os.path.join(train_root, study_id))\n#         for series_id in series_ids:\n#             os.makedirs(os.path.join(target_root, study_id, series_id), exist_ok=True)\n            \n#             file_names = os.listdir(os.path.join(train_root, study_id, series_id))\n#             for file_name in file_names:\n#                 dicom = pydicom.dcmread(os.path.join(train_root, study_id, series_id, file_name))\n#                 arr = dicom_preprocess(dicom, True)\n#                 img = Image.fromarray(arr)\n#                 target_file_name = os.path.splitext(file_name)[0] + '.png'\n#                 img.save(os.path.join(target_root, study_id, series_id, target_file_name))\n\n# convert_to_png(train_root, './train_images')","metadata":{"execution":{"iopub.status.busy":"2024-07-19T18:04:08.782087Z","iopub.execute_input":"2024-07-19T18:04:08.782572Z","iopub.status.idle":"2024-07-19T18:04:26.768844Z","shell.execute_reply.started":"2024-07-19T18:04:08.782533Z","shell.execute_reply":"2024-07-19T18:04:26.767030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df.loc[~train_df['study_id'].isin(delete_study_ids)].to_csv('train.csv', index=False)\n# label_coordinates_df\n# series_descriptions_df","metadata":{"execution":{"iopub.status.busy":"2024-07-20T08:07:04.202739Z","iopub.execute_input":"2024-07-20T08:07:04.203176Z","iopub.status.idle":"2024-07-20T08:07:04.255468Z","shell.execute_reply.started":"2024-07-20T08:07:04.203143Z","shell.execute_reply":"2024-07-20T08:07:04.254304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Correct Axial T2","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/rsna2024-check-incorrectly-labeled-data/train.csv')\nlabel_coordinates_df = pd.read_csv('/kaggle/input/rsna2024-check-incorrectly-labeled-data/train_label_coordinates.csv')\n\nseries_to_coords_dict = dict()\nfor i, row in tqdm(label_coordinates_df.iterrows(), total=len(label_coordinates_df)):\n    key = str(row['series_id'])\n    item = series_to_coords_dict.get(key, [])\n    series_to_coords_dict[key] = item\n    item.append(row)\nseries_to_coords_dict = {series_id: pd.DataFrame([row.tolist() for row in rows], columns=rows[0].keys()) for series_id, rows in series_to_coords_dict.items()}\n","metadata":{"execution":{"iopub.status.busy":"2024-08-20T11:26:45.943003Z","iopub.execute_input":"2024-08-20T11:26:45.943379Z","iopub.status.idle":"2024-08-20T11:26:52.685691Z","shell.execute_reply.started":"2024-08-20T11:26:45.943353Z","shell.execute_reply":"2024-08-20T11:26:52.684430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unpaired_axiel_ids = []\n\nfor series_id, coord_rows in series_to_coords_dict.items():\n    if series_description_dict[series_id] == 'Axial T2':\n        dice = dict()\n        for _, row in coord_rows[['condition', 'level']].iterrows():\n            dice[row['level']] = dice.get(row['level'], 0)\n            dice[row['level']] += 1\n        b = all([c == 2 for level, c in dice.items()])\n        if not b:\n            unpaired_axiel_ids.append(series_id)\n            \nresults = dict()\nfor series_id in unpaired_axiel_ids:\n    study_id = series_id2study_id[series_id]\n    results[study_id] = results.get(study_id, [])\n    results[study_id].append(series_id)           \n# results = results.iloc[results['study_id'].argsort()]\ncoord_rows","metadata":{"execution":{"iopub.status.busy":"2024-08-20T11:30:53.202995Z","iopub.execute_input":"2024-08-20T11:30:53.203410Z","iopub.status.idle":"2024-08-20T11:30:55.882379Z","shell.execute_reply.started":"2024-08-20T11:30:53.203377Z","shell.execute_reply":"2024-08-20T11:30:55.881173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def image_to_realword_coordinate(coords, positions, orientations, pixel_spacing):\n    mat = np.array([\n        [orientations[0] * pixel_spacing[0], orientations[3] * pixel_spacing[1], 0, positions[0]],\n        [orientations[1] * pixel_spacing[0], orientations[4] * pixel_spacing[1], 0, positions[1]],\n        [orientations[2] * pixel_spacing[0], orientations[5] * pixel_spacing[1], 0, positions[2]],\n        [0, 0, 0, 1],\n    ])\n    coords = np.array([coords[0], coords[1], 0, 1]).reshape(4, 1)\n    new_coords = mat @ coords\n    return new_coords.ravel()[:3]\n\ndef realword_to_image_coordinate(coords, positions, orientations, pixel_spacing):\n    mat = np.array([\n        [orientations[0] / pixel_spacing[0], orientations[3] / pixel_spacing[1]],\n        [orientations[1] / pixel_spacing[0], orientations[4] / pixel_spacing[1]],\n        [orientations[2] / pixel_spacing[0], orientations[5] / pixel_spacing[1]],\n    ]).T\n    offset = np.array(positions).reshape(3, 1)\n    coords = np.array([coords[0], coords[1], coords[2]]).reshape(3, 1)\n    new_coords = mat @ (coords - offset)\n    return new_coords.ravel()\n\nnew_rows = []\nfor study_id, series_ids in results.items():\n    levels_dict = dict()\n    instances_dict = dict()\n    for series_id in series_ids:\n        coord_rows = series_to_coords_dict[series_id]\n        for _, row in coord_rows.iterrows():\n            level = row['level']\n            condition = row['condition']\n            position = row[['x', 'y']]\n            dice = levels_dict.get(level, {'Left Subarticular Stenosis':[], 'Right Subarticular Stenosis': []})\n            levels_dict[level] = dice\n            meta = to_meta_dict[study_id][series_id][str(row[\"instance_number\"])]\n            position = image_to_realword_coordinate(list(position), \n                                     [meta[f'ImagePositionPatient_{i}'] for i in range(3)], \n                                     [meta[f'ImageOrientationPatient_{i}'] for i in range(6)], \n                                     [meta[f'PixelSpacing_{i}'] for i in range(2)])\n            dice[condition].append(position)\n            instances_dict[level] = instances_dict.get(level, dict())\n            instances_dict[level][series_id] = (meta, row[\"instance_number\"])\n    \n    \n    for level, dice in levels_dict.items():\n        for series_id in instances_dict[level].keys():\n#             if series_id in ['983684394']:\n#                 continue\n            meta, instance_number = instances_dict[level][series_id]\n            \n            condition = 'Left Subarticular Stenosis'\n#             key = '-'.join(series_id, condition, level)\n            positions = dice[condition]\n            position = [sum(p_list) / len(p_list) for p_list in zip(*positions) if len(p_list) > 0]\n            if len(position) > 0:\n                position = realword_to_image_coordinate(position, \n                                     [meta[f'ImagePositionPatient_{i}'] for i in range(3)], \n                                     [meta[f'ImageOrientationPatient_{i}'] for i in range(6)], \n                                     [meta[f'PixelSpacing_{i}'] for i in range(2)])\n                new_rows.append((int(study_id), int(series_id), instance_number, condition, level, *position))\n            \n            condition = 'Right Subarticular Stenosis'\n#             key = '-'.join(series_id, condition, level)\n            positions = dice[condition]\n            position = [sum(p_list) / len(p_list) for p_list in zip(*positions) if len(p_list) > 0]\n            if len(position) > 0:\n                position = realword_to_image_coordinate(position, \n                                     [meta[f'ImagePositionPatient_{i}'] for i in range(3)], \n                                     [meta[f'ImageOrientationPatient_{i}'] for i in range(6)], \n                                     [meta[f'PixelSpacing_{i}'] for i in range(2)])\n\n                new_rows.append((int(study_id), int(series_id), instance_number, condition, level, *position))","metadata":{"execution":{"iopub.status.busy":"2024-08-20T11:33:28.868642Z","iopub.execute_input":"2024-08-20T11:33:28.869527Z","iopub.status.idle":"2024-08-20T11:33:30.639975Z","shell.execute_reply.started":"2024-08-20T11:33:28.869491Z","shell.execute_reply":"2024-08-20T11:33:30.638784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_df = pd.DataFrame(new_rows, \n             columns=['study_id', 'series_id', 'instance_number', 'condition', 'level', 'x', 'y'])\n","metadata":{"execution":{"iopub.status.busy":"2024-08-20T11:33:30.641834Z","iopub.execute_input":"2024-08-20T11:33:30.642265Z","iopub.status.idle":"2024-08-20T11:33:30.661073Z","shell.execute_reply.started":"2024-08-20T11:33:30.642222Z","shell.execute_reply":"2024-08-20T11:33:30.659833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"old_rows = []\nfor series_id, coord_rows in series_to_coords_dict.items():\n    if series_id in unpaired_axiel_ids:\n        continue\n    else:\n        old_rows.append(coord_rows)\nold_df = pd.concat(old_rows, axis=0)\nnew_df = pd.concat([old_df, new_df], axis=0)\nnew_df = new_df.iloc[new_df['study_id'].argsort()]\nnew_df.to_csv('train_label_coordinates.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-08-20T11:33:30.662799Z","iopub.execute_input":"2024-08-20T11:33:30.663455Z","iopub.status.idle":"2024-08-20T11:33:31.543905Z","shell.execute_reply.started":"2024-08-20T11:33:30.663416Z","shell.execute_reply":"2024-08-20T11:33:31.542684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls","metadata":{"execution":{"iopub.status.busy":"2024-08-20T11:33:55.035422Z","iopub.execute_input":"2024-08-20T11:33:55.035912Z","iopub.status.idle":"2024-08-20T11:33:56.208916Z","shell.execute_reply.started":"2024-08-20T11:33:55.035873Z","shell.execute_reply":"2024-08-20T11:33:56.207570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}