{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nimport cv2\nimport pydicom\nimport numpy as np\nimport os\nimport glob\nfrom tqdm import tqdm\nimport warnings","metadata":{"execution":{"iopub.status.busy":"2024-05-21T12:08:08.766806Z","iopub.execute_input":"2024-05-21T12:08:08.767230Z","iopub.status.idle":"2024-05-21T12:08:09.523960Z","shell.execute_reply.started":"2024-05-21T12:08:08.767199Z","shell.execute_reply":"2024-05-21T12:08:09.522717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train.csv')","metadata":{"execution":{"iopub.status.busy":"2024-05-21T12:08:11.067690Z","iopub.execute_input":"2024-05-21T12:08:11.068504Z","iopub.status.idle":"2024-05-21T12:08:11.094890Z","shell.execute_reply.started":"2024-05-21T12:08:11.068466Z","shell.execute_reply":"2024-05-21T12:08:11.093528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n### Grab metadata for each scan.\n\nMany thanks to [this starting](https://www.kaggle.com/code/abhinavsuri/anatomy-image-visualization-overview-rsna-raids/) notebook authors, from where Meta object construction is taken from! \n\nFor each scan let's create an object with the following structure:\n\n```\nmeta_obj = {\n    StudyInstanceUID: {\n        'folder_path': ... # path to the folder,\n        'SeriesInstanceUIDs': [ Array of the SeriesInstanceUIDs ],\n        'SeriesDescriptions' [ Array of the Series Descriptions ]\n    }, ...\n}\n```","metadata":{}},{"cell_type":"code","source":"# List out all of the Studies we have on patients.\npart_1 = os.listdir('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images')\npart_1 = list(filter(lambda x: x.find('.DS') == -1, part_1))","metadata":{"execution":{"iopub.status.busy":"2024-05-21T12:08:14.940757Z","iopub.execute_input":"2024-05-21T12:08:14.942485Z","iopub.status.idle":"2024-05-21T12:08:14.952705Z","shell.execute_reply.started":"2024-05-21T12:08:14.942421Z","shell.execute_reply":"2024-05-21T12:08:14.950443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_meta_f = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_series_descriptions.csv')","metadata":{"execution":{"iopub.status.busy":"2024-05-21T12:09:04.146748Z","iopub.execute_input":"2024-05-21T12:09:04.147212Z","iopub.status.idle":"2024-05-21T12:09:04.163987Z","shell.execute_reply.started":"2024-05-21T12:09:04.147176Z","shell.execute_reply":"2024-05-21T12:09:04.162227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p1 = [(x, f\"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/{x}\") for x in part_1]\nmeta_obj = { p[0]: { 'folder_path': p[1], \n                    'SeriesInstanceUIDs': [] \n                   } \n            for p in p1 }","metadata":{"execution":{"iopub.status.busy":"2024-05-21T12:09:05.992068Z","iopub.execute_input":"2024-05-21T12:09:05.994191Z","iopub.status.idle":"2024-05-21T12:09:06.005224Z","shell.execute_reply.started":"2024-05-21T12:09:05.994111Z","shell.execute_reply":"2024-05-21T12:09:06.003392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for m in meta_obj:\n    meta_obj[m]['SeriesInstanceUIDs'] = list(\n        filter(lambda x: x.find('.DS') == -1, \n               os.listdir(meta_obj[m]['folder_path'])\n              )\n    )","metadata":{"execution":{"iopub.status.busy":"2024-05-21T12:09:07.600932Z","iopub.execute_input":"2024-05-21T12:09:07.601397Z","iopub.status.idle":"2024-05-21T12:09:08.557693Z","shell.execute_reply.started":"2024-05-21T12:09:07.601332Z","shell.execute_reply":"2024-05-21T12:09:08.556622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# grabs the correspoding series descriptions\nfor k in tqdm(meta_obj):\n    for s in meta_obj[k]['SeriesInstanceUIDs']:\n        if 'SeriesDescriptions' not in meta_obj[k]:\n            meta_obj[k]['SeriesDescriptions'] = []\n        try:\n            meta_obj[k]['SeriesDescriptions'].append(\n                df_meta_f[(df_meta_f['study_id'] == int(k)) & \n                (df_meta_f['series_id'] == int(s))]['series_description'].iloc[0])\n        except:\n            print(\"Failed on\", s, k)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T12:09:10.572698Z","iopub.execute_input":"2024-05-21T12:09:10.573659Z","iopub.status.idle":"2024-05-21T12:09:14.539171Z","shell.execute_reply.started":"2024-05-21T12:09:10.573619Z","shell.execute_reply":"2024-05-21T12:09:14.537827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_obj[list(meta_obj.keys())[1]]","metadata":{"execution":{"iopub.status.busy":"2024-05-21T12:09:16.435828Z","iopub.execute_input":"2024-05-21T12:09:16.436239Z","iopub.status.idle":"2024-05-21T12:09:16.445439Z","shell.execute_reply.started":"2024-05-21T12:09:16.436207Z","shell.execute_reply":"2024-05-21T12:09:16.443919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Data Wrangling: Construct training images vs labels","metadata":{}},{"cell_type":"markdown","source":"Lets simplify all this data by crunching it into 2 lists of training images and corresponmding labels. \n\n\nPlease note that, each row in `train.csv` corresponds to 1 patient and we must output 25 labels.\n\n\n5 conditions : (['Spinal Canal Stenosis', 'Right Neural Foraminal Narrowing',\n       'Left Neural Foraminal Narrowing', 'Left Subarticular Stenosis',\n       'Right Subarticular Stenosis'])\n       \n       \n5 levels of spine: (['L1/L2', 'L2/L3', 'L3/L4', 'L4/L5', 'L5/S1'])\n\n\nso in total 25 labels , and each label has 3 severity levels (Normal/Mild, Moderate, Severe) (3 x 1) per label.\n\nWe will need to build a model which will take in a single image and output 25 labels(25 **heads**...) each being (3 x 1)","metadata":{}},{"cell_type":"code","source":"train_df = train.fillna(\"Normal/Mild\")","metadata":{"execution":{"iopub.status.busy":"2024-05-21T12:09:20.319920Z","iopub.execute_input":"2024-05-21T12:09:20.320467Z","iopub.status.idle":"2024-05-21T12:09:20.340794Z","shell.execute_reply.started":"2024-05-21T12:09:20.320422Z","shell.execute_reply":"2024-05-21T12:09:20.339527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_images = []\ntrain_labels = []\n\nfor idx in tqdm(range(len(train_df))):\n    patient = train_df.iloc[idx]\n    ptobj = meta_obj[str(patient['study_id'])]\n    \n    label_dict = {\n        \"Normal/Mild\": [1, 0, 0],\n        \"Moderate\": [0, 1, 0],\n        \"Severe\": [0, 0, 1]\n    }\n    labels = []\n    for x,y in patient.items():\n        if \"study\" in x:\n            continue\n        labels.append(label_dict[y])\n    \n    for idx, i in enumerate(ptobj['SeriesInstanceUIDs']):\n        images = glob.glob(f\"{ptobj['folder_path']}/{ptobj['SeriesInstanceUIDs'][idx]}/*.dcm\")\n        for image_path in sorted(images, key=lambda x: int(x.split('/')[-1].replace('.dcm', ''))):\n            train_images.append(image_path)\n            train_labels.append(np.array(labels))\n    ","metadata":{"execution":{"iopub.status.busy":"2024-05-21T12:09:22.440487Z","iopub.execute_input":"2024-05-21T12:09:22.440939Z","iopub.status.idle":"2024-05-21T12:09:31.398062Z","shell.execute_reply.started":"2024-05-21T12:09:22.440904Z","shell.execute_reply":"2024-05-21T12:09:31.396695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Total images/samples: \", len(train_images))\nprint(\"Total labels: \", len(train_labels))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}