{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"}],"dockerImageVersionId":30746,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"All credits go to [this notebook](https://www.kaggle.com/code/itsuki9180/rsna2024-lsdc-making-dataset/notebook#Export-png-from-dcm). Please comment if you see that I am incorrect about something. Thank you.\n\n# Helpers","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nimport cv2\nimport os\nfrom tqdm import tqdm\n\nimport pydicom\nimport numpy as np\n\ndf_train_series_descriptions = pd.read_csv(\"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_series_descriptions.csv\")\nfolder_train_images = os.listdir(\"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images\")\n# exclude .DS_store files\nfolder_train_images = list(filter(lambda x: x.find('.DS') == -1, folder_train_images))\n\ndef get_metadata_object(folder_images, df_series_descriptions):\n    '''\n    for intially the train_images folder and train_series_descriptions.csv,\n    later for the test_images folder and test_series_description.csv\n    '''\n    \n    # a list of tuples like (study_id, study_id's path location)\n    images_study_id_dirs = [(int(study_id),    # integer the study_id\n                                   f\"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/{study_id}\") \n                                  for study_id in folder_images]\n\n    # convert the list of tuples into dictionary/metadata\n    metadata_object = {study_id: {'study_id_folder_path': path, \n                                    'SeriesInstanceUIDs': [],\n                                    'SeriesDescriptions': []\n                                 }\n                       for study_id, path in images_study_id_dirs\n                      }\n\n    # remove all the .DS files/folders (MacOS) from SeriesInstanceUIDs or series_ids folders\n    # then put the names for series_ids as well as the corresponding description to metadata\n    for study_id in tqdm(metadata_object):\n\n        # SERIES_ID\n        # get all series directories/folders inside each study_id directory\n        series_ids_dirs = os.listdir(metadata_object[study_id]['study_id_folder_path'])\n        filtered_series_ids_dirs = [int(x) for x in series_ids_dirs if x.find('.DS') == -1]   # integer(series_id)\n        # put to metadata_object\n        metadata_object[study_id]['SeriesInstanceUIDs'] = filtered_series_ids_dirs\n\n        # SERIES_DESCRIPTIONS\n        series_desc_df = df_series_descriptions[df_series_descriptions.study_id==study_id][[\"series_id\", \"series_description\"]]\n        for series_id in metadata_object[study_id]['SeriesInstanceUIDs']:\n\n            series_desc_list = series_desc_df[series_desc_df.series_id==series_id].series_description.values\n\n            if len(series_desc_list) == 0:\n                metadata_object[study_id]['SeriesDescriptions'].append(\"\")\n            else:\n                metadata_object[study_id]['SeriesDescriptions'].append(series_desc_list[0])\n                \n    return metadata_object\n\n\nmetadata_object_train = get_metadata_object(folder_train_images, df_train_series_descriptions)\n\n\n\n\n\ndef get_series_metadata_object_given_study_id(metadata_object, study_id):\n    \"\"\"\n    one study_id can have many series => this func gets all series metadata for this \n    particular study_id\n    \"\"\"\n    \n    metadata_for_study_id = metadata_object[study_id]\n    \n    series_metadata_object_given_study_id = {}\n\n    for idx, series_id in enumerate(metadata_for_study_id[\"SeriesInstanceUIDs\"]):\n\n        # create bases for the series_images_metadata: each series contains desc and images files\n        series_metadata_object_given_study_id[series_id] = {'image_series_description': metadata_for_study_id[\"SeriesDescriptions\"][idx], \n                                                            'image_files': []\n                                                            }\n\n        # glob for patten matching as we want to get image files ending with .dcm\n        folder_path_study_id = metadata_for_study_id[\"study_id_folder_path\"]\n        # rmb: \"SeriesInstancesUIDs\" is just a list of images and series_id is the actual series-id\n        image_files = glob(f\"{folder_path_study_id}/{series_id}/*.dcm\")\n\n\n        # inside image_files, create a metadata for id_image and corresponding dicom readable image file\n        sorted_image_files = sorted(image_files, key = lambda x: int(x.split('/')[-1].replace('.dcm', '')))\n\n        \n        # iterate through all image files (sorting to make sense, not very necessarily)\n        for image_file in sorted_image_files:\n            \n            dicom_image_id = image_file.split('/')[-1].replace(\".dcm\", '')\n            dicom_image_read = pydicom.dcmread(image_file)\n            \n            # metadata for one image instance\n            one_image_metadata = {\"SOPInstanceUID\": dicom_image_id,      # id of the dicom image instance file\n                                  \"dicom_image_file\": dicom_image_read}  # actual read of the dcm instance file\n\n            # append this image_metadata to list of image_files\n            series_metadata_object_given_study_id[series_id][\"image_files\"].append(one_image_metadata)\n\n    return series_metadata_object_given_study_id\n    \n    \n    \n    \n    \n    \ndef display_images_given_study_id(metadata_object, study_id): \n    '''\n    inside there is another function specifically for this function\n    '''\n    \n    # view images for this study_id = 4003253 for a particular series_description\n    def display_images(image_files, series_description, max_images_per_row=5):\n\n        # grid for display\n        num_images = len(image_files)\n        num_rows = (num_images + max_images_per_row - 1) // max_images_per_row  # ceiling division (ignore the remainder, extra)\n\n        # subplot grid\n        fig, axes = plt.subplots(nrows=num_rows, ncols=max_images_per_row)\n\n        # flatten axes for easy looping if there are multiple rows\n        if num_rows > 1:\n            axes = axes.flatten()\n        else:\n            axes = [axes] # iterable for consistency\n\n        # plot each image\n        for idx, image_file in enumerate(image_files):\n            ax = axes[idx]\n            ax.imshow(image_file, cmap='gray') # Assuming grayscale for simplicity, change cmap as needed\n            ax.axis('off')\n\n        # turn off unused subplots\n        for idx in range(num_images, len(axes)):\n            axes[idx].axis(\"off\")\n\n        fig.suptitle(series_description, fontsize=12)\n        plt.show()\n        \n        \n        \n    series_metadata_object = get_series_metadata_object_given_study_id(metadata_object, study_id)\n    \n    for series_id in series_metadata_object:\n\n        series_description = series_metadata_object[series_id][\"image_series_description\"]\n\n        # get the image_files for this particular series_id\n        image_files = series_metadata_object[series_id][\"image_files\"]\n\n        # get the dicom files to a list\n        dicom_images = []\n        for image_metadata in image_files:\n            dicom_image = image_metadata['dicom_image_file'].pixel_array\n            dicom_images.append(dicom_image)\n\n        # display images for each series\n        display_images(dicom_images, series_description)\n        \n        \n\n        \n        \n        \ndf_label_coord = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_label_coordinates.csv')\n# add the columns in coordinates regarding condition+level to match with train_df\ncols = ['condition', 'level']\ndf_label_coord['m_condition'] = df_label_coord[cols].apply(lambda row: ' '.join(row.values.astype(str)).lower(), axis=1)\ndf_label_coord['m_condition'] = df_label_coord['m_condition'].str.replace(r'[ /]', '_', regex=True)\n\ndf_train = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train.csv')\n\n\n\ndef display_images_with_coord(metadata_object, study_id):\n    '''\n    again, there is an inner function to display imges\n    '''\n    \n    # this function is for one coord, one image, and one title/description/condition/level)\n    def display_image_with_coord(center_coord, image_instance_dicom, title):\n        '''\n        coord_entry is a particular coord for a particular image \n        image_meta is the image that specifically in the series, containing SOPInstanceUID and dicom_image_file, i.e. \n        {2448190387: {'image_series_description': 'Axial T2',\n                      'image_files': [{'SOPInstanceUID': '1',\n                                        'dicom_image_file': Dataset.file_meta -------------------------------\n                                        (0002, 0001) File Meta Informa\n        '''\n        radius = 10\n        color = (255, 0, 0)\n        thickness = 2\n\n        # for what?\n        image_normalized = cv2.normalize(image_instance_dicom, None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_8U)\n        # circling\n        image_circle = cv2.circle(image_normalized.copy(), center_coord, radius, color, thickness)\n\n        # convert image from BGR to RGB for correct color display in matplotlib\n        image_circle = cv2.cvtColor(image_circle, cv2.COLOR_BAYER_BG2BGR)\n\n        # display\n        plt.imshow(image_circle)\n        plt.axis('off')\n        plt.title(title)\n        plt.show()\n    \n    \n    \n    # example_study_id = 4003253\n    series_meta =  get_series_metadata_object_given_study_id(metadata_object, study_id) # series_meta all based on given study_id\n    example_train_df = train_df[train_df.study_id==study_id]\n    example_coord_entries_df = df_label_coord[df_label_coord.study_id==study_id]\n\n    for d, coord_entry in example_coord_entries_df.iterrows():\n\n        center_coord = (int(coord_entry['x']), int(coord_entry['y']))\n\n        # search for the image available for coord in all imgs in series\n        image_meta_instances_list = series_meta[coord_entry.series_id][\"image_files\"]\n        # check matching\n        image_instance_id = coord_entry.instance_number\n\n        for image_instance in image_meta_instances_list:\n\n            if int(image_instance[\"SOPInstanceUID\"]) == int(image_instance_id):\n\n                image_instance_dicom = image_instance[\"dicom_image_file\"].pixel_array\n\n                severity = train_df.loc[example_train_df.index[0], coord_entry.m_condition]\n                title = f'image_instance_id: {image_instance_id} \\n {coord_entry.m_condition} - severity: {severity}'\n\n                display_image_with_coord(center_coord, image_instance_dicom, title)","metadata":{"_kg_hide-input":true,"jupyter":{"source_hidden":true}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. Preparing","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport pydicom\nimport matplotlib.pyplot as plt\nimport os\nimport cv2\nfrom tqdm import tqdm\nimport re\nimport shutil\nfrom glob import glob","metadata":{"execution":{"iopub.status.busy":"2024-07-16T20:15:02.115211Z","iopub.execute_input":"2024-07-16T20:15:02.115651Z","iopub.status.idle":"2024-07-16T20:15:02.122624Z","shell.execute_reply.started":"2024-07-16T20:15:02.115621Z","shell.execute_reply":"2024-07-16T20:15:02.121152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv(\"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train.csv\")\ndf_train_series_descriptions = pd.read_csv(\"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_series_descriptions.csv\")\ndf_label_coord = pd.read_csv(\"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_label_coordinates.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-07-16T20:13:11.015903Z","iopub.execute_input":"2024-07-16T20:13:11.016318Z","iopub.status.idle":"2024-07-16T20:13:11.145381Z","shell.execute_reply.started":"2024-07-16T20:13:11.016287Z","shell.execute_reply":"2024-07-16T20:13:11.144199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Note that, most `study_id` has 3 series which correspond to 3 MRI scan type and also correspond to 3 conditions (without levels). Some has more Axial T2 than just one => consider all of these Axial T2 things as one for consistency.","metadata":{}},{"cell_type":"markdown","source":"# 2. Write dicom to png images, set new directory names too\n\n### General structure of new image folder\n\ntrain_images_png > study_id > [Axial_T2, Sagittal_T1, Sagittal_T2_STIR] > instance_ids of the images (in png)\n\nAxial T2 images don't follow any order of imaging? => keep them as it aka not take any subset out of it. For the two Sagittal images, they seem to follow order so we can just take randomly 10 images at equal interval. Doing so helps with computational resources.\n\n<br>","metadata":{}},{"cell_type":"code","source":"# collect main folder names (fixed and standard)\n\n# unique study_id\nstudy_ids = list(df_train.study_id.unique())\nseries_descs = list(df_train_series_descriptions.series_description.unique())\nseries_descs = list(map(lambda x: x.replace(' ', '_').replace('/','_'), series_descs))\nprint(study_ids[:5], ',... and more study_ids. And', series_descs)","metadata":{"execution":{"iopub.status.busy":"2024-07-16T20:13:16.648842Z","iopub.execute_input":"2024-07-16T20:13:16.649277Z","iopub.status.idle":"2024-07-16T20:13:16.662485Z","shell.execute_reply.started":"2024-07-16T20:13:16.649243Z","shell.execute_reply":"2024-07-16T20:13:16.661336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def readdcm_writepng_image(src_dicom_pixelarray, dest_path_png):\n    '''\n    read image in dicom, then re-write it to png with my own construction of directories\n    '''\n    \n    # standardize to 0 to 255 pixels range for saving purpose in png (1e-10 to avoid divided by 0)\n    standardized_image_data = ((src_dicom_pixelarray - src_dicom_pixelarray.min()) / \n                               (src_dicom_pixelarray.max() - src_dicom_pixelarray.min() + 1e-10)) * 255\n    \n    # resize to 512 x 512 dimension (same to all images)\n    final_image_to_png = cv2.resize(standardized_image_data, (512, 512),interpolation=cv2.INTER_CUBIC)\n    \n    # note:\n    # 255 pixels represent intensity (contrast of the image) and 512x512 represent dimension\n    \n    # automatically write to new dest path with a png image when given a file path ended with\".png\"\n    cv2.imwrite(dest_path_png, final_image_to_png)\n    ","metadata":{"execution":{"iopub.status.busy":"2024-07-16T20:13:20.315611Z","iopub.execute_input":"2024-07-16T20:13:20.316144Z","iopub.status.idle":"2024-07-16T20:13:20.324901Z","shell.execute_reply.started":"2024-07-16T20:13:20.316100Z","shell.execute_reply":"2024-07-16T20:13:20.323470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# use my own metadata_object_train with other's to make dataset\nmain_path = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/'\noutput_path = '/kaggle/working/train_images_png/'\nnp.random.seed(123)\n\n# remove for fresh writing\nif os.path.isdir(output_path):\n    shutil.rmtree(output_path) \n\nfor study_id in tqdm(metadata_object_train, total=len(metadata_object_train)):\n     \n    # series_metadata for this study_id\n    metadata_series = get_series_metadata_object_given_study_id(metadata_object_train, study_id)\n    \n    # for zipped series_id and series_descriptions\n    series_ids_list = metadata_object_train[study_id][\"SeriesInstanceUIDs\"]\n    series_descs_list = metadata_object_train[study_id][\"SeriesDescriptions\"]\n    series_descs_list = list(map(lambda x: x.replace(' ', '_').replace('/','_'), series_descs_list))\n    \n    for series_id, series_desc in zip(series_ids_list, series_descs_list):\n        \n        # make new folder for each series_desc for each study_id\n        os.makedirs(f'{output_path}/{study_id}/{series_desc}', exist_ok=True)\n        \n        # get list of dicom images from this series\n        images_dicom_list = metadata_series[series_id][\"image_files\"]\n        len_images = len(images_dicom_list)\n        \n        if len_images==0:\n            print(study_id, series_id, series_desc, 'has no images.')\n            continue\n        \n        # Axial T2: write all and randomly subset for training later\n        elif series_desc == 'Axial_T2':\n        \n            # convert these dicom images into png and write to new folder\n            for metadata_image_dicom in images_dicom_list:\n                image_filename = metadata_image_dicom[\"SOPInstanceUID\"]\n                image_dicom_pixelarray = metadata_image_dicom[\"dicom_image_file\"].pixel_array\n\n                dest_path = f'{output_path}/{study_id}/{series_desc}/{image_filename}.png'\n\n                readdcm_writepng_image(image_dicom_pixelarray, dest_path)\n                \n        # Other two Sagittals: write 10 subset from the images at equal interval\n        else:\n            \n            # determine intervals\n            if len_images < 10:\n                sampled_indices = np.arange(0, len_images)\n            else: # other cases: 10 images, equal interval\n                step_size = len_images // 10\n                sampled_indices = np.arange(0, len_images, step_size)\n                sampled_indices = np.random.choice(sampled_indices, size=10, replace=False)\n                \n                # more check for 10 images\n                sampled_indices = sampled_indices[:10]\n\n            # loop through indices in this list\n            for idx in sampled_indices:\n                \n                metadata_image_dicom = images_dicom_list[int(idx)]\n                \n                image_filename = metadata_image_dicom[\"SOPInstanceUID\"]\n                image_dicom_pixelarray = metadata_image_dicom[\"dicom_image_file\"].pixel_array\n\n                dest_path = f'{output_path}/{study_id}/{series_desc}/{image_filename}.png'\n\n                readdcm_writepng_image(image_dicom_pixelarray, dest_path)\n            \n            \n            # check\n            num_png_files = len(glob(f'{output_path}/{study_id}/{series_desc}/*.png'))\n            if num_png_files > 10:\n                print(study_id, series_id, series_desc, \"has more than 10 images\")\n            assert 8 <= num_png_files <= 14, f'{len_images} got {num_png_files} images'","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2024-07-16T20:15:18.476297Z","iopub.execute_input":"2024-07-16T20:15:18.477350Z","iopub.status.idle":"2024-07-16T21:23:03.327721Z","shell.execute_reply.started":"2024-07-16T20:15:18.477305Z","shell.execute_reply":"2024-07-16T21:23:03.325315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Mandatory checking output path","metadata":{}},{"cell_type":"code","source":"metadata_object_train[100206310]","metadata":{"execution":{"iopub.status.busy":"2024-07-16T22:42:37.896256Z","iopub.execute_input":"2024-07-16T22:42:37.897064Z","iopub.status.idle":"2024-07-16T22:42:37.905036Z","shell.execute_reply.started":"2024-07-16T22:42:37.897027Z","shell.execute_reply":"2024-07-16T22:42:37.903721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from PIL import Image\nimport matplotlib.pyplot as plt\n\noutput_path = '/kaggle/working/train_images_png'\n\nstudy_id = 100206310\nstudy_id_folder = f'{output_path}/{study_id}'\n\nfor series_desc in os.listdir(study_id_folder):\n    \n    print(f'Series description: {series_desc}')\n    \n    png_files = glob(f'{study_id_folder}/{series_desc}/*.png')\n    \n    for png_file in png_files:\n       \n        # Open and display the image\n        img = Image.open(png_file)\n        plt.imshow(img, cmap='gray')\n        plt.title(f'{series_desc} - {os.path.basename(png_file)}')\n        plt.axis('off')\n        plt.show()\n        ","metadata":{"execution":{"iopub.status.busy":"2024-07-16T21:55:33.337862Z","iopub.execute_input":"2024-07-16T21:55:33.338348Z","iopub.status.idle":"2024-07-16T21:55:51.142308Z","shell.execute_reply.started":"2024-07-16T21:55:33.338312Z","shell.execute_reply":"2024-07-16T21:55:51.141053Z"},"trusted":true},"execution_count":null,"outputs":[]}]}