{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"}],"dockerImageVersionId":30732,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Multiprocessing version of ITK 8191's original preprocessing notebook\nI just added multiprocessing to accelerate ITK8191's original notebook.\n\n\n# RSNA2024 LSDC Making Dataset\nIn this competition, handling the dataset images seems a little difficult.\n\nIn ITK8191's method, the instance numbers specified for each condition and level are used as part of the input image, and a total of 25 channels of input images are collected for each study ID.\n\n### ITK 8191's other Notebooks\n- [RSNA2024 LSDC Making Dataset](https://www.kaggle.com/code/itsuki9180/rsna2024-lsdc-making-dataset) <- you're reading now\n- [RSNA2024 LSDC Training Baseline](https://www.kaggle.com/code/itsuki9180/rsna2024-lsdc-training-baseline)\n- [RSNA2024 LSDC Submission Baseline](https://www.kaggle.com/code/itsuki9180/rsna2024-lsdc-submission-baseline)\n\n","metadata":{"execution":{"iopub.status.busy":"2024-06-12T23:49:53.776252Z","iopub.execute_input":"2024-06-12T23:49:53.777034Z","iopub.status.idle":"2024-06-12T23:49:55.048899Z","shell.execute_reply.started":"2024-06-12T23:49:53.777Z","shell.execute_reply":"2024-06-12T23:49:55.047589Z"}}},{"cell_type":"markdown","source":"# Import Libralies","metadata":{}},{"cell_type":"code","source":"import pydicom\nimport glob, os\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport numpy as np\nimport cv2\nfrom tqdm import tqdm\nimport re\nfrom joblib import Parallel, delayed","metadata":{"execution":{"iopub.status.busy":"2024-07-09T19:43:17.108537Z","iopub.execute_input":"2024-07-09T19:43:17.109611Z","iopub.status.idle":"2024-07-09T19:43:17.940145Z","shell.execute_reply.started":"2024-07-09T19:43:17.109568Z","shell.execute_reply":"2024-07-09T19:43:17.938971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rd = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification'\nN_PROCESSES = 4","metadata":{"execution":{"iopub.status.busy":"2024-07-09T19:43:17.942515Z","iopub.execute_input":"2024-07-09T19:43:17.942983Z","iopub.status.idle":"2024-07-09T19:43:17.949050Z","shell.execute_reply.started":"2024-07-09T19:43:17.942948Z","shell.execute_reply":"2024-07-09T19:43:17.947781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def atoi(text):\n    return int(text) if text.isdigit() else text\n\ndef natural_keys(text):\n    return [ atoi(c) for c in re.split(r'(\\d+)', text) ]","metadata":{"execution":{"iopub.status.busy":"2024-07-09T19:43:17.950443Z","iopub.execute_input":"2024-07-09T19:43:17.950834Z","iopub.status.idle":"2024-07-09T19:43:17.959248Z","shell.execute_reply.started":"2024-07-09T19:43:17.950800Z","shell.execute_reply":"2024-07-09T19:43:17.958075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reading and Taking a look csv","metadata":{}},{"cell_type":"code","source":"dfc = pd.read_csv(f'{rd}/train_label_coordinates.csv')","metadata":{"execution":{"iopub.status.busy":"2024-07-09T19:43:17.960538Z","iopub.execute_input":"2024-07-09T19:43:17.960907Z","iopub.status.idle":"2024-07-09T19:43:18.083012Z","shell.execute_reply.started":"2024-07-09T19:43:17.960875Z","shell.execute_reply":"2024-07-09T19:43:18.081677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(f'{rd}/train_series_descriptions.csv')\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2024-07-09T19:43:18.085844Z","iopub.execute_input":"2024-07-09T19:43:18.086231Z","iopub.status.idle":"2024-07-09T19:43:18.117583Z","shell.execute_reply.started":"2024-07-09T19:43:18.086201Z","shell.execute_reply":"2024-07-09T19:43:18.116513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"For each study_id, we can observe 3 to 6 series_ids.","metadata":{}},{"cell_type":"code","source":"df['series_description'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-07-09T19:43:18.118833Z","iopub.execute_input":"2024-07-09T19:43:18.119176Z","iopub.status.idle":"2024-07-09T19:43:18.132969Z","shell.execute_reply.started":"2024-07-09T19:43:18.119116Z","shell.execute_reply":"2024-07-09T19:43:18.131659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We note that most study_ids with four or more series_ids have two or more Axial T2s.","metadata":{}},{"cell_type":"code","source":"df[df['study_id']==4096820034]","metadata":{"execution":{"iopub.status.busy":"2024-07-09T19:43:18.134703Z","iopub.execute_input":"2024-07-09T19:43:18.135384Z","iopub.status.idle":"2024-07-09T19:43:18.158490Z","shell.execute_reply.started":"2024-07-09T19:43:18.135343Z","shell.execute_reply":"2024-07-09T19:43:18.157181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfc[dfc['study_id']==4096820034]","metadata":{"execution":{"iopub.status.busy":"2024-07-09T19:43:18.159944Z","iopub.execute_input":"2024-07-09T19:43:18.160369Z","iopub.status.idle":"2024-07-09T19:43:18.184736Z","shell.execute_reply.started":"2024-07-09T19:43:18.160333Z","shell.execute_reply":"2024-07-09T19:43:18.183350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see that it corresponds as follows:\n\n- Axial T2 => (Left|Right) Subarticular Stenosis (10 classes)\n- Sagittal T2/STIR => Spinal Canal Stenosis (5 Classes)\n- Sagittal T1 => (Left|Right) Neural Foraminal Narrowing (10 classes)","metadata":{}},{"cell_type":"markdown","source":"I found it difficult to order Axial T2 well, so we decided to select them randomly during training. For the other two, we will save images at equal intervals.","metadata":{}},{"cell_type":"markdown","source":"# Export png from dcm\n.dcm format files have various pixel values and image shapes. To use them in a deep learning framework, we will make the values ​​fall within a certain range and resize the shapes to 512px.","metadata":{}},{"cell_type":"code","source":"def imread_and_imwirte(src_path, dst_path):\n    dicom_data = pydicom.dcmread(src_path)\n    image = dicom_data.pixel_array\n    image = (image - image.min()) / (image.max() - image.min() +1e-6) * 255\n    img = cv2.resize(image, (512, 512),interpolation=cv2.INTER_CUBIC)\n    assert img.shape==(512,512)\n    cv2.imwrite(dst_path, img)","metadata":{"execution":{"iopub.status.busy":"2024-07-09T19:43:18.186644Z","iopub.execute_input":"2024-07-09T19:43:18.187052Z","iopub.status.idle":"2024-07-09T19:43:18.196831Z","shell.execute_reply.started":"2024-07-09T19:43:18.187016Z","shell.execute_reply":"2024-07-09T19:43:18.195811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"st_ids = df['study_id'].unique()\nst_ids[:3], len(st_ids)","metadata":{"execution":{"iopub.status.busy":"2024-07-09T19:43:18.198816Z","iopub.execute_input":"2024-07-09T19:43:18.199233Z","iopub.status.idle":"2024-07-09T19:43:18.216499Z","shell.execute_reply.started":"2024-07-09T19:43:18.199199Z","shell.execute_reply":"2024-07-09T19:43:18.215408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"desc = list(df['series_description'].unique())\ndesc","metadata":{"execution":{"iopub.status.busy":"2024-07-09T19:43:18.217956Z","iopub.execute_input":"2024-07-09T19:43:18.218912Z","iopub.status.idle":"2024-07-09T19:43:18.227734Z","shell.execute_reply.started":"2024-07-09T19:43:18.218872Z","shell.execute_reply":"2024-07-09T19:43:18.226678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#for idx, si in enumerate(tqdm(st_ids, total=len(st_ids))):\ndef process_study(si):\n    pdf = df[df['study_id']==si]\n    for ds in desc:\n        ds_ = ds.replace('/', '_')\n        pdf_ = pdf[pdf['series_description']==ds]\n        os.makedirs(f'cvt_png/{si}/{ds_}', exist_ok=True)\n        allimgs = []\n        for i, row in pdf_.iterrows():\n            pimgs = glob.glob(f'{rd}/train_images/{row[\"study_id\"]}/{row[\"series_id\"]}/*.dcm')\n            pimgs = sorted(pimgs, key=natural_keys)\n            allimgs.extend(pimgs)\n            \n        if len(allimgs)==0:\n            print(si, ds, 'has no images')\n            continue\n\n        if ds == 'Axial T2':\n            for j, impath in enumerate(allimgs):\n                dst = f'cvt_png/{si}/{ds}/{j:03d}.png'\n                imread_and_imwirte(impath, dst)\n                \n        elif ds == 'Sagittal T2/STIR':\n            \n            step = len(allimgs) / 10.0\n            st = len(allimgs)/2.0 - 4.0*step\n            end = len(allimgs)+0.0001\n            for j, i in enumerate(np.arange(st, end, step)):\n                dst = f'cvt_png/{si}/{ds_}/{j:03d}.png'\n                ind2 = max(0, int((i-0.5001).round()))\n                imread_and_imwirte(allimgs[ind2], dst)\n                \n            assert len(glob.glob(f'cvt_png/{si}/{ds_}/*.png'))==10\n                \n        elif ds == 'Sagittal T1':\n            step = len(allimgs) / 10.0\n            st = len(allimgs)/2.0 - 4.0*step\n            end = len(allimgs)+0.0001\n            for j, i in enumerate(np.arange(st, end, step)):\n                dst = f'cvt_png/{si}/{ds}/{j:03d}.png'\n                ind2 = max(0, int((i-0.5001).round()))\n                imread_and_imwirte(allimgs[ind2], dst)\n                \n            assert len(glob.glob(f'cvt_png/{si}/{ds}/*.png'))==10\n            \nParallel(n_jobs=N_PROCESSES)(delayed(process_study)(si) for si in st_ids)","metadata":{"execution":{"iopub.status.busy":"2024-07-09T19:43:18.228931Z","iopub.execute_input":"2024-07-09T19:43:18.229345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Continuing with the [Training Baseline...](https://www.kaggle.com/code/itsuki9180/rsna2024-lsdc-training-baseline)","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}