{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"},{"sourceId":9103215,"sourceType":"datasetVersion","datasetId":5494026}],"dockerImageVersionId":30746,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import wandb\nfrom pytorch_lightning.loggers import WandbLogger\nimport os\nimport gc\nimport yaml\nimport sys\nimport random\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport torch\nfrom glob import glob\nfrom torch.utils.data import Dataset\nfrom torch.utils.data import DataLoader\nimport torch.nn as nn\nimport pytorch_lightning as pl\nfrom pytorch_lightning.callbacks import ModelCheckpoint, EarlyStopping, TQDMProgressBar\nimport torchvision.transforms as T\nimport albumentations as A\nimport pandas.api.types\nimport timm\nimport scipy\nfrom torchvision.transforms import v2\nfrom torchvision import models\n#from einops import repeat\n#from einops.layers.torch import Rearrange\nfrom tqdm.auto import tqdm\n#sys.path.append('/kaggle/input/coat-git')\n#from src.models.coat import coat_lite_medium\nfrom joblib import Parallel, delayed\nfrom torch.utils.data import default_collate\nimport pydicom as dcm","metadata":{"execution":{"iopub.status.busy":"2024-08-07T20:38:25.761001Z","iopub.execute_input":"2024-08-07T20:38:25.761447Z","iopub.status.idle":"2024-08-07T20:38:39.493507Z","shell.execute_reply.started":"2024-08-07T20:38:25.761407Z","shell.execute_reply":"2024-08-07T20:38:39.492012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mode='train'","metadata":{"execution":{"iopub.status.busy":"2024-08-07T21:54:24.847061Z","iopub.execute_input":"2024-08-07T21:54:24.847529Z","iopub.status.idle":"2024-08-07T21:54:24.854629Z","shell.execute_reply.started":"2024-08-07T21:54:24.847469Z","shell.execute_reply":"2024-08-07T21:54:24.853033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_dcm_df(study_id, series_id, series_desc): \n    path_list = glob(f'/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/{mode}_images/{study_id}/{series_id}/*.dcm')\n    in_list = sorted([int(s.split('/')[-1].split('.')[0]) for s in path_list])\n    #dmc_list = []\n    #for i in in_list: \n    #    dcm_list.append(dcm.dcmread(f'/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/{mode}_images/{study_id}/{series_id}/{i}.dcm')\n    dcm_list = [dcm.dcmread(f'/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/{mode}_images/{study_id}/{series_id}/{i}.dcm') for i in in_list]\n    ipp = np.asarray([d.ImagePositionPatient for d in dcm_list]).astype('float')\n    iop = [d.ImageOrientationPatient for d in dcm_list]\n    iop = [[float(d[0]), float(d[1]), float(d[2]), \n            float(d[3]), float(d[4]), float(d[5])] for d in iop]\n    ipp_x = ipp[:, 0]\n    ipp_y = ipp[:, 1]\n    ipp_z = ipp[:, 2]\n    shape = np.array([d.pixel_array.shape for d in dcm_list])\n    sbs = np.asarray([d.SpacingBetweenSlices for d in dcm_list]).astype('float')\n    ps = np.asarray([d.PixelSpacing for d in dcm_list]).astype('float')\n    ps_x = ps[:, 0]\n    ps_y = ps[:, 1]\n    meta_dict = {'instance_number': in_list, 'ipp_x': ipp_x, 'ipp_y': ipp_y, 'ipp_z': ipp_z, 'sbs': sbs, 'ps_x': ps_x, 'ps_y': ps_y}\n    meta_df = pd.DataFrame(meta_dict)\n    meta_df['series_id'] = series_id\n    meta_df['study_id'] = study_id\n    meta_df['series_description'] = series_desc\n    meta_df['height'] = shape[:, 0]\n    meta_df['width'] = shape[:, 1]\n    meta_df['iop'] = pd.Series(iop)\n    del dcm_list\n    gc.collect()\n    return meta_df[['study_id', 'series_id', 'series_description', 'instance_number', 'height', 'width', 'ipp_x', 'ipp_y', 'ipp_z', 'iop', 'sbs', 'ps_x', 'ps_y']]","metadata":{"execution":{"iopub.status.busy":"2024-08-07T21:53:58.448666Z","iopub.execute_input":"2024-08-07T21:53:58.450647Z","iopub.status.idle":"2024-08-07T21:53:58.469998Z","shell.execute_reply.started":"2024-08-07T21:53:58.450528Z","shell.execute_reply":"2024-08-07T21:53:58.468882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import joblib","metadata":{"execution":{"iopub.status.busy":"2024-08-07T21:54:00.505606Z","iopub.execute_input":"2024-08-07T21:54:00.506063Z","iopub.status.idle":"2024-08-07T21:54:00.512063Z","shell.execute_reply.started":"2024-08-07T21:54:00.506028Z","shell.execute_reply":"2024-08-07T21:54:00.510657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntest_series = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_series_descriptions.csv')\nmeta_df_list = []\nmeta_df_list = joblib.Parallel(n_jobs=-1)([joblib.delayed(create_dcm_df)(row.study_id, row.series_id, row.series_description) for _, row in test_series.iterrows()])\n#for _, row in tqdm(test_series.iterrows(), total=len(test_series)): \n#    meta_df_list.append(create_dcm_df(row.study_id, row.series_id, row.series_description))\nmeta_df = pd.concat(meta_df_list)\ndel meta_df_list\ngc.collect()\nmeta_df.to_parquet('meta.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-08-07T21:54:27.194422Z","iopub.execute_input":"2024-08-07T21:54:27.195341Z","iopub.status.idle":"2024-08-07T21:54:30.154472Z","shell.execute_reply.started":"2024-08-07T21:54:27.195303Z","shell.execute_reply":"2024-08-07T21:54:30.153144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"meta_df = pd.read_csv('/kaggle/input/rsna-newmeta/meta.csv', index_col=0)\ncoor = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_label_coordinates.csv')\ndisplay(coor.head(3))\nmeta_df.head(3)","metadata":{"execution":{"iopub.status.busy":"2024-08-07T20:38:39.495556Z","iopub.execute_input":"2024-08-07T20:38:39.496133Z","iopub.status.idle":"2024-08-07T20:38:40.352240Z","shell.execute_reply.started":"2024-08-07T20:38:39.496099Z","shell.execute_reply":"2024-08-07T20:38:40.350800Z"}}},{"cell_type":"markdown","source":"study_id = 4003253\nseries_id = 702807833\nscs_coor = coor.loc[coor.condition=='Spinal Canal Stenosis']\nscs_coor = scs_coor.merge(meta_df, on=['study_id', 'series_id', 'instance_number'], how='left')\nsub_coor = scs_coor.loc[scs_coor.study_id==study_id]\ndisplay(sub_coor)\nsub_meta = meta_df.loc[(meta_df.study_id==study_id) & (meta_df.series_description=='Axial T2')]","metadata":{"execution":{"iopub.status.busy":"2024-08-07T20:50:34.283339Z","iopub.execute_input":"2024-08-07T20:50:34.283808Z","iopub.status.idle":"2024-08-07T20:50:34.364785Z","shell.execute_reply.started":"2024-08-07T20:50:34.283769Z","shell.execute_reply":"2024-08-07T20:50:34.363582Z"}}},{"cell_type":"markdown","source":"# thanks @hengck\n# customized for my pipeline\n# project 2d to 3d\ndef project_to_3d(row):\n    #d = df.iloc[z]\n    #H, W = d.H, d.W\n    #sx, sy, sz = [float(v) for v in d.ImagePositionPatient]\n    sx, sy, sz = row.ipp_x, row.ipp_y, row.ipp_z\n    print(len(row.iop))\n    iop = [float(i) for i, in row.iop]\n    print(iop)\n    o0, o1, o2, o3, o4, o5 = row.iop#[float(v) for v in d.ImageOrientationPatient]\n    delx, dely = row.ps_x, row.ps_y#d.PixelSpacing\n\n    xx = o0 * delx * x + o3 * dely * y + sx\n    yy = o1 * delx * x + o4 * dely * y + sy\n    zz = o2 * delx * x + o5 * dely * y + sz\n    return xx,yy,zz\n\npoint = sub_coor[['ipp_x', 'ipp_y', 'ipp_z']].values #2d\nprint(point)\n# here we project 2d to 3d\n#df = valid_data[0].sagittal_t2[0].df #get dicom header\ncenter=[] \nfor _, row in sub_coor.iterrows():\n    #x,y,z = point[i]\n    xx,yy,zz = project_to_3d(row)\n    center.append([xx,yy,zz])\ncenter = np.array(center) #3d\n\n# == 2. we get closest axial slices to the CSC points =================\n#df = valid_data[0].axial_t2[0].df\ndisplay(meta_df)\norientation = np.array(meta_df.iop.values.tolist())\nposition= np.array(meta_df[['ipp_x', 'ipp_y', 'ipp_z']].values.tolist())\nox = orientation[:, :3]\noy = orientation[:, 3:]\noz = np.cross(ox,oy)\nt = center.reshape(-1,1,3) - position.reshape(1,-1,3)\ndis = (oz.reshape(1,-1,3) * t).sum(-1)  # np.dot(point-s,oz)\ndis = np.fabs(dis)\nclosest = dis.argmin(-1)\nclosest_df = meta_df.iloc[closest]\ndisplay(closest_df)\n","metadata":{"execution":{"iopub.status.busy":"2024-08-07T21:14:26.814919Z","iopub.execute_input":"2024-08-07T21:14:26.815362Z","iopub.status.idle":"2024-08-07T21:14:26.960604Z","shell.execute_reply.started":"2024-08-07T21:14:26.815328Z","shell.execute_reply":"2024-08-07T21:14:26.958756Z"}}},{"cell_type":"markdown","source":"# thanks @hengck\n# project 2d to 3d\ndef project_to_3d(x,y,z, df):\n    d = df.iloc[z]\n    H, W = d.H, d.W\n    sx, sy, sz = [float(v) for v in d.ImagePositionPatient]\n    o0, o1, o2, o3, o4, o5, = [float(v) for v in d.ImageOrientationPatient]\n    delx, dely = d.PixelSpacing\n\n    xx = o0 * delx * x + o3 * dely * y + sx\n    yy = o1 * delx * x + o4 * dely * y + sy\n    zz = o2 * delx * x + o5 * dely * y + sz\n    return xx,yy,zz\n\npoint = point[0,20:] #2d\nprint(point)\n# here we project 2d to 3d\ndf = valid_data[0].sagittal_t2[0].df #get dicom header\ncenter=[] \nfor i in range(5):\n    x,y,z = point[i]\n    xx,yy,zz = project_to_3d(x, y, z, df)\n    center.append([xx,yy,zz])\ncenter = np.array(center) #3d\n\n# == 2. we get closest axial slices to the CSC points =================\ndf = valid_data[0].axial_t2[0].df\ndisplay(df)\norientation = np.array(df.ImageOrientationPatient.values.tolist())\nposition= np.array(df.ImagePositionPatient.values.tolist())\nox = orientation[:, :3]\noy = orientation[:, 3:]\noz = np.cross(ox,oy)\nt = center.reshape(-1,1,3) - position.reshape(1,-1,3)\ndis = (oz.reshape(1,-1,3) * t).sum(-1)  # np.dot(point-s,oz)\ndis = np.fabs(dis)\nclosest = dis.argmin(-1)\nclosest_df = df.iloc[closest]\ndisplay(closest_df)","metadata":{}}]}