{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Script to search for the in dicom images (in the relevant folders) in the order of their appearance in the csv file\n\n%cd /kaggle/input/siim-covid19-detection/\n%matplotlib inline\n\nimport pydicom\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\n\n# Can't get Andres' fun to work\n# def read_xray(path, voi_lut = True, fix_monochrome = True):\n  #  dicom = pydicom.read_file(path)\n    \n  #  # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to \"human-friendly\" view\n  #  if voi_lut:\n  #      data = apply_voi_lut(dicom.pixel_array, dicom)\n  #  else:\n  #      data = dicom.pixel_array\n               \n  #  # depending on this value, X-ray may look inverted - fix that:\n  #  if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n  #      data = np.amax(data) - data\n        \n  #  data = data - np.min(data)\n  #  data = data / np.max(data)\n  #  data = (data * 255).astype(np.uint8)\n        \n  #   return data","metadata":{"execution":{"iopub.status.busy":"2021-06-30T08:24:49.919726Z","iopub.execute_input":"2021-06-30T08:24:49.920086Z","iopub.status.idle":"2021-06-30T08:24:50.028135Z","shell.execute_reply.started":"2021-06-30T08:24:49.920053Z","shell.execute_reply":"2021-06-30T08:24:50.027074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Where are the .dcm files? What do the folders look like?\n\n# We loop over all folders and if it ends with \".dcm\" we print it.\npathdf = pd.DataFrame(columns=('root', 'file'))\ni = 0\nimport os\nfor root, dirs, files in os.walk(\".\"):\n    for file in files:\n        # print(root)   # ./test/2fb11712bc93/b056067b8455\n        # print(file) # a29c5a68b07b.dcm\n        # if file.endswith(\".dcm\"):                              # mixes train and test files\n        if file.endswith(\".dcm\") and root.startswith(\"./train\"): # We only need dcm files and train objects for now\n            path_file = os.path.join(root,file)\n            #  print(path_file)             # enable to see all the files\n            # ./train/cd5dd5e6f3f5/b2ee36aa2df5/d8ba599611e5.dcm [...]\n            # These are the paths that will be used to load in the dicom.\n            # Don't only print the paths, save them so that we can use them later on. \n            pathdf.loc[i] = [root,file]\n            i = i + 1\n            \n# We have \"test\" and \"train\" folders \n# Within we have 2 subfolders\n# And then we have the dcm file.\n# We probably just need the \"train\" flders for now\n\n# path_file is the path of one picture, we overwrite the \"path_file\" each time\n# print(len(path_file)) # 50\n# type(path_file)       # str\n# print(path_file)      # 1 [the last ]path: ./train/22353f3ff2d1/191c168c55c0/058d56025fa2.dcm","metadata":{"execution":{"iopub.status.busy":"2021-06-30T09:03:18.48582Z","iopub.execute_input":"2021-06-30T09:03:18.486172Z","iopub.status.idle":"2021-06-30T09:03:51.19216Z","shell.execute_reply.started":"2021-06-30T09:03:18.486143Z","shell.execute_reply":"2021-06-30T09:03:51.191042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data.frame of paths\npathdf.head\nprint(pathdf.shape) # Great: we indeed have 6334 images\npathdf.file\n","metadata":{"execution":{"iopub.status.busy":"2021-06-30T09:04:59.329719Z","iopub.execute_input":"2021-06-30T09:04:59.330108Z","iopub.status.idle":"2021-06-30T09:04:59.34054Z","shell.execute_reply.started":"2021-06-30T09:04:59.330077Z","shell.execute_reply":"2021-06-30T09:04:59.339241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Aside from all the pictures, we have 2 csv's : train_image_level and train_study_level","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#1 train_study_level\ntrainstudy = pd.read_csv('train_study_level.csv')\ntrainstudy.head(5)\n# trainstudy.shape \n# (6054, 5)\n# cols: id (unique study identifier), negative, typical, indeterminate, atypical\n# rows: 1 row for each study\n\n# these are the study level detections. \n# These are the ouputs in the sample_submission","metadata":{"execution":{"iopub.status.busy":"2021-06-30T09:05:02.181395Z","iopub.execute_input":"2021-06-30T09:05:02.181911Z","iopub.status.idle":"2021-06-30T09:05:02.206203Z","shell.execute_reply.started":"2021-06-30T09:05:02.18188Z","shell.execute_reply":"2021-06-30T09:05:02.205468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Separate study objects. These are the output ids for the submission file\ntrainstudy.iloc[:,0]\n# _study should be removed later on when we prepare the submission file","metadata":{"execution":{"iopub.status.busy":"2021-06-30T09:05:06.330548Z","iopub.execute_input":"2021-06-30T09:05:06.330951Z","iopub.status.idle":"2021-06-30T09:05:06.340639Z","shell.execute_reply.started":"2021-06-30T09:05:06.330919Z","shell.execute_reply":"2021-06-30T09:05:06.339252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"#2 train_image_level: AHA the one we're working with now.\ntrainimage = pd.read_csv('train_image_level.csv')\n\ntrainimage.shape\n# (6334, 4)\n\ntrainimage.head(5)\n# cols: id (6334 unique values), boxes, label, StudyInstanceUiD (6054 unique vals)\n# rows: 1 row for each image\n# The train data set comprises of 6,334 chest scans. \n# StudyInstance are the ppl? We hebben dus meerdere waarden per study object.\n# In the folders we have 6054 directories\n\n# These are the image level predictions. \n\n# type(trainimage) # dataframe\n\n# BFMA wants to order them in this way\n\n","metadata":{"execution":{"iopub.status.busy":"2021-06-30T09:05:09.00152Z","iopub.execute_input":"2021-06-30T09:05:09.001913Z","iopub.status.idle":"2021-06-30T09:05:09.0407Z","shell.execute_reply.started":"2021-06-30T09:05:09.001882Z","shell.execute_reply":"2021-06-30T09:05:09.039632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# There are multiple pictures per study object. There is an Id to link both csvs\n\ntrainimage.StudyInstanceUID # last col \"StudyInstanceUID\" with 6334 inputs\n# There's no \"_study\" at the end of the name\n\ntemp = trainimage.StudyInstanceUID.unique() # gives back an array\nprint(temp) # no underscores here.\n\nlen(temp) # 6054\n#StudyInstanceImageUnique = np.unique(StudyInstanceImage) # gives back an array\n#StudyInstanceImageUnique\n\n# But that's for later, we're here to load in the images. \n","metadata":{"execution":{"iopub.status.busy":"2021-06-30T09:05:12.960187Z","iopub.execute_input":"2021-06-30T09:05:12.960594Z","iopub.status.idle":"2021-06-30T09:05:12.971973Z","shell.execute_reply.started":"2021-06-30T09:05:12.960555Z","shell.execute_reply":"2021-06-30T09:05:12.970504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Each row the trainimage csv corresponds to a figure. How do we call that? \nprint(trainimage.id) # col vector of images (eg. id for one row: 000a312787f2_image). \n\n# Drop the image suffix for the first line [if we are working in a loop]\ni = 0                   # indicator to do this for all rows\ntemp = str(trainimage.id[i])\nprint(temp)             # 000a312787f2_image\nprint(temp[0:12])       # 000a312787f2we only take the first 12 letters\nprint(len(temp[0:12]))  # size 12\n\n# Do this for all images [if we want to make a new variable]\n# print(trainimage['id'])\n# trainimage_idsubstr = trainimage['id'].str[:12] # only take out first 12 letters to drop \"_image\" suffix\n# print(trainimage_idsubstr)","metadata":{"execution":{"iopub.status.busy":"2021-06-30T09:05:16.418778Z","iopub.execute_input":"2021-06-30T09:05:16.419147Z","iopub.status.idle":"2021-06-30T09:05:16.429709Z","shell.execute_reply.started":"2021-06-30T09:05:16.419114Z","shell.execute_reply":"2021-06-30T09:05:16.428426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# BFMA's new names\nnewId = [\"%04d\" % x for x in range(len(trainimage.id))] # 4 digits\n# print(newId)\nlen(newId) # 6334","metadata":{"execution":{"iopub.status.busy":"2021-06-30T09:11:57.611075Z","iopub.execute_input":"2021-06-30T09:11:57.611676Z","iopub.status.idle":"2021-06-30T09:11:57.622442Z","shell.execute_reply.started":"2021-06-30T09:11:57.61163Z","shell.execute_reply":"2021-06-30T09:11:57.621283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# block to look for the correct images in our folders. It prints the path in the order of the csv\n\n\n\n# Drop the image suffix for the first line [if we are working in a loop]\ni = 0                   # indicator to do this for all rows. We can loop over all rows\n# Let's try writing it for one image. It will not work on Kaggle, but it might on BFMA's machine. \n\n# Image name in the csv\nid_csv_i = str(trainimage.id[i])[0:12]\nprint(id_csv_i)\n\n# Match image name in the csv with the image name in pathdf\n# extract the file names wo/ \".dcm\" suffix\nid_df = pathdf.file.str[:12]\n# print(id_df)\n# We look for the id in the csv in a list of file names\nstates = id_csv_i == id_df\nj = np.where(states)[0] # position in the df\nprint(id_df[j])\n# Extract the relevant folders\nprint(pathdf.iloc[j,:])\npath = str(pathdf.root.values[j] + '/' + pathdf.file.values[j]) # .values so that we drop the index\nprint(path) # It adds square brackets. I don't want those square brackets\npath = path[2:len(path)-2]\nprint(path) # N00b solution\ntype(path) # str\n\n# Let's look at the dicom file\n# path = './train/cd5dd5e6f3f5/b2ee36aa2df5/d8ba599611e5.dcm' # manual\n# print(path) # ./train/cd5dd5e6f3f5/b2ee36aa2df5/d8ba599611e5.dcm [no square brackets]\n# type(path_file)  # str\nds = pydicom.read_file(path)\nds # A lot of meta data\nds.PatientName # anonymized\nds.dir(\"setup\") # empty\n\n# Save under a differnet path\n# I can read one image. Let's loop over all images and write them according to a newly chosen name.\nsuffix = '.dcm'\nnewName = str(newId[i]) + suffix\nprint(newName)\n# ds.save_as(newName) # Doesn't work on Kaggle.","metadata":{"execution":{"iopub.status.busy":"2021-06-30T09:13:09.968255Z","iopub.execute_input":"2021-06-30T09:13:09.968655Z","iopub.status.idle":"2021-06-30T09:13:10.026388Z","shell.execute_reply.started":"2021-06-30T09:13:09.968622Z","shell.execute_reply":"2021-06-30T09:13:10.024905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Show images\n\n# Let's use Andres' read_xray. Disabled for now. BFMA requires the real dcims. \n\n# path = \"b74f626d63fa.dcm\" # FileNotFound\n# path = 'test/2fb11712bc93/b056067b8455/a29c5a68b07b.dcm' # fix path:  Runtime error:  The following handlers are available to decode the pixel data however they are missing required dependencies: GDCM (req. GDCM), pylibjpeg (req. )\n#img = read_xray(path, fix_monochrome = False) \n#plt.figure(figsize = (12,12))\n#plt.imshow(img, 'gray')","metadata":{"execution":{"iopub.status.busy":"2021-06-28T09:46:58.488934Z","iopub.execute_input":"2021-06-28T09:46:58.489314Z","iopub.status.idle":"2021-06-28T09:46:58.89486Z","shell.execute_reply.started":"2021-06-28T09:46:58.489283Z","shell.execute_reply":"2021-06-28T09:46:58.892175Z"},"trusted":true},"execution_count":null,"outputs":[]}]}