{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Importing Libraries and Reading CSV Files","metadata":{}},{"cell_type":"code","source":"import time\nsince = time.time()\n!pip3 install python-gdcm","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:39:42.923285Z","iopub.execute_input":"2021-06-25T05:39:42.923750Z","iopub.status.idle":"2021-06-25T05:39:53.538094Z","shell.execute_reply.started":"2021-06-25T05:39:42.923706Z","shell.execute_reply":"2021-06-25T05:39:53.536980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport seaborn as sn\nimport pydicom as dicom # Dicom (Digital Imaging in Medicine) - medical image datasets, storage and transfer\nimport os\nfrom tqdm import tqdm # allows you to output a smart progress bar by wrapping around any iterable\nimport glob # retrieve files/pathnames matching a specified pattern\nimport pprint # pretty-print” arbitrary Python data structures\nimport ast # \nfrom pydicom.pixel_data_handlers.util import apply_voi_lut #\nimport wandb #\n\nfrom PIL import Image\nimport shutil\nimport gdcm\n\npath = '/kaggle/input/siim-covid19-detection/'\ntrain_image_level = pd.read_csv(path + \"train_image_level.csv\")\ntrain_study_level = pd.read_csv(path + \"train_study_level.csv\")\n\ntrain_image_level.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:39:53.540140Z","iopub.execute_input":"2021-06-25T05:39:53.540571Z","iopub.status.idle":"2021-06-25T05:39:55.624609Z","shell.execute_reply.started":"2021-06-25T05:39:53.540520Z","shell.execute_reply":"2021-06-25T05:39:55.623684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_image_level.describe()","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:39:55.627031Z","iopub.execute_input":"2021-06-25T05:39:55.627718Z","iopub.status.idle":"2021-06-25T05:39:55.686441Z","shell.execute_reply.started":"2021-06-25T05:39:55.627670Z","shell.execute_reply":"2021-06-25T05:39:55.685735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n2040 ids without box","metadata":{}},{"cell_type":"code","source":"train_study_level_key = train_study_level.id.str[:-6]\ntraining_set = pd.merge(left = train_study_level, right = train_image_level, how = 'right', left_on = train_study_level_key, right_on = 'StudyInstanceUID')\nprint(training_set.shape)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:39:55.687649Z","iopub.execute_input":"2021-06-25T05:39:55.688029Z","iopub.status.idle":"2021-06-25T05:39:55.712776Z","shell.execute_reply.started":"2021-06-25T05:39:55.688001Z","shell.execute_reply":"2021-06-25T05:39:55.711720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(training_set.loc[0])","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:39:55.714149Z","iopub.execute_input":"2021-06-25T05:39:55.714491Z","iopub.status.idle":"2021-06-25T05:39:55.722991Z","shell.execute_reply.started":"2021-06-25T05:39:55.714436Z","shell.execute_reply":"2021-06-25T05:39:55.721858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read Image in DCM format\n\n## Searching Desired StudyInstanceUID in all Subfolders","metadata":{}},{"cell_type":"code","source":"def read_dcm(i):\n    path_train = path + 'train/' + training_set.loc[i, 'StudyInstanceUID']\n    #print(os.listdir(path_train))\n    img_id = training_set.loc[i, 'id_y'].replace('_image','.dcm')\n    \n    for dirname, _, filenames in os.walk(path_train):\n        for filename in filenames:\n            path_img_id = os.path.join(dirname, filename)\n            if path_img_id[-16:-4] == img_id:\n                #print(path_img_id[-16:-4])\n                break\n      \n    last_folder_in_path = os.listdir(path_train)[0]\n    path_train = path_train + '/{}/'.format(last_folder_in_path)\n    data_file = dicom.dcmread(path_img_id)#(path_train + img_id)\n    return img_id, data_file\n\ndef Image_resize(img,len_x):\n    img = img.resize((len_x, len_x), Image.ANTIALIAS)\n    return img\n\n_,img = read_dcm(1)\nprint(img)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:39:55.724237Z","iopub.execute_input":"2021-06-25T05:39:55.724579Z","iopub.status.idle":"2021-06-25T05:39:55.918490Z","shell.execute_reply.started":"2021-06-25T05:39:55.724550Z","shell.execute_reply":"2021-06-25T05:39:55.917749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# DCM Image as Pixel Array","metadata":{}},{"cell_type":"code","source":"img_arr = img.pixel_array\nplt.imshow(img_arr)\nprint('Shape:', img_arr.shape)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:39:55.919717Z","iopub.execute_input":"2021-06-25T05:39:55.920226Z","iopub.status.idle":"2021-06-25T05:39:56.806919Z","shell.execute_reply.started":"2021-06-25T05:39:55.920195Z","shell.execute_reply":"2021-06-25T05:39:56.805719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(img_arr.ravel()) #calculating histogram","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:39:56.809935Z","iopub.execute_input":"2021-06-25T05:39:56.810360Z","iopub.status.idle":"2021-06-25T05:39:57.069065Z","shell.execute_reply.started":"2021-06-25T05:39:56.810313Z","shell.execute_reply":"2021-06-25T05:39:57.067913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_,img = read_dcm(5) # Another Image\nimg_arr = img.pixel_array\n\nplt.imshow(img_arr)\nprint('Image Shape:',img_arr.shape)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:39:57.071013Z","iopub.execute_input":"2021-06-25T05:39:57.071472Z","iopub.status.idle":"2021-06-25T05:39:57.694312Z","shell.execute_reply.started":"2021-06-25T05:39:57.071403Z","shell.execute_reply":"2021-06-25T05:39:57.692678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(img_arr.ravel()) ","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:39:57.695254Z","iopub.status.idle":"2021-06-25T05:39:57.695688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Observation\n### Some images have an intensity range of 0 to 255\n### Some images have a higher intensity range.(can be 16 bit images/ 12 bit allocated)","metadata":{}},{"cell_type":"markdown","source":"\n# Histogram Equalization + Converting to RGB Images","metadata":{}},{"cell_type":"code","source":"def To_16bit(img_arr):\n    min_arr = np.amin(img_arr)\n    max_arr = np.amax(img_arr)\n    range_array = max_arr - min_arr\n\n    return np.round((img_arr-min_arr)/range_array*(np.power(2,16)*3-1)) \n\n\ndef To_RGB(img_arr): #Extend to 24 bit then segment by 8 bit\n    min_arr = np.amin(img_arr)\n    max_arr = np.amax(img_arr)\n    range_array = max_arr - min_arr\n    \n    lenx, leny = img_arr.shape\n    rgbArray = np.zeros((lenx,leny,3), 'uint8')\n\n    arr2 = np.round((img_arr-min_arr)/range_array*(np.power(2,24)-1))\n    rgbArray[:,:, 0] = arr2 % np.power(2,8)\n    arr2 = (arr2-rgbArray[:,:, 0])/np.power(2,8)\n    rgbArray[:,:, 1] = arr2 % np.power(2,8)\n    arr2 = (arr2-rgbArray[:,:, 1])/np.power(2,8)\n    rgbArray[:,:, 2] = arr2 % np.power(2,8)\n    \n    return rgbArray\n    \ndef To_RGB2(img_arr): #Extend to 16 bit then segment by 8 bit top(G), 8 bit bottom(R), 8 bit overall(B)\n    min_arr = np.amin(img_arr)\n    max_arr = np.amax(img_arr)\n    range_array = max_arr - min_arr\n    \n    lenx, leny = img_arr.shape\n    rgbArray = np.zeros((lenx,leny,3), 'uint8')\n\n    arr2 = np.round((img_arr-min_arr)/range_array*(np.power(2,16)-1))\n    rgbArray[:,:, 0] = arr2 % np.power(2,8)                 #8 bit bottom\n    arr2 = np.floor(arr2/np.power(2,8))                     #8 bit top\n    rgbArray[:,:, 1] = arr2 \n    \n    rgbArray[:,:, 2] = np.round((img_arr-min_arr)/range_array*(np.power(2,8)-1)) #8 bit overall\n    \n    \n    return rgbArray    ","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:39:57.696724Z","iopub.status.idle":"2021-06-25T05:39:57.697136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rgb_arr = To_RGB2(img_arr)\nplt.imshow(rgb_arr)\nprint('Image Shape:', rgb_arr.shape)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:39:57.698021Z","iopub.status.idle":"2021-06-25T05:39:57.698420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plotting\n\n### A few images are still grey (All of R,G,B components have the same value)","metadata":{}},{"cell_type":"code","source":"fig, axes = plt.subplots(3,3, figsize=(20,16))\nfig.subplots_adjust(hspace=.1, wspace=.1)\naxes = axes.ravel()\n\nstart_index = 0\n\nfor row in range(9):\n    img_id, img = read_dcm(row + start_index)\n    img = img.pixel_array\n    img = To_RGB2(img)\n    print(img_id, training_set.loc[row, 'label'].split(' ')[0])\n    if (training_set.loc[row + start_index,'boxes'] == training_set.loc[row + start_index,'boxes']):\n        boxes = ast.literal_eval(training_set.loc[row + start_index,'boxes'])\n        for box in boxes:\n            p = matplotlib.patches.Rectangle((box['x'], box['y']),\n                                              box['width'], box['height'],\n                                              ec = 'r', fc = 'none', lw = 2.\n                                            )\n            axes[row].add_patch(p)\n    axes[row].imshow(img, cmap = 'gray')\n    axes[row].set_title(training_set.loc[row, 'label'].split(' ')[0] + '  Image Shape:' +str(img.shape))\n    axes[row].set_xticklabels([])\n    axes[row].set_yticklabels([])","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:39:57.699394Z","iopub.status.idle":"2021-06-25T05:39:57.699811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"time_elapsed = time.time() - since\nprint('Time from start {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:39:57.700673Z","iopub.status.idle":"2021-06-25T05:39:57.701071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"folderlocation = './Images'\nif not os.path.exists(folderlocation):\n        os.mkdir(folderlocation)\n\nfor iter_name in ['train', 'val', 'test']:\n    folderlocation = './Images/' + iter_name\n    if not os.path.exists(folderlocation):\n            os.mkdir(folderlocation)\n            \n    folderlocation = './Images/' + iter_name +'/none'\n    if not os.path.exists(folderlocation):\n            os.mkdir(folderlocation)\n            \n    folderlocation = './Images/' + iter_name +'/opacity'\n    if not os.path.exists(folderlocation):\n            os.mkdir(folderlocation)\n    \n","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:39:57.701857Z","iopub.status.idle":"2021-06-25T05:39:57.702259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"iter_split=0\nfolderlocation = './Images/'\n\nfor row in range(len(training_set)): # len(training_set)\n    img_id, img = read_dcm(row)\n    img = img.pixel_array\n    img = To_RGB2(img)\n    iter_split = iter_split +1\n    \n    img_destination = folderlocation +'train/'+ training_set.loc[row, 'label'].split(' ')[0] +'/'+img_id[:-4] +'.jpeg'\n    \n    if iter_split%20 == 19:\n        img_destination = folderlocation +'test/'+ training_set.loc[row, 'label'].split(' ')[0] +'/'+img_id[:-4] +'.jpeg'\n    if iter_split%20 == 18:\n        img_destination = folderlocation +'val/'+ training_set.loc[row, 'label'].split(' ')[0] +'/'+img_id[:-4] +'.jpeg'\n        \n    im = Image.fromarray(img)\n    im = Image_resize(im,2048)\n    im.save(img_destination)\n    \n    \n    if row%500 == 499:\n        time_elapsed = time.time() - since\n        print('Time from start {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))\n        print('Percentage Complete:', np.round(10000*(row+1)/6334)/100)\n    \n    \nprint('100% Complete.')    ","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:39:57.703105Z","iopub.status.idle":"2021-06-25T05:39:57.703519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shutil.make_archive('SIIM-FISABIO-RSNA-JPEG', 'zip', folderlocation)\nshutil.rmtree(folderlocation)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:39:57.704432Z","iopub.status.idle":"2021-06-25T05:39:57.704838Z"},"trusted":true},"execution_count":null,"outputs":[]}]}