{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Importing Libraries and Reading CSV Files","metadata":{}},{"cell_type":"code","source":"import time\nsince = time.time()\n!pip3 install python-gdcm","metadata":{"execution":{"iopub.status.busy":"2021-07-08T10:54:06.958337Z","iopub.execute_input":"2021-07-08T10:54:06.95878Z","iopub.status.idle":"2021-07-08T10:54:13.391478Z","shell.execute_reply.started":"2021-07-08T10:54:06.958744Z","shell.execute_reply":"2021-07-08T10:54:13.390578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport seaborn as sn\nimport pydicom as dicom # Dicom (Digital Imaging in Medicine) - medical image datasets, storage and transfer\nimport os\nfrom tqdm import tqdm # allows you to output a smart progress bar by wrapping around any iterable\nimport glob # retrieve files/pathnames matching a specified pattern\nimport pprint # pretty-print” arbitrary Python data structures\nimport ast # \nfrom pydicom.pixel_data_handlers.util import apply_voi_lut #\nimport wandb #\nfrom scipy import ndimage\nfrom PIL import Image\nimport shutil\nimport gdcm\n\npath = '/kaggle/input/siim-covid19-detection/'\ntrain_image_level = pd.read_csv(path + \"train_image_level.csv\")\ntrain_study_level = pd.read_csv(path + \"train_study_level.csv\")\n\ntrain_image_level.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T10:54:13.392923Z","iopub.execute_input":"2021-07-08T10:54:13.393203Z","iopub.status.idle":"2021-07-08T10:54:13.447051Z","shell.execute_reply.started":"2021-07-08T10:54:13.393172Z","shell.execute_reply":"2021-07-08T10:54:13.446094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_image_level.describe()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T10:54:13.44911Z","iopub.execute_input":"2021-07-08T10:54:13.449505Z","iopub.status.idle":"2021-07-08T10:54:13.490603Z","shell.execute_reply.started":"2021-07-08T10:54:13.449464Z","shell.execute_reply":"2021-07-08T10:54:13.489721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n2040 ids without box","metadata":{}},{"cell_type":"code","source":"train_study_level_key = train_study_level.id.str[:-6]\ntraining_set = pd.merge(left = train_study_level, right = train_image_level, how = 'right', left_on = train_study_level_key, right_on = 'StudyInstanceUID')\nprint(training_set.shape)","metadata":{"execution":{"iopub.status.busy":"2021-07-08T10:54:13.492061Z","iopub.execute_input":"2021-07-08T10:54:13.492364Z","iopub.status.idle":"2021-07-08T10:54:13.509655Z","shell.execute_reply.started":"2021-07-08T10:54:13.492328Z","shell.execute_reply":"2021-07-08T10:54:13.508647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(training_set.loc[0])","metadata":{"execution":{"iopub.status.busy":"2021-07-08T10:54:13.511018Z","iopub.execute_input":"2021-07-08T10:54:13.511307Z","iopub.status.idle":"2021-07-08T10:54:13.519618Z","shell.execute_reply.started":"2021-07-08T10:54:13.511275Z","shell.execute_reply":"2021-07-08T10:54:13.51875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read Image in DCM format\n\n## Searching Desired StudyInstanceUID in all Subfolders","metadata":{}},{"cell_type":"code","source":"def read_dcm(i):\n    path_train = path + 'train/' + training_set.loc[i, 'StudyInstanceUID']\n    #print(os.listdir(path_train))\n    img_id = training_set.loc[i, 'id_y'].replace('_image','.dcm')\n    \n    for dirname, _, filenames in os.walk(path_train):\n        for filename in filenames:\n            path_img_id = os.path.join(dirname, filename)\n            if path_img_id[-16:-4] == img_id:\n                #print(path_img_id[-16:-4])\n                break\n      \n    last_folder_in_path = os.listdir(path_train)[0]\n    path_train = path_train + '/{}/'.format(last_folder_in_path)\n    data_file = dicom.dcmread(path_img_id)#(path_train + img_id)\n    return img_id, data_file\n\ndef Image_resize(img,len_x):\n    img = img.resize((len_x, len_x), Image.ANTIALIAS)\n    return img\n\n_,img = read_dcm(1)\nprint(img)","metadata":{"execution":{"iopub.status.busy":"2021-07-08T10:54:13.52087Z","iopub.execute_input":"2021-07-08T10:54:13.521172Z","iopub.status.idle":"2021-07-08T10:54:13.546053Z","shell.execute_reply.started":"2021-07-08T10:54:13.521136Z","shell.execute_reply":"2021-07-08T10:54:13.54496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# DCM Image as Pixel Array","metadata":{}},{"cell_type":"code","source":"img_arr = img.pixel_array\nplt.imshow(img_arr)\nprint('Shape:', img_arr.shape)","metadata":{"execution":{"iopub.status.busy":"2021-07-08T10:54:13.547461Z","iopub.execute_input":"2021-07-08T10:54:13.547881Z","iopub.status.idle":"2021-07-08T10:54:14.401864Z","shell.execute_reply.started":"2021-07-08T10:54:13.547837Z","shell.execute_reply":"2021-07-08T10:54:14.400734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(img_arr.ravel()) #calculating histogram","metadata":{"execution":{"iopub.status.busy":"2021-07-08T10:54:14.405094Z","iopub.execute_input":"2021-07-08T10:54:14.405523Z","iopub.status.idle":"2021-07-08T10:54:14.668066Z","shell.execute_reply.started":"2021-07-08T10:54:14.405479Z","shell.execute_reply":"2021-07-08T10:54:14.667099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_,img = read_dcm(5) # Another Image\nimg_arr = img.pixel_array\n\nplt.imshow(img_arr)\nprint('Image Shape:',img_arr.shape)","metadata":{"execution":{"iopub.status.busy":"2021-07-08T10:54:14.669446Z","iopub.execute_input":"2021-07-08T10:54:14.669729Z","iopub.status.idle":"2021-07-08T10:54:15.827363Z","shell.execute_reply.started":"2021-07-08T10:54:14.6697Z","shell.execute_reply":"2021-07-08T10:54:15.82624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(img_arr.ravel()) ","metadata":{"execution":{"iopub.status.busy":"2021-07-08T10:54:15.828562Z","iopub.execute_input":"2021-07-08T10:54:15.828916Z","iopub.status.idle":"2021-07-08T10:54:16.117965Z","shell.execute_reply.started":"2021-07-08T10:54:15.828883Z","shell.execute_reply":"2021-07-08T10:54:16.116788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Observation\n### Some images have an intensity range of 0 to 255\n### Some images have a higher intensity range.(can be 16 bit images/ 12 bit allocated)","metadata":{}},{"cell_type":"markdown","source":"# Edge Detection","metadata":{}},{"cell_type":"code","source":"'''\n'sobel_filters' Copied from \nhttps://towardsdatascience.com/canny-edge-detection-step-by-step-in-python-computer-vision-b49c3a2d8123\n'''\n\ndef sobel_filters(img):\n    Kx = np.array([[-1, 0, 1], [-2, 0, 2], [-1, 0, 1]], np.float32)\n    Ky = np.array([[1, 2, 1], [0, 0, 0], [-1, -2, -1]], np.float32)\n\n    \n    Ix = ndimage.filters.convolve(img, Kx)\n    Iy = ndimage.filters.convolve(img, Ky)\n    \n    G = np.hypot(Ix, Iy)\n    G = G / G.max() * 255\n    theta = np.arctan2(Iy, Ix)\n    return (G, theta)\n\nmg_arr = np.array(img_arr)\n#rint(img_arr.shape)\nimg_arr2,A = sobel_filters(img_arr)\nplt.imshow(img_arr2)\n\nprint(A.shape)","metadata":{"execution":{"iopub.status.busy":"2021-07-08T10:54:16.119217Z","iopub.execute_input":"2021-07-08T10:54:16.119497Z","iopub.status.idle":"2021-07-08T10:54:17.755771Z","shell.execute_reply.started":"2021-07-08T10:54:16.11947Z","shell.execute_reply":"2021-07-08T10:54:17.754905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(A)","metadata":{"execution":{"iopub.status.busy":"2021-07-08T10:54:17.757162Z","iopub.execute_input":"2021-07-08T10:54:17.757468Z","iopub.status.idle":"2021-07-08T10:54:18.684563Z","shell.execute_reply.started":"2021-07-08T10:54:17.757436Z","shell.execute_reply":"2021-07-08T10:54:18.68351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# Histogram Equalization + Converting to RGB Images","metadata":{}},{"cell_type":"code","source":"def To_16bit(img_arr):\n    min_arr = np.amin(img_arr)\n    max_arr = np.amax(img_arr)\n    range_array = max_arr - min_arr\n\n    return np.round((img_arr-min_arr)/range_array*(np.power(2,16)*3-1)) \n\n\n    \ndef To_RGB(img_arr): #Extend to 16 bit then segment by 8 bit top(G), 8 bit bottom(R), 8 bit overall(B)\n    min_arr = np.amin(img_arr)\n    max_arr = np.amax(img_arr)\n    range_array = max_arr - min_arr\n   \n    img_arr2, A = sobel_filters(img_arr)\n    \n    lenx, leny = img_arr.shape\n    rgbArray = np.zeros((lenx,leny,3), 'uint8')\n\n    arr2 = np.round((img_arr-min_arr)/range_array*(np.power(2,8)-1)) #Histogram equalization\n    rgbArray[:,:, 0] = arr2      #8 bit top    \n    rgbArray[:,:, 1] = img_arr2  #edge magnitude     \n    rgbArray[:,:, 2] = A         #edge angle\n    \n    \n    return rgbArray    ","metadata":{"execution":{"iopub.status.busy":"2021-07-08T10:54:18.686351Z","iopub.execute_input":"2021-07-08T10:54:18.686763Z","iopub.status.idle":"2021-07-08T10:54:18.695873Z","shell.execute_reply.started":"2021-07-08T10:54:18.686725Z","shell.execute_reply":"2021-07-08T10:54:18.695152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rgb_arr = To_RGB(img_arr)\nplt.imshow(rgb_arr)\nprint('Image Shape:', rgb_arr.shape)","metadata":{"execution":{"iopub.status.busy":"2021-07-08T10:54:44.687094Z","iopub.execute_input":"2021-07-08T10:54:44.687491Z","iopub.status.idle":"2021-07-08T10:54:46.385948Z","shell.execute_reply.started":"2021-07-08T10:54:44.68746Z","shell.execute_reply":"2021-07-08T10:54:46.384901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plotting\n","metadata":{}},{"cell_type":"code","source":"fig, axes = plt.subplots(3,3, figsize=(20,16))\nfig.subplots_adjust(hspace=.1, wspace=.1)\naxes = axes.ravel()\n\nstart_index = 0\n\nfor row in range(9):\n    img_id, img = read_dcm(row + start_index)\n    img = img.pixel_array\n    img = To_RGB(img)\n    \n    print(img_id, training_set.loc[row, 'label'].split(' ')[0])\n    if (training_set.loc[row + start_index,'boxes'] == training_set.loc[row + start_index,'boxes']):\n        boxes = ast.literal_eval(training_set.loc[row + start_index,'boxes'])\n        for box in boxes:\n            p = matplotlib.patches.Rectangle((box['x'], box['y']),\n                                              box['width'], box['height'],\n                                              ec = 'r', fc = 'none', lw = 2.\n                                            )\n            axes[row].add_patch(p)\n    axes[row].imshow(img, cmap = 'gray')\n    axes[row].set_title(training_set.loc[row, 'label'].split(' ')[0] + '  Image Shape:' +str(img.shape))\n    axes[row].set_xticklabels([])\n    axes[row].set_yticklabels([])","metadata":{"execution":{"iopub.status.busy":"2021-07-08T11:08:42.088234Z","iopub.execute_input":"2021-07-08T11:08:42.088625Z","iopub.status.idle":"2021-07-08T11:09:04.039094Z","shell.execute_reply.started":"2021-07-08T11:08:42.088589Z","shell.execute_reply":"2021-07-08T11:09:04.037705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"time_elapsed = time.time() - since\nprint('Time from start {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))","metadata":{"execution":{"iopub.status.busy":"2021-07-08T10:54:18.728299Z","iopub.status.idle":"2021-07-08T10:54:18.728742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"folderlocation = './Images'\nif not os.path.exists(folderlocation):\n        os.mkdir(folderlocation)\n\nfor iter_name in ['train', 'val', 'test']:\n    folderlocation = './Images/' + iter_name\n    if not os.path.exists(folderlocation):\n            os.mkdir(folderlocation)\n            \n    folderlocation = './Images/' + iter_name +'/none'\n    if not os.path.exists(folderlocation):\n            os.mkdir(folderlocation)\n            \n    folderlocation = './Images/' + iter_name +'/opacity'\n    if not os.path.exists(folderlocation):\n            os.mkdir(folderlocation)\n    \n","metadata":{"execution":{"iopub.status.busy":"2021-07-08T10:54:18.729765Z","iopub.status.idle":"2021-07-08T10:54:18.730238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"iter_split= 0 \nfolderlocation = './Images/'\n\nfor row in range(len(training_set)): # len(training_set)\n    img_id, img = read_dcm(row)\n    img = img.pixel_array\n    img = To_RGB(img)\n    iter_split = iter_split +1\n    \n    if iter_split%20 < 18:\n        img_destination = folderlocation +'train/'+ training_set.loc[row, 'label'].split(' ')[0] +'/'+img_id[:-4] +'.jpeg'\n    \n    if iter_split%20 == 19:\n        img_destination = folderlocation +'test/'+ training_set.loc[row, 'label'].split(' ')[0] +'/'+img_id[:-4] +'.jpeg'\n    if iter_split%20 == 18:\n        img_destination = folderlocation +'val/'+ training_set.loc[row, 'label'].split(' ')[0] +'/'+img_id[:-4] +'.jpeg'\n        \n    im = Image.fromarray(img)\n    im = Image_resize(im,512)\n    im.save(img_destination)\n    \n    \n    if row%500 == 499:\n        time_elapsed = time.time() - since\n        print('Time from start {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))\n        print('Percentage Complete:', np.round(10000*(row+1)/6334)/100)\n    \n    \nprint('Percentage Complete: 100')    ","metadata":{"execution":{"iopub.status.busy":"2021-07-08T10:54:18.731452Z","iopub.status.idle":"2021-07-08T10:54:18.731891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shutil.make_archive('SIIM-FISABIO-RSNA-JPEG', 'zip', folderlocation)\nshutil.rmtree(folderlocation)","metadata":{"execution":{"iopub.status.busy":"2021-07-08T10:54:18.732975Z","iopub.status.idle":"2021-07-08T10:54:18.733383Z"},"trusted":true},"execution_count":null,"outputs":[]}]}