{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Go to the colab notebook for details\n","metadata":{}},{"cell_type":"code","source":"pip install tensorflow-io","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install python-gdcm","metadata":{"execution":{"iopub.status.busy":"2021-07-29T09:52:10.418551Z","iopub.execute_input":"2021-07-29T09:52:10.418930Z","iopub.status.idle":"2021-07-29T09:52:19.230838Z","shell.execute_reply.started":"2021-07-29T09:52:10.418839Z","shell.execute_reply":"2021-07-29T09:52:19.229801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nt_image_fnames = []\npath = \"/kaggle/input/siim-covid19-detection/train/\"\nimport os\nlen(os.listdir(path))\nfor root, dirs, filenames in os.walk(path):\n    for fname in filenames:\n        t_image_fnames.append(os.path.join(root,fname))\n\ntrain_image_level = pd.read_csv(\"/kaggle/input/siim-covid19-detection/train_image_level.csv\")\ntrain_study_level = pd.read_csv(\"/kaggle/input/siim-covid19-detection/train_study_level.csv\")    \nlen(t_image_fnames)\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":false,"_kg_hide-input":false,"execution":{"iopub.status.busy":"2021-07-29T09:52:19.234064Z","iopub.execute_input":"2021-07-29T09:52:19.234319Z","iopub.status.idle":"2021-07-29T09:52:37.823029Z","shell.execute_reply.started":"2021-07-29T09:52:19.234291Z","shell.execute_reply":"2021-07-29T09:52:37.822292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if len(train_image_level.id) == len(t_image_fnames):\n    print(\"length is almost the same\")\n    \nelse:\n    print(\"you fucked up\")\n    \ntrain_image_level.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-29T09:52:37.824792Z","iopub.execute_input":"2021-07-29T09:52:37.825158Z","iopub.status.idle":"2021-07-29T09:52:37.849289Z","shell.execute_reply.started":"2021-07-29T09:52:37.825123Z","shell.execute_reply":"2021-07-29T09:52:37.848500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_image_level.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2021-07-29T09:52:37.850804Z","iopub.execute_input":"2021-07-29T09:52:37.851156Z","iopub.status.idle":"2021-07-29T09:52:37.860931Z","shell.execute_reply.started":"2021-07-29T09:52:37.851121Z","shell.execute_reply":"2021-07-29T09:52:37.859902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# verifying the directory stucture.\nfor i in train_image_level[\"StudyInstanceUID\"]:\n    if i == \"0135d267f462\":\n        print(\"its here\")\n        break\ntrain_image_level.id = train_image_level.id.str.split(\"_\",expand = True,n=0)\n\n# The number of cases with more than one images\n\ntrain_image_level.StudyInstanceUID.duplicated().sum()        ","metadata":{"execution":{"iopub.status.busy":"2021-07-29T09:52:37.862468Z","iopub.execute_input":"2021-07-29T09:52:37.862904Z","iopub.status.idle":"2021-07-29T09:52:37.887528Z","shell.execute_reply.started":"2021-07-29T09:52:37.862869Z","shell.execute_reply":"2021-07-29T09:52:37.886630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Creating our own validation set\n\nNow gotta create the datasets and create our own validation set\n","metadata":{}},{"cell_type":"code","source":"NUM_IMAGES = 200 #@param {type:\"slider\", min:100, max:6334, step: 10 }","metadata":{"execution":{"iopub.status.busy":"2021-07-29T09:52:37.893087Z","iopub.execute_input":"2021-07-29T09:52:37.893630Z","iopub.status.idle":"2021-07-29T09:52:37.898809Z","shell.execute_reply.started":"2021-07-29T09:52:37.893596Z","shell.execute_reply":"2021-07-29T09:52:37.896366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_image_level.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-29T09:52:37.902332Z","iopub.execute_input":"2021-07-29T09:52:37.903552Z","iopub.status.idle":"2021-07-29T09:52:37.931012Z","shell.execute_reply.started":"2021-07-29T09:52:37.903521Z","shell.execute_reply":"2021-07-29T09:52:37.930120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Setup X & y variables\n\nX = t_image_fnames\ny = train_image_level.label\nbboxes = train_image_level.boxes","metadata":{"execution":{"iopub.status.busy":"2021-07-29T09:52:37.934354Z","iopub.execute_input":"2021-07-29T09:52:37.934693Z","iopub.status.idle":"2021-07-29T09:52:37.943353Z","shell.execute_reply.started":"2021-07-29T09:52:37.934657Z","shell.execute_reply":"2021-07-29T09:52:37.941381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y[1]","metadata":{"execution":{"iopub.status.busy":"2021-07-29T09:52:37.946643Z","iopub.execute_input":"2021-07-29T09:52:37.947178Z","iopub.status.idle":"2021-07-29T09:52:37.957901Z","shell.execute_reply.started":"2021-07-29T09:52:37.947138Z","shell.execute_reply":"2021-07-29T09:52:37.956847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Lets see what improting an image looks like:\n\n","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nimport pydicom\n%matplotlib inline\nplt.figure(figsize = (10,8))\nimage = pydicom.dcmread(X[21])\nplt.imshow(image.pixel_array,cmap=plt.cm.bone)\n","metadata":{"execution":{"iopub.status.busy":"2021-07-29T09:52:37.960408Z","iopub.execute_input":"2021-07-29T09:52:37.961364Z","iopub.status.idle":"2021-07-29T09:52:39.896658Z","shell.execute_reply.started":"2021-07-29T09:52:37.961317Z","shell.execute_reply":"2021-07-29T09:52:39.895866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Function to show 25 Images\n","metadata":{"execution":{"iopub.status.busy":"2021-07-12T17:04:12.938691Z","iopub.execute_input":"2021-07-12T17:04:12.939058Z","iopub.status.idle":"2021-07-12T17:04:13.063005Z","shell.execute_reply.started":"2021-07-12T17:04:12.93903Z","shell.execute_reply":"2021-07-12T17:04:13.06206Z"}}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport pydicom\n\ndef show_25_images(images):\n    \"\"\"\n    Displays a plot of 25 images and their labes for training images\n    \"\"\"\n    \n    # setup the figure\n    plt.figure(figsize = (10,10))\n    \n    # loop through 25 files to display 25 images\n    for i in range(25):\n        # Create subplots ( 5 rows , 5 columns)\n        ax = plt.subplot(5,5,i+1)\n        # display an image\n        image = pydicom.dcmread(images[i])\n        plt.imshow(image.pixel_array,cmap = plt.cm.bone)\n        plt.axis(\"off\")\n        \n        ","metadata":{"execution":{"iopub.status.busy":"2021-07-29T09:52:39.897693Z","iopub.execute_input":"2021-07-29T09:52:39.898047Z","iopub.status.idle":"2021-07-29T09:52:39.905215Z","shell.execute_reply.started":"2021-07-29T09:52:39.897991Z","shell.execute_reply":"2021-07-29T09:52:39.904434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_25_images(X)","metadata":{"execution":{"iopub.status.busy":"2021-07-29T09:52:39.906582Z","iopub.execute_input":"2021-07-29T09:52:39.907157Z","iopub.status.idle":"2021-07-29T09:52:55.094899Z","shell.execute_reply.started":"2021-07-29T09:52:39.907122Z","shell.execute_reply":"2021-07-29T09:52:55.093962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(X), len(y), len(bboxes)","metadata":{"execution":{"iopub.status.busy":"2021-07-29T09:52:55.096201Z","iopub.execute_input":"2021-07-29T09:52:55.096556Z","iopub.status.idle":"2021-07-29T09:52:55.103375Z","shell.execute_reply.started":"2021-07-29T09:52:55.096519Z","shell.execute_reply":"2021-07-29T09:52:55.101378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preprocessing Images (Turning images into tensors)\n\nWrite a function to preprocess images\n* 1. Take an image filepath as input\n* 2. Use Tensorflow to read the file and save it to a variable, `image`\n* 3. Turn our image (dicom) into Tensors\n* 4. Resize the image to a shape of (416,416) # Hyper parameter to experiment with\n* 5. Return the modified image","metadata":{"execution":{"iopub.status.busy":"2021-07-13T12:25:46.496994Z","iopub.execute_input":"2021-07-13T12:25:46.497365Z","iopub.status.idle":"2021-07-13T12:25:46.501647Z","shell.execute_reply.started":"2021-07-13T12:25:46.497326Z","shell.execute_reply":"2021-07-13T12:25:46.500385Z"}}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport pydicom \n%matplotlib inline\nplt.figure(figsize = (10,8))\nimage = pydicom.dcmread(X[2])\nplt.imshow(image.pixel_array,cmap=plt.cm.bone)","metadata":{"execution":{"iopub.status.busy":"2021-07-29T09:52:55.104658Z","iopub.execute_input":"2021-07-29T09:52:55.105206Z","iopub.status.idle":"2021-07-29T09:52:56.468814Z","shell.execute_reply.started":"2021-07-29T09:52:55.105154Z","shell.execute_reply":"2021-07-29T09:52:56.467929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import ast\nboxes = ast.literal_eval(train_image_level.loc[3,'boxes'])\nboxes","metadata":{"execution":{"iopub.status.busy":"2021-07-29T09:52:56.470161Z","iopub.execute_input":"2021-07-29T09:52:56.470511Z","iopub.status.idle":"2021-07-29T09:52:56.478577Z","shell.execute_reply.started":"2021-07-29T09:52:56.470475Z","shell.execute_reply":"2021-07-29T09:52:56.477612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = pydicom.dcmread(X[3])\nimg = img.pixel_array\nimg","metadata":{"execution":{"iopub.status.busy":"2021-07-29T09:52:56.480132Z","iopub.execute_input":"2021-07-29T09:52:56.480804Z","iopub.status.idle":"2021-07-29T09:52:56.498661Z","shell.execute_reply.started":"2021-07-29T09:52:56.480767Z","shell.execute_reply":"2021-07-29T09:52:56.497929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib\nfig,ax = plt.subplots(figsize=(10,8))\n\nfor box in boxes:\n    p = matplotlib.patches.Rectangle((box['x'], box['y']),box['width'], box['height'],\n                                     ec='r' , fc = 'none',lw = 2.)\n    ax.add_patch(p)\n    \nax.imshow(img, cmap= plt.cm.bone)\nplt.show()\n                                     ","metadata":{"execution":{"iopub.status.busy":"2021-07-29T09:52:56.499794Z","iopub.execute_input":"2021-07-29T09:52:56.500156Z","iopub.status.idle":"2021-07-29T09:52:57.280448Z","shell.execute_reply.started":"2021-07-29T09:52:56.500131Z","shell.execute_reply":"2021-07-29T09:52:57.279519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axs = plt.subplots(3,3,figsize = (20,16))\nfig.subplots_adjust(hspace = .1 , wspace = .1)\naxs = axs.ravel()\n\nfor row in range(9):\n    study = train_image_level.loc[row, 'StudyInstanceUID']\n    dt_file =pydicom.dcmread( X[row])\n    img = dt_file.pixel_array\n    if(train_image_level.loc[row,'boxes']!= train_image_level.loc[row,'boxes']) == False:\n        boxes = ast.literal_eval(train_image_level.loc[row,'boxes'])\n        \n        for box in boxes:\n            p = matplotlib.patches.Rectangle((box['x'],box['y']), box['width'], box['height'],\n                                           ec = 'r',fc = 'none', lw = 2.)\n            axs[row].add_patch(p)\n    axs[row].imshow(img,cmap = plt.cm.bone)\n    axs[row].set_title(train_image_level.loc[row,'label'].split(' ')[0])\n    axs[row].set_xticklabels([])\n    axs[row].set_yticklabels([])\n    ","metadata":{"execution":{"iopub.status.busy":"2021-07-29T09:52:57.281835Z","iopub.execute_input":"2021-07-29T09:52:57.282181Z","iopub.status.idle":"2021-07-29T09:53:10.636201Z","shell.execute_reply.started":"2021-07-29T09:52:57.282148Z","shell.execute_reply":"2021-07-29T09:53:10.635317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2021-07-29T09:53:10.637616Z","iopub.execute_input":"2021-07-29T09:53:10.638104Z","iopub.status.idle":"2021-07-29T09:53:56.543711Z","shell.execute_reply.started":"2021-07-29T09:53:10.638066Z","shell.execute_reply":"2021-07-29T09:53:56.542725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import PIL\nfrom PIL import Image, ImageDraw, ImageFont\nimport tensorflow as tf\nimport tensorflow_io as tfio\nimg1 = pydicom.dcmread(X[0])\nb = tf.constant(img1.pixel_array)\n","metadata":{"execution":{"iopub.status.busy":"2021-07-29T10:04:13.480860Z","iopub.execute_input":"2021-07-29T10:04:13.481219Z","iopub.status.idle":"2021-07-29T10:04:13.498349Z","shell.execute_reply.started":"2021-07-29T10:04:13.481188Z","shell.execute_reply":"2021-07-29T10:04:13.497535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create a function to preprocess the images\n1. It should Take an image filepath as input i.e. X[0]\n2. Using pydicom it should convert the read file into `TENSORS`\n3. Normalise the image ( convert colour channel from 0-225 to 0-1 ).\n4. Resize the image to be of the shape of (416,416)\n5. Return the modified image.\n\n* Had to convert the images in to RGB to convert them into acceptable Tensor\n","metadata":{}},{"cell_type":"code","source":"# Define image size\nIMAGE_SIZE = 416\n\n# create a function for preprocessing the images\n\ndef process_image(path):\n    \"\"\"\n    takes an image file and turn image into a tensor\n    \"\"\"\n    img=pydicom.dcmread(path)\n    img = img.pixel_array\n    img = np.array(img)\n    \n    if len(img.shape)<3 :\n        w, h = img.shape\n        ret = np.empty((w, h, 3), dtype=np.uint8)\n        ret[:, :, 2] =  ret[:, :, 1] =  ret[:, :, 0] =  img\n        img=ret \n    # read in image file path and turn image into a tensor\n    image = img\n    image = tf.convert_to_tensor(image, np.float32)\n     #convert the colour channel values from 0-255 to 0-1\n    image = tf.image.convert_image_dtype(image,tf.float32)\n     # Resize the image to our desired value (224,224)\n    image = tf.image.resize(image, size=[IMAGE_SIZE, IMAGE_SIZE])\n    \n    \n    return image","metadata":{"execution":{"iopub.status.busy":"2021-07-29T09:53:58.956132Z","iopub.execute_input":"2021-07-29T09:53:58.956507Z","iopub.status.idle":"2021-07-29T09:53:58.963379Z","shell.execute_reply.started":"2021-07-29T09:53:58.956462Z","shell.execute_reply":"2021-07-29T09:53:58.962682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a = process_image(X[0])","metadata":{"execution":{"iopub.status.busy":"2021-07-29T10:03:37.650166Z","iopub.execute_input":"2021-07-29T10:03:37.650494Z","iopub.status.idle":"2021-07-29T10:03:37.759175Z","shell.execute_reply.started":"2021-07-29T10:03:37.650463Z","shell.execute_reply":"2021-07-29T10:03:37.758332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Turning our data into batches\n\nwe do about 32 (This is the batch size) images at a time (you can manually adjust the batchsize if need to)\n\nIn order to use TensorFLow effectively, we need our data in the form of Tensor Tuples which look like this: (image , label)","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a simple function to return a tuple( image , label)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}