{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**SIIM-FISABIO-RSNA COVID-19 Detection**\n\nIdentify and localize COVID-19 abnormalities on chest radiographs","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n%lsmagic\n%matplotlib inline\nimport os\nimport cv2\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport pydicom\nimport pydicom.data\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\n#import glob\n#from tqdm.notebook import tqdm\nfrom fastai.vision.all import *\nfrom fastai.medical.imaging import *\nimport seaborn as sns\nsns.set(style=\"whitegrid\")\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-22T13:16:36.523807Z","iopub.execute_input":"2021-06-22T13:16:36.524535Z","iopub.status.idle":"2021-06-22T13:16:41.203034Z","shell.execute_reply.started":"2021-06-22T13:16:36.524431Z","shell.execute_reply":"2021-06-22T13:16:41.202033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! conda install -c conda-forge gdcm -y","metadata":{"execution":{"iopub.status.busy":"2021-06-22T13:16:41.204545Z","iopub.execute_input":"2021-06-22T13:16:41.205126Z","iopub.status.idle":"2021-06-22T13:17:57.103686Z","shell.execute_reply.started":"2021-06-22T13:16:41.205087Z","shell.execute_reply":"2021-06-22T13:17:57.102238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_folder='../input/siim-covid19-detection'","metadata":{"execution":{"iopub.status.busy":"2021-06-22T13:17:57.108288Z","iopub.execute_input":"2021-06-22T13:17:57.108701Z","iopub.status.idle":"2021-06-22T13:17:57.114771Z","shell.execute_reply.started":"2021-06-22T13:17:57.108659Z","shell.execute_reply":"2021-06-22T13:17:57.112657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Take a quick look at the Image level CSV**","metadata":{}},{"cell_type":"code","source":"import pandas as pd\ntrain_image=pd.read_csv('../input/siim-covid19-detection/train_image_level.csv')\ntrain_image.head(10)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T13:17:57.116687Z","iopub.execute_input":"2021-06-22T13:17:57.117052Z","iopub.status.idle":"2021-06-22T13:17:57.23597Z","shell.execute_reply.started":"2021-06-22T13:17:57.117015Z","shell.execute_reply":"2021-06-22T13:17:57.234846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_image.info()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T13:17:57.237474Z","iopub.execute_input":"2021-06-22T13:17:57.237861Z","iopub.status.idle":"2021-06-22T13:17:57.263036Z","shell.execute_reply.started":"2021-06-22T13:17:57.237825Z","shell.execute_reply":"2021-06-22T13:17:57.261887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_image.columns)\nprint(train_image.shape)\nprint(train_image.dtypes)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T13:17:57.264706Z","iopub.execute_input":"2021-06-22T13:17:57.265071Z","iopub.status.idle":"2021-06-22T13:17:57.278691Z","shell.execute_reply.started":"2021-06-22T13:17:57.265035Z","shell.execute_reply":"2021-06-22T13:17:57.277517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_image.describe(include='all')","metadata":{"execution":{"iopub.status.busy":"2021-06-22T13:17:57.28066Z","iopub.execute_input":"2021-06-22T13:17:57.281224Z","iopub.status.idle":"2021-06-22T13:17:57.334126Z","shell.execute_reply.started":"2021-06-22T13:17:57.281169Z","shell.execute_reply":"2021-06-22T13:17:57.333257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_image.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T13:24:52.342769Z","iopub.execute_input":"2021-06-22T13:24:52.34323Z","iopub.status.idle":"2021-06-22T13:24:52.358359Z","shell.execute_reply.started":"2021-06-22T13:24:52.343194Z","shell.execute_reply":"2021-06-22T13:24:52.357489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**A quick look at the Study level**","metadata":{}},{"cell_type":"code","source":"train_study=pd.read_csv('../input/siim-covid19-detection/train_study_level.csv')\ntrain_study.head(10)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T13:17:57.351348Z","iopub.execute_input":"2021-06-22T13:17:57.351722Z","iopub.status.idle":"2021-06-22T13:17:57.389811Z","shell.execute_reply.started":"2021-06-22T13:17:57.351686Z","shell.execute_reply":"2021-06-22T13:17:57.388617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_study.info()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T13:17:57.39164Z","iopub.execute_input":"2021-06-22T13:17:57.392354Z","iopub.status.idle":"2021-06-22T13:17:57.414093Z","shell.execute_reply.started":"2021-06-22T13:17:57.392296Z","shell.execute_reply":"2021-06-22T13:17:57.412885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_study.columns)\nprint(train_study.shape)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T13:17:57.417965Z","iopub.execute_input":"2021-06-22T13:17:57.418393Z","iopub.status.idle":"2021-06-22T13:17:57.425138Z","shell.execute_reply.started":"2021-06-22T13:17:57.41835Z","shell.execute_reply":"2021-06-22T13:17:57.423681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_study.describe(include='all')","metadata":{"execution":{"iopub.status.busy":"2021-06-22T13:17:57.426759Z","iopub.execute_input":"2021-06-22T13:17:57.427117Z","iopub.status.idle":"2021-06-22T13:17:57.475388Z","shell.execute_reply.started":"2021-06-22T13:17:57.427081Z","shell.execute_reply":"2021-06-22T13:17:57.474244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_study.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T13:17:57.47718Z","iopub.execute_input":"2021-06-22T13:17:57.477656Z","iopub.status.idle":"2021-06-22T13:17:57.489762Z","shell.execute_reply.started":"2021-06-22T13:17:57.477615Z","shell.execute_reply":"2021-06-22T13:17:57.488214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train_study['Negative for Pneumonia'].value_counts())\ndisplay(train_study['Typical Appearance'].value_counts())\ndisplay(train_study['Indeterminate Appearance'].value_counts())\ndisplay(train_study['Atypical Appearance'].value_counts())","metadata":{"execution":{"iopub.status.busy":"2021-06-22T13:17:57.491374Z","iopub.execute_input":"2021-06-22T13:17:57.491739Z","iopub.status.idle":"2021-06-22T13:17:57.51829Z","shell.execute_reply.started":"2021-06-22T13:17:57.491702Z","shell.execute_reply":"2021-06-22T13:17:57.517105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20, 6))\nsns.kdeplot(train_study['Negative for Pneumonia'], shade=False,label='Negative for Pneumonia')\nsns.kdeplot(train_study['Typical Appearance'], shade=False, label='Typical Appearance')\nsns.kdeplot(train_study['Indeterminate Appearance'], shade=False, label='Indeterminate Appearance')\nsns.kdeplot(train_study['Atypical Appearance'], shade=False,label= 'Atypical Appearance')\n\n# Labeling of plot\nplt.xlabel('Labels for each study'); plt.ylabel('Density'); plt.title('Distribution of Study level');\nplt.legend(loc='upper left')","metadata":{"execution":{"iopub.status.busy":"2021-06-22T13:25:34.065849Z","iopub.execute_input":"2021-06-22T13:25:34.066502Z","iopub.status.idle":"2021-06-22T13:25:34.565747Z","shell.execute_reply.started":"2021-06-22T13:25:34.066463Z","shell.execute_reply":"2021-06-22T13:25:34.564702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_classes = ['Negative for Pneumonia', 'Typical Appearance', 'Indeterminate Appearance', 'Atypical Appearance']\nnp.unique(train_study[study_classes].values, axis=0)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T13:17:58.167505Z","iopub.execute_input":"2021-06-22T13:17:58.168007Z","iopub.status.idle":"2021-06-22T13:17:58.192838Z","shell.execute_reply.started":"2021-06-22T13:17:58.167955Z","shell.execute_reply":"2021-06-22T13:17:58.191301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.figure(figsize = (10,5))\nplt.bar([1,2,3,4], train_study[study_classes].values.sum(axis=0))\nplt.xticks([1,2,3,4],study_classes)\nplt.ylabel('Frequency')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T13:17:58.194782Z","iopub.execute_input":"2021-06-22T13:17:58.195323Z","iopub.status.idle":"2021-06-22T13:17:58.383154Z","shell.execute_reply.started":"2021-06-22T13:17:58.19527Z","shell.execute_reply":"2021-06-22T13:17:58.381918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We have our bounding box labels provided in the `label` column. The format is as follows:\n\n`[class ID] [confidence score] [bounding box]`\n\n* class ID - either `opacity` or `none`\n* confidence score - confidence from your neural network model. If none, the confidence is `1`.\n* bounding box - typical `xmin ymin xmax ymax` format. If class ID is none, the bounding box is `1 0 0 1 1`.\n\nThe bounding boxes are also provided in easily readable dictionary format in column `boxes`, and the study that each image is a part of is provided in`StudyInstanceUID`.\n\nLet's quick look at the distribution of opacity vs none:","metadata":{}},{"cell_type":"code","source":"train_image['split_label'] = train_image.label.apply(lambda x: [x.split()[offs:offs+6] for offs in range(0, len(x.split()), 6)])","metadata":{"execution":{"iopub.status.busy":"2021-06-22T13:17:58.385093Z","iopub.execute_input":"2021-06-22T13:17:58.385558Z","iopub.status.idle":"2021-06-22T13:17:58.41807Z","shell.execute_reply.started":"2021-06-22T13:17:58.385509Z","shell.execute_reply":"2021-06-22T13:17:58.416915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classes_freq = []\nfor i in range(len(train_image)):\n    for j in train_image.iloc[i].split_label: classes_freq.append(j[0])\nplt.hist(classes_freq)\nplt.ylabel('Frequency')","metadata":{"execution":{"iopub.status.busy":"2021-06-22T13:17:58.41951Z","iopub.execute_input":"2021-06-22T13:17:58.419878Z","iopub.status.idle":"2021-06-22T13:17:59.570692Z","shell.execute_reply.started":"2021-06-22T13:17:58.419843Z","shell.execute_reply":"2021-06-22T13:17:59.569436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bbox_areas = []\nfor i in range(len(train_image)):\n    for j in train_image.iloc[i].split_label:\n        bbox_areas.append((float(j[4])-float(j[2]))*(float(j[5])*float(j[3])))\nplt.hist(bbox_areas)\nplt.ylabel('Frequency')","metadata":{"execution":{"iopub.status.busy":"2021-06-22T13:17:59.572436Z","iopub.execute_input":"2021-06-22T13:17:59.572901Z","iopub.status.idle":"2021-06-22T13:18:00.704867Z","shell.execute_reply.started":"2021-06-22T13:17:59.572852Z","shell.execute_reply":"2021-06-22T13:18:00.703678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def dicom2array(path, voi_lut=True, fix_monochrome=True):\n    dicom = pydicom.read_file(path)\n    # VOI LUT (if available by DICOM device) is used to\n    # transform raw DICOM data to \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    return data\n        \n    \ndef plot_img(img, size=(7, 7), is_rgb=True, title=\"\", cmap='gray'):\n    plt.figure(figsize=size)\n    plt.imshow(img, cmap=cmap)\n    plt.suptitle(title)\n    plt.show()\n\n\ndef plot_imgs(imgs, cols=4, size=7, is_rgb=True, title=\"\", cmap='gray', img_size=(500,500)):\n    rows = len(imgs)//cols + 1\n    fig = plt.figure(figsize=(cols*size, rows*size))\n    for i, img in enumerate(imgs):\n        if img_size is not None:\n            img = cv2.resize(img, img_size)\n        fig.add_subplot(rows, cols, i+1)\n        plt.imshow(img, cmap=cmap)\n    plt.suptitle(title)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T13:18:00.706443Z","iopub.execute_input":"2021-06-22T13:18:00.706977Z","iopub.status.idle":"2021-06-22T13:18:00.71938Z","shell.execute_reply.started":"2021-06-22T13:18:00.70691Z","shell.execute_reply":"2021-06-22T13:18:00.7185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dicom_paths = get_dicom_files('../input/siim-covid19-detection/train')\nimgs = [dicom2array(path) for path in dicom_paths[:4]]\nplot_imgs(imgs)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T13:18:00.723042Z","iopub.execute_input":"2021-06-22T13:18:00.723428Z","iopub.status.idle":"2021-06-22T13:18:46.762525Z","shell.execute_reply.started":"2021-06-22T13:18:00.723392Z","shell.execute_reply":"2021-06-22T13:18:46.761738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_images_per_study = []\nfor i in (Path('../input/siim-covid19-detection/train')).ls():\n    num_images_per_study.append(len(get_dicom_files(i)))\n    if len(get_dicom_files(i)) > 5:\n        print(f'Study {i} had {len(get_dicom_files(i))} images')","metadata":{"execution":{"iopub.status.busy":"2021-06-22T13:18:46.763932Z","iopub.execute_input":"2021-06-22T13:18:46.76452Z","iopub.status.idle":"2021-06-22T13:19:07.937835Z","shell.execute_reply.started":"2021-06-22T13:18:46.764484Z","shell.execute_reply":"2021-06-22T13:19:07.936643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Reference: \n","metadata":{}}]}