{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Welcome to the [SIIM-FISABIO-RSNA COVID-19 Detection](http://www.kaggle.com/c/siim-covid19-detection) competition to identify and localize COVID-19 abnormalities on chest radiographs.\n\nThis notebook is a work in progress EDA notebook. \n\n","metadata":{}},{"cell_type":"code","source":"#! conda install -c conda-forge gdcm -y","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Libraries","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib\nimport pydicom #as dicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nimport cv2\nimport ast\nimport seaborn as sns\n\nfrom fastai.vision.all import *\nfrom fastai.medical.imaging import *\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load","metadata":{}},{"cell_type":"code","source":"path = '/kaggle/input/siim-covid19-detection/'\nos.listdir(path)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_image = pd.read_csv(path+'train_image_level.csv')\ntrain_study = pd.read_csv(path+'train_study_level.csv')\nsamp_subm = pd.read_csv(path+'sample_submission.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So we have:\n\n* `train_study_level.csv` - the train study-level metadata, with one row for each study, including correct labels.\n* `train_image_level.csv` - the train image-level metadata, with one row for each image, including both correct labels and any bounding boxes in a dictionary format. Some images in both test and train have multiple bounding boxes.\n* `sample_submission.csv` - a sample submission file containing all image- and study-level IDs.\n* `train` folder - comprises 6,334 chest scans in DICOM format, stored in paths with the form `study`/`series`/`image`\n* `test` folder - The hidden test dataset is of roughly the same scale as the training dataset.\n","metadata":{}},{"cell_type":"markdown","source":"## Overview","metadata":{}},{"cell_type":"code","source":"txt = \"The shape of image samples, study samples and sample submissions is: {}, {}, {}\"\nprint(txt.format(train_image.shape, train_study.shape, samp_subm.shape))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_image['StudyInstanceUID'].nunique()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_study['id'].nunique()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"studyGroups = train_image.groupby(['StudyInstanceUID']).size().sort_values(ascending=False)\n#groupsDist = studyGroups","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ImsPerStudy = studyGroups.value_counts().sort_values(ascending=False)\nlist(range(11, 17))\nprint(ImsPerStudy)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize=(8, 4))\naxes = ImsPerStudy.plot.bar(rot=0, subplots=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are 6054 unique imaging studies. The majority of these contain one image. train_image contains 6334 samples. 280 more samples than train_study","metadata":{}},{"cell_type":"code","source":"train_image.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_study.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"samp_subm.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ts_cols = train_study.columns\n\n# counting the training labels\nm = []\nn = []\nfor i in range(1,5):\n    m.append(train_study[ts_cols[i]].sum())\n    n.append(ts_cols[i])\nprint(m)\nprint(n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize=(8, 4))\n\nax.bar(n, m)\nax.set_xticklabels(n, rotation=30)\nax.set_title('Distribution pneumonia and COVID types')\nplt.grid()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_image.columns","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_image['id'][0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_image['label'][0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_image['boxes'][0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Images","metadata":{}},{"cell_type":"code","source":"def dicom2array(path, voi_lut=True, fix_monochrome=True):\n    dicom = pydicom.read_file(path)\n    # VOI LUT (if available by DICOM device) is used to\n    # transform raw DICOM data to \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    return data\n        \n    \ndef plot_img(img, size=(7, 7), is_rgb=True, title=\"\", cmap='gray'):\n    plt.figure(figsize=size)\n    plt.imshow(img, cmap=cmap)\n    plt.suptitle(title)\n    plt.show()\n\n\ndef plot_imgs(imgs, cols=4, size=7, is_rgb=True, title=\"\", cmap='gray', img_size=(500,500)):\n    rows = len(imgs)//cols + 1\n    fig = plt.figure(figsize=(cols*size, rows*size))\n    for i, img in enumerate(imgs):\n        if img_size is not None:\n            img = cv2.resize(img, img_size)\n        fig.add_subplot(rows, cols, i+1)\n        plt.imshow(img, cmap=cmap)\n    plt.suptitle(title)\n    plt.show()\n    \n# thanks to https://www.kaggle.com/tanlikesmath/siim-covid-19-detection-a-simple-eda","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dicom_paths = get_dicom_files(path+'train')\nprint(dicom_paths[:4])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imgs = [dicom2array(path) for path in dicom_paths[:4]]\nplot_imgs(imgs)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"--- If this was useful, please upvote! :)","metadata":{}},{"cell_type":"markdown","source":"## Thanks for reading! More to follow!","metadata":{}}]}