{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Introdution\nThis Notebook is how anyone can get started with this compition \n\nThis notebook is heavily inspired  by : https://www.kaggle.com/ruchi798/siim-covid-19-detection-eda-data-augmentation\n\n**Problem Statement:**\n* You’ll identify and localize COVID-19 abnormalities on chest radiographs. In particular, you'll categorize the radiographs as negative for pneumonia or typical, indeterminate, or atypical for COVID-19. \n\n**Evaluation Metric**\n*  mean Average Precision (mAP)\n\n","metadata":{}},{"cell_type":"markdown","source":"## Importing Nessesary Libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom tqdm import tqdm \nimport glob \nimport os\nimport matplotlib.pyplot as plt\nimport matplotlib.pylab as pylab\nimport seaborn as sns\nimport pprint\nimport pydicom as dicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nimport albumentations as A\nimport cv2\nimport wandb\nfrom PIL import Image","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Getting the Input Data \n\nWe are given with 2 csv file: train_study_level and train_image_level and Folder train with all the training images \n\nWe are supposed to merge the 2 csv files and make an column called \"path\" (Path to Image) into a DataFrame called train","metadata":{}},{"cell_type":"code","source":"train_study_level=pd.read_csv(\"../input/siim-covid19-detection/train_study_level.csv\")\ntrain_study_level.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_image_level=pd.read_csv(\"../input/siim-covid19-detection/train_image_level.csv\")\ntrain_image_level.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_study_level[\"StudyInstanceUID\"]=train_study_level[\"id\"].apply(lambda x:x.replace('_study','') )\ndel train_study_level['id']\ntrain=train_image_level.merge(train_study_level,on=\"StudyInstanceUID\")\ntrain.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"glob :Module used to retrieve files/pathnames matching a specified pattern\n","metadata":{}},{"cell_type":"code","source":"train_dir=\"../input/siim-covid19-detection/train/\"\ntest_dir=\"../input/siim-covid19-detection/test/\"\n\ntraining_paths = []\n\nfor i in tqdm(train['StudyInstanceUID']):\n    training_paths.append(glob.glob(os.path.join(train_dir, i +\"/*/*\"))[0])\n\ntrain['path'] = training_paths","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Exploration ","metadata":{}},{"cell_type":"code","source":"fig,ax=plt.subplots(4,1)\n\nsns.kdeplot(train[\"Negative for Pneumonia\"],shade=True,ax=ax[0])\nax[0].set_title(\"Negative for Pneumonia\")\nsns.kdeplot(train[\"Typical Appearance\"],shade=True,ax=ax[1])\nax[1].set_title(\"Typical Appearance\")\nsns.kdeplot(train[\"Indeterminate Appearance\"],shade=True,ax=ax[2])\nax[2].set_title(\"Indeterminate Appearance\")\nsns.kdeplot(train[\"Atypical Appearance\"],shade=True,ax=ax[3])\nax[3].set_title(\"Atypical Appearance\")\n\n\n\nfig.subplots_adjust( hspace=2.0, top=2.0)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig,ax=plt.subplots(4,1)\n\nsns.countplot(x=train[\"Negative for Pneumonia\"],ax=ax[0])\nax[0].set_title(\"Negative for Pneumonia\")\nsns.countplot(x=train[\"Typical Appearance\"],ax=ax[1])\nax[1].set_title(\"Typical Appearance\")\nsns.countplot(x=train[\"Indeterminate Appearance\"],ax=ax[2])\nax[2].set_title(\"Indeterminate Appearance\")\nsns.countplot(x=train[\"Atypical Appearance\"],ax=ax[3])\nax[3].set_title(\"Atypical Appearance\")\n\n\n\nfig.subplots_adjust( hspace=2.0, top=2.0)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dicom Data\n* Images given to us are in the dicom format \n* Digital Imaging and Communications in Medicine is the standard for the communication and management of medical imaging information and related data\n* We will need to read and convert dicom images to jpeg for convinience using dicom python library","metadata":{}},{"cell_type":"markdown","source":"#### Images and Metadata","metadata":{}},{"cell_type":"code","source":"voi_lut=True\nfix_monochrome=True\n\ndef dicom_dataset_to_dict(filename):\n    \"\"\"Credit: https://github.com/pydicom/pydicom/issues/319\n               https://www.kaggle.com/raddar/convert-dicom-to-np-array-the-correct-way\n    \"\"\"\n    \n    dicom_header = dicom.dcmread(filename) \n    \n    #====== DICOM FILE DATA ======\n    dicom_dict = {}\n    repr(dicom_header)\n    for dicom_value in dicom_header.values():\n        if dicom_value.tag == (0x7fe0, 0x0010):\n            #discard pixel data\n            continue\n        if type(dicom_value.value) == dicom.dataset.Dataset:\n            dicom_dict[dicom_value.name] = dicom_dataset_to_dict(dicom_value.value)\n        else:\n            v = _convert_value(dicom_value.value)\n            dicom_dict[dicom_value.name] = v\n      \n    del dicom_dict['Pixel Representation']\n    \n    #====== DICOM IMAGE DATA ======\n    # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom_header.pixel_array, dicom_header)\n    else:\n        data = dicom_header.pixel_array\n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom_header.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n    data = data - np.min(data)\n    data = data / np.max(data)\n    modified_image_data = (data * 255).astype(np.uint8)\n    \n    return dicom_dict, modified_image_data\n\ndef _sanitise_unicode(s):\n    return s.replace(u\"\\u0000\", \"\").strip()\n\ndef _convert_value(v):\n    t = type(v)\n    if t in (list, int, float):\n        cv = v\n    elif t == str:\n        cv = _sanitise_unicode(v)\n    elif t == bytes:\n        s = v.decode('ascii', 'replace')\n        cv = _sanitise_unicode(s)\n    elif t == dicom.valuerep.DSfloat:\n        cv = float(v)\n    elif t == dicom.valuerep.IS:\n        cv = int(v)\n    else:\n        cv = repr(v)\n    return cv","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for filename in train.path[0:5]:\n    df, img_array = dicom_dataset_to_dict(filename)\n    \n    fig, ax = plt.subplots(1, 2, figsize=[10, 8])\n    ax[0].imshow(img_array, cmap=plt.cm.gray)\n    ax[1].imshow(img_array, cmap=plt.cm.plasma)    \n    plt.show()\n    \n    pprint.pprint(df)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Converting and storing Dicom images in jpeg format ","metadata":{}},{"cell_type":"code","source":"classes = ['Negative for Pneumonia','Typical Appearance', 'Indeterminate Appearance', 'Atypical Appearance']\nfor siim_class in classes:\n    print(siim_class)\n    for _, row in train[train[siim_class]==1].iloc[:2].iterrows():\n        filename = row['path']\n        df, img_array = dicom_dataset_to_dict(filename)\n        \n        fig, ax = plt.subplots(1, 2, figsize=[10, 8])\n        ax[0].imshow(img_array, cmap=plt.cm.gray)\n        ax[1].imshow(img_array, cmap=plt.cm.plasma)   \n        plt.show()\n        \n        img_object = Image.fromarray(img_array)\n        \n        \n\n        ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Converted Dataset : https://www.kaggle.com/xhlulu/siim-covid19-resized-to-256px-jpg","metadata":{}},{"cell_type":"code","source":"train_jpg_directory = '../input/siim-covid19-resized-to-256px-jpg/train'\ntest_jpg_directory = '../input/siim-covid19-resized-to-256px-jpg/test'\n\ndef getImagePaths(path):\n    image_names = []\n    for dirname, _, filenames in os.walk(path):\n        for filename in filenames:\n            fullpath = os.path.join(dirname, filename)\n            image_names.append(fullpath)\n    return image_names\n\ntrain_images_path = getImagePaths(train_jpg_directory)\ntest_images_path = getImagePaths(test_jpg_directory)\n\nprint(f\"Number of train images: {len(train_images_path)}\\n\")\nprint(f\"Number of test images:  {len(test_images_path)}\\n\")\n\ndef getShape(data, images_paths):\n    shape = cv2.imread(images_paths[0]).shape\n    for image_path in images_paths:\n        image_shape=cv2.imread(image_path).shape\n        if (image_shape!=shape):\n            return data +\" - Different image shape\"\n        else:\n            return data +\" - Same image shape \" + str(shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}