{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport pydicom as dicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\nfrom ast import literal_eval\nimport os\nimport glob\nimport seaborn as sns\nsns.set_style(\"whitegrid\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"voi_lut=True\nfix_monochrome=True\n\ndef dicom_dataset_to_dict(filename):\n    \"\"\"Credit: https://github.com/pydicom/pydicom/issues/319\n               https://www.kaggle.com/raddar/convert-dicom-to-np-array-the-correct-way\n    \"\"\"\n    \n    dicom_header = dicom.dcmread(filename) \n    \n    #====== DICOM FILE DATA ======\n    dicom_dict = {}\n    repr(dicom_header)\n    for dicom_value in dicom_header.values():\n        if dicom_value.tag == (0x7fe0, 0x0010):\n            #discard pixel data\n            continue\n        if type(dicom_value.value) == dicom.dataset.Dataset:\n            dicom_dict[dicom_value.name] = dicom_dataset_to_dict(dicom_value.value)\n        else:\n            v = _convert_value(dicom_value.value)\n            dicom_dict[dicom_value.name] = v\n      \n    del dicom_dict['Pixel Representation']\n    \n    #====== DICOM IMAGE DATA ======\n    # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom_header.pixel_array, dicom_header)\n    else:\n        data = dicom_header.pixel_array\n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom_header.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n    data = data - np.min(data)\n    data = data / np.max(data)\n    modified_image_data = (data * 255).astype(np.uint8)\n    \n    return dicom_dict, modified_image_data\n\ndef _sanitise_unicode(s):\n    return s.replace(u\"\\u0000\", \"\").strip()\n\ndef _convert_value(v):\n    t = type(v)\n    if t in (list, int, float):\n        cv = v\n    elif t == str:\n        cv = _sanitise_unicode(v)\n    elif t == bytes:\n        s = v.decode('ascii', 'replace')\n        cv = _sanitise_unicode(s)\n    elif t == dicom.valuerep.DSfloat:\n        cv = float(v)\n    elif t == dicom.valuerep.IS:\n        cv = int(v)\n    else:\n        cv = repr(v)\n    return cv","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# SIIM-FISABIO-RSNA COVID-19 Detection\n### Identify and localize COVID-19 abnormalities on chest radiographs","metadata":{}},{"cell_type":"markdown","source":"**In this competition, we are identifying and localizing COVID-19 abnormalities on chest radiographs. This is an object detection and classification problem.**","metadata":{}},{"cell_type":"markdown","source":"# Load data","metadata":{}},{"cell_type":"code","source":"train_image_level = pd.read_csv('../input/siim-covid19-detection/train_image_level.csv')\ntrain_study_level =  pd.read_csv('../input/siim-covid19-detection/train_study_level.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_image_level.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_study_level.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Checking for NaN","metadata":{}},{"cell_type":"code","source":"train_image_level.isna().sum()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_study_level.isna().sum()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Merge ","metadata":{}},{"cell_type":"code","source":"train_study_level['StudyInstanceUID'] = train_study_level['id'].apply(lambda x : x.split('_')[0])\ntrain_study_level = train_study_level.drop('id',axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result = pd.merge(train_image_level, train_study_level, on=\"StudyInstanceUID\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Creating a Categorical Column","metadata":{}},{"cell_type":"code","source":"labels = result[['Negative for Pneumonia', 'Typical Appearance',\n                 'Indeterminate Appearance', 'Atypical Appearance']]\n\nresult['category'] = labels.apply(lambda x: x[x==1].index.values[0], axis=1)\n\nresult.drop(['Negative for Pneumonia', 'Typical Appearance',\n             'Indeterminate Appearance', 'Atypical Appearance','label'],\n           axis=1, inplace=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Fill NaN","metadata":{}},{"cell_type":"code","source":"result['boxes'] = result['boxes'].fillna(0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nans = result[result['boxes']==0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result['boxes'] = result['boxes'].apply(lambda x: literal_eval(x) if x != 0 else [{'x':0,'y':0,'width':0,'height':0}])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(10,4))\nsns.countplot(x='category', data=nans, ax=ax)\nax.set_title('Total Count of NaN per Category')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Get Image Directory","metadata":{}},{"cell_type":"code","source":"training_paths = []\ntrain_directory = '../input/siim-covid19-detection/train'\n\nfor UID in result['StudyInstanceUID']:\n    training_paths.append(glob.glob(os.path.join(train_directory, UID +\"/*/*\"))[0])\n\nresult['path'] = training_paths","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA","metadata":{}},{"cell_type":"markdown","source":"## Categories","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(10,4))\nsns.countplot(x='category', data=result, ax=ax)\nax.set_title('Total Count of Categories')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Need to deal with unbalanced dataset","metadata":{}},{"cell_type":"markdown","source":"## Visualizations","metadata":{}},{"cell_type":"code","source":"def visualize_image(sample):\n\n    df, img_array = dicom_dataset_to_dict(sample['path'])\n\n    fig, ax = plt.subplots(figsize=(8,8))\n    ax.imshow(img_array, cmap='jet')\n    ax.set_xticks([])\n    ax.set_yticks([])\n\n    for i,location in enumerate(sample.boxes):\n\n        rect = patches.Rectangle((location['x'], location['y']),\n                                 location['width'], location['height'],\n                                 linewidth=1, edgecolor='k',\n                                 facecolor='none')\n        ax.add_patch(rect)\n    \n    ax.set_title(f'ID: {sample.id} Label: {sample.category}')\n    plt.show()\n    ","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Negative for Pneumonia","metadata":{}},{"cell_type":"code","source":"sample = result.iloc[1]\nvisualize_image(sample)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample = result.iloc[6]\nvisualize_image(sample)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample = result.iloc[27]\nvisualize_image(sample)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Typical Appearance","metadata":{}},{"cell_type":"code","source":"sample = result.iloc[0]\nvisualize_image(sample)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample = result.iloc[2]\nvisualize_image(sample)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample = result.iloc[4]\nvisualize_image(sample)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Indeterminate Appearance","metadata":{}},{"cell_type":"code","source":"sample = result.iloc[5]\nvisualize_image(sample)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample = result.iloc[18]\nvisualize_image(sample)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample = result.iloc[40]\nvisualize_image(sample)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Atypical Appearance","metadata":{}},{"cell_type":"code","source":"sample = result.iloc[3]\nvisualize_image(sample)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample = result.iloc[46]\nvisualize_image(sample)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample = result.iloc[53]\nvisualize_image(sample)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Work in Progress**","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}