{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Table of Contents\n\n- [Let's look at the distribution of number of opacities](#Let's-look-at-the-distribution-of-number-of-opacities)\n- [Pneumonia detected even if no opacity is observed](#Pneumonia-detected-even-if-no-opacity-is-observed)\n- [Opacities vs Pneumonia](#Opacities-vs-Pneumonia)\n    - [Couple of things to be clarified from the data](#Couple-of-things-to-be-clarified-from-the-data)\n- [Quick sanity check on images and annotations](#Quick-sanity-check-on-images-and-annotations)\n- [Reference](#Reference)\n","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n'''for dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n'''\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-06-10T02:19:13.046915Z","iopub.execute_input":"2021-06-10T02:19:13.047408Z","iopub.status.idle":"2021-06-10T02:19:13.054797Z","shell.execute_reply.started":"2021-06-10T02:19:13.047365Z","shell.execute_reply":"2021-06-10T02:19:13.053762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install python-gdcm\n!pip install pylibjpeg-openjpeg\n!pip install pylibjpeg","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-06-10T02:19:17.717215Z","iopub.execute_input":"2021-06-10T02:19:17.717572Z","iopub.status.idle":"2021-06-10T02:19:37.609335Z","shell.execute_reply.started":"2021-06-10T02:19:17.717541Z","shell.execute_reply":"2021-06-10T02:19:37.608212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip uninstall numpy -y\n!pip install numpy --no-cache-dir","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-06-10T02:19:37.611182Z","iopub.execute_input":"2021-06-10T02:19:37.611477Z","iopub.status.idle":"2021-06-10T02:19:52.221177Z","shell.execute_reply.started":"2021-06-10T02:19:37.611444Z","shell.execute_reply":"2021-06-10T02:19:52.219949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip uninstall pydicom -y\n!pip install pydicom","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-06-10T02:19:52.223771Z","iopub.execute_input":"2021-06-10T02:19:52.224243Z","iopub.status.idle":"2021-06-10T02:20:03.407174Z","shell.execute_reply.started":"2021-06-10T02:19:52.224188Z","shell.execute_reply":"2021-06-10T02:20:03.405759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nimport numpy as np\nimport pydicom\nimport pydicom.pixel_data_handlers.util as dicom_utils\nimport cv2\nimport gdcm\nimport pylibjpeg\nimport glob","metadata":{"_kg_hide-input":false,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-06-10T02:20:19.757766Z","iopub.execute_input":"2021-06-10T02:20:19.758164Z","iopub.status.idle":"2021-06-10T02:20:19.764328Z","shell.execute_reply.started":"2021-06-10T02:20:19.758120Z","shell.execute_reply":"2021-06-10T02:20:19.763074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROOT_DIR = \"/kaggle/input/siim-covid19-detection/\"\ntrain_image = pd.read_csv(ROOT_DIR+'train_image_level.csv')\ntrain_study = pd.read_csv(ROOT_DIR+'train_study_level.csv')","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-06-10T02:20:22.837772Z","iopub.execute_input":"2021-06-10T02:20:22.838127Z","iopub.status.idle":"2021-06-10T02:20:22.881631Z","shell.execute_reply.started":"2021-06-10T02:20:22.838097Z","shell.execute_reply":"2021-06-10T02:20:22.880526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_image.head()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-10T02:20:25.706811Z","iopub.execute_input":"2021-06-10T02:20:25.707192Z","iopub.status.idle":"2021-06-10T02:20:25.736099Z","shell.execute_reply.started":"2021-06-10T02:20:25.707156Z","shell.execute_reply":"2021-06-10T02:20:25.734958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Let's look at the distribution of number of opacities\n[Back to top](#Table-of-Contents)","metadata":{}},{"cell_type":"code","source":"def get_opacity_count(item):\n    return item.split(' ').count('opacity')","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2021-06-10T02:20:30.056821Z","iopub.execute_input":"2021-06-10T02:20:30.057242Z","iopub.status.idle":"2021-06-10T02:20:30.062024Z","shell.execute_reply.started":"2021-06-10T02:20:30.057196Z","shell.execute_reply":"2021-06-10T02:20:30.060936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_image['opacity_count'] = train_image['label'].apply(get_opacity_count)\ntrain_image.head()","metadata":{"_kg_hide-input":false,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-06-10T02:20:33.116952Z","iopub.execute_input":"2021-06-10T02:20:33.117285Z","iopub.status.idle":"2021-06-10T02:20:33.144933Z","shell.execute_reply.started":"2021-06-10T02:20:33.117257Z","shell.execute_reply":"2021-06-10T02:20:33.143780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_image['opacity_count'].value_counts()\nax = sns.countplot(x = 'opacity_count', data = train_image)","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2021-06-10T02:20:35.806925Z","iopub.execute_input":"2021-06-10T02:20:35.807334Z","iopub.status.idle":"2021-06-10T02:20:35.953657Z","shell.execute_reply.started":"2021-06-10T02:20:35.807298Z","shell.execute_reply":"2021-06-10T02:20:35.952518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are upto 8 opacities observed in an X-Ray.","metadata":{}},{"cell_type":"markdown","source":"# Pneumonia detected even if no opacity is observed\n[Back to top](#Table-of-Contents)","metadata":{}},{"cell_type":"code","source":"train_study.head()","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-06-10T02:20:39.457799Z","iopub.execute_input":"2021-06-10T02:20:39.458197Z","iopub.status.idle":"2021-06-10T02:20:39.470523Z","shell.execute_reply.started":"2021-06-10T02:20:39.458165Z","shell.execute_reply":"2021-06-10T02:20:39.468973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def map_study_observation(item):\n    target_map = {'Negative for Pneumonia':0,'Typical Appearance':1,'Indeterminate Appearance':2,'Atypical Appearance':3}\n    item += '_study'\n    record = train_study.loc[train_study['id']==item, ['Negative for Pneumonia','Typical Appearance','Indeterminate Appearance','Atypical Appearance']]\n    df = (record != 0).any()\n    target = df.index[df]\n    #print(target_map[target[0]])\n    return target_map[target[0]]","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2021-06-10T02:20:43.536913Z","iopub.execute_input":"2021-06-10T02:20:43.537279Z","iopub.status.idle":"2021-06-10T02:20:43.543217Z","shell.execute_reply.started":"2021-06-10T02:20:43.537248Z","shell.execute_reply":"2021-06-10T02:20:43.542100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_image['target'] = train_image['StudyInstanceUID'].apply(map_study_observation)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T02:20:44.277449Z","iopub.execute_input":"2021-06-10T02:20:44.277830Z","iopub.status.idle":"2021-06-10T02:21:02.167916Z","shell.execute_reply.started":"2021-06-10T02:20:44.277794Z","shell.execute_reply":"2021-06-10T02:21:02.166831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_image.head()","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-06-10T02:21:02.169633Z","iopub.execute_input":"2021-06-10T02:21:02.170086Z","iopub.status.idle":"2021-06-10T02:21:02.185470Z","shell.execute_reply.started":"2021-06-10T02:21:02.170039Z","shell.execute_reply":"2021-06-10T02:21:02.184210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_no_opacity_but_has_target = train_image[(train_image['opacity_count'] == 0) & (train_image['target'] != 0)]","metadata":{"execution":{"iopub.status.busy":"2021-06-10T02:21:03.637117Z","iopub.execute_input":"2021-06-10T02:21:03.637485Z","iopub.status.idle":"2021-06-10T02:21:03.644942Z","shell.execute_reply.started":"2021-06-10T02:21:03.637454Z","shell.execute_reply":"2021-06-10T02:21:03.643790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Total images: {len(train_image)}\")\nprint(f\"Total images with no opacities: {sum(train_image['opacity_count']==0)}\")\nprint(f\"Total images with no opacity but identified pneumonia: {len(df_no_opacity_but_has_target)}\")","metadata":{"execution":{"iopub.status.busy":"2021-06-10T02:21:07.116849Z","iopub.execute_input":"2021-06-10T02:21:07.117218Z","iopub.status.idle":"2021-06-10T02:21:07.124587Z","shell.execute_reply.started":"2021-06-10T02:21:07.117189Z","shell.execute_reply":"2021-06-10T02:21:07.123261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see that, 304 out of 2040 were considered pneumonia though no opacities found. Can a pneumonia be diagnosed without spotting an opacity?","metadata":{}},{"cell_type":"markdown","source":"I feel the above records need to be discarded when modelling, as it may induce a data bias. Now let's observe the distribution of opacity count over different target cats. ","metadata":{}},{"cell_type":"markdown","source":"# Opacities vs Pneumonia\n[Back to top](#Table-of-Contents)","metadata":{}},{"cell_type":"code","source":"grouped_opacities = train_image.groupby(['opacity_count','target'])\ngrouped_df2 = grouped_opacities['id'].count()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-10T02:21:11.176744Z","iopub.execute_input":"2021-06-10T02:21:11.177096Z","iopub.status.idle":"2021-06-10T02:21:11.185497Z","shell.execute_reply.started":"2021-06-10T02:21:11.177065Z","shell.execute_reply":"2021-06-10T02:21:11.184749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grouped_df2.unstack()","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-06-10T02:21:12.416835Z","iopub.execute_input":"2021-06-10T02:21:12.417368Z","iopub.status.idle":"2021-06-10T02:21:12.435137Z","shell.execute_reply.started":"2021-06-10T02:21:12.417320Z","shell.execute_reply":"2021-06-10T02:21:12.434200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"legends = ['Negative for Pneumonia','Typical Appearance','Indeterminate Appearance','Atypical Appearance']\nfig = plt.figure(figsize=(10,10))\nplt.suptitle(\"Distribution of opacity count over different targets\")\nfor i in range(7):\n    plt.subplot(3,3,i+1)\n    df = grouped_df2.unstack().iloc[i]\n    sns.barplot(x=df.index, y=df.values)\n    plt.title(f'No. of opacities: {i}')\n    plt.xlabel('target')\n    plt.ylabel('count')\nfig.text(0.5, 0.2, s='**********  Target Mappings  *******\\n Negative for Pneumonia - 0\\n Typical - 1\\n Indeterminate - 2\\n Atypical - 3', fontsize='x-large')\nfig.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2021-06-10T02:21:13.406737Z","iopub.execute_input":"2021-06-10T02:21:13.407222Z","iopub.status.idle":"2021-06-10T02:21:14.521425Z","shell.execute_reply.started":"2021-06-10T02:21:13.407191Z","shell.execute_reply":"2021-06-10T02:21:14.520193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Couple of things to be clarified from the data\n[Back to top](#Table-of-Contents)","metadata":{}},{"cell_type":"markdown","source":"The above distribution raises couple of questions:\n1. Pneumonia is identified even if there are no opacities. Can a pneumonia be diagnosed without spotting a single opacity?\n2. If an opacity is identified, we do not see 'Negative for Pneumonia'. Does that mean opacity is always a sign of pneumonia? Can't there be opacities in the lung due to other factors? Or the experts annotated this dataset consider opacities only due to pneumonia and left the same due to other factors?","metadata":{}},{"cell_type":"markdown","source":"# Quick sanity check on images and annotations\n[Back to top](#Table-of-Contents)","metadata":{}},{"cell_type":"code","source":"def get_bboxes(labels):\n    opacities = labels.strip().split('opacity')\n    opacity_list = []\n    #print(opacities)\n    for opacity in opacities:\n        #print(opacity)\n        if 'none' in opacity:\n            continue\n        if opacity != '':\n            opacity_list.append(opacity.strip().split(' ')[1:])\n    return opacity_list","metadata":{"execution":{"iopub.status.busy":"2021-06-10T02:21:19.986578Z","iopub.execute_input":"2021-06-10T02:21:19.987100Z","iopub.status.idle":"2021-06-10T02:21:19.993708Z","shell.execute_reply.started":"2021-06-10T02:21:19.987067Z","shell.execute_reply":"2021-06-10T02:21:19.992852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\ntarget_map = ['Negative for Pneumonia','Typical Appearance','Indeterminate Appearance','Atypical Appearance']\nfig = plt.figure(figsize=(50,50))\nfor i in range(9):\n    random_record = train_image.sample()\n    #print(type(random_record.index[0]))\n    study_id = random_record['StudyInstanceUID'].values[0]\n    image_id = random_record['id'].values[0]\n    labels = random_record['label'].values[0]\n    target = random_record['target'].values[0]\n    bboxes = get_bboxes(labels)\n    #print(image_id.values[0])\n    img = str(image_id).split('_image')[0]+'.dcm'\n    file_ = glob.glob(ROOT_DIR+'train/'+study_id+'/*/'+img)\n    dicom = pydicom.dcmread(file_[0])\n    img = dicom.pixel_array\n    if 'PhotometricInterpretation' in dicom:\n        if dicom.PhotometricInterpretation == 'MONOCHROME1':\n            print(\"It's a monochrome1 image\")\n            img = np.amax(img) - img\n    img = dicom_utils.apply_modality_lut(img, dicom)\n    img = dicom_utils.apply_voi_lut(img, dicom)\n    #print(img.max())\n    img = np.stack([img, img, img])\n    img = img.astype('float32')\n    img = img - img.min()\n    img = img / img.max()\n    #img = img * 255\n    img = img.transpose(1,2,0)\n    #print(img.shape)\n    #img = img / 255\n    plt.subplot(9,1,i+1)\n    #plt.subplot(3,3,i+1)\n    for bbox in bboxes:\n        #img = img.copy()\n        xmin = int(float(bbox[0]))\n        ymin = int(float(bbox[1]))\n        xmax = int(float(bbox[2]))\n        ymax = int(float(bbox[3]))\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        img = cv2.rectangle(img, (xmin, ymin), (xmax, ymax), (255,0,0), 5)\n    plt.imshow(img)\n    plt.axis('off')\n    plt.title(target_map[target])\nplt.suptitle(\"Random lookup of images and its annotations\")\nfig.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2021-06-10T02:21:22.967219Z","iopub.execute_input":"2021-06-10T02:21:22.967919Z","iopub.status.idle":"2021-06-10T02:21:48.009682Z","shell.execute_reply.started":"2021-06-10T02:21:22.967866Z","shell.execute_reply":"2021-06-10T02:21:48.008223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Reference\n\nHere's a very good resource to learn more about lung opacities and pneumonia - https://www.kaggle.com/zahaviguy/what-are-lung-opacities\n\n[Back to top](#Table-of-Contents)","metadata":{}}]}