{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\n# import numpy as np # linear algebra\n# import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load libraries\nfrom pathlib import Path\nimport pandas as pd\nimport numpy as np\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nimport ast\nimport copy\nfrom collections import Counter","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"HOME = Path('/kaggle/input/siim-covid19-detection/')\nSTUDY_ANNOTATION = Path('train_study_level.csv')\nIMG_ANNOTATION = Path('train_image_level.csv')\n\n\n\n# /kaggle/input/siim-covid19-detection/test/2fb11712bc93/b056067b8455/a29c5a68b07b.dcm\n# /kaggle/input/siim-covid19-detection/test/19c66935e737/41b28d4fa278/9850b5470fd6.dcm","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We start from the general, with inspecting the <b>Study level annotations</b>","metadata":{}},{"cell_type":"code","source":"study_annotation = pd.read_csv(HOME/STUDY_ANNOTATION)\nstudy_annotation.head(3)","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"assert len(study_annotation[study_annotation.duplicated() == True]) == 0, \"There are duplicated rows.\" ","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_counts = study_annotation.iloc[:,1:].sum(axis=0)\n\nfig, ax = plt.subplots( figsize = (10,5))\nax = sns.barplot(x = label_counts.values, y = label_counts.index)\nax.set_title('Label frequency at study level')\nax.set_xlabel('Number of studies')\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Extract all study ids from the study_annotation file and store them in a new column\nstudy_ids_study_annotation = [study_id.split('_')[0] for study_id in study_annotation.id]\nstudy_annotation['StudyInstanceUID'] = study_ids_study_annotation","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Transform the dataframe for further usage\nstudy_annotation_tr = copy.deepcopy(study_annotation.iloc[:,1:])\n\nstudy_annotation_tr = study_annotation_tr.mask(study_annotation_tr.eq(1), study_annotation_tr.columns.to_series(), axis=1)\nstudy_annotation_tr = study_annotation_tr.replace(0,'')\n\n\nstudy_annotation_tr['Diagnosis'] = (study_annotation_tr['Negative for Pneumonia'] +\n                                    study_annotation_tr['Typical Appearance'] + \n                                    study_annotation_tr['Indeterminate Appearance'] + \n                                    study_annotation_tr['Atypical Appearance'])\n\n# Rename the labels as in the submission file.\nstudy_annotation_tr.replace(\"Negative for Pneumonia\", \"negative 1 0 0 1 1\", inplace=True)\nstudy_annotation_tr.replace(\"Typical Appearance\", \"typical 1 0 0 1 1\", inplace=True)\nstudy_annotation_tr.replace(\"Atypical Appearance\", \"atypical 1 0 0 1 1\", inplace=True)\nstudy_annotation_tr.replace(\"Indeterminate Appearance\", \"indeterminate 1 0 0 1 1\", inplace=True)\n\nstudy_annotation_tr = study_annotation_tr[['StudyInstanceUID', 'Diagnosis']]\nstudy_annotation_tr.head(3)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"And continue with the particular: <b>image level annotations</b>.","metadata":{}},{"cell_type":"code","source":"# Read the annotation file\nimg_annotation = pd.read_csv(HOME/IMG_ANNOTATION)\nimg_annotation.head(3)","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# How many images does each study have? \n\nimg_per_study = img_annotation[['id', 'StudyInstanceUID']].groupby('StudyInstanceUID').count().reset_index()\n\nstudies_with_n_imgs = img_per_study.groupby('id').count().reset_index()\nstudies_with_n_imgs.columns = ['nr_images_per_study', 'occurences']\nstudies_with_n_imgs","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The number of images in one study ranges between 1 and 9. The majority of patients (5822 studies) have one annotated image, but there are also some studies with more than 3 images. Those are:","metadata":{}},{"cell_type":"code","source":"over3_imgs = img_per_study[img_per_study.id > 3]\nover3_imgs","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's see if there is a connection between the opacity features and the patient's (study) diagnosis. For this we merge the two dataframes and perform some operations (calculate the number of opacities in each image and the total estimated area of the opacities in every image).","metadata":{}},{"cell_type":"code","source":"# Merge the image and study annotations for further investigations\nprint(len(img_annotation))\nprint(len(study_annotation_tr))\nmerged_df = pd.merge(img_annotation, study_annotation_tr, on = 'StudyInstanceUID')\nmerged_df.head(3)","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"col_nr_opacities = []\ncol_opacity_area = []\n\nfor image in merged_df.boxes:\n    if not pd.isnull(image):\n        box_list = ast.literal_eval(image)\n        nr_opacities = len(box_list)\n        \n        opacity_area = 0\n        for box in box_list:\n            coord = box\n            area = coord['width'] * coord['height']\n            opacity_area = opacity_area + area\n        \n    else:\n        nr_opacities = 0\n        opacity_area = 0\n        \n    col_nr_opacities.append(nr_opacities)\n    col_opacity_area.append(opacity_area)\n    \n\nmerged_df['opacity_area'] = col_opacity_area\nmerged_df['nr_opacities'] = col_nr_opacities\nmerged_df.head(3)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The majority of the 'Typical appearance' cases have 2 opacities. \nThere is no indication of a clear difference between the 'Atypical' and 'Indeterminate' cases based on the number of opacities.\n<b>One observation</b> that we will investigate further refers to the cases with <b>NO OPACITIES</b> and a diagnosis other than <b>Negative for Pneumonia</b>. Please see the <b>Observation 1</b> snippet.","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 2, figsize = (15, 7))\n\nsns.histplot(ax = ax[1],\n            x = merged_df['nr_opacities'], \n            hue = merged_df['Diagnosis'], \n            multiple = 'dodge')\n\nsns.boxplot(ax = ax[0],\n            x = merged_df['nr_opacities'], \n            y = merged_df['Diagnosis'])\n\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Relationship between the type of diagnosis and the total surface of the opacities\n\nIn the case of No Pneumonia, there is no opacity identified on the lungs. \nThe Typical Appearance cases differ from the other groups, exhibiting more surface with opacities.\nThe Atypical and Indeterminate appearance are similar.","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 2, figsize = (15, 7))\n\nsns.histplot(ax = ax[1],\n            x = merged_df['opacity_area'], \n            hue = merged_df['Diagnosis'], \n            multiple = 'dodge')\n\nsns.boxplot(ax = ax[0],\n            x = merged_df['opacity_area'], \n            y = merged_df['Diagnosis'])\n\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From the boxplot below we can see that having one opacity implies less affected surface of the lungs. The majority of the cases\nwith one opacity are 'Indeterminate appearance'. \nOne cannot draw a clear line between the cases with 2, 3 or 4 opacities, as there is no relationship between how many opacities there are\nand the affected area.","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(nrows = 1, ncols = 2, figsize = (15, 7))\nsns.scatterplot(ax = ax[0],\n                x = merged_df['opacity_area'], \n                y = merged_df['nr_opacities'],\n                hue = merged_df['Diagnosis'])\n\nsns.boxplot(ax = ax[1],\n            x = merged_df['nr_opacities'], \n            y = merged_df['opacity_area'],\n            orient = 'v')\n\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merged_df[['Diagnosis', 'opacity_area']].groupby('Diagnosis').median()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# merged_df[merged_df.StudyInstanceUID == 'bf35d36cf961']\nstudy_annotation[study_annotation.StudyInstanceUID == '4189ab7c6027']\nimg_annotation[img_annotation.StudyInstanceUID ==  '4189ab7c6027']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Observation 1\n\nHere, we look into those images which have no opacities, but have a diagnosis other than \"Negative for Pneumonia\". \nOne source for these images is, naturally, the fact that a study with multiple images, has opacities on just one image. The diagnosis is placed based on that single image, so all other images will be tagged with the same diagnostic, even if they have no opacities.\n\n<b>Conclusion for postprocessing:</b> if at least one image of a study has opacities, the study will be diagnoses with a label other than \"Negative for Pneumonia\". \n\nBut unfortunately, this still leaves us with unsolved 82 study ids (patients) which:\n- have only one image to assess, \n- that images has 0 opacities\n- that study has a label different from \"Negative for Pneumonia\" (81 cases with \"atypical\" and 1 with \"typical\")","metadata":{}},{"cell_type":"code","source":"st_id_pos_0opac = merged_df[(merged_df.Diagnosis != \"Negative for Pneumonia\") & (merged_df.nr_opacities == 0)].StudyInstanceUID.values","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"diag_ =[]\nstid_ = []\nstd_id_multiple_images = []\n\nfor i in set(st_id_pos_0opac):\n    \n    study_images = merged_df[merged_df.StudyInstanceUID == i]\n    \n    if len(study_images) < 2:\n        diag_.append(study_images.Diagnosis.values[0])\n        stid_.append(study_images.StudyInstanceUID.values[0])\n    if len(study_images) > 1:\n        std_id_multiple_images.append(study_images.StudyInstanceUID.values[0])","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"number of unadressed patients: {}\".format(len(set(stid_))))\nprint(Counter(diag_))","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#stid_","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualize images\n\n1. Studies with \"Atypical appearance\" label which have no bounding boxes localizing opacities\n2. Studies with \"Negative for Pneumonia\"\n3. Studies with \"Typical Appearance\"","metadata":{}},{"cell_type":"code","source":"import os\nimport glob\nimport pydicom\nimport random","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN = '/kaggle/input/siim-covid19-detection/train/'\nCOLS = 4\nROWS = 2\n\nfig, ax = plt.subplots(ncols = COLS, nrows = ROWS, figsize = (25, 10))\nplt.suptitle(\"Studies with 'Atypical appearance' without opacity bounding boxes\")\n\nstudies = random.sample(stid_, COLS * ROWS)\n\nc = 0\nr = 0\n\nfor study in studies:\n    dcm_file = pydicom.dcmread(glob.glob(os.path.join(TRAIN, study, \"*/*.dcm\"))[0])\n    dcm_array = dcm_file.pixel_array\n    \n    ax[r, c].imshow(dcm_array, cmap= 'gray')\n    ax[r, c].set_title(study)\n    \n    \n    c = c + 1\n    if (c) % COLS == 0:\n        c = 0\n        r = r + 1\n      ","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stid_negative_for_pneumonia = list(set(merged_df[merged_df.Diagnosis == \"Negative for Pneumonia\"].StudyInstanceUID))\n\nTRAIN = '/kaggle/input/siim-covid19-detection/train/'\nCOLS = 4\nROWS = 2\n\nfig, ax = plt.subplots(ncols = COLS, nrows = ROWS, figsize = (25, 10))\nplt.suptitle(\"Studies with 'Negative for Pneumonia' without opacity bounding boxes\")\n\nstudies = random.sample(stid_negative_for_pneumonia, COLS * ROWS)\n\nc = 0\nr = 0\n\nfor study in studies:\n    dcm_file = pydicom.dcmread(glob.glob(os.path.join(TRAIN, study, \"*/*.dcm\"))[0])\n    dcm_array = dcm_file.pixel_array\n    \n    ax[r, c].imshow(dcm_array, cmap = 'gray')\n    ax[r, c].set_title(study)\n    \n    \n    c = c + 1\n    if (c) % COLS == 0:\n        c = 0\n        r = r + 1","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stid_negative_for_pneumonia = list(set(merged_df[merged_df.Diagnosis == \"Typical Appearance\"].StudyInstanceUID))\n\nTRAIN = '/kaggle/input/siim-covid19-detection/train/'\nCOLS = 4\nROWS = 2\n\nfig, ax = plt.subplots(ncols = COLS, nrows = ROWS, figsize = (25, 10))\nplt.suptitle(\"Studies with 'Typical Appearance' without opacity bounding boxes\")\n\nstudies = random.sample(stid_negative_for_pneumonia, COLS * ROWS)\n\nc = 0\nr = 0\n\nfor study in studies:\n    dcm_file = pydicom.dcmread(glob.glob(os.path.join(TRAIN, study, \"*/*.dcm\"))[0])\n    dcm_array = dcm_file.pixel_array\n    \n    ax[r, c].imshow(dcm_array, cmap = 'gray')\n    ax[r, c].set_title(study)\n    \n    \n    c = c + 1\n    if (c) % COLS == 0:\n        c = 0\n        r = r + 1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stid_negative_for_pneumonia = list(set(merged_df[merged_df.Diagnosis == \"Indeterminate Appearance\"].StudyInstanceUID))\n\nTRAIN = '/kaggle/input/siim-covid19-detection/train/'\nCOLS = 4\nROWS = 2\n\nfig, ax = plt.subplots(ncols = COLS, nrows = ROWS, figsize = (25, 10))\nplt.suptitle(\"Studies with 'Indeterminate Appearance' without opacity bounding boxes\")\n\nstudies = random.sample(stid_negative_for_pneumonia, COLS * ROWS)\n\nc = 0\nr = 0\n\nfor study in studies:\n    dcm_file = pydicom.dcmread(glob.glob(os.path.join(TRAIN, study, \"*/*.dcm\"))[0])\n    dcm_array = dcm_file.pixel_array\n    \n    ax[r, c].imshow(dcm_array, cmap = 'gray')\n    ax[r, c].set_title(study)\n    \n    \n    c = c + 1\n    if (c) % COLS == 0:\n        c = 0\n        r = r + 1","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}