{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\ncurrent_path='/kaggle/input/siim-covid19-detection'\nprint (os.listdir(current_path))\nprint (1)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!wget 'https://anaconda.org/conda-forge/libjpeg-turbo/2.1.0/download/linux-64/libjpeg-turbo-2.1.0-h7f98852_0.tar.bz2' -q\n!wget 'https://anaconda.org/conda-forge/libgcc-ng/9.3.0/download/linux-64/libgcc-ng-9.3.0-h2828fa1_19.tar.bz2' -q\n!wget 'https://anaconda.org/conda-forge/gdcm/2.8.9/download/linux-64/gdcm-2.8.9-py37h500ead1_1.tar.bz2' -q\n!wget 'https://anaconda.org/conda-forge/conda/4.10.1/download/linux-64/conda-4.10.1-py37h89c1867_0.tar.bz2' -q\n!wget 'https://anaconda.org/conda-forge/certifi/2020.12.5/download/linux-64/certifi-2020.12.5-py37h89c1867_1.tar.bz2' -q\n!wget 'https://anaconda.org/conda-forge/openssl/1.1.1k/download/linux-64/openssl-1.1.1k-h7f98852_0.tar.bz2' -q\n!conda install 'libjpeg-turbo-2.1.0-h7f98852_0.tar.bz2' -c conda-forge -y\n!conda install 'libgcc-ng-9.3.0-h2828fa1_19.tar.bz2' -c conda-forge -y\n!conda install 'gdcm-2.8.9-py37h500ead1_1.tar.bz2' -c conda-forge -y\n!conda install 'conda-4.10.1-py37h89c1867_0.tar.bz2' -c conda-forge -y\n!conda install 'certifi-2020.12.5-py37h89c1867_1.tar.bz2' -c conda-forge -y\n!conda install 'openssl-1.1.1k-h7f98852_0.tar.bz2' -c conda-forge -y","metadata":{"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n<b><a href='#1'>1:  Reading Files</a></b>\n<br>\n<b><a href='#2'>2:  Adding Images Path to DataFrame</a></b><br>\n<b><a href='#3'>3:  Combining Image Level df and Study Level df</a></b><br>\n<b><a href='#4'>4:  Adding Data from Image File</a></b><br>\n<b><a href='#5'>5:  What exactly are we PREDICTING?</a></b><br>\n<b><a href='#6'>6:  Distribution of Gender</a></b><br>\n<b><a href='#7'>7:  Xray Categories</a></b><br>\n<b> &nbsp;&nbsp;  <a href='#7.1'>7.1: Chest</a></b><br>\n<b> &nbsp;&nbsp;  <a href='#7.2'>7.2: Thorax</a></b><br>\n<b> &nbsp;&nbsp;  <a href='#7.3'>7.3: Empty Label</a></b><br>\n<b> &nbsp;&nbsp;  <a href='#7.4'>7.4: Skull</a></b><br>\n<b> &nbsp;&nbsp;  <a href='#7.5'>7.5: Port Chest</a></b><br>\n<br>\n<b><a href='#8'>8:  Distribution Of Labels</a></b><br>","metadata":{}},{"cell_type":"markdown","source":"<a id='1'></a>\n#### Reading Files\n","metadata":{}},{"cell_type":"code","source":"import pandas as pd\ntrain_image_level=pd.read_csv(os.path.join(current_path, 'train_image_level.csv'))\ntrain_study_level=pd.read_csv(os.path.join(current_path,'train_study_level.csv'))\nsample_submission=pd.read_csv(os.path.join(current_path,'sample_submission.csv'))\ntrain_image_level.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_study_level.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Note that all images are stored in paths with the form study/series/image. The study ID here relates directly to the study-level predictions, and the image ID is the ID used for image-level predictions.","metadata":{}},{"cell_type":"markdown","source":"<a id='2'></a>\n#### Adding Images paths to dataframe\nNow we will add the image path to the dataframe so we have the complete dataset","metadata":{}},{"cell_type":"code","source":"all_files_train=[]\nall_files_test=[]\nall_files_train=[os.path.join(dirname,filename) for dirname,_,filenames in os.walk(os.path.join(current_path,'train')) for filename in filenames]\nall_files_test=[os.path.join(dirname,filename) for dirname,_,filenames in os.walk(os.path.join(current_path,'test')) for filename in filenames]\n\nall_files_train_dict={x.split('/')[-1].replace('.dcm','_image'): x for x in all_files_train}\nall_files_test_dict={x.split('/')[-1].replace('.dcm','_image'): x for x in all_files_test}\nprint (len(all_files_train)+len(all_files_test))\nprint (\"All files present\")\n\ntrain_image_level['path']=train_image_level.id.map(all_files_train_dict)\ntrain_image_level['id']=train_image_level['id'].apply(lambda x: x.replace('_image',''))\ntrain_image_level['simplified_path']=train_image_level['path'].apply(lambda x: '/'.join(x.split('/')[5:]))\ntrain_study_level=train_study_level.rename(columns={'id':'StudyInstanceUID'}, inplace=False)\n\ntrain_study_level['StudyInstanceUID']=train_study_level['StudyInstanceUID'].apply(lambda x: x.replace('_study',''))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We have now added image path to the train_image_level dataframe. ","metadata":{}},{"cell_type":"code","source":"train_image_level.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='3'></a>\n#### Merging Study level Dataset and Image level dataset\nWe will now merge it with train_study level so that we have one complete dataset to analyse\n\n* PLEASE NOTE that each study can have MULTIPLE IMAGES\n* Because of this reason we are joining the train_image_level dataframe which have all the images id with the train_study_level which have all the study id.\n* Because two images can have ONE studyID hence in complete_data dataframe we might see duplicate StudyId.","metadata":{}},{"cell_type":"code","source":"complete_data=train_image_level.merge(train_study_level, how='inner', on='StudyInstanceUID')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"complete_data.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Reordering columns for better view","metadata":{}},{"cell_type":"code","source":"columns_reordered=['id',\n 'StudyInstanceUID',\n 'boxes',\n 'label',\n 'Negative for Pneumonia',\n 'Typical Appearance',\n 'Indeterminate Appearance',\n 'Atypical Appearance',\n 'path',\n 'simplified_path']\ncomplete_data=complete_data[columns_reordered]\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='4'></a>\n#### Adding Data from image file (.dcm)\nAdding data from image file as well","metadata":{}},{"cell_type":"code","source":"import pydicom\nfrom pydicom import dcmread\ndef process_dicom(dicom_obj):\n    pixel_data=(0x7fe0, 0x0010) #ignore the pixel data\n    data_dict={}\n    for x in dicom_obj:\n        if x.tag==pixel_data:\n            continue\n        value=dicom_obj[x.tag].value\n        name=x.name\n        data_dict[name]=value\n    return data_dict","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\ndicom_dict={}\nneeded_columns=[\"Patient ID\",\"Patient's Sex\",\"Body Part Examined\",\"Imager Pixel Spacing\",\n                \"Photometric Interpretation\"]\n# \"Study Instance UID\",\"Study ID\"\n\nfor i,x in tqdm(complete_data.iterrows()):\n    dicom_obj=pydicom.dcmread(x['path'],stop_before_pixels=True)\n    dicom_obj_dict=process_dicom(dicom_obj)\n    for key in dicom_obj_dict:\n        if type(dicom_obj_dict[key])==list:\n            continue\n        if key in needed_columns:\n            if key not in dicom_dict:\n                dicom_dict[key]=[]\n            dicom_dict[key].append(dicom_obj_dict[key])\n            \nfor col in needed_columns:\n    complete_data[col]=dicom_dict[col]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='5'></a>\n#### What exactly are we predicting?\n#### Its important here to distinguish BETWEEN study and image data.\n* for example there are 6054 studies hence each study can be on more than one image as well. But there are total 6334 images. There can be study on more than ONE image as well. \n\n* For prediction we need to make TWO prediction\n    * For first one on train_image_level\n        * If there is an object on the image we will predict the confidence in our prediction and the corresponding bounding box \n        * for example: \"opacity 1 1543 341 2484 1002 opacity 0.9 222 22 22 22\"\n        * this means that there are two objects in the image we are predicting and first object we are predicting with 100% confidence and other one we are predicting with 90% confidence. \n        * The four numbers after the confidence score are the coordinates of the bounding box corresponding to xmin, ymin, xmax,ymax. \n        * In general our prediction will be like this \"opacity confidence_score xmin ymin xmax ymax\" OR if there is no object then our prediction should be like this \"ImageId none 1 0 0 1 1\"\n    * For second one train_study_level\n        * In this one OUR model have to predict one of the following labels \"negative\", \"typical\", \"indeterminate\", \"atypical\"\n        * The prediction have to be in the format \"StudyID prediction confidencescore 0 0 1 1\"\n        * For Example if our prediction is negative with a confidence score of 60% then our prediction sould be like as follows\n        * \"StudyID negative 0.6 0 0 1 1\"\n    * Please note that with train_study_level prediction we will add StudyID and for the train_image_level we will add Image to distinguish our predictions\n        \n\n\n\nSeems like we have a good enough distribution of MALE and FEMALE","metadata":{}},{"cell_type":"markdown","source":"<a id='6'></a>\n#### Distribution of MALE AND FEMALE chest XRAY images","metadata":{}},{"cell_type":"code","source":"import seaborn as sns\nsns.countplot(x=\"Patient's Sex\",data=complete_data)\nprint (complete_data[\"Patient's Sex\"].value_counts())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"On Body Part Examined We need to do some bit of Data Cleaning.\n\n### I just found out that PECHO is chest in spanish. :D","metadata":{}},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pylab as plt\nplt.xticks(rotation=45)\nsns.countplot(x=\"Body Part Examined\",data=complete_data)\nprint (complete_data[\"Body Part Examined\"].value_counts())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='7'></a>\n#### Lets View some XRAY photos with different categories, Head, Thorax, Skull, etc","metadata":{}},{"cell_type":"code","source":"def replace_with_correct(x):\n    x=x.strip()\n    \n    if x.find('PECHO')!=-1 or x.lower().find('pecho')!=-1:\n        # PECHO is chest in spanish ;P\n        return 'chest'\n    if x=='TORAX' or x=='TÒRAX' or x=='2- TORAX' or x=='T?RAX':\n        return 'THORAX'.lower()\n    elif x=='':\n        return \"EMPTY\".lower()\n    else:\n        return x.lower()\n    \ncomplete_data['Body Part Examined']=complete_data['Body Part Examined'].apply(replace_with_correct)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### We see that most of the data is of the chest but there are skull xrays, thorax xrays as well.\n### Lets try to view some of the sample images of each part","metadata":{}},{"cell_type":"code","source":"plt.clf()\nplt.xticks(rotation=45)\nsns.countplot(x=\"Body Part Examined\",data=complete_data)\nprint (complete_data[\"Body Part Examined\"].value_counts())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_figures(figures, nrows = 1, ncols=1):\n    fig, axeslist = plt.subplots(ncols=ncols, nrows=nrows)\n    for ind,title in enumerate(figures):\n        axeslist.ravel()[ind].imshow(figures[title], cmap=plt.gray())\n        axeslist.ravel()[ind].set_title(title)\n        axeslist.ravel()[ind].set_axis_off()\n    plt.tight_layout() # optional\n\nall_cats=list(complete_data[\"Body Part Examined\"].value_counts().index)\nprint (all_cats)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='7.1'></a>\n#### SAMPLE IMAGES OF CHEST","metadata":{}},{"cell_type":"code","source":"complete_data_chest=complete_data[complete_data['Body Part Examined']=='chest']\ncomplete_data_chest=complete_data_chest.sample(8)\nfigures={x['id']:dcmread(x['path']).pixel_array for i,x in complete_data_chest.iterrows()}\nplot_figures(figures,2,4)\n","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='7.2'></a>\n#### SAMPLE IMAGES OF THORAX","metadata":{}},{"cell_type":"code","source":"complete_data_thorax=complete_data[complete_data['Body Part Examined']=='thorax'].sample(8)\nfigures={x['id']:dcmread(x['path']).pixel_array for i,x in complete_data_thorax.iterrows()}\nplot_figures(figures,2,4)\n","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='7.3'></a>\n#### SAMPLE IMAGES OF (EMPTY) LABEL","metadata":{}},{"cell_type":"code","source":"complete_data_empty=complete_data[complete_data['Body Part Examined']=='empty'].sample(8)\nfigures={x['id']:dcmread(x['path']).pixel_array for i,x in complete_data_empty.iterrows()}\nplot_figures(figures,2,4)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='7.4'></a>\n#### SAMPLE IMAGES OF SKULL","metadata":{}},{"cell_type":"code","source":"complete_data_skull=complete_data[complete_data['Body Part Examined']=='skull'].sample(8)\nfigures={x['id']:dcmread(x['path']).pixel_array for i,x in complete_data_skull.iterrows()}\nplot_figures(figures,2,4)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='7.5'></a>\n#### SAMPLE IMAGES OF PORT CHEST","metadata":{}},{"cell_type":"code","source":"complete_data_portchest=complete_data[complete_data['Body Part Examined']=='port chest'].sample(8)\nfigures={x['id']:dcmread(x['path']).pixel_array for i,x in complete_data_portchest.iterrows()}\nplot_figures(figures,2,4)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### We see that irrespective of the category the images are of chest, Hence we can deduce that Body Part Examined field is useless as it doesnt categorises it correctly. ","metadata":{}},{"cell_type":"markdown","source":"\n<a id='8'></a>\n#### Lets analyse the classes in Train Study Level\n#### Distribution of Labels","metadata":{}},{"cell_type":"code","source":"columns=list(train_study_level.columns)[1:]\ntrain_study_bar=train_study_level[columns]\ncounts = train_study_bar.apply(lambda x: x.value_counts()).transpose()# / len(x)).transpose()\nfig = plt.figure()\nax = fig.add_subplot(111)\ncounts.plot(ax=ax,kind='bar', stacked=True, rot=0)\nvals = ax.get_yticks()\nplt.xticks(rotation=45)\nax.set_yticklabels(['{:3.2f}'.format(x) for x in vals])\nax.yaxis.grid(True)\nax.set_axisbelow(True)\nplt.title(\"Distribution of Labels\")\nplt.show()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}