{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nimport cv2\nfrom PIL import Image\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-13T07:26:39.105530Z","iopub.execute_input":"2021-06-13T07:26:39.105909Z","iopub.status.idle":"2021-06-13T07:26:39.110064Z","shell.execute_reply.started":"2021-06-13T07:26:39.105878Z","shell.execute_reply":"2021-06-13T07:26:39.109273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Version 3- Added Anamoly ids  and Area computation for Bounding boxes","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport pydicom\nimport pydicom.data  # we need pydicom to read all the dicom files some thing new for beginners","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:26:39.115506Z","iopub.execute_input":"2021-06-13T07:26:39.116040Z","iopub.status.idle":"2021-06-13T07:26:39.127281Z","shell.execute_reply.started":"2021-06-13T07:26:39.115998Z","shell.execute_reply":"2021-06-13T07:26:39.126370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base='../input/siim-covid19-detection/train/'\n#../input/siim-covid19-detection/train/00086460a852/9e8302230c91/65761e66de9f.dcm","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:26:39.128466Z","iopub.execute_input":"2021-06-13T07:26:39.128873Z","iopub.status.idle":"2021-06-13T07:26:39.141338Z","shell.execute_reply.started":"2021-06-13T07:26:39.128842Z","shell.execute_reply":"2021-06-13T07:26:39.140351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file='65761e66de9f.dcm'\n#filename = pydicom.data.data_manager.get_files(base,  '00')[0]\n#print(file_name)  \nds = pydicom.dcmread('../input/siim-covid19-detection/train/00086460a852/9e8302230c91/65761e66de9f.dcm')\n  \nplt.imshow(ds.pixel_array, cmap=plt.cm.bone)  # set the color map to bone\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:26:39.152277Z","iopub.execute_input":"2021-06-13T07:26:39.153119Z","iopub.status.idle":"2021-06-13T07:26:39.980419Z","shell.execute_reply.started":"2021-06-13T07:26:39.153051Z","shell.execute_reply":"2021-06-13T07:26:39.979320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import fastai\nimport fastai.medical\nfrom fastai.basics import *\nfrom fastai.vision.all import *\nfrom fastai.medical.imaging import *\n\nimport pydicom\n\nimport pandas as pd","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:26:39.981900Z","iopub.execute_input":"2021-06-13T07:26:39.982240Z","iopub.status.idle":"2021-06-13T07:26:39.988328Z","shell.execute_reply.started":"2021-06-13T07:26:39.982207Z","shell.execute_reply":"2021-06-13T07:26:39.987206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\n# Lets check the meta data for pydicom\n#items=get_dicom_files(base)\n#print(items)\n#df_trn = pd.DataFrame.from_dicoms(items,px_summ=False)\n'''\ncol_list=[ 'StudyID','PatientID','ImageType', 'StudyInstanceUID', 'SeriesInstanceUID',\n        'SOPClassUID', 'SOPInstanceUID', 'PhotometricInterpretation', 'Rows', 'Columns', 'BitsAllocated',\n       'BitsStored', 'HighBit', 'PixelRepresentation', 'fname','BodyPartExamined',]\n\ndf_trn[[ 'StudyID','PatientID','ImageType', 'StudyInstanceUID', 'SeriesInstanceUID',\n        'SOPClassUID', 'SOPInstanceUID', 'PhotometricInterpretation', 'Rows', 'Columns', 'BitsAllocated',\n       'BitsStored', 'HighBit', 'PixelRepresentation', 'fname','BodyPartExamined',]]\n'''","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:26:39.990394Z","iopub.execute_input":"2021-06-13T07:26:39.990723Z","iopub.status.idle":"2021-06-13T07:26:40.005472Z","shell.execute_reply.started":"2021-06-13T07:26:39.990691Z","shell.execute_reply":"2021-06-13T07:26:40.004269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df_trn.to_csv('df_trn_dicom.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:26:40.008310Z","iopub.execute_input":"2021-06-13T07:26:40.008627Z","iopub.status.idle":"2021-06-13T07:26:40.021308Z","shell.execute_reply.started":"2021-06-13T07:26:40.008597Z","shell.execute_reply":"2021-06-13T07:26:40.020188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df_trn.values[10]","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:26:40.022821Z","iopub.execute_input":"2021-06-13T07:26:40.023433Z","iopub.status.idle":"2021-06-13T07:26:40.031917Z","shell.execute_reply.started":"2021-06-13T07:26:40.023399Z","shell.execute_reply":"2021-06-13T07:26:40.030959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df_trn.StudyID.nunique()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:26:40.033252Z","iopub.execute_input":"2021-06-13T07:26:40.033592Z","iopub.status.idle":"2021-06-13T07:26:40.044152Z","shell.execute_reply.started":"2021-06-13T07:26:40.033559Z","shell.execute_reply":"2021-06-13T07:26:40.043108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Mapping directory structure to Ids in Dicom\n#df_trn[df_trn.SeriesInstanceUID=='9e8302230c91'][col_list] #series instance uid is sub dir,studyinstanceuid is upper directory,SOPinstanceUID is image id","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:26:40.045689Z","iopub.execute_input":"2021-06-13T07:26:40.046029Z","iopub.status.idle":"2021-06-13T07:26:40.054946Z","shell.execute_reply.started":"2021-06-13T07:26:40.045999Z","shell.execute_reply":"2021-06-13T07:26:40.054041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#items[0]","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:26:40.056434Z","iopub.execute_input":"2021-06-13T07:26:40.056739Z","iopub.status.idle":"2021-06-13T07:26:40.066853Z","shell.execute_reply.started":"2021-06-13T07:26:40.056711Z","shell.execute_reply":"2021-06-13T07:26:40.065894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\ntrain_image=pd.read_csv('../input/siim-covid19-detection/train_image_level.csv')\ntrain_image[train_image.StudyInstanceUID=='00292f8c37bd'].label.values #Non Pneumonia label look like this","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:26:40.069429Z","iopub.execute_input":"2021-06-13T07:26:40.069881Z","iopub.status.idle":"2021-06-13T07:26:40.106276Z","shell.execute_reply.started":"2021-06-13T07:26:40.069847Z","shell.execute_reply":"2021-06-13T07:26:40.105512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"*Every Covid 19 is considered Pneuomonia  but every pneumonia is not a Covid\nSo if Pneumonia is 0  it means there cannot be covid . If Pneuomonia  is 1 it means there can be various kinds of Covid 19 Pneumonia patterns such as Typical /Atypical or Indertiminate *\n[https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7297525/] )","metadata":{}},{"cell_type":"code","source":"!cp ../input/basic-exploration-eda-duplicate-nonduplicate/df_trn_dicom.csv /kaggle/working/","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:26:40.107796Z","iopub.execute_input":"2021-06-13T07:26:40.108206Z","iopub.status.idle":"2021-06-13T07:26:40.859762Z","shell.execute_reply.started":"2021-06-13T07:26:40.108176Z","shell.execute_reply":"2021-06-13T07:26:40.858452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# All values are 0 if its  Not a penuomonia\ntrain_study=pd.read_csv('../input/siim-covid19-detection/train_study_level.csv')\ntrain_study['StudyInstanceUID']=train_study.id.apply(lambda x: x.split('_')[0])\n\n\ntrain_study[train_study[\"Negative for Pneumonia\"]==1].sum(0)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:26:40.861657Z","iopub.execute_input":"2021-06-13T07:26:40.862099Z","iopub.status.idle":"2021-06-13T07:26:40.896312Z","shell.execute_reply.started":"2021-06-13T07:26:40.862051Z","shell.execute_reply":"2021-06-13T07:26:40.895380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"''' \nOpacityCount = train_image['label'].str.count('opacity')\n#np.where(OpacityCount==0)[0].shape\n#train_image[train_image.boxes.isnull()]#.StudyInstanceUID.nunique()\n#set(train_image[train_image.boxes.isnull()].StudyInstanceUID.unique())-set(count_df_train_img[count_df_train_img.boxescnt==0].StudyInstanceUID.values)\ntrain_image['opacity_count']=OpacityCount\nnp.where(OpacityCount==0)[0].shape,train_image[train_image.boxes.isnull()].StudyInstanceUID.unique().shape\n train_image[(train_image['opacity_count']==0) ].StudyInstanceUID.nunique() \ntrain_image['anamoly_flag']=train_image.apply(lambda x:1 if  (x['opacity_count']==0 and  x['Negative for Pneumonia']  !=0)  or (x['opacity_count']==0 and  x['Negative for Pneumonia']  !=1) \n                                              else 0,axis=1 ) \n'''","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:26:40.897807Z","iopub.execute_input":"2021-06-13T07:26:40.898220Z","iopub.status.idle":"2021-06-13T07:26:40.904817Z","shell.execute_reply.started":"2021-06-13T07:26:40.898177Z","shell.execute_reply":"2021-06-13T07:26:40.903833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Computing  the presence of BBx there or not  and areas of various bboxes\nimport ast,numpy as np\ntrain_image.boxes.fillna('',inplace=True)\n\ntrain_image['bbx_cnt']=train_image.boxes.apply(lambda x: len(ast.literal_eval(x) ) if  x !=''  else 0)\ntrain_image['area']=train_image.boxes.apply(lambda x : float(np.sum([ a['width']*a['height'] for a in \n                                                               ast.literal_eval(x)])) if  x !='' else 0)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:26:40.906479Z","iopub.execute_input":"2021-06-13T07:26:40.906929Z","iopub.status.idle":"2021-06-13T07:26:41.513379Z","shell.execute_reply.started":"2021-06-13T07:26:40.906885Z","shell.execute_reply":"2021-06-13T07:26:41.512208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(train_image['area'])  #total area for all the bounding boxes for Xray","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:26:41.514532Z","iopub.execute_input":"2021-06-13T07:26:41.514837Z","iopub.status.idle":"2021-06-13T07:26:41.674780Z","shell.execute_reply.started":"2021-06-13T07:26:41.514807Z","shell.execute_reply":"2021-06-13T07:26:41.673492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Getting anamoly ids the ones which are Covid positive but no BBX but other way round is 0 so we are good\nanamoly_ids=pd.merge(train_image[train_image.bbx_cnt==0],train_study[train_study[\"Negative for Pneumonia\"]==0],on='StudyInstanceUID').id_x.values.tolist()\nprint('Total No of Anamoly ids ',len(anamoly_ids))\ntrain_image[~train_image.id.isin(anamoly_ids)].shape\n#train_image.shape","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:28:00.792741Z","iopub.execute_input":"2021-06-13T07:28:00.793138Z","iopub.status.idle":"2021-06-13T07:28:00.815377Z","shell.execute_reply.started":"2021-06-13T07:28:00.793104Z","shell.execute_reply":"2021-06-13T07:28:00.814242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_image=pd.merge(train_image, train_study, on='StudyInstanceUID')\n#train_image.index.size","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:26:41.703212Z","iopub.execute_input":"2021-06-13T07:26:41.703554Z","iopub.status.idle":"2021-06-13T07:26:41.707877Z","shell.execute_reply.started":"2021-06-13T07:26:41.703525Z","shell.execute_reply":"2021-06-13T07:26:41.706788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_image['anamoly_flag']=train_image.apply(lambda x:1 if  (x['opacity_count']==0 and  train_study[train_study.StudyInstanceUID==x['StudyInstanceUID'] ]['Negative for Pneumonia'].values[0] !=0) else 0,axis=1 ) ","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:26:41.709260Z","iopub.execute_input":"2021-06-13T07:26:41.709632Z","iopub.status.idle":"2021-06-13T07:26:41.728211Z","shell.execute_reply.started":"2021-06-13T07:26:41.709601Z","shell.execute_reply":"2021-06-13T07:26:41.727125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# All values are 0 if its  Not a penuomonia\ntrain_study=pd.read_csv('../input/siim-covid19-detection/train_study_level.csv')\ntrain_study[train_study[\"Negative for Pneumonia\"]==1].sum(0)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:26:41.729780Z","iopub.execute_input":"2021-06-13T07:26:41.730130Z","iopub.status.idle":"2021-06-13T07:26:41.758825Z","shell.execute_reply.started":"2021-06-13T07:26:41.730099Z","shell.execute_reply":"2021-06-13T07:26:41.757828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Looks like Atypical patterns goes by its meaning interms of its occurances  in Images , and Typical also same . However some thing to notics is that there are around 1k appearance those are categorized as  Indeterminate. ","metadata":{}},{"cell_type":"code","source":"#So all patterns that are typical have No Indeterminate and Atypical patterns\ntrain_study[(train_study[\"Negative for Pneumonia\"]==0) & \n            ((train_study[\"Typical Appearance\"]==1))].sum(0)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:26:41.760065Z","iopub.execute_input":"2021-06-13T07:26:41.760379Z","iopub.status.idle":"2021-06-13T07:26:41.776867Z","shell.execute_reply.started":"2021-06-13T07:26:41.760351Z","shell.execute_reply":"2021-06-13T07:26:41.775382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Same goes for Indeterminate\ntrain_study[(train_study[\"Negative for Pneumonia\"]==0) & \n            ((train_study[\"Indeterminate Appearance\"]==1))].sum(0)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:26:41.778369Z","iopub.execute_input":"2021-06-13T07:26:41.778670Z","iopub.status.idle":"2021-06-13T07:26:41.790118Z","shell.execute_reply.started":"2021-06-13T07:26:41.778641Z","shell.execute_reply":"2021-06-13T07:26:41.789088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Same goes for Atypical\ntrain_study[(train_study[\"Negative for Pneumonia\"]==0) & \n            ((train_study[\"Atypical Appearance\"]==1))].sum(0)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:26:41.791317Z","iopub.execute_input":"2021-06-13T07:26:41.791708Z","iopub.status.idle":"2021-06-13T07:26:41.801860Z","shell.execute_reply.started":"2021-06-13T07:26:41.791660Z","shell.execute_reply":"2021-06-13T07:26:41.801106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Conclusion is This Problem Is multiclass objection Detection  Not Multilabel Multiclass ","metadata":{}},{"cell_type":"code","source":"train_image.StudyInstanceUID.count() ,train_study.id.count() \n#more rows than train study","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:26:41.803213Z","iopub.execute_input":"2021-06-13T07:26:41.803502Z","iopub.status.idle":"2021-06-13T07:26:41.814452Z","shell.execute_reply.started":"2021-06-13T07:26:41.803475Z","shell.execute_reply":"2021-06-13T07:26:41.813089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_study.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:26:41.816301Z","iopub.execute_input":"2021-06-13T07:26:41.817117Z","iopub.status.idle":"2021-06-13T07:26:41.834815Z","shell.execute_reply.started":"2021-06-13T07:26:41.817064Z","shell.execute_reply":"2021-06-13T07:26:41.833742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count_df_train_img","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:26:41.836407Z","iopub.execute_input":"2021-06-13T07:26:41.836732Z","iopub.status.idle":"2021-06-13T07:26:41.857682Z","shell.execute_reply.started":"2021-06-13T07:26:41.836697Z","shell.execute_reply":"2021-06-13T07:26:41.856446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#checking if a given StudyUID can have more than one DCMs\ncount_df_train_img=train_image.iloc[:,:-2].groupby('StudyInstanceUID').count().reset_index()\ncount_df_train_img.columns=['StudyInstanceUID','idcnt','boxescnt','labelcnt' ]\ncount_df_train_img","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:26:41.859505Z","iopub.execute_input":"2021-06-13T07:26:41.860005Z","iopub.status.idle":"2021-06-13T07:26:41.900298Z","shell.execute_reply.started":"2021-06-13T07:26:41.859938Z","shell.execute_reply":"2021-06-13T07:26:41.899226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Lets check one  \ntrain_image[train_image.StudyInstanceUID=='ffcb4630f46f']","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:26:41.901942Z","iopub.execute_input":"2021-06-13T07:26:41.902379Z","iopub.status.idle":"2021-06-13T07:26:41.918760Z","shell.execute_reply.started":"2021-06-13T07:26:41.902336Z","shell.execute_reply":"2021-06-13T07:26:41.917519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"One sop instance UID that is upper most dir will have multiple Series Instance UID which could possibly mean multiple x ray reports one may have infection other instance may not have an infection..negative report","metadata":{}},{"cell_type":"code","source":"!ls -l ../input/siim-covid19-detection/train/ffcb4630f46f/","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:26:41.922996Z","iopub.execute_input":"2021-06-13T07:26:41.923641Z","iopub.status.idle":"2021-06-13T07:26:42.676697Z","shell.execute_reply.started":"2021-06-13T07:26:41.923600Z","shell.execute_reply":"2021-06-13T07:26:42.675550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls -l ../input/siim-covid19-detection/train/ffcb4630f46f/4a4a9ae34a66","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:26:42.678801Z","iopub.execute_input":"2021-06-13T07:26:42.679139Z","iopub.status.idle":"2021-06-13T07:26:43.418103Z","shell.execute_reply.started":"2021-06-13T07:26:42.679103Z","shell.execute_reply":"2021-06-13T07:26:43.416757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" count_df_train_img.shape  ","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:26:43.422202Z","iopub.execute_input":"2021-06-13T07:26:43.422555Z","iopub.status.idle":"2021-06-13T07:26:43.430646Z","shell.execute_reply.started":"2021-06-13T07:26:43.422522Z","shell.execute_reply":"2021-06-13T07:26:43.429789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Loooking ids greater than 2 count","metadata":{}},{"cell_type":"code","source":"count_df_train_img[count_df_train_img.idcnt>1  ].head()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:26:43.431762Z","iopub.execute_input":"2021-06-13T07:26:43.432129Z","iopub.status.idle":"2021-06-13T07:26:43.449504Z","shell.execute_reply.started":"2021-06-13T07:26:43.432096Z","shell.execute_reply":"2021-06-13T07:26:43.448416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Looking at ones having boxes greater than 1 real duplicates with no other labels \n\ntrain_image[train_image.StudyInstanceUID=='7416b5cbc531']","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:26:43.450703Z","iopub.execute_input":"2021-06-13T07:26:43.451150Z","iopub.status.idle":"2021-06-13T07:26:43.470193Z","shell.execute_reply.started":"2021-06-13T07:26:43.451119Z","shell.execute_reply":"2021-06-13T07:26:43.469066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\ncount_df_train_img[(count_df_train_img.idcnt==1) ] ","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:26:43.471484Z","iopub.execute_input":"2021-06-13T07:26:43.472024Z","iopub.status.idle":"2021-06-13T07:26:43.492552Z","shell.execute_reply.started":"2021-06-13T07:26:43.471959Z","shell.execute_reply":"2021-06-13T07:26:43.491384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count_df_train_unique= count_df_train_img[(count_df_train_img.idcnt==1) ] ","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:26:43.493875Z","iopub.execute_input":"2021-06-13T07:26:43.494205Z","iopub.status.idle":"2021-06-13T07:26:43.499722Z","shell.execute_reply.started":"2021-06-13T07:26:43.494175Z","shell.execute_reply":"2021-06-13T07:26:43.498922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_image.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:26:43.500760Z","iopub.execute_input":"2021-06-13T07:26:43.501217Z","iopub.status.idle":"2021-06-13T07:26:43.522913Z","shell.execute_reply.started":"2021-06-13T07:26:43.501186Z","shell.execute_reply":"2021-06-13T07:26:43.521871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#merge with train img id having \ntrain_image_unique=pd.merge(count_df_train_unique,train_image,on='StudyInstanceUID')\ntrain_image_unique.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:26:43.524053Z","iopub.execute_input":"2021-06-13T07:26:43.524473Z","iopub.status.idle":"2021-06-13T07:26:43.556018Z","shell.execute_reply.started":"2021-06-13T07:26:43.524440Z","shell.execute_reply":"2021-06-13T07:26:43.554968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_image_unique[train_image_unique.boxes.isnull() ].shape # all Nan have no boxes numbering 1705","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:26:43.557371Z","iopub.execute_input":"2021-06-13T07:26:43.557673Z","iopub.status.idle":"2021-06-13T07:26:43.568149Z","shell.execute_reply.started":"2021-06-13T07:26:43.557645Z","shell.execute_reply":"2021-06-13T07:26:43.566761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#validating with train study\ntrain_study['StudyInstanceUID']=train_study.id.apply(lambda x: x.split('_')[0])\n\n ","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:26:43.569813Z","iopub.execute_input":"2021-06-13T07:26:43.570302Z","iopub.status.idle":"2021-06-13T07:26:43.586634Z","shell.execute_reply.started":"2021-06-13T07:26:43.570256Z","shell.execute_reply":"2021-06-13T07:26:43.585538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_study_unique=pd.merge(count_df_train_unique,train_study,on='StudyInstanceUID')\ntrain_study_unique.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:26:43.587929Z","iopub.execute_input":"2021-06-13T07:26:43.588298Z","iopub.status.idle":"2021-06-13T07:26:43.612063Z","shell.execute_reply.started":"2021-06-13T07:26:43.588264Z","shell.execute_reply":"2021-06-13T07:26:43.610873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_study_unique[train_study_unique.boxescnt==0 ][\"Negative for Pneumonia\"].value_counts()\n# oh so there are  82 odd instances where are there no bbox provided even though patients have pneumonia","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:26:43.613243Z","iopub.execute_input":"2021-06-13T07:26:43.613534Z","iopub.status.idle":"2021-06-13T07:26:43.626475Z","shell.execute_reply.started":"2021-06-13T07:26:43.613506Z","shell.execute_reply":"2021-06-13T07:26:43.625486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Further classifying them\ntrain_study_unique[train_study_unique.boxescnt==0 ].sum(0)\n#mostly are atypical  \n#open question from domain expert why should that be","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:26:43.628296Z","iopub.execute_input":"2021-06-13T07:26:43.629126Z","iopub.status.idle":"2021-06-13T07:26:43.640850Z","shell.execute_reply.started":"2021-06-13T07:26:43.629076Z","shell.execute_reply":"2021-06-13T07:26:43.640090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_study.sum(0)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:26:43.641958Z","iopub.execute_input":"2021-06-13T07:26:43.642423Z","iopub.status.idle":"2021-06-13T07:26:43.682816Z","shell.execute_reply.started":"2021-06-13T07:26:43.642377Z","shell.execute_reply":"2021-06-13T07:26:43.681756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#lets check regular Atypical cases \n\ntrain_study_unique[train_study_unique.boxescnt!=0 ].sum(0)\n#ok that is fine 386 out of 474 have no irregularity","metadata":{"execution":{"iopub.status.busy":"2021-06-13T07:26:43.684128Z","iopub.execute_input":"2021-06-13T07:26:43.684425Z","iopub.status.idle":"2021-06-13T07:26:43.725428Z","shell.execute_reply.started":"2021-06-13T07:26:43.684397Z","shell.execute_reply":"2021-06-13T07:26:43.724102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"That is it very  Simple EDA . Could help in building a good CV .Please post your opinions or any questions\n","metadata":{}}]}