{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"REFRENCES \n\nhttps://www.kaggle.com/piantic/siim-fisabio-rsna-covid-19-detection-basic-eda/notebook#7.-Etc.---Pandas-Profiling-%F0%9F%8C%A4%EF%B8%8F by @Heroseo\n\nhttps://www.kaggle.com/andradaolteanu/siim-covid-19-3d-analysis-augmentations by @andradaolteanu\n\nhttps://www.kaggle.com/piantic/siim-fisabio-rsna-covid-19-detection-basic-eda by @piantic\n\nhttps://www.kaggle.com/tanlikesmath/siim-covid-19-detection-a-simple-eda by @tanlikesmath\n\n","metadata":{}},{"cell_type":"code","source":"import glob,tqdm\nimport pandas as pd,numpy as np,os,ast,seaborn as sns,matplotlib.pyplot as plt,matplotlib as mpl,cv2\n# !pip install plotly\nimport plotly_express as px\nfrom colorama import Fore, Back, Style\nimport pydicom as dicom\nsample_sub_loc = '../input/siim-covid19-detection/sample_submission.csv'\nimage_level_loc = '../input/siim-covid19-detection/train_image_level.csv'\nstudy_level_loc  = '../input/siim-covid19-detection/train_study_level.csv'\ntrain_images_loc = '../input/siim-covid19-detection/train'\ntest_images_loc = '../input/siim-covid19-detection/test'","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:47:11.34052Z","iopub.execute_input":"2021-06-11T08:47:11.340922Z","iopub.status.idle":"2021-06-11T08:47:11.347655Z","shell.execute_reply.started":"2021-06-11T08:47:11.340888Z","shell.execute_reply":"2021-06-11T08:47:11.346737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Number of Train Images',len(glob.glob(os.path.join(train_images_loc,'*','*','*.dcm'))),'Test Images',len(glob.glob(os.path.join(test_images_loc,'*','*','*'))))\nprint(len(glob.glob(os.path.join(train_images_loc,'*'))),len(glob.glob(os.path.join(train_images_loc,'*','*'))))","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:47:11.351293Z","iopub.execute_input":"2021-06-11T08:47:11.351811Z","iopub.status.idle":"2021-06-11T08:47:50.115153Z","shell.execute_reply.started":"2021-06-11T08:47:11.351765Z","shell.execute_reply":"2021-06-11T08:47:50.113612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Study level\n\n\n6054 IDs(Points to a unique dcm)\n\nSingle class multiple options\n","metadata":{}},{"cell_type":"code","source":"study_level = pd.read_csv(study_level_loc)\nstudy_level['id'] = study_level['id'].str.split('_',expand=True)[0]\nprint('Shape of study' ,study_level.shape,'Check for Nulls ',study_level.isnull().sum(),sep='\\n')\nprint('Unique IDS are ',study_level['id'].nunique())\n\n# glob.glob(os.path.join(train_images_loc,'00086460a852','*','*.dcm'))\n\nviz = study_level[['Negative for Pneumonia', 'Typical Appearance','Indeterminate Appearance', 'Atypical Appearance']].idxmax(axis=1)\nstudy_level['type'] = study_level[['Negative for Pneumonia', 'Typical Appearance','Indeterminate Appearance', 'Atypical Appearance']].idxmax(axis=1)\nstudy_level['Images_in_study_file']=study_level['id'].apply(lambda x:len(glob.glob(os.path.join(train_images_loc,x,'*','*.dcm'))))\ndisplay(study_level.head())\nplt.pie(viz.value_counts(),labels=viz.value_counts().index,autopct=lambda x:f\"{x:.2f}% \")","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:47:50.116934Z","iopub.execute_input":"2021-06-11T08:47:50.117248Z","iopub.status.idle":"2021-06-11T08:47:56.635155Z","shell.execute_reply.started":"2021-06-11T08:47:50.117219Z","shell.execute_reply":"2021-06-11T08:47:56.634118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"px.histogram(study_level['Images_in_study_file'],histnorm='percent')","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:47:56.63658Z","iopub.execute_input":"2021-06-11T08:47:56.636911Z","iopub.status.idle":"2021-06-11T08:47:57.796748Z","shell.execute_reply.started":"2021-06-11T08:47:56.636867Z","shell.execute_reply":"2021-06-11T08:47:57.795522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in study_level['id'].iteritems():\n    if len(glob.glob(os.path.join(train_images_loc,'00086460a852','*','*.dcm')))>1:\n        print(i[1]) \n# Hence study id 1:1 image","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:48:12.903195Z","iopub.execute_input":"2021-06-11T08:48:12.903743Z","iopub.status.idle":"2021-06-11T08:48:18.732663Z","shell.execute_reply.started":"2021-06-11T08:48:12.903696Z","shell.execute_reply":"2021-06-11T08:48:18.731567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Image level \n\nSame length as the number of images we have.All ids are unique.\n\nSome Images have no boxes or NAN with label none 1 0 0 1 1 .\n\nTo access a picture we have to use study instance uid which are on many:1 relation with id.","metadata":{}},{"cell_type":"code","source":"image_level = pd.read_csv(image_level_loc)\nimage_level[['id','t']]= image_level['id'].str.split('_',expand=True)\nprint('Shape of Image level',image_level.shape,'Checking Unique IDS and also (Study Instance ID)',image_level['id'].nunique(),\n     image_level['StudyInstanceUID'].nunique())\ndisplay(image_level)\n#glob.glob(os.path.join(train_images_loc,'5776db0cec75','*','*.dcm')) Work only on SUID\n#glob.glob(os.path.join(train_images_loc,'*','*','000a312787f2.dcm')) Use id like this","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:48:18.734457Z","iopub.execute_input":"2021-06-11T08:48:18.73479Z","iopub.status.idle":"2021-06-11T08:48:18.840251Z","shell.execute_reply.started":"2021-06-11T08:48:18.734757Z","shell.execute_reply":"2021-06-11T08:48:18.83889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(image_level[image_level.StudyInstanceUID=='0fd2db233deb'])\nprint(glob.glob(os.path.join(train_images_loc,'0fd2db233deb','*','*.dcm')))","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:48:18.84292Z","iopub.execute_input":"2021-06-11T08:48:18.843379Z","iopub.status.idle":"2021-06-11T08:48:18.873671Z","shell.execute_reply.started":"2021-06-11T08:48:18.843332Z","shell.execute_reply":"2021-06-11T08:48:18.872427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set_style('darkgrid')\nfig, axs = plt.subplots(figsize=(20,10),ncols=3)\nimage_level['boxes'].fillna(0,inplace=True)\nimage_level['count_boxes'] = image_level['boxes'].apply(lambda x:len(ast.literal_eval(x)) if x else 0)\nsns.countplot(image_level.count_boxes,ax=axs[0])\n\nimage_level['label_type'] = image_level.label.str.split(expand=True)[0]\nsns.countplot(image_level['label_type'],ax=axs[1])\n#px.histogram(image_level.StudyInstanceUID.value_counts())\n\n\n# fig = px.pie(image_level,values='label_type',names='count_boxes')\n# fig.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:48:18.876044Z","iopub.execute_input":"2021-06-11T08:48:18.876484Z","iopub.status.idle":"2021-06-11T08:48:19.606358Z","shell.execute_reply.started":"2021-06-11T08:48:18.876438Z","shell.execute_reply":"2021-06-11T08:48:19.604934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gg = image_level[image_level['StudyInstanceUID'].isin (image_level['StudyInstanceUID'].value_counts()[image_level['StudyInstanceUID'].value_counts()>1].index)]\ndisplay(gg[gg.label_type=='opacity'].groupby('StudyInstanceUID')['label_type'].describe())\ndisplay(gg[gg.label_type!='opacity'].groupby('StudyInstanceUID')['label_type'].describe())\nprint('Maximum Frequency of opacity instances for any StudyUID is ',gg[gg.label_type=='opacity'].groupby('StudyInstanceUID')['label_type'].describe()['freq'].max(),\n     'and Maximum Frequency of None instances for any StudyUID is ',gg[gg.label_type!='opacity'].groupby('StudyInstanceUID')['label_type'].describe()['freq'].max())","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:48:19.607898Z","iopub.execute_input":"2021-06-11T08:48:19.608345Z","iopub.status.idle":"2021-06-11T08:48:21.067166Z","shell.execute_reply.started":"2021-06-11T08:48:19.6083Z","shell.execute_reply":"2021-06-11T08:48:21.065731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"MERGE THE DATAFRAME\n\nStudy level has 6054 rows and Image level has 6334 .\n\nAs in Image level we have multiple rows with same StudyInstanceUID.(0fd2db233deb)","metadata":{}},{"cell_type":"code","source":"print(image_level.columns,study_level.columns)\nmerged_levels = pd.merge(image_level, study_level, left_on='StudyInstanceUID',right_on='id')\nmerged_levels['boxes'] = merged_levels['boxes'].apply(lambda x :ast.literal_eval(x) if x else x)\n# merged_levels.drop(columns=['id_x','id_y'],inplace=True)\nmerged_levels.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:48:21.068897Z","iopub.execute_input":"2021-06-11T08:48:21.069304Z","iopub.status.idle":"2021-06-11T08:48:21.289149Z","shell.execute_reply.started":"2021-06-11T08:48:21.069261Z","shell.execute_reply":"2021-06-11T08:48:21.28803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So mutliple Rows with same SIUID seems to have maximum of only one row instance to be of type opacity and rest are None","metadata":{}},{"cell_type":"code","source":"display(merged_levels[merged_levels['StudyInstanceUID']=='0fd2db233deb'])","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:48:21.290596Z","iopub.execute_input":"2021-06-11T08:48:21.291028Z","iopub.status.idle":"2021-06-11T08:48:21.321849Z","shell.execute_reply.started":"2021-06-11T08:48:21.29098Z","shell.execute_reply":"2021-06-11T08:48:21.320581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Submission Files\n\nGiven a study id , find the image and make the prediction(class Confidence bbox)\n","metadata":{}},{"cell_type":"code","source":"print(image_level.columns)\nprint(study_level.columns)","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:48:21.325176Z","iopub.execute_input":"2021-06-11T08:48:21.325556Z","iopub.status.idle":"2021-06-11T08:48:21.332572Z","shell.execute_reply.started":"2021-06-11T08:48:21.325517Z","shell.execute_reply":"2021-06-11T08:48:21.330985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub = pd.read_csv(sample_sub_loc)\nprint('Shape of Sample submission',sample_sub.shape)\nsample_sub[['id','type']] = sample_sub['id'].str.split('_',expand=True)\ndisplay(sample_sub)\npx.bar(sample_sub['type'].value_counts())\n\n#glob.glob(os.path.join(test_images_loc,'00188a671292','*','*')) Works for study ids only\n#glob.glob(os.path.join(test_images_loc,'*','*','46719b856de1.dcm')) For Image ids","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:48:21.335869Z","iopub.execute_input":"2021-06-11T08:48:21.336334Z","iopub.status.idle":"2021-06-11T08:48:21.462514Z","shell.execute_reply.started":"2021-06-11T08:48:21.336285Z","shell.execute_reply":"2021-06-11T08:48:21.461694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Images ","metadata":{}},{"cell_type":"code","source":"for i in merged_levels.StudyInstanceUID.unique():\n    if len(glob.glob(os.path.join(train_images_loc,i,'*','*.dcm')))>8:\n        print(i,len(glob.glob(os.path.join(train_images_loc,i,'*','*.dcm'))))","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:48:21.463623Z","iopub.execute_input":"2021-06-11T08:48:21.464069Z","iopub.status.idle":"2021-06-11T08:48:27.973335Z","shell.execute_reply.started":"2021-06-11T08:48:21.464038Z","shell.execute_reply":"2021-06-11T08:48:27.972154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_cord(boxes):\n    boxes = ast.literal_eval(boxes)\n    b = list()\n    for b in boxes:\n        x,y,w,h = j['x'],j['y'],j['width'],j['height']\n        b.append(x,y,w,h)\n    return b\ndef metadata(study_id):\n    bbox,label =[],[]\n    df = merged_levels[merged_levels.id_y==study_id][['boxes','type']]\n    for i in df.iterrows():\n        a,b = i[1][0],i[1][1]\n        if a :\n            bboxes = [[x['x'],x['y'],x['x']+x['width'],x['y']+x['height']] for x in a]\n            bbox.append(bboxes)\n        else:\n            bbox.append([[-1,-1,-1,-1]])\n        label.append(b)\n        bbox\n    return bbox,label\n\ndef show_img(studyid,figsize=20):\n    if not isinstance(studyid, str):\n        for i in studyid:\n            show_img(i,figsize//len(studyid))\n        return\n    dcms = glob.glob(os.path.join(train_images_loc,studyid,'*','*.dcm'))\n    row = ((len(dcms)-1)//3)+1\n    ncols = 3 if len(dcms)>3 else len(dcms)\n    fig,axes = plt.subplots(figsize=(figsize,figsize),ncols=ncols,nrows =row)\n    meta_data = metadata(studyid)\n    for cnt,path in enumerate(dcms):\n        data = dicom.dcmread(path)\n        msg = meta_data[1][cnt].split()[0]+' Body Part '+data['BodyPartExamined'].value +' SEX  '+ data[\"PatientSex\"].value\n        img = data.pixel_array\n#         print(meta_data[0][cnt],meta_data[1][cnt])\n#         Loc '/'.join(path.split('/')[-2:])\n        if meta_data[0][cnt]:       \n            for x1, y1, x2, y2 in meta_data[0][cnt]:\n                x1, y1, x2, y2 = int(x1), int(y1), int(x2), int(y2)\n                img = cv2.rectangle(img,(x1, y1), (x2, y2), color = [0,0,255], thickness = 10)\n        if len(dcms) == 1:\n            axes.imshow(img,cmap=\"gray\")\n            axes.axis('off')\n            axes.title.set_text(msg)\n        else:\n            axes[cnt//3,cnt%3].imshow(img,cmap=\"gray\")\n            axes[cnt//3,cnt%3].axis('off')\n            axes[cnt//3,cnt%3].title.set_text(msg)","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:48:27.974808Z","iopub.execute_input":"2021-06-11T08:48:27.975116Z","iopub.status.idle":"2021-06-11T08:48:27.993966Z","shell.execute_reply.started":"2021-06-11T08:48:27.975086Z","shell.execute_reply":"2021-06-11T08:48:27.992653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_img('0fd2db233deb')","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:48:27.995534Z","iopub.execute_input":"2021-06-11T08:48:27.995996Z","iopub.status.idle":"2021-06-11T08:48:40.115469Z","shell.execute_reply.started":"2021-06-11T08:48:27.99595Z","shell.execute_reply":"2021-06-11T08:48:40.114266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Some Images on ALL TYPES ","metadata":{}},{"cell_type":"code","source":"for i in merged_levels[(merged_levels['type']=='Negative for Pneumonia')]['StudyInstanceUID'].unique()[:3]:\n    show_img(i,20)","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:48:40.116945Z","iopub.execute_input":"2021-06-11T08:48:40.117371Z","iopub.status.idle":"2021-06-11T08:48:46.62726Z","shell.execute_reply.started":"2021-06-11T08:48:40.117322Z","shell.execute_reply":"2021-06-11T08:48:46.625885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in merged_levels[(merged_levels['count_boxes']>3)& (merged_levels['type']=='Typical Appearance')]['StudyInstanceUID'].unique()[:3]:\n    show_img(i,4)","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:48:46.628907Z","iopub.execute_input":"2021-06-11T08:48:46.629258Z","iopub.status.idle":"2021-06-11T08:48:50.723691Z","shell.execute_reply.started":"2021-06-11T08:48:46.629225Z","shell.execute_reply":"2021-06-11T08:48:50.722221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in merged_levels[(merged_levels['count_boxes']>3)& (merged_levels['type']=='Typical Appearance')]['StudyInstanceUID'].unique()[:3]:\n    show_img(i,4)","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:48:50.726031Z","iopub.execute_input":"2021-06-11T08:48:50.72652Z","iopub.status.idle":"2021-06-11T08:48:53.519142Z","shell.execute_reply.started":"2021-06-11T08:48:50.726468Z","shell.execute_reply":"2021-06-11T08:48:53.517834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_img('53b92c44a02e',10)\nshow_img('341a8f794069',10)","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:48:53.521231Z","iopub.execute_input":"2021-06-11T08:48:53.521717Z","iopub.status.idle":"2021-06-11T08:48:56.177892Z","shell.execute_reply.started":"2021-06-11T08:48:53.521649Z","shell.execute_reply":"2021-06-11T08:48:56.176335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(image_level[image_level['StudyInstanceUID'].isin(image_level.groupby('StudyInstanceUID').count()[image_level.groupby('StudyInstanceUID').count()['id']>1].index)])","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:48:56.180041Z","iopub.execute_input":"2021-06-11T08:48:56.180471Z","iopub.status.idle":"2021-06-11T08:48:56.235282Z","shell.execute_reply.started":"2021-06-11T08:48:56.18043Z","shell.execute_reply":"2021-06-11T08:48:56.234089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"DATA FROM IMAGE\n\nos.path.join(train_images_loc,image_level.StudyInstanceUID[2],'*','*.dcm')","metadata":{}},{"cell_type":"code","source":"# dicom.dcmread(glob.glob(os.path.join(train_images_loc,image_level.StudyInstanceUID[0],'*','*.dcm'))[0])\nelem = dicom.dcmread(glob.glob(os.path.join(train_images_loc,image_level.StudyInstanceUID[2],'*','*.dcm'))[0])\nfor i in ['BodyPartExamined','ImageType',\"PatientSex\"]:\n    print(elem[i])\nelem","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:48:56.237156Z","iopub.execute_input":"2021-06-11T08:48:56.237596Z","iopub.status.idle":"2021-06-11T08:48:56.701598Z","shell.execute_reply.started":"2021-06-11T08:48:56.237551Z","shell.execute_reply":"2021-06-11T08:48:56.700304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(elem)\nprint(elem.dir())","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:48:56.703519Z","iopub.execute_input":"2021-06-11T08:48:56.704029Z","iopub.status.idle":"2021-06-11T08:48:56.714718Z","shell.execute_reply.started":"2021-06-11T08:48:56.703975Z","shell.execute_reply":"2021-06-11T08:48:56.713531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"attrs =['AccessionNumber', 'BitsAllocated', 'BitsStored', 'BodyPartExamined','Columns', 'DeidentificationMethod', 'HighBit', 'ImageType', 'ImagerPixelSpacing', 'InstanceNumber', 'Modality', 'PatientID', 'PatientName', 'PatientSex', 'PhotometricInterpretation', 'PixelRepresentation','Rows', 'SOPClassUID', 'SOPInstanceUID', 'SamplesPerPixel', 'SeriesInstanceUID', 'SeriesNumber', 'SpecificCharacterSet', 'StudyDate', 'StudyID', 'StudyInstanceUID', 'StudyTime']\ndef get_metadata(path):\n    elem = dicom.dcmread(path)\n#     img = elem.get('PixelData')\n    attrs =['AccessionNumber', 'BitsAllocated', 'BitsStored', 'BodyPartExamined','Columns', 'DeidentificationMethod', 'HighBit', 'ImageType', 'ImagerPixelSpacing', 'InstanceNumber', 'Modality', 'PatientID', 'PatientName', 'PatientSex', 'PhotometricInterpretation', 'PixelRepresentation','Rows', 'SOPClassUID', 'SOPInstanceUID', 'SamplesPerPixel', 'SeriesInstanceUID', 'SeriesNumber', 'SpecificCharacterSet', 'StudyDate', 'StudyID', 'StudyInstanceUID', 'StudyTime']\n    df = [path]\n    for i in attrs:\n        df.append(elem.get(i))\n    return df","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:48:56.71685Z","iopub.execute_input":"2021-06-11T08:48:56.717317Z","iopub.status.idle":"2021-06-11T08:48:56.727337Z","shell.execute_reply.started":"2021-06-11T08:48:56.71727Z","shell.execute_reply":"2021-06-11T08:48:56.726456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Datasets Saved\nmeta_data_loc = '../input/all-metadata/AllMeta.csv'\nif os.path.isfile(meta_data_loc):\n    full_meta = pd.read_csv(meta_data_loc,usecols=['path']+attrs)\nelse:\n    full_meta = []\n    for i in tqdm.tqdm(glob.glob(os.path.join(train_images_loc,'*','*','*.dcm'))):\n        full_meta.append(get_metadata(i))\n    full_meta = pd.DataFrame(full_meta,columns=['path']+attrs)\n    full_meta.to_csv('AllMeta.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:49:26.168373Z","iopub.execute_input":"2021-06-11T08:49:26.168851Z","iopub.status.idle":"2021-06-11T08:49:26.21132Z","shell.execute_reply.started":"2021-06-11T08:49:26.168815Z","shell.execute_reply":"2021-06-11T08:49:26.209379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(attrs)","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:48:56.958703Z","iopub.status.idle":"2021-06-11T08:48:56.959155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So around 56% percent entries aare of Men \n\nAnd most have examined their chest about 80%.","metadata":{}},{"cell_type":"code","source":"#full_meta.groupby(['PatientSex','BodyPartExamined']).size().reset_index().pivot(columns='PatientSex',index='BodyPartExamined',values=0).plot(kind='bar', stacked=True)\nprint(full_meta.PatientSex.value_counts()/len(full_meta))\nprint(full_meta.BodyPartExamined.value_counts()/len(full_meta))\nviz = full_meta.groupby(['PatientSex','BodyPartExamined']).size().reset_index().pivot(columns='PatientSex',index='BodyPartExamined',values=0)\npx.bar(viz,y=['F','M'])","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:48:56.960462Z","iopub.status.idle":"2021-06-11T08:48:56.961013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in full_meta.columns:\n    print(i,full_meta[i].nunique())","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:48:56.962197Z","iopub.status.idle":"2021-06-11T08:48:56.962872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merged_meta = pd.merge(merged_levels,full_meta,left_on='id_x',right_on='SOPInstanceUID')\ndisplay(merged_meta)","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:48:56.964215Z","iopub.status.idle":"2021-06-11T08:48:56.964889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"viz = merged_meta.groupby(['type','BodyPartExamined']).size().reset_index().pivot(columns='BodyPartExamined',index='type',values=0)\npx.bar(viz,orientation='h')","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:48:56.966161Z","iopub.status.idle":"2021-06-11T08:48:56.966831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Multiple Patients have same Name , so Even though their identity are anonymous the names have been encoded in a particular way.\n\nAbout 60% percent of patients have only one entry and as 96% of the total patients have exactly 1 image of them , we can say for sure that a good size of sample around (40-4 = 36%) of entries are made by a person who had a test before .\n\nLet me explain this in simple step:\n1. 96-4 is the ratio of patient having multiple images of them ,(as multiple images are a product of artifacts{small problems while taking the image } .\n2. 60-40 is the ratio of patiens having single entries ,but already 4% of entries have more than 1 image so that information is of not much use.\n3. So the main calculation here is that the 4% out of that 40% are the ones have multiple images due to error so we have 36% of actual double entries .","metadata":{}},{"cell_type":"code","source":"px.histogram(merged_meta['PatientName'].value_counts(),histnorm='percent')","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:48:56.968387Z","iopub.status.idle":"2021-06-11T08:48:56.969075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pandas_profiling import ProfileReport\nprofile_study = ProfileReport(merged_levels, title=\"Pandas Profiling Report - Train Study df\")\nprofile_study.to_widgets()","metadata":{"execution":{"iopub.status.busy":"2021-06-11T04:02:53.853802Z","iopub.execute_input":"2021-06-11T04:02:53.854178Z","iopub.status.idle":"2021-06-11T04:03:42.121654Z","shell.execute_reply.started":"2021-06-11T04:02:53.854143Z","shell.execute_reply":"2021-06-11T04:03:42.120288Z"},"trusted":true},"execution_count":null,"outputs":[]}]}