{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"! conda install -c conda-forge gdcm -y","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:21:53.92614Z","iopub.execute_input":"2021-05-28T02:21:53.926829Z","iopub.status.idle":"2021-05-28T02:22:57.021299Z","shell.execute_reply.started":"2021-05-28T02:21:53.92674Z","shell.execute_reply":"2021-05-28T02:22:57.020151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport re\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nimport cv2\nimport albumentations as A\nfrom tqdm.notebook import tqdm\n\nfrom fastai.vision.all import *\nfrom fastai.medical.imaging import *","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-05-28T02:23:01.854011Z","iopub.execute_input":"2021-05-28T02:23:01.854426Z","iopub.status.idle":"2021-05-28T02:23:06.027953Z","shell.execute_reply.started":"2021-05-28T02:23:01.854376Z","shell.execute_reply":"2021-05-28T02:23:06.026964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from matplotlib.pyplot import figure\nfigure(figsize=(12, 10), dpi=80)","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:06.031808Z","iopub.execute_input":"2021-05-28T02:23:06.032143Z","iopub.status.idle":"2021-05-28T02:23:06.043304Z","shell.execute_reply.started":"2021-05-28T02:23:06.032113Z","shell.execute_reply":"2021-05-28T02:23:06.042642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = Path('../input/siim-covid19-detection')","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:06.044766Z","iopub.execute_input":"2021-05-28T02:23:06.045206Z","iopub.status.idle":"2021-05-28T02:23:06.057962Z","shell.execute_reply.started":"2021-05-28T02:23:06.045177Z","shell.execute_reply":"2021-05-28T02:23:06.057124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path.ls()","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:06.059299Z","iopub.execute_input":"2021-05-28T02:23:06.059714Z","iopub.status.idle":"2021-05-28T02:23:06.075583Z","shell.execute_reply.started":"2021-05-28T02:23:06.059682Z","shell.execute_reply":"2021-05-28T02:23:06.074651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path_train_image = path/'train'\npath_test_image = path/'test'\nss = pd.read_csv(path/'sample_submission.csv')\ntrain_image = pd.read_csv(path/'train_image_level.csv')\ntrain_study = pd.read_csv(path/'train_study_level.csv')","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:06.076936Z","iopub.execute_input":"2021-05-28T02:23:06.077304Z","iopub.status.idle":"2021-05-28T02:23:06.170125Z","shell.execute_reply.started":"2021-05-28T02:23:06.077267Z","shell.execute_reply":"2021-05-28T02:23:06.168051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Understanding train_study DF","metadata":{}},{"cell_type":"code","source":"train_study.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:06.171698Z","iopub.execute_input":"2021-05-28T02:23:06.172153Z","iopub.status.idle":"2021-05-28T02:23:06.200525Z","shell.execute_reply.started":"2021-05-28T02:23:06.172108Z","shell.execute_reply":"2021-05-28T02:23:06.199811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#number of individual studies\nlen(train_study)","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:06.201573Z","iopub.execute_input":"2021-05-28T02:23:06.202024Z","iopub.status.idle":"2021-05-28T02:23:06.208064Z","shell.execute_reply.started":"2021-05-28T02:23:06.201993Z","shell.execute_reply":"2021-05-28T02:23:06.207022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#sanity check to make sure no repeats for individual studies\nassert len(train_study) == train_study.id.nunique()","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:06.21072Z","iopub.execute_input":"2021-05-28T02:23:06.211035Z","iopub.status.idle":"2021-05-28T02:23:06.226903Z","shell.execute_reply.started":"2021-05-28T02:23:06.211005Z","shell.execute_reply":"2021-05-28T02:23:06.225254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_dist = train_study.iloc[:,1:].sum(axis=0); class_dist/sum(class_dist)","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:06.228378Z","iopub.execute_input":"2021-05-28T02:23:06.228694Z","iopub.status.idle":"2021-05-28T02:23:06.248555Z","shell.execute_reply.started":"2021-05-28T02:23:06.228664Z","shell.execute_reply":"2021-05-28T02:23:06.247328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Number of classes = 4","metadata":{}},{"cell_type":"code","source":"plt.bar(class_dist.index, class_dist.values)\n\nfor x, y in zip(class_dist.index, class_dist.values):\n    plt.text(x, y+50, f'{y/len(train_study)*100:.3}%', fontsize=14)\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:06.24972Z","iopub.execute_input":"2021-05-28T02:23:06.250017Z","iopub.status.idle":"2021-05-28T02:23:06.480247Z","shell.execute_reply.started":"2021-05-28T02:23:06.249988Z","shell.execute_reply":"2021-05-28T02:23:06.479313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# are there studies with more than one label? No\n(train_study.iloc[:, 1:].sum(1) > 1).sum()","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:06.481615Z","iopub.execute_input":"2021-05-28T02:23:06.481958Z","iopub.status.idle":"2021-05-28T02:23:06.489534Z","shell.execute_reply.started":"2021-05-28T02:23:06.481918Z","shell.execute_reply":"2021-05-28T02:23:06.488544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Understanding train_image DF","metadata":{}},{"cell_type":"code","source":"train_image.head(5)","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:06.491167Z","iopub.execute_input":"2021-05-28T02:23:06.491519Z","iopub.status.idle":"2021-05-28T02:23:06.510415Z","shell.execute_reply.started":"2021-05-28T02:23:06.491436Z","shell.execute_reply":"2021-05-28T02:23:06.509446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# there are more images than study. This is because one study can have more than one image.\nlen(train_image)","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:06.594206Z","iopub.execute_input":"2021-05-28T02:23:06.594706Z","iopub.status.idle":"2021-05-28T02:23:06.599493Z","shell.execute_reply.started":"2021-05-28T02:23:06.594675Z","shell.execute_reply":"2021-05-28T02:23:06.598789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# checks if the number of studies in the image level is same as the train_study. Yes, it is\nassert train_image.StudyInstanceUID.nunique() == len(train_study)","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:06.833196Z","iopub.execute_input":"2021-05-28T02:23:06.833692Z","iopub.status.idle":"2021-05-28T02:23:06.840855Z","shell.execute_reply.started":"2021-05-28T02:23:06.833659Z","shell.execute_reply":"2021-05-28T02:23:06.839805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numimagesperstudy = train_image.groupby('StudyInstanceUID').count()['id'].sort_values(ascending=False)","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:07.029058Z","iopub.execute_input":"2021-05-28T02:23:07.029412Z","iopub.status.idle":"2021-05-28T02:23:07.049343Z","shell.execute_reply.started":"2021-05-28T02:23:07.029381Z","shell.execute_reply":"2021-05-28T02:23:07.048443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numimagesperstudycounts = numimagesperstudy.value_counts(); numimagesperstudycounts","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:07.260969Z","iopub.execute_input":"2021-05-28T02:23:07.261328Z","iopub.status.idle":"2021-05-28T02:23:07.269778Z","shell.execute_reply.started":"2021-05-28T02:23:07.261297Z","shell.execute_reply":"2021-05-28T02:23:07.269018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = plt.bar(numimagesperstudycounts.index, numimagesperstudycounts.values)\n\nfor x, y in zip(numimagesperstudycounts.index, numimagesperstudycounts.values):\n    plt.text(x, y+50, f'{y}', fontsize=14)\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:07.490346Z","iopub.execute_input":"2021-05-28T02:23:07.490933Z","iopub.status.idle":"2021-05-28T02:23:07.789222Z","shell.execute_reply.started":"2021-05-28T02:23:07.490896Z","shell.execute_reply":"2021-05-28T02:23:07.788038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Most studies have only one image. About 230 studies have more than one image. There is one study with 9 images. Lets take a look at this.","metadata":{}},{"cell_type":"code","source":"numimagesperstudy[numimagesperstudy >8].index[0]","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:07.94265Z","iopub.execute_input":"2021-05-28T02:23:07.943009Z","iopub.status.idle":"2021-05-28T02:23:07.949923Z","shell.execute_reply.started":"2021-05-28T02:23:07.942979Z","shell.execute_reply":"2021-05-28T02:23:07.948705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_image[train_image['StudyInstanceUID'] == numimagesperstudy[numimagesperstudy >8].index[0]]","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:08.190646Z","iopub.execute_input":"2021-05-28T02:23:08.190992Z","iopub.status.idle":"2021-05-28T02:23:08.20628Z","shell.execute_reply.started":"2021-05-28T02:23:08.190963Z","shell.execute_reply":"2021-05-28T02:23:08.205334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This is interesting. Out of the 9 images, only one has a bbox. Other images have label none. Let's take a look at the study level label for this.","metadata":{}},{"cell_type":"code","source":"train_study[train_study['id'] == '0fd2db233deb_study']","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:08.631923Z","iopub.execute_input":"2021-05-28T02:23:08.632256Z","iopub.status.idle":"2021-05-28T02:23:08.647333Z","shell.execute_reply.started":"2021-05-28T02:23:08.632227Z","shell.execute_reply":"2021-05-28T02:23:08.646411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"At study level, the label is 'Intermediate Appearace'.\n\nNow, lets take a look at the distribution of number of bboxes per image.","metadata":{}},{"cell_type":"code","source":"train_image['n_boxes'] = train_image['boxes'].apply(lambda x: sum(1 for _ in re.findall('width', str(x))))","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:09.094819Z","iopub.execute_input":"2021-05-28T02:23:09.095197Z","iopub.status.idle":"2021-05-28T02:23:09.119249Z","shell.execute_reply.started":"2021-05-28T02:23:09.09516Z","shell.execute_reply":"2021-05-28T02:23:09.118516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_boxes = train_image['n_boxes'].value_counts(); n_boxes/sum(n_boxes)*100","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:09.333439Z","iopub.execute_input":"2021-05-28T02:23:09.333931Z","iopub.status.idle":"2021-05-28T02:23:09.342121Z","shell.execute_reply.started":"2021-05-28T02:23:09.3339Z","shell.execute_reply":"2021-05-28T02:23:09.341135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = plt.bar(n_boxes.index, n_boxes.values)\n\nfor x, y in zip(n_boxes.index, n_boxes.values):\n    plt.text(x, y+50, f'{y/len(train_image)*100:.3}%', fontsize=14)\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:09.583142Z","iopub.execute_input":"2021-05-28T02:23:09.583518Z","iopub.status.idle":"2021-05-28T02:23:09.802671Z","shell.execute_reply.started":"2021-05-28T02:23:09.583488Z","shell.execute_reply":"2021-05-28T02:23:09.801641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Most images have <= 2 bboxes. ","metadata":{}},{"cell_type":"code","source":"train_image['label_only'] = train_image['label'].apply(lambda x: x.split()[0])","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:10.061472Z","iopub.execute_input":"2021-05-28T02:23:10.061885Z","iopub.status.idle":"2021-05-28T02:23:10.074929Z","shell.execute_reply.started":"2021-05-28T02:23:10.061848Z","shell.execute_reply":"2021-05-28T02:23:10.073889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = train_image['label_only'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:10.298692Z","iopub.execute_input":"2021-05-28T02:23:10.299082Z","iopub.status.idle":"2021-05-28T02:23:10.306276Z","shell.execute_reply.started":"2021-05-28T02:23:10.299048Z","shell.execute_reply":"2021-05-28T02:23:10.305187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = plt.bar(labels.index, labels.values)\n\nfor x, y in zip(labels.index, labels.values):\n    plt.text(x, y+50, f'{y/len(train_study)*100:.3}%', fontsize=14)\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:10.533962Z","iopub.execute_input":"2021-05-28T02:23:10.534349Z","iopub.status.idle":"2021-05-28T02:23:10.698588Z","shell.execute_reply.started":"2021-05-28T02:23:10.534319Z","shell.execute_reply":"2021-05-28T02:23:10.697468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"70.9% of the images have label opacity.","metadata":{}},{"cell_type":"code","source":"train_study.columns = ['StudyInstanceUID', 'Negative for Pneumonia', 'Typical Appearance',\n       'Indeterminate Appearance', 'Atypical Appearance']","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:10.990706Z","iopub.execute_input":"2021-05-28T02:23:10.991263Z","iopub.status.idle":"2021-05-28T02:23:10.994798Z","shell.execute_reply.started":"2021-05-28T02:23:10.991227Z","shell.execute_reply":"2021-05-28T02:23:10.994077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_image['StudyInstanceUID'] = train_image['StudyInstanceUID'].apply(lambda x: f'{x}_study')","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:11.218781Z","iopub.execute_input":"2021-05-28T02:23:11.219801Z","iopub.status.idle":"2021-05-28T02:23:11.228719Z","shell.execute_reply.started":"2021-05-28T02:23:11.219754Z","shell.execute_reply":"2021-05-28T02:23:11.227779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's merge the dataframes. ","metadata":{}},{"cell_type":"code","source":"df = train_image.merge(train_study, on='StudyInstanceUID')","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:11.67047Z","iopub.execute_input":"2021-05-28T02:23:11.670826Z","iopub.status.idle":"2021-05-28T02:23:11.689051Z","shell.execute_reply.started":"2021-05-28T02:23:11.670798Z","shell.execute_reply":"2021-05-28T02:23:11.687979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:11.893395Z","iopub.execute_input":"2021-05-28T02:23:11.894819Z","iopub.status.idle":"2021-05-28T02:23:11.910396Z","shell.execute_reply.started":"2021-05-28T02:23:11.89476Z","shell.execute_reply":"2021-05-28T02:23:11.909454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# DICOM Metadata","metadata":{}},{"cell_type":"markdown","source":"The train images are stored in this format\n```\n--root [../input/siim-covid19-detection/train]\n    |--StudyID01\n        |--ImageID01\n        |--ImageID02\n    |--StudyID02\n        |--ImageID01\n        |--ImageID02\n```","metadata":{}},{"cell_type":"code","source":"dcm_fns = get_dicom_files(path_train_image)","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:12.82611Z","iopub.execute_input":"2021-05-28T02:23:12.82671Z","iopub.status.idle":"2021-05-28T02:23:38.906776Z","shell.execute_reply.started":"2021-05-28T02:23:12.826672Z","shell.execute_reply":"2021-05-28T02:23:38.905479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"samp = dcm_fns[0].dcmread()","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:38.908025Z","iopub.execute_input":"2021-05-28T02:23:38.908306Z","iopub.status.idle":"2021-05-28T02:23:38.954303Z","shell.execute_reply.started":"2021-05-28T02:23:38.908277Z","shell.execute_reply":"2021-05-28T02:23:38.953391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"samp","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:38.956039Z","iopub.execute_input":"2021-05-28T02:23:38.956346Z","iopub.status.idle":"2021-05-28T02:23:39.513169Z","shell.execute_reply.started":"2021-05-28T02:23:38.956319Z","shell.execute_reply":"2021-05-28T02:23:39.512085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"MONOCHROME1\nPixel data represent a single monochrome image plane. The minimum sample value is intended to be displayed as white after any VOI gray scale transformations have been performed. \n\nMONOCHROME2\nPixel data represent a single monochrome image plane. The minimum sample value is intended to be displayed as black after any VOI gray scale transformations have been performed.","metadata":{}},{"cell_type":"markdown","source":"Let's create dicom metadata df. This takes a while hence I generated and uploaded the csv.","metadata":{}},{"cell_type":"code","source":"#to generate dicom metadata for train images\n#df_dcm = pd.DataFrame.from_dicoms(dcm_fns, px_summ=True)\n#df_dcm.to_csv('COVID_dcm_metadata.csv')","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:39.514813Z","iopub.execute_input":"2021-05-28T02:23:39.51509Z","iopub.status.idle":"2021-05-28T02:23:39.51852Z","shell.execute_reply.started":"2021-05-28T02:23:39.515064Z","shell.execute_reply":"2021-05-28T02:23:39.517419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#to generate dicom metadata for test images\n#dcm_test_fns = get_dicom_files(path_test_image)\n#df_dcm_test = pd.DataFrame.from_dicoms(dcm_test_fns, px_summ=True)\n#df_dcm_test.to_csv('COVID_dcm_metadata_test.csv')","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:39.519838Z","iopub.execute_input":"2021-05-28T02:23:39.520131Z","iopub.status.idle":"2021-05-28T02:23:39.53167Z","shell.execute_reply.started":"2021-05-28T02:23:39.520102Z","shell.execute_reply":"2021-05-28T02:23:39.530813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_dcm = pd.read_csv('../input/covid-dataframes/COVID_dcm_metadata.csv')\ndf_dcm_test = pd.read_csv('../input/covid-dataframes/COVID_dcm_metadata_test.csv')","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:39.532969Z","iopub.execute_input":"2021-05-28T02:23:39.533386Z","iopub.status.idle":"2021-05-28T02:23:39.898993Z","shell.execute_reply.started":"2021-05-28T02:23:39.533347Z","shell.execute_reply":"2021-05-28T02:23:39.897909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_dcm.columns","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:39.900411Z","iopub.execute_input":"2021-05-28T02:23:39.900984Z","iopub.status.idle":"2021-05-28T02:23:39.907203Z","shell.execute_reply.started":"2021-05-28T02:23:39.90095Z","shell.execute_reply":"2021-05-28T02:23:39.906352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's look at the modality\nprint(f\"{df_dcm['Modality'].value_counts()}\\n\\n{df_dcm_test['Modality'].value_counts()}\")","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:39.90948Z","iopub.execute_input":"2021-05-28T02:23:39.909796Z","iopub.status.idle":"2021-05-28T02:23:39.925845Z","shell.execute_reply.started":"2021-05-28T02:23:39.909768Z","shell.execute_reply":"2021-05-28T02:23:39.924564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"{df_dcm['PatientSex'].value_counts()}\\n\\n{df_dcm_test['PatientSex'].value_counts()}\")","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:39.927737Z","iopub.execute_input":"2021-05-28T02:23:39.928137Z","iopub.status.idle":"2021-05-28T02:23:39.942125Z","shell.execute_reply.started":"2021-05-28T02:23:39.928096Z","shell.execute_reply":"2021-05-28T02:23:39.941095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"{df_dcm['PhotometricInterpretation'].value_counts()}\\n\\n{df_dcm_test['PhotometricInterpretation'].value_counts()}\")","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:39.944378Z","iopub.execute_input":"2021-05-28T02:23:39.944735Z","iopub.status.idle":"2021-05-28T02:23:39.96864Z","shell.execute_reply.started":"2021-05-28T02:23:39.944702Z","shell.execute_reply":"2021-05-28T02:23:39.967325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We have to consider this when we are preparing the dataset.","metadata":{}},{"cell_type":"code","source":"print(f\"{df_dcm['PixelRepresentation'].value_counts()}\\n\\n{df_dcm_test['PixelRepresentation'].value_counts()}\")","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:39.97017Z","iopub.execute_input":"2021-05-28T02:23:39.970657Z","iopub.status.idle":"2021-05-28T02:23:39.978998Z","shell.execute_reply.started":"2021-05-28T02:23:39.970612Z","shell.execute_reply":"2021-05-28T02:23:39.977928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"All dicom are `unsigned`. ","metadata":{}},{"cell_type":"code","source":"df_dcm['BitsAllocated'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:39.980661Z","iopub.execute_input":"2021-05-28T02:23:39.98107Z","iopub.status.idle":"2021-05-28T02:23:39.99345Z","shell.execute_reply.started":"2021-05-28T02:23:39.981026Z","shell.execute_reply":"2021-05-28T02:23:39.992694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Some are 16bits while others are 8bits","metadata":{}},{"cell_type":"code","source":"#the following codes are from https://www.kaggle.com/tanlikesmath/siim-covid-19-detection-a-simple-eda\n\ndef dicom2array(path, voi_lut=True, fix_monochrome=True):\n    dicom = pydicom.read_file(path)\n    # VOI LUT (if available by DICOM device) is used to\n    # transform raw DICOM data to \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    return data\n        \n    \ndef plot_img(img, size=(7, 7), is_rgb=True, title=\"\", cmap='gray'):\n    plt.figure(figsize=size)\n    plt.imshow(img, cmap=cmap)\n    plt.suptitle(title)\n    plt.grid(False)\n    plt.axis('off')\n    plt.show()\n\n\ndef plot_imgs(imgs, cols=3, size=7, is_rgb=True, title=\"\", cmap='gray', img_size=(500,500), label=[]):\n    rows = len(imgs)//cols + 1\n    fig = plt.figure(figsize=(cols*size, rows*size))\n    for i, img in enumerate(imgs):\n        if img_size is not None:\n            img = cv2.resize(img, img_size)\n        fig.add_subplot(rows, cols, i+1)\n        plt.imshow(img, cmap=cmap)\n        plt.title(label[i])\n        plt.grid(False)\n        plt.axis('off')\n\n    plt.suptitle(title)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:39.994667Z","iopub.execute_input":"2021-05-28T02:23:39.994941Z","iopub.status.idle":"2021-05-28T02:23:40.008933Z","shell.execute_reply.started":"2021-05-28T02:23:39.994914Z","shell.execute_reply":"2021-05-28T02:23:40.007756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_img(dicom2array(dcm_fns[0]))","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:40.009925Z","iopub.execute_input":"2021-05-28T02:23:40.0102Z","iopub.status.idle":"2021-05-28T02:23:40.922826Z","shell.execute_reply.started":"2021-05-28T02:23:40.010173Z","shell.execute_reply":"2021-05-28T02:23:40.921901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's take a look at the 9 images from study - `0fd2db233deb`","metadata":{}},{"cell_type":"code","source":"imgs_path = get_dicom_files(path_train_image/'0fd2db233deb')\nimgs_id = [f\"{str(img).split('/')[-1].split('.dcm')[0]}_image\" for img in imgs_path]\nimgs_label = list(df[df['id'].isin(imgs_id)]['label'].apply(lambda x: x.split()[0]).values)","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:40.923838Z","iopub.execute_input":"2021-05-28T02:23:40.924192Z","iopub.status.idle":"2021-05-28T02:23:40.938211Z","shell.execute_reply.started":"2021-05-28T02:23:40.924163Z","shell.execute_reply":"2021-05-28T02:23:40.937289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_imgs([dicom2array(img) for img in imgs_path], label=imgs_label)","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:40.939601Z","iopub.execute_input":"2021-05-28T02:23:40.939966Z","iopub.status.idle":"2021-05-28T02:23:45.075332Z","shell.execute_reply.started":"2021-05-28T02:23:40.939938Z","shell.execute_reply":"2021-05-28T02:23:45.07437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"They all look same to me :(","metadata":{}},{"cell_type":"markdown","source":"# Resize and create Smaller images for prototyping","metadata":{}},{"cell_type":"markdown","source":"The following codes are from this wonderful [notebook](https://www.kaggle.com/konradb/diy-rescaled-images-with-bboxes/output).","metadata":{}},{"cell_type":"code","source":"df.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:45.076696Z","iopub.execute_input":"2021-05-28T02:23:45.077007Z","iopub.status.idle":"2021-05-28T02:23:45.093146Z","shell.execute_reply.started":"2021-05-28T02:23:45.076978Z","shell.execute_reply":"2021-05-28T02:23:45.092191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path_list = []\nimage_list = []\nsplits = []\n\nfor split in ['train']:   \n    for dirname, _, filenames in tqdm(os.walk(f'../input/siim-covid19-detection/{split}')):\n        for file in filenames:\n            fullpath = dirname + '/' + file\n            path_list.append(fullpath)\n            image_list.append(file)\n            \ntemp_df = pd.DataFrame(image_list, columns =['image_id'])\ntemp_df['image_path'] = path_list","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:23:50.811121Z","iopub.execute_input":"2021-05-28T02:23:50.811516Z","iopub.status.idle":"2021-05-28T02:23:55.750738Z","shell.execute_reply.started":"2021-05-28T02:23:50.811479Z","shell.execute_reply":"2021-05-28T02:23:55.749874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"size=512\ntransform = A.Compose(\n    [\n        A.Resize(height = size , width = size, p=1),\n    ], \n    p=1.0,  bbox_params=A.BboxParams( format='pascal_voc', min_area=0,  min_visibility=0, label_fields=['labels']  ))        ","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:24:02.387464Z","iopub.execute_input":"2021-05-28T02:24:02.387829Z","iopub.status.idle":"2021-05-28T02:24:02.392938Z","shell.execute_reply.started":"2021-05-28T02:24:02.387801Z","shell.execute_reply":"2021-05-28T02:24:02.391982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['image_id']  = df['id'].apply(lambda s: s.replace('_image','') + '.dcm')\ndf = pd.merge(left = df, right = temp_df, on = 'image_id')","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:24:06.185311Z","iopub.execute_input":"2021-05-28T02:24:06.185703Z","iopub.status.idle":"2021-05-28T02:24:06.205988Z","shell.execute_reply.started":"2021-05-28T02:24:06.185667Z","shell.execute_reply":"2021-05-28T02:24:06.205184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir train512","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:24:08.783743Z","iopub.execute_input":"2021-05-28T02:24:08.784285Z","iopub.status.idle":"2021-05-28T02:24:09.519065Z","shell.execute_reply.started":"2021-05-28T02:24:08.78425Z","shell.execute_reply":"2021-05-28T02:24:09.517873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"OUTPUT_DIRECTORY = Path('./train512')","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:24:09.521057Z","iopub.execute_input":"2021-05-28T02:24:09.521703Z","iopub.status.idle":"2021-05-28T02:24:09.526542Z","shell.execute_reply.started":"2021-05-28T02:24:09.521652Z","shell.execute_reply":"2021-05-28T02:24:09.525524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_list = []\nlabel_list = []\n\n# loop over files\nfor ii in tqdm(range(len(df)), total=len(df)):\n    # get the image\n    row = df.loc[ii]\n    img_path = row['image_path']\n    img = dicom2array(path=img_path)\n    newname = img_path.split('/')[-1].replace('dcm', 'jpg')\n    img_list.append(newname)\n    \n    # get the bounding boxes\n    bboxes = []\n    bbox = []\n    labels = []\n    confidences = []\n\n    for i, l in enumerate(row['label'].split(' ')):\n        if (i % 6 == 0) :\n            labels.append(l)\n        if (i % 6 == 1):\n            confidences.append(l)\n        if (i % 6 > 1):\n            bbox.append(np.clip(float(l), a_min = 0, a_max = None ))\n        if i % 6 == 5:\n            bboxes.append(bbox)\n            bbox = []    \n\n    # transform both\n    result = transform(image = img, bboxes = bboxes, labels = np.ones(len(bboxes)))\n    new_image = result['image']\n    new_bboxes = np.array(result['bboxes']).tolist()\n\n    # format the output\n    # print('orig label: ' + row['label'])\n    newlabel = ''\n    if labels[0] == 'none':\n        newlabel = 'none 1 0 0 1 1'\n    else:\n        for j in range(len(labels)):\n            newlabel += labels[j] + ' ' + confidences[j] + ' ' +  ' '.join([str(np.round(f,5)) for f in new_bboxes[j]]) + ' '\n    #print('new label:' + newlabel)\n    label_list.append(newlabel)\n    \n    # store the new image\n    cv2.imwrite(str(OUTPUT_DIRECTORY/newname), new_image)","metadata":{"execution":{"iopub.status.busy":"2021-05-28T02:33:09.70231Z","iopub.execute_input":"2021-05-28T02:33:09.702942Z","iopub.status.idle":"2021-05-28T02:33:13.883266Z","shell.execute_reply.started":"2021-05-28T02:33:09.702907Z","shell.execute_reply":"2021-05-28T02:33:13.882167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# store the new boxes with image_ids\nxmeta = pd.DataFrame(img_list, columns =['image_id'])\nxmeta['label'] = label_list\nxmeta.to_csv('bounding_boxes512.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2021-05-27T10:34:09.905682Z","iopub.execute_input":"2021-05-27T10:34:09.906049Z","iopub.status.idle":"2021-05-27T10:34:09.935293Z","shell.execute_reply.started":"2021-05-27T10:34:09.906017Z","shell.execute_reply":"2021-05-27T10:34:09.934412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# wrap it up\n!zip -rm -qq rescaled_with_bb512.zip train512 bounding_boxes_512.csv","metadata":{"execution":{"iopub.status.busy":"2021-05-27T10:34:12.286115Z","iopub.execute_input":"2021-05-27T10:34:12.286511Z","iopub.status.idle":"2021-05-27T10:34:18.070759Z","shell.execute_reply.started":"2021-05-27T10:34:12.286477Z","shell.execute_reply":"2021-05-27T10:34:18.069533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}