{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-22T10:29:43.291962Z","iopub.execute_input":"2021-06-22T10:29:43.292394Z","iopub.status.idle":"2021-06-22T10:29:43.302124Z","shell.execute_reply.started":"2021-06-22T10:29:43.292333Z","shell.execute_reply":"2021-06-22T10:29:43.300982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install glob2","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install pydicom kornia opencv-python scikit-image nbdev","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!conda install -c conda-forge gdcm -y","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_df = pd.read_csv('../input/siim-covid19-detection/train_image_level.csv')\nstudy_df = pd.read_csv('../input/siim-covid19-detection/train_study_level.csv')\nsub_df = pd.read_csv('../input/siim-covid19-detection/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:29:54.035088Z","iopub.execute_input":"2021-06-22T10:29:54.035460Z","iopub.status.idle":"2021-06-22T10:29:54.116033Z","shell.execute_reply.started":"2021-06-22T10:29:54.035430Z","shell.execute_reply":"2021-06-22T10:29:54.115021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(image_df.shape)\nprint(study_df.shape)\nprint(sub_df.shape)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:29:54.895536Z","iopub.execute_input":"2021-06-22T10:29:54.895946Z","iopub.status.idle":"2021-06-22T10:29:54.902460Z","shell.execute_reply.started":"2021-06-22T10:29:54.895914Z","shell.execute_reply":"2021-06-22T10:29:54.901133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:29:55.551745Z","iopub.execute_input":"2021-06-22T10:29:55.552128Z","iopub.status.idle":"2021-06-22T10:29:55.580283Z","shell.execute_reply.started":"2021-06-22T10:29:55.552096Z","shell.execute_reply":"2021-06-22T10:29:55.579171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_df['StudyInstanceUID'] = [x + '_study' for x in image_df['StudyInstanceUID']]\nimage_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:29:57.947431Z","iopub.execute_input":"2021-06-22T10:29:57.948021Z","iopub.status.idle":"2021-06-22T10:29:57.966727Z","shell.execute_reply.started":"2021-06-22T10:29:57.947984Z","shell.execute_reply":"2021-06-22T10:29:57.965422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:30:01.019685Z","iopub.execute_input":"2021-06-22T10:30:01.020223Z","iopub.status.idle":"2021-06-22T10:30:01.035057Z","shell.execute_reply.started":"2021-06-22T10:30:01.020170Z","shell.execute_reply":"2021-06-22T10:30:01.033482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_df = study_df.rename(columns={'id': 'StudyInstanceUID'})\nstudy_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:30:03.108975Z","iopub.execute_input":"2021-06-22T10:30:03.109384Z","iopub.status.idle":"2021-06-22T10:30:03.125943Z","shell.execute_reply.started":"2021-06-22T10:30:03.109349Z","shell.execute_reply":"2021-06-22T10:30:03.124188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(image_df.StudyInstanceUID.tolist()))\nprint(len(study_df.StudyInstanceUID.tolist()))","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:30:05.727832Z","iopub.execute_input":"2021-06-22T10:30:05.728233Z","iopub.status.idle":"2021-06-22T10:30:05.733967Z","shell.execute_reply.started":"2021-06-22T10:30:05.728203Z","shell.execute_reply":"2021-06-22T10:30:05.732953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merger = pd.merge(image_df,study_df,on='StudyInstanceUID')\nmerger.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:30:14.406576Z","iopub.execute_input":"2021-06-22T10:30:14.407010Z","iopub.status.idle":"2021-06-22T10:30:14.445211Z","shell.execute_reply.started":"2021-06-22T10:30:14.406972Z","shell.execute_reply":"2021-06-22T10:30:14.443522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merger.StudyInstanceUID.value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:30:23.534796Z","iopub.execute_input":"2021-06-22T10:30:23.535160Z","iopub.status.idle":"2021-06-22T10:30:23.557927Z","shell.execute_reply.started":"2021-06-22T10:30:23.535126Z","shell.execute_reply":"2021-06-22T10:30:23.556457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count = merger.StudyInstanceUID.value_counts().reset_index()\ncount.columns = ['StudyInstanceUID', 'number of imgs']\ncount = count.sort_values(by = 'number of imgs', ascending=False)\ncount.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:30:29.227382Z","iopub.execute_input":"2021-06-22T10:30:29.227815Z","iopub.status.idle":"2021-06-22T10:30:29.248740Z","shell.execute_reply.started":"2021-06-22T10:30:29.227780Z","shell.execute_reply":"2021-06-22T10:30:29.247672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merger[\"Nb_boxes\"] = merger.boxes.str.split('},')\nmerger['Nb_boxes'] = merger['Nb_boxes'].fillna('0 0 1 1')\nmerger[\"len\"] = [1 if box=='0 0 1 1' else len(box) for box in merger['Nb_boxes'].values]\nmerger.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:30:48.061779Z","iopub.execute_input":"2021-06-22T10:30:48.062183Z","iopub.status.idle":"2021-06-22T10:30:48.100249Z","shell.execute_reply.started":"2021-06-22T10:30:48.062148Z","shell.execute_reply":"2021-06-22T10:30:48.099452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merger['Nb_boxes'][0]","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:30:57.712038Z","iopub.execute_input":"2021-06-22T10:30:57.712638Z","iopub.status.idle":"2021-06-22T10:30:57.719892Z","shell.execute_reply.started":"2021-06-22T10:30:57.712560Z","shell.execute_reply":"2021-06-22T10:30:57.718717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new = merger.loc[merger.index.repeat(merger['len'])]\nnew = merger.explode('Nb_boxes').reset_index(drop=True)\nnew.head()\n","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:31:05.502701Z","iopub.execute_input":"2021-06-22T10:31:05.503296Z","iopub.status.idle":"2021-06-22T10:31:05.546600Z","shell.execute_reply.started":"2021-06-22T10:31:05.503237Z","shell.execute_reply":"2021-06-22T10:31:05.545846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#'\\[{|{|}\\]'\nnew['Nb_boxes'] = new['Nb_boxes'].str.replace(\"\\[{|{|}\\]|'x':||'y':|'width':|'height':|,\",'')\nnew.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:31:17.854049Z","iopub.execute_input":"2021-06-22T10:31:17.854655Z","iopub.status.idle":"2021-06-22T10:31:18.025341Z","shell.execute_reply.started":"2021-06-22T10:31:17.854598Z","shell.execute_reply":"2021-06-22T10:31:18.024542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nnew[[\"x\", \"y\",\"h\", \"w\"]] = pd.DataFrame([x.split() for x in new.Nb_boxes.tolist()], index= new.index)\nxmin = new.loc[:, 'x'].astype('float')\nymin = new.loc[:, 'y'].astype('float')\nnew['x'] = new.loc[:, 'x'].astype('float')\nnew['y'] = new.loc[:, 'y'].astype('float')\nnew['xmax'] = xmin + new.loc[:, 'w'].astype('float')\nnew['ymax'] = ymin + new.loc[:, 'h'].astype('float')\nnew.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:31:24.224633Z","iopub.execute_input":"2021-06-22T10:31:24.225261Z","iopub.status.idle":"2021-06-22T10:31:24.301372Z","shell.execute_reply.started":"2021-06-22T10:31:24.225200Z","shell.execute_reply":"2021-06-22T10:31:24.300244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\n\nnew['label'] = new['label'].str.replace(r\"[0-9]|[.]|-\",'')\nnew['label'] = new['label'].str.replace(r'\\b(\\w+)(\\s+\\1)+\\b', r'\\1')\n\nnew.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:31:33.075625Z","iopub.execute_input":"2021-06-22T10:31:33.076002Z","iopub.status.idle":"2021-06-22T10:31:33.310064Z","shell.execute_reply.started":"2021-06-22T10:31:33.075969Z","shell.execute_reply":"2021-06-22T10:31:33.308825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new.label.value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:31:38.870552Z","iopub.execute_input":"2021-06-22T10:31:38.870966Z","iopub.status.idle":"2021-06-22T10:31:38.882702Z","shell.execute_reply.started":"2021-06-22T10:31:38.870934Z","shell.execute_reply":"2021-06-22T10:31:38.881681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new = new.drop(columns=['boxes','Nb_boxes','len'])\nnew.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:31:44.030245Z","iopub.execute_input":"2021-06-22T10:31:44.030641Z","iopub.status.idle":"2021-06-22T10:31:44.056681Z","shell.execute_reply.started":"2021-06-22T10:31:44.030609Z","shell.execute_reply":"2021-06-22T10:31:44.055805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new['id'] = new['id'].str.replace('_image','.dcm')\nnew['StudyInstanceUID'] = new['StudyInstanceUID'].str.replace('_study','')\nnew.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:31:47.391677Z","iopub.execute_input":"2021-06-22T10:31:47.392230Z","iopub.status.idle":"2021-06-22T10:31:47.435363Z","shell.execute_reply.started":"2021-06-22T10:31:47.392193Z","shell.execute_reply":"2021-06-22T10:31:47.434502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new['label'] = new['label'].str.replace(' ','')","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:31:53.759822Z","iopub.execute_input":"2021-06-22T10:31:53.760209Z","iopub.status.idle":"2021-06-22T10:31:53.775220Z","shell.execute_reply.started":"2021-06-22T10:31:53.760178Z","shell.execute_reply":"2021-06-22T10:31:53.773574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob2\n\nimg_path = glob2.glob('../input/siim-covid19-detection/train/**/*.dcm')","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:31:56.501284Z","iopub.execute_input":"2021-06-22T10:31:56.501683Z","iopub.status.idle":"2021-06-22T10:32:31.905531Z","shell.execute_reply.started":"2021-06-22T10:31:56.501648Z","shell.execute_reply":"2021-06-22T10:32:31.904541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new['path'] = [path for file in new['id'].tolist() for path in img_path if file==str(path).split('/')[-1]]\nnew.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:32:47.800690Z","iopub.execute_input":"2021-06-22T10:32:47.801187Z","iopub.status.idle":"2021-06-22T10:33:23.759855Z","shell.execute_reply.started":"2021-06-22T10:32:47.801138Z","shell.execute_reply":"2021-06-22T10:33:23.759062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# save the new csv and add it to your data\nnew.to_csv('siim_final2.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## from fastai.basics import *\nfrom fastai.callback.all import *\nfrom fastai.vision.all import *\nfrom fastai.medical.imaging import *\n\nimport pydicom\nimport gdcm\n\n","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:33:48.511514Z","iopub.execute_input":"2021-06-22T10:33:48.512150Z","iopub.status.idle":"2021-06-22T10:33:51.806967Z","shell.execute_reply.started":"2021-06-22T10:33:48.512094Z","shell.execute_reply":"2021-06-22T10:33:51.805856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('../input/siim-competition/siim_final2.csv')\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:34:04.561083Z","iopub.execute_input":"2021-06-22T10:34:04.561467Z","iopub.status.idle":"2021-06-22T10:34:04.646913Z","shell.execute_reply.started":"2021-06-22T10:34:04.561438Z","shell.execute_reply":"2021-06-22T10:34:04.646048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imgs =[]\nbbox = []\nlbl_box = []\n\nfor item in df['id'].unique():\n    data = df[df['id'] == item]\n    boxes = data[['x','y','xmax','ymax']].values.tolist()\n    lbl = data['label'].values.tolist()\n    imgs.append(item)\n    bbox.append(boxes)\n    lbl_box.append(lbl)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:34:17.899956Z","iopub.execute_input":"2021-06-22T10:34:17.900489Z","iopub.status.idle":"2021-06-22T10:34:35.364201Z","shell.execute_reply.started":"2021-06-22T10:34:17.900452Z","shell.execute_reply":"2021-06-22T10:34:35.362955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imgs[0]","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:34:37.638661Z","iopub.execute_input":"2021-06-22T10:34:37.639020Z","iopub.status.idle":"2021-06-22T10:34:37.644078Z","shell.execute_reply.started":"2021-06-22T10:34:37.638991Z","shell.execute_reply":"2021-06-22T10:34:37.643388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lbl_bbox = list(zip(bbox,lbl_box))\nlbl_bbox[0]","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:34:38.353418Z","iopub.execute_input":"2021-06-22T10:34:38.353994Z","iopub.status.idle":"2021-06-22T10:34:38.362654Z","shell.execute_reply.started":"2021-06-22T10:34:38.353942Z","shell.execute_reply":"2021-06-22T10:34:38.361739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img2bbox = dict(zip(imgs, lbl_bbox))","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:34:40.352882Z","iopub.execute_input":"2021-06-22T10:34:40.353476Z","iopub.status.idle":"2021-06-22T10:34:40.359025Z","shell.execute_reply.started":"2021-06-22T10:34:40.353440Z","shell.execute_reply":"2021-06-22T10:34:40.357787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img2bbox['fed009ec0e3f.dcm'][0]","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:34:41.163785Z","iopub.execute_input":"2021-06-22T10:34:41.164307Z","iopub.status.idle":"2021-06-22T10:34:41.170688Z","shell.execute_reply.started":"2021-06-22T10:34:41.164273Z","shell.execute_reply":"2021-06-22T10:34:41.169713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"first = {k: img2bbox[k] for k in list(img2bbox)[:1]}; first","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:34:41.972055Z","iopub.execute_input":"2021-06-22T10:34:41.972774Z","iopub.status.idle":"2021-06-22T10:34:41.979981Z","shell.execute_reply.started":"2021-06-22T10:34:41.972734Z","shell.execute_reply":"2021-06-22T10:34:41.978949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class PILDicom(PILBase):\n    _open_args,_tensor_cls,_show_args = {},TensorDicom,TensorDicom._show_args\n    @classmethod\n    def create(cls, fn:(Path,str,bytes), mode='RGB')->None:\n        \"Open a `DICOM file` from path `fn` or bytes `fn` and load it as a `PIL Image`\"\n        if isinstance(fn,bytes): im = Image.fromarray(pydicom.dcmread(pydicom.filebase.DicomBytesIO(fn)).pixel_array)\n        if isinstance(fn,(Path,str)): im = pydicom.dcmread(fn).pixel_array\n        im = (im / np.max(im)) * 255\n        #im = (im.windowed(w=40, l=80))*255\n        im = Image.fromarray(im.astype(np.uint8))\n        #im = Image.fromarray(im)\n        im.load()\n        im = im._new(im.im)\n        return cls(im.convert(mode) if mode else im)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:35:08.069432Z","iopub.execute_input":"2021-06-22T10:35:08.070098Z","iopub.status.idle":"2021-06-22T10:35:08.079936Z","shell.execute_reply.started":"2021-06-22T10:35:08.070043Z","shell.execute_reply":"2021-06-22T10:35:08.078677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pneumothorax = DataBlock(blocks=(ImageBlock(cls=PILDicom), BBoxBlock,BBoxLblBlock),\n                   get_items=get_dicom_files,\n                   splitter=RandomSplitter(),    \n                   get_y=[lambda o: img2bbox[o.name][0], lambda o: img2bbox[o.name][1]],\n                   item_tfms=Resize(224),\n                   batch_tfms=aug_transforms(),\n                 n_inp=1)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:35:16.510141Z","iopub.execute_input":"2021-06-22T10:35:16.510518Z","iopub.status.idle":"2021-06-22T10:35:16.519088Z","shell.execute_reply.started":"2021-06-22T10:35:16.510487Z","shell.execute_reply":"2021-06-22T10:35:16.518140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls = pneumothorax.dataloaders('../input/siim-covid19-detection/train')\ndls.show_batch(max_n=16)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:35:31.439258Z","iopub.execute_input":"2021-06-22T10:35:31.439856Z","iopub.status.idle":"2021-06-22T10:36:06.757702Z","shell.execute_reply.started":"2021-06-22T10:35:31.439816Z","shell.execute_reply":"2021-06-22T10:36:06.756704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x,y,z = next(iter(dls[0]))","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:36:48.693249Z","iopub.execute_input":"2021-06-22T10:36:48.693684Z","iopub.status.idle":"2021-06-22T10:37:37.416011Z","shell.execute_reply.started":"2021-06-22T10:36:48.693639Z","shell.execute_reply":"2021-06-22T10:37:37.413879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#x,y,z","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:39:19.372055Z","iopub.execute_input":"2021-06-22T10:39:19.372466Z","iopub.status.idle":"2021-06-22T10:39:19.376292Z","shell.execute_reply.started":"2021-06-22T10:39:19.372434Z","shell.execute_reply":"2021-06-22T10:39:19.375417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}