{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"In this notebook I created a .csv file, which consists of 10 columns: labels for the type of disease,the image id, the path of the .dcm file in the file system, the coordinates and size of the bounding boxes.The entries in the table are repeated because there can be several bounding boxes for one picture. Coordinates of bounding boxes are calculated for the size of pictures 512х512. You can change the coordinates and sizes of the bounding boxes by making elementary proportions :)","metadata":{}},{"cell_type":"code","source":"import pydicom\nimport glob\nimport pandas as pd\nimport ast\nimport os","metadata":{"execution":{"iopub.status.busy":"2021-06-30T16:28:18.520988Z","iopub.execute_input":"2021-06-30T16:28:18.521687Z","iopub.status.idle":"2021-06-30T16:28:18.526878Z","shell.execute_reply.started":"2021-06-30T16:28:18.521645Z","shell.execute_reply":"2021-06-30T16:28:18.525996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image = pd.read_csv('../input/siim-covid19-detection/train_image_level.csv')\nimage['StudyInstanceUID'] = image['StudyInstanceUID'] + '_study'\nimage","metadata":{"execution":{"iopub.status.busy":"2021-06-30T16:28:18.533559Z","iopub.execute_input":"2021-06-30T16:28:18.534485Z","iopub.status.idle":"2021-06-30T16:28:18.614845Z","shell.execute_reply.started":"2021-06-30T16:28:18.534436Z","shell.execute_reply":"2021-06-30T16:28:18.613181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study = pd.read_csv('../input/siim-covid19-detection/train_study_level.csv')\nstudy = study.set_index(['id'])\nstudy","metadata":{"execution":{"iopub.status.busy":"2021-06-30T16:28:18.617192Z","iopub.execute_input":"2021-06-30T16:28:18.617668Z","iopub.status.idle":"2021-06-30T16:28:18.653599Z","shell.execute_reply.started":"2021-06-30T16:28:18.617618Z","shell.execute_reply":"2021-06-30T16:28:18.652239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = study.loc[image['StudyInstanceUID'].to_numpy()].reset_index()\nlabels","metadata":{"execution":{"iopub.status.busy":"2021-06-30T16:28:18.656414Z","iopub.execute_input":"2021-06-30T16:28:18.656884Z","iopub.status.idle":"2021-06-30T16:28:18.689549Z","shell.execute_reply.started":"2021-06-30T16:28:18.656834Z","shell.execute_reply":"2021-06-30T16:28:18.688337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.merge(image.rename(columns={'id': 'image_id'}), labels, how='inner', left_on='StudyInstanceUID', right_on='id')\ndf = df.drop_duplicates()\ndf = df.reset_index(drop=True)\ndf = df.drop(['StudyInstanceUID', 'label', 'id'], axis=1)\ndf['image_id'] = df['image_id'].apply(lambda x: x[:-6])\ndf","metadata":{"execution":{"iopub.status.busy":"2021-06-30T16:28:18.691708Z","iopub.execute_input":"2021-06-30T16:28:18.692356Z","iopub.status.idle":"2021-06-30T16:28:18.741185Z","shell.execute_reply.started":"2021-06-30T16:28:18.692304Z","shell.execute_reply":"2021-06-30T16:28:18.740114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"paths = glob.glob('../input/siim-covid19-detection/train/*/*/*.dcm')\n\nimage_idx = [os.path.split(path)[1][:-4] for path in paths]\nimage_idx[:5]","metadata":{"execution":{"iopub.status.busy":"2021-06-30T16:28:18.742656Z","iopub.execute_input":"2021-06-30T16:28:18.743283Z","iopub.status.idle":"2021-06-30T16:28:46.381652Z","shell.execute_reply.started":"2021-06-30T16:28:18.743236Z","shell.execute_reply":"2021-06-30T16:28:46.380560Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"paths_df = pd.DataFrame({'image_id': image_idx, 'Path': paths})\npaths_df","metadata":{"execution":{"iopub.status.busy":"2021-06-30T16:28:46.383376Z","iopub.execute_input":"2021-06-30T16:28:46.383678Z","iopub.status.idle":"2021-06-30T16:28:46.402729Z","shell.execute_reply.started":"2021-06-30T16:28:46.383649Z","shell.execute_reply":"2021-06-30T16:28:46.401705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = df.merge(paths_df, left_on='image_id', right_on='image_id')\ndata","metadata":{"execution":{"iopub.status.busy":"2021-06-30T16:28:46.404309Z","iopub.execute_input":"2021-06-30T16:28:46.404609Z","iopub.status.idle":"2021-06-30T16:28:46.433626Z","shell.execute_reply.started":"2021-06-30T16:28:46.404580Z","shell.execute_reply":"2021-06-30T16:28:46.432777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_wh(path):\n    ds = pydicom.dcmread(path)\n    # .Rows = height .Columns = width\n    return {'w': ds.Columns, 'h': ds.Rows}","metadata":{"execution":{"iopub.status.busy":"2021-06-30T16:28:46.436036Z","iopub.execute_input":"2021-06-30T16:28:46.436604Z","iopub.status.idle":"2021-06-30T16:28:46.441432Z","shell.execute_reply.started":"2021-06-30T16:28:46.436551Z","shell.execute_reply":"2021-06-30T16:28:46.440394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# this cell is executed for a long time\nwh = pd.DataFrame(list(data['Path'].apply(get_wh)))\nwh","metadata":{"execution":{"iopub.status.busy":"2021-06-30T16:28:46.443157Z","iopub.execute_input":"2021-06-30T16:28:46.443610Z","iopub.status.idle":"2021-06-30T16:29:16.429620Z","shell.execute_reply.started":"2021-06-30T16:28:46.443562Z","shell.execute_reply":"2021-06-30T16:29:16.427581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data[['w', 'h']] = wh\ndata","metadata":{"execution":{"iopub.status.busy":"2021-06-30T16:29:19.040476Z","iopub.execute_input":"2021-06-30T16:29:19.040918Z","iopub.status.idle":"2021-06-30T16:29:19.073045Z","shell.execute_reply.started":"2021-06-30T16:29:19.040883Z","shell.execute_reply":"2021-06-30T16:29:19.072010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data['boxes'] = data['boxes'].apply(lambda x: ast.literal_eval(x) if not pd.isna(x) else x)\ndata = data.explode('boxes').reset_index(drop=True)\ndata","metadata":{"execution":{"iopub.status.busy":"2021-06-30T16:29:19.931537Z","iopub.execute_input":"2021-06-30T16:29:19.932112Z","iopub.status.idle":"2021-06-30T16:29:20.167290Z","shell.execute_reply.started":"2021-06-30T16:29:19.932062Z","shell.execute_reply":"2021-06-30T16:29:20.166169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.loc[data['boxes'].isna(), 'boxes'] = str(dict(x=0, y=0, width=0, height=0))\ndata","metadata":{"execution":{"iopub.status.busy":"2021-06-30T16:29:21.642937Z","iopub.execute_input":"2021-06-30T16:29:21.643330Z","iopub.status.idle":"2021-06-30T16:29:21.676780Z","shell.execute_reply.started":"2021-06-30T16:29:21.643298Z","shell.execute_reply":"2021-06-30T16:29:21.675495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def str_to_dict(s):\n    if type(s) is str:\n        s = ast.literal_eval(s)\n    return s","metadata":{"execution":{"iopub.status.busy":"2021-06-30T16:29:23.442529Z","iopub.execute_input":"2021-06-30T16:29:23.443329Z","iopub.status.idle":"2021-06-30T16:29:23.449197Z","shell.execute_reply.started":"2021-06-30T16:29:23.443288Z","shell.execute_reply":"2021-06-30T16:29:23.447643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data['boxes'] = data['boxes'].apply(str_to_dict)","metadata":{"execution":{"iopub.status.busy":"2021-06-30T16:29:23.769801Z","iopub.execute_input":"2021-06-30T16:29:23.770309Z","iopub.status.idle":"2021-06-30T16:29:23.829006Z","shell.execute_reply.started":"2021-06-30T16:29:23.770266Z","shell.execute_reply":"2021-06-30T16:29:23.827479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xywh = pd.DataFrame(list(data['boxes']))\nxywh","metadata":{"execution":{"iopub.status.busy":"2021-06-30T16:29:25.252668Z","iopub.execute_input":"2021-06-30T16:29:25.253062Z","iopub.status.idle":"2021-06-30T16:29:25.294483Z","shell.execute_reply.started":"2021-06-30T16:29:25.253029Z","shell.execute_reply":"2021-06-30T16:29:25.293057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data[['x', 'y', 'width', 'height']] = xywh\ndata","metadata":{"execution":{"iopub.status.busy":"2021-06-30T16:29:25.362968Z","iopub.execute_input":"2021-06-30T16:29:25.363331Z","iopub.status.idle":"2021-06-30T16:29:25.398070Z","shell.execute_reply.started":"2021-06-30T16:29:25.363300Z","shell.execute_reply":"2021-06-30T16:29:25.397025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data['x'] = data['x'] / data['w'] * 512\ndata['y'] = data['y'] / data['h'] * 512\ndata['width'] = data['width'] / data['w'] * 512\ndata['height'] = data['height'] / data['h'] * 512\ndata","metadata":{"execution":{"iopub.status.busy":"2021-06-30T16:29:26.277256Z","iopub.execute_input":"2021-06-30T16:29:26.277738Z","iopub.status.idle":"2021-06-30T16:29:26.311981Z","shell.execute_reply.started":"2021-06-30T16:29:26.277706Z","shell.execute_reply":"2021-06-30T16:29:26.310745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_to_save = data.drop(['boxes', 'w', 'h'], axis=1).rename(columns={'width': 'w', 'height': 'h'})\ndf_to_save","metadata":{"execution":{"iopub.status.busy":"2021-06-30T16:29:28.661115Z","iopub.execute_input":"2021-06-30T16:29:28.661534Z","iopub.status.idle":"2021-06-30T16:29:28.687359Z","shell.execute_reply.started":"2021-06-30T16:29:28.661495Z","shell.execute_reply":"2021-06-30T16:29:28.686246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_to_save.to_csv('train.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-30T16:29:28.762225Z","iopub.execute_input":"2021-06-30T16:29:28.762570Z","iopub.status.idle":"2021-06-30T16:29:28.896961Z","shell.execute_reply.started":"2021-06-30T16:29:28.762540Z","shell.execute_reply":"2021-06-30T16:29:28.895826Z"},"trusted":true},"execution_count":null,"outputs":[]}]}