{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!conda install '/kaggle/input/pydicom-conda-helper/gdcm-2.8.9-py37h500ead1_1.tar.bz2' -c conda-forge -y","metadata":{"execution":{"iopub.status.busy":"2021-06-02T15:19:21.274314Z","iopub.execute_input":"2021-06-02T15:19:21.274832Z","iopub.status.idle":"2021-06-02T15:19:52.128376Z","shell.execute_reply.started":"2021-06-02T15:19:21.274730Z","shell.execute_reply":"2021-06-02T15:19:52.126740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport pydicom\nfrom tqdm.notebook import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-02T15:19:52.133495Z","iopub.execute_input":"2021-06-02T15:19:52.133938Z","iopub.status.idle":"2021-06-02T15:19:52.493662Z","shell.execute_reply.started":"2021-06-02T15:19:52.133891Z","shell.execute_reply":"2021-06-02T15:19:52.492186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"You can also use the outputs of this notebook as data to import instead of running the code yourself.<br>\nMake sure to use .loc[ ] when indexing custom indexes :)<br>\nHope you find something useful, have fun!","metadata":{}},{"cell_type":"code","source":"#Train image shapes and paths\nimages = []\nwidths = []\nheights = []\npaths = []\n\ntq = tqdm()\n\nfor dirname, _, filenames in os.walk('../input/siim-covid19-detection/train'):\n    for filename in filenames:\n        dicom = pydicom.read_file(dirname+\"/\"+filename,specific_tags=[\"Rows\",\"Columns\"])\n        heights.append(dicom.Rows)\n        widths.append(dicom.Columns)\n        images.append(filename[:-4])\n        paths.append(dirname+\"/\"+filename)\n        tq.update(1)\n\nid2path_train = pd.DataFrame({\"Image_id\":images,\"Path\":paths})    \nimg_shapes_train = pd.DataFrame({\"Image_id\":images,\"Width\":widths,\"Height\":heights}).set_index(\"Image_id\")\nimg_shapes_train","metadata":{"execution":{"iopub.status.busy":"2021-06-02T15:19:52.495796Z","iopub.execute_input":"2021-06-02T15:19:52.496205Z","iopub.status.idle":"2021-06-02T15:22:24.000561Z","shell.execute_reply.started":"2021-06-02T15:19:52.496168Z","shell.execute_reply":"2021-06-02T15:22:23.999311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"id2path_train","metadata":{"execution":{"iopub.status.busy":"2021-06-02T15:22:24.003622Z","iopub.execute_input":"2021-06-02T15:22:24.004338Z","iopub.status.idle":"2021-06-02T15:22:24.020328Z","shell.execute_reply.started":"2021-06-02T15:22:24.004275Z","shell.execute_reply":"2021-06-02T15:22:24.019207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Study to Image\ntrain_image_level = pd.read_csv(\"../input/siim-covid19-detection/train_image_level.csv\")\nimage_study_train = pd.DataFrame({\"Image_id\":train_image_level.id.str[:-6],\"Study_id\":train_image_level.StudyInstanceUID})\nimage_study_train","metadata":{"execution":{"iopub.status.busy":"2021-06-02T15:22:24.021875Z","iopub.execute_input":"2021-06-02T15:22:24.022239Z","iopub.status.idle":"2021-06-02T15:22:24.099521Z","shell.execute_reply.started":"2021-06-02T15:22:24.022204Z","shell.execute_reply":"2021-06-02T15:22:24.098601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Numeric Label and removed _study\ntrain_study_level = pd.read_csv(\"../input/siim-covid19-detection/train_study_level.csv\")\ntrain_study_level.id = train_study_level.id.str[:-6]\ntrain_study_level[\"Label\"] = train_study_level.apply(lambda l: np.argmax(l.values[1:]),axis=1)\ntrain_study_level","metadata":{"execution":{"iopub.status.busy":"2021-06-02T15:22:24.100758Z","iopub.execute_input":"2021-06-02T15:22:24.101264Z","iopub.status.idle":"2021-06-02T15:22:24.224609Z","shell.execute_reply.started":"2021-06-02T15:22:24.101227Z","shell.execute_reply":"2021-06-02T15:22:24.223180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Image id to Classes \nimg_classes_train = pd.DataFrame({\"Image_id\":image_study_train.Image_id,\"Label\":train_study_level.set_index(\"id\").Label.loc[image_study_train.Study_id].values})\nimg_classes_train","metadata":{"execution":{"iopub.status.busy":"2021-06-02T15:22:24.226441Z","iopub.execute_input":"2021-06-02T15:22:24.226827Z","iopub.status.idle":"2021-06-02T15:22:24.255438Z","shell.execute_reply.started":"2021-06-02T15:22:24.226778Z","shell.execute_reply":"2021-06-02T15:22:24.253634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Helper function\ndef getBoxes(x):\n    if not pd.isnull(x.boxes):\n        new = []\n        boxes = eval(x.boxes)\n        width = img_shapes_train.loc[x.id[:-6]].Width\n        height = img_shapes_train.loc[x.id[:-6]].Height\n        for box in boxes:\n            new.append({\"x\":max(0,box[\"x\"]/width),\n                        \"y\":max(0,box[\"y\"]/height),\n                        \"width\":max(0,box[\"width\"]/width),\n                        \"height\":max(0,box[\"height\"]/height)})# There is one value below 0 --> max(0,value)\n        return new\n    else:\n        return ","metadata":{"execution":{"iopub.status.busy":"2021-06-02T15:22:24.258700Z","iopub.execute_input":"2021-06-02T15:22:24.259157Z","iopub.status.idle":"2021-06-02T15:22:24.272092Z","shell.execute_reply.started":"2021-06-02T15:22:24.259116Z","shell.execute_reply":"2021-06-02T15:22:24.270968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Normalized Boxes\n#Use eval() to turn the string into a list\nimg_boxes_train =  pd.DataFrame({\"Image_id\":train_image_level.id.str[:-6], \"Boxes\":train_image_level.apply(getBoxes,axis=1),\"has_box\":~pd.isnull(train_image_level.boxes)})\nimg_boxes_train","metadata":{"execution":{"iopub.status.busy":"2021-06-02T15:22:24.274205Z","iopub.execute_input":"2021-06-02T15:22:24.274638Z","iopub.status.idle":"2021-06-02T15:22:26.267758Z","shell.execute_reply.started":"2021-06-02T15:22:24.274600Z","shell.execute_reply":"2021-06-02T15:22:26.266720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Test set paths, shapes and image to study\nstudies = []\nimages = []\npaths = []\nwidths = []\nheights = []\ntq = tqdm()\n\nfor dirname, _, filenames in os.walk('../input/siim-covid19-detection/test'):\n    for filename in filenames:\n        dicom = pydicom.read_file(dirname+\"/\"+filename,specific_tags=[\"Rows\",\"Columns\"])\n        heights.append(dicom.Rows)\n        widths.append(dicom.Columns)\n        studies.append(dirname.split(\"/\")[-2])\n        images.append(filename[:-4])\n        paths.append(dirname+\"/\"+filename)\n        tq.update(1)\n\nid2path_test = pd.DataFrame({\"Image_id\":images,\"Path\":paths})\nimg_study_test = pd.DataFrame({\"Study_id\":studies,\"Image_id\":images})    \nimg_shapes_test = pd.DataFrame({\"Image_id\":images,\"Width\":widths,\"Height\":heights}).set_index(\"Image_id\")","metadata":{"execution":{"iopub.status.busy":"2021-06-02T15:22:26.269633Z","iopub.execute_input":"2021-06-02T15:22:26.270431Z","iopub.status.idle":"2021-06-02T15:22:56.167135Z","shell.execute_reply.started":"2021-06-02T15:22:26.270375Z","shell.execute_reply":"2021-06-02T15:22:56.165997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_study_test","metadata":{"execution":{"iopub.status.busy":"2021-06-02T15:22:56.168807Z","iopub.execute_input":"2021-06-02T15:22:56.169303Z","iopub.status.idle":"2021-06-02T15:22:56.187175Z","shell.execute_reply.started":"2021-06-02T15:22:56.169258Z","shell.execute_reply":"2021-06-02T15:22:56.185641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_shapes_test","metadata":{"execution":{"iopub.status.busy":"2021-06-02T15:22:56.189086Z","iopub.execute_input":"2021-06-02T15:22:56.189584Z","iopub.status.idle":"2021-06-02T15:22:56.210889Z","shell.execute_reply.started":"2021-06-02T15:22:56.189533Z","shell.execute_reply":"2021-06-02T15:22:56.209742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"id2path_test","metadata":{"execution":{"iopub.status.busy":"2021-06-02T15:22:56.212421Z","iopub.execute_input":"2021-06-02T15:22:56.213092Z","iopub.status.idle":"2021-06-02T15:22:56.234572Z","shell.execute_reply.started":"2021-06-02T15:22:56.213038Z","shell.execute_reply":"2021-06-02T15:22:56.233412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Output\nid2path_train.to_csv(\"id2path_train.csv\",index=False)\nimage_study_train.to_csv(\"image_study_train.csv\",index=False)\nimg_shapes_train.to_csv(\"img_shapes_train.csv\")\n\nimg_classes_train.to_csv(\"img_classes_train.csv\",index=False)\nimg_boxes_train.to_csv(\"img_boxes_train.csv\",index=False)\n\nid2path_test.to_csv(\"id2path_test.csv\",index=False)\nimg_study_test.to_csv(\"img_study_test.csv\",index=False)\nimg_shapes_test.to_csv(\"img_shapes_test.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-06-02T15:25:39.086959Z","iopub.execute_input":"2021-06-02T15:25:39.087431Z","iopub.status.idle":"2021-06-02T15:25:39.276995Z","shell.execute_reply.started":"2021-06-02T15:25:39.087395Z","shell.execute_reply":"2021-06-02T15:25:39.275882Z"},"trusted":true},"execution_count":null,"outputs":[]}]}