{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**Analysis of the training labels/boxes for the [SIIM COVID-19 Detection](https://www.kaggle.com/c/siim-covid19-detection/overview) competition**.\n\n**Conclusions**:\n- There is no duplicates.\n- 32% of images don't have a box.\n- There can be multiple images per study!\n- There are two recurring areas: one for each lung.","metadata":{}},{"cell_type":"markdown","source":"# CONFS","metadata":{}},{"cell_type":"code","source":"IMAGES = '/kaggle/input/siim-covid19-detection/train_image_level.csv'\nSTUDIES = '/kaggle/input/siim-covid19-detection/train_study_level.csv'","metadata":{"_uuid":"ad69751f-8936-4ea4-83dc-bd1af0957c29","_cell_guid":"a2671d37-c826-4a54-ab9d-4e8374d5b7e3","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-27T13:41:47.572845Z","iopub.execute_input":"2021-06-27T13:41:47.573415Z","iopub.status.idle":"2021-06-27T13:41:47.577662Z","shell.execute_reply.started":"2021-06-27T13:41:47.573366Z","shell.execute_reply":"2021-06-27T13:41:47.576531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# IMPORTS","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport plotly.express as px","metadata":{"execution":{"iopub.status.busy":"2021-06-27T13:41:47.579581Z","iopub.execute_input":"2021-06-27T13:41:47.580079Z","iopub.status.idle":"2021-06-27T13:41:47.592216Z","shell.execute_reply.started":"2021-06-27T13:41:47.579998Z","shell.execute_reply":"2021-06-27T13:41:47.590889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# DATASETS","metadata":{}},{"cell_type":"code","source":"studies = pd.read_csv(STUDIES, index_col='id')\n# assert studies['id'].nunique() == len(studies)\nstudies = studies.idxmax(axis=1).to_frame(name='case')\nstudies.index = studies.index.str.replace('_study', '')\nstudies.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-27T13:41:47.594395Z","iopub.execute_input":"2021-06-27T13:41:47.594715Z","iopub.status.idle":"2021-06-27T13:41:47.646708Z","shell.execute_reply.started":"2021-06-27T13:41:47.594685Z","shell.execute_reply":"2021-06-27T13:41:47.645710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = pd.read_csv(IMAGES, index_col='id')\n# assert labels['id'].nunique() == len(labels)\n# assert set(labels['StudyInstanceUID']) ^ set(studies.index) == set()\nlabels = labels.merge(studies, left_on='StudyInstanceUID', right_index=True)\nlabels.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-27T13:41:47.649286Z","iopub.execute_input":"2021-06-27T13:41:47.649623Z","iopub.status.idle":"2021-06-27T13:41:47.699490Z","shell.execute_reply.started":"2021-06-27T13:41:47.649588Z","shell.execute_reply":"2021-06-27T13:41:47.698214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"boxes = []\nfor i, row in labels.iterrows():\n    value = row['boxes']\n    if isinstance(value, float): # nan\n        print('x', end='')\n        continue\n    values = eval(value)\n    print('.', end='')\n    for val in values:\n        box = {\n            'image': i,\n            'study': row['StudyInstanceUID'],\n            'case': row['case'],\n            'xmin': val['x'],\n            'ymin': val['y'],\n            'xmax': val['x'] + val['width'],\n            'ymax': val['y'] + val['height'],\n            'width': val['width'],\n            'height': val['height'],\n        }\n        boxes.append(box)\nboxes = pd.DataFrame(boxes)\nboxes.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-27T13:41:47.701167Z","iopub.execute_input":"2021-06-27T13:41:47.701548Z","iopub.status.idle":"2021-06-27T13:41:49.602353Z","shell.execute_reply.started":"2021-06-27T13:41:47.701513Z","shell.execute_reply":"2021-06-27T13:41:49.601315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ANALYSIS","metadata":{}},{"cell_type":"markdown","source":"## Labels","metadata":{}},{"cell_type":"code","source":"labels.info()","metadata":{"execution":{"iopub.status.busy":"2021-06-27T13:41:49.603535Z","iopub.execute_input":"2021-06-27T13:41:49.603818Z","iopub.status.idle":"2021-06-27T13:41:49.624251Z","shell.execute_reply.started":"2021-06-27T13:41:49.603791Z","shell.execute_reply":"2021-06-27T13:41:49.623106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- Most studies contain a single image.\n- The maximum number of images per study is 9.","metadata":{}},{"cell_type":"code","source":"images_per_study = labels['StudyInstanceUID'].value_counts()\nimages_per_study","metadata":{"execution":{"iopub.status.busy":"2021-06-27T13:41:49.625521Z","iopub.execute_input":"2021-06-27T13:41:49.625828Z","iopub.status.idle":"2021-06-27T13:41:49.640602Z","shell.execute_reply.started":"2021-06-27T13:41:49.625796Z","shell.execute_reply":"2021-06-27T13:41:49.639333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"px.histogram(images_per_study)","metadata":{"execution":{"iopub.status.busy":"2021-06-27T13:41:49.641960Z","iopub.execute_input":"2021-06-27T13:41:49.642382Z","iopub.status.idle":"2021-06-27T13:41:51.041882Z","shell.execute_reply.started":"2021-06-27T13:41:49.642337Z","shell.execute_reply":"2021-06-27T13:41:51.041091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 32% of images don't have boxes.","metadata":{}},{"cell_type":"code","source":"images_no_boxes = labels[labels['label']=='none 1 0 0 1 1']\nlen(images_no_boxes) / len(labels) * 100","metadata":{"execution":{"iopub.status.busy":"2021-06-27T13:41:51.044506Z","iopub.execute_input":"2021-06-27T13:41:51.045079Z","iopub.status.idle":"2021-06-27T13:41:51.054116Z","shell.execute_reply.started":"2021-06-27T13:41:51.045025Z","shell.execute_reply":"2021-06-27T13:41:51.053092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Boxes","metadata":{}},{"cell_type":"code","source":"boxes.info()","metadata":{"execution":{"iopub.status.busy":"2021-06-27T13:41:51.055681Z","iopub.execute_input":"2021-06-27T13:41:51.056415Z","iopub.status.idle":"2021-06-27T13:41:51.080771Z","shell.execute_reply.started":"2021-06-27T13:41:51.056364Z","shell.execute_reply":"2021-06-27T13:41:51.079595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- We see two modes, one for each lungs\n- The distribution of cases if about the same.\n- There seems to be more boxes on the left lung.","metadata":{}},{"cell_type":"code","source":"px.histogram(boxes, x='xmin', color='case')","metadata":{"execution":{"iopub.status.busy":"2021-06-27T13:41:51.082420Z","iopub.execute_input":"2021-06-27T13:41:51.082827Z","iopub.status.idle":"2021-06-27T13:41:51.203869Z","shell.execute_reply.started":"2021-06-27T13:41:51.082789Z","shell.execute_reply":"2021-06-27T13:41:51.202655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"px.histogram(boxes, x='ymin', color='case')","metadata":{"execution":{"iopub.status.busy":"2021-06-27T13:41:51.205489Z","iopub.execute_input":"2021-06-27T13:41:51.205917Z","iopub.status.idle":"2021-06-27T13:41:51.314383Z","shell.execute_reply.started":"2021-06-27T13:41:51.205871Z","shell.execute_reply":"2021-06-27T13:41:51.313365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"px.histogram(boxes, x='xmax', color='case')","metadata":{"execution":{"iopub.status.busy":"2021-06-27T13:41:51.315712Z","iopub.execute_input":"2021-06-27T13:41:51.316101Z","iopub.status.idle":"2021-06-27T13:41:51.427456Z","shell.execute_reply.started":"2021-06-27T13:41:51.316064Z","shell.execute_reply":"2021-06-27T13:41:51.426421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"px.histogram(boxes, x='ymax', color='case')","metadata":{"execution":{"iopub.status.busy":"2021-06-27T13:41:51.429089Z","iopub.execute_input":"2021-06-27T13:41:51.429537Z","iopub.status.idle":"2021-06-27T13:41:51.540527Z","shell.execute_reply.started":"2021-06-27T13:41:51.429491Z","shell.execute_reply":"2021-06-27T13:41:51.539227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"px.histogram(boxes, x='width', color='case')","metadata":{"execution":{"iopub.status.busy":"2021-06-27T13:41:51.542015Z","iopub.execute_input":"2021-06-27T13:41:51.542442Z","iopub.status.idle":"2021-06-27T13:41:51.650287Z","shell.execute_reply.started":"2021-06-27T13:41:51.542402Z","shell.execute_reply":"2021-06-27T13:41:51.649050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"px.histogram(boxes, x='height', color='case')","metadata":{"execution":{"iopub.status.busy":"2021-06-27T13:41:51.652313Z","iopub.execute_input":"2021-06-27T13:41:51.652745Z","iopub.status.idle":"2021-06-27T13:41:51.765205Z","shell.execute_reply.started":"2021-06-27T13:41:51.652697Z","shell.execute_reply":"2021-06-27T13:41:51.763926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- There is a linear relation between the width and the height.\n- The heigth is often greater than the width (explained by the size of human lungs).","metadata":{}},{"cell_type":"code","source":"px.scatter(boxes, x='width', y='height', color='case')","metadata":{"execution":{"iopub.status.busy":"2021-06-27T13:41:51.766721Z","iopub.execute_input":"2021-06-27T13:41:51.767182Z","iopub.status.idle":"2021-06-27T13:41:51.914233Z","shell.execute_reply.started":"2021-06-27T13:41:51.767122Z","shell.execute_reply":"2021-06-27T13:41:51.913327Z"},"trusted":true},"execution_count":null,"outputs":[]}]}