{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport plotly.express as px\nimport glob\nimport cv2\nimport pydicom","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-03-15T14:14:26.179569Z","iopub.execute_input":"2022-03-15T14:14:26.180021Z","iopub.status.idle":"2022-03-15T14:14:29.129943Z","shell.execute_reply.started":"2022-03-15T14:14:26.179896Z","shell.execute_reply":"2022-03-15T14:14:29.129189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Study Level EDA","metadata":{}},{"cell_type":"code","source":"train_study_df = pd.read_csv('../input/siim-covid19-detection/train_study_level.csv')\ntrain_study_df['id'] = train_study_df.apply(lambda row: row.id.split('_')[0], axis=1)\ntrain_study_df.columns = ['StudyInstanceUID', 'Negative for Pneumonia', 'Typical Appearance', 'Indeterminate Appearance', 'Atypical Appearance']\ntrain_study_df.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-03-15T14:14:29.131682Z","iopub.execute_input":"2022-03-15T14:14:29.132477Z","iopub.status.idle":"2022-03-15T14:14:29.288764Z","shell.execute_reply.started":"2022-03-15T14:14:29.132435Z","shell.execute_reply":"2022-03-15T14:14:29.287866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classes = ['Negative for Pneumonia', 'Typical Appearance', 'Indeterminate Appearance', 'Atypical Appearance']","metadata":{"execution":{"iopub.status.busy":"2022-03-15T14:14:29.289965Z","iopub.execute_input":"2022-03-15T14:14:29.290235Z","iopub.status.idle":"2022-03-15T14:14:29.294932Z","shell.execute_reply.started":"2022-03-15T14:14:29.290204Z","shell.execute_reply":"2022-03-15T14:14:29.294052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.unique(train_study_df[classes].values, axis=0)","metadata":{"execution":{"iopub.status.busy":"2022-03-15T14:14:29.296359Z","iopub.execute_input":"2022-03-15T14:14:29.296651Z","iopub.status.idle":"2022-03-15T14:14:29.327259Z","shell.execute_reply.started":"2022-03-15T14:14:29.296609Z","shell.execute_reply":"2022-03-15T14:14:29.326094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classes_sum = train_study_df[classes].values.sum(axis=0)\nclasses_sum","metadata":{"execution":{"iopub.status.busy":"2022-03-15T14:14:29.329866Z","iopub.execute_input":"2022-03-15T14:14:29.330149Z","iopub.status.idle":"2022-03-15T14:14:29.337730Z","shell.execute_reply.started":"2022-03-15T14:14:29.330115Z","shell.execute_reply":"2022-03-15T14:14:29.337134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#  = np.array(images)\n# label = np.array(label)\nclass_count_df = pd.DataFrame({'classes': classes, 'count': classes_sum}, columns=['classes', 'count'])\nclass_count_df","metadata":{"execution":{"iopub.status.busy":"2022-03-15T14:14:29.339165Z","iopub.execute_input":"2022-03-15T14:14:29.339646Z","iopub.status.idle":"2022-03-15T14:14:29.356939Z","shell.execute_reply.started":"2022-03-15T14:14:29.339607Z","shell.execute_reply":"2022-03-15T14:14:29.356286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.bar(class_count_df.sort_values('count'), \n             x=class_count_df[\"classes\"].sort_index(), \n             y=\"count\", \n             color=class_count_df[\"classes\"], \n             opacity=0.5,\n            title=\"<b>Number of Data per Labeled Category</b>\",)\n\nfig.update_layout(legend_title=None,\n                  xaxis_title=\"\",\n                  yaxis_title=\"<b>Count</b>\")\n\nfig","metadata":{"execution":{"iopub.status.busy":"2022-03-15T14:14:29.358437Z","iopub.execute_input":"2022-03-15T14:14:29.358687Z","iopub.status.idle":"2022-03-15T14:14:30.389906Z","shell.execute_reply.started":"2022-03-15T14:14:29.358657Z","shell.execute_reply":"2022-03-15T14:14:30.389062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Image Level EDA\nImage level has studies with more than one image","metadata":{}},{"cell_type":"code","source":"train_image_df = pd.read_csv('../input/siim-covid19-detection/train_image_level.csv')\ntrain_image_df","metadata":{"execution":{"iopub.status.busy":"2022-03-15T14:14:30.391026Z","iopub.execute_input":"2022-03-15T14:14:30.391237Z","iopub.status.idle":"2022-03-15T14:14:30.449203Z","shell.execute_reply.started":"2022-03-15T14:14:30.391211Z","shell.execute_reply":"2022-03-15T14:14:30.448267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_image_df.StudyInstanceUID.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-03-15T14:14:30.450741Z","iopub.execute_input":"2022-03-15T14:14:30.450983Z","iopub.status.idle":"2022-03-15T14:14:30.464988Z","shell.execute_reply.started":"2022-03-15T14:14:30.450953Z","shell.execute_reply":"2022-03-15T14:14:30.463942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.histogram(train_image_df.StudyInstanceUID.value_counts(),\n                   log_y=True,\n                   opacity=0.5,\n                   text_auto=True,\n                   title=\"<b>Number of Images per Study</b>\",)\n\nfig.update_layout(showlegend=False,\n                  xaxis_title=\"<b>Number of Images</b>\",\n                  yaxis_title=\"<b>Count</b>\")\n\nfig","metadata":{"execution":{"iopub.status.busy":"2022-03-15T14:14:30.466656Z","iopub.execute_input":"2022-03-15T14:14:30.467091Z","iopub.status.idle":"2022-03-15T14:14:30.590763Z","shell.execute_reply.started":"2022-03-15T14:14:30.467047Z","shell.execute_reply":"2022-03-15T14:14:30.589942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_image_df['opacity'] = train_image_df.label.apply(lambda x: x.split()[0])\ntrain_image_df","metadata":{"execution":{"iopub.status.busy":"2022-03-15T14:14:30.592318Z","iopub.execute_input":"2022-03-15T14:14:30.592682Z","iopub.status.idle":"2022-03-15T14:14:30.614936Z","shell.execute_reply.started":"2022-03-15T14:14:30.592652Z","shell.execute_reply":"2022-03-15T14:14:30.613880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.bar(train_image_df, \n             x=train_image_df['opacity'].unique(), \n             y=train_image_df['opacity'].value_counts(), \n             color=train_image_df['opacity'].unique(), \n             opacity=0.5,\n            title=\"<b>Data with Opacity vs None</b>\",)\n\nfig.update_layout(legend_title=None,\n                  xaxis_title=\"\",\n                  yaxis_title=\"<b>Count</b>\")\n\nfig","metadata":{"execution":{"iopub.status.busy":"2022-03-15T14:14:30.617053Z","iopub.execute_input":"2022-03-15T14:14:30.617484Z","iopub.status.idle":"2022-03-15T14:14:30.701475Z","shell.execute_reply.started":"2022-03-15T14:14:30.617441Z","shell.execute_reply":"2022-03-15T14:14:30.700883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Merge image-level and study-level\ntrain_study_df = train_study_df.merge(train_image_df, on='StudyInstanceUID',how=\"left\")\ntrain_study_df.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-03-15T14:14:30.702594Z","iopub.execute_input":"2022-03-15T14:14:30.703244Z","iopub.status.idle":"2022-03-15T14:14:30.734051Z","shell.execute_reply.started":"2022-03-15T14:14:30.703208Z","shell.execute_reply":"2022-03-15T14:14:30.733227Z"},"trusted":true},"execution_count":null,"outputs":[]}]}