{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns\nimport glob\nimport plotly as py\nimport plotly.graph_objects as go\nimport plotly.express as px\nfrom plotly.subplots import make_subplots\nimport pydicom as dicom\nimport matplotlib.pylab as plt\nimport matplotlib\nimport cv2\nimport ast\nimport json\nimport os","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-25T05:21:35.262702Z","iopub.execute_input":"2021-06-25T05:21:35.263180Z","iopub.status.idle":"2021-06-25T05:21:38.272921Z","shell.execute_reply.started":"2021-06-25T05:21:35.263150Z","shell.execute_reply":"2021-06-25T05:21:38.271347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In this competition, we are identifying and localizing COVID-19 abnormalities on chest radiographs. This is an object detection and classification problem.\n\nFor each test image, you will be predicting a bounding box and class for all findings. If you predict that there are no findings, you should create a prediction of \"none 1 0 0 1 1\" (\"none\" is the class ID for no finding, and this provides a one-pixel bounding box with a confidence of 1.0).\n\nFurther, for each test study, you should make a determination within the following labels:\n\n'Negative for Pneumonia' 'Typical Appearance' 'Indeterminate Appearance' 'Atypical Appearance'","metadata":{}},{"cell_type":"markdown","source":"**train_study_level.csv**\n\nid - unique study identifier\nNegative for Pneumonia - 1 if the study is negative for pneumonia, 0 otherwise\nTypical Appearance - 1 if the study has this appearance, 0 otherwise\nIndeterminate Appearance  - 1 if the study has this appearance, 0 otherwise\nAtypical Appearance  - 1 if the study has this appearance, 0 otherwise\n\n**train_image_level.csv**\n\nid - unique image identifier\nboxes - bounding boxes in easily-readable dictionary format\nlabel - the correct prediction label for the provided bounding boxes","metadata":{}},{"cell_type":"code","source":"train_study = pd.read_csv(\"../input/siim-covid19-detection/train_study_level.csv\")\ntrain_study.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:21:38.275182Z","iopub.execute_input":"2021-06-25T05:21:38.275914Z","iopub.status.idle":"2021-06-25T05:21:38.324582Z","shell.execute_reply.started":"2021-06-25T05:21:38.275870Z","shell.execute_reply":"2021-06-25T05:21:38.323332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_study.info()","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:21:38.326792Z","iopub.execute_input":"2021-06-25T05:21:38.327419Z","iopub.status.idle":"2021-06-25T05:21:38.352848Z","shell.execute_reply.started":"2021-06-25T05:21:38.327369Z","shell.execute_reply":"2021-06-25T05:21:38.351699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns = train_study.columns[1:] # ignore id\nx0 = [columns[0],columns[1],columns[2],columns[3]]\n# fetch all '0's\ny0 = [str(len(train_study[train_study[columns[0]] == 0])), str(len(train_study[train_study[columns[1]] == 0])), str(len(train_study[train_study[columns[2]] == 0])), str(len(train_study[train_study[columns[3]] == 0]))]\n\nx1 = [columns[0],columns[1],columns[2],columns[3]]\n# fetch all '1's\ny1 = [str(len(train_study[train_study[columns[0]] == 1])), str(len(train_study[train_study[columns[1]] == 1])), str(len(train_study[train_study[columns[2]] == 1])), str(len(train_study[train_study[columns[3]] == 1]))]\n\nfig = go.Figure()\nfig.add_trace(go.Histogram(histfunc=\"sum\", y=y0, x=x0, name=\"0\"))\nfig.add_trace(go.Histogram(histfunc=\"sum\", y=y1, x=x1, name=\"1\"))\npy.offline.iplot(fig)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:21:38.354691Z","iopub.execute_input":"2021-06-25T05:21:38.355319Z","iopub.status.idle":"2021-06-25T05:21:39.574215Z","shell.execute_reply.started":"2021-06-25T05:21:38.355244Z","shell.execute_reply":"2021-06-25T05:21:39.573165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_img = pd.read_csv(\"../input/siim-covid19-detection/train_image_level.csv\")\ntrain_img.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:21:39.576121Z","iopub.execute_input":"2021-06-25T05:21:39.576691Z","iopub.status.idle":"2021-06-25T05:21:39.642255Z","shell.execute_reply.started":"2021-06-25T05:21:39.576642Z","shell.execute_reply":"2021-06-25T05:21:39.640906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_img['label'][0]","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:21:39.643902Z","iopub.execute_input":"2021-06-25T05:21:39.644330Z","iopub.status.idle":"2021-06-25T05:21:39.651616Z","shell.execute_reply.started":"2021-06-25T05:21:39.644284Z","shell.execute_reply":"2021-06-25T05:21:39.650562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_img['boxes'][0]","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:21:39.653357Z","iopub.execute_input":"2021-06-25T05:21:39.653699Z","iopub.status.idle":"2021-06-25T05:21:39.667390Z","shell.execute_reply.started":"2021-06-25T05:21:39.653668Z","shell.execute_reply":"2021-06-25T05:21:39.665834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_img.info()","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:21:39.669209Z","iopub.execute_input":"2021-06-25T05:21:39.669607Z","iopub.status.idle":"2021-06-25T05:21:39.692883Z","shell.execute_reply.started":"2021-06-25T05:21:39.669575Z","shell.execute_reply":"2021-06-25T05:21:39.691819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# count of missing values in each column\ntrain_img.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:21:39.694639Z","iopub.execute_input":"2021-06-25T05:21:39.695107Z","iopub.status.idle":"2021-06-25T05:21:39.713879Z","shell.execute_reply.started":"2021-06-25T05:21:39.695069Z","shell.execute_reply":"2021-06-25T05:21:39.712937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub= pd.read_csv('../input/siim-covid19-detection/sample_submission.csv')\nsample_sub.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:22:43.446489Z","iopub.execute_input":"2021-06-25T05:22:43.446907Z","iopub.status.idle":"2021-06-25T05:22:43.467424Z","shell.execute_reply.started":"2021-06-25T05:22:43.446876Z","shell.execute_reply":"2021-06-25T05:22:43.466335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Merge train image and train study data based on UID , and append train image path with the merged dataframe","metadata":{}},{"cell_type":"code","source":"train_path = '../input/siim-covid19-detection/train'\ntest_path = '../input/siim-covid19-detection/test'","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:22:44.032338Z","iopub.execute_input":"2021-06-25T05:22:44.032742Z","iopub.status.idle":"2021-06-25T05:22:44.036804Z","shell.execute_reply.started":"2021-06-25T05:22:44.032710Z","shell.execute_reply":"2021-06-25T05:22:44.035891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_imgs = glob.glob('/kaggle/input/siim-covid19-detection/train/*/*/*.dcm')\ntest_imgs = glob.glob('/kaggle/input/siim-covid19-detection/test/*/*/*.dcm')\n\nprint('Total train images',len(train_imgs) ,'\\nSample train image',train_imgs[0])\nprint('Total test images',len(test_imgs) ,'\\nSample test image',test_imgs [0])","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:22:44.387408Z","iopub.execute_input":"2021-06-25T05:22:44.387898Z","iopub.status.idle":"2021-06-25T05:23:12.632455Z","shell.execute_reply.started":"2021-06-25T05:22:44.387867Z","shell.execute_reply":"2021-06-25T05:23:12.631628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_image_id(path):\n    image_name = path.split('/')[-1].replace('.dcm', '_image')\n    return image_name","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:23:12.633781Z","iopub.execute_input":"2021-06-25T05:23:12.634313Z","iopub.status.idle":"2021-06-25T05:23:12.639769Z","shell.execute_reply.started":"2021-06-25T05:23:12.634256Z","shell.execute_reply":"2021-06-25T05:23:12.638199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_images_df = pd.DataFrame(train_imgs, columns =['training_images_path'])","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:23:12.641993Z","iopub.execute_input":"2021-06-25T05:23:12.642595Z","iopub.status.idle":"2021-06-25T05:23:12.654208Z","shell.execute_reply.started":"2021-06-25T05:23:12.642558Z","shell.execute_reply":"2021-06-25T05:23:12.653195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_images_df['image_id'] = training_images_df.apply(lambda rows: get_image_id(\n                                rows['training_images_path']), axis=1)\n\ntraining_images_df = pd.merge(training_images_df, train_img, left_on='image_id', right_on='id', \n                              how='left')\n\ntraining_images_df.head(3)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:23:12.656025Z","iopub.execute_input":"2021-06-25T05:23:12.656625Z","iopub.status.idle":"2021-06-25T05:23:12.780495Z","shell.execute_reply.started":"2021-06-25T05:23:12.656527Z","shell.execute_reply":"2021-06-25T05:23:12.779679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_images_df = training_images_df.drop(['id'],axis=1)\ntraining_images_df.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:23:12.781640Z","iopub.execute_input":"2021-06-25T05:23:12.782039Z","iopub.status.idle":"2021-06-25T05:23:12.798695Z","shell.execute_reply.started":"2021-06-25T05:23:12.782010Z","shell.execute_reply":"2021-06-25T05:23:12.797628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Merge train img df with train study df","metadata":{}},{"cell_type":"code","source":"training_images = pd.DataFrame(train_imgs, columns =['training_images_path'])","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:23:12.800298Z","iopub.execute_input":"2021-06-25T05:23:12.800648Z","iopub.status.idle":"2021-06-25T05:23:12.812934Z","shell.execute_reply.started":"2021-06-25T05:23:12.800617Z","shell.execute_reply":"2021-06-25T05:23:12.811708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_images_df['StudyInstance'] = training_images_df.apply(lambda rows: get_image_id(rows['StudyInstanceUID']) + \"_study\", axis=1)\ntraining_images_df = pd.merge(training_images_df, train_study, left_on='StudyInstance', right_on='id', how='left').drop(['StudyInstance','id'], axis=1)\ntraining_images_df.info()","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:23:12.814413Z","iopub.execute_input":"2021-06-25T05:23:12.815039Z","iopub.status.idle":"2021-06-25T05:23:12.936780Z","shell.execute_reply.started":"2021-06-25T05:23:12.814990Z","shell.execute_reply":"2021-06-25T05:23:12.935988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_images_df.head(3)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:23:12.938979Z","iopub.execute_input":"2021-06-25T05:23:12.939425Z","iopub.status.idle":"2021-06-25T05:23:12.954173Z","shell.execute_reply.started":"2021-06-25T05:23:12.939393Z","shell.execute_reply":"2021-06-25T05:23:12.952892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Save Merged training data df in - output dir","metadata":{}},{"cell_type":"code","source":"training_images_df.to_csv('./merged_train_df.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:28:50.314776Z","iopub.execute_input":"2021-06-25T05:28:50.315151Z","iopub.status.idle":"2021-06-25T05:28:50.403247Z","shell.execute_reply.started":"2021-06-25T05:28:50.315122Z","shell.execute_reply":"2021-06-25T05:28:50.402027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_images_df['training_images_path'][0]","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:23:13.071202Z","iopub.execute_input":"2021-06-25T05:23:13.071617Z","iopub.status.idle":"2021-06-25T05:23:13.078631Z","shell.execute_reply.started":"2021-06-25T05:23:13.071584Z","shell.execute_reply":"2021-06-25T05:23:13.077593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"#### Checking for duplicate ID","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:23:13.079899Z","iopub.execute_input":"2021-06-25T05:23:13.080337Z","iopub.status.idle":"2021-06-25T05:23:13.092039Z","shell.execute_reply.started":"2021-06-25T05:23:13.080306Z","shell.execute_reply":"2021-06-25T05:23:13.091101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dup_ids= training_images_df.groupby(\"StudyInstanceUID\").count().reset_index()\n# dup_ids","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:23:13.093492Z","iopub.execute_input":"2021-06-25T05:23:13.093960Z","iopub.status.idle":"2021-06-25T05:23:13.105711Z","shell.execute_reply.started":"2021-06-25T05:23:13.093923Z","shell.execute_reply":"2021-06-25T05:23:13.104782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# no_dup = dup_ids[dup_ids[\"image_id\"]==1] \n# no_dup","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:23:13.107017Z","iopub.execute_input":"2021-06-25T05:23:13.107654Z","iopub.status.idle":"2021-06-25T05:23:13.118734Z","shell.execute_reply.started":"2021-06-25T05:23:13.107609Z","shell.execute_reply":"2021-06-25T05:23:13.117899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#training_images_df[training_images_df.StudyInstanceUID == 'fa9ea207e240']","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:23:13.120831Z","iopub.execute_input":"2021-06-25T05:23:13.121299Z","iopub.status.idle":"2021-06-25T05:23:13.133451Z","shell.execute_reply.started":"2021-06-25T05:23:13.121225Z","shell.execute_reply":"2021-06-25T05:23:13.132218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_images_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:23:13.136462Z","iopub.execute_input":"2021-06-25T05:23:13.136940Z","iopub.status.idle":"2021-06-25T05:23:13.157928Z","shell.execute_reply.started":"2021-06-25T05:23:13.136896Z","shell.execute_reply":"2021-06-25T05:23:13.156889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"neg_pnemonia_no = training_images_df[training_images_df['Negative for Pneumonia']==0]\nneg_pnemonia_yes = training_images_df[training_images_df['Negative for Pneumonia']==1]","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:23:13.159600Z","iopub.execute_input":"2021-06-25T05:23:13.159937Z","iopub.status.idle":"2021-06-25T05:23:13.173808Z","shell.execute_reply.started":"2021-06-25T05:23:13.159908Z","shell.execute_reply":"2021-06-25T05:23:13.172261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"neg_pnemonia_no.info()","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:23:13.175402Z","iopub.execute_input":"2021-06-25T05:23:13.175877Z","iopub.status.idle":"2021-06-25T05:23:13.205809Z","shell.execute_reply.started":"2021-06-25T05:23:13.175845Z","shell.execute_reply":"2021-06-25T05:23:13.204038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"neg_pnemonia_yes.info()","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:23:13.207217Z","iopub.execute_input":"2021-06-25T05:23:13.207538Z","iopub.status.idle":"2021-06-25T05:23:13.231900Z","shell.execute_reply.started":"2021-06-25T05:23:13.207510Z","shell.execute_reply":"2021-06-25T05:23:13.230673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"neg_pnemonia_no[neg_pnemonia_no['boxes'].isna()]","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:23:13.233861Z","iopub.execute_input":"2021-06-25T05:23:13.234295Z","iopub.status.idle":"2021-06-25T05:23:13.263927Z","shell.execute_reply.started":"2021-06-25T05:23:13.234192Z","shell.execute_reply":"2021-06-25T05:23:13.263040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Total 304 images which are NOT negative for pnemonia (means having pnemonia) has 'none'lables","metadata":{}},{"cell_type":"code","source":"neg_pnemonia_yes[neg_pnemonia_yes['boxes'].isna()]","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:23:13.264996Z","iopub.execute_input":"2021-06-25T05:23:13.265313Z","iopub.status.idle":"2021-06-25T05:23:13.292869Z","shell.execute_reply.started":"2021-06-25T05:23:13.265258Z","shell.execute_reply":"2021-06-25T05:23:13.291628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"All images which are negative for pnemonia (means who do not have pnemonia is equiv to not having covid) has 'none' lables","metadata":{}},{"cell_type":"markdown","source":"#### Image Analysis","metadata":{}},{"cell_type":"code","source":"training_images_df.loc[1, 'boxes']","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:23:13.294550Z","iopub.execute_input":"2021-06-25T05:23:13.294888Z","iopub.status.idle":"2021-06-25T05:23:13.308107Z","shell.execute_reply.started":"2021-06-25T05:23:13.294860Z","shell.execute_reply":"2021-06-25T05:23:13.307282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"boxes = ast.literal_eval(training_images_df.loc[1, 'boxes'])\nprint(boxes)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:23:13.309688Z","iopub.execute_input":"2021-06-25T05:23:13.310413Z","iopub.status.idle":"2021-06-25T05:23:13.319523Z","shell.execute_reply.started":"2021-06-25T05:23:13.310369Z","shell.execute_reply":"2021-06-25T05:23:13.318416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visualize single img with boxes\n\nimage_path = train_imgs[1]\nds = dicom.dcmread(image_path).pixel_array\nfig, ax = plt.subplots(1,1, figsize=(8,4))\nfor box in boxes:\n    print('box',box)\n    p = matplotlib.patches.Rectangle((box['x'], box['y']),\n                                      box['width'], box['height'],\n                                      ec='r', fc='none', lw=1.5)\n    ax.add_patch(p)\nax.imshow(ds, cmap='gray')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:23:13.324298Z","iopub.execute_input":"2021-06-25T05:23:13.324804Z","iopub.status.idle":"2021-06-25T05:23:15.806621Z","shell.execute_reply.started":"2021-06-25T05:23:13.324769Z","shell.execute_reply":"2021-06-25T05:23:15.805019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_images_df.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:23:15.808399Z","iopub.execute_input":"2021-06-25T05:23:15.808748Z","iopub.status.idle":"2021-06-25T05:23:15.824313Z","shell.execute_reply.started":"2021-06-25T05:23:15.808715Z","shell.execute_reply":"2021-06-25T05:23:15.823460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_images_df.columns","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:23:15.825919Z","iopub.execute_input":"2021-06-25T05:23:15.826586Z","iopub.status.idle":"2021-06-25T05:23:15.842118Z","shell.execute_reply.started":"2021-06-25T05:23:15.826541Z","shell.execute_reply":"2021-06-25T05:23:15.840590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def visualization(class_name,df,color):\n    df.dropna(inplace=True)\n    fig,axes = plt.subplots(3,3,figsize=(20,16))\n    fig.subplots_adjust(hspace=.1, wspace=.1)\n    axes = axes.ravel()\n    records = df[\n        df[class_name]==0].iloc[:9].reset_index(drop=True)\n    \n    for _, row in records.iterrows():\n        img = row['training_images_path']\n        img = dicom.dcmread(image_path).pixel_array\n\n        if (row['boxes'] == row['boxes']):\n            boxes = ast.literal_eval(row['boxes'])\n            for box in boxes:\n                p = matplotlib.patches.Rectangle((box['x'], box['y']),\n                                                  box['width'], box['height'],\n                                                  ec=color, fc='none', lw=2.\n                                                )\n                axes[_].add_patch(p)\n\n\n            axes[_].imshow(img, cmap='gray')\n            axes[_].set_title(row['StudyInstanceUID'].split(' ')[0])\n            axes[_].set_xticklabels([])\n            axes[_].set_yticklabels([])","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:23:15.843757Z","iopub.execute_input":"2021-06-25T05:23:15.844103Z","iopub.status.idle":"2021-06-25T05:23:15.856827Z","shell.execute_reply.started":"2021-06-25T05:23:15.844072Z","shell.execute_reply":"2021-06-25T05:23:15.855840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Indeterminate Appearance ","metadata":{}},{"cell_type":"code","source":"visualization('Indeterminate Appearance',training_images_df,'g')","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:23:15.858490Z","iopub.execute_input":"2021-06-25T05:23:15.858972Z","iopub.status.idle":"2021-06-25T05:23:31.711135Z","shell.execute_reply.started":"2021-06-25T05:23:15.858924Z","shell.execute_reply":"2021-06-25T05:23:31.710304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Typical Appearance","metadata":{}},{"cell_type":"code","source":"visualization('Typical Appearance',training_images_df,'r')","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:23:31.712352Z","iopub.execute_input":"2021-06-25T05:23:31.712782Z","iopub.status.idle":"2021-06-25T05:23:47.313943Z","shell.execute_reply.started":"2021-06-25T05:23:31.712747Z","shell.execute_reply":"2021-06-25T05:23:47.313157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Atypical Appearance","metadata":{}},{"cell_type":"code","source":"visualization('Atypical Appearance',training_images_df,'b')","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:23:47.315069Z","iopub.execute_input":"2021-06-25T05:23:47.315505Z","iopub.status.idle":"2021-06-25T05:24:02.897386Z","shell.execute_reply.started":"2021-06-25T05:23:47.315469Z","shell.execute_reply":"2021-06-25T05:24:02.896193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### ************ *** EDA DONE ***************** ** ####","metadata":{}},{"cell_type":"markdown","source":"### Data Preperation","metadata":{}},{"cell_type":"markdown","source":"#### Replacing boxes with nan value to [{'x': 0, 'y': 0, 'width': 1, 'height': 1}]","metadata":{}},{"cell_type":"code","source":"df= pd.read_csv('./training_images_df.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:24:02.898782Z","iopub.execute_input":"2021-06-25T05:24:02.899118Z","iopub.status.idle":"2021-06-25T05:24:02.938878Z","shell.execute_reply.started":"2021-06-25T05:24:02.899085Z","shell.execute_reply":"2021-06-25T05:24:02.937721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:24:02.940181Z","iopub.execute_input":"2021-06-25T05:24:02.940554Z","iopub.status.idle":"2021-06-25T05:24:02.957380Z","shell.execute_reply.started":"2021-06-25T05:24:02.940522Z","shell.execute_reply":"2021-06-25T05:24:02.956480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"none_ = [{'x': 0, 'y': 0, 'width': 1, 'height': 1}]","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:24:02.958574Z","iopub.execute_input":"2021-06-25T05:24:02.958926Z","iopub.status.idle":"2021-06-25T05:24:02.973156Z","shell.execute_reply.started":"2021-06-25T05:24:02.958892Z","shell.execute_reply":"2021-06-25T05:24:02.971682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.boxes[0] = none_","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:24:02.975354Z","iopub.execute_input":"2021-06-25T05:24:02.976107Z","iopub.status.idle":"2021-06-25T05:24:02.989229Z","shell.execute_reply.started":"2021-06-25T05:24:02.976061Z","shell.execute_reply":"2021-06-25T05:24:02.987921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"boxes = ast.literal_eval(json.dumps(df.loc[0, 'boxes']))\nprint(boxes)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:24:02.990984Z","iopub.execute_input":"2021-06-25T05:24:02.991333Z","iopub.status.idle":"2021-06-25T05:24:03.000991Z","shell.execute_reply.started":"2021-06-25T05:24:02.991295Z","shell.execute_reply":"2021-06-25T05:24:02.999858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visualize single img with box coordinate - 0,0,1,1\n\nimage_path = train_imgs[0]\nds = dicom.dcmread(image_path).pixel_array\nfig, ax = plt.subplots(1,1, figsize=(8,4))\nfor box in boxes:\n    print('box',box)\n    p = matplotlib.patches.Rectangle((box['x'], box['y']),\n                                      box['width'], box['height'],\n                                      ec='r', fc='none')#, lw=1.5)\n    ax.add_patch(p)\nax.imshow(ds, cmap='gray')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:24:03.002362Z","iopub.execute_input":"2021-06-25T05:24:03.002674Z","iopub.status.idle":"2021-06-25T05:24:04.398620Z","shell.execute_reply.started":"2021-06-25T05:24:03.002636Z","shell.execute_reply":"2021-06-25T05:24:04.397507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Creating train_df with class lables and boxes","metadata":{}},{"cell_type":"code","source":"df[\"one_hot\"] = df.apply(lambda x : np.array([x[\"Negative for Pneumonia\"],\n                                                        x[\"Typical Appearance\"],\n                                                        x[\"Indeterminate Appearance\"],\n                                                        x[\"Atypical Appearance\"]]),axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:24:04.399989Z","iopub.execute_input":"2021-06-25T05:24:04.400546Z","iopub.status.idle":"2021-06-25T05:24:04.618954Z","shell.execute_reply.started":"2021-06-25T05:24:04.400501Z","shell.execute_reply":"2021-06-25T05:24:04.617682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:24:04.620984Z","iopub.execute_input":"2021-06-25T05:24:04.621659Z","iopub.status.idle":"2021-06-25T05:24:04.640442Z","shell.execute_reply.started":"2021-06-25T05:24:04.621608Z","shell.execute_reply":"2021-06-25T05:24:04.638756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classes_dict = {\n    0 : \"Negative for Pneumonia\",\n    1  : \"Typical Appearance\",\n    2  : \"Indeterminate Appearance\",\n    3  : \"Atypical Appearance\"\n}","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:24:04.644210Z","iopub.execute_input":"2021-06-25T05:24:04.644656Z","iopub.status.idle":"2021-06-25T05:24:04.654536Z","shell.execute_reply.started":"2021-06-25T05:24:04.644612Z","shell.execute_reply":"2021-06-25T05:24:04.652723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[\"Class\"] = df[\"one_hot\"].map(lambda x : classes_dict[np.argmax(x)]) # argmax returns the index of max value\ndf[\"Class\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:24:04.656762Z","iopub.execute_input":"2021-06-25T05:24:04.657131Z","iopub.status.idle":"2021-06-25T05:24:04.712987Z","shell.execute_reply.started":"2021-06-25T05:24:04.657100Z","shell.execute_reply":"2021-06-25T05:24:04.711736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df.drop([\"Negative for Pneumonia\",\"Typical Appearance\",\"Indeterminate Appearance\",\"Atypical Appearance\",\"one_hot\"],axis=1)\ndf.head(1)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:24:04.714374Z","iopub.execute_input":"2021-06-25T05:24:04.714677Z","iopub.status.idle":"2021-06-25T05:24:04.732189Z","shell.execute_reply.started":"2021-06-25T05:24:04.714649Z","shell.execute_reply":"2021-06-25T05:24:04.731360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Replace nan boxes with one pixel coordinates\ndf[\"boxes\"].fillna(\"[{'x':0,'y':0,'width':1,'height':1}]\",inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:24:04.733325Z","iopub.execute_input":"2021-06-25T05:24:04.733788Z","iopub.status.idle":"2021-06-25T05:24:04.749627Z","shell.execute_reply.started":"2021-06-25T05:24:04.733758Z","shell.execute_reply":"2021-06-25T05:24:04.748525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[\"boxes\"] = df[\"boxes\"].map(lambda x : (ast.literal_eval(json.dumps(x))))","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:24:04.751194Z","iopub.execute_input":"2021-06-25T05:24:04.751545Z","iopub.status.idle":"2021-06-25T05:24:04.823529Z","shell.execute_reply.started":"2021-06-25T05:24:04.751513Z","shell.execute_reply":"2021-06-25T05:24:04.822431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.info()","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:24:04.825138Z","iopub.execute_input":"2021-06-25T05:24:04.825474Z","iopub.status.idle":"2021-06-25T05:24:04.848016Z","shell.execute_reply.started":"2021-06-25T05:24:04.825433Z","shell.execute_reply":"2021-06-25T05:24:04.846703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"raw","source":"df","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:24:04.849797Z","iopub.execute_input":"2021-06-25T05:24:04.850213Z","iopub.status.idle":"2021-06-25T05:24:04.871198Z","shell.execute_reply.started":"2021-06-25T05:24:04.850146Z","shell.execute_reply":"2021-06-25T05:24:04.870295Z"}}},{"cell_type":"code","source":"df.to_csv('train_df_after_eda.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:27:13.317651Z","iopub.execute_input":"2021-06-25T05:27:13.318048Z","iopub.status.idle":"2021-06-25T05:27:13.429607Z","shell.execute_reply.started":"2021-06-25T05:27:13.318017Z","shell.execute_reply":"2021-06-25T05:27:13.428631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['training_images_path'][0]","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:24:33.643686Z","iopub.execute_input":"2021-06-25T05:24:33.644054Z","iopub.status.idle":"2021-06-25T05:24:33.651094Z","shell.execute_reply.started":"2021-06-25T05:24:33.644024Z","shell.execute_reply":"2021-06-25T05:24:33.649804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['training_images_path'][1]","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:24:47.353112Z","iopub.execute_input":"2021-06-25T05:24:47.353547Z","iopub.status.idle":"2021-06-25T05:24:47.362170Z","shell.execute_reply.started":"2021-06-25T05:24:47.353515Z","shell.execute_reply":"2021-06-25T05:24:47.360254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.read_csv('./train_df_after_eda.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-25T05:24:04.994840Z","iopub.execute_input":"2021-06-25T05:24:04.995141Z","iopub.status.idle":"2021-06-25T05:24:05.049155Z","shell.execute_reply.started":"2021-06-25T05:24:04.995110Z","shell.execute_reply":"2021-06-25T05:24:05.048117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}