{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport ast\n\nimport re\nimport os","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-24T11:48:09.813239Z","iopub.execute_input":"2021-06-24T11:48:09.813694Z","iopub.status.idle":"2021-06-24T11:48:09.81794Z","shell.execute_reply.started":"2021-06-24T11:48:09.81365Z","shell.execute_reply":"2021-06-24T11:48:09.816989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ls ../input/siim-covid19-resized-to-512px-jpg/meta.csv","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:09.823161Z","iopub.execute_input":"2021-06-24T11:48:09.823457Z","iopub.status.idle":"2021-06-24T11:48:10.598452Z","shell.execute_reply.started":"2021-06-24T11:48:09.823431Z","shell.execute_reply":"2021-06-24T11:48:10.597595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FOLDER = '../input/siim-covid19-detection'","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:10.599916Z","iopub.execute_input":"2021-06-24T11:48:10.600212Z","iopub.status.idle":"2021-06-24T11:48:10.60428Z","shell.execute_reply.started":"2021-06-24T11:48:10.600173Z","shell.execute_reply":"2021-06-24T11:48:10.603249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train set\ntrain_study_df = pd.read_csv('../input/siim-covid19-detection/train_study_level.csv')\ntrain_image_df = pd.read_csv('../input/siim-covid19-detection/train_image_level.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:10.60626Z","iopub.execute_input":"2021-06-24T11:48:10.606635Z","iopub.status.idle":"2021-06-24T11:48:10.659144Z","shell.execute_reply.started":"2021-06-24T11:48:10.606598Z","shell.execute_reply":"2021-06-24T11:48:10.658363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_study_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:10.660772Z","iopub.execute_input":"2021-06-24T11:48:10.661128Z","iopub.status.idle":"2021-06-24T11:48:10.672055Z","shell.execute_reply.started":"2021-06-24T11:48:10.66109Z","shell.execute_reply":"2021-06-24T11:48:10.671109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_image_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:10.673443Z","iopub.execute_input":"2021-06-24T11:48:10.673824Z","iopub.status.idle":"2021-06-24T11:48:10.688205Z","shell.execute_reply.started":"2021-06-24T11:48:10.673787Z","shell.execute_reply":"2021-06-24T11:48:10.68733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. Basic info","metadata":{}},{"cell_type":"code","source":"print('# studies in train set:', train_image_df.StudyInstanceUID.nunique())\nprint('# images in train set:', train_image_df.id.nunique())","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:10.690961Z","iopub.execute_input":"2021-06-24T11:48:10.691228Z","iopub.status.idle":"2021-06-24T11:48:10.705124Z","shell.execute_reply.started":"2021-06-24T11:48:10.691203Z","shell.execute_reply":"2021-06-24T11:48:10.704256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test set\ndef get_df_from_folder(phase='test'):\n    list_test_studies = os.listdir(os.path.join(FOLDER, phase))\n\n    ret_studies = []\n    ret_images = []\n\n    for study in list_test_studies:\n        series = os.path.join(FOLDER, phase, study)\n        for sr in os.listdir(series):\n            images = os.path.join(FOLDER, phase, study, sr)\n            for image in os.listdir(images):\n                ret_studies.append(study+'_study')\n                ret_images.append(image.replace('.dcm','')+'_image')\n    return pd.DataFrame({'id':ret_images, 'StudyInstanceUID':ret_studies})","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:10.706091Z","iopub.execute_input":"2021-06-24T11:48:10.706343Z","iopub.status.idle":"2021-06-24T11:48:10.713654Z","shell.execute_reply.started":"2021-06-24T11:48:10.706319Z","shell.execute_reply":"2021-06-24T11:48:10.712403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_image_df = get_df_from_folder('test')","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:10.715604Z","iopub.execute_input":"2021-06-24T11:48:10.715898Z","iopub.status.idle":"2021-06-24T11:48:11.75686Z","shell.execute_reply.started":"2021-06-24T11:48:10.715873Z","shell.execute_reply":"2021-06-24T11:48:11.756088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sam_sub = pd.read_csv('../input/siim-covid19-detection/sample_submission.csv')\nsam_sub['id'].map(lambda x: x.endswith('image')).sum()","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:11.758266Z","iopub.execute_input":"2021-06-24T11:48:11.758587Z","iopub.status.idle":"2021-06-24T11:48:11.775678Z","shell.execute_reply.started":"2021-06-24T11:48:11.758535Z","shell.execute_reply":"2021-06-24T11:48:11.7746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_image_df.id.isin(sam_sub[sam_sub['id'].map(lambda x: x.endswith('image'))].id).mean()","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:11.77679Z","iopub.execute_input":"2021-06-24T11:48:11.777071Z","iopub.status.idle":"2021-06-24T11:48:11.785437Z","shell.execute_reply.started":"2021-06-24T11:48:11.777029Z","shell.execute_reply":"2021-06-24T11:48:11.784391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('# studies in test set:', test_image_df.StudyInstanceUID.nunique())\nprint('# images in test set:', test_image_df.id.nunique())","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:11.786652Z","iopub.execute_input":"2021-06-24T11:48:11.786991Z","iopub.status.idle":"2021-06-24T11:48:11.799052Z","shell.execute_reply.started":"2021-06-24T11:48:11.786962Z","shell.execute_reply":"2021-06-24T11:48:11.798072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wh_df = pd.read_csv('../input/siim-covid19-resized-to-512px-jpg/meta.csv')\nwh_df.columns = ['image_id', 'height', 'width', 'split']\nwh_train = wh_df[wh_df.split=='train']\nwh_test = wh_df[wh_df.split=='test']","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:11.800097Z","iopub.execute_input":"2021-06-24T11:48:11.800369Z","iopub.status.idle":"2021-06-24T11:48:11.827587Z","shell.execute_reply.started":"2021-06-24T11:48:11.800333Z","shell.execute_reply":"2021-06-24T11:48:11.826776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Number of unique study:', train_image_df.StudyInstanceUID.nunique())\nassert train_study_df.id.nunique() == train_image_df.StudyInstanceUID.nunique() # cf the consistent number","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:11.828742Z","iopub.execute_input":"2021-06-24T11:48:11.829012Z","iopub.status.idle":"2021-06-24T11:48:11.839503Z","shell.execute_reply.started":"2021-06-24T11:48:11.828988Z","shell.execute_reply":"2021-06-24T11:48:11.838511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# a study may contain multiple images\ntrain_img_cnt_study = train_image_df.StudyInstanceUID.value_counts()\nprint('#study havin more than 1 images in train set', (train_img_cnt_study > 1).sum())\nprint('Max number of images a study can have in train set', train_img_cnt_study.max())","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:11.840911Z","iopub.execute_input":"2021-06-24T11:48:11.841278Z","iopub.status.idle":"2021-06-24T11:48:11.866128Z","shell.execute_reply.started":"2021-06-24T11:48:11.84124Z","shell.execute_reply":"2021-06-24T11:48:11.865277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_img_cnt_study = test_image_df.StudyInstanceUID.value_counts()\nprint('#study havin more than 1 images in test set', (test_img_cnt_study > 1).sum())\nprint('Max number of images a study can have in test set', test_img_cnt_study.max())","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:11.867118Z","iopub.execute_input":"2021-06-24T11:48:11.867427Z","iopub.status.idle":"2021-06-24T11:48:11.87354Z","shell.execute_reply.started":"2021-06-24T11:48:11.867397Z","shell.execute_reply":"2021-06-24T11:48:11.872967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# confirm that bounding boxes only takes 2 categories: opacity/none\ntrain_image_df.label.map(lambda x: set(re.findall('[a-z]+', x))).astype(str).value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:11.87434Z","iopub.execute_input":"2021-06-24T11:48:11.874676Z","iopub.status.idle":"2021-06-24T11:48:11.912936Z","shell.execute_reply.started":"2021-06-24T11:48:11.874652Z","shell.execute_reply":"2021-06-24T11:48:11.911834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"4294+2040","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:11.913958Z","iopub.execute_input":"2021-06-24T11:48:11.914223Z","iopub.status.idle":"2021-06-24T11:48:11.92061Z","shell.execute_reply.started":"2021-06-24T11:48:11.914199Z","shell.execute_reply":"2021-06-24T11:48:11.9196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_image_df['StudyInstanceUID'] = train_image_df.StudyInstanceUID + '_study'","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:11.921823Z","iopub.execute_input":"2021-06-24T11:48:11.922187Z","iopub.status.idle":"2021-06-24T11:48:11.93013Z","shell.execute_reply.started":"2021-06-24T11:48:11.922148Z","shell.execute_reply":"2021-06-24T11:48:11.929445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Counting #study for each classification label')\nprint(train_study_df.drop('id', axis=1).sum(axis=0))","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:11.931268Z","iopub.execute_input":"2021-06-24T11:48:11.931803Z","iopub.status.idle":"2021-06-24T11:48:11.945165Z","shell.execute_reply.started":"2021-06-24T11:48:11.931773Z","shell.execute_reply":"2021-06-24T11:48:11.944192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Generate OD annotations","metadata":{}},{"cell_type":"code","source":"anns = []\nfor i, row in train_image_df.iterrows():\n    image_id = row['id']\n    if(pd.notna(row['boxes'])):\n        labels = ast.literal_eval(row['boxes'])\n        df = pd.DataFrame(labels)\n        df['id'] = image_id\n        anns.append(df)\n    else:\n        anns.append(pd.DataFrame())\n    \nanns = pd.concat(anns)","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:11.946406Z","iopub.execute_input":"2021-06-24T11:48:11.946695Z","iopub.status.idle":"2021-06-24T11:48:18.828145Z","shell.execute_reply.started":"2021-06-24T11:48:11.946669Z","shell.execute_reply":"2021-06-24T11:48:18.827194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"anns['x_min'] = anns['x']\nanns['y_min'] = anns['y']\nanns['x_max'] = anns['x_min'] + anns['width']\nanns['y_max'] = anns['y_min'] + anns['height']\n\nanns['class_name'] = 'opacity'\nanns['class_id'] = 1","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:18.831931Z","iopub.execute_input":"2021-06-24T11:48:18.832218Z","iopub.status.idle":"2021-06-24T11:48:18.840843Z","shell.execute_reply.started":"2021-06-24T11:48:18.832189Z","shell.execute_reply":"2021-06-24T11:48:18.839987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"anns = anns.merge(train_image_df[['id']], on='id', how='right')","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:18.841816Z","iopub.execute_input":"2021-06-24T11:48:18.842065Z","iopub.status.idle":"2021-06-24T11:48:18.865303Z","shell.execute_reply.started":"2021-06-24T11:48:18.842041Z","shell.execute_reply":"2021-06-24T11:48:18.864498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"anns['class_id'] = anns['class_id'].fillna(0)\nanns['class_name'] = anns['class_name'].fillna('none')\n\nanns['class_id'] = anns['class_id'].astype(int)","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:18.868216Z","iopub.execute_input":"2021-06-24T11:48:18.868483Z","iopub.status.idle":"2021-06-24T11:48:18.874873Z","shell.execute_reply.started":"2021-06-24T11:48:18.868457Z","shell.execute_reply":"2021-06-24T11:48:18.873864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"anns = anns.drop(['x','y','width','height'],axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:18.876708Z","iopub.execute_input":"2021-06-24T11:48:18.876949Z","iopub.status.idle":"2021-06-24T11:48:18.889809Z","shell.execute_reply.started":"2021-06-24T11:48:18.876926Z","shell.execute_reply":"2021-06-24T11:48:18.888705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"anns['image_id'] = anns['id'].map(lambda x: x.replace('_image',''))","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:18.891246Z","iopub.execute_input":"2021-06-24T11:48:18.891603Z","iopub.status.idle":"2021-06-24T11:48:18.901498Z","shell.execute_reply.started":"2021-06-24T11:48:18.891574Z","shell.execute_reply":"2021-06-24T11:48:18.900604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"anns_wh = anns.merge(wh_train, on='image_id')","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:18.902893Z","iopub.execute_input":"2021-06-24T11:48:18.90316Z","iopub.status.idle":"2021-06-24T11:48:18.921576Z","shell.execute_reply.started":"2021-06-24T11:48:18.903136Z","shell.execute_reply":"2021-06-24T11:48:18.920627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"anns_wh.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:18.922792Z","iopub.execute_input":"2021-06-24T11:48:18.923092Z","iopub.status.idle":"2021-06-24T11:48:18.942725Z","shell.execute_reply.started":"2021-06-24T11:48:18.923062Z","shell.execute_reply":"2021-06-24T11:48:18.941751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"anns_wh = anns_wh[['image_id', 'width', 'height', 'class_id', 'class_name', 'x_min', 'y_min', 'x_max', 'y_max']]","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:18.94416Z","iopub.execute_input":"2021-06-24T11:48:18.944475Z","iopub.status.idle":"2021-06-24T11:48:18.953519Z","shell.execute_reply.started":"2021-06-24T11:48:18.944445Z","shell.execute_reply":"2021-06-24T11:48:18.952651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SIZE = 512\n\nfor col in [ 'x_min', 'x_max']:\n    anns_wh[col] = anns_wh[col] / anns_wh['width'] * 512\n    \nfor col in ['y_min', 'y_max']:\n    anns_wh[col] = anns_wh[col] / anns_wh['height'] * 512\n    \n    \nanns_wh['width'] = 512\nanns_wh['height'] = 512","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:18.954637Z","iopub.execute_input":"2021-06-24T11:48:18.954886Z","iopub.status.idle":"2021-06-24T11:48:18.965909Z","shell.execute_reply.started":"2021-06-24T11:48:18.954861Z","shell.execute_reply":"2021-06-24T11:48:18.965251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('#images having at least 1 bbox not none:', (anns_wh.groupby('image_id').class_id.sum() > 0 ).sum())","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:18.966933Z","iopub.execute_input":"2021-06-24T11:48:18.967323Z","iopub.status.idle":"2021-06-24T11:48:18.981232Z","shell.execute_reply.started":"2021-06-24T11:48:18.967285Z","shell.execute_reply":"2021-06-24T11:48:18.980291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"anns_wh.to_csv('bbox_train_meta_512.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:18.982537Z","iopub.execute_input":"2021-06-24T11:48:18.982909Z","iopub.status.idle":"2021-06-24T11:48:19.054479Z","shell.execute_reply.started":"2021-06-24T11:48:18.98288Z","shell.execute_reply":"2021-06-24T11:48:19.053754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wh_test = wh_test[['image_id']]\nwh_test['width'] = 512\nwh_test['height'] = 512","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:19.055993Z","iopub.execute_input":"2021-06-24T11:48:19.056626Z","iopub.status.idle":"2021-06-24T11:48:19.063419Z","shell.execute_reply.started":"2021-06-24T11:48:19.056582Z","shell.execute_reply":"2021-06-24T11:48:19.062751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wh_test.to_csv('bbox_test_meta_512.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:19.064789Z","iopub.execute_input":"2021-06-24T11:48:19.065216Z","iopub.status.idle":"2021-06-24T11:48:19.077641Z","shell.execute_reply.started":"2021-06-24T11:48:19.065187Z","shell.execute_reply":"2021-06-24T11:48:19.076841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport cv2\n\ncolor_palletes = [\n    (123, 141, 245),\n    (123, 193, 239),\n    (230, 193, 239),\n    (230, 79, 239),\n    (71, 79, 239),\n    (71, 170, 90),\n    (201, 170, 38),\n    (201, 47, 46),\n    (201, 240, 219),\n    (0, 43, 255),\n    (0, 255, 255),\n    (102, 114, 41),\n    (255, 242, 235),\n    (54, 0, 0)\n]\n\n\ndef draw_boxes(img_id, meta_df, img_folder, plot_rad=False, figsize=(10,15), return_img=False):\n    df = meta_df[meta_df.image_id == img_id]\n    img = cv2.imread(img_folder + '/' + img_id + '.jpg')\n    if(not return_img):\n        print(meta_df[meta_df.image_id==img_id][['class_name', 'x_min', 'y_min', 'x_max', 'y_max', 'width', 'height']])\n    for i, row in meta_df[meta_df.image_id == img_id].iterrows():\n        x1, y1, x2, y2, cls_id, cls_name = int(row['x_min']), int(row['y_min']), \\\n                                            int(row['x_max']), int(row['y_max']), \\\n                                            int(row['class_id']), row['class_name']\n        \n        c = color_palletes[cls_id]\n        cv2.rectangle(img, (x1, y1), (x2, y2), c, 2)\n        if(plot_rad):\n            rad_id = row['rad_id']\n            cv2.putText(img, cls_name + '-' + rad_id, (x1, y1), cv2.FONT_HERSHEY_SIMPLEX, 0.7, c, 2)\n        else:\n            cv2.putText(img, cls_name, (x1, y1), cv2.FONT_HERSHEY_SIMPLEX, 0.7, c, 2)\n    \n    if(not return_img):\n        plt.figure(figsize=figsize)\n        plt.imshow(img)\n    else:\n        return img\n    \ndef show_values_on_bars(axs, h_v=\"v\", space=0.4):\n    def _show_on_single_plot(ax):\n        if h_v == \"v\":\n            for p in ax.patches:\n                _x = p.get_x() + p.get_width() / 2\n                _y = p.get_y() + p.get_height()\n                value = int(p.get_height())\n                ax.text(_x, _y, value, ha=\"center\") \n        elif h_v == \"h\":\n            for p in ax.patches:\n                _x = p.get_x() + p.get_width() + float(space)\n                _y = p.get_y() + p.get_height()\n                value = int(p.get_width())\n                ax.text(_x, _y, value, ha=\"left\")\n\n    if isinstance(axs, np.ndarray):\n        for idx, ax in np.ndenumerate(axs):\n            _show_on_single_plot(ax)\n    else:\n        _show_on_single_plot(axs)\n        ","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:19.078936Z","iopub.execute_input":"2021-06-24T11:48:19.079571Z","iopub.status.idle":"2021-06-24T11:48:19.241521Z","shell.execute_reply.started":"2021-06-24T11:48:19.079512Z","shell.execute_reply":"2021-06-24T11:48:19.240802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"im_folder = '../input/siim-covid19-resized-to-512px-jpg/train'\ndraw_boxes('ffd9b6cf2961', anns_wh, im_folder)","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:19.242517Z","iopub.execute_input":"2021-06-24T11:48:19.24298Z","iopub.status.idle":"2021-06-24T11:48:19.596048Z","shell.execute_reply.started":"2021-06-24T11:48:19.242939Z","shell.execute_reply":"2021-06-24T11:48:19.595389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nax = sns.countplot(anns_wh[anns_wh.class_id==1].image_id.value_counts())\nplt.xlabel('k')\nplt.ylabel('Number of images having k bboxes')\nplt.title('How many bboxes in 1 images (only consider postive image)')\nshow_values_on_bars(ax)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:19.597009Z","iopub.execute_input":"2021-06-24T11:48:19.597413Z","iopub.status.idle":"2021-06-24T11:48:20.458442Z","shell.execute_reply.started":"2021-06-24T11:48:19.597372Z","shell.execute_reply":"2021-06-24T11:48:20.457542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. The relation ship between classification label and object detection label","metadata":{}},{"cell_type":"code","source":"study_lookup = train_image_df.set_index('id')['StudyInstanceUID'].to_dict()","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:20.459478Z","iopub.execute_input":"2021-06-24T11:48:20.459763Z","iopub.status.idle":"2021-06-24T11:48:20.468639Z","shell.execute_reply.started":"2021-06-24T11:48:20.459735Z","shell.execute_reply":"2021-06-24T11:48:20.467667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"anns_wh_study = anns_wh.copy()\nanns_wh_study['StudyInstanceUID'] = anns_wh_study['image_id'].map(lambda x: study_lookup[x+'_image'])\nanns_wh_study = anns_wh_study.merge(train_study_df.rename({'id':'StudyInstanceUID'}, axis=1), on='StudyInstanceUID')","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:20.470264Z","iopub.execute_input":"2021-06-24T11:48:20.470717Z","iopub.status.idle":"2021-06-24T11:48:20.493743Z","shell.execute_reply.started":"2021-06-24T11:48:20.470677Z","shell.execute_reply":"2021-06-24T11:48:20.492857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"anns_wh_study[anns_wh_study['Negative for Pneumonia']==1]","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:20.495179Z","iopub.execute_input":"2021-06-24T11:48:20.495554Z","iopub.status.idle":"2021-06-24T11:48:20.520751Z","shell.execute_reply.started":"2021-06-24T11:48:20.495514Z","shell.execute_reply":"2021-06-24T11:48:20.519901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"anns_wh_study[anns_wh_study.class_id==1]['Negative for Pneumonia'].max()","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:20.521938Z","iopub.execute_input":"2021-06-24T11:48:20.522178Z","iopub.status.idle":"2021-06-24T11:48:20.530142Z","shell.execute_reply.started":"2021-06-24T11:48:20.522153Z","shell.execute_reply":"2021-06-24T11:48:20.529499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CLF_COLS = ['Negative for Pneumonia', 'Typical Appearance', 'Indeterminate Appearance', 'Atypical Appearance']","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:20.531239Z","iopub.execute_input":"2021-06-24T11:48:20.531488Z","iopub.status.idle":"2021-06-24T11:48:20.537623Z","shell.execute_reply.started":"2021-06-24T11:48:20.531463Z","shell.execute_reply":"2021-06-24T11:48:20.536954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"anns_wh_study.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:20.538881Z","iopub.execute_input":"2021-06-24T11:48:20.539427Z","iopub.status.idle":"2021-06-24T11:48:20.560537Z","shell.execute_reply.started":"2021-06-24T11:48:20.539386Z","shell.execute_reply":"2021-06-24T11:48:20.55975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Number of images (clf label = x | the image does not have opacity bbox(es)):')\nprint(anns_wh_study[anns_wh_study.class_id==0].drop_duplicates('image_id')[CLF_COLS].sum().rename('x'))","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:20.561655Z","iopub.execute_input":"2021-06-24T11:48:20.562154Z","iopub.status.idle":"2021-06-24T11:48:20.574713Z","shell.execute_reply.started":"2021-06-24T11:48:20.562114Z","shell.execute_reply":"2021-06-24T11:48:20.573571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Number of images (clf label = x | the image has opacity bbox(es)):')\nprint(anns_wh_study[anns_wh_study.class_id==1].drop_duplicates('image_id')[CLF_COLS].sum().rename('x'))","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:20.575793Z","iopub.execute_input":"2021-06-24T11:48:20.576067Z","iopub.status.idle":"2021-06-24T11:48:20.587367Z","shell.execute_reply.started":"2021-06-24T11:48:20.576012Z","shell.execute_reply":"2021-06-24T11:48:20.586675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Number of images (the image has/does not have opacity bbox(es)) | clf label = Negative for Pneumonia:')\nprint(anns_wh_study[anns_wh_study['Negative for Pneumonia']==1].drop_duplicates('image_id').class_id.value_counts().sort_index())","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:20.5883Z","iopub.execute_input":"2021-06-24T11:48:20.58853Z","iopub.status.idle":"2021-06-24T11:48:20.600089Z","shell.execute_reply.started":"2021-06-24T11:48:20.588508Z","shell.execute_reply":"2021-06-24T11:48:20.599288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Number of images (the image has/does not have opacity bbox(es)) | clf label = Typical Appearance:')\nprint(anns_wh_study[anns_wh_study['Typical Appearance']==1].drop_duplicates('image_id').class_id.value_counts().sort_index())","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:20.601099Z","iopub.execute_input":"2021-06-24T11:48:20.601704Z","iopub.status.idle":"2021-06-24T11:48:20.611551Z","shell.execute_reply.started":"2021-06-24T11:48:20.601672Z","shell.execute_reply":"2021-06-24T11:48:20.610969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Number of images (the image has/does not have opacity bbox(es)) | clf label = Indeterminate Appearance:')\nprint(anns_wh_study[anns_wh_study['Indeterminate Appearance']==1].drop_duplicates('image_id').class_id.value_counts().sort_index())","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:20.612452Z","iopub.execute_input":"2021-06-24T11:48:20.612878Z","iopub.status.idle":"2021-06-24T11:48:20.62091Z","shell.execute_reply.started":"2021-06-24T11:48:20.612837Z","shell.execute_reply":"2021-06-24T11:48:20.620274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Number of images (the image has/does not have opacity bbox(es)) | clf label = Atypical Appearance:')\nprint(anns_wh_study[anns_wh_study['Atypical Appearance']==1].drop_duplicates('image_id').class_id.value_counts().sort_index())","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:20.621803Z","iopub.execute_input":"2021-06-24T11:48:20.62218Z","iopub.status.idle":"2021-06-24T11:48:20.635067Z","shell.execute_reply.started":"2021-06-24T11:48:20.622141Z","shell.execute_reply":"2021-06-24T11:48:20.634123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_image_df","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:20.636144Z","iopub.execute_input":"2021-06-24T11:48:20.636404Z","iopub.status.idle":"2021-06-24T11:48:20.650365Z","shell.execute_reply.started":"2021-06-24T11:48:20.636381Z","shell.execute_reply":"2021-06-24T11:48:20.649605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Counting number of images having bbox opacity in each study\nprint('How many images with opacity bboxes on train set for each study? Maximum number is:')\ntrain_image_df.groupby('StudyInstanceUID').apply(lambda df: df.boxes.notnull().sum()).max()","metadata":{"execution":{"iopub.status.busy":"2021-06-24T11:48:20.651325Z","iopub.execute_input":"2021-06-24T11:48:20.651616Z","iopub.status.idle":"2021-06-24T11:48:22.442277Z","shell.execute_reply.started":"2021-06-24T11:48:20.651583Z","shell.execute_reply":"2021-06-24T11:48:22.441645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}