{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# SIIM-FISABIO-RSNA COVID-19 Detection\n\n> **Identify and localize COVID-19 abnormalities on chest radiographs**\n\nIn this competition, you’ll identify and localize COVID-19 abnormalities on chest radiographs. In particular, you'll categorize the radiographs as negative for pneumonia or typical, indeterminate, or atypical for COVID-19. You and your model will work with imaging data and annotations from a group of radiologists.\n","metadata":{}},{"cell_type":"markdown","source":"**If you liked this notebook, please feel free to upvote. It is too much appreciated.**","metadata":{}},{"cell_type":"code","source":"!pip -q install gdcm","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:55:44.139493Z","iopub.execute_input":"2021-05-23T03:55:44.139926Z","iopub.status.idle":"2021-05-23T03:55:50.753076Z","shell.execute_reply.started":"2021-05-23T03:55:44.139891Z","shell.execute_reply":"2021-05-23T03:55:50.751559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"import os\nimport random\nimport pydicom\nfrom pathlib import Path\nfrom glob import glob\nfrom tqdm.notebook import tqdm\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport cv2\nfrom skimage import exposure\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2021-05-23T01:09:44.200143Z","iopub.execute_input":"2021-05-23T01:09:44.20057Z","iopub.status.idle":"2021-05-23T01:09:44.207764Z","shell.execute_reply.started":"2021-05-23T01:09:44.200536Z","shell.execute_reply":"2021-05-23T01:09:44.206182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Some color settings for the output plots.\nLABEL2COLOR = [(249, 192, 12), (0, 185, 241), (114, 0, 218), (249,50,12)]\nCOLOR_PALETTE = [\"#F9C00C\", \"#00B9F1\", \"#7200DA\", \"#F9320C\"]","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:07:55.896498Z","iopub.execute_input":"2021-05-23T03:07:55.896842Z","iopub.status.idle":"2021-05-23T03:07:55.902125Z","shell.execute_reply.started":"2021-05-23T03:07:55.896812Z","shell.execute_reply":"2021-05-23T03:07:55.900777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reading the input data\nBASE_DATA_PATH = Path(\"../input/siim-covid19-detection/\")\n!ls {BASE_DATA_PATH}\n\ndf_train_img = pd.read_csv(BASE_DATA_PATH / \"train_image_level.csv\")\ndf_train_study = pd.read_csv(BASE_DATA_PATH / \"train_study_level.csv\")\ndf_sub = pd.read_csv(BASE_DATA_PATH / \"sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:48:50.687395Z","iopub.execute_input":"2021-05-23T03:48:50.687846Z","iopub.status.idle":"2021-05-23T03:48:51.475662Z","shell.execute_reply.started":"2021-05-23T03:48:50.687812Z","shell.execute_reply":"2021-05-23T03:48:51.474324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Bbox labels per image.\ndf_train_img.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:49:48.511589Z","iopub.execute_input":"2021-05-23T03:49:48.511989Z","iopub.status.idle":"2021-05-23T03:49:48.525267Z","shell.execute_reply.started":"2021-05-23T03:49:48.511957Z","shell.execute_reply":"2021-05-23T03:49:48.524398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Labels of the studies\ndf_train_study.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:49:19.145036Z","iopub.execute_input":"2021-05-23T03:49:19.145673Z","iopub.status.idle":"2021-05-23T03:49:19.156922Z","shell.execute_reply.started":"2021-05-23T03:49:19.145634Z","shell.execute_reply":"2021-05-23T03:49:19.155741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Merging the Image and Study CSVs","metadata":{}},{"cell_type":"code","source":"CLASS_MAP = {\n    0: \"Negative for Pneumonia\",\n    1: \"Typical Appearance\",\n    2: \"Indeterminate Appearance\",\n    3: \"Atypical Appearance\"\n}\n\n# Create a label column.\ndf_train_study[\"class_id\"] = df_train_study.iloc[:, 1:].values.argmax(1)\n\n# Remove the study part from the ids.\ndf_train_study[\"StudyInstanceUID\"] = df_train_study[\"id\"].apply(lambda x: x[:-6])\ndf_train_study = df_train_study[[\"StudyInstanceUID\", \"class_id\"]]\n\n# Merge the two train csvs together.\ndf_train = pd.merge(df_train_img, df_train_study, on=\"StudyInstanceUID\")\n\n# Map the class ids to original names for plotting.\ndf_train[\"class_label\"] = df_train[\"class_id\"].map(CLASS_MAP)\n\n# Generating the image paths from given StudyInstanceUID.\ntrain_dir = BASE_DATA_PATH / \"train\"\ndf_train[\"path\"] = df_train[\"StudyInstanceUID\"].apply(lambda s_id: glob(os.path.join(train_dir, s_id + \"/*/*\"))[0])","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:25:02.4383Z","iopub.execute_input":"2021-05-23T03:25:02.438878Z","iopub.status.idle":"2021-05-23T03:25:02.463781Z","shell.execute_reply.started":"2021-05-23T03:25:02.438818Z","shell.execute_reply":"2021-05-23T03:25:02.462797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:25:04.090053Z","iopub.execute_input":"2021-05-23T03:25:04.090684Z","iopub.status.idle":"2021-05-23T03:25:04.111345Z","shell.execute_reply.started":"2021-05-23T03:25:04.090619Z","shell.execute_reply":"2021-05-23T03:25:04.109207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[\"class_id\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-05-23T02:50:57.845514Z","iopub.execute_input":"2021-05-23T02:50:57.845879Z","iopub.status.idle":"2021-05-23T02:50:57.857694Z","shell.execute_reply.started":"2021-05-23T02:50:57.845846Z","shell.execute_reply":"2021-05-23T02:50:57.856287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Class Distributions","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(21, 10))\ndf_train[\"label\"]\nax = sns.countplot(x=\"class_label\", data=df_train, palette=COLOR_PALETTE)\nplt.title('Percentage of the Classes', fontsize=20)\n\ntotal = len(df_train)\n\nfor p in ax.patches:\n    percentage = '{:.1f}%'.format(100 * p.get_height()/total)\n    x = p.get_x() + p.get_width() / 3\n    y = p.get_height() + 10\n    ax.annotate(percentage, (x, y), weight=\"bold\", fontsize=20)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:25:19.968905Z","iopub.execute_input":"2021-05-23T03:25:19.969253Z","iopub.status.idle":"2021-05-23T03:25:20.180152Z","shell.execute_reply.started":"2021-05-23T03:25:19.969223Z","shell.execute_reply":"2021-05-23T03:25:20.179139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Bbox Distributions","metadata":{}},{"cell_type":"code","source":"bbox_counts = df_train.label.str.count(\"opacity\")\nopacity_count = (bbox_counts > 0).sum()\nnone_count = len(df_train) - opacity_count\n\ndf_bbox_counts = pd.DataFrame({\"label\": [\"opacity\", \"none\"], \"count\": [opacity_count, none_count]})\ndf_bbox_counts","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:45:20.506849Z","iopub.execute_input":"2021-05-23T03:45:20.507309Z","iopub.status.idle":"2021-05-23T03:45:20.529974Z","shell.execute_reply.started":"2021-05-23T03:45:20.507266Z","shell.execute_reply":"2021-05-23T03:45:20.528764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(14, 8))\nsns.barplot(data=df_bbox_counts, x=\"count\", y=\"label\");","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:45:58.201536Z","iopub.execute_input":"2021-05-23T03:45:58.201918Z","iopub.status.idle":"2021-05-23T03:45:58.329203Z","shell.execute_reply.started":"2021-05-23T03:45:58.201886Z","shell.execute_reply":"2021-05-23T03:45:58.32842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# BBox Distribution per Image","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(14, 8))\nsns.histplot(bbox_counts);","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:46:03.920123Z","iopub.execute_input":"2021-05-23T03:46:03.920526Z","iopub.status.idle":"2021-05-23T03:46:04.18798Z","shell.execute_reply.started":"2021-05-23T03:46:03.920491Z","shell.execute_reply":"2021-05-23T03:46:04.186651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Minimum number of bboxes per image: {min(bbox_counts)}\")\nprint(f\"Maximum number of bboxes per image: {max(bbox_counts)}\")","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:44:42.302087Z","iopub.execute_input":"2021-05-23T03:44:42.30245Z","iopub.status.idle":"2021-05-23T03:44:42.31082Z","shell.execute_reply.started":"2021-05-23T03:44:42.302418Z","shell.execute_reply":"2021-05-23T03:44:42.309027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper Functions","metadata":{}},{"cell_type":"code","source":"def dicom2array(path, voi_lut=True, fix_monochrome=True):\n    dicom = pydicom.read_file(path)\n    # VOI LUT (if available by DICOM device) is used to\n    # transform raw DICOM data to \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    return data\n        \n    \ndef plot_img(img, size=(7, 7), is_rgb=True, title=\"\", cmap='gray'):\n    plt.figure(figsize=size)\n    plt.imshow(img, cmap=cmap)\n    plt.suptitle(title)\n    plt.show()\n\n\ndef plot_imgs(imgs, labels=None, cols=4, size=7, is_rgb=True, title=\"\", cmap='gray', img_size=(500,500)):\n    rows = len(imgs)//cols + 1\n    fig = plt.figure(figsize=(cols*size, rows*size))\n    if labels is None: labels = [None] * len(imgs)\n        \n    for i, (img, label) in enumerate(zip(imgs, labels)):\n        if img_size is not None:\n            img = cv2.resize(img, img_size)\n        fig.add_subplot(rows, cols, i+1)\n        plt.imshow(img, cmap=cmap)\n        if label is not None:\n            plt.title(label)\n    plt.suptitle(title)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-05-23T02:40:09.68452Z","iopub.execute_input":"2021-05-23T02:40:09.685022Z","iopub.status.idle":"2021-05-23T02:40:09.697303Z","shell.execute_reply.started":"2021-05-23T02:40:09.68499Z","shell.execute_reply":"2021-05-23T02:40:09.696022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plotting X-ray Images","metadata":{}},{"cell_type":"code","source":"imgs = [dicom2array(path) for path in df_train[\"path\"][:4]]\nplot_imgs(imgs)","metadata":{"execution":{"iopub.status.busy":"2021-05-23T02:40:37.11259Z","iopub.execute_input":"2021-05-23T02:40:37.113047Z","iopub.status.idle":"2021-05-23T02:40:38.582087Z","shell.execute_reply.started":"2021-05-23T02:40:37.113002Z","shell.execute_reply":"2021-05-23T02:40:38.58095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plotting the Bounding Boxes","metadata":{}},{"cell_type":"code","source":"img_ids = df_train['id'].values\nclass_ids = df_train['class_id'].unique()\n\nscale = 5\nthickness = 7\n\npaths = df_train[\"path\"]\nall_boxes = df_train.label.apply(lambda x: [x.split()[idx:idx+6] for idx in range(0, len(x.split()), 6)])\n\nimgs, labels = [], []\n\nfor i in np.random.choice(range(len(df_train)), 8):\n    img = dicom2array(path=paths[i])\n    img = cv2.resize(img, None, fx=1/scale, fy=1/scale)\n    img = np.stack([img, img, img], axis=-1)\n    \n    boxes = all_boxes[i]\n    img_labels = [df_train.class_id[i]] * len(boxes)\n    for label_id, box in zip(img_labels, boxes):\n        color = LABEL2COLOR[label_id]\n        img = cv2.rectangle(\n            img,\n            (int(float(box[2]) / scale), int(float(box[3]) / scale)),\n            (int(float(box[4]) / scale), int(float(box[5]) / scale)),\n            color, thickness\n    )\n    img = cv2.resize(img, (500, 500))\n    imgs.append(img)\n    labels.append(CLASS_MAP[label_id])\n    \nplot_imgs(imgs, labels, cmap=None)","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:08:07.646836Z","iopub.execute_input":"2021-05-23T03:08:07.647179Z","iopub.status.idle":"2021-05-23T03:08:12.629623Z","shell.execute_reply.started":"2021-05-23T03:08:07.647148Z","shell.execute_reply":"2021-05-23T03:08:12.627443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### To be continued...","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:47:21.56382Z","iopub.execute_input":"2021-05-23T03:47:21.564255Z","iopub.status.idle":"2021-05-23T03:47:21.568763Z","shell.execute_reply.started":"2021-05-23T03:47:21.564217Z","shell.execute_reply":"2021-05-23T03:47:21.567909Z"}}}]}