{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"

Visual In-Depth EDA – SIIM – COVID19 Detection

\n\n

Exploratory Data Analysis (EDA)

\n
CREATED BY: DARIEN SCHETTLER

\n\n
\n\n
💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠
\n
🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫
\n
💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠
\n
🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪
\n
🩺🩺🩺🩺🦠🦠🦠🩺🩺🩺🩺
\n
🩺🩺🩺
\n\n
\n","metadata":{"papermill":{"duration":0.044805,"end_time":"2021-02-05T02:17:44.538251","exception":false,"start_time":"2021-02-05T02:17:44.493446","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"
\n\n

TABLE OF CONTENTS

\n\n---\n\n

0    IMPORTS

\n\n---\n\n

1    BACKGROUND INFORMATION

\n\n---\n\n

2    SETUP

\n\n---\n\n

3    HELPER FUNCTIONS

\n\n---\n\n

4    TABULAR DATA

\n\n---\n\n

5    IMAGE DATA

\n\n---\n\n

6    IDENTIFY DUPLICATES

\n\n---\n\n
","metadata":{"papermill":{"duration":0.043006,"end_time":"2021-02-05T02:17:44.624815","exception":false,"start_time":"2021-02-05T02:17:44.581809","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"
\n\n\n\n

0  IMPORTS

","metadata":{"papermill":{"duration":0.04267,"end_time":"2021-02-05T02:17:44.71043","exception":false,"start_time":"2021-02-05T02:17:44.66776","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print(\"\\n... PIP/APT INSTALLS STARTING ...\")\n!cp /kaggle/input/gdcm-conda-install/gdcm.tar .\n!tar -xvzf gdcm.tar\n!conda install --offline ./gdcm/gdcm-2.8.9-py37h71b2a6d_0.tar.bz2\n!rm -rf ./gdcm.tar\n\n# !conda install -c conda-forge gdcm -y\n!pip install pandarallel\nprint(\"... PIP/APT INSTALLS COMPLETE ...\\n\")\n\n\nprint(\"\\n... IMPORTS STARTING ...\\n\")\nprint(\"\\n\\tVERSION INFORMATION\")\n# Machine Learning and Data Science Imports\nimport tensorflow as tf; print(f\"\\t\\t– TENSORFLOW VERSION: {tf.__version__}\");\nimport tensorflow_addons as tfa; print(f\"\\t\\t– TENSORFLOW ADDONS VERSION: {tfa.__version__}\");\nimport pandas as pd; pd.options.mode.chained_assignment = None; pd.set_option('max_columns', 100);\nimport numpy as np; print(f\"\\t\\t– NUMPY VERSION: {np.__version__}\");\n\n# Other Competition Related Imports\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nfrom pandarallel import pandarallel; pandarallel.initialize();\n\n# Built In Imports\nfrom kaggle_datasets import KaggleDatasets\nfrom collections import Counter\nfrom datetime import datetime\nfrom glob import glob\nimport warnings\nimport requests\nimport imageio\nimport IPython\nimport urllib\nimport zipfile\nimport pickle\nimport random\nimport shutil\nimport string\nimport math\nimport time\nimport gzip\nimport ast\nimport sys\nimport io\nimport os\nimport gc\nimport re\n\n# Visualization Imports\nfrom matplotlib.colors import ListedColormap\nimport matplotlib.patches as patches\nimport plotly.graph_objects as go\nimport matplotlib.pyplot as plt\nfrom tqdm.notebook import tqdm; tqdm.pandas();\nimport plotly.express as px\nimport seaborn as sns\nfrom PIL import Image\nimport matplotlib; print(f\"\\t\\t– MATPLOTLIB VERSION: {matplotlib.__version__}\");\nimport plotly\nimport PIL\nimport cv2\nprint(\"\\n\\n... IMPORTS COMPLETE ...\\n\")\n \n\nprint(\"\\n... SEEDING FOR DETERMINISTIC BEHAVIOUR ...\")\ndef seed_it_all(seed=7):\n \"\"\" Attempt to be Reproducible \"\"\"\n os.environ['PYTHONHASHSEED'] = str(seed)\n random.seed(seed)\n np.random.seed(seed)\n tf.random.set_seed(seed)\n\nseed_it_all()\nprint(\"... SEEDING COMPLETE ...\\n\\n\")\n\n\nprint(\"\\n... SETTING PRESETS STARTING...\")\nFIG_FONT = dict(family=\"Helvetica, Arial\", size=14, color=\"#7f7f7f\")\nLABEL_LIST = ['atypical', 'indeterminate', 'negative', 'typical']\nLABEL_COLORS = [px.colors.label_rgb(px.colors.convert_to_RGB_255(x)) for x in sns.color_palette(\"Spectral\", 4)]\n# LABEL_COLORS_WOUT_NO_FINDING = LABEL_COLORS[:8]+LABEL_COLORS[9:]\nprint(\"... SETTING PRESETS COMPLETE...\\n\\n\")","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":41.017298,"end_time":"2021-02-05T02:18:25.770952","exception":false,"start_time":"2021-02-05T02:17:44.753654","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-04T20:44:39.419211Z","iopub.execute_input":"2021-06-04T20:44:39.419734Z","iopub.status.idle":"2021-06-04T20:45:28.480711Z","shell.execute_reply.started":"2021-06-04T20:44:39.419640Z","shell.execute_reply":"2021-06-04T20:45:28.479668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n\n\n\n

1  BACKGROUND INFORMATION

","metadata":{"papermill":{"duration":0.045609,"end_time":"2021-02-05T02:18:25.862323","exception":false,"start_time":"2021-02-05T02:18:25.816714","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"**
\n\n

1.1 THE DATA

\n\n---\n\nBACKGROUND INFORMATION\n\nIn this competition, we are identifying and localizing COVID-19 abnormalities on chest radiographs.
**This is an object detection and classification problem.**\n\nFor each test image, you will be predicting a bounding box and class for all findings. \n* If you predict that there are no findings, you should create a prediction of **`\"none 1 0 0 1 1\"`** \n * \"none\" is the class ID for no finding, and this provides a one-pixel bounding box with a confidence of 1.0\n\nFurther, for each test study, you should make a determination within the following labels:\n\n> **`'Negative for Pneumonia', 'Typical Appearance', 'Indeterminate Appearance', 'Atypical Appearance'`**\n\nTo make a prediction of one of the above labels, create a prediction string similar to the \"none\" class above: \n* i.e. **`atypical 1 0 0 1 1`**\n\n---\n\n**MESSAGE FROM THE COMPETITION HOST ON LABEL AND BBOX DETAILS:**\n\nIn this challenge, the chest radiographs (CXRs) were categorized using a specific grading schema, based on a published paper:\n\n[**Litmanovich DE, Chung M, Kirkbride RR, Kicska G, Kanne JP. Review of chest radiograph findings of COVID-19 pneumonia and suggested reporting language. Journal of thoracic imaging. 2020 Nov 14;35(6):354-60.**](https://journals.lww.com/thoracicimaging/Fulltext/2020/11000/Review_of_Chest_Radiograph_Findings_of_COVID_19.4.aspx)\n\nPer the grading schema, chest radiographs are classified into one of four categories, which are mutually exclusive:\n\n1. **Typical Appearance**: Multifocal bilateral, peripheral opacities with rounded morphology, lower lung–predominant distribution\n2. **Indeterminate Appearance**: Absence of typical findings AND unilateral, central or upper lung predominant distribution\n3. **Atypical Appearance**: Pneumothorax, pleural effusion, pulmonary edema, lobar consolidation, solitary lung nodule or mass, diffuse tiny nodules, cavity\n4. **Negative for Pneumonia**: No lung opacities\n\nBounding boxes were placed on lung opacities, whether typical or indeterminate. Bounding boxes were also placed on some atypical findings including solitary lobar consolidation, nodules/masses, and cavities. Bounding boxes were not placed on pleural effusions, or pneumothoraces. No bounding boxes were placed for the negative for pneumonia category.\n\nIn cases of multiple adjacent opacities, we opted for one large bounding box, rather than multiple adjacent smaller boxes, to improve consistency in the labeling.\n\nAnnotators did have access to the COVID status for each patient, but were asked to adhere to the grading system above irrespective of the status. As such, some patients who were COVID negative still had chest radiographs with typical appearances. Similarly, some patients who were COVID positive had atypical appearances, or were negative for pneumonia (no lung opacities), because the grading system is based off the chest radiographic findings alone.\n\nThe goal in this challenge is to determine the appropriate category for each radiograph, as well as localize the lung opacities with a bounding box prediction.\n\n---\n\nThe images are in DICOM format, which means they contain additional data that might be useful for visualizing and classifying.\nNote that the images are in **DICOM** format, which means they contain additional data that might be useful for visualizing and classifying.\n\n![Example Radiographs](https://i.imgur.com/QWmbhXx.png)\n\n
\n\nDATASET INFORMATION\n\nThe **train dataset** comprises **`6,334`** chest scans in **DICOM** format, which were de-identified to protect patient privacy. \n\nNote that all images are stored in paths with the form **`study/series/image`**. \n* The **`study`** ID here relates directly to the study-level predictions\n* the **`image`** ID is the ID used for image-level predictions\n\nThe **test dataset** is of roughly the same scale as the training dataset. \n* As this is a kernels only competition we shsould plan accordingly\n* i.e. we should be able to infer on the entirety of the training dataset within the submission kernel\n\n
\n\nDATA FILES\n> **`train_study_level.csv`**\n> * **`id`** - unique study identifier\n> * **Negative for Pneumonia** - **`1`** if the study is negative for pneumonia, **`0`** otherwise\n> * **Typical Appearance** - **`1`** if the study has this appearance, **`0`** otherwise\n> * **Indeterminate Appearance** - **`1`** if the study has this appearance, **`0`** otherwise\n> * **Atypical Appearance** - **`1`** if the study has this appearance, **`0`** otherwise\n\n> **`train_image_level.csv`**\n> * **`id`** - unique image identifier\n> * **`boxes`** - bounding boxes in easily-readable dictionary format\n> * **`label`** - the correct prediction label for the provided bounding boxes","metadata":{"papermill":{"duration":0.04413,"end_time":"2021-02-05T02:18:25.953034","exception":false,"start_time":"2021-02-05T02:18:25.908904","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"
\n\n

1.2 THE GOAL

\n\n---\n\nIn this competition, you’ll identify and localize COVID-19 abnormalities on chest radiographs. In particular, you'll categorize the radiographs as one of a possible **`4`** categories. \n\nIn this competition, we are making predictions at both a study (multi-image) and image level.\n* **`negative for pneumonia`** or **`typical`**, **`indeterminate`**, or **`atypical`** \n \nYou'll work with a dataset consisting of **`8,781`** scans that have been annotated by experienced radiologists. You can train your model with **`6,334`** independently-labeled images and you will be evaluated on a test set of **`2,447`** images. \n\nThe challenge uses the standard PASCAL VOC 2010 mean Average Precision (mAP) at IoU > 0.5.\n* Note that the linked document describes VOC 2012, which differs in some minor ways (e.g. there is no concept of \"difficult\" classes in VOC 2010). The P/R curve and AP calculations remain the same.\n\n
\n\n
🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺
\n\n
If successful, you'll help radiologists diagnose the millions of COVID-19 patients more confidently and quickly.
\n\n
🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺
","metadata":{"papermill":{"duration":0.044853,"end_time":"2021-02-05T02:18:26.044977","exception":false,"start_time":"2021-02-05T02:18:26.000124","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"
\n\n

1.3 ADDITIONAL INFORMATION ON ABNORMALITIES

\n\n---\n\n
Negative for Pneumonia\n* No lung opacities\n\n
Typical Appearance\n* Multifocal bilateral, peripheral opacities with rounded morphology, lower lung–predominant distribution\n\n
Indeterminate Appearance\n* Absence of typical findings AND unilateral, central or upper lung predominant distribution\n\n
Atypical Appearance\n* Pneumothorax, pleural effusion, pulmonary edema, lobar consolidation, solitary lung nodule or mass, diffuse tiny nodules, cavity","metadata":{"papermill":{"duration":0.046066,"end_time":"2021-02-05T02:18:26.137054","exception":false,"start_time":"2021-02-05T02:18:26.090988","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"
\n\n\n\n

2  NOTEBOOK SETUP

","metadata":{"papermill":{"duration":0.045324,"end_time":"2021-02-05T02:18:26.226344","exception":false,"start_time":"2021-02-05T02:18:26.18102","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def unpack_bbox_column(df_row):\n \"\"\" go from xmin,ymin,width,height --> xmin,ymin,xmax,ymax \"\"\"\n df_row[\"xmin\"] = df_row[\"boxes\"][\"x\"]\n df_row[\"ymin\"] = df_row[\"boxes\"][\"y\"]\n df_row[\"xmax\"] = df_row[\"boxes\"][\"x\"]+df_row[\"boxes\"][\"width\"]\n df_row[\"ymax\"] = df_row[\"boxes\"][\"y\"]+df_row[\"boxes\"][\"height\"]\n return df_row\n\ndef get_human_label(row):\n \"\"\" Get human readable label for visualization purposes \"\"\"\n for lbl in [\"negative\", \"typical\", \"indeterminate\", \"atypical\"]:\n if row[lbl]:\n row[\"human_label\"] = lbl\n return row\n\ndef get_dicom_data(row):\n \"\"\" Return relevant dicom data \"\"\"\n dcm = pydicom.read_file(row.dcm_path)\n row[\"height\"] = dcm.Rows\n row[\"width\"] = dcm.Columns\n row[\"pixel_spacing\"] = float(dcm.ImagerPixelSpacing[0])\n try:\n row[\"series_number\"] = int(dcm.SeriesNumber)\n except:\n row[\"series_number\"] = -1\n row[\"monochrome\"] = int(dcm.PhotometricInterpretation[-1])\n row[dcm.PatientSex] = 1\n row[\"SOP\"] = dcm.SOPClassUID\n try:\n row[\"image_type\"] = \"-\".join(sorted([x for x in dcm.ImageType if x]))\n except:\n row[\"image_type\"] = dcm.ImageType\n row[\"modality\"] = dcm.Modality\n \n # Triple Checks\n row[\"StudyInstanceUID\"] = dcm.StudyInstanceUID\n row[\"SeriesInstanceUID\"] = dcm.SeriesInstanceUID\n row[\"ImageInstanceUID\"] = row.dcm_path[:-4].rsplit(\"/\", 1)[1]\n \n return row\n\ndef create_fractional_bbox_coordinates(row):\n \"\"\" Function to return bbox coordiantes as fractions from DF row \"\"\"\n row[\"frac_xmin\"] = row[\"xmin\"]/row[\"width\"]\n row[\"frac_xmax\"] = row[\"xmax\"]/row[\"width\"]\n row[\"frac_ymin\"] = row[\"ymin\"]/row[\"height\"]\n row[\"frac_ymax\"] = row[\"ymax\"]/row[\"height\"]\n return row\n\ndef get_absolute_file_paths(directory):\n all_abs_file_paths = []\n for dirpath,_,filenames in tqdm(os.walk(directory)):\n for f in filenames:\n all_abs_file_paths.append(os.path.abspath(os.path.join(dirpath, f)))\n return all_abs_file_paths\n\n# Define the root data directory\nROOT_DIR = \"/kaggle/input\"\nDATA_DIR = os.path.join(ROOT_DIR,\"siim-covid19-detection\")\n\n# Define the paths to the training and testing dicom folders respectively\nTRAIN_DIR = os.path.join(DATA_DIR, \"train\")\nTEST_DIR = os.path.join(DATA_DIR, \"test\")\n\n# Define paths to the relevant csv files\nTRAIN_IMAGE_LVL_CSV = os.path.join(DATA_DIR, \"train_image_level.csv\")\nTRAIN_STUDY_LVL_CSV = os.path.join(DATA_DIR, \"train_study_level.csv\")\nSS_CSV = os.path.join(DATA_DIR, \"sample_submission.csv\")\n\n# Create the relevant dataframe objects\nprint(\"\\n... OPEN THE IMAGE & STUDY LEVEL DATAFRAME ...\\n\")\ntrain_image_level_df = pd.read_csv(TRAIN_IMAGE_LVL_CSV, usecols=[\"id\", \"boxes\", \"StudyInstanceUID\"]).sort_values(by=\"StudyInstanceUID\").reset_index(drop=True)\ntrain_study_level_df = pd.read_csv(TRAIN_STUDY_LVL_CSV)\n\nprint(\"\\n... CONVERT BOUNDING BOX STRING REPRESENTATION TO LITERAL ...\\n\")\ntrain_image_level_df[\"boxes\"] = train_image_level_df[\"boxes\"].fillna('{\"x\":0, \"y\":0, \"width\":1, \"height\":1}')\ntrain_image_level_df[\"boxes\"] = train_image_level_df[\"boxes\"].progress_apply(lambda x: ast.literal_eval(x))\n\nprint(\"\\n... RENAME AND FIX SOME COLUMNS IN BOTH DATAFRAMES ...\\n\")\ntrain_study_level_df.columns = [\"StudyInstanceUID\", \"negative\", \"typical\", \"indeterminate\", \"atypical\"]\ntrain_study_level_df[\"StudyInstanceUID\"] = train_study_level_df[\"StudyInstanceUID\"].str.replace(\"_study\", \"\")\ntrain_image_level_df[\"id\"] = train_image_level_df[\"id\"].str.replace(\"_image\", \"\")\n\nprint(\"\\n... OPEN THE SUBMISSION DATAFRAME AND CREATE TRAIN DATAFRAME ...\\n\")\nss_df = pd.read_csv(SS_CSV)\ntrain_df = pd.merge(train_image_level_df, train_study_level_df, on=\"StudyInstanceUID\")\n\nprint(\"\\n... GET THE IMAGE PATHS FOR BOTH DATA FRAMES ...\\n\")\nall_dcm_paths_train = get_absolute_file_paths(TRAIN_DIR)\nall_dcm_paths_test = get_absolute_file_paths(TEST_DIR)\nss_df[\"dcm_path\"] = ss_df[\"id\"].str.replace(\"_study\", \"\")\\\n .str.replace(\"_image\", \"\")\\\n .progress_apply(lambda x: [z for z in all_dcm_paths_test if x in z][-1])\ntrain_df[\"dcm_path\"] = train_df[\"id\"].progress_apply(lambda x: [z for z in all_dcm_paths_train if x in z][-1])\n\nprint(\"\\n... CAPTURE RELEVANT DICOM DATA FOR TRAIN DATAFRAME (~5-10 MINUTES)...\")\ntrain_df = train_df.parallel_apply(lambda row: get_dicom_data(row), axis=1)\n\nprint(\"... CAPTURE RELEVANT DICOM DATA FOR SUBMISSION DATAFRAME (~2-4 MINUTES)...\\n\")\nss_df = ss_df.parallel_apply(lambda row: get_dicom_data(row), axis=1)\n\nprint(\"\\n... EXPLODE LIST OF BOUNDING BOXES INTO INDIVIDUAL ROWS FOR TRAIN DATAFRAME ...\\n\")\ntrain_df[\"boxes\"] = train_df.boxes.progress_apply(lambda x: [x] if type(x)==dict else x)\ntrain_df = train_df.explode(\"boxes\", ignore_index=True).reset_index(drop=True)\n\nprint(\"\\n... CONVERT BBOX COORDINATES INTO RESPECTIVE COLUMNS ...\\n\")\ntrain_df = train_df.progress_apply(unpack_bbox_column, axis=1)\ntrain_df = train_df.sort_values(by=\"id\").reset_index(drop=True)\n\nprint(\"\\n... CREATE HUMAN LABEL COLUMN ...\\n\")\ntrain_df = train_df.progress_apply(get_human_label, axis=1)\n\nprint(\"\\n... INITIALIZE THE GENDER COLUMNS ...\\n\")\ntrain_df[\"M\"] = train_df[\"F\"] = ss_df[\"M\"] = ss_df[\"F\"] = 0\n\nprint(\"\\n... REORDER TRAIN DATAFRAME ...\\n\")\ntrain_col_order = [\n \"id\", \"dcm_path\", \n \"xmin\", \"ymin\", \"xmax\", \"ymax\", \"width\", \"height\",\n \"human_label\", \"negative\", \"typical\", \"indeterminate\", \"atypical\", \n \"StudyInstanceUID\", \"SeriesInstanceUID\", \"ImageInstanceUID\", \n \"series_number\", \"modality\", \"monochrome\", \"M\", \"F\", \"image_type\", \"pixel_spacing\", \"SOP\"]\ntrain_df = train_df[train_col_order]\n\nprint(\"\\n... REORDER SUBMISSION DATAFRAME ...\\n\")\nss_df = ss_df[[col for col in train_col_order if col in ss_df.columns]+[\"PredictionString\",]]\n\nprint(\"\\n... CREATE FRACTIONAL BBOX COORDINATES ...\\n\")\ntrain_df = train_df.progress_apply(create_fractional_bbox_coordinates, axis=1)\n\nprint(\"\\n... CREATE INTEGER LABEL COLUMN ...\\n\")\ntrain_df[\"integer_label\"] = train_df[\"human_label\"].progress_apply(lambda x: LABEL_LIST.index(x))\n\nprint(\"\\n\\nCOMBINED AND EXPLODED TRAIN DATAFRAME\\n\\n\")\ndisplay(train_df)\n\nprint(\"\\n\\nSAMPLE SUBMISSION DATAFRAME\\n\\n\")\ndisplay(ss_df)","metadata":{"papermill":{"duration":0.717244,"end_time":"2021-02-05T02:18:26.987446","exception":false,"start_time":"2021-02-05T02:18:26.270202","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-04T20:45:28.482318Z","iopub.execute_input":"2021-06-04T20:45:28.482605Z","iopub.status.idle":"2021-06-04T20:46:45.425146Z","shell.execute_reply.started":"2021-06-04T20:45:28.482573Z","shell.execute_reply":"2021-06-04T20:46:45.423179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n\n\n\n

3  HELPER FUNCTIONS

","metadata":{"papermill":{"duration":0.045205,"end_time":"2021-02-05T02:18:27.079444","exception":false,"start_time":"2021-02-05T02:18:27.034239","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def dicom2array(path, voi_lut=True, fix_monochrome=True):\n \"\"\" Convert dicom file to numpy array \n \n Args:\n path (str): Path to the dicom file to be converted\n voi_lut (bool): Whether or not VOI LUT is available\n fix_monochrome (bool): Whether or not to apply monochrome fix\n \n Returns:\n Numpy array of the respective dicom file \n \n \"\"\"\n # Use the pydicom library to read the dicom file\n dicom = pydicom.read_file(path)\n \n # VOI LUT (if available by DICOM device) is used to \n # transform raw DICOM data to \"human-friendly\" view\n if voi_lut:\n data = apply_voi_lut(dicom.pixel_array, dicom)\n else:\n data = dicom.pixel_array\n \n # The XRAY may look inverted\n # - If we want to fix this we can\n if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n data = np.amax(data) - data\n \n # Normalize the image array and return\n data = (data-np.min(data))/(np.max(data)-np.min(data))\n return data\n\n\ndef draw_bboxes(img, tl, br, rgb, label=\"\", label_location=\"tl\", opacity=0.1, line_thickness=0):\n \"\"\" TBD \n \n Args:\n TBD\n \n Returns:\n TBD \n \"\"\"\n rect = np.uint8(np.ones((br[1]-tl[1], br[0]-tl[0], 3))*rgb)\n sub_combo = cv2.addWeighted(img[tl[1]:br[1],tl[0]:br[0],:], 1-opacity, rect, opacity, 1.0) \n img[tl[1]:br[1],tl[0]:br[0],:] = sub_combo\n\n if line_thickness>0:\n img = cv2.rectangle(img, tuple(tl), tuple(br), rgb, line_thickness)\n \n if label:\n # DEFAULTS\n FONT = cv2.FONT_HERSHEY_SIMPLEX\n FONT_SCALE = 1.666\n FONT_THICKNESS = 3\n FONT_LINE_TYPE = cv2.LINE_AA\n \n if type(label)==str:\n LABEL = label.upper().replace(\" \", \"_\")\n else:\n LABEL = f\"CLASS_{label:02}\"\n \n text_width, text_height = cv2.getTextSize(LABEL, FONT, FONT_SCALE, FONT_THICKNESS)[0]\n \n label_origin = {\"tl\":tl, \"br\":br, \"tr\":(br[0],tl[1]), \"bl\":(tl[0],br[1])}[label_location]\n label_offset = {\n \"tl\":np.array([0, -10]), \"br\":np.array([-text_width, text_height+10]), \n \"tr\":np.array([-text_width, -10]), \"bl\":np.array([0, text_height+10])\n }[label_location]\n img = cv2.putText(img, LABEL, tuple(label_origin+label_offset), \n FONT, FONT_SCALE, rgb, FONT_THICKNESS, FONT_LINE_TYPE)\n \n return img","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","papermill":{"duration":0.083595,"end_time":"2021-02-05T02:18:27.209879","exception":false,"start_time":"2021-02-05T02:18:27.126284","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-04T20:51:37.719299Z","iopub.execute_input":"2021-06-04T20:51:37.719978Z","iopub.status.idle":"2021-06-04T20:51:37.734521Z","shell.execute_reply.started":"2021-06-04T20:51:37.719938Z","shell.execute_reply":"2021-06-04T20:51:37.733163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n\n\n\n

4  TABULAR DATA

\n\nWE ARE GIVEN (AND HAVE CREATED) THE FOLLOWING TRAIN COLUMNS\n> **`id`** - unique image identifier (UID)
\n**`dcm_path`** - Path to the respective detection's dicom image
\n**`xmin`** - minimum X coordinate of the object's bounding box
\n**`ymin`** - minimum Y coordinate of the object's bounding box
\n**`xmax`** - maximum X coordinate of the object's bounding box
\n**`ymax`** - maximum Y coordinate of the object's bounding box
\n**`human_label`** - Which category is this rows detection
\n**`negative`** - boolean label
\n**`typical`** - boolean label
\n**`indeterminate`** - boolean label
\n**`atypical`** - boolean label
\n**`StudyInstanceUID`** - Study UID
\n**`SeriesInstanceUID`** - Series UID
\n**`ImageInstanceUID`** - Series UID
\n**`image_shape`** - Image shape from dicom meta
\n**`width`** - Self explanatory
\n**`height`** - Self explanatory
\n**`integer_label`** - Integer label instead of human readable... all the versions..
","metadata":{"papermill":{"duration":0.048134,"end_time":"2021-02-05T02:18:27.306076","exception":false,"start_time":"2021-02-05T02:18:27.257942","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"

4.1 IMAGE_ID COLUMN EXPLORATION

\n\n---\n\nThe **`id`** column contains a **U**nique **ID**entifier (**UID**) that indicates which deidentified patient the respective row (object) relates to.\n\nAs there can be up multiple objects within a single radiograph, the **`id`** column can contain duplicates. If you grab all the rows for a single **`id`** you will have all of the objects detected within that patients radiograph.\n\n

\n\nTOTAL OPACITIES PER IMAGE/PATIENT\n\nLet's count the distribution of the amount of annotations per unique **`id`** value. \n\n*Note that we use a log-axis for the count axis to handle the large number of values where their is only 1 or 2 opacities present.*\n\n---","metadata":{"papermill":{"duration":0.045257,"end_time":"2021-02-05T02:18:27.397897","exception":false,"start_time":"2021-02-05T02:18:27.35264","status":"completed"},"tags":[]}},{"cell_type":"code","source":"fig = px.histogram(train_df.id.value_counts(), \n log_y=True, color_discrete_sequence=['indianred'], opacity=0.7,\n labels={\"value\":\"Number of Annotations Per Image\"},\n title=\"DISTRIBUTION OF # OF OPACITIES PER IMAGE/PATIENT \" \\\n \"(Log Scale for Y-Axis)\",\n )\nfig.update_layout(showlegend=False,\n xaxis_title=\"Number of Unique Images\",\n yaxis_title=\"Count of All Annotated Opacities\",\n font=FIG_FONT,)\nfig.show()","metadata":{"papermill":{"duration":1.486546,"end_time":"2021-02-05T02:18:28.930971","exception":false,"start_time":"2021-02-05T02:18:27.444425","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-31T02:17:28.799415Z","iopub.execute_input":"2021-05-31T02:17:28.799814Z","iopub.status.idle":"2021-05-31T02:17:30.116923Z","shell.execute_reply.started":"2021-05-31T02:17:28.799782Z","shell.execute_reply":"2021-05-31T02:17:30.115915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

4.2 HUMAN LABEL COLUMN EXPLORATION

\n\n---\n\nThe **`human_label`** column indicates the label as a string for the respective object/annotation (each row is for one object/annotation). \n

\n\nANNOTATIONS PER CLASS\n\nWe know there are 4 different possible **`human_label`**(s). To identify the distribution of counts across the labels we will use a bar-chart.","metadata":{"papermill":{"duration":0.055678,"end_time":"2021-02-05T02:18:30.697455","exception":false,"start_time":"2021-02-05T02:18:30.641777","status":"completed"},"tags":[]}},{"cell_type":"code","source":"fig = px.bar(train_df.human_label.value_counts().sort_index(), \n color=train_df.human_label.value_counts().sort_index().index, opacity=0.7,\n color_discrete_sequence=LABEL_COLORS,\n labels={\"y\":\"Annotations Per Class\", \"x\":\"\"},\n title=\"Annotations Per Class\",)\nfig.update_layout(legend_title=None,\n font=FIG_FONT,\n xaxis_title=\"\",\n yaxis_title=\"Annotations Per Class\")\n\nfig.show()","metadata":{"papermill":{"duration":0.256023,"end_time":"2021-02-05T02:18:31.00785","exception":false,"start_time":"2021-02-05T02:18:30.751827","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-31T02:17:30.576257Z","iopub.execute_input":"2021-05-31T02:17:30.576658Z","iopub.status.idle":"2021-05-31T02:17:30.698465Z","shell.execute_reply.started":"2021-05-31T02:17:30.576614Z","shell.execute_reply":"2021-05-31T02:17:30.697199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"NUMBER OF ANNOTATIONS FOR THE SAME CLASS PER IMAGE\n\nWe know there are 4 different possible **`human_label`**(s). For a given patient/image we also know that all the detections are the same label. Let's visualize what this looks like by the numbers...","metadata":{}},{"cell_type":"code","source":"data_list = []\nfor clr, lbl in zip(LABEL_COLORS, LABEL_LIST):\n data_list.append(go.Bar(\n x=train_df.groupby(\"id\")[lbl].sum().unique(), \n y=list(train_df.groupby(\"id\")[lbl].sum().value_counts().to_dict().values()), \n name=lbl.title(),\n marker_color=clr,\n ))\nfig = go.Figure(data=data_list)\n# Change the bar mode\nfig.update_layout(font=FIG_FONT,\n title=\"Number Of Annotations For The Same Class Per Image\",\n yaxis_title=\"Annotations Per Image\",\n xaxis_title=\"Number of Occuerences in Dataset\",\n legend_title_text=\"Class List\",\n barmode='group')\n\nfig.update_yaxes(type=\"log\")\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-05-31T02:17:31.430792Z","iopub.execute_input":"2021-05-31T02:17:31.43121Z","iopub.status.idle":"2021-05-31T02:17:31.522059Z","shell.execute_reply.started":"2021-05-31T02:17:31.431156Z","shell.execute_reply":"2021-05-31T02:17:31.520995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

4.3 EXPLORATION OF BBOX COORDINATE COLUMNS

\n\n---\n\nThe **`xmin`**, **`ymin`**, **`xmax`**, and **`ymax`** columns indicate the location of the annotated object bounding box, where the top-left corner is represented by the tuple (**`xmin`**, **`ymin`**) and the bottom-right corner is represented by the tuple (**`xmax`**, **`ymax`**).\n\nA value of **`(0, 0, 1, 1)`** indicates that no opacities are detected.\n\n
\n\nNOTE: You can have no opacities detected and still have an **`atypical`** study-level label.\n\n
\n\nWHAT PERCENTAGE OF IMAGES HAVE BOUNDING BOXES\n","metadata":{"papermill":{"duration":0.086324,"end_time":"2021-02-05T02:18:33.775555","exception":false,"start_time":"2021-02-05T02:18:33.689231","status":"completed"},"tags":[]}},{"cell_type":"code","source":"no_bb_train_df = train_df[(train_df.xmin==0.0) & (train_df.xmax==1.0)]\nonly_bb_train_df = train_df[(train_df.xmin!=0.0)]\n\nno_bb = no_bb_train_df.id.drop_duplicates().count()\nhas_bb = (train_df.id.drop_duplicates().count()-no_bb)\n\npx.pie(names=[\"Has Bounding Boxes\", \"No Bounding Boxess\"],\n values=[has_bb, no_bb], \n title=\"Comparison Of Images With Bounding Boxes\")","metadata":{"execution":{"iopub.status.busy":"2021-05-31T02:17:31.895841Z","iopub.execute_input":"2021-05-31T02:17:31.896263Z","iopub.status.idle":"2021-05-31T02:17:31.985649Z","shell.execute_reply.started":"2021-05-31T02:17:31.896223Z","shell.execute_reply":"2021-05-31T02:17:31.984371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.bar(only_bb_train_df.human_label.value_counts().sort_index(), \n color=only_bb_train_df.human_label.value_counts().sort_index().index, opacity=0.7,\n color_discrete_sequence=LABEL_COLORS,\n labels={\"y\":\"Annotations Per Class\", \"x\":\"\"},\n title=\"Bounding Box Only Images - Annotations Per Class\",)\nfig.update_layout(legend_title=None,\n font=FIG_FONT,\n xaxis_title=\"\",\n yaxis_title=\"Annotations Per Class\")\nfig.show()\n\nprint(\"\\n\",\"-\"*125,\"\\n\\n\")\n\nfig = px.bar(no_bb_train_df.human_label.value_counts().sort_index(), \n color=no_bb_train_df.human_label.value_counts().sort_index().index, opacity=0.7,\n color_discrete_sequence=LABEL_COLORS,\n labels={\"y\":\"Annotations Per Class\", \"x\":\"\"},\n title=\"No Bounding Box Images Only - Annotations Per Class\",)\nfig.update_layout(legend_title=None,\n font=FIG_FONT,\n xaxis_title=\"\",\n yaxis_title=\"Annotations Per Class\")\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-05-31T02:17:32.177665Z","iopub.execute_input":"2021-05-31T02:17:32.178094Z","iopub.status.idle":"2021-05-31T02:17:32.385164Z","shell.execute_reply.started":"2021-05-31T02:17:32.178059Z","shell.execute_reply":"2021-05-31T02:17:32.383626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"PLOT HEATMAP REPRESENTING BOUNDING BOXES FOR VARIOUS CLASSES\n\nThere's a lot to digest within these plots. The important thing to focus on will be identifying for each class the approximate range of locations the annotations are found in and the intensity of the locations within the heatmap.\n\n---\n\n**From the heatmaps plotted below we can ascertain the following information**
\nTBD","metadata":{}},{"cell_type":"code","source":"# # Record some important values for later\nave_src_img_height = train_df.height.mean().astype(np.int32)\nave_src_img_width = train_df.width.mean().astype(np.int32)\nprint(f\"\\n... The Average Shape Of The Images Is : {(ave_src_img_width, ave_src_img_height)} ...\\n\")\n\n# DEFAULT\nHEATMAP_SIZE = (ave_src_img_height, ave_src_img_width, 4)\n\n# Initialize\nheatmap = np.zeros(HEATMAP_SIZE, dtype=np.int32)\nbbox_np = only_bb_train_df[[\"integer_label\", \"frac_xmin\", \"frac_xmax\", \"frac_ymin\", \"frac_ymax\"]].to_numpy()\nbbox_np[:, 1:3] *= ave_src_img_width\nbbox_np[:, 3:5] *= ave_src_img_height\nbbox_np = np.floor(bbox_np).astype(np.int32)\n\n# Color map stuff\ncustom_cmaps = [\n matplotlib.colors.LinearSegmentedColormap.from_list(\n colors=[(0.,0.,0.), c, (0.95,0.95,0.95)], \n name=f\"custom_{i}\") for i,c in enumerate(sns.color_palette(\"Spectral\", 4))\n]\n\nfor row in tqdm(bbox_np, total=bbox_np.shape[0]):\n heatmap[row[3]:row[4]+1, row[1]:row[2]+1, row[0]] += 1\n \nfig = plt.figure(figsize=(18,20))\nplt.suptitle(\"Heatmaps Showing Bounding Box Placement\\n \", fontweight=\"bold\", fontsize=16)\nfor i in range(4):\n plt.subplot(2, 2, i+1)\n if i==0:\n plt.imshow(heatmap.mean(axis=-1), cmap=\"bone\")\n plt.title(f\"Average of All Classes\", fontweight=\"bold\")\n elif i==3:\n plt.imshow(heatmap[:, :, i], cmap=custom_cmaps[i])\n plt.title(f\"{LABEL_LIST[i]} – ({i})\", fontweight=\"bold\")\n else:\n plt.imshow(heatmap[:, :, i-1], cmap=custom_cmaps[i-1])\n plt.title(f\"{LABEL_LIST[i-1]} – ({i-1})\", fontweight=\"bold\")\n plt.axis(False)\n \nfig.tight_layout(rect=[0, 0.03, 1, 0.97])\nplt.show()\n\nprint(\"\\n\\n... NO HEATMAP FOR `TYPICAL` AS NO BOUNDING BOXES EXIST FOR THIS CLASS ...\\n\\n\")","metadata":{"papermill":{"duration":40.569565,"end_time":"2021-02-05T02:27:30.658936","exception":false,"start_time":"2021-02-05T02:26:50.089371","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-31T02:17:32.585919Z","iopub.execute_input":"2021-05-31T02:17:32.586472Z","iopub.status.idle":"2021-05-31T02:17:58.333086Z","shell.execute_reply.started":"2021-05-31T02:17:32.586426Z","shell.execute_reply":"2021-05-31T02:17:58.331455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"INVESTIGATE THE SIZES OF BOUNDING BOXES AND THE IMPACT OF CLASS\n\n---\n\n**From the boxplot plotted below we can ascertain the following information**\nTBD","metadata":{"papermill":{"duration":0.110799,"end_time":"2021-02-05T02:27:30.881424","exception":false,"start_time":"2021-02-05T02:27:30.770625","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Update bbox dataframe to make this easier\nonly_bb_train_df[\"frac_bbox_area\"] = (only_bb_train_df[\"frac_xmax\"]-only_bb_train_df[\"frac_xmin\"])*(only_bb_train_df[\"frac_ymax\"]-only_bb_train_df[\"frac_ymin\"])\n\nfig = px.box(only_bb_train_df.sort_values(by=\"human_label\"), x=\"human_label\", y=\"frac_bbox_area\", color=\"human_label\", \n color_discrete_sequence=LABEL_COLORS[:2]+LABEL_COLORS[3:], notched=True,\n labels={\"human_label\":\"Class Name\", \"frac_bbox_area\":\"BBox Area (%)\"},\n title=\"DISTRIBUTION OF BBOX AREAS AS % OF SOURCE IMAGE AREA \" \\\n \"(Some Upper Outliers Excluded For Better Visualization)\")\n\nfig.update_layout(showlegend=True,\n yaxis_range=[-0.025,0.4],\n legend_title_text=None,\n xaxis_title=\"\",\n yaxis_title=\"Bounding Box Area %\",\n font=FIG_FONT,)\nfig.show()","metadata":{"papermill":{"duration":0.769755,"end_time":"2021-02-05T02:27:31.758621","exception":false,"start_time":"2021-02-05T02:27:30.988866","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-31T02:17:58.335413Z","iopub.execute_input":"2021-05-31T02:17:58.335799Z","iopub.status.idle":"2021-05-31T02:17:58.545432Z","shell.execute_reply.started":"2021-05-31T02:17:58.335766Z","shell.execute_reply":"2021-05-31T02:17:58.539994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"INVESTIGATE THE ASPECT RATIO OF BOUNDING BOXES AND THE IMPACT OF CLASS\n\nWe want to understand the average shape (wide-narrow, square, etc.) of the bouning-boxes associated with each class, and to do this we will use a bar chart with some pre-drawn lines.\n\n---\n\n**From the bar chart plotted below we can ascertain the following information:**","metadata":{"papermill":{"duration":0.150595,"end_time":"2021-02-05T02:27:32.074798","exception":false,"start_time":"2021-02-05T02:27:31.924203","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Aspect Ratio is Calculated as Width/Height\nonly_bb_train_df[\"aspect_ratio\"] = (only_bb_train_df[\"xmax\"]-only_bb_train_df[\"xmin\"])/(only_bb_train_df[\"ymax\"]-only_bb_train_df[\"ymin\"])\n\n# Generate the bar plot\nfig = px.bar(x=['atypical', 'indeterminate', 'typical'], y=only_bb_train_df.groupby(\"human_label\").mean()[\"aspect_ratio\"], \n color=['atypical', 'indeterminate', 'typical'], opacity=0.85,\n color_discrete_sequence=LABEL_COLORS[:2]+LABEL_COLORS[3:], \n labels={\"x\":\"Class Name\", \"y\":\"Aspect Ratio (W/H)\"},\n title=\"Aspect Ratios For Bounding Boxes By Class\",)\nfig.update_layout(font=FIG_FONT,\n yaxis_title=\"Aspect Ratio (W/H)\",\n xaxis_title=None,\n legend_title_text=None)\nfig.add_hline(y=1, line_width=2, line_dash=\"dot\", \n annotation_font_size=10, \n annotation_text=\"SQUARE ASPECT RATIO\", \n annotation_position=\"bottom left\", \n annotation_font_color=\"black\")\nfig.add_hrect(y0=0, y1=0.5, line_width=0, fillcolor=\"red\", opacity=0.125,\n annotation_text=\">2:1 WIDE RECTANGLE REGION\", \n annotation_position=\"bottom right\", \n annotation_font_size=10,\n annotation_font_color=\"red\")\nfig.show()","metadata":{"papermill":{"duration":0.396096,"end_time":"2021-02-05T02:27:32.612364","exception":false,"start_time":"2021-02-05T02:27:32.216268","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-31T02:17:58.548478Z","iopub.execute_input":"2021-05-31T02:17:58.548975Z","iopub.status.idle":"2021-05-31T02:17:58.701508Z","shell.execute_reply.started":"2021-05-31T02:17:58.548929Z","shell.execute_reply":"2021-05-31T02:17:58.699816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n\n\n\n

5  IMAGE DATA

\n\nRecall that the image data is stored in DICOM format and the annotations are stored in our **`train_df`** Dataframe.","metadata":{"papermill":{"duration":0.144158,"end_time":"2021-02-05T02:27:32.900106","exception":false,"start_time":"2021-02-05T02:27:32.755948","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"

5.1 CLASS TO HELP PARSE THE DATA

\n\n---\n\nClasses are very useful when we need to bind data to functionality. In this case, I have created a class (unwieldy as it may be currently in it's initial version) to help with that called **`TrainData`**.\n\nI will get into the details of how the methods work at a later time... and for today I will simply generate the outputs using each method to show their functionality.","metadata":{"papermill":{"duration":0.141972,"end_time":"2021-02-05T02:27:33.185785","exception":false,"start_time":"2021-02-05T02:27:33.043813","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# class TrainData():\n# def __init__(self, df, train_dir, cmap=\"Spectral\"):\n# # Initialize\n# self.df = df\n# self.train_dir = train_dir\n \n# # Visualization\n# self.cmap = cmap\n# self.pal = [tuple([int(x) for x in np.array(c)*(255,255,255)]) for c in sns.color_palette(cmap, 4)]\n# self.pal.pop(2)\n \n# # Store df components in individual numpy arrays for easy access based on index\n# tmp_numpy = self.df.to_numpy()\n# image_ids = tmp_numpy[0]\n# class_ids = tmp_numpy[1]\n# rad_ids = tmp_numpy[2]\n# bboxes = tmp_numpy[3:]\n \n# self.img_annotations = self.get_annotations(get_all=True)\n \n# # Clean-Up\n# del tmp_numpy; gc.collect();\n \n \n# def get_annotations(self, get_all=False, image_ids=None, class_ids=None, rad_ids=None, index=None):\n# \"\"\" TBD \n \n# Args:\n# get_all (bool, optional): TBD\n# image_ids (list of strs, optional): TBD\n# class_ids (list of ints, optional): TBD\n# rad_ids (list of strs, optional): TBD\n# index (int, optional):\n \n# Returns:\n \n \n# \"\"\"\n# if not get_all and image_ids is None and class_ids is None and rad_ids is None and index is None:\n# raise ValueError(\"Expected one of the following arguments to be passed:\" \\\n# \"\\n\\t\\t– `get_all`, `image_id`, `class_id`, `rad_id`, or `index`\")\n# # Initialize\n# tmp_df = self.df.copy()\n \n# if not get_all:\n# if image_ids is not None:\n# tmp_df = tmp_df[tmp_df.image_id.isin(image_ids)]\n# if class_ids is not None:\n# tmp_df = tmp_df[tmp_df.class_id.isin(class_ids)]\n# if rad_ids is not None:\n# tmp_df = tmp_df[tmp_df.rad_id.isin(rad_ids)]\n# if index is not None:\n# tmp_df = tmp_df.iloc[index]\n \n# annotations = {image_id:[] for image_id in tmp_df.image_id.to_list()}\n# for row in tmp_df.to_numpy():\n \n# # Update annotations dictionary\n# annotations[row[0]].append(dict(\n# img_path=os.path.join(self.train_dir, row[0]+\".dicom\"),\n# image_id=row[0],\n# class_id=int(row[1]),\n# rad_id=int(row[2][1:]),\n# ))\n \n# # Catch to convert float array to integer array\n# if row[1]==14:\n# annotations[row[0]][-1][\"bbox\"]=row[3:]\n# else:\n# annotations[row[0]][-1][\"bbox\"]=row[3:].astype(np.int32)\n# return annotations\n \n# def get_annotated_image(self, image_id, annots=None, plot=False, plot_size=(18,25), plot_title=\"\"):\n# if annots is None:\n# annots = self.img_annotations.copy()\n \n# if type(annots) != list:\n# image_annots = annots[image_id]\n# else:\n# image_annots = annots\n \n# img = cv2.cvtColor(dicom2array(image_annots[0][\"img_path\"]),cv2.COLOR_GRAY2RGB)\n# for ann in image_annots:\n# if ann[\"class_id\"] != 14:\n# img = draw_bboxes(img, \n# ann[\"bbox\"][:2], ann[\"bbox\"][-2:], \n# rgb=self.pal[ann[\"class_id\"]], \n# label=int_2_str[ann[\"class_id\"]], \n# opacity=0.08, line_thickness=4)\n# if plot:\n# plot_image(img, title=plot_title, figsize=plot_size)\n \n# return img\n \n# def plot_image_ids(self, image_id_list, height_multiplier=6, verbose=True):\n# annotations = self.get_annotations(image_ids=image_id_list)\n# annotated_imgs = []\n# n = len(image_id_list)\n \n# plt.figure(figsize=(20, height_multiplier*n))\n# for i, (image_id, annots) in enumerate(annotations.items()):\n# if i >= n:\n# break\n# if verbose:\n# print(f\".\", end=\"\")\n# plt.subplot(n//2,2,i+1)\n# plt.imshow(self.get_annotated_image(image_id, annots))\n# plt.axis(False)\n# plt.title(f\"Image ID – {image_id}\")\n# plt.tight_layout(rect=[0, 0.03, 1, 0.97])\n# plt.show()\n \n# def plot_classes(self, class_list, n=4, height_multiplier=6, verbose=True):\n# annotations = self.get_annotations(class_ids=class_list)\n# annotated_imgs = []\n\n# plt.figure(figsize=(20, height_multiplier*n))\n# for i, (image_id, annots) in enumerate(annotations.items()):\n# if i >= n:\n# break\n# if verbose:\n# print(f\".\", end=\"\")\n# plt.subplot(n//2,2,i+1)\n# plt.imshow(self.get_annotated_image(image_id, annots))\n# plt.axis(False)\n# plt.title(f\"Image ID – {image_id}\")\n# plt.tight_layout(rect=[0, 0.03, 1, 0.97])\n# plt.show()\n\n# def plot_radiologists(self, rad_id_list, n=4, height_multiplier=6, verbose=True):\n# annotations = self.get_annotations(rad_ids=rad_id_list)\n# annotated_imgs = []\n\n# plt.figure(figsize=(20, height_multiplier*n))\n# for i, (image_id, annots) in enumerate(annotations.items()):\n# if i >= n:\n# break\n# if verbose:\n# print(f\".\", end=\"\")\n# plt.subplot(n//2,2,i+1)\n# plt.imshow(self.get_annotated_image(image_id, annots))\n# plt.axis(False)\n# plt.title(f\"Image ID – {image_id}\")\n# plt.tight_layout(rect=[0, 0.03, 1, 0.97])\n# plt.show()\n\n# train_data = TrainData(train_df, TRAIN_DIR)","metadata":{"papermill":{"duration":0.96872,"end_time":"2021-02-05T02:27:34.298764","exception":false,"start_time":"2021-02-05T02:27:33.330044","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-31T02:17:58.703334Z","iopub.execute_input":"2021-05-31T02:17:58.703689Z","iopub.status.idle":"2021-05-31T02:17:58.710768Z","shell.execute_reply.started":"2021-05-31T02:17:58.703655Z","shell.execute_reply":"2021-05-31T02:17:58.709632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"PLOT IMAGES\n\nSummary to be done later\n\n---\n\nMore detail to come later","metadata":{"papermill":{"duration":0.143297,"end_time":"2021-02-05T02:27:34.584752","exception":false,"start_time":"2021-02-05T02:27:34.441455","status":"completed"},"tags":[]}},{"cell_type":"code","source":"plt.figure(figsize=(18,14))\n\nfor i, (_, row) in enumerate(train_df.sample(4).iterrows()):\n arr = dicom2array(row.dcm_path)\n plt.subplot(2,2,i+1)\n plt.imshow(arr, cmap=\"bone\")\n plt.title(f\"LABEL = {row.human_label.title()}\", fontweight=\"bold\")\n \nplt.tight_layout()\nplt.show()","metadata":{"papermill":{"duration":10.204767,"end_time":"2021-02-05T02:27:44.933342","exception":false,"start_time":"2021-02-05T02:27:34.728575","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-31T02:17:58.712308Z","iopub.execute_input":"2021-05-31T02:17:58.712633Z","iopub.status.idle":"2021-05-31T02:18:08.072992Z","shell.execute_reply.started":"2021-05-31T02:17:58.712598Z","shell.execute_reply":"2021-05-31T02:18:08.071286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"PLOT IMAGES CONTAINING A SINGLE CLASS\n\nSummary to be done later... \n\n**NOTE: Only the bounding boxes for the specified classes will be drawn... probably a TBD in the future as a possible arg**\n\n---\n\nMore detail to come later","metadata":{"papermill":{"duration":0.195289,"end_time":"2021-02-05T02:27:45.326034","exception":false,"start_time":"2021-02-05T02:27:45.130745","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# train_data.plot_classes(class_list=[7,], n=2, verbose=False)","metadata":{"papermill":{"duration":4.089116,"end_time":"2021-02-05T02:27:49.60833","exception":false,"start_time":"2021-02-05T02:27:45.519214","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-31T02:18:08.075425Z","iopub.execute_input":"2021-05-31T02:18:08.075826Z","iopub.status.idle":"2021-05-31T02:18:08.081786Z","shell.execute_reply.started":"2021-05-31T02:18:08.07579Z","shell.execute_reply":"2021-05-31T02:18:08.079679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"PLOT IMAGES CONTAINING ONE OR MORE CLASSES FROM A LIST\n\nSummary to be done later... \n\n**NOTE: Images need not contain ALL the classes... potential future improvement or option.**\n\n---\n\nMore detail to come later","metadata":{"papermill":{"duration":0.206209,"end_time":"2021-02-05T02:27:50.029509","exception":false,"start_time":"2021-02-05T02:27:49.8233","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# train_data.plot_classes(class_list=[5,8,11], n=4, verbose=False)","metadata":{"papermill":{"duration":10.625197,"end_time":"2021-02-05T02:28:00.863859","exception":false,"start_time":"2021-02-05T02:27:50.238662","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-31T02:18:08.084028Z","iopub.execute_input":"2021-05-31T02:18:08.084481Z","iopub.status.idle":"2021-05-31T02:18:08.106024Z","shell.execute_reply.started":"2021-05-31T02:18:08.084442Z","shell.execute_reply":"2021-05-31T02:18:08.104846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

5.2 VISUALIZE EACH TYPE OF BBOX

\n\n---\n\nTBD","metadata":{"papermill":{"duration":0.29884,"end_time":"2021-02-05T02:28:11.291874","exception":false,"start_time":"2021-02-05T02:28:10.993034","status":"completed"},"tags":[]}},{"cell_type":"code","source":"LABEL_LIST","metadata":{"execution":{"iopub.status.busy":"2021-05-31T02:18:08.110926Z","iopub.execute_input":"2021-05-31T02:18:08.111682Z","iopub.status.idle":"2021-05-31T02:18:08.13316Z","shell.execute_reply.started":"2021-05-31T02:18:08.111637Z","shell.execute_reply":"2021-05-31T02:18:08.131778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"ATYPICAL - (0)\n\nSummary to be done later... \n\n---\n\nMore detail to come later","metadata":{"papermill":{"duration":0.294173,"end_time":"2021-02-05T02:28:11.881205","exception":false,"start_time":"2021-02-05T02:28:11.587032","status":"completed"},"tags":[]}},{"cell_type":"code","source":"plt.figure(figsize=(18,14))\n\nfor i, (_, row) in enumerate(train_df[train_df.human_label==\"atypical\"].sample(4).iterrows()):\n arr = dicom2array(row.dcm_path)\n plt.subplot(2,2,i+1)\n plt.imshow(arr, cmap=\"bone\")\n plt.title(f\"LABEL = {row.human_label.title()}\", fontweight=\"bold\")\n \nplt.tight_layout()\nplt.show()","metadata":{"papermill":{"duration":10.958611,"end_time":"2021-02-05T02:28:23.132565","exception":false,"start_time":"2021-02-05T02:28:12.173954","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-31T02:18:08.135222Z","iopub.execute_input":"2021-05-31T02:18:08.135609Z","iopub.status.idle":"2021-05-31T02:18:15.200479Z","shell.execute_reply.started":"2021-05-31T02:18:08.135575Z","shell.execute_reply":"2021-05-31T02:18:15.199316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"INDETERMINATE - (1)\n\nSummary to be done later... \n\n---\n\nMore detail to come later","metadata":{"papermill":{"duration":0.33362,"end_time":"2021-02-05T02:28:23.817096","exception":false,"start_time":"2021-02-05T02:28:23.483476","status":"completed"},"tags":[]}},{"cell_type":"code","source":"plt.figure(figsize=(18,14))\n\nfor i, (_, row) in enumerate(train_df[train_df.human_label==\"indeterminate\"].sample(4).iterrows()):\n arr = dicom2array(row.dcm_path)\n plt.subplot(2,2,i+1)\n plt.imshow(arr, cmap=\"bone\")\n plt.title(f\"LABEL = {row.human_label.title()}\", fontweight=\"bold\")\n \nplt.tight_layout()\nplt.show()","metadata":{"papermill":{"duration":9.982755,"end_time":"2021-02-05T02:28:34.128931","exception":false,"start_time":"2021-02-05T02:28:24.146176","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-31T02:18:15.201992Z","iopub.execute_input":"2021-05-31T02:18:15.202631Z","iopub.status.idle":"2021-05-31T02:18:20.760497Z","shell.execute_reply.started":"2021-05-31T02:18:15.20258Z","shell.execute_reply":"2021-05-31T02:18:20.758159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"NEGATIVE - (2)\n\nSummary to be done later... \n\n---\n\nMore detail to come later","metadata":{"papermill":{"duration":0.374339,"end_time":"2021-02-05T02:28:34.893534","exception":false,"start_time":"2021-02-05T02:28:34.519195","status":"completed"},"tags":[]}},{"cell_type":"code","source":"plt.figure(figsize=(18,14))\n\nfor i, (_, row) in enumerate(train_df[train_df.human_label==\"negative\"].sample(4).iterrows()):\n arr = dicom2array(row.dcm_path)\n plt.subplot(2,2,i+1)\n plt.imshow(arr, cmap=\"bone\")\n plt.title(f\"LABEL = {row.human_label.title()}\", fontweight=\"bold\")\n \nplt.tight_layout()\nplt.show()","metadata":{"papermill":{"duration":13.263982,"end_time":"2021-02-05T02:28:48.528066","exception":false,"start_time":"2021-02-05T02:28:35.264084","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-31T02:18:20.762066Z","iopub.status.idle":"2021-05-31T02:18:20.763022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"TYPICAL - (3)\n\nSummary to be done later... \n\n---\n\nMore detail to come later","metadata":{"papermill":{"duration":0.428688,"end_time":"2021-02-05T02:28:49.419262","exception":false,"start_time":"2021-02-05T02:28:48.990574","status":"completed"},"tags":[]}},{"cell_type":"code","source":"plt.figure(figsize=(18,14))\n\nfor i, (_, row) in enumerate(train_df[train_df.human_label==\"typical\"].sample(4).iterrows()):\n arr = dicom2array(row.dcm_path)\n plt.subplot(2,2,i+1)\n plt.imshow(arr, cmap=\"bone\")\n plt.title(f\"LABEL = {row.human_label.title()}\", fontweight=\"bold\")\n \nplt.tight_layout()\nplt.show()","metadata":{"papermill":{"duration":8.112312,"end_time":"2021-02-05T02:28:58.040324","exception":false,"start_time":"2021-02-05T02:28:49.928012","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-31T02:18:20.764391Z","iopub.status.idle":"2021-05-31T02:18:20.765329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n\n6  IDENTIFY DUPLICATES\n\nFind duplicate images using perceptual hash.\n\nTBD","metadata":{"papermill":{"duration":1.059028,"end_time":"2021-02-05T02:31:14.465774","exception":false,"start_time":"2021-02-05T02:31:13.406746","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# def calc_iou(bbox_1, bbox_2):\n# # determine the coordinates of the intersection rectangle\n# x_left = max(bbox_1[0], bbox_2[0])\n# y_top = max(bbox_1[1], bbox_2[1])\n# x_right = min(bbox_1[2], bbox_2[2])\n# y_bottom = min(bbox_1[3], bbox_2[3])\n\n# # Check if bboxes overlap at all (if not return 0)\n# if x_right < x_left or y_bottom < y_top:\n# return 0.0\n \n# # The intersection of two axis-aligned bounding boxes is always an\n# # axis-aligned bounding box\n# else:\n# intersection_area = (x_right - x_left) * (y_bottom - y_top)\n \n# # compute the area of both AABBs\n# bbox_1_area = (bbox_1[2] - bbox_1[0]) * (bbox_1[3] - bbox_1[1])\n# bbox_2_area = (bbox_2[2] - bbox_2[0]) * (bbox_2[3] - bbox_2[1])\n\n# # compute the intersection over union by taking the intersection\n# # area and dividing it by the sum of prediction + ground-truth\n# # areas - the interesection area\n# iou = intersection_area / float(bbox_1_area + bbox_2_area - intersection_area)\n# return iou\n\n# def redux_bboxes(annots):\n# def _get_inner_box(bboxes):\n# xmin = max([box[0] for box in bboxes])\n# ymin = max([box[1] for box in bboxes])\n# xmax = min([box[2] for box in bboxes])\n# ymax = min([box[3] for box in bboxes])\n# if (xmax<=xmin) or (ymax<=ymin):\n# return None\n# else:\n# return [xmin, ymin, xmax, ymax]\n \n# valid_list_indices = [] \n# new_bboxes = []\n# new_class_ids = []\n# new_rad_ids = []\n \n# for i, (class_id, rad_id, bbox) in enumerate(zip(annots[\"class_id\"], annots[\"rad_id\"], annots[\"bbox\"])):\n# intersecting_boxes = [bbox,]\n# other_bboxes = [x for j,x in enumerate(annots[\"bbox\"]) if j!=i]\n# other_classes = [x for j,x in enumerate(annots[\"class_id\"]) if j!=i]\n# for j, (other_class_id, other_bbox) in enumerate(zip(other_classes, other_bboxes)):\n# if class_id==other_class_id:\n# iou = calc_iou(bbox, other_bbox)\n# if iou>0.:\n# intersecting_boxes.append(other_bbox)\n\n# if len(intersecting_boxes)>1:\n# inner_box = _get_inner_box(intersecting_boxes)\n# if inner_box and inner_box not in new_bboxes:\n# new_bboxes.append(inner_box)\n# new_class_ids.append(class_id)\n# new_rad_ids.append(rad_id) \n\n# annots[\"bbox\"] = new_bboxes\n# annots[\"rad_id\"] = new_rad_ids\n# annots[\"class_id\"] = new_class_ids\n \n# return annots\n\n# # Make GT Dataframe\n# gt_df = train_df[train_df.class_id!=14]\n\n# # Apply Manipulations and Merger Functions\n# gt_df[\"bbox\"] = gt_df.loc[:, [\"x_min\",\"y_min\",\"x_max\",\"y_max\"]].values.tolist()\n# gt_df.drop(columns=[\"x_min\",\"y_min\",\"x_max\",\"y_max\"], inplace=True)\n# gt_df = gt_df.groupby([\"image_id\"]).agg({k:list for k in gt_df.columns if k !=\"image_id\"}).reset_index()\n# gt_df = gt_df.apply(redux_bboxes, axis=1)\n\n# # Recreate the Original Dataframe Style\n# gt_df = gt_df.apply(pd.Series.explode).reset_index(drop=True).dropna()\n# gt_df[\"x_min\"] = gt_df[\"bbox\"].apply(lambda x: x[0])\n# gt_df[\"y_min\"] = gt_df[\"bbox\"].apply(lambda x: x[1])\n# gt_df[\"x_max\"] = gt_df[\"bbox\"].apply(lambda x: x[2])\n# gt_df[\"y_max\"] = gt_df[\"bbox\"].apply(lambda x: x[3])\n# gt_df.drop(columns=[\"bbox\"], inplace=True)\n\n# # Add back in NaN Rows As A Single Annotation\n# gt_df = pd.concat([\n# gt_df, train_df.loc[train_df['class_id'] == 14].drop_duplicates(subset=[\"image_id\"])\n# ]).reset_index(drop=True)","metadata":{"papermill":{"duration":3.455263,"end_time":"2021-02-05T02:31:18.92604","exception":false,"start_time":"2021-02-05T02:31:15.470777","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-31T02:18:20.766705Z","iopub.status.idle":"2021-05-31T02:18:20.767349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# gt_data = TrainData(gt_df, TRAIN_DIR)\n# IMAGE_ID_LIST = gt_df[gt_df.class_id!=14].groupby(\"image_id\") \\\n# .count() \\\n# .sort_values(by=\"class_id\", ascending=False) \\\n# .index[0:100:20]\n\n# for i, IMAGE_ID in enumerate(IMAGE_ID_LIST):\n# train_data.get_annotated_image(IMAGE_ID, annots=None, plot=True, plot_size=(18,22), plot_title=f\"ORIGINAL – IMG #{i+1}\")\n# gt_data.get_annotated_image(IMAGE_ID, annots=None, plot=True, plot_size=(18,22), plot_title=f\"REDUX VERSION – IMG #{i+1}\")","metadata":{"papermill":{"duration":24.605919,"end_time":"2021-02-05T02:31:44.535441","exception":false,"start_time":"2021-02-05T02:31:19.929522","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-05-31T02:18:20.768295Z","iopub.status.idle":"2021-05-31T02:18:20.768847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.to_csv(\"./updated_train_labels.csv\", index=False)\nss_df.to_csv(\"./updated_sample_submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-05-31T02:18:20.769734Z","iopub.status.idle":"2021-05-31T02:18:20.770327Z"},"trusted":true},"execution_count":null,"outputs":[]}]}