{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# --- CSS STYLE ---\nfrom IPython.core.display import HTML\ndef css_styling():\n    styles = open(\"../input/2020-cost-of-living/alerts.css\", \"r\").read()\n    return HTML(\"<style>\"+styles+\"</style>\")\ncss_styling()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-06-07T18:46:32.368505Z","iopub.execute_input":"2021-06-07T18:46:32.369194Z","iopub.status.idle":"2021-06-07T18:46:32.391802Z","shell.execute_reply.started":"2021-06-07T18:46:32.369075Z","shell.execute_reply":"2021-06-07T18:46:32.391007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<img src=\"https://i.imgur.com/YcUJEhW.png\">\n\n<center><h1>SIIM-FISABIO-RSNA COVID-19 Detection</h1></center>\n\nThis is **an object detection and classification problem**, meaning that for each instance we'll have to *predict* a **bounding box** and a **class**.\n\n<div class=\"alert success-alert\">\n📌 <b>Competition Goal</b>: Categorize chest radiographs as negative for pneumonia, typical, indeterminate, or atypical for COVID-19. If some abnormalities are found, provide the bounding boxes.\n</div>\n\nOK! Covid-19 sympthoms look very similar to other viral or bacterial pneumonias/ chest radiographs, hence it's much harder to correctly (and I might add quickly) diagnose.\n\n<div class=\"alert simple-alert\">\n🎉 <b>Fun Bonus</b>: <a href=\"https://www8.hp.com/us/en/workstations/overview.html\">Z by HP</a> is providing the top student team with a ZBook G8 Data Science Workstation!\n</div>\n\nI have been working on these for quite some time and I absolutely LOVE it. So I encourage everybody to try, have fun and ... good luck! ❤️\n\n### ⬇️ Libraries\n* Link to my W&B Dashboard here: https://wandb.ai/andrada/siim-covid19\n* How to use W&B: [Experiment Tracking with Weights and Biases](https://www.kaggle.com/ayuraj/experiment-tracking-with-weights-and-biases)","metadata":{}},{"cell_type":"code","source":"# Libraries\nimport os\nimport re\nimport wandb\nimport tqdm\nimport warnings\nimport glob\nimport ast\nimport cv2\nimport math\nimport pandas as pd\nimport numpy as np\nfrom IPython.display import display_html\nimport seaborn as sns\nimport matplotlib as mpl\nimport matplotlib.patches as patches\nimport matplotlib.pyplot as plt\nfrom scipy.stats import pearsonr\nfrom matplotlib.offsetbox import AnnotationBbox, OffsetImage\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nfrom sklearn.cluster import KMeans\nfrom skimage import morphology, measure\n\n# Environment check\nwarnings.filterwarnings(\"ignore\")\nos.environ[\"WANDB_SILENT\"] = \"true\"\nCONFIG = {'competition': 'siim-fisabio-rsna', '_wandb_kernel': 'aot'}\n\n# Secrets 🤫\nfrom kaggle_secrets import UserSecretsClient\nuser_secrets = UserSecretsClient()\nsecret_value_0 = user_secrets.get_secret(\"wandb\")\n\n# Custom colors\nmy_colors = [\"#E97777\", \"#E9B077\", \"#E9E977\", \n             \"#77E977\", \"#7777E9\", \"#6F6BAC\", \"#B677E9\"]\nsns.palplot(sns.color_palette(my_colors))\n\n# Set Style\nsns.set_style(\"white\")\nmpl.rcParams['xtick.labelsize'] = 18\nmpl.rcParams['ytick.labelsize'] = 18\nmpl.rcParams['axes.spines.left'] = False\nmpl.rcParams['axes.spines.right'] = False\nmpl.rcParams['axes.spines.top'] = False\nplt.rcParams.update({'font.size': 22})\n\nclass color:\n    BOLD = '\\033[1m' + '\\033[93m'\n    END = '\\033[0m'","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-07T18:46:32.393256Z","iopub.execute_input":"2021-06-07T18:46:32.393799Z","iopub.status.idle":"2021-06-07T18:46:36.236564Z","shell.execute_reply.started":"2021-06-07T18:46:32.393757Z","shell.execute_reply":"2021-06-07T18:46:36.235899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> 📌 **Note**: If this line throws an error, try using `wandb.login()` instead. It will ask for the API key to login, which you can get from your W&B profile (click on Profile -> Settings -> scroll to API keys).","metadata":{}},{"cell_type":"code","source":"! wandb login $secret_value_0","metadata":{"_kg_hide-input":false,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-06-07T18:46:36.238595Z","iopub.execute_input":"2021-06-07T18:46:36.239242Z","iopub.status.idle":"2021-06-07T18:46:38.592824Z","shell.execute_reply.started":"2021-06-07T18:46:36.239200Z","shell.execute_reply":"2021-06-07T18:46:38.591877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### ⬇️ Handy Functions","metadata":{}},{"cell_type":"code","source":"def show_values_on_bars(axs, h_v=\"v\", space=0.4):\n    '''Plots the value at the end of the a seaborn barplot.\n    axs: the ax of the plot\n    h_v: weather or not the barplot is vertical/ horizontal'''\n    \n    def _show_on_single_plot(ax):\n        if h_v == \"v\":\n            for p in ax.patches:\n                _x = p.get_x() + p.get_width() / 2\n                _y = p.get_y() + p.get_height()\n                value = int(p.get_height())\n                ax.text(round(_x, 5), round(_y, 5), format(round(value, 5), ','), ha=\"center\") \n        elif h_v == \"h\":\n            for p in ax.patches:\n                _x = p.get_x() + p.get_width() + float(space)\n                _y = p.get_y() + p.get_height()\n                value = int(p.get_width())\n                ax.text(_x, _y, format(value, ','), ha=\"left\")\n\n    if isinstance(axs, np.ndarray):\n        for idx, ax in np.ndenumerate(axs):\n            _show_on_single_plot(ax)\n    else:\n        _show_on_single_plot(axs)\n        \n        \ndef offset_png(x, y, path, ax, zoom, offset, border=2):\n    '''For adding other .png images to the graph.\n    source: https://stackoverflow.com/questions/61971090/how-can-i-add-images-to-bars-in-axes-matplotlib'''\n    \n    img = plt.imread(path)\n    im = OffsetImage(img, zoom=zoom)\n    im.image.axes = ax\n    x_offset = offset\n    ab = AnnotationBbox(im, (x, y), xybox=(x_offset, 0), frameon=False,\n                        xycoords='data', boxcoords=\"offset points\", pad=0)\n    ax.add_artist(ab)\n    \n    \ndef get_image_metadata(study_id, df):\n    '''Returns the label and bounding boxes (if any)\n    for a speciffic study id.'''\n    \n    data = df[df[\"study_id\"] == study_id]\n    \n    if data[\"Negative for Pneumonia\"].values == 1:\n        label = \"negative_for_pneumonia\"\n    elif data[\"Typical Appearance\"].values == 1:\n        label = \"typical\"\n    elif data[\"Indeterminate Appearance\"].values == 1:\n        label = \"indeterminate\"\n    else:\n        label = \"atypical\"\n        \n    bbox = list(data[\"boxes\"].values)\n    \n    return label, bbox\n\n\ndef save_dataset_artifact(run_name, artifact_name, path):\n    '''Saves dataset to W&B Artifactory.\n    run_name: name of the experiment\n    artifact_name: under what name should the dataset be stored\n    path: path to the dataset'''\n    \n    run = wandb.init(project='siim-covid19', \n                     name=run_name, \n                     config=CONFIG, anonymous=\"allow\")\n    artifact = wandb.Artifact(name=artifact_name, \n                              type='dataset')\n    artifact.add_file(path)\n\n    wandb.log_artifact(artifact)\n    wandb.finish()\n    print(\"Artifact has been saved successfully.\")\n    \n    \ndef create_wandb_plot(x_data=None, y_data=None, x_name=None, y_name=None, title=None, log=None, plot=\"line\"):\n    '''Create and save lineplot/barplot in W&B Environment.\n    x_data & y_data: Pandas Series containing x & y data\n    x_name & y_name: strings containing axis names\n    title: title of the graph\n    log: string containing name of log'''\n    \n    data = [[label, val] for (label, val) in zip(x_data, y_data)]\n    table = wandb.Table(data=data, columns = [x_name, y_name])\n    \n    if plot == \"line\":\n        wandb.log({log : wandb.plot.line(table, x_name, y_name, title=title)})\n    elif plot == \"bar\":\n        wandb.log({log : wandb.plot.bar(table, x_name, y_name, title=title)})\n    elif plot == \"scatter\":\n        wandb.log({log : wandb.plot.scatter(table, x_name, y_name, title=title)})\n        \n        \ndef create_wandb_hist(x_data=None, x_name=None, title=None, log=None):\n    '''Create and save histogram in W&B Environment.\n    x_data: Pandas Series containing x values\n    x_name: strings containing axis name\n    title: title of the graph\n    log: string containing name of log'''\n    \n    data = [[x] for x in x_data]\n    table = wandb.Table(data=data, columns=[x_name])\n    wandb.log({log : wandb.plot.histogram(table, x_name, title=title)})\n    \n    \ndef return_coords(box):\n    '''Returns coordinates from a bbox'''\n    # Get the list of dictionaries\n    box = ast.literal_eval(box)[0]\n    # Get the exact x and y coordinates\n    x1, y1, x2, y2 = box[\"x\"], box[\"y\"], box[\"x\"] + box[\"width\"], box[\"y\"] + box[\"height\"]\n    # Save coordinates\n    return (int(x1), int(y1), int(x2), int(y2))\n\n\ndef fix_inverted_radiograms(data, img):\n    '''Fixes inverted radiograms - with PhotometricInterpretation == \"MONOCHROME1\"\n    data: the .dcm dataset\n    img: the .dcm pixel_array'''\n    \n    if data.PhotometricInterpretation == \"MONOCHROME1\":\n        img = np.amax(img) - img\n    \n    img = img - np.min(img)\n    img = img / np.max(img)\n    img = (img * 255).astype(np.uint8)\n    \n    return img","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2021-06-07T18:46:38.594320Z","iopub.execute_input":"2021-06-07T18:46:38.594613Z","iopub.status.idle":"2021-06-07T18:46:38.615034Z","shell.execute_reply.started":"2021-06-07T18:46:38.594584Z","shell.execute_reply":"2021-06-07T18:46:38.613909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. 🗃 Metadata\n\nOur data consists of images + `.csv` files, containing custom information for each radiography.\n\n**Metadata structure**:\n1. `train_study_level.csv` - contains one row for each study, including correct labels.\n2. `train_image_level.csv` - containing one row for each image, including both correct labels and any bounding boxes in a dictionary format\n\n<center><img src=\"https://i.imgur.com/WFXxolI.png\" width=650></center>\n\n> 📌 **Important**: An image can have *multiple bounding boxes*.","metadata":{}},{"cell_type":"code","source":"# Save data to W&B Dashboard\nsave_dataset_artifact(run_name='save-train1',\n                      artifact_name='train_study_level', \n                      path=\"../input/siim-covid19-detection/train_study_level.csv\")\n\nsave_dataset_artifact(run_name='save-train2',\n                      artifact_name='train_image_level', \n                      path=\"../input/siim-covid19-detection/train_image_level.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-06-07T18:46:38.618058Z","iopub.execute_input":"2021-06-07T18:46:38.618332Z","iopub.status.idle":"2021-06-07T18:47:10.735831Z","shell.execute_reply.started":"2021-06-07T18:46:38.618304Z","shell.execute_reply":"2021-06-07T18:47:10.734583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read in metadata\ntrain_study = pd.read_csv(\"../input/siim-covid19-detection/train_study_level.csv\")\ntrain_image = pd.read_csv(\"../input/siim-covid19-detection/train_image_level.csv\")\n\nprint(color.BOLD + \"Train Study Shape:\" + color.END, train_study.shape, \"\\n\" +\n      color.BOLD + \"Train Image Shape:\" + color.END, train_image.shape, \"\\n\" +\n      \"\\n\" +\n      \"Note: There are {} missing values in train_image.\".\\\n                              format(train_image[\"boxes\"].isna().sum()), \"\\n\" +\n      \"This happens for labels = 'none' - no checkboxes.\", 3*\"\\n\")\n\n\n# Head of our 2 training metadata\ndf1_styler = train_study.head(3).style.set_table_attributes(\"style='display:inline'\").\\\n                                set_caption('TRAIN STUDY')\ndf2_styler = train_image.head(3).style.set_table_attributes(\"style='display:inline'\").\\\n                                set_caption('TRAIN IMAGE')\n\ndisplay_html(df1_styler._repr_html_() + df2_styler._repr_html_(), raw=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-07T18:47:10.737557Z","iopub.execute_input":"2021-06-07T18:47:10.737999Z","iopub.status.idle":"2021-06-07T18:47:10.819124Z","shell.execute_reply.started":"2021-06-07T18:47:10.737949Z","shell.execute_reply":"2021-06-07T18:47:10.818085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1.1 train_study analysis\n\n🔎 **Findings**:\n1. `id`: there are 6,054 unique ids - there are **no duplicates**\n2. `target`: one of our targets is to predict is the radiography is `negative_for_pneumonia`, has `typical` appearance, has `indeterminate` appearance or it's just `atypical`.\n3. an image can have **positive value for only 1 label**. For example, there aren't any images which are both `negative_for_pneumonia` and `indeterminate appearance` in the same time. It's only one or the other.\n4. **class imbalance is present** - especially for `Indeterminate Appearance` and `Atypical Appearance`\n\n### Target labels distribution\n> Now let's see how the **labels we'll have to predict** are layed out.","metadata":{}},{"cell_type":"code","source":"run = wandb.init(project='siim-covid19', name='metadata_eda', config=CONFIG, anonymous=\"allow\")","metadata":{"execution":{"iopub.status.busy":"2021-06-07T18:47:10.820279Z","iopub.execute_input":"2021-06-07T18:47:10.820530Z","iopub.status.idle":"2021-06-07T18:47:15.802657Z","shell.execute_reply.started":"2021-06-07T18:47:10.820504Z","shell.execute_reply":"2021-06-07T18:47:15.801759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Process id\ntrain_study[\"study_id\"] = train_study[\"id\"].apply(lambda x: x.split(\"_\")[0])\n\n# Data for plots\npneumonia = train_study[\"Negative for Pneumonia\"]\ntypical = train_study[\"Typical Appearance\"]\nindeterminate = train_study[\"Indeterminate Appearance\"]\natypical = train_study[\"Atypical Appearance\"]\n\n# Plotting\nfig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(21,20))\naxs = [ax1, ax2, ax3, ax4]\ndfs = [pneumonia, typical, indeterminate, atypical]\ntitles = [\"Pneumonia\", \"Typical\", \"Indeterminate\", \"Atypical\"]\n\nfor ax, df, title in zip(axs, dfs, titles):\n    sns.countplot(y=df, ax=ax, palette=my_colors[1:])\n    ax.set_title(title, fontsize=25, weight='bold')\n    show_values_on_bars(ax, h_v=\"h\", space=0.4)\n    ax.set_xticklabels([])\n    ax.set_ylabel('')\n    ax.set_xlabel('')\n    \n# Virus png\npath='../input/siimfisabiorsna-covid-2021/PinClipart.com_virus-clip-art_742280.png'\noffset_png(x=4378, y=1, path=path, ax=ax1, zoom=0.05, offset=-360, border=1)\noffset_png(x=2855, y=1, path=path, ax=ax2, zoom=0.05, offset=-50, border=1)\noffset_png(x=1049, y=1, path=path, ax=ax3, zoom=0.05, offset=-43, border=1)\noffset_png(x=474, y=1, path=path, ax=ax4, zoom=0.05, offset=40, border=1)","metadata":{"execution":{"iopub.status.busy":"2021-06-07T18:47:15.805819Z","iopub.execute_input":"2021-06-07T18:47:15.806453Z","iopub.status.idle":"2021-06-07T18:47:18.364590Z","shell.execute_reply.started":"2021-06-07T18:47:15.806402Z","shell.execute_reply":"2021-06-07T18:47:18.363456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save plots into W&B Dashboard\nfor title, df in zip(titles, dfs):\n    create_wandb_plot(x_data=[0, 1], \n                      y_data=df.value_counts().values, \n                      x_name=\"Flag\", y_name=\"Freq\", title=title, \n                      log=title, plot=\"bar\")","metadata":{"execution":{"iopub.status.busy":"2021-06-07T18:47:18.366667Z","iopub.execute_input":"2021-06-07T18:47:18.366952Z","iopub.status.idle":"2021-06-07T18:47:41.308441Z","shell.execute_reply.started":"2021-06-07T18:47:18.366923Z","shell.execute_reply":"2021-06-07T18:47:41.307204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### How many instances per label?\n\n> 📌 **Note**: 50% of our images have typical appearance. The rest 50% is split in order into *negative for pneumonia* (28%), *indeterminate* (17%) and the rest *atypical*.","metadata":{}},{"cell_type":"code","source":"# Get data and transform frequencies to percentages\ndf = train_study.groupby(['Negative for Pneumonia', 'Typical Appearance',\n       'Indeterminate Appearance', 'Atypical Appearance']).count().reset_index()\n\ndf[\"label\"] = ['Atypical Appearance', 'Indeterminate Appearance',\n               'Typical Appearance', 'Negative for Pneumonia']\ndf[\"perc\"] = df[\"id\"]/df[\"id\"].sum()*100\n\n# Plot\nbar,ax = plt.subplots(figsize=(21,10))\nax = sns.barplot(x=df[\"label\"], y=df[\"perc\"], \n                 ci=None, palette=my_colors, orient='v')\nax.set_title(\"Label % in Total Observations\", fontsize=25,\n             weight = \"bold\")\nax.set_xlabel(\" \")\nax.set_ylabel(\"Percentage\")\nfor rect in ax.patches:\n    ax.text (rect.get_x() + rect.get_width() / 2,rect.get_height(),\n             \"%.1f%%\"% rect.get_height(), weight='bold')","metadata":{"execution":{"iopub.status.busy":"2021-06-07T18:47:41.309651Z","iopub.execute_input":"2021-06-07T18:47:41.309960Z","iopub.status.idle":"2021-06-07T18:47:42.896560Z","shell.execute_reply.started":"2021-06-07T18:47:41.309925Z","shell.execute_reply":"2021-06-07T18:47:42.895706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_wandb_plot(x_data=df[\"label\"].values,\n                  y_data=df[\"perc\"].values,\n                  x_name=\"Label\", y_name=\"Percentage\", \n                  title=\"Label % in Total Observations\", \n                  log=\"perc\", plot=\"bar\")","metadata":{"execution":{"iopub.status.busy":"2021-06-07T18:47:42.897616Z","iopub.execute_input":"2021-06-07T18:47:42.897863Z","iopub.status.idle":"2021-06-07T18:47:53.493832Z","shell.execute_reply.started":"2021-06-07T18:47:42.897838Z","shell.execute_reply":"2021-06-07T18:47:53.493034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wandb.finish()","metadata":{"execution":{"iopub.status.busy":"2021-06-07T18:47:53.494851Z","iopub.execute_input":"2021-06-07T18:47:53.495285Z","iopub.status.idle":"2021-06-07T18:48:04.497013Z","shell.execute_reply.started":"2021-06-07T18:47:53.495257Z","shell.execute_reply":"2021-06-07T18:48:04.495937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1.2 train_image analysis\n\n🔎 **Findings**:\n1. `StudyInstanceUID` corresponds 1:1 to `new_id` created for `train_study`.\n2. `image_id` is unique in the `image_train` data.\n3. There are images with multiple bounding boxes!\n4. There can be multiple images per study!\n\n### How many images have some sort of abnormality?\n\nTo have a bounding box we need to have something weird detected in the scan. If there is nothing weird ... then we don't need a bounding box!","metadata":{}},{"cell_type":"code","source":"run = wandb.init(project='siim-covid19', name='train_imgs_eda', config=CONFIG, anonymous=\"allow\")","metadata":{"execution":{"iopub.status.busy":"2021-06-07T18:48:04.498433Z","iopub.execute_input":"2021-06-07T18:48:04.498737Z","iopub.status.idle":"2021-06-07T18:48:09.568620Z","shell.execute_reply.started":"2021-06-07T18:48:04.498705Z","shell.execute_reply":"2021-06-07T18:48:09.567443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Process id\ntrain_image[\"image_id\"] = train_image[\"id\"].apply(lambda x: x.split(\"_\")[0])\n\n# Data for plotting\ndf = train_image[\"label\"].apply(lambda x: x.split(\" \")[0]).\\\n                                    value_counts().reset_index()\n\n# Plot\nplt.figure(figsize=(21, 15))\nax = sns.barplot(data=df, y=\"index\", x=\"label\", palette=my_colors[3:])\nshow_values_on_bars(ax, h_v=\"h\", space=0.4)\nplt.title(\"How many images have a bounding box present?\", \n          fontsize=30, weight='bold')\nplt.xticks([])\nplt.ylabel('')\nplt.xlabel('');\n\n# Virus png\npath='../input/siimfisabiorsna-covid-2021/PinClipart.com_virus-clip-art_742280.png'\noffset_png(x=4294, y=0, path=path, ax=ax, zoom=0.1, offset=-110, border=1)","metadata":{"execution":{"iopub.status.busy":"2021-06-07T18:48:09.570610Z","iopub.execute_input":"2021-06-07T18:48:09.571038Z","iopub.status.idle":"2021-06-07T18:48:12.195245Z","shell.execute_reply.started":"2021-06-07T18:48:09.570991Z","shell.execute_reply":"2021-06-07T18:48:12.194416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_wandb_plot(x_data=df[\"index\"], \n                  y_data=df[\"label\"].values, \n                  x_name=\"BBox\", y_name=\"Freq\", \n                  title=\"Images with bbox\",\n                  log=\"bbox\", plot=\"bar\")","metadata":{"execution":{"iopub.status.busy":"2021-06-07T18:48:12.197924Z","iopub.execute_input":"2021-06-07T18:48:12.198302Z","iopub.status.idle":"2021-06-07T18:48:21.634158Z","shell.execute_reply.started":"2021-06-07T18:48:12.198268Z","shell.execute_reply":"2021-06-07T18:48:21.633392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### How many images per each study?\n\n> Majority of studies have only 1 images. That being said, we have ~230 studies that have multiple images available (up to 9).","metadata":{}},{"cell_type":"code","source":"# Crate df\ndf = train_image[\"StudyInstanceUID\"].value_counts().reset_index().\\\n                        sort_values(\"StudyInstanceUID\", ascending=False)\nprint(color.BOLD + \"Max number of images available per study:\" + color.END, \n      df[\"StudyInstanceUID\"].max(), \"\\n\" +\n      color.BOLD + \"Min number of images available per study:\" + color.END, \n      df[\"StudyInstanceUID\"].min(), 2*\"\\n\")\n# Plot\nplt.figure(figsize=(21, 14))\nsns.distplot(a=df[\"StudyInstanceUID\"], color=my_colors[6], \n             hist=False, kde_kws=dict(lw=7, ls=\"-\"))\nplt.title(\"How many images per study?\", \n          fontsize=30, weight='bold')\nplt.xticks([])\nplt.ylabel('Study Frequency')\nplt.xlabel('Image Ids');","metadata":{"execution":{"iopub.status.busy":"2021-06-07T18:48:21.635453Z","iopub.execute_input":"2021-06-07T18:48:21.636051Z","iopub.status.idle":"2021-06-07T18:48:24.184247Z","shell.execute_reply.started":"2021-06-07T18:48:21.636008Z","shell.execute_reply":"2021-06-07T18:48:24.183268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wandb.log({\"max_images_on_study\" : df[\"StudyInstanceUID\"].max()})\ncreate_wandb_hist(x_data=df[\"StudyInstanceUID\"], \n                  x_name=\"Image Freq\", title=\"No. images per study\", \n                  log=\"hist\")","metadata":{"execution":{"iopub.status.busy":"2021-06-07T18:48:24.185577Z","iopub.execute_input":"2021-06-07T18:48:24.185968Z","iopub.status.idle":"2021-06-07T18:48:34.994047Z","shell.execute_reply.started":"2021-06-07T18:48:24.185926Z","shell.execute_reply":"2021-06-07T18:48:34.992986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wandb.finish()","metadata":{"execution":{"iopub.status.busy":"2021-06-07T18:48:34.995292Z","iopub.execute_input":"2021-06-07T18:48:34.995539Z","iopub.status.idle":"2021-06-07T18:48:46.108360Z","shell.execute_reply.started":"2021-06-07T18:48:34.995514Z","shell.execute_reply":"2021-06-07T18:48:46.107331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> After EDA, my [W&B Dashboard](https://wandb.ai/andrada/siim-covid19?workspace=user-andrada) looks like this:\n\n<center><img src=\"https://i.imgur.com/HQBdtSd.gif\" width=600></center>\n\n### Create the full train dataset\n\nThis is also how the `final_label` will need to look before submission.\n\n<center><img src=\"https://i.imgur.com/8Ckg1L0.png\" width=600></center>","metadata":{}},{"cell_type":"code","source":"# Merge all info together\ntrain = pd.merge(train_image, train_study, \n                 left_on=\"StudyInstanceUID\", right_on=\"study_id\")\n\ntrain.drop([\"id_x\", \"StudyInstanceUID\", \"id_y\"], axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-07T18:48:46.109538Z","iopub.execute_input":"2021-06-07T18:48:46.109937Z","iopub.status.idle":"2021-06-07T18:48:46.130650Z","shell.execute_reply.started":"2021-06-07T18:48:46.109882Z","shell.execute_reply":"2021-06-07T18:48:46.129702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> Let's also look at the study id `0fd2db233deb`, which has most of the images available. 8 images have nothing unusual in them and 1 is labeled as `Indeterminate Appearance`.","metadata":{}},{"cell_type":"code","source":"train[train[\"study_id\"] == \"0fd2db233deb\"]","metadata":{"execution":{"iopub.status.busy":"2021-06-07T18:48:46.131940Z","iopub.execute_input":"2021-06-07T18:48:46.132222Z","iopub.status.idle":"2021-06-07T18:48:46.153546Z","shell.execute_reply.started":"2021-06-07T18:48:46.132188Z","shell.execute_reply":"2021-06-07T18:48:46.152876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. 📷 Images\n\nGood, now that we've explored the metadata and the `target` labels, we can start focusing on the good stuff - meaning the **CT scans**.\n\n<center><img src=\"https://i.imgur.com/DFDDdvD.png\" width=550></center>\n\n### WHAT ARE CT SCANS?\n\n\"A **Computerized Tomography** scan (CT or CAT scan) uses computers and rotating X-ray machines to **create cross-sectional images** of the body. These images provide **more detailed** information than normal X-ray images. They can show the **soft tissues**, **blood vessels**, and **bones** in various parts of the body.\"\n\n<center><img src=\"https://i.imgur.com/5WUxAHZ.png\" width=550></center>","metadata":{}},{"cell_type":"code","source":"run = wandb.init(project='siim-covid19', name='image_explore', config=CONFIG, anonymous=\"allow\")","metadata":{"execution":{"iopub.status.busy":"2021-06-07T18:48:46.154670Z","iopub.execute_input":"2021-06-07T18:48:46.155139Z","iopub.status.idle":"2021-06-07T18:48:52.130330Z","shell.execute_reply.started":"2021-06-07T18:48:46.155107Z","shell.execute_reply":"2021-06-07T18:48:52.129044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_dcm_info(study_ids, df):\n    '''Show .dcm images along with description.'''\n    wandb_logs = []\n    \n    fig, axes = plt.subplots(nrows=2, ncols=3, figsize=(21,10))\n\n    # Get .dcm paths\n    dcm_paths = [glob.glob(f\"../input/siim-covid19-detection/train/{study_id}/*/*\")[0]\n                 for study_id in study_ids]\n    datasets = [pydicom.dcmread(path) for path in dcm_paths]\n    images = [apply_voi_lut(dataset.pixel_array, dataset) for dataset in datasets]\n\n    # Loop through the information\n    for study_id, data, img, i in zip(study_ids, datasets, images, range(2*3)):\n        # Fix inverted images\n        img = fix_inverted_radiograms(data, img)\n\n        # Below function available in functions section ;)\n        label, bbox = get_image_metadata(study_id, df)\n        \n        # Check for bounding box and add if it's the case\n        try: \n            # For no bbox, the list is [nan]\n            no_box = math.isnan(bbox[0])\n            pass\n        except TypeError:\n            # Retrieve the bounding box\n            all_coords = []\n            for box in bbox:\n                all_coords.append(return_coords(box))\n\n            for (x1, y1, x2, y2) in all_coords:\n                cv2.rectangle(img, (x1, y1), (x2, y2), (0, 80, 255), 15)\n                cv2.putText(img, label, (x1, y1-14), \n                            cv2.FONT_HERSHEY_SIMPLEX, 3, (0, 0, 0), 4)\n                \n        # Plot the image\n        x = i // 3\n        y = i % 3\n        \n        axes[x, y].imshow(img, cmap=\"rainbow\")\n        axes[x, y].set_title(f\"Label: {label} \\n Sex: {data.PatientSex} | Body Part: {data.BodyPartExamined}\", \n                  fontsize=14, weight='bold')\n        axes[x, y].axis('off');\n        \n        # Save to W&B\n        wandb_logs.append(wandb.Image(img, \n                                      caption=f\"Label: {label} \\n Sex: {data.PatientSex} | Body Part: {data.BodyPartExamined}\"))\n          \n    wandb.log({f\"{label}\": wandb_logs})","metadata":{"execution":{"iopub.status.busy":"2021-06-07T18:48:52.132273Z","iopub.execute_input":"2021-06-07T18:48:52.132693Z","iopub.status.idle":"2021-06-07T18:48:53.454664Z","shell.execute_reply.started":"2021-06-07T18:48:52.132636Z","shell.execute_reply":"2021-06-07T18:48:53.453819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Typical Appearance","metadata":{}},{"cell_type":"code","source":"show_dcm_info(study_ids=[\"72044bb44d41\", \"5b65a69885b6\", \"6aa32e76f998\",\n                         \"c9ffe6312921\", \"082cafb03942\", \"d3e83031ebea\"], \n              df=train)","metadata":{"execution":{"iopub.status.busy":"2021-06-07T18:48:53.459686Z","iopub.execute_input":"2021-06-07T18:48:53.460001Z","iopub.status.idle":"2021-06-07T18:49:23.610116Z","shell.execute_reply.started":"2021-06-07T18:48:53.459971Z","shell.execute_reply":"2021-06-07T18:49:23.609083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Atypical Appearance","metadata":{}},{"cell_type":"code","source":"show_dcm_info(study_ids=[\"f807cd855d31\", \"8087e3bc0efe\", \"7249de10ed69\",\n                         \"e300a4e86207\", \"4bac6c7da8b8\", \"f2d30ac37f7b\"], \n              df=train)","metadata":{"execution":{"iopub.status.busy":"2021-06-07T18:49:23.611962Z","iopub.execute_input":"2021-06-07T18:49:23.612643Z","iopub.status.idle":"2021-06-07T18:49:51.761082Z","shell.execute_reply.started":"2021-06-07T18:49:23.612610Z","shell.execute_reply":"2021-06-07T18:49:51.760311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Indeterminate Appearance","metadata":{}},{"cell_type":"code","source":"show_dcm_info(study_ids=[\"b949689a9ef1\", \"fe7e6015560d\", \"feffa20fac13\",\n                         \"747483509d0e\", \"c70369caef91\", \"1e1b4b1b53cb\"], \n              df=train)","metadata":{"execution":{"iopub.status.busy":"2021-06-07T18:49:51.762063Z","iopub.execute_input":"2021-06-07T18:49:51.762437Z","iopub.status.idle":"2021-06-07T18:50:24.732424Z","shell.execute_reply.started":"2021-06-07T18:49:51.762410Z","shell.execute_reply":"2021-06-07T18:50:24.731478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Negative for Pneumonia","metadata":{}},{"cell_type":"code","source":"show_dcm_info(study_ids=[\"612ea5194007\", \"db14e640e037\", \"d4ab797396b4\",\n                         \"6ae8a88c4b0c\", \"b3cf474bee3b\", \"0ba55e5422ab\"], \n              df=train)","metadata":{"execution":{"iopub.status.busy":"2021-06-07T18:50:24.734248Z","iopub.execute_input":"2021-06-07T18:50:24.734636Z","iopub.status.idle":"2021-06-07T18:50:41.462431Z","shell.execute_reply.started":"2021-06-07T18:50:24.734595Z","shell.execute_reply":"2021-06-07T18:50:41.461647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2.2 Magic - save images & bounding boxes to W&B\n\n> 🔗 **Thank you very much to Ayush**, who inspired the following experiment. :) You can find [his full notebook with explanations here](https://www.kaggle.com/ayuraj/visualize-bounding-boxes-interactively).","metadata":{}},{"cell_type":"code","source":"def wandb_bbox(image, bboxes, true_label, class_id_to_label):\n    '''Source: https://www.kaggle.com/ayuraj/visualize-bounding-boxes-interactively\n    Thank you Ayush for all your hard work! ^^'''\n    all_boxes = []\n    for bbox in bboxes:\n        box_data = {\"position\": {\n                        \"minX\": bbox[0],\n                        \"minY\": bbox[1],\n                        \"maxX\": bbox[2],\n                        \"maxY\": bbox[3]\n                    },\n                     \"class_id\" : int(true_label),\n                     \"box_caption\": class_id_to_label[true_label],\n                     \"domain\" : \"pixel\"}\n        all_boxes.append(box_data)\n    \n\n    return wandb.Image(image, boxes={\n        \"ground_truth\": {\n            \"box_data\": all_boxes,\n          \"class_labels\": class_id_to_label\n        }\n    })\n\n\ndef resize_img_and_coord(img, coord, resize):\n    '''Resizes the image and its coordinates.\n    img: the pixel.array image\n    coord: the speciffic coordinates from return_coordinates() function\n    resize: and integer specifying the desired size of new image'''\n    \n    # Resize the image\n    w_old, h_old = img.shape\n\n    # Resize the coordinates\n    img = cv2.resize(img, (resize,resize))\n\n    new_x1 = int(coord[0][0] / (w_old/resize))\n    new_y1 = int(coord[0][1] / (h_old/resize))\n    new_x2 = int(coord[0][2] / (w_old/resize))\n    new_y2 = int(coord[0][3] / (h_old/resize))\n    new_coord = [(new_x1, new_y1, new_x2, new_y2)]\n    \n    return img, new_coord","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2021-06-07T18:50:41.464208Z","iopub.execute_input":"2021-06-07T18:50:41.464783Z","iopub.status.idle":"2021-06-07T18:50:42.882865Z","shell.execute_reply.started":"2021-06-07T18:50:41.464734Z","shell.execute_reply":"2021-06-07T18:50:42.882183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's look at an **example of 10 images** (below you can see how all the code outputs in the [W&B Dashboard](https://wandb.ai/andrada/siim-covid19?workspace=user-andrada)!)\n<center><video src=\"https://i.imgur.com/q0jCbza.mp4\" width=650 controls></center>","metadata":{}},{"cell_type":"code","source":"# Get a few example ids (image_id)\nexample_ids = [\"000a312787f2\", \"0012ff7358bc\", \"001398f4ff4f\",\n               \"001bd15d1891\", \"002e9b2128d0\", \"ffbeafe30b77\",\n               \"0022227f5adf\", \"00a129830f4e\", \"01376c1ba556\", \"008ca392cff3\"]\n\n# Read in datas\nstudy_ids = train[train[\"image_id\"].isin(example_ids)][\"study_id\"].values\npaths = [glob.glob(f\"../input/siim-covid19-detection/train/{i}/*/*\")[0]\n         for i in study_ids]\n\n# Retrieve resized information\nimages, coords, labels = [], [], []\nfor path, study_id in zip(paths, study_ids):\n    try:\n        # Read data file\n        data = pydicom.dcmread(path)\n        # Get image data\n        img = apply_voi_lut(data.pixel_array, data)\n        # Get image coordinates\n        label, bbox = get_image_metadata(study_id=study_id, df=train)\n        coord = [return_coords(box) for box in bbox]\n\n        # Fix inverted radiograms + resize\n        img = fix_inverted_radiograms(data, img)\n        resized_img, resized_coord = resize_img_and_coord(img, coord, resize=200)\n\n        images.append(resized_img)\n        coords.append(resized_coord)\n        labels.append(label)\n    except RuntimeError:\n        pass","metadata":{"execution":{"iopub.status.busy":"2021-06-07T18:50:42.883986Z","iopub.execute_input":"2021-06-07T18:50:42.884514Z","iopub.status.idle":"2021-06-07T18:50:49.592375Z","shell.execute_reply.started":"2021-06-07T18:50:42.884475Z","shell.execute_reply":"2021-06-07T18:50:49.591395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Map each label to a number\nclass_label_to_id = {'atypical': 0, 'indeterminate': 1, 'typical': 2}\n# And each number to a label\nclass_id_to_label = {val: key for key, val in class_label_to_id.items()}\n\n# Log each image\nwandb_bbox_list = []\n\nfor image, coord, label in zip(images, coords, labels):\n    wandb_bbox_list.append(wandb_bbox(image=image,\n                                      bboxes=coord, \n                                      true_label=class_label_to_id[label],\n                                      class_id_to_label=class_id_to_label))\n\n# Save images to W&B Dashboard\nwandb.log({\"radiograph\": wandb_bbox_list})\n\nprint(color.BOLD + \"Finished! Your Images were uploaded in your W&B Dashboard!\" + color.END)","metadata":{"execution":{"iopub.status.busy":"2021-06-07T18:50:49.593475Z","iopub.execute_input":"2021-06-07T18:50:49.593719Z","iopub.status.idle":"2021-06-07T18:50:51.989604Z","shell.execute_reply.started":"2021-06-07T18:50:49.593694Z","shell.execute_reply":"2021-06-07T18:50:51.988682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wandb.finish()","metadata":{"execution":{"iopub.status.busy":"2021-06-07T18:50:51.990645Z","iopub.execute_input":"2021-06-07T18:50:51.990922Z","iopub.status.idle":"2021-06-07T18:51:08.924780Z","shell.execute_reply.started":"2021-06-07T18:50:51.990882Z","shell.execute_reply":"2021-06-07T18:51:08.923976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Extract Metadata from .dcm\n\n> 📌 **Important**: We can create *more* **metadata** (more information on the images) from the information stored in the `.dcm` files. Below I am extracting all features stored in each `.dcm` file and storing them into a sepparate dataframe.\n\n## 3.1 Store and Save Metadata\n\n<center><img src=\"https://i.imgur.com/8Fhupoe.png\" width=650></center>","metadata":{}},{"cell_type":"code","source":"def get_observation_data(path):\n    \"\"\"Get information from the .dcm files.\n    path: complete path to the .dcm file\"\"\"\n\n    image_data = pydicom.read_file(path)\n    \n    # Dictionary to store the information from the image\n    observation_data = {\n        \"FileNumber\" : path.split(\"/\")[5],\n        \"Rows\" : image_data.get(\"Rows\"),\n        \"Columns\" : image_data.get(\"Columns\"),\n        \"PatientID\" : image_data.get(\"PatientID\"),\n        \"PatientName\" : image_data.get(\"PatientName\"),\n        \"PhotometricInterpretation\" : image_data.get(\"PhotometricInterpretation\"),\n        \"StudyInstanceUID\" : image_data.get(\"StudyInstanceUID\"),\n        \"SamplesPerPixel\" : image_data.get(\"SamplesPerPixel\"),\n        \"BitsAllocated\" : image_data.get(\"BitsAllocated\"),\n        \"BitsStored\" : image_data.get(\"BitsStored\"),\n        \"HighBit\" : image_data.get(\"HighBit\"),\n        \"PixelRepresentation\" : image_data.get(\"PixelRepresentation\"),\n    }\n\n    # String columns\n    str_columns = [\"ImageType\", \"Modality\", \"PatientSex\", \"BodyPartExamined\"]\n    for k in str_columns:\n        observation_data[k] = str(image_data.get(k)) if k in image_data else None\n\n    \n    return observation_data","metadata":{"execution":{"iopub.status.busy":"2021-06-07T18:51:08.925856Z","iopub.execute_input":"2021-06-07T18:51:08.926269Z","iopub.status.idle":"2021-06-07T18:51:08.932729Z","shell.execute_reply.started":"2021-06-07T18:51:08.926240Z","shell.execute_reply":"2021-06-07T18:51:08.932078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # An example\n# p = \"../input/siim-covid19-detection/train/00792b5c8852/1f52bcb3143e/3fadf4b48db3.dcm\"\n# example = get_observation_data(p)\n# print(example)\n\n# # === GET ALL METADATA ===\n# # Get all paths to .dcm files\n# all_paths = glob.glob(\"../input/siim-covid19-detection/train/*/*/*\")\n\n# # === Get metadata ===\n# exceptions = 0\n# dicts = []\n\n# for path in tqdm.tqdm(all_paths):\n#     # Get .dcm metadata\n#     ### TODO: add .dcm id\n#     try:\n#         d = get_observation_data(path)\n#         dicts.append(d)\n#     except Exception as e:\n#         exceptions += 1\n#         continue\n        \n# # === SAVE METADATA ===\n# # Convert into df\n# meta_train_data = pd.DataFrame(data=dicts, columns=example.keys())\n# meta_train_data[\"\"]\n# # Export information\n# meta_train_data.to_csv(\"meta_train.csv\", index=False)\n\n# print(\"Metadata processed & saved successfuly :)\")","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2021-06-07T18:51:08.933775Z","iopub.execute_input":"2021-06-07T18:51:08.934210Z","iopub.status.idle":"2021-06-07T18:51:08.951511Z","shell.execute_reply.started":"2021-06-07T18:51:08.934171Z","shell.execute_reply":"2021-06-07T18:51:08.950714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3.2 Let's Analyse the new information","metadata":{}},{"cell_type":"code","source":"# Save data to W&B Dashboard\nsave_dataset_artifact(run_name='dave-dcm-meta',\n                      artifact_name='dcm_metadata', \n                      path=\"../input/siimfisabiorsna-covid-2021/meta_train.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-06-07T18:51:08.952652Z","iopub.execute_input":"2021-06-07T18:51:08.953123Z","iopub.status.idle":"2021-06-07T18:51:22.278162Z","shell.execute_reply.started":"2021-06-07T18:51:08.953049Z","shell.execute_reply":"2021-06-07T18:51:22.277231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import\ndcm_meta = pd.read_csv(\"../input/siimfisabiorsna-covid-2021/meta_train.csv\")\ndcm_meta = pd.concat([dcm_meta, train], axis=1)\n\ndcm_meta.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-06-07T18:51:22.279499Z","iopub.execute_input":"2021-06-07T18:51:22.279766Z","iopub.status.idle":"2021-06-07T18:51:22.330036Z","shell.execute_reply.started":"2021-06-07T18:51:22.279738Z","shell.execute_reply":"2021-06-07T18:51:22.328961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Patient's Gender","metadata":{}},{"cell_type":"code","source":"# Get the Data\nlabels = ['Negative for Pneumonia', 'Typical Appearance', \n          'Indeterminate Appearance', 'Atypical Appearance']\ndt = dcm_meta.groupby(\"PatientSex\")[labels].sum().reset_index()\n\n# Plotting\nfig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(21,20))\naxs = [ax1, ax2, ax3, ax4]\n\nfor ax, title in zip(axs, labels):\n    sns.barplot(data=dt, x=title, y=\"PatientSex\", \n                ax=ax, palette=my_colors[0:])\n    ax.set_title(title, fontsize=25, weight='bold')\n    show_values_on_bars(ax, h_v=\"h\", space=0.4)\n    ax.set_xticklabels([])\n    ax.set_ylabel('')\n    ax.set_xlabel('')","metadata":{"execution":{"iopub.status.busy":"2021-06-07T18:51:22.331349Z","iopub.execute_input":"2021-06-07T18:51:22.331639Z","iopub.status.idle":"2021-06-07T18:51:22.933626Z","shell.execute_reply.started":"2021-06-07T18:51:22.331610Z","shell.execute_reply":"2021-06-07T18:51:22.932679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Location of X-rays","metadata":{}},{"cell_type":"code","source":"# Get the Data\nlabels = ['Negative for Pneumonia', 'Typical Appearance', \n          'Indeterminate Appearance', 'Atypical Appearance']\ndt = dcm_meta.groupby(\"BodyPartExamined\")[labels].sum().reset_index()\ndt = dt.sort_values(\"Negative for Pneumonia\", ascending=False)\n\n# Plotting\nfig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(21,20))\naxs = [ax1, ax2, ax3, ax4]\n\nfor ax, title in zip(axs, labels):\n    sns.barplot(data=dt, x=title, y=\"BodyPartExamined\", \n                ax=ax, palette=my_colors)\n    ax.set_title(title, fontsize=25, weight='bold')\n    show_values_on_bars(ax, h_v=\"h\", space=0.4)\n    ax.set_xticklabels([])\n    ax.set_ylabel('')\n    ax.set_xlabel('')\n    fig.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2021-06-07T18:51:22.935037Z","iopub.execute_input":"2021-06-07T18:51:22.935326Z","iopub.status.idle":"2021-06-07T18:51:24.038665Z","shell.execute_reply.started":"2021-06-07T18:51:22.935296Z","shell.execute_reply":"2021-06-07T18:51:24.037668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### MONOCHROME1 or MONOCHROME2? That's the question ...","metadata":{}},{"cell_type":"code","source":"# Get the Data\nlabels = ['Negative for Pneumonia', 'Typical Appearance', \n          'Indeterminate Appearance', 'Atypical Appearance']\ndt = dcm_meta.groupby(\"PhotometricInterpretation\")[labels].sum().reset_index()\n\n# Plotting\nfig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(21,20))\naxs = [ax1, ax2, ax3, ax4]\n\nfor ax, title in zip(axs, labels):\n    sns.barplot(data=dt, x=title, y=\"PhotometricInterpretation\", \n                ax=ax, palette=my_colors[5:])\n    ax.set_title(title, fontsize=25, weight='bold')\n    show_values_on_bars(ax, h_v=\"h\", space=0.4)\n    ax.set_xticklabels([])\n    ax.set_ylabel('')\n    ax.set_xlabel('')\n    fig.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2021-06-07T18:51:24.040159Z","iopub.execute_input":"2021-06-07T18:51:24.040544Z","iopub.status.idle":"2021-06-07T18:51:24.744957Z","shell.execute_reply.started":"2021-06-07T18:51:24.040503Z","shell.execute_reply":"2021-06-07T18:51:24.743874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> The Exploratory Dashboard in W&B!\n<center><video src=\"https://i.imgur.com/P8tdXpU.mp4\" width=650 controls></center>\n\n<center><img src=\"https://i.imgur.com/cUQXtS7.png\"></center>\n\n# ⌨️🎨 My Specs\n\n* **Z8 G4** Workstation 🖥\n* 2 CPUs & 96GB Memory 💾\n* NVIDIA **Quadro RTX 8000** 🎮\n* **RAPIDS** version 0.17 🏃🏾•♀️","metadata":{}}]}