{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"-------------------------------------------------------------\n**Basic data exploration, paying particular attention to checking relationships in the data against our expectations, or to find out how things actually work. Gaining insight and checking for inconsistencies or glitches.**\n\n-------------------------------------------------------------","metadata":{}},{"cell_type":"code","source":"%%capture\n!conda install gdcm -c conda-forge -y","metadata":{"execution":{"iopub.status.busy":"2021-05-30T20:00:04.819813Z","iopub.execute_input":"2021-05-30T20:00:04.820273Z","iopub.status.idle":"2021-05-30T20:00:43.007836Z","shell.execute_reply.started":"2021-05-30T20:00:04.820232Z","shell.execute_reply":"2021-05-30T20:00:43.006124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Packages\n\nimport os, time, glob, re, pprint, random, math\nfrom collections import defaultdict, Counter\n\nimport pandas as pd \nimport numpy as np\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport matplotlib.ticker as ticker\nimport matplotlib.patches as patches\n\nimport seaborn as sns\n\nimport cv2\nimport gdcm\n\nimport pydicom\nfrom pydicom import dcmread\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\n\nimport imagehash\nfrom PIL import Image as im\n\nimport tqdm","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-05-30T20:36:38.211218Z","iopub.execute_input":"2021-05-30T20:36:38.211580Z","iopub.status.idle":"2021-05-30T20:36:38.218284Z","shell.execute_reply.started":"2021-05-30T20:36:38.211548Z","shell.execute_reply":"2021-05-30T20:36:38.217292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Magic\n\n%matplotlib inline ","metadata":{"execution":{"iopub.status.busy":"2021-05-30T20:00:43.024172Z","iopub.execute_input":"2021-05-30T20:00:43.024715Z","iopub.status.idle":"2021-05-30T20:00:43.040123Z","shell.execute_reply.started":"2021-05-30T20:00:43.024662Z","shell.execute_reply":"2021-05-30T20:00:43.038916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Pandas options\n\npd.set_option(\"display.expand_frame_repr\", True)\npd.set_option('display.max_colwidth', 0)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T20:00:43.042217Z","iopub.execute_input":"2021-05-30T20:00:43.042557Z","iopub.status.idle":"2021-05-30T20:00:43.056649Z","shell.execute_reply.started":"2021-05-30T20:00:43.042526Z","shell.execute_reply":"2021-05-30T20:00:43.054608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Global values\n\nCODE_PATH = \"./\"\nDATA_PATH = \"../input/siim-covid19-detection\"\nTRAIN_PATH = os.path.join(DATA_PATH, \"train\")\nTEST_PATH = os.path.join(DATA_PATH, \"test\")","metadata":{"execution":{"iopub.status.busy":"2021-05-30T20:00:43.058493Z","iopub.execute_input":"2021-05-30T20:00:43.058938Z","iopub.status.idle":"2021-05-30T20:00:43.072119Z","shell.execute_reply.started":"2021-05-30T20:00:43.058896Z","shell.execute_reply":"2021-05-30T20:00:43.071091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Utility functions\n\ndef print_title(s):\n    \n    # Print an underlined title\n    \n    print(\"%s\\n%s\" % (s, '-'*len(s)))\n    \n    \ndef na_fill(dtype):\n    \n    # Fill in missing values so that Seaborn countplot() will include them.\n    # Also make sure the missing value for a string column is itself a string.\n    \n    if dtype == np.float64:\n        result = -999\n    elif dtype == np.int64:\n        result = -999\n    elif dtype == np.object:\n        result = \"NA\"\n    else:\n        result = -999\n\n    return result\n\n\ndef explode_list_col(df, col):\n    \n    other_cols = list(df.columns)\n    other_cols.remove(col)\n    out_df = df.set_index(other_cols)[col]\n    out_df = out_df.apply(pd.Series)\n    out_df = out_df.stack()\n    out_df = out_df.reset_index()\n    out_df = out_df.drop(f\"level_{len(other_cols)}\", axis=1)\n    out_df = out_df.rename(columns={0 : col})\n    \n    return out_df\n    \n\ndef cat_order(train_df, test_df, cols):\n    \n    # For comparison, we want the categories to be the same between train and test. \n    # We also want the categories to be in the same order for both. \n    # We need to make sure missing value types are consistent with the column data\n    # type (else sort will complain), and make sure that Seaborn countplot() will\n    # include them (it won't include np.nan). Dealing with the latter two issues \n    # is the job of fillna(). When the data is plotted, it needs to be run through\n    # fillna() as well.\n\n    category_order = {}\n    n = len(cols)\n\n    for col in cols:\n\n        if col in train_cols:\n            train_dtype = train_df[col].dtype\n        else:\n            train_dtype = None     \n        if col in test_cols:\n            test_dtype = test_df[col].dtype\n        else:\n            test_dtype = None\n        if (train_dtype != None) and (test_dtype != None) and (train_dtype != test_dtype):\n            print(f\"***Incompatible train/test data types for column {col}\")\n            break\n        \n        col_vals = []\n        if col in train_cols:\n            col_vals += list(train_df[col].fillna(na_fill(train_dtype)).unique())\n        if col in test_cols:\n            col_vals += list(test_df[col].fillna(na_fill(test_dtype)).unique())\n        col_vals = list(set(col_vals))\n        col_vals.sort()\n    \n        category_order[col] = col_vals\n        \n    return category_order\n\n\ndef plot_categories(df, col, order, title, ax, xlabel=None, ylabel=None, pallete=\"Set3\", scale=\"log\"):\n\n    data = df[col].fillna(na_fill(df[col].dtype))\n    ax = sns.countplot(x=data, order=order, palette=pallete, ax=ax)\n    ax.set_xticklabels(ax.get_xticklabels(), rotation=90, ha=\"right\")\n    ax.set_yscale(scale)\n    if scale == \"log\":\n        ax.set_ylim(bottom=0.5)\n    else:\n        ax.set_ylim(bottom=0)\n    counts = data.value_counts()\n    for i, p in enumerate(ax.patches):\n        px = p.get_x()+0.05\n        py = max(p.get_height()*0.9, 1.0)\n        ax.annotate(counts.get(order[i], \"\"), (px, py), fontsize=10, rotation=90)\n    ax.set_title(title, fontsize=15)\n    if xlabel: ax.set_xlabel(xlabel)\n    if ylabel: ax.set_ylabel(ylabel)\n    \n    return\n\n\ndef plot_cat_sub(df, col, sub_col, title, ax, scale=\"log\"):\n\n    ax = sns.countplot(data=df, x=col, hue=sub_col, ax=ax)\n    ax.set_xticklabels(ax.get_xticklabels(), rotation=90, ha=\"right\")\n    ax.set_yscale(scale)\n    if scale == \"log\":\n        ax.set_ylim(bottom=0.5)\n    else:\n        ax.set_ylim(bottom=0)\n    ax.set_title(title, fontsize=15)\n        \n    return\n\n\ndef plot_number_uniq_per(df, df_name, attr1, attr2, ax, title=None, palette=None, scale=\"log\"):\n    \n    uniq_df = df.groupby(attr1)[[attr2]].nunique().reset_index()\n    ax = sns.barplot(x=attr1, y=attr2, data=uniq_df, ax=ax, palette=palette)\n    ax.set_yscale(scale)\n    if scale == \"log\":\n        ax.set_ylim(bottom=0.5)\n    else:\n        ax.set_ylim(bottom=0)\n        \n    counts = uniq_df[attr2]\n    for i, p in enumerate(ax.patches):\n        px = p.get_x()+0.05\n        py = max(p.get_height()*0.9, 1.0)\n        ax.annotate(uniq_df[attr2].iloc[i], (px, py), fontsize=10, rotation=90)\n\n    if title:\n        ax.set_title(title, fontsize=15)\n    else:\n        ax.set_title(f\"{df_name}\\nNumber of unique '{attr2}' values \\nper '{attr1}' value\", fontsize=15)\n    ax.set_xlabel(attr1)\n    ax.set_ylabel(f\"# unique '{attr2}'\")\n    \n    return uniq_df\n\n\ndef plot_number_having_n_uniq(df, df_name, attr1, attr2, ax, order=None, palette=None, scale=\"log\"):\n\n    nuniq = df.groupby(attr1)[attr2].nunique()\n    ax = sns.countplot(x=nuniq, order=order, ax=ax, palette=palette)\n    ax.set_yscale(scale)\n    if scale == \"log\":\n        ax.set_ylim(bottom=0.5)\n    else:\n        ax.set_ylim(bottom=0)\n    ax.set_title(f\"{df_name}\\nNumber of '{attr1}' values \\nhaving N unique values of '{attr2}'\", fontsize=15)\n    ax.set_xlabel(f\"N = # of unique '{attr2}' values\")\n    ax.set_ylabel(f\"# of '{attr1}' having N\")\n\n    return nuniq\n\n\ndef get_img(path, img_id):\n\n    img_path = os.path.join(path, f\"{img_id}.jpg\")\n    im = cv2.imread(img_path)\n    im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)\n    im = cv2.normalize(im, None, 0, 255, cv2.NORM_MINMAX)\n    \n    return im\n\n\ndef image_subplots(images, r, c, axes, titles=None):\n    \n    for idx, img in enumerate(images, 0):\n        i = idx // c\n        j = idx % c\n        if titles:\n            imshow(img, axes[i, j], title=titles[idx])\n        else:\n            imshow(img, axes[i, j])\n\n    return\n        \n\ndef imshow(image, ax=None, title=None, cmap=None):\n\n    if ax is None:\n        fig, ax = plt.subplots()\n    ax.imshow(image, cmap=cmap)\n    ax.spines['top'].set_visible(False)\n    ax.spines['right'].set_visible(False)\n    ax.spines['left'].set_visible(False)\n    ax.spines['bottom'].set_visible(False)\n    ax.tick_params(axis='both', length=0)\n    ax.set_xticklabels('')\n    ax.set_yticklabels('')\n    if title:\n        ax.set_title(title)\n\n    return ax\n","metadata":{"execution":{"iopub.status.busy":"2021-05-30T20:00:43.073608Z","iopub.execute_input":"2021-05-30T20:00:43.073945Z","iopub.status.idle":"2021-05-30T20:00:43.110894Z","shell.execute_reply.started":"2021-05-30T20:00:43.073915Z","shell.execute_reply":"2021-05-30T20:00:43.109925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"----\n### Training Data\n\nThe train dataset comprises 6,334 chest scans in DICOM format, which were de-identified to protect patient privacy. All images were labeled by a panel of experienced radiologists for the presence of opacities as well as overall appearance.\n\nNote that all images are stored in paths with the form `study/series/image`. The study ID here relates directly to the study-level predictions, and the image ID is the ID used for image-level predictions.\n\n----","metadata":{}},{"cell_type":"code","source":"# Read the image metadata\n\ntrain_image_df = pd.read_csv(os.path.join(DATA_PATH,'train_image_level.csv'))\ntrain_study_df =  pd.read_csv(os.path.join(DATA_PATH,'train_study_level.csv'))","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-05-30T20:00:43.112104Z","iopub.execute_input":"2021-05-30T20:00:43.112386Z","iopub.status.idle":"2021-05-30T20:00:43.208718Z","shell.execute_reply.started":"2021-05-30T20:00:43.112358Z","shell.execute_reply":"2021-05-30T20:00:43.207505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"----\n`train_study_level.csv`\n\n- **id** --- unique study identifier  \n- **Negative for Pneumonia** --- 1 if the study is negative for pneumonia, 0 otherwise  \n- **Typical Appearance** --- 1 if the study has this appearance, 0 otherwise  \n- **Indeterminate Appearance**  --- 1 if the study has this appearance, 0 otherwise  \n- **Atypical Appearance**  --- 1 if the study has this appearance, 0 otherwise  \n\n----","metadata":{}},{"cell_type":"code","source":"train_study_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-30T20:00:43.212662Z","iopub.execute_input":"2021-05-30T20:00:43.213151Z","iopub.status.idle":"2021-05-30T20:00:43.245339Z","shell.execute_reply.started":"2021-05-30T20:00:43.213108Z","shell.execute_reply":"2021-05-30T20:00:43.243976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"id_col = train_study_df.loc[:, \"id\"].apply(lambda s: s.split('_')[0])\nother_cols = train_study_df[[\"Negative for Pneumonia\", \"Typical Appearance\", \n                             \"Indeterminate Appearance\", \"Atypical Appearance\"]]\ntrain_study_df = pd.concat([id_col, other_cols], axis=1)\ntrain_study_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-30T20:00:43.247262Z","iopub.execute_input":"2021-05-30T20:00:43.247560Z","iopub.status.idle":"2021-05-30T20:00:43.270519Z","shell.execute_reply.started":"2021-05-30T20:00:43.247530Z","shell.execute_reply":"2021-05-30T20:00:43.269826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"----\n`train_image_level.csv`\n\n- **id** --- unique image identifier  \n- **boxes** --- bounding boxes in easily-readable dictionary format  \n- **label** --- the correct prediction label for the provided bounding boxes  \n----","metadata":{}},{"cell_type":"code","source":"train_image_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-30T20:00:43.271611Z","iopub.execute_input":"2021-05-30T20:00:43.271921Z","iopub.status.idle":"2021-05-30T20:00:43.294363Z","shell.execute_reply.started":"2021-05-30T20:00:43.271892Z","shell.execute_reply":"2021-05-30T20:00:43.293178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get the image and study ID's from the file paths.\n\nimage_paths = glob.glob(os.path.join(TRAIN_PATH, \"*/*/*\"))\n\np = re.compile(os.path.join(TRAIN_PATH, \"([^/]*)/([^/]*)/([^.]*)(.dcm)\"))\n\nstudy_ids = []\nimage_ids = []\nid_dict = defaultdict(list)\nfor ip in image_paths:\n    m = p.match(ip)\n    study_ids.append((m.group(1)))\n    image_ids.append((m.group(3)))\n    id_dict[m.group(1)].append((m.group(3), ip))\n    \nif (len(image_ids) != len(set(image_ids))):\n    print(\"there are duplicate image IDs\")\nelse:\n    print(\"there are no duplicate image IDs\")\n    \nprint(f\"number of image IDs: {len(set(image_ids))}\")\nprint(f\"number of studies: {len(set(study_ids))}\")","metadata":{"execution":{"iopub.status.busy":"2021-05-30T20:00:43.295860Z","iopub.execute_input":"2021-05-30T20:00:43.296467Z","iopub.status.idle":"2021-05-30T20:01:05.664535Z","shell.execute_reply.started":"2021-05-30T20:00:43.296424Z","shell.execute_reply":"2021-05-30T20:01:05.663321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check that the metadata is consistent with the file path derived study and image ID's.\n\nmeta_image_ids = [id.split('_')[0] for id in train_image_df[\"id\"]]\nmeta_study_ids = [id.split('_')[0] for id in train_study_df[\"id\"]]\nmeta_study_instance_uids = list(set(train_image_df[\"StudyInstanceUID\"]))\n\nprint(f\"there are {len(meta_image_ids)} metadata image IDs\")\nprint(f\"there are {len(meta_study_ids)} metadata study IDs\")\nprint(f\"there are {len(meta_study_instance_uids)} metadata StudyInstanceUIDs\")\nprint()\n\nif (len(meta_image_ids) != len(set(meta_image_ids))):\n    print(\"there are duplicate metadata image IDs\")\nif (set(meta_image_ids) != set(image_ids)):\n    print(\"metadata image IDs mismatch file path image IDs\")\n    s1 = set(meta_image_ids)\n    s2 = set(image_ids)\n    print(f\"There are {len(list(s1.difference(s2)))} extra metadata image IDs\")\n    print(f\"There are {len(list(s2.difference(s1)))} extra file path image IDs\")\n    print()\n    \nif (set(meta_study_ids) != set(study_ids)):\n    print(\"metadata study IDs mismatch file path study IDs\")\n    s1 = set(meta_study_ids)\n    s2 = set(study_ids)\n    print(f\"There are {len(list(s1.difference(s2)))} extra metadata study IDs\")\n    print(f\"There are {len(list(s2.difference(s1)))} extra file path study IDs\")\n    print()\n    \nif (set(meta_study_instance_uids) != set(study_ids)):\n    print(\"metadata StudyInstanceUIDs mismatch file path study IDs\")\n    s1 = set(meta_study_instance_uids)\n    s2 = set(study_ids)\n    print(f\"There are {len(list(s1.difference(s2)))} extra metadata study instace UIDs\")\n    print(f\"There are {len(list(s2.difference(s1)))} extra file path study IDs\")\n    print()\n    \nif (set(meta_study_instance_uids) != set(meta_study_ids)):\n    print(\"metadata StudyInstanceUIDs mismatch metadata study IDs\")\n    s1 = set(meta_study_instance_uids)\n    s2 = set(meta_study_ids)\n    print(f\"There are {len(list(s1.difference(s2)))} extra metadata StudyInstanceUIDs\")\n    print(f\"There are {len(list(s2.difference(s1)))} extra metadata study IDs\")\n    print()\n    \n# Check consistency of metadata mapping between studies and images with file mapping\nincon = False\nfor study_id, image_info in id_dict.items():\n    for (image_id, _) in image_info:\n        meta_image_id = image_id + \"_image\"\n        meta_study_id = train_image_df.loc[train_image_df[\"id\"] == meta_image_id, \"StudyInstanceUID\"].iloc[0]\n        if study_id != meta_study_id:\n            incon = True\n            print(\"metadata ID mapping does not match file mapping\")\n            print(image_id, study_id, meta_study_id)\nif (not incon):\n    print(\"study/image ID relationships are consistent between metadata and file paths\")","metadata":{"execution":{"iopub.status.busy":"2021-05-30T20:01:05.666120Z","iopub.execute_input":"2021-05-30T20:01:05.666418Z","iopub.status.idle":"2021-05-30T20:01:13.485004Z","shell.execute_reply.started":"2021-05-30T20:01:05.666388Z","shell.execute_reply":"2021-05-30T20:01:13.483769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get all the field names in the DICOM data.\n\ndicom_field_names = set([])\nfor ip in image_paths:\n    dicom_dataset = pydicom.dcmread(ip)\n    for dfld in dicom_dataset:\n        dicom_field_names.add(dfld.name)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T20:01:13.486473Z","iopub.execute_input":"2021-05-30T20:01:13.486777Z","iopub.status.idle":"2021-05-30T20:17:47.857888Z","shell.execute_reply.started":"2021-05-30T20:01:13.486747Z","shell.execute_reply":"2021-05-30T20:17:47.856400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dicom_field_names = list(dicom_field_names)\npprint.pprint(dicom_field_names)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T20:17:47.859579Z","iopub.execute_input":"2021-05-30T20:17:47.859909Z","iopub.status.idle":"2021-05-30T20:17:47.870541Z","shell.execute_reply.started":"2021-05-30T20:17:47.859878Z","shell.execute_reply":"2021-05-30T20:17:47.869520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Display typical DICOM dataset content.\n\nimage_file_path = image_paths[0]\ndicom_dataset = pydicom.dcmread(image_file_path)\nprint_title(image_file_path)\nfor dfld in dicom_dataset:\n    print(dfld)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T20:17:47.871909Z","iopub.execute_input":"2021-05-30T20:17:47.872287Z","iopub.status.idle":"2021-05-30T20:17:48.015595Z","shell.execute_reply.started":"2021-05-30T20:17:47.872253Z","shell.execute_reply":"2021-05-30T20:17:48.014217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Skip fields not wanted in the DICOM dataframe.\n\nskip_fields = [\"Specific Character Set\", \n               \"De-identification Method\", \n               \"De-identification Method Code Sequence\",\n               \"Pixel Data\"]","metadata":{"execution":{"iopub.status.busy":"2021-05-30T20:17:48.017131Z","iopub.execute_input":"2021-05-30T20:17:48.017473Z","iopub.status.idle":"2021-05-30T20:17:48.022468Z","shell.execute_reply.started":"2021-05-30T20:17:48.017442Z","shell.execute_reply":"2021-05-30T20:17:48.020984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import DICOM data into dataframe.\n\np = re.compile(os.path.join(TRAIN_PATH, \"([^/]*)/([^/]*)/([^.]*)(.dcm)\"))\n\ndata_dict = defaultdict(list)\n\nfor ip in image_paths:\n    m = p.match(ip)\n    study_id = m.group(1)\n    series_id = m.group(2)\n    image_id = m.group(3)\n    dicom_dataset = pydicom.dcmread(ip)\n    file_fields = dict([(dfld.name, dfld) for dfld in dicom_dataset])\n    data_dict[\"study_id\"].append(study_id)\n    data_dict[\"series_id\"].append(series_id)\n    data_dict[\"image_id\"].append(image_id)\n    for dfld_name in dicom_field_names:\n        if (dfld_name in skip_fields):\n            continue\n        dfld = file_fields.get(dfld_name, None)\n        if dfld:\n            data_dict[dfld_name].append(str(dfld.value))\n        else:\n            data_dict[dfld_name].append(np.nan)\n            \ndicom_df = pd.DataFrame(data_dict)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T20:17:48.024054Z","iopub.execute_input":"2021-05-30T20:17:48.024407Z","iopub.status.idle":"2021-05-30T20:34:22.301076Z","shell.execute_reply.started":"2021-05-30T20:17:48.024375Z","shell.execute_reply":"2021-05-30T20:34:22.300006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"there are {len(dicom_df)} DICOM rows\")\ndicom_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-30T20:34:22.302836Z","iopub.execute_input":"2021-05-30T20:34:22.303283Z","iopub.status.idle":"2021-05-30T20:34:22.333545Z","shell.execute_reply.started":"2021-05-30T20:34:22.303237Z","shell.execute_reply":"2021-05-30T20:34:22.332190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# DICOM attribute value summary\n\ndicom_df.describe().transpose()","metadata":{"execution":{"iopub.status.busy":"2021-05-30T20:34:22.335340Z","iopub.execute_input":"2021-05-30T20:34:22.335713Z","iopub.status.idle":"2021-05-30T20:34:22.583281Z","shell.execute_reply.started":"2021-05-30T20:34:22.335680Z","shell.execute_reply":"2021-05-30T20:34:22.582168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Test consistency of ID fields\n\nprint(set(dicom_df.loc[:, \"study_id\"] == dicom_df.loc[:, \"Study Instance UID\"]))\nprint(set(dicom_df.loc[:, \"series_id\"] == dicom_df.loc[:, \"Series Instance UID\"]))\nprint(set(dicom_df.loc[:, \"image_id\"] == dicom_df.loc[:, \"SOP Instance UID\"]))","metadata":{"execution":{"iopub.status.busy":"2021-05-30T20:34:22.584927Z","iopub.execute_input":"2021-05-30T20:34:22.585377Z","iopub.status.idle":"2021-05-30T20:34:22.605071Z","shell.execute_reply.started":"2021-05-30T20:34:22.585334Z","shell.execute_reply":"2021-05-30T20:34:22.603353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Display info on NA values\n\ndicom_df.isna().describe().transpose()","metadata":{"execution":{"iopub.status.busy":"2021-05-30T20:34:22.607531Z","iopub.execute_input":"2021-05-30T20:34:22.608145Z","iopub.status.idle":"2021-05-30T20:34:22.709756Z","shell.execute_reply.started":"2021-05-30T20:34:22.608077Z","shell.execute_reply":"2021-05-30T20:34:22.709017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Display the unique values and their counts for fields with <100 unique values\n\nunique_col_vals = {}\nfor col in dicom_df.columns:\n    colcount = Counter(list(dicom_df.loc[:, col]))\n    if len(colcount) < 100:\n        unique_col_vals[col] = colcount\n        print_title(f\"{col}:\")\n        for val, count in sorted(colcount.items(), key=lambda item: item[1], reverse=True):\n            print(f\"\\t{count}:\\t{val}\")\n        print()","metadata":{"execution":{"iopub.status.busy":"2021-05-30T20:34:22.711076Z","iopub.execute_input":"2021-05-30T20:34:22.711382Z","iopub.status.idle":"2021-05-30T20:34:22.842099Z","shell.execute_reply.started":"2021-05-30T20:34:22.711353Z","shell.execute_reply":"2021-05-30T20:34:22.840717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Show some key relationships among DICOM features\n\nfig, ax = plt.subplots(8, 2, figsize=(15, 25))\n\n_ = plot_number_having_n_uniq(dicom_df, \"\", \"Patient ID\", \"Patient's Sex\", ax[0, 0], palette=\"Set3\")\n_ = plot_number_having_n_uniq(dicom_df, \"\", \"Patient ID\", \"Patient's Name\", ax[0, 1], palette=\"Set3\")\n_ = plot_number_having_n_uniq(dicom_df, \"\", \"Patient ID\", \"study_id\", ax[1, 0], palette=\"Set3\")\n_ = plot_number_having_n_uniq(dicom_df, \"\", \"study_id\", \"Study Date\", ax[1, 1], palette=\"Set3\")\n_ = plot_number_having_n_uniq(dicom_df, \"\", \"study_id\", \"series_id\", ax[2, 0], palette=\"Set3\")\n_ = plot_number_having_n_uniq(dicom_df, \"\", \"Patient ID\", \"series_id\", ax[2, 1], palette=\"Set3\")\n_ = plot_number_having_n_uniq(dicom_df, \"\", \"series_id\", \"image_id\", ax[3, 0], palette=\"Set3\")\n_ = plot_number_having_n_uniq(dicom_df, \"\", \"study_id\", \"image_id\", ax[3, 1], palette=\"Set3\")\n_ = plot_number_having_n_uniq(dicom_df, \"\", \"Patient ID\", \"image_id\", ax[4, 0], palette=\"Set3\")\n_ = plot_number_having_n_uniq(dicom_df, \"\", \"study_id\", \"Patient ID\", ax[4, 1], palette=\"Set3\")\n_ = plot_number_having_n_uniq(dicom_df, \"\", \"study_id\", \"Imager Pixel Spacing\", ax[5, 0], palette=\"Set3\")\n_ = plot_number_having_n_uniq(dicom_df, \"\", \"study_id\", \"Private Creator\", ax[5, 1], palette=\"Set3\")\n_ = plot_number_having_n_uniq(dicom_df, \"\", \"study_id\", \"Image Type\", ax[6, 0], palette=\"Set3\")\n_ = plot_number_having_n_uniq(dicom_df, \"\", \"study_id\", \"Photometric Interpretation\", ax[6, 1], palette=\"Set3\")\n_ = plot_number_having_n_uniq(dicom_df, \"\", \"study_id\", \"Modality\", ax[7, 0], palette=\"Set3\")\n_ = plot_number_having_n_uniq(dicom_df, \"\", \"study_id\", \"Body Part Examined\", ax[7, 1], palette=\"Set3\")\n\nfig.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2021-05-30T20:34:22.846447Z","iopub.execute_input":"2021-05-30T20:34:22.846783Z","iopub.status.idle":"2021-05-30T20:34:31.435008Z","shell.execute_reply.started":"2021-05-30T20:34:22.846752Z","shell.execute_reply":"2021-05-30T20:34:31.434057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Join the image and study metadata on study ID\n\nimage_study_df = train_image_df.merge(train_study_df, left_on=\"StudyInstanceUID\", right_on=\"id\")\nimage_study_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-30T20:34:31.436841Z","iopub.execute_input":"2021-05-30T20:34:31.437311Z","iopub.status.idle":"2021-05-30T20:34:31.469008Z","shell.execute_reply.started":"2021-05-30T20:34:31.437276Z","shell.execute_reply":"2021-05-30T20:34:31.468182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check that each study has one and only one determination.\n\nidx1 = image_study_df[image_study_df[\"Negative for Pneumonia\"] + image_study_df[\"Typical Appearance\"] +  \n                image_study_df[\"Indeterminate Appearance\"] + image_study_df[\"Atypical Appearance\"] > 1].index\nif (len(idx1) > 1): \n    print(f\"{len(idx1)} studies have multiple determinations\")\nelse:\n    print(\"No studies have multiple determinations\")\n\nidx2 = image_study_df[image_study_df[\"Negative for Pneumonia\"] + image_study_df[\"Typical Appearance\"] +  \n                image_study_df[\"Indeterminate Appearance\"] + image_study_df[\"Atypical Appearance\"] < 1].index\nif (len(idx2) > 1): \n    print(f\"{len(idx2)} studies have zero determinations\")\nelse:\n    print(\"No studies have zero determinations\")","metadata":{"execution":{"iopub.status.busy":"2021-05-30T20:34:31.470195Z","iopub.execute_input":"2021-05-30T20:34:31.470620Z","iopub.status.idle":"2021-05-30T20:34:31.486585Z","shell.execute_reply.started":"2021-05-30T20:34:31.470589Z","shell.execute_reply":"2021-05-30T20:34:31.485047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Combine image and study metadata and massage the data a bit\n\nlabel_re = re.compile(\"(opacity 1(\\s(\\-){0,1}\\d*(\\.){0,1}\\d*){4})\")\n\ndef reformat_label(old_label):\n    new_label =[op[0].split() for op in label_re.findall(old_label)]\n    return new_label\n\ndef reformat_boxes(old_boxes):\n    new_boxes = [(box[\"x\"], box[\"y\"], box[\"width\"], box[\"height\"]) for box in eval(old_boxes)]\n    return new_boxes\n\ndef determ_string(s):\n    if s[0] == 1: return \"NegPnu\"\n    if s[1] == 1: return \"Typ\"\n    if s[2] == 1: return \"Indeterm\"\n    if s[3] == 1: return \"ATyp\"\n    return \"none\"\n\nid_col = image_study_df[\"id_x\"].apply(lambda s: s.split('_')[0]).rename(\"image_id\")\nbox_col = image_study_df[\"boxes\"].fillna('[]').rename(\"new_boxes\").apply(reformat_boxes)\nbox_count_col = image_study_df[\"boxes\"].fillna('[]').apply(lambda s: len(eval(s))).rename(\"box_count\")\nnew_label_col = image_study_df[\"label\"].fillna('[]').apply(reformat_label).rename(\"new_label\")\nlabel_count_col = new_label_col.apply(lambda s: len(s)).rename(\"label_count\")\ndeterm_col = image_study_df[[\"Negative for Pneumonia\", \n                             \"Typical Appearance\",\n                             \"Indeterminate Appearance\",\n                             \"Atypical Appearance\"]].apply(determ_string, axis=1).rename(\"determ\")\n\nother_cols = image_study_df[[\"boxes\", \"label\", \"StudyInstanceUID\", \"Negative for Pneumonia\", \n                             \"Typical Appearance\", \"Indeterminate Appearance\", \"Atypical Appearance\"]]\nimage_study_df = pd.concat([id_col, box_col, box_count_col, new_label_col, label_count_col, determ_col, \n                            other_cols], axis=1)\n\nimage_study_df = image_study_df[[\"image_id\", \"boxes\", \"new_boxes\", \"box_count\", \"label\", \"new_label\", \n                                 \"label_count\", \"determ\", \"StudyInstanceUID\", \n                                \"Negative for Pneumonia\", \"Typical Appearance\", \n                                \"Indeterminate Appearance\", \"Atypical Appearance\"]]\n\nimage_study_df = image_study_df.rename(columns={\"StudyInstanceUID\": \"study_id\"})\nimage_study_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-30T20:34:31.488654Z","iopub.execute_input":"2021-05-30T20:34:31.489077Z","iopub.status.idle":"2021-05-30T20:34:32.191238Z","shell.execute_reply.started":"2021-05-30T20:34:31.489038Z","shell.execute_reply":"2021-05-30T20:34:32.189991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check that each image has the same number of boxes and labels.\n\nidx = image_study_df[image_study_df[\"box_count\"] != image_study_df[\"label_count\"]].index\n        \nif (len(idx) > 0): \n    print(f\"{len(idx)} box/label mismatch found\")\nelse:\n    print(\"No studies have box/label mismatch\")","metadata":{"execution":{"iopub.status.busy":"2021-05-30T20:34:32.192830Z","iopub.execute_input":"2021-05-30T20:34:32.193166Z","iopub.status.idle":"2021-05-30T20:34:32.201221Z","shell.execute_reply.started":"2021-05-30T20:34:32.193135Z","shell.execute_reply":"2021-05-30T20:34:32.199918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check that the label regex we used hasn't thown out any labels (and so our regex accurately capture the label syntax)    \n\njoin_label = image_study_df[\"new_label\"].rename(\"join_label\").apply(lambda ll: ' '.join([' '.join(l) for l in ll]))\ntemp_df = pd.concat([image_study_df[\"label\"], join_label], axis=1)\nidx = temp_df[~((temp_df[\"label\"] == temp_df[\"join_label\"]) | \n              ((temp_df[\"label\"] == \"none 1 0 0 1 1\") & (temp_df[\"join_label\"] == \"\")))].index\n\nif (len(idx) > 0): \n    print(f\"{len(idx)} missing labels found\")\nelse:\n    print(\"No missing labels\")","metadata":{"execution":{"iopub.status.busy":"2021-05-30T20:34:32.203223Z","iopub.execute_input":"2021-05-30T20:34:32.203701Z","iopub.status.idle":"2021-05-30T20:34:32.232079Z","shell.execute_reply.started":"2021-05-30T20:34:32.203655Z","shell.execute_reply":"2021-05-30T20:34:32.231050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Remove unneeded columns from this dataframe\n\nimage_study_df = image_study_df[[\"image_id\", \"new_boxes\", \"box_count\",\n                                 \"determ\", \"study_id\"]].rename(columns={\"new_boxes\" : \"boxes\"})\nimage_study_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-30T20:34:32.233660Z","iopub.execute_input":"2021-05-30T20:34:32.233999Z","iopub.status.idle":"2021-05-30T20:34:32.263920Z","shell.execute_reply.started":"2021-05-30T20:34:32.233967Z","shell.execute_reply":"2021-05-30T20:34:32.262711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Join in some patient and image features\n\njoin_in_df = dicom_df[[\"image_id\", \"Patient ID\", \"Patient's Sex\", \"Patient's Name\", \n                        \"Body Part Examined\", \"Rows\", \"Columns\"]]\n    \nimage_study_patient_df = image_study_df.merge(join_in_df, on=\"image_id\")\n\nimage_study_patient_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-30T20:34:32.265233Z","iopub.execute_input":"2021-05-30T20:34:32.265525Z","iopub.status.idle":"2021-05-30T20:34:32.305865Z","shell.execute_reply.started":"2021-05-30T20:34:32.265498Z","shell.execute_reply":"2021-05-30T20:34:32.304550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the number of images, number of studies per determination\n\nfig, ax = plt.subplots(1, 2, figsize=(15, 5))\n\n_ = plot_number_uniq_per(image_study_patient_df, \"\", \"determ\", \"image_id\", ax[0], palette=\"Set3\", scale=\"linear\")\n_ = plot_number_uniq_per(image_study_patient_df, \"\", \"determ\", \"study_id\", ax[1], palette=\"Set3\", scale=\"linear\")\n\nfig.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2021-05-30T20:34:32.307214Z","iopub.execute_input":"2021-05-30T20:34:32.307505Z","iopub.status.idle":"2021-05-30T20:34:32.760433Z","shell.execute_reply.started":"2021-05-30T20:34:32.307479Z","shell.execute_reply":"2021-05-30T20:34:32.759275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Within each determination, how many images have a given number of boxes.\n\nfig, ax = plt.subplots(2, 2, figsize=(15, 8))\nfor i, determ in enumerate([\"Typ\", \"NegPnu\", \"ATyp\", \"Indeterm\"]):\n    df = image_study_patient_df[image_study_patient_df[\"determ\"] == determ]\n    plot_categories(df, \"box_count\", range(0, 5), f\"{determ} # images with box_count\", ax[i%2, i//2], \\\n                    ylabel=\"image count\", scale=\"linear\")\nfig.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2021-05-30T20:34:32.761799Z","iopub.execute_input":"2021-05-30T20:34:32.762134Z","iopub.status.idle":"2021-05-30T20:34:33.568395Z","shell.execute_reply.started":"2021-05-30T20:34:32.762106Z","shell.execute_reply":"2021-05-30T20:34:33.567190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Number of patients having a given number of determinations.\n# Number of studies having a given number of determinations.\n\nfig, ax = plt.subplots(1, 2, figsize=(15, 5))\n\n_ = plot_number_having_n_uniq(image_study_patient_df, \"\", \"Patient ID\", \"determ\", ax[0], \n                              palette=\"Set3\", scale=\"linear\")\n_ = plot_number_having_n_uniq(image_study_patient_df, \"\", \"study_id\", \"determ\", ax[1], \n                              palette=\"Set3\", scale=\"linear\")\nfig.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2021-05-30T20:34:33.570061Z","iopub.execute_input":"2021-05-30T20:34:33.570404Z","iopub.status.idle":"2021-05-30T20:34:33.957694Z","shell.execute_reply.started":"2021-05-30T20:34:33.570372Z","shell.execute_reply":"2021-05-30T20:34:33.956581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Distributions of determinations by gender. Number of studies per gender.\n\nfig, ax = plt.subplots(3, 1, figsize=(8, 10))\nfor i, sex in enumerate([\"M\", \"F\"]):\n    df = image_study_patient_df[image_study_patient_df[\"Patient's Sex\"] == sex]\n    \n    _ = plot_number_uniq_per(df, \"\", \"determ\", \"study_id\", ax[i], \n                         title=f\"Patient's Sex={sex}:\\n # studies with given determination\", palette=\"Set3\", \n                         scale=\"linear\")\n\n_ = plot_number_uniq_per(image_study_patient_df, \"\", \"Patient's Sex\", \"study_id\", ax[2], \n                         title=\"# studies per gender\", palette=\"Set3\", scale=\"linear\")    \n\nfig.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2021-05-30T20:34:33.959048Z","iopub.execute_input":"2021-05-30T20:34:33.959361Z","iopub.status.idle":"2021-05-30T20:34:34.531601Z","shell.execute_reply.started":"2021-05-30T20:34:33.959330Z","shell.execute_reply":"2021-05-30T20:34:34.530842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Explode the box lists into seperate rows\n\nimage_study_patient_boxes_df = explode_list_col(image_study_patient_df, \"boxes\")\nimage_study_patient_boxes_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-30T20:34:34.533052Z","iopub.execute_input":"2021-05-30T20:34:34.533701Z","iopub.status.idle":"2021-05-30T20:34:36.517641Z","shell.execute_reply.started":"2021-05-30T20:34:34.533652Z","shell.execute_reply":"2021-05-30T20:34:36.516582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check that all boxes in the metadata are valid\n\ndef valid_boxes(s):\n    (x1, y1, w, h) = s[0]\n    x2 = x1 + w\n    y2 = y1 + h\n    r = float(s[1])\n    c = float(s[2])\n    invalid = not((w > 0) and (h > 0) and\n    (x1 >= 0) and (x1 <= c) and (y1 >= 0) and (y1 <= r) and\n    (x2 >= 0) and (x2 <= c) and (y2 >= 0) and (y2 <= r))\n    return invalid\n    \nimage_study_patient_boxes_df[\n    image_study_patient_boxes_df[[\"boxes\", \"Rows\", \"Columns\"]].apply(valid_boxes, axis=1)]","metadata":{"execution":{"iopub.status.busy":"2021-05-30T20:34:36.519080Z","iopub.execute_input":"2021-05-30T20:34:36.519379Z","iopub.status.idle":"2021-05-30T20:34:36.677760Z","shell.execute_reply.started":"2021-05-30T20:34:36.519349Z","shell.execute_reply":"2021-05-30T20:34:36.677011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot the image sizes\n\nfig, ax = plt.subplots(1, 1, figsize=(15, 15))\nax.grid(True)\n\nsize_df = dicom_df.loc[:, [\"Rows\", \"Columns\"]].astype({\"Rows\": \"int\", \"Columns\": \"int\"})\n\nsns.scatterplot(x=\"Rows\", y=\"Columns\", palette=\"pastel\", data=size_df, ax=ax)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T20:34:36.678850Z","iopub.execute_input":"2021-05-30T20:34:36.679267Z","iopub.status.idle":"2021-05-30T20:34:36.983897Z","shell.execute_reply.started":"2021-05-30T20:34:36.679236Z","shell.execute_reply":"2021-05-30T20:34:36.982806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to display images for a patient\n\ndef display_patient_images(patient_id, study_ids, image_boxes, study_determ):\n    \n    image_paths = []\n    for study_id in study_ids:\n        image_paths += glob.glob(os.path.join(TRAIN_PATH, f\"{study_id}/*/*\"))\n        \n    p = re.compile(os.path.join(TRAIN_PATH, \"([^/]*)/([^/]*)/([^/]*)\"))\n\n    image_data = []\n    for ip in image_paths:\n        m = p.match(ip)\n        study_id = m.group(1)\n        series_id = m.group(2)\n        image_id = m.group(3).split('.')[0]\n        image_data.append((study_id, series_id, image_id, ip, image_boxes[image_id], study_determ[study_id]))\n\n    num_images = len(image_data)\n    nrows = int(math.ceil(num_images/2))\n    fig, axs = plt.subplots(nrows, 2, figsize=(15, nrows*8), squeeze=False)\n    clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8,8))\n    i = 0\n    for (study_id, series_id, image_id, image_path, boxes, determ) in image_data:\n        dicom_dataset = pydicom.dcmread(image_path)\n        image = apply_voi_lut(dicom_dataset.pixel_array, dicom_dataset)\n        pr = dicom_dataset.data_element('PhotometricInterpretation').value\n        if pr == \"MONOCHROME1\":\n            image = 255.0 - image\n        image = image - np.min(image)\n        image = (image/np.max(image)) * 255.0\n        image = image.astype('uint8')\n        \n        rows = dicom_dataset.data_element('Rows').value\n        cols = dicom_dataset.data_element('Columns').value\n        \n        cl1 = clahe.apply(image)\n\n        title = f\"Patient {patient_id}\\nStudy {study_id}\\nDetermination {determ}\\n\" + \\\n                f\"Study Date:{dicom_dataset.data_element('StudyDate').value}\"\n        ax = axs[int((i/2)), i%2]\n        ax.set_title(title)\n        imshow(cl1, cmap='gray', ax=ax)\n        for (x, y, w, h) in boxes:\n            ax.add_patch(patches.Rectangle(xy=(x, y), width=w, height=h, linewidth=1, color='red', fill=False))\n        ax.add_patch(patches.Rectangle(xy=(0, 0), width=cols, height=rows, linewidth=4, color='blue', fill=False))\n        i += 1\n    if (num_images % 2 != 0):\n        axs[int((i - 1)/2), 1].set_visible(False)\n            \n    fig.tight_layout()\n","metadata":{"execution":{"iopub.status.busy":"2021-05-30T20:34:36.985463Z","iopub.execute_input":"2021-05-30T20:34:36.985810Z","iopub.status.idle":"2021-05-30T20:34:37.002782Z","shell.execute_reply.started":"2021-05-30T20:34:36.985777Z","shell.execute_reply":"2021-05-30T20:34:37.001586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Explore patient image sets. Rerun cell to sample patients.\n\npatient_id = image_study_patient_df.iloc[random.randrange(len(image_study_patient_df))][\"Patient ID\"]\nprint(f\"Patient ID = {patient_id}\")\n\nstudy_image_boxes = image_study_patient_df[\n            image_study_patient_df[\"Patient ID\"] == patient_id][[\"study_id\", \"image_id\", \"determ\", \"boxes\"]]\n\nstudy_ids = study_image_boxes[\"study_id\"].unique()\nimage_boxes = [(r[1][\"image_id\"], r[1][\"boxes\"]) for r in study_image_boxes.iterrows()]\nimage_boxes = dict(image_boxes)\nstudy_determ = [(r[1][\"study_id\"], r[1][\"determ\"]) for r in study_image_boxes.iterrows()]\nstudy_determ = dict(study_determ)\n\n\nprint(f\"Number of studies: {len(study_ids)}\")\nprint(f\"Number of images: {len(image_boxes)}\")\n    \ndisplay_patient_images(patient_id, study_ids, image_boxes, study_determ)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T20:34:37.004372Z","iopub.execute_input":"2021-05-30T20:34:37.004687Z","iopub.status.idle":"2021-05-30T20:35:06.234675Z","shell.execute_reply.started":"2021-05-30T20:34:37.004656Z","shell.execute_reply":"2021-05-30T20:35:06.232667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get the studies with multiple images\n\nuniq_df = image_study_patient_df.groupby([\"Patient ID\", \"study_id\"])[[\"image_id\"]].nunique().reset_index()\nmulti_image_studies_df = uniq_df[uniq_df[\"image_id\"] > 1]\nmulti_image_studies_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-30T20:35:06.236730Z","iopub.execute_input":"2021-05-30T20:35:06.237137Z","iopub.status.idle":"2021-05-30T20:35:06.280259Z","shell.execute_reply.started":"2021-05-30T20:35:06.237098Z","shell.execute_reply":"2021-05-30T20:35:06.279075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Explore studies that have multiple images. Rerun the cell to sample studies with multiple images.\n\n(patient_id, study_id) = multi_image_studies_df.iloc[random.randrange(len(multi_image_studies_df))][[\"Patient ID\", \"study_id\"]]\nprint(f\"Patient ID = {patient_id}\")\nprint(f\"study_id = {study_id}\")\n\nstudy_image_boxes = image_study_patient_df[(image_study_patient_df[\"Patient ID\"] == patient_id) &\n            (image_study_patient_df[\"study_id\"] == study_id)][[\"study_id\", \"image_id\", \"determ\", \"boxes\"]]\n\nstudy_ids = study_image_boxes[\"study_id\"].unique()\nimage_boxes = [(r[1][\"image_id\"], r[1][\"boxes\"]) for r in study_image_boxes.iterrows()]\nimage_boxes = dict(image_boxes)\nstudy_determ = [(r[1][\"study_id\"], r[1][\"determ\"]) for r in study_image_boxes.iterrows()]\nstudy_determ = dict(study_determ)\n\n\nprint(f\"Number of studies: {len(study_ids)}\")\nprint(f\"Number of images: {len(image_boxes)}\")\n    \ndisplay_patient_images(patient_id, study_ids, image_boxes, study_determ)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T20:35:06.281794Z","iopub.execute_input":"2021-05-30T20:35:06.282132Z","iopub.status.idle":"2021-05-30T20:35:09.439750Z","shell.execute_reply.started":"2021-05-30T20:35:06.282101Z","shell.execute_reply":"2021-05-30T20:35:09.438851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Determine whether studies have the expected number of images with boxes.\n\nstudy_image_count_df = image_study_df[[\"study_id\", \"determ\", \"image_id\", \"box_count\"]]\nstudy_boxes = defaultdict(list)\nstudy_determ = {}\nfor r in study_image_count_df.iterrows():\n    (study_id, determ, image_id, box_count) = (r[1][\"study_id\"], r[1][\"determ\"], r[1][\"image_id\"], r[1][\"box_count\"])\n    study_boxes[study_id].append((image_id, box_count))\n    study_determ[study_id] = determ\nstudy_box_count = {}\nfor study_id in study_boxes.keys():\n    nonzero_count = sum(int(bc > 0) for (_, bc) in study_boxes[study_id])\n    image_count = len(study_boxes[study_id])\n    study_box_count[study_id] = (image_count, nonzero_count)\n\nnegpnu = []\nnegpnu_nonzero_boxes = []\n\ntia = []\ntia_one_img = []\ntia_one_img_no_boxes = []\n\ntia_mult_img = []\ntia_mult_img_no_boxes = []\ntia_mult_img_one_with_boxes = []\ntia_mult_img_mult_with_boxes = []\ntia_mult_img_all_with_boxes = []\n\nfor study_id in study_box_count.keys():\n    if (study_determ[study_id] == \"NegPnu\"):\n        negpnu.append((study_id, study_box_count[study_id]))\n        if (study_box_count[study_id][1] > 0):\n            negpnu_nonzero_boxes.append((study_id, study_box_count[study_id]))\n    else:\n        tia.append((study_id, study_box_count[study_id]))\n        if (study_box_count[study_id][0] > 1):\n            tia_mult_img.append((study_id, study_box_count[study_id]))\n            if (study_box_count[study_id][1] == 0):\n                tia_mult_img_no_boxes.append((study_id, study_box_count[study_id]))\n            elif (study_box_count[study_id][1] == 1):\n                tia_mult_img_one_with_boxes.append((study_id, study_box_count[study_id]))\n            else:\n                tia_mult_img_mult_with_boxes.append((study_id, study_box_count[study_id]))\n            if (study_box_count[study_id][0] == study_box_count[study_id][1]):\n                tia_mult_img_all_with_boxes.append((study_id, study_box_count[study_id]))\n        else:\n            tia_one_img.append((study_id, study_box_count[study_id]))\n            if (study_box_count[study_id][1] == 0):\n                tia_one_img_no_boxes.append((study_id, study_box_count[study_id]))\n\nprint(f\"There are {len(study_boxes.keys())} studies total (in the metadata)\")\nprint()\nprint(f\"There are {len(negpnu)} NegPnu studies. Of these:\")\nprint(f\"    {len(negpnu_nonzero_boxes)} have images with boxes\")\nprint()\nprint(f\"There are {len(tia)} Typ, Indeterm, or ATyp studies. Of these:\")\nprint(f\"    {len(tia_one_img)} Type, Indeterm, or AType studies have exactly one image. Of these:\")\nprint(f\"        {len(tia_one_img_no_boxes)} have no boxes in the image\")\nprint()\nprint(f\"    {len(tia_mult_img)} Typ, Indeterm, or ATyp studies have multiple images. Of these:\")\nprint(f\"        {len(tia_mult_img_no_boxes)} have no boxes in any image\")\nprint(f\"        {len(tia_mult_img_one_with_boxes)} have exactly one image with boxes\")\nprint(f\"        {len(tia_mult_img_mult_with_boxes)} have more than one image with boxes\")\nprint(f\"        {len(tia_mult_img_all_with_boxes)} have all images with boxes\")\n","metadata":{"execution":{"iopub.status.busy":"2021-05-30T20:35:09.441136Z","iopub.execute_input":"2021-05-30T20:35:09.441629Z","iopub.status.idle":"2021-05-30T20:35:10.118433Z","shell.execute_reply.started":"2021-05-30T20:35:09.441576Z","shell.execute_reply":"2021-05-30T20:35:10.117196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We're going to explore possible image duplication using image hashes.\n# Create an image hash dataframe.\n\nclahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8,8))\nimage_hashes = []\n\nfor row in dicom_df[[\"study_id\", \"series_id\", \"image_id\", \"Patient ID\"]].iterrows():\n    (study_id, series_id, image_id, patient_id) = (row[1][:])\n    image_path = os.path.join(TRAIN_PATH, f\"{study_id}/{series_id}/{image_id}.dcm\")\n    dicom_dataset = pydicom.dcmread(image_path)\n    image = apply_voi_lut(dicom_dataset.pixel_array, dicom_dataset)\n    pi = dicom_dataset.data_element('PhotometricInterpretation').value\n    if pi == \"MONOCHROME1\":\n        image = 255.0 - image\n    image = image - np.min(image)\n    image = (image/np.max(image)) * 255.0\n    image = image.astype('uint8')\n    image = clahe.apply(image)\n    image_hash = imagehash.phash(im.fromarray(image))\n    row = {\"patient_id\":patient_id, \"study_id\":study_id, \"series_id\":series_id, \"image_id\":image_id, \"image_hash\":image_hash}\n    image_hashes.append(row)\n\nimage_hashes_df = pd.DataFrame(image_hashes, columns=[\"patient_id\", \"study_id\", \"series_id\", \"image_id\", \"image_hash\"])\nimage_hashes_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-30T20:36:45.826338Z","iopub.execute_input":"2021-05-30T20:36:45.827053Z","iopub.status.idle":"2021-05-30T21:18:43.661466Z","shell.execute_reply.started":"2021-05-30T20:36:45.827002Z","shell.execute_reply":"2021-05-30T21:18:43.660173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# How many patients have duplicate images.\n\nimage_count_df = image_hashes_df.groupby([\"patient_id\"])[[\"image_id\"]].nunique().reset_index().rename(columns={\"image_id\": \"image_count\"})\nmulti_image_df = image_count_df[image_count_df[\"image_count\"] > 1]\nprint(f\"{len(multi_image_df)} patients have multiple images\")\n\nhash_count_df = image_hashes_df.groupby([\"patient_id\"])[[\"image_hash\"]].nunique().reset_index().rename(columns={\"image_hash\": \"image_hash_count\"})\n\nmulti_image_hash_df = multi_image_df.merge(hash_count_df, on=\"patient_id\")\ndupl_image_df = multi_image_hash_df[multi_image_hash_df[\"image_hash_count\"] < multi_image_hash_df[\"image_count\"]]\n\nprint(f\"{len(dupl_image_df)} patients have duplicate images\")\ndupl_image_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-30T21:19:38.305489Z","iopub.execute_input":"2021-05-30T21:19:38.305865Z","iopub.status.idle":"2021-05-30T21:19:39.880976Z","shell.execute_reply.started":"2021-05-30T21:19:38.305835Z","shell.execute_reply":"2021-05-30T21:19:39.879771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# See whether any images are duplicated across patients or studies.\n\nimage_hashes_df['image_hash'] = image_hashes_df['image_hash'].astype(str)\npatients_per_hash_df = \\\n    image_hashes_df.groupby([\"image_hash\"])[[\"patient_id\"]].nunique()\\\n        .reset_index().rename(columns={\"patient_id\": \"patient_count\"})\n\nprint(f\"{len(patients_per_hash_df[patients_per_hash_df['patient_count'] > 1])} image hashes appear in multiple patients\")\nprint()\n\nstudies_per_hash_df = \\\n    image_hashes_df.groupby([\"image_hash\"])[[\"study_id\"]].nunique()\\\n        .reset_index().rename(columns={\"study_id\": \"study_count\"})\n\nprint(f\"{len(studies_per_hash_df[studies_per_hash_df['study_count'] > 1])} image hashes appear in multiple studies\")\nstudies_per_hash_df[studies_per_hash_df['study_count'] > 1]","metadata":{"execution":{"iopub.status.busy":"2021-05-30T21:20:08.968586Z","iopub.execute_input":"2021-05-30T21:20:08.969019Z","iopub.status.idle":"2021-05-30T21:20:09.616310Z","shell.execute_reply.started":"2021-05-30T21:20:08.968984Z","shell.execute_reply":"2021-05-30T21:20:09.615317Z"},"trusted":true},"execution_count":null,"outputs":[]}]}