{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"

SIIM – COVID19 Detection

\n\n

Study Level Image Classification K-Fold TFRecords

\n
CREATED BY: DARIEN SCHETTLER

\n\n
\n\n
💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠
\n
🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫
\n
💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠
\n
🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪🔬🩺💊🧬🦠🧫🧪
\n
🩺🩺🩺🩺🦠🦠🦠🩺🩺🩺🩺
\n
🩺🩺🩺
\n\n
\n","metadata":{"papermill":{"duration":0.044805,"end_time":"2021-02-05T02:17:44.538251","exception":false,"start_time":"2021-02-05T02:17:44.493446","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"
\n\n

TABLE OF CONTENTS

\n\n---\n\n

0    IMPORTS

\n\n---\n\n

1    BACKGROUND INFORMATION

\n\n---\n\n

2    SETUP

\n\n---\n\n

3    HELPER FUNCTIONS

\n\n---\n\n

4    TABULAR DATA

\n\n---\n\n

5    IMAGE DATA

\n\n---\n\n

6    IDENTIFY DUPLICATES

\n\n---\n\n
","metadata":{"papermill":{"duration":0.043006,"end_time":"2021-02-05T02:17:44.624815","exception":false,"start_time":"2021-02-05T02:17:44.581809","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"
\n\n\n\n

0  IMPORTS

","metadata":{"papermill":{"duration":0.04267,"end_time":"2021-02-05T02:17:44.71043","exception":false,"start_time":"2021-02-05T02:17:44.66776","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Installs\n!cp /kaggle/input/gdcm-conda-install/gdcm.tar .\n!tar -xvzf gdcm.tar\n!conda install --offline ./gdcm/gdcm-2.8.9-py37h71b2a6d_0.tar.bz2\n!rm -rf ./gdcm.tar\n\nprint(\"\\n... IMPORTS STARTING ...\\n\")\nprint(\"\\n\\tVERSION INFORMATION\")\n# Machine Learning and Data Science Imports\nimport tensorflow as tf; print(f\"\\t\\t– TENSORFLOW VERSION: {tf.__version__}\");\nimport tensorflow_addons as tfa; print(f\"\\t\\t– TENSORFLOW ADDONS VERSION: {tfa.__version__}\");\nimport pandas as pd; pd.options.mode.chained_assignment = None;\nimport numpy as np; print(f\"\\t\\t– NUMPY VERSION: {np.__version__}\");\nfrom sklearn.model_selection import GroupKFold\n\nimport pydicom\nfrom pydicom import dcmread\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\n\n# Built In Imports\nfrom kaggle_datasets import KaggleDatasets\nfrom collections import Counter\nfrom datetime import datetime\nfrom glob import glob\nimport warnings\nimport requests\nimport imageio\nimport IPython\nimport urllib\nimport zipfile\nimport pickle\nimport random\nimport shutil\nimport string\nimport math\nimport time\nimport gzip\nimport ast\nimport sys\nimport io\nimport os\nimport gc\nimport re\n\n# Visualization Imports\nfrom matplotlib.colors import ListedColormap\nimport matplotlib.patches as patches\nimport plotly.graph_objects as go\nimport matplotlib.pyplot as plt\nfrom tqdm.notebook import tqdm; tqdm.pandas();\nimport plotly.express as px\nimport seaborn as sns\nfrom PIL import Image\nimport matplotlib; print(f\"\\t\\t– MATPLOTLIB VERSION: {matplotlib.__version__}\");\nimport plotly\nimport PIL\nimport cv2\n \nprint(\"\\n\\n... IMPORTS COMPLETE ...\\n\")\n\n# To give access to automl files\nsys.path.append(\"/kaggle/input/automl-efficientdet-efficientnetv2/automl\")\nsys.path.append(\"/kaggle/input/automl-efficientdet-efficientnetv2/automl/brain_automl\")\nsys.path.append(\"/kaggle/input/automl-efficientdet-efficientnetv2/automl/brain_automl/efficientdet\")\nsys.path.append(\"/kaggle/input/automl-efficientdet-efficientnetv2/automl/brain_automl/efficientnetv2\")","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":41.017298,"end_time":"2021-02-05T02:18:25.770952","exception":false,"start_time":"2021-02-05T02:17:44.753654","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-07T21:01:53.331732Z","iopub.execute_input":"2021-06-07T21:01:53.332244Z","iopub.status.idle":"2021-06-07T21:01:54.792870Z","shell.execute_reply.started":"2021-06-07T21:01:53.332206Z","shell.execute_reply":"2021-06-07T21:01:54.790732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n\n\n\n

1  BACKGROUND INFORMATION

","metadata":{"papermill":{"duration":0.045609,"end_time":"2021-02-05T02:18:25.862323","exception":false,"start_time":"2021-02-05T02:18:25.816714","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"**
\n\n

1.1 THE DATA

\n\n---\n\nBACKGROUND INFORMATION\n\nIn this competition, we are identifying and localizing COVID-19 abnormalities on chest radiographs.
**This is an object detection and classification problem.**\n\nFor each test image, you will be predicting a bounding box and class for all findings. \n* If you predict that there are no findings, you should create a prediction of **`\"none 1 0 0 1 1\"`** \n * \"none\" is the class ID for no finding, and this provides a one-pixel bounding box with a confidence of 1.0\n\nFurther, for each test study, you should make a determination within the following labels:\n\n> **`'Negative for Pneumonia', 'Typical Appearance', 'Indeterminate Appearance', 'Atypical Appearance'`**\n\nTo make a prediction of one of the above labels, create a prediction string similar to the \"none\" class above: \n* i.e. **`atypical 1 0 0 1 1`**\n\n---\n\n**MESSAGE FROM THE COMPETITION HOST ON LABEL AND BBOX DETAILS:**\n\nIn this challenge, the chest radiographs (CXRs) were categorized using a specific grading schema, based on a published paper:\n\n[**Litmanovich DE, Chung M, Kirkbride RR, Kicska G, Kanne JP. Review of chest radiograph findings of COVID-19 pneumonia and suggested reporting language. Journal of thoracic imaging. 2020 Nov 14;35(6):354-60.**](https://journals.lww.com/thoracicimaging/Fulltext/2020/11000/Review_of_Chest_Radiograph_Findings_of_COVID_19.4.aspx)\n\nPer the grading schema, chest radiographs are classified into one of four categories, which are mutually exclusive:\n\n1. **Typical Appearance**: Multifocal bilateral, peripheral opacities with rounded morphology, lower lung–predominant distribution\n2. **Indeterminate Appearance**: Absence of typical findings AND unilateral, central or upper lung predominant distribution\n3. **Atypical Appearance**: Pneumothorax, pleural effusion, pulmonary edema, lobar consolidation, solitary lung nodule or mass, diffuse tiny nodules, cavity\n4. **Negative for Pneumonia**: No lung opacities\n\nBounding boxes were placed on lung opacities, whether typical or indeterminate. Bounding boxes were also placed on some atypical findings including solitary lobar consolidation, nodules/masses, and cavities. Bounding boxes were not placed on pleural effusions, or pneumothoraces. No bounding boxes were placed for the negative for pneumonia category.\n\nIn cases of multiple adjacent opacities, we opted for one large bounding box, rather than multiple adjacent smaller boxes, to improve consistency in the labeling.\n\nAnnotators did have access to the COVID status for each patient, but were asked to adhere to the grading system above irrespective of the status. As such, some patients who were COVID negative still had chest radiographs with typical appearances. Similarly, some patients who were COVID positive had atypical appearances, or were negative for pneumonia (no lung opacities), because the grading system is based off the chest radiographic findings alone.\n\nThe goal in this challenge is to determine the appropriate category for each radiograph, as well as localize the lung opacities with a bounding box prediction.\n\n---\n\nThe images are in DICOM format, which means they contain additional data that might be useful for visualizing and classifying.\nNote that the images are in **DICOM** format, which means they contain additional data that might be useful for visualizing and classifying.\n\n![Example Radiographs](https://i.imgur.com/QWmbhXx.png)\n\n
\n\nDATASET INFORMATION\n\nThe **train dataset** comprises **`6,334`** chest scans in **DICOM** format, which were de-identified to protect patient privacy. \n\nNote that all images are stored in paths with the form **`study/series/image`**. \n* The **`study`** ID here relates directly to the study-level predictions\n* the **`image`** ID is the ID used for image-level predictions\n\nThe **test dataset** is of roughly the same scale as the training dataset. \n* As this is a kernels only competition we shsould plan accordingly\n* i.e. we should be able to infer on the entirety of the training dataset within the submission kernel\n\n
\n\nDATA FILES\n> **`train_study_level.csv`**\n> * **`id`** - unique study identifier\n> * **Negative for Pneumonia** - **`1`** if the study is negative for pneumonia, **`0`** otherwise\n> * **Typical Appearance** - **`1`** if the study has this appearance, **`0`** otherwise\n> * **Indeterminate Appearance** - **`1`** if the study has this appearance, **`0`** otherwise\n> * **Atypical Appearance** - **`1`** if the study has this appearance, **`0`** otherwise\n\n> **`train_image_level.csv`**\n> * **`id`** - unique image identifier\n> * **`boxes`** - bounding boxes in easily-readable dictionary format\n> * **`label`** - the correct prediction label for the provided bounding boxes","metadata":{"papermill":{"duration":0.04413,"end_time":"2021-02-05T02:18:25.953034","exception":false,"start_time":"2021-02-05T02:18:25.908904","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"
\n\n

1.2 THE GOAL

\n\n---\n\nIn this competition, you’ll identify and localize COVID-19 abnormalities on chest radiographs. In particular, you'll categorize the radiographs as one of a possible **`4`** categories. \n\nIn this competition, we are making predictions at both a study (multi-image) and image level.\n* **`negative for pneumonia`** or **`typical`**, **`indeterminate`**, or **`atypical`** \n \nYou'll work with a dataset consisting of **`8,781`** scans that have been annotated by experienced radiologists. You can train your model with **`6,334`** independently-labeled images and you will be evaluated on a test set of **`2,447`** images. \n\nThe challenge uses the standard PASCAL VOC 2010 mean Average Precision (mAP) at IoU > 0.5.\n* Note that the linked document describes VOC 2012, which differs in some minor ways (e.g. there is no concept of \"difficult\" classes in VOC 2010). The P/R curve and AP calculations remain the same.\n\n
\n\n
🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺
\n\n
If successful, you'll help radiologists diagnose the millions of COVID-19 patients more confidently and quickly.
\n\n
🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺🩺
","metadata":{"papermill":{"duration":0.044853,"end_time":"2021-02-05T02:18:26.044977","exception":false,"start_time":"2021-02-05T02:18:26.000124","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"
\n\n

1.3 ADDITIONAL INFORMATION ON ABNORMALITIES

\n\n---\n\n
Negative for Pneumonia\n* No lung opacities\n\n
Typical Appearance\n* Multifocal bilateral, peripheral opacities with rounded morphology, lower lung–predominant distribution\n\n
Indeterminate Appearance\n* Absence of typical findings AND unilateral, central or upper lung predominant distribution\n\n
Atypical Appearance\n* Pneumothorax, pleural effusion, pulmonary edema, lobar consolidation, solitary lung nodule or mass, diffuse tiny nodules, cavity","metadata":{"papermill":{"duration":0.046066,"end_time":"2021-02-05T02:18:26.137054","exception":false,"start_time":"2021-02-05T02:18:26.090988","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"
\n\n\n\n

2  NOTEBOOK SETUP

","metadata":{"papermill":{"duration":0.045324,"end_time":"2021-02-05T02:18:26.226344","exception":false,"start_time":"2021-02-05T02:18:26.18102","status":"completed"},"tags":[]}},{"cell_type":"code","source":"TRAIN_CSV_PATH = \"/kaggle/input/siim-covid19-updated-train-labels/updated_train_labels.csv\"\nSS_CSV_PATH = \"/kaggle/input/siim-covid19-updated-train-labels/updated_sample_submission.csv\"\n\nprint(\"\\n\\nCOMBINED AND EXPLODED TRAIN DATAFRAME\\n\\n\")\ntrain_df = pd.read_csv(TRAIN_CSV_PATH)\ndisplay(train_df)\n\nprint(\"\\n\\nSAMPLE SUBMISSION DATAFRAME\\n\\n\")\nss_df = pd.read_csv(SS_CSV_PATH)\ndisplay(ss_df)\n\nstudy_df = pd.read_csv(\"/kaggle/input/siim-covid19-detection/train_study_level.csv\")\nimage_df = pd.read_csv(\"/kaggle/input/siim-covid19-detection/train_image_level.csv\")\n\nall_image_ids = image_df.id.str.replace(\"_image\", \"\")\nbbox_image_ids = image_df.dropna().id.str.replace(\"_image\", \"\")","metadata":{"papermill":{"duration":0.717244,"end_time":"2021-02-05T02:18:26.987446","exception":false,"start_time":"2021-02-05T02:18:26.270202","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-07T21:01:54.794155Z","iopub.status.idle":"2021-06-07T21:01:54.794811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n\n\n\n

3  HELPER FUNCTIONS

","metadata":{"papermill":{"duration":0.045205,"end_time":"2021-02-05T02:18:27.079444","exception":false,"start_time":"2021-02-05T02:18:27.034239","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def get_absolute_file_paths(directory):\n all_abs_file_paths = []\n for dirpath,_,filenames in os.walk(directory):\n for f in filenames:\n all_abs_file_paths.append(os.path.abspath(os.path.join(dirpath, f)))\n return all_abs_file_paths\n\ndef dicom2array(path, voi_lut=True, fix_monochrome=True):\n \"\"\" Convert dicom file to numpy array \n \n Args:\n path (str): Path to the dicom file to be converted\n voi_lut (bool): Whether or not VOI LUT is available\n fix_monochrome (bool): Whether or not to apply monochrome fix\n \n Returns:\n Numpy array of the respective dicom file \n \n \"\"\"\n # Use the pydicom library to read the dicom file\n dicom = pydicom.read_file(path)\n \n # VOI LUT (if available by DICOM device) is used to \n # transform raw DICOM data to \"human-friendly\" view\n if voi_lut:\n data = apply_voi_lut(dicom.pixel_array, dicom)\n else:\n data = dicom.pixel_array\n \n # The XRAY may look inverted\n # - If we want to fix this we can\n if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n data = np.amax(data) - data\n \n # Normalize the image array and return\n data = data - np.min(data)\n data = data / np.max(data)\n data = (data*65535).astype(np.uint16)\n return data\n\n\ndef dicom2array_2(fname, target_size=300, use_clahe=True, clip_limit=2., grid_size=(8,8)):\n if tf.is_tensor(fname):\n fname = str(fname.numpy().decode())\n \n dicom = pydicom.dcmread(fname)\n data = apply_voi_lut(dicom.pixel_array, dicom)\n im = 65535*((data-np.min(data))/(np.max(data)-np.min(data)))\n \n if dicom.PhotometricInterpretation == \"MONOCHROME1\": # check for inverted image\n im = 65535 - im\n \n if use_clahe:\n clahe = cv2.createCLAHE(clipLimit=clip_limit, tileGridSize=grid_size)\n im = clahe.apply(im.astype('uint16'))\n \n return np.expand_dims(cv2.resize(im, (target_size, target_size)), axis=-1)\n\ndef flatten_l_o_l(nested_list):\n \"\"\" Flatten a list of lists \"\"\"\n return [item for sublist in nested_list for item in sublist]","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","papermill":{"duration":0.083595,"end_time":"2021-02-05T02:18:27.209879","exception":false,"start_time":"2021-02-05T02:18:27.126284","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-07T21:01:54.795845Z","iopub.status.idle":"2021-06-07T21:01:54.796457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Prepare\n\n- Study ID\n- Image ID\n- Path To Image\n- Label","metadata":{}},{"cell_type":"code","source":"study_df = study_df[study_df.id.str.contains(\"study\")]\nstudy_df[\"id\"] = study_df[\"id\"].str.replace(\"_study\", \"\")\nstudy_df[\"study_dir\"] = \"/kaggle/input/siim-covid19-detection/train/\"+study_df[\"id\"]\nall_study_images = study_df.study_dir.progress_apply(lambda x: get_absolute_file_paths(x)).to_list()\nlabels = study_df[['Negative for Pneumonia', 'Typical Appearance', 'Indeterminate Appearance', 'Atypical Appearance']].to_numpy()\nlabels = np.argmax(labels, axis=1)\nlbl_map = {i:v for i,v in enumerate(['Negative for Pneumonia', 'Typical Appearance', 'Indeterminate Appearance', 'Atypical Appearance'])}\n\ndef get_labelled_data(all_study_images, all_labels):\n image_paths = []\n labels = []\n for label, images in tqdm(zip(all_labels,all_study_images), total=len(all_labels)):\n image_paths += images\n labels += [label,]*len(images)\n return image_paths, labels\n\ntrain_image_paths, train_gt_labels = get_labelled_data(all_study_images, labels)\ntest_image_paths = flatten_l_o_l((\"/kaggle/input/siim-covid19-detection/test/\"+ss_df[ss_df.id.str.contains(\"_study\")].id.str.replace(\"_study\", \"\")).progress_apply(lambda x: get_absolute_file_paths(x)).to_list())\n\ntrain_image_ids = [fname.rsplit(\"train/\", 1)[1][:-4].replace(\"/\", \"-\") for fname in train_image_paths]\ntest_image_ids = [fname.rsplit(\"test/\", 1)[1][:-4].replace(\"/\", \"-\") for fname in test_image_paths]\n\n# For k-fold splitting (study's stay together)\ntrain_study_ids = [path.rsplit(\"/\", 3)[1] for path in train_image_paths]\ntest_study_ids = [path.rsplit(\"/\", 3)[1] for path in test_image_paths]\n\nprint(len(train_image_paths), len(train_image_ids), len(train_gt_labels), len(train_study_ids))\nprint(train_image_paths[0], train_image_ids[0], train_gt_labels[0], train_study_ids[0])","metadata":{"execution":{"iopub.status.busy":"2021-06-07T21:01:54.797522Z","iopub.status.idle":"2021-06-07T21:01:54.797994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Create K Spltis","metadata":{}},{"cell_type":"code","source":"# Use Group fold to keep studies together\nK_FOLDS = 5\ngroup_kfold = GroupKFold(n_splits=K_FOLDS,)\ngroup_kfold.get_n_splits(train_image_paths, train_gt_labels, train_study_ids)\n\n# Visualize The Folds\nfor i, (train_index, val_index) in enumerate(group_kfold.split(train_image_paths, train_gt_labels, train_study_ids)):\n print(f\"\\n\\n\\n------------ SPLIT: {i+1} ------------\\n\\n\\t--> TRAIN FOLD INDICES : {train_index}\\n\\t--> VAL FOLD INDICES : {val_index}\\n\\n\")\n x_train, x_val = np.array(train_image_paths)[train_index], np.array(train_image_paths)[val_index]\n y_train, y_val = np.array(train_gt_labels)[train_index], np.array(train_gt_labels)[val_index]\n print(\">> TRAIN PATHS << :\\n\\n\", x_train)\n print(\"\\n>> VAL PATHS << :\\n\\n\", x_val)\n print(\"\\n>> TRAIN LABELS << :\\n\\n\", y_train)\n print(\"\\n>> VAL LABELS << :\\n\\n\", y_val)","metadata":{"execution":{"iopub.status.busy":"2021-06-07T21:01:54.799086Z","iopub.status.idle":"2021-06-07T21:01:54.799511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def _bytes_feature(value, is_list=False):\n \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n if isinstance(value, type(tf.constant(0))):\n value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n \n if not is_list:\n value = [value]\n \n return tf.train.Feature(bytes_list=tf.train.BytesList(value=value))\n\ndef _float_feature(value, is_list=False):\n \"\"\"Returns a float_list from a float / double.\"\"\"\n \n if not is_list:\n value = [value]\n \n return tf.train.Feature(float_list=tf.train.FloatList(value=value))\n\ndef _int64_feature(value, is_list=False):\n \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n \n if not is_list:\n value = [value]\n \n return tf.train.Feature(int64_list=tf.train.Int64List(value=value))\n\ndef create_tf_dataset(paths, ids, labels=None):\n ds = tf.data.Dataset.from_tensor_slices(paths)\n ds_ids = tf.data.Dataset.from_tensor_slices(ids)\n ds = tf.data.Dataset.zip((ds, ds_ids))\n if labels is None:\n return ds\n else:\n ds_label = tf.data.Dataset.from_tensor_slices(tf.constant(labels, dtype=tf.uint8))\n return tf.data.Dataset.zip((ds, ds_label))\n\n# def prep_tf_dataset_w_target(img_path, bbox, target, img_shape,\n# invert=True, \n# rotate_trick=True, \n# repair_images=False):\n \n# img_tensor = tf_load_image(img_path, bbox,\n# img_size=img_shape[:2], \n# tile_to_3_channel=img_shape[-1]==3, \n# invert=invert, \n# rotate_trick=rotate_trick, \n# repair_images=repair_images)\n# return img_tensor, target\n\n# def prep_tf_dataset_wo_target(img_path, bbox, img_shape,\n# invert=True, \n# rotate_trick=True, \n# repair_images=False):\n \n# img_tensor = tf_load_image(img_path, bbox,\n# img_size=img_shape[:2], \n# tile_to_3_channel=img_shape[-1]==3, \n# invert=invert, \n# rotate_trick=rotate_trick, \n# repair_images=repair_images)\n# img_id = tf.strings.split(tf.strings.split(img_path, \".png\")[0], \"/\")[-1]\n# return img_tensor, img_id\n\ndef serialize_raw(image, image_id, target=None):\n \"\"\"\n Creates a tf.Example message ready to be written to a file from 4 features.\n\n Args:\n image (TBD): TBD\n image_id (str): TBD\n target (str): | delimited integers\n \n Returns:\n A tf.Example Message ready to be written to file\n \"\"\"\n \n # Create a dictionary mapping the feature name to the \n # tf.Example-compatible data type.\n feature = {'image': _bytes_feature(tf.io.encode_png(image), is_list=False)}\n \n if target is not None:\n feature[\"label\"] = _int64_feature(target, is_list=False)\n else:\n feature[\"image_id\"] = _bytes_feature(image_id, is_list=False)\n \n # Create a Features message using tf.train.Example.\n example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n return example_proto.SerializeToString()","metadata":{"execution":{"iopub.status.busy":"2021-06-07T21:01:54.800491Z","iopub.status.idle":"2021-06-07T21:01:54.800947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kfold_datasets = []\nfold_lengths = []\n\n# TRAIN AND VAL\nfor i, (train_indices, val_indices) in tqdm(enumerate(group_kfold.split(train_image_paths, train_gt_labels, train_study_ids)), total=K_FOLDS):\n k_fold_val_image_paths = np.array(train_image_paths)[val_indices]\n k_fold_val_image_ids = np.array(train_image_ids)[val_indices]\n k_fold_val_gt_labels = np.array(train_gt_labels)[val_indices] \n\n kfold_datasets.append(create_tf_dataset(k_fold_val_image_paths, k_fold_val_image_ids, k_fold_val_gt_labels))\n fold_lengths.append(len(k_fold_val_image_paths))\n \nkfold_datasets = [\n ds.map(lambda x,y: (tf.py_function(func=dicom2array_2, inp=[x[0]], Tout=tf.uint16), x[1], y)) \\\n for ds in tqdm(kfold_datasets, total=K_FOLDS)\n]\n\n# TEST\ntest_ds = create_tf_dataset(test_image_paths, test_image_ids)\ntest_ds = test_ds.map(lambda x,y: (tf.py_function(func=dicom2array_2, inp=[x], Tout=tf.uint16), y))\n\nimg,img_id,lbl = next(iter(kfold_datasets[0]))\nplt.imshow(img)\nplt.title(str(lbl.numpy())+\" ––– \"+str(img_id.numpy().decode()))\nplt.show()\n\nplt.imshow(next(iter(test_ds))[0])","metadata":{"execution":{"iopub.status.busy":"2021-06-07T21:01:54.801863Z","iopub.status.idle":"2021-06-07T21:01:54.802289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def write_tfrecords(ds, n_ex, n_ex_per_rec=2000, serialize_fn=serialize_raw, out_dir=\"/kaggle/working/300x300\", ds_type=\"train\", fold=None):\n \"\"\"\"\"\"\n n_recs = int(np.ceil(n_ex/n_ex_per_rec))\n \n # Make dataset iterable\n ds = iter(ds)\n \n out_dir = os.path.join(out_dir, ds_type)\n # Create folder\n if not os.path.isdir(out_dir):\n os.makedirs(out_dir, exist_ok=True)\n \n # Create tfrecords\n for i in tqdm(range(n_recs), total=n_recs):\n print(f\"\\n... Writing {ds_type.title()} TFRecord {i+1} of {n_recs} For Fold #{fold}...\\n\")\n if fold is not None:\n tfrec_path = os.path.join(out_dir, f\"{ds_type}_fold_{fold}__{(i+1):02}_{n_recs:02}.tfrec\")\n else:\n tfrec_path = os.path.join(out_dir, f\"{ds_type}__{(i+1):02}_{n_recs:02}.tfrec\")\n \n # This makes the tfrecord\n with tf.io.TFRecordWriter(tfrec_path) as writer:\n for ex in tqdm(range(n_ex_per_rec), total=n_ex_per_rec):\n try:\n example = serialize_fn(*next(ds))\n writer.write(example)\n except:\n break","metadata":{"execution":{"iopub.status.busy":"2021-06-07T21:01:54.803326Z","iopub.status.idle":"2021-06-07T21:01:54.803781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Remove if already existing\n!rm -rf /kaggle/working/*\n\nfor i, k_ds in tqdm(enumerate(kfold_datasets), total=K_FOLDS):\n write_tfrecords(k_ds, fold_lengths[i], n_ex_per_rec=1500, ds_type=\"train\", fold=i+1)\n\nwrite_tfrecords(test_ds, len(test_image_ids), n_ex_per_rec=1500, ds_type=\"test\")","metadata":{"execution":{"iopub.status.busy":"2021-06-07T21:01:54.804750Z","iopub.status.idle":"2021-06-07T21:01:54.805181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}