{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!conda install '/kaggle/input/pydicom-conda-helper/libjpeg-turbo-2.1.0-h7f98852_0.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/libgcc-ng-9.3.0-h2828fa1_19.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/gdcm-2.8.9-py37h500ead1_1.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/conda-4.10.1-py37h89c1867_0.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/certifi-2020.12.5-py37h89c1867_1.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/openssl-1.1.1k-h7f98852_0.tar.bz2' -c conda-forge -y","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-07-02T18:36:56.102113Z","iopub.execute_input":"2021-07-02T18:36:56.102820Z","iopub.status.idle":"2021-07-02T18:38:17.044347Z","shell.execute_reply.started":"2021-07-02T18:36:56.102690Z","shell.execute_reply":"2021-07-02T18:38:17.043030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMAGE_SIZE = 1024","metadata":{"execution":{"iopub.status.busy":"2021-07-02T18:39:42.131013Z","iopub.execute_input":"2021-07-02T18:39:42.131532Z","iopub.status.idle":"2021-07-02T18:39:42.137177Z","shell.execute_reply.started":"2021-07-02T18:39:42.131483Z","shell.execute_reply":"2021-07-02T18:39:42.135790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nimport os\nimport pydicom\nimport glob\nfrom tqdm.notebook import tqdm\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nimport matplotlib.pyplot as plt\nfrom skimage import exposure\nimport cv2\nimport warnings\nfrom fastai.vision.all import *\nfrom fastai.medical.imaging import *\nfrom tqdm.auto import tqdm\nfrom sklearn.model_selection import StratifiedKFold\nimport tensorflow as tf\nimport multiprocessing as mp\n\ntqdm.pandas()\nwarnings.filterwarnings('ignore')\npd.set_option('display.max_colwidth', None)","metadata":{"execution":{"iopub.status.busy":"2021-07-02T18:39:42.908921Z","iopub.execute_input":"2021-07-02T18:39:42.909349Z","iopub.status.idle":"2021-07-02T18:39:52.532123Z","shell.execute_reply.started":"2021-07-02T18:39:42.909315Z","shell.execute_reply":"2021-07-02T18:39:52.530796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def dicom2array(path, voi_lut=True, fix_monochrome=True):\n    dicom = pydicom.read_file(path)\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    return data","metadata":{"execution":{"iopub.status.busy":"2021-07-02T18:39:52.534096Z","iopub.execute_input":"2021-07-02T18:39:52.534489Z","iopub.status.idle":"2021-07-02T18:39:52.542373Z","shell.execute_reply.started":"2021-07-02T18:39:52.534445Z","shell.execute_reply":"2021-07-02T18:39:52.541091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_path = Path('../input/siim-covid19-detection')\ndataset_path.ls()","metadata":{"execution":{"iopub.status.busy":"2021-07-02T18:39:52.544472Z","iopub.execute_input":"2021-07-02T18:39:52.544928Z","iopub.status.idle":"2021-07-02T18:39:52.560832Z","shell.execute_reply.started":"2021-07-02T18:39:52.544880Z","shell.execute_reply":"2021-07-02T18:39:52.559589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data_path = dataset_path/'train'\ntrain_data_path.ls()","metadata":{"execution":{"iopub.status.busy":"2021-07-02T18:39:52.562434Z","iopub.execute_input":"2021-07-02T18:39:52.562780Z","iopub.status.idle":"2021-07-02T18:39:52.755668Z","shell.execute_reply.started":"2021-07-02T18:39:52.562746Z","shell.execute_reply":"2021-07-02T18:39:52.754554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_study_level  = pd.read_csv('../input/kagglesiimcovid/train_stratified_group_5xfolds_clean.csv')\ndf_study_level.drop(columns=['has_bbox'], inplace=True)\ndf_study_level.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-02T18:39:55.272406Z","iopub.execute_input":"2021-07-02T18:39:55.272907Z","iopub.status.idle":"2021-07-02T18:39:55.343047Z","shell.execute_reply.started":"2021-07-02T18:39:55.272864Z","shell.execute_reply":"2021-07-02T18:39:55.341931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_dicoms = get_dicom_files(train_data_path)\nimage_ids  = df_study_level.id.unique().tolist()\nall_dicoms = all_dicoms.filter(lambda x : str(x).split(os.path.sep)[-1].split('.')[0] in image_ids)\ndicom_dict = all_dicoms.map_dict(lambda x: str(x).split(os.path.sep)[-1].split('.')[0])\ndicom_dict = {v:str(os.path.abspath(k)) for k,v in dicom_dict.items()}","metadata":{"execution":{"iopub.status.busy":"2021-07-02T18:39:55.619205Z","iopub.execute_input":"2021-07-02T18:39:55.619705Z","iopub.status.idle":"2021-07-02T18:40:29.736162Z","shell.execute_reply.started":"2021-07-02T18:39:55.619658Z","shell.execute_reply":"2021-07-02T18:40:29.735037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_study_level['dcm_path'] = df_study_level['id'].map(lambda x: dicom_dict[x])\ndf_study_level.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-02T18:40:31.402734Z","iopub.execute_input":"2021-07-02T18:40:31.403203Z","iopub.status.idle":"2021-07-02T18:40:31.427487Z","shell.execute_reply.started":"2021-07-02T18:40:31.403157Z","shell.execute_reply":"2021-07-02T18:40:31.426329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = df_study_level\nprint('Images WITH Negative for Pneumonia')\nimgs = train.loc[train.negative == 1].sample(10).dcm_path.values\nplt.figure(figsize=(20,8))\nfor i,k in enumerate(imgs):\n    img = dicom2array(k)\n    img = cv2.resize(img, (128,128))\n    plt.subplot(2,5,i+1)\n    plt.axis('off')\n    plt.imshow(img, cmap='gray')\nplt.show()\n\nprint('Images WITH Typical Appearance')\nimgs = train.loc[train.typical == 1].sample(10).dcm_path.values\nplt.figure(figsize=(20,8))\nfor i,k in enumerate(imgs):\n    img = dicom2array(k)\n    img = cv2.resize(img, (128,128))\n    plt.subplot(2,5,i+1)\n    plt.axis('off')\n    plt.imshow(img, cmap='gray')\nplt.show()\n\nprint('Images WITH Indeterminate Appearance')\nimgs = train.loc[train.indeterminate == 1].sample(10).dcm_path.values\nplt.figure(figsize=(20,8))\nfor i,k in enumerate(imgs):\n    img = dicom2array(k)\n    img = cv2.resize(img, (128,128))\n    plt.subplot(2,5,i+1)\n    plt.axis('off')\n    plt.imshow(img, cmap='gray')\nplt.show()\n\nprint('Images WITH Atypical Appearance')\nimgs = train.loc[train.atypical == 1].sample(10).dcm_path.values\nplt.figure(figsize=(20,8))\nfor i,k in enumerate(imgs):\n    img = dicom2array(k)\n    img = cv2.resize(img, (128,128))\n    plt.subplot(2,5,i+1)\n    plt.axis('off')\n    plt.imshow(img, cmap='gray')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-02T18:40:34.523376Z","iopub.execute_input":"2021-07-02T18:40:34.523911Z","iopub.status.idle":"2021-07-02T18:40:50.194095Z","shell.execute_reply.started":"2021-07-02T18:40:34.523862Z","shell.execute_reply":"2021-07-02T18:40:50.193207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def convert_to_feature(value, value_type=None):\n    \"\"\"Converts the given python object to a tf.train.Feature.\n  Args:\n    value: int, float, bytes or a list of them.\n    value_type: optional, if specified, forces the feature to be of the given\n      type. Otherwise, type is inferred automatically. Can be one of\n      ['bytes', 'int64', 'float', 'bytes_list', 'int64_list', 'float_list']\n  Returns:\n    feature: A tf.train.Feature object.\n  \"\"\"\n\n    if value_type is None:\n\n        element = value[0] if isinstance(value, list) else value\n\n        if isinstance(element, bytes):\n            value_type = 'bytes'\n\n        elif isinstance(element, (int, np.integer)):\n            value_type = 'int64'\n\n        elif isinstance(element, (float, np.floating)):\n            value_type = 'float'\n\n        else:\n            raise ValueError('Cannot convert type {} to feature'.format(\n                type(element)))\n\n        if isinstance(value, list):\n            value_type = value_type + '_list'\n\n    if value_type == 'int64':\n        return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n\n    elif value_type == 'int64_list':\n        value = np.asarray(value).astype(np.int64).reshape(-1)\n        return tf.train.Feature(int64_list=tf.train.Int64List(value=value))\n\n    elif value_type == 'float':\n        return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\n    elif value_type == 'float_list':\n        value = np.asarray(value).astype(np.float32).reshape(-1)\n        return tf.train.Feature(float_list=tf.train.FloatList(value=value))\n\n    elif value_type == 'bytes':\n        return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\n    elif value_type == 'bytes_list':\n        return tf.train.Feature(bytes_list=tf.train.BytesList(value=value))\n\n    else:\n        raise ValueError('Unknown value_type parameter - {}'.format(value_type))","metadata":{"execution":{"iopub.status.busy":"2021-07-02T18:41:51.150828Z","iopub.execute_input":"2021-07-02T18:41:51.151897Z","iopub.status.idle":"2021-07-02T18:41:51.167353Z","shell.execute_reply.started":"2021-07-02T18:41:51.151813Z","shell.execute_reply":"2021-07-02T18:41:51.165766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def dicom2array_2(fname, target_size=IMAGE_SIZE, use_clahe=True, clip_limit=2.0, grid_size=(8, 8)):\n    dicom = pydicom.dcmread(fname)\n    data = apply_voi_lut(dicom.pixel_array, dicom)\n    im = data - np.min(data)\n    im = 255.0 * im / np.max(im)\n\n    if dicom.PhotometricInterpretation == \"MONOCHROME1\":  # check for inverted image\n        im = 255.0 - im\n\n    if use_clahe:\n        clahe = cv2.createCLAHE(clipLimit=clip_limit, tileGridSize=grid_size)\n        im = clahe.apply(im.astype(\"uint8\"))\n    return (\n        np.expand_dims(\n            cv2.resize(im, (target_size, target_size)).astype(dtype=np.uint8), axis=-1\n        ),\n        im.shape,\n    )\n\n\ndef image_info_to_feature_dict(image_id, height, width, encoded_str, labels):\n    \"\"\"Convert image information to a dict of features.\"\"\"\n    \n    # [\"atypical\", \"indeterminate\", \"negative\", \"typical\"]\n    return {\n        \"image/encoded\"       : convert_to_feature(encoded_str.numpy(), 'bytes'),\n        \"image/image_id\"      : convert_to_feature(image_id, 'bytes'),\n        \"image/height\"        : convert_to_feature(height),\n        \"image/width\"         : convert_to_feature(width),\n        \"label/atypical\"      : convert_to_feature(labels[0], value_type=\"int64\"),\n        \"label/indeterminate\" : convert_to_feature(labels[1], value_type=\"int64\"),\n        \"label/negative\"      : convert_to_feature(labels[2], value_type=\"int64\"),\n        \"label/typical\"       : convert_to_feature(labels[3], value_type=\"int64\"),\n    }\n\n\ndef serialize_sample(dcm_path, labels):\n    \"\"\"Serializes a single dicom image and its corresponding label\"\"\"\n    assert os.path.exists(dcm_path)\n    img, (h, w) = dicom2array_2(dcm_path)\n    image_id = str(dcm_path).split(os.path.sep)[-1].split('.')[0]\n    encoded_image = tf.io.encode_png(img)\n    feature_dict = image_info_to_feature_dict(image_id.encode(), h, w, encoded_image, labels)\n    sample = tf.train.Example(features=tf.train.Features(feature=feature_dict))\n    return sample","metadata":{"execution":{"iopub.status.busy":"2021-07-02T18:41:55.904813Z","iopub.execute_input":"2021-07-02T18:41:55.905298Z","iopub.status.idle":"2021-07-02T18:41:55.920804Z","shell.execute_reply.started":"2021-07-02T18:41:55.905249Z","shell.execute_reply":"2021-07-02T18:41:55.919155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def check_and_make_dir(directory):\n    \"\"\"Creates the directory if it doesn't exist.\"\"\"\n    if not tf.io.gfile.isdir(directory):\n        tf.io.gfile.makedirs(directory)\n\ndef write_tf_record_dataset(\n    output_path,\n    annotation_iterator,\n    process_func,\n    num_shards,\n    use_multiprocessing=True,\n    unpack_arguments=True,\n):\n    \"\"\"Iterates over annotations, processes them and writes into TFRecords.\n    Args:\n      output_path: The prefix path to create TF record files.\n      annotation_iterator: An iterator of tuples containing details about the\n        dataset.\n      process_func: A function which takes the elements from the tuples of\n        annotation_iterator as arguments and returns a tuple of (tf.train.Example,\n        int). The integer indicates the number of annotations that were skipped.\n      num_shards: int, the number of shards to write for the dataset.\n      use_multiprocessing:\n        Whether or not to use multiple processes to write TF Records.\n      unpack_arguments:\n        Whether to unpack the tuples from annotation_iterator as individual\n          arguments to the process func or to pass the returned value as it is.\n    Returns:\n      num_skipped: The total number of skipped annotations.\n    \"\"\"\n    writers = [\n        tf.io.TFRecordWriter(output_path + \"%05d-of-%05d.tfrecord\" % (i, num_shards))\n        for i in range(num_shards)\n    ]\n\n    total_num_annotations_skipped = 0\n\n    if use_multiprocessing:\n        pool = mp.Pool(processes=mp.cpu_count())\n        if unpack_arguments:\n            tf_example_iterator = pool.starmap(process_func, annotation_iterator)\n        else:\n            tf_example_iterator = pool.imap(process_func, annotation_iterator)\n    else:\n        if unpack_arguments:\n            tf_example_iterator = itertools.starmap(process_func, annotation_iterator)\n        else:\n            tf_example_iterator = map(process_func, annotation_iterator)\n    \n    iterator = tqdm(tf_example_iterator, total=len(annotation_iterator))\n\n    for idx, (tf_example) in enumerate(iterator):\n        writers[idx % num_shards].write(tf_example.SerializeToString())\n\n    if use_multiprocessing:\n        pool.close()\n        pool.join()\n\n    for writer in writers:\n        writer.close()","metadata":{"execution":{"iopub.status.busy":"2021-07-02T18:41:59.119826Z","iopub.execute_input":"2021-07-02T18:41:59.120489Z","iopub.status.idle":"2021-07-02T18:41:59.132623Z","shell.execute_reply.started":"2021-07-02T18:41:59.120436Z","shell.execute_reply":"2021-07-02T18:41:59.131019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CLASSES = [\"atypical\", \"indeterminate\", \"negative\", \"typical\"]\ntrain = df_study_level.copy()\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-02T18:41:59.923103Z","iopub.execute_input":"2021-07-02T18:41:59.923539Z","iopub.status.idle":"2021-07-02T18:41:59.944594Z","shell.execute_reply.started":"2021-07-02T18:41:59.923502Z","shell.execute_reply":"2021-07-02T18:41:59.943345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"folds = 5\nnum_shards = 20\n\nfor fold in tqdm(range(folds)):\n    df_fold = train.query(f'kfold=={fold}').reset_index(inplace=False, drop=True)\n    df_fold_labels = df_fold[CLASSES].values.tolist()\n    df_fold_labels = [list(o) for o in df_fold_labels]\n    df_fold_images = df_fold['dcm_path'].values.tolist()\n    iterator = [(df_fold_images[i], df_fold_labels[i]) for i in range(len(df_fold))]\n    check_and_make_dir(f'fold-{fold}/')\n\n    write_tf_record_dataset(\n        output_path=f'fold-{fold}/',\n        annotation_iterator=iterator, \n        process_func=serialize_sample, \n        num_shards=num_shards, \n        use_multiprocessing=True\n    )","metadata":{"execution":{"iopub.status.busy":"2021-07-02T18:42:00.826886Z","iopub.execute_input":"2021-07-02T18:42:00.827369Z","iopub.status.idle":"2021-07-02T18:45:24.898417Z","shell.execute_reply.started":"2021-07-02T18:42:00.827326Z","shell.execute_reply":"2021-07-02T18:45:24.896934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_feature_description = {\n    \"image/encoded\"       : tf.io.FixedLenFeature([], tf.string),\n    \"image/image_id\"      : tf.io.FixedLenFeature([], tf.string),\n    \"image/height\"        : tf.io.FixedLenFeature([], tf.int64),\n    \"image/width\"         : tf.io.FixedLenFeature([], tf.int64),\n    f\"label/{CLASSES[0]}\" : tf.io.FixedLenFeature([], tf.int64),\n    f\"label/{CLASSES[1]}\" : tf.io.FixedLenFeature([], tf.int64),\n    f\"label/{CLASSES[2]}\" : tf.io.FixedLenFeature([], tf.int64),\n    f\"label/{CLASSES[3]}\" : tf.io.FixedLenFeature([], tf.int64),\n}\n\n\nraw_image_dataset = tf.data.TFRecordDataset(tf.io.gfile.glob('./fold-0/*.tfrecord'))\n\ndef _parse_image_function(example_proto):\n    return tf.io.parse_single_example(example_proto, image_feature_description)\n\nparsed_image_dataset = raw_image_dataset.map(_parse_image_function)\n\nplt.figure(figsize=(20,8))\nfor i,features in enumerate(parsed_image_dataset.take(10)):\n    # [\"atypical\", \"indeterminate\", \"negative\", \"typical\"]\n    image_raw = features['image/encoded'].numpy()\n    img = PILImage.create(image_raw)\n    \n    atypical      = features[f'label/{CLASSES[0]}'].numpy()\n    indeterminate = features[f'label/{CLASSES[1]}'].numpy()\n    negative      = features[f'label/{CLASSES[2]}'].numpy()\n    typical       = features[f'label/{CLASSES[3]}'].numpy()\n    label_str = [atypical, indeterminate, negative, typical]\n    \n    plt.subplot(2,5, i+1)\n    plt.axis('off')\n    plt.imshow(img, cmap='gray')\n    plt.title(label_str, color='green' if negative else 'red')","metadata":{"execution":{"iopub.status.busy":"2021-07-02T18:51:05.713953Z","iopub.execute_input":"2021-07-02T18:51:05.714404Z","iopub.status.idle":"2021-07-02T18:51:07.837336Z","shell.execute_reply.started":"2021-07-02T18:51:05.714356Z","shell.execute_reply":"2021-07-02T18:51:07.836027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}