{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook is the third part of [RANZCR 1st Place Solution by TF](https://www.kaggle.com/tt195361/ranzcr-1st-place-solution-by-tf-1-make-masks).\nThis notebook is based on [RANZCR 1st Place Soluiton Seg Model (small ver.)](https://www.kaggle.com/haqishen/ranzcr-1st-place-soluiton-seg-model-small-ver).\n\nThe third step is to generate masks by using the trained segmentation model in the [previous step](https://www.kaggle.com/tt195361/ranzcr-1st-place-solution-by-tf-2-seg-model). Then, TFRecord files are made from images, generated masks, labels, and folds. The files will be used to train the classification model at the [next step](https://www.kaggle.com/tt195361/ranzcr-1st-place-solution-by-tf-4-cls-model).\n\nThe original notebook uses 5 fold results to make masks. In this notebook, only 1 fold is used, because it took long time (about 4.5 hours) to train the segmentation model.\nI used the result of [Version 10](https://www.kaggle.com/tt195361/ranzcr-1st-place-solution-by-tf-2-seg-model?scriptVersionId=61479593).","metadata":{}},{"cell_type":"markdown","source":"## Install Segmentation Models Locally","metadata":{}},{"cell_type":"code","source":"%env SM_FRAMEWORK=tf.keras\n!pip install ../input/segmentation-models-keras/Keras_Applications-1.0.8-py3-none-any.whl --quiet\n!pip install ../input/segmentation-models-keras/image_classifiers-1.0.0-py3-none-any.whl --quiet\n!pip install ../input/segmentation-models-keras/efficientnet-1.0.0-py3-none-any.whl --quiet\n!pip install ../input/segmentation-models-keras/segmentation_models-1.0.1-py3-none-any.whl --quiet\n\nprint(\"Segmentation Models installed.\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Config and Libraries","metadata":{}},{"cell_type":"code","source":"DEBUG = False","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# libraries\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport segmentation_models as sm\nfrom kaggle_datasets import KaggleDatasets\n\nprint(tf.__version__)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_image_size = 1024\ngen_image_size = 512\nbatch_size = 16\njpeg_quality = 100","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sm.set_framework('tf.keras')\ntf.keras.backend.set_image_data_format('channels_last')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## TPU","metadata":{}},{"cell_type":"code","source":"try: # detect TPUs\n    # NEW: in Tensorflow 2.4\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect() \n    strategy = tf.distribute.TPUStrategy(tpu)\nexcept ValueError: # otherwise detect GPUs\n    strategy = tf.distribute.MirroredStrategy() # single-GPU or multi-GPU\n    \nprint(f\"Running on {strategy.num_replicas_in_sync} replicas\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ranzcr_name = 'ranzcr-clip-catheter-line-classification'\nranzcr_fold_dir = '../input/ranzcr-fold/'\nranzcr_model_dir = '../input/ranzcr-1st-place-solution-by-tf-models/'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seg_model_name = 'seg_model_V10_0.hdf5'\n\nAUTOTUNE = tf.data.experimental.AUTOTUNE","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GCS_DS_PATH = KaggleDatasets().get_gcs_path(ranzcr_name)\n\nGCS_DS_PATH","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train_V2","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv(ranzcr_fold_dir + 'train_v2.csv')\n\ndf_train","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"uid_fold_dict = dict(\n    zip(df_train['StudyInstanceUID'], df_train['fold']))\n\nlen(uid_fold_dict)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train TFRecords","metadata":{}},{"cell_type":"code","source":"def decode_train_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    return image\n\ndef read_train_tfrecord(example):\n    TFREC_FORMAT = {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'StudyInstanceUID': tf.io.FixedLenFeature([], tf.string),\n        'ETT - Abnormal': tf.io.FixedLenFeature([], tf.int64),\n        'ETT - Borderline': tf.io.FixedLenFeature([], tf.int64),\n        'ETT - Normal': tf.io.FixedLenFeature([], tf.int64),\n        'NGT - Abnormal': tf.io.FixedLenFeature([], tf.int64),\n        'NGT - Borderline': tf.io.FixedLenFeature([], tf.int64),\n        'NGT - Incompletely Imaged': tf.io.FixedLenFeature([], tf.int64),\n        'NGT - Normal': tf.io.FixedLenFeature([], tf.int64),\n        'CVC - Abnormal': tf.io.FixedLenFeature([], tf.int64),\n        'CVC - Borderline': tf.io.FixedLenFeature([], tf.int64),\n        'CVC - Normal': tf.io.FixedLenFeature([], tf.int64),\n        'Swan Ganz Catheter Present': tf.io.FixedLenFeature([], tf.int64),\n    }\n    \n    example = tf.io.parse_single_example(example, TFREC_FORMAT)\n    image = decode_train_image(example['image'])\n    study_inst_id = example['StudyInstanceUID']\n    ett_abnormal = example['ETT - Abnormal']\n    ett_borderline = example['ETT - Borderline']\n    ett_normal = example['ETT - Normal']\n    ngt_abnormal = example['NGT - Abnormal']\n    ngt_borderline = example['NGT - Borderline']\n    ngt_inc_imaged = example['NGT - Incompletely Imaged']\n    ngt_normal = example['NGT - Normal']\n    cvc_abnormal = example['CVC - Abnormal']\n    cvc_borderline = example['CVC - Borderline']\n    cvc_normal = example['CVC - Normal']\n    swan_ganz_cat_present = example['Swan Ganz Catheter Present']\n    labels = [\n        ett_abnormal, ett_borderline, ett_normal,\n        ngt_abnormal, ngt_borderline, ngt_inc_imaged, ngt_normal,\n        cvc_abnormal, cvc_borderline, cvc_normal,\n        swan_ganz_cat_present,\n    ]\n    return image, labels, study_inst_id\n\ndef load_train_dataset(filenames):\n    ds = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTOTUNE)\n    ds = ds.map(read_train_tfrecord, num_parallel_calls=AUTOTUNE)\n    return ds","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model","metadata":{}},{"cell_type":"code","source":"def load_model(model_name):\n    with strategy.scope():\n        model_path = ranzcr_model_dir + model_name\n        model = tf.keras.models.load_model(model_path)\n\n    model.summary()\n    return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = load_model(seg_model_name)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset","metadata":{}},{"cell_type":"code","source":"def make_predict_dataset(image_batch):\n    def _preprocess_image(image):\n        image = tf.cast(image, tf.float32) / 255.0  # convert image to floats in [0, 1] range\n        image = tf.image.resize(image, (train_image_size, train_image_size))\n        return image\n    \n    ds = tf.data.Dataset.from_tensor_slices(image_batch)\n    ds = ds.map(_preprocess_image, num_parallel_calls=AUTOTUNE)\n    ds = ds.batch(batch_size)\n    return ds","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def convert_image(image):\n    image = tf.image.resize(image, (gen_image_size, gen_image_size))\n    image = tf.cast(image, dtype=tf.uint8)\n    image = tf.image.encode_jpeg(image, quality=jpeg_quality)\n    return image\n\ndef convert_mask(mask):\n    mask = tf.image.resize(mask, (gen_image_size, gen_image_size))\n    # Generated mask is 0..1, so change to 0..255.\n    mask = mask * 255.0\n    # Make 3 channels for encoding as PNG.\n    zeros = tf.zeros((gen_image_size, gen_image_size, 1))\n    mask = tf.concat([mask, zeros], axis=-1) \n    mask = tf.cast(mask, dtype=tf.uint8)\n    mask = tf.io.encode_png(mask)\n    return mask","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def _bytes_feature(value):\n    \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n    if isinstance(value, type(tf.constant(0))):\n        # BytesList won't unpack a string from an EagerTensor.\n        value = value.numpy() \n    elif isinstance(value, str):\n        # string needs to be encoded to bytes.\n        value = value.encode('utf-8')\n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _int64_feature(value):\n    \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def serialize_example(image, mask, labels, fold):\n    feature = {\n        'image': _bytes_feature(image),\n        'mask': _bytes_feature(mask),\n        'ETT - Abnormal': _int64_feature(labels[0]),\n        'ETT - Borderline': _int64_feature(labels[1]),\n        'ETT - Normal': _int64_feature(labels[2]),\n        'NGT - Abnormal': _int64_feature(labels[3]),\n        'NGT - Borderline': _int64_feature(labels[4]),\n        'NGT - Incompletely Imaged': _int64_feature(labels[5]),\n        'NGT - Normal': _int64_feature(labels[6]),\n        'CVC - Abnormal': _int64_feature(labels[7]),\n        'CVC - Borderline': _int64_feature(labels[8]),\n        'CVC - Normal': _int64_feature(labels[9]),\n        'Swan Ganz Catheter Present': _int64_feature(labels[10]),\n        'fold': _int64_feature(fold),\n    }\n    \n    example_proto = tf.train.Example(\n        features=tf.train.Features(feature=feature))\n    return example_proto.SerializeToString()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_tfrec_paths = GCS_DS_PATH + '/train_tfrecords/*.tfrec'\ntrain_tfrec_file_names = sorted(tf.io.gfile.glob(train_tfrec_paths))\ntrain_tfrec_file_names = \\\n    train_tfrec_file_names[:2] if DEBUG else train_tfrec_file_names\n\nlen(train_tfrec_file_names)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Only one fold...","metadata":{}},{"cell_type":"code","source":"for train_file_i, train_file_name in enumerate(train_tfrec_file_names):\n    train_ds = load_train_dataset(train_file_name)\n    train_ds = train_ds.take(100) if DEBUG else train_ds\n    \n    gen_item_count = 0\n    for train_item in train_ds:\n        gen_item_count += 1\n    gen_file_name = \\\n        \"{0:02d}-{1:04d}.tfrec\".format(train_file_i, gen_item_count)\n\n    print(\"Writing {0}...\".format(gen_file_name))\n    with tf.io.TFRecordWriter(gen_file_name) as writer:\n        train_batch_ds = train_ds.batch(batch_size)\n        for image_batch, labels_batch, study_inst_id_batch in train_batch_ds:\n            print('.', end='', flush=True)\n            pred_ds = make_predict_dataset(image_batch)\n            mask_batch = model.predict(pred_ds, verbose=0)\n            for image, mask, labels, study_inst_id in \\\n                    zip(image_batch, mask_batch,\n                        labels_batch, study_inst_id_batch):\n                image = convert_image(image)\n                mask = convert_mask(mask)\n                uid = study_inst_id.numpy().decode('utf-8')\n                fold = uid_fold_dict[uid]\n                example = serialize_example(image, mask, labels, fold)\n                writer.write(example)\n    print()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! ls -l","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Verify Generated TFRecords","metadata":{}},{"cell_type":"code","source":"def decode_gen_image(image_bytes):\n    image = tf.image.decode_jpeg(image_bytes, channels=3)\n    return image\n\ndef decode_gen_mask(mask_bytes):\n    mask = tf.io.decode_png(mask_bytes, channels=3)\n    return mask\n\ndef read_gen_tfrecord(example):\n    TFREC_FORMAT = {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'mask': tf.io.FixedLenFeature([], tf.string),\n        'ETT - Abnormal': tf.io.FixedLenFeature([], tf.int64),\n        'ETT - Borderline': tf.io.FixedLenFeature([], tf.int64),\n        'ETT - Normal': tf.io.FixedLenFeature([], tf.int64),\n        'NGT - Abnormal': tf.io.FixedLenFeature([], tf.int64),\n        'NGT - Borderline': tf.io.FixedLenFeature([], tf.int64),\n        'NGT - Incompletely Imaged': tf.io.FixedLenFeature([], tf.int64),\n        'NGT - Normal': tf.io.FixedLenFeature([], tf.int64),\n        'CVC - Abnormal': tf.io.FixedLenFeature([], tf.int64),\n        'CVC - Borderline': tf.io.FixedLenFeature([], tf.int64),\n        'CVC - Normal': tf.io.FixedLenFeature([], tf.int64),\n        'Swan Ganz Catheter Present': tf.io.FixedLenFeature([], tf.int64),\n        'fold': tf.io.FixedLenFeature([], tf.int64),\n    }\n    \n    example = tf.io.parse_single_example(example, TFREC_FORMAT)\n    image = decode_gen_image(example['image'])\n    mask = decode_gen_mask(example['mask'])\n    ett_abnormal = example['ETT - Abnormal']\n    ett_borderline = example['ETT - Borderline']\n    ett_normal = example['ETT - Normal']\n    ngt_abnormal = example['NGT - Abnormal']\n    ngt_borderline = example['NGT - Borderline']\n    ngt_inc_imaged = example['NGT - Incompletely Imaged']\n    ngt_normal = example['NGT - Normal']\n    cvc_abnormal = example['CVC - Abnormal']\n    cvc_borderline = example['CVC - Borderline']\n    cvc_normal = example['CVC - Normal']\n    swan_ganz_cat_present = example['Swan Ganz Catheter Present']\n    labels = [\n        ett_abnormal, ett_borderline, ett_normal,\n        ngt_abnormal, ngt_borderline, ngt_inc_imaged, ngt_normal,\n        cvc_abnormal, cvc_borderline, cvc_normal,\n        swan_ganz_cat_present,\n    ]\n    fold = example['fold']\n    return image, mask, labels, fold\n\ndef load_gen_dataset(filenames):\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=None)\n    dataset = dataset.map(read_gen_tfrecord, num_parallel_calls=None)\n    return dataset","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gen_tfrec_file_names = tf.io.gfile.glob('*.tfrec')\ngen_ds = load_gen_dataset(gen_tfrec_file_names)\n\nprint(gen_ds)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pylab import rcParams\nrcParams['figure.figsize'] = 20,10\n\nf, axarr = plt.subplots(1,5)\nmasks = []\ngen_ds_iter = iter(gen_ds)\nfor p in range(5):\n    img, mask, labels, fold = next(gen_ds_iter)\n    axarr[p].imshow(img)\n    title = \"{0}{1}{2}:{3}{4}{5}{6}:{7}{8}{9}:{10}-{11}\".format(\n        labels[0], labels[1], labels[2],\n        labels[3], labels[4], labels[5], labels[6],\n        labels[7], labels[8], labels[9], labels[10],\n        fold)\n    axarr[p].set_title(title)\n    masks.append(mask)\n\nf, axarr = plt.subplots(1,5)\nfor p in range(5):\n    axarr[p].imshow(masks[p][ : , : , 0])\n\nf, axarr = plt.subplots(1,5)\nfor p in range(5):\n    axarr[p].imshow(masks[p][ : , : , 1])","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}