{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook trains a Tensorflow Keras model running on TPU.  Summaries are as follows:\n\n* Model: FPN in [Segmentation Models](https://github.com/qubvel/segmentation_models)\n* Backbone: EfficientNet B4\n* Image Size: 640\n* Learning Rate: maximum 1e-3, cosine decay with warmup\n* Epochs: 30 (2 for warmup and the rests are cosine decay)\n* Batch Size: 64\n* Folds: 5 (trains 1 fold only)\n* Loss: 0.5 * BCE + 0.5 * Dice\n* Data Augmentations: [Albumentations](https://albumentations.ai/) like\n* Oversampling: 3 times for hard samples (DICE coefficients < 0.05)\n* 2.5D stride 2\n\n# Reference\n\nThanks a lot to the authors for sharing the valuable information.\n\n* [UWMGI: UNet Keras [Train] with EDA](https://www.kaggle.com/code/ammarnassanalhajali/uwmgi-unet-keras-train-with-eda)\n* [UWMGI: Unet [Train] [PyTorch]](https://www.kaggle.com/code/awsaf49/uwmgi-unet-train-pytorch)\n\nThe followings are my related notebooks.\n\n* [UWMGI Image Segmentation EDA](https://www.kaggle.com/code/tt195361/uwmgi-image-segmentation-eda)\n* [UWMGI Image Segmentation Make TFRecords](https://www.kaggle.com/code/tt195361/uwmgi-image-segmentation-make-tfrecords)\n* [UWMGI Image Segmentation Inference](https://www.kaggle.com/code/tt195361/uwmgi-image-segmentation-inference)\n\n# Preparation","metadata":{"papermill":{"duration":0.053534,"end_time":"2022-06-25T10:40:10.63677","exception":false,"start_time":"2022-06-25T10:40:10.583236","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!git clone https://github.com/tt195361/TfDataAugmentation.git\n\nimport sys\nsys.path.append('TfDataAugmentation')\n\nimport TfDataAugmentation as Tfda","metadata":{"execution":{"iopub.execute_input":"2022-06-25T10:40:10.727873Z","iopub.status.busy":"2022-06-25T10:40:10.726565Z","iopub.status.idle":"2022-06-25T10:40:19.645816Z","shell.execute_reply":"2022-06-25T10:40:19.645055Z","shell.execute_reply.started":"2022-06-24T20:42:15.949058Z"},"papermill":{"duration":8.962183,"end_time":"2022-06-25T10:40:19.646005","exception":false,"start_time":"2022-06-25T10:40:10.683822","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%env SM_FRAMEWORK=tf.keras\n!pip install ../input/segmentation-models-keras/Keras_Applications-1.0.8-py3-none-any.whl --quiet\n!pip install ../input/segmentation-models-keras/image_classifiers-1.0.0-py3-none-any.whl --quiet\n!pip install ../input/segmentation-models-keras/efficientnet-1.0.0-py3-none-any.whl --quiet\n!pip install ../input/segmentation-models-keras/segmentation_models-1.0.1-py3-none-any.whl --quiet\n\nprint(\"Segmentation Models installed.\")","metadata":{"execution":{"iopub.execute_input":"2022-06-25T10:40:19.736683Z","iopub.status.busy":"2022-06-25T10:40:19.735929Z","iopub.status.idle":"2022-06-25T10:40:54.147526Z","shell.execute_reply":"2022-06-25T10:40:54.14682Z","shell.execute_reply.started":"2022-06-24T20:42:24.218591Z"},"papermill":{"duration":34.460867,"end_time":"2022-06-25T10:40:54.147676","exception":false,"start_time":"2022-06-25T10:40:19.686809","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEBUG = False","metadata":{"execution":{"iopub.execute_input":"2022-06-25T10:40:54.23782Z","iopub.status.busy":"2022-06-25T10:40:54.236926Z","iopub.status.idle":"2022-06-25T10:40:54.240019Z","shell.execute_reply":"2022-06-25T10:40:54.239505Z","shell.execute_reply.started":"2022-06-24T20:42:58.646289Z"},"papermill":{"duration":0.04979,"end_time":"2022-06-25T10:40:54.24021","exception":false,"start_time":"2022-06-25T10:40:54.19042","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport segmentation_models as sm\nimport tensorflow as tf\nimport tensorflow.keras.layers as L\nimport tensorflow.keras.backend as K\nfrom kaggle_datasets import KaggleDatasets\nimport os\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nimport math\nimport joblib\n\nprint(tf.__version__)","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2022-06-25T10:40:54.330166Z","iopub.status.busy":"2022-06-25T10:40:54.329293Z","iopub.status.idle":"2022-06-25T10:40:55.413091Z","shell.execute_reply":"2022-06-25T10:40:55.412307Z","shell.execute_reply.started":"2022-06-24T20:42:58.658382Z"},"papermill":{"duration":1.131578,"end_time":"2022-06-25T10:40:55.413268","exception":false,"start_time":"2022-06-25T10:40:54.28169","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SEG_MODEL = sm.FPN\nBACKBONE = 'efficientnetb4'\nIMAGE_SIZE = 640\nBATCH_SIZE = 64\nINIT_LR = 1e-4\nWARMUP_EPO = 2 if not DEBUG else 1\nCOSINE_EPO = 28 if not DEBUG else 2\nN_EPOCHS = WARMUP_EPO + COSINE_EPO\nN_FOLDS = 5\nOVERSAMPLE_DICE_THRESHOLD = 0.05\nOVERSAMPLE_COUNT = 3\nSTRIDE_25D = 2\n\nVID = 'V68'\nFOLD_I_LIST = [4]\nFOLD_I_LIST = FOLD_I_LIST[:2] if DEBUG else FOLD_I_LIST\n\nprint(\"N_EPOCHS:   \", N_EPOCHS)\nprint(\"FOLD_I_LIST:\", FOLD_I_LIST)","metadata":{"execution":{"iopub.execute_input":"2022-06-25T10:40:55.522314Z","iopub.status.busy":"2022-06-25T10:40:55.521328Z","iopub.status.idle":"2022-06-25T10:40:55.532333Z","shell.execute_reply":"2022-06-25T10:40:55.531601Z","shell.execute_reply.started":"2022-06-24T20:42:59.715751Z"},"papermill":{"duration":0.076611,"end_time":"2022-06-25T10:40:55.532483","exception":false,"start_time":"2022-06-25T10:40:55.455872","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_SRC = 'uwmgi-image-segmentation-tfrecords'\nAUTOTUNE = tf.data.experimental.AUTOTUNE","metadata":{"execution":{"iopub.execute_input":"2022-06-25T10:40:55.633576Z","iopub.status.busy":"2022-06-25T10:40:55.632804Z","iopub.status.idle":"2022-06-25T10:40:55.63554Z","shell.execute_reply":"2022-06-25T10:40:55.634865Z","shell.execute_reply.started":"2022-06-24T20:42:59.726604Z"},"papermill":{"duration":0.056803,"end_time":"2022-06-25T10:40:55.635684","exception":false,"start_time":"2022-06-25T10:40:55.578881","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TPU","metadata":{"papermill":{"duration":0.042077,"end_time":"2022-06-25T10:40:55.720449","exception":false,"start_time":"2022-06-25T10:40:55.678372","status":"completed"},"tags":[]}},{"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect() \n    strategy = tf.distribute.TPUStrategy(tpu)\nexcept ValueError: # otherwise detect GPUs\n    strategy = tf.distribute.MirroredStrategy() # single-GPU or multi-GPU\n    \nREPLICAS = strategy.num_replicas_in_sync\n\nprint(f\"Running on {REPLICAS} replicas\")","metadata":{"execution":{"iopub.execute_input":"2022-06-25T10:40:55.913423Z","iopub.status.busy":"2022-06-25T10:40:55.912334Z","iopub.status.idle":"2022-06-25T10:41:01.695143Z","shell.execute_reply":"2022-06-25T10:41:01.696825Z","shell.execute_reply.started":"2022-06-24T20:42:59.737159Z"},"papermill":{"duration":5.921932,"end_time":"2022-06-25T10:41:01.697169","exception":false,"start_time":"2022-06-25T10:40:55.775237","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GCS_DS_PATH = KaggleDatasets().get_gcs_path(DATA_SRC)\n\nGCS_DS_PATH","metadata":{"execution":{"iopub.execute_input":"2022-06-25T10:41:01.793051Z","iopub.status.busy":"2022-06-25T10:41:01.792324Z","iopub.status.idle":"2022-06-25T10:41:02.274499Z","shell.execute_reply":"2022-06-25T10:41:02.275033Z","shell.execute_reply.started":"2022-06-24T20:43:05.7916Z"},"papermill":{"duration":0.53271,"end_time":"2022-06-25T10:41:02.275252","exception":false,"start_time":"2022-06-25T10:41:01.742542","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{"papermill":{"duration":0.042612,"end_time":"2022-06-25T10:41:02.360583","exception":false,"start_time":"2022-06-25T10:41:02.317971","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def decode_image(image_bytes, height, width):\n    image_raw_bytes = tf.io.decode_raw(image_bytes, out_type=tf.uint8)\n    image_len = height * width\n    image_bytes = image_raw_bytes[-image_len: ]\n    image = tf.reshape(image_bytes, [width, height, 1])\n    image = tf.cast(image, dtype=tf.float32) / 255.0\n    return image\n    \ndef decode_mask(mask_bytes, height, width):\n    mask_png = tf.image.decode_png(mask_bytes)\n    # loaded image's shape is [width, height, channel]\n    mask_png = tf.reshape(mask_png, [width, height, 3])\n    mask_float = tf.cast(mask_png, dtype=tf.float32)\n    return mask_float\n\ndef resize_image(image):\n    resized_image = tf.image.resize(\n        image, [IMAGE_SIZE, IMAGE_SIZE],\n        method=tf.image.ResizeMethod.BILINEAR)\n    return resized_image","metadata":{"execution":{"iopub.execute_input":"2022-06-25T10:41:02.452345Z","iopub.status.busy":"2022-06-25T10:41:02.451589Z","iopub.status.idle":"2022-06-25T10:41:02.459468Z","shell.execute_reply":"2022-06-25T10:41:02.459938Z","shell.execute_reply.started":"2022-06-24T20:43:06.152376Z"},"papermill":{"duration":0.055369,"end_time":"2022-06-25T10:41:02.460144","exception":false,"start_time":"2022-06-25T10:41:02.404775","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_tfrecord(example):\n    TFREC_FORMAT = {\n        'id': tf.io.FixedLenFeature([], tf.string),\n        'case_no': tf.io.FixedLenFeature([], tf.int64),\n        'day_no': tf.io.FixedLenFeature([], tf.int64),\n        'slice_no': tf.io.FixedLenFeature([], tf.int64),\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'mask': tf.io.FixedLenFeature([], tf.string),\n        'fold': tf.io.FixedLenFeature([], tf.int64),\n        'height': tf.io.FixedLenFeature([], tf.int64),\n        'width': tf.io.FixedLenFeature([], tf.int64),\n        'space_h': tf.io.FixedLenFeature([], tf.float32),\n        'space_w': tf.io.FixedLenFeature([], tf.float32),\n        'large_bowel_dice_coef': tf.io.FixedLenFeature([], tf.float32),\n        'small_bowel_dice_coef': tf.io.FixedLenFeature([], tf.float32),\n        'stomach_dice_coef': tf.io.FixedLenFeature([], tf.float32),\n        'slice_count': tf.io.FixedLenFeature([], tf.int64),\n    }\n    \n    example = tf.io.parse_single_example(example, TFREC_FORMAT)\n    sample_id = example['id']\n    height = example['height']\n    width = example['width']\n    image = decode_image(example['image'], height, width)\n    mask = decode_mask(example['mask'], height, width)\n    fold = example['fold']\n    large_bowel_dice_coef = example['large_bowel_dice_coef']\n    small_bowel_dice_coef = example['small_bowel_dice_coef']\n    stomach_dice_coef = example['stomach_dice_coef']\n    slice_no = example['slice_no']\n    slice_count = example['slice_count']\n    \n    resized_image = resize_image(image)\n    resized_mask = resize_image(mask)\n    return resized_image, \\\n        (resized_mask, (sample_id, height, width), fold, \\\n        [large_bowel_dice_coef, small_bowel_dice_coef, stomach_dice_coef], \\\n        (slice_no, slice_count))\n\ndef make_raw_ds(filenames):\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=None)\n    dataset = dataset.map(read_tfrecord, num_parallel_calls=None)\n    return dataset","metadata":{"execution":{"iopub.execute_input":"2022-06-25T10:41:02.552351Z","iopub.status.busy":"2022-06-25T10:41:02.551598Z","iopub.status.idle":"2022-06-25T10:41:02.566379Z","shell.execute_reply":"2022-06-25T10:41:02.566865Z","shell.execute_reply.started":"2022-06-24T20:43:06.164269Z"},"papermill":{"duration":0.063669,"end_time":"2022-06-25T10:41:02.567089","exception":false,"start_time":"2022-06-25T10:41:02.50342","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tfrec_file_pattern = os.path.join(GCS_DS_PATH, '*.tfrec')\ntfrec_file_names = sorted(tf.io.gfile.glob(tfrec_file_pattern))\nraw_ds = make_raw_ds(tfrec_file_names)\n\nprint(raw_ds)","metadata":{"execution":{"iopub.execute_input":"2022-06-25T10:41:02.66323Z","iopub.status.busy":"2022-06-25T10:41:02.662459Z","iopub.status.idle":"2022-06-25T10:41:03.528791Z","shell.execute_reply":"2022-06-25T10:41:03.529347Z","shell.execute_reply.started":"2022-06-24T20:43:06.185002Z"},"papermill":{"duration":0.915427,"end_time":"2022-06-25T10:41:03.529544","exception":false,"start_time":"2022-06-25T10:41:02.614117","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Data in TFRecords are in order.\nfor ds_item in raw_ds.take(20):\n    _, (_, (sample_id, _, _), _, _, (slice_no, slice_count)) = ds_item\n    sample_id = sample_id.numpy().decode('utf-8')\n    slice_no = slice_no.numpy()\n    slice_count = slice_count.numpy()\n    print(\"{0}, {1:3d}, {2:3d}\".format(sample_id, slice_no, slice_count))","metadata":{"execution":{"iopub.execute_input":"2022-06-25T10:41:03.628886Z","iopub.status.busy":"2022-06-25T10:41:03.626674Z","iopub.status.idle":"2022-06-25T10:41:05.229706Z","shell.execute_reply":"2022-06-25T10:41:05.229103Z","shell.execute_reply.started":"2022-06-24T20:43:07.044157Z"},"papermill":{"duration":1.654631,"end_time":"2022-06-25T10:41:05.229849","exception":false,"start_time":"2022-06-25T10:41:03.575218","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2.5D","metadata":{"papermill":{"duration":0.044154,"end_time":"2022-06-25T10:41:05.318572","exception":false,"start_time":"2022-06-25T10:41:05.274418","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def make_image_ds(ds):\n    image_ds = ds.map(\n        lambda image, rest: image, num_parallel_calls=None)\n    return image_ds\n\ndef make_skip_ds(ds, n_skips):\n    take_ds = ds.take(n_skips)\n    skip_ds = ds \\\n        .skip(n_skips) \\\n        .concatenate(take_ds)\n    return skip_ds","metadata":{"execution":{"iopub.execute_input":"2022-06-25T10:41:05.416477Z","iopub.status.busy":"2022-06-25T10:41:05.415664Z","iopub.status.idle":"2022-06-25T10:41:05.418783Z","shell.execute_reply":"2022-06-25T10:41:05.419328Z","shell.execute_reply.started":"2022-06-24T20:43:08.893322Z"},"papermill":{"duration":0.054668,"end_time":"2022-06-25T10:41:05.419521","exception":false,"start_time":"2022-06-25T10:41:05.364853","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_25D_ds(image_m2, image_m1, image_zero_rest, image_p1, image_p2):\n    image_zero, rest = image_zero_rest\n    mask, size_info, fold, dice_coefs, slice_info = rest\n    slice_zero_no, slice_count = slice_info\n    image_array = tf.stack(\n        [image_m2, image_m1, image_zero, image_p1, image_p2])\n    \n    def _get_image(slice_no_offset):\n        slice_no = slice_zero_no + slice_no_offset\n        clipped_slice_no = tf.clip_by_value(slice_no, 1, slice_count)\n        slice_index = clipped_slice_no - slice_zero_no + STRIDE_25D\n        return image_array[slice_index]\n    \n    image_m = _get_image(-STRIDE_25D)\n    image_p = _get_image(STRIDE_25D)\n    image_25D = tf.concat([image_p, image_zero, image_m], axis=-1)\n    return image_25D, mask, size_info, fold, dice_coefs","metadata":{"execution":{"iopub.execute_input":"2022-06-25T10:41:05.518281Z","iopub.status.busy":"2022-06-25T10:41:05.517247Z","iopub.status.idle":"2022-06-25T10:41:05.520293Z","shell.execute_reply":"2022-06-25T10:41:05.519756Z","shell.execute_reply.started":"2022-06-24T20:43:08.90279Z"},"papermill":{"duration":0.055718,"end_time":"2022-06-25T10:41:05.520431","exception":false,"start_time":"2022-06-25T10:41:05.464713","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_m2_ds = make_image_ds(raw_ds)\nimage_m1_ds = make_skip_ds(make_image_ds(raw_ds), 1)\nimage_zero_ds = make_skip_ds(raw_ds, 2)\nimage_p1_ds = make_skip_ds(make_image_ds(raw_ds), 3)\nimage_p2_ds = make_skip_ds(make_image_ds(raw_ds), 4)\n\nraw_25D_ds = tf.data.Dataset.zip((\n    image_m2_ds, image_m1_ds, image_zero_ds, image_p1_ds, image_p2_ds)) \\\n    .map(make_25D_ds, num_parallel_calls=None)\n\nraw_25D_ds","metadata":{"execution":{"iopub.execute_input":"2022-06-25T10:41:05.63103Z","iopub.status.busy":"2022-06-25T10:41:05.629929Z","iopub.status.idle":"2022-06-25T10:41:05.854851Z","shell.execute_reply":"2022-06-25T10:41:05.854296Z","shell.execute_reply.started":"2022-06-24T20:43:08.91636Z"},"papermill":{"duration":0.285986,"end_time":"2022-06-25T10:41:05.854991","exception":false,"start_time":"2022-06-25T10:41:05.569005","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Folding and Oversampling","metadata":{"papermill":{"duration":0.044181,"end_time":"2022-06-25T10:41:05.944191","exception":false,"start_time":"2022-06-25T10:41:05.90001","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train_data_file_path = os.path.join(GCS_DS_PATH, 'train_data.csv')\ntrain_data_df = pd.read_csv(train_data_file_path)\nfold_count_dict = \\\n    train_data_df['fold'] \\\n        .value_counts() \\\n        .sort_index() \\\n        .to_dict()\n\nfold_count_dict","metadata":{"execution":{"iopub.execute_input":"2022-06-25T10:41:06.043356Z","iopub.status.busy":"2022-06-25T10:41:06.042241Z","iopub.status.idle":"2022-06-25T10:41:16.566314Z","shell.execute_reply":"2022-06-25T10:41:16.565617Z","shell.execute_reply.started":"2022-06-24T20:43:09.147079Z"},"papermill":{"duration":10.577617,"end_time":"2022-06-25T10:41:16.566487","exception":false,"start_time":"2022-06-25T10:41:05.98887","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oversample_count_df = pd.DataFrame()\noversample_count_df['fold'] = train_data_df['fold']\noversample_count_df['oversampled'] = \\\n    (train_data_df['large_bowel_dice_coef'] <= OVERSAMPLE_DICE_THRESHOLD) \\\n    | (train_data_df['small_bowel_dice_coef'] <= OVERSAMPLE_DICE_THRESHOLD) \\\n    | (train_data_df['stomach_dice_coef'] <= OVERSAMPLE_DICE_THRESHOLD)\n\noversample_count_df","metadata":{"execution":{"iopub.execute_input":"2022-06-25T10:41:16.680737Z","iopub.status.busy":"2022-06-25T10:41:16.680009Z","iopub.status.idle":"2022-06-25T10:41:16.707865Z","shell.execute_reply":"2022-06-25T10:41:16.708387Z","shell.execute_reply.started":"2022-06-24T20:43:19.904078Z"},"papermill":{"duration":0.082798,"end_time":"2022-06-25T10:41:16.708599","exception":false,"start_time":"2022-06-25T10:41:16.625801","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oversample_count_dict = \\\n    oversample_count_df \\\n        .groupby('fold') \\\n        .sum() \\\n        ['oversampled'] \\\n        .to_dict()\n\noversample_count_dict","metadata":{"execution":{"iopub.execute_input":"2022-06-25T10:41:16.81208Z","iopub.status.busy":"2022-06-25T10:41:16.811279Z","iopub.status.idle":"2022-06-25T10:41:16.82396Z","shell.execute_reply":"2022-06-25T10:41:16.824579Z","shell.execute_reply.started":"2022-06-24T20:43:19.937542Z"},"papermill":{"duration":0.069362,"end_time":"2022-06-25T10:41:16.824798","exception":false,"start_time":"2022-06-25T10:41:16.755436","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_train_count(fold_i):\n    count = 0\n    for i in range(N_FOLDS):\n        if i != fold_i:\n            count += fold_count_dict[i] + oversample_count_dict[i]\n    return count\n\ndef get_val_count(fold_i):\n    return fold_count_dict[fold_i]","metadata":{"execution":{"iopub.execute_input":"2022-06-25T10:41:16.92569Z","iopub.status.busy":"2022-06-25T10:41:16.924887Z","iopub.status.idle":"2022-06-25T10:41:16.929966Z","shell.execute_reply":"2022-06-25T10:41:16.930474Z","shell.execute_reply.started":"2022-06-24T20:43:19.963461Z"},"papermill":{"duration":0.057482,"end_time":"2022-06-25T10:41:16.930661","exception":false,"start_time":"2022-06-25T10:41:16.873179","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pick_image_mask_dice_coefs(image, mask, info, fold, dice_coefs):\n    return image, mask, dice_coefs\n\ndef pick_image_mask(image, mask, info, fold, dice_coefs):\n    return image, mask\n\ndef pick_image_mask_info(image, mask, info, fold, dice_coefs):\n    return image, mask, info\n\ndef select_train(ds, fold_i):\n    ds = ds.filter(lambda image, mask, info, fold, dice_coefs: fold != fold_i)\n    return ds\n    \ndef select_val(ds, fold_i):\n    ds = ds.filter(lambda image, mask, info, fold, dice_coefs: fold == fold_i)\n    return ds","metadata":{"execution":{"iopub.execute_input":"2022-06-25T10:41:17.029528Z","iopub.status.busy":"2022-06-25T10:41:17.028785Z","iopub.status.idle":"2022-06-25T10:41:17.035776Z","shell.execute_reply":"2022-06-25T10:41:17.036322Z","shell.execute_reply.started":"2022-06-24T20:43:19.972136Z"},"papermill":{"duration":0.057691,"end_time":"2022-06-25T10:41:17.036518","exception":false,"start_time":"2022-06-25T10:41:16.978827","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://stackoverflow.com/questions/47236465/\n# oversampling-functionality-in-tensorflow-dataset-api\n\ndef oversample(image, mask, dice_coefs):\n    repeat_count = tf.cond(\n        tf.math.reduce_any(\n            dice_coefs <= OVERSAMPLE_DICE_THRESHOLD),\n        lambda: OVERSAMPLE_COUNT,\n        lambda: 1)\n    repeat_count = tf.cast(repeat_count, dtype=tf.int64)\n    oversample_ds = \\\n        tf.data.Dataset.from_tensors((image, mask)) \\\n            .repeat(repeat_count)\n    return oversample_ds","metadata":{"execution":{"iopub.execute_input":"2022-06-25T10:41:17.138335Z","iopub.status.busy":"2022-06-25T10:41:17.134375Z","iopub.status.idle":"2022-06-25T10:41:17.140617Z","shell.execute_reply":"2022-06-25T10:41:17.1411Z","shell.execute_reply.started":"2022-06-24T20:43:19.986862Z"},"papermill":{"duration":0.056823,"end_time":"2022-06-25T10:41:17.14129","exception":false,"start_time":"2022-06-25T10:41:17.084467","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cut_size = IMAGE_SIZE // 10\n\ntransforms = Tfda.Compose([\n    Tfda.HorizontalFlip(p=0.5),\n    Tfda.VerticalFlip(p=0.5),\n    Tfda.RandomBrightnessContrast(\n        brightness_limit=0.2, contrast_limit=0.2, p=0.75),\n    Tfda.OneOf([\n        Tfda.GridDistortion(\n            num_steps=10, distort_limit=0.5,\n            interpolation='bilinear', border_mode='constant', p=0.5),        \n        Tfda.OpticalDistortion(\n            distort_limit=1.0, shift_limit=0.05,\n            interpolation='bilinear', border_mode='constant', p=0.5),\n        ], p=0.75),\n    Tfda.ShiftScaleRotate(\n        shift_limit=0.125, scale_limit=0.1, rotate_limit=20,\n        interpolation='bilinear', border_mode='constant', p=0.75),\n    Tfda.Cutout(\n        num_holes=8, max_h_size=cut_size, max_w_size=cut_size, p=0.75),\n])\n\ndef data_augment(image, mask):\n    result = transforms(image=image, mask=mask)\n    aug_image = result[\"image\"]\n    aug_mask = result[\"mask\"]\n    return aug_image, aug_mask","metadata":{"execution":{"iopub.execute_input":"2022-06-25T10:41:17.244775Z","iopub.status.busy":"2022-06-25T10:41:17.243749Z","iopub.status.idle":"2022-06-25T10:41:17.246786Z","shell.execute_reply":"2022-06-25T10:41:17.247333Z","shell.execute_reply.started":"2022-06-24T20:43:20.001204Z"},"papermill":{"duration":0.059635,"end_time":"2022-06-25T10:41:17.247535","exception":false,"start_time":"2022-06-25T10:41:17.1879","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def cast_image_to_uint8(image, mask):\n    image_ui8 = tf.cast(image * 255.0, dtype=tf.uint8)\n    return image_ui8, mask\n\ndef cast_image_to_float32(image, mask):\n    image_f32 = tf.cast(image, dtype=tf.float32) / 255.0\n    return image_f32, mask","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_datasets(fold_i):\n    # When caching, save memory by casting to uint8.\n    train_ds = select_train(raw_25D_ds, fold_i) \\\n        .map(pick_image_mask_dice_coefs, num_parallel_calls=AUTOTUNE) \\\n        .flat_map(oversample) \\\n        .map(cast_image_to_uint8, num_parallel_calls=AUTOTUNE) \\\n        .cache() \\\n        .map(cast_image_to_float32, num_parallel_calls=AUTOTUNE) \\\n        .repeat() \\\n        .shuffle(1024) \\\n        .map(data_augment, num_parallel_calls=AUTOTUNE) \\\n        .batch(BATCH_SIZE) \\\n        .prefetch(AUTOTUNE)\n        \n    val_ds = select_val(raw_25D_ds, fold_i) \\\n        .map(pick_image_mask, num_parallel_calls=AUTOTUNE) \\\n        .batch(BATCH_SIZE) \\\n        .map(cast_image_to_uint8, num_parallel_calls=AUTOTUNE) \\\n        .cache() \\\n        .map(cast_image_to_float32, num_parallel_calls=AUTOTUNE) \\\n        .prefetch(AUTOTUNE)\n    \n    train_steps = get_train_count(fold_i) // BATCH_SIZE\n    val_steps = get_val_count(fold_i) // BATCH_SIZE\n\n    return train_ds, val_ds, train_steps, val_steps","metadata":{"execution":{"iopub.execute_input":"2022-06-25T10:41:17.452547Z","iopub.status.busy":"2022-06-25T10:41:17.45168Z","iopub.status.idle":"2022-06-25T10:41:17.453713Z","shell.execute_reply":"2022-06-25T10:41:17.454335Z","shell.execute_reply.started":"2022-06-24T20:43:20.035112Z"},"papermill":{"duration":0.0577,"end_time":"2022-06-25T10:41:17.454512","exception":false,"start_time":"2022-06-25T10:41:17.396812","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_pred_dataset(fold_i):\n    pred_ds = select_val(raw_25D_ds, fold_i) \\\n        .map(pick_image_mask_info, num_parallel_calls=AUTOTUNE) \\\n        .batch(BATCH_SIZE) \\\n        .prefetch(AUTOTUNE)\n    return pred_ds","metadata":{"execution":{"iopub.execute_input":"2022-06-25T10:41:17.550693Z","iopub.status.busy":"2022-06-25T10:41:17.549965Z","iopub.status.idle":"2022-06-25T10:41:17.55432Z","shell.execute_reply":"2022-06-25T10:41:17.554934Z","shell.execute_reply.started":"2022-06-24T20:43:20.048519Z"},"papermill":{"duration":0.054175,"end_time":"2022-06-25T10:41:17.555148","exception":false,"start_time":"2022-06-25T10:41:17.500973","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualization","metadata":{"papermill":{"duration":0.045579,"end_time":"2022-06-25T10:41:17.647721","exception":false,"start_time":"2022-06-25T10:41:17.602142","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def draw_images_masks(ds):\n    rows = 6\n    cols = 5\n    n_imgs = cols * rows\n    plt.figure(figsize=(12, 2.5 * rows))\n    for i, (image, mask) in enumerate(ds.take(n_imgs)):\n        plt.subplot(rows, cols, i + 1)\n        plt.imshow(image, cmap=\"gray\")\n        plt.imshow(mask, alpha=0.5)\n        plt.axis(\"off\")\n\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.execute_input":"2022-06-25T10:41:17.743516Z","iopub.status.busy":"2022-06-25T10:41:17.742827Z","iopub.status.idle":"2022-06-25T10:41:17.749361Z","shell.execute_reply":"2022-06-25T10:41:17.749929Z","shell.execute_reply.started":"2022-06-24T20:43:20.064081Z"},"papermill":{"duration":0.056441,"end_time":"2022-06-25T10:41:17.750135","exception":false,"start_time":"2022-06-25T10:41:17.693694","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_ds, val_ds, _, _ = make_datasets(0)\n\n# print(\"train_ds\")\n# draw_images_masks(train_ds.unbatch().skip(95))\n\n# print(\"val_ds\")\n# draw_images_masks(val_ds.unbatch().skip(80))","metadata":{"execution":{"iopub.execute_input":"2022-06-25T10:41:17.848652Z","iopub.status.busy":"2022-06-25T10:41:17.847912Z","iopub.status.idle":"2022-06-25T10:41:17.850991Z","shell.execute_reply":"2022-06-25T10:41:17.851574Z","shell.execute_reply.started":"2022-06-24T20:43:20.07905Z"},"papermill":{"duration":0.05475,"end_time":"2022-06-25T10:41:17.851761","exception":false,"start_time":"2022-06-25T10:41:17.797011","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{"papermill":{"duration":0.045385,"end_time":"2022-06-25T10:41:17.943804","exception":false,"start_time":"2022-06-25T10:41:17.898419","status":"completed"},"tags":[]}},{"cell_type":"code","source":"dice_loss_fun = sm.losses.DiceLoss()\nbce_loss_fun = sm.losses.BinaryCELoss()\n\ndef bce_dice_loss(y_true, y_pred):\n    dice_loss = dice_loss_fun(y_true, y_pred)\n    bce_loss = bce_loss_fun(y_true, y_pred)\n    return 0.5 * dice_loss + 0.5 * bce_loss","metadata":{"execution":{"iopub.execute_input":"2022-06-25T10:41:18.041072Z","iopub.status.busy":"2022-06-25T10:41:18.04033Z","iopub.status.idle":"2022-06-25T10:41:18.044834Z","shell.execute_reply":"2022-06-25T10:41:18.04545Z","shell.execute_reply.started":"2022-06-24T20:43:20.091385Z"},"papermill":{"duration":0.05591,"end_time":"2022-06-25T10:41:18.04564","exception":false,"start_time":"2022-06-25T10:41:17.98973","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://www.kaggle.com/code/ammarnassanalhajali/uwmgi-unet-keras-train-with-eda\ndef dice_coef(y_true, y_pred, smooth=1):\n    y_true_f = K.flatten(y_true)\n    y_pred_f = K.flatten(y_pred)\n    intersection = K.sum(y_true_f * y_pred_f)\n    coef = (2. * intersection + smooth) \\\n        / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)\n    return coef","metadata":{"execution":{"iopub.execute_input":"2022-06-25T10:41:18.143435Z","iopub.status.busy":"2022-06-25T10:41:18.142683Z","iopub.status.idle":"2022-06-25T10:41:18.147978Z","shell.execute_reply":"2022-06-25T10:41:18.148553Z","shell.execute_reply.started":"2022-06-24T20:43:20.099326Z"},"papermill":{"duration":0.055814,"end_time":"2022-06-25T10:41:18.148769","exception":false,"start_time":"2022-06-25T10:41:18.092955","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_model():\n    seg_model = SEG_MODEL(\n        BACKBONE, encoder_weights='imagenet', \n        classes=3, activation='sigmoid')\n    \n    inputs = tf.keras.Input(\n        shape=(IMAGE_SIZE, IMAGE_SIZE, 3), name=\"inputs\")\n    outputs = seg_model(inputs)\n    model = tf.keras.Model(\n        inputs=inputs, outputs=outputs, name=\"seg_model\")\n\n    # 'steps_per_execution' instructs to send multiple batches to TPU\n    # at once. Each core in TPU should receive 128 elements.\n    steps_per_execution = 128 // (BATCH_SIZE // REPLICAS)\n    print(\"steps_per_execution: \", steps_per_execution)\n\n    model.compile(\n        optimizer=tf.keras.optimizers.Adam(),\n        loss=bce_dice_loss,\n        metrics=[dice_coef],\n        steps_per_execution=steps_per_execution)\n    return model","metadata":{"execution":{"iopub.execute_input":"2022-06-25T10:41:18.246714Z","iopub.status.busy":"2022-06-25T10:41:18.246006Z","iopub.status.idle":"2022-06-25T10:41:18.253159Z","shell.execute_reply":"2022-06-25T10:41:18.253739Z","shell.execute_reply.started":"2022-06-24T20:43:20.112424Z"},"papermill":{"duration":0.057738,"end_time":"2022-06-25T10:41:18.253958","exception":false,"start_time":"2022-06-25T10:41:18.19622","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    model = make_model()\n    \ninitial_weights = model.get_weights()\nmodel.summary()","metadata":{"execution":{"iopub.execute_input":"2022-06-25T10:41:18.356462Z","iopub.status.busy":"2022-06-25T10:41:18.353319Z","iopub.status.idle":"2022-06-25T10:41:51.326621Z","shell.execute_reply":"2022-06-25T10:41:51.325895Z","shell.execute_reply.started":"2022-06-24T20:43:20.127734Z"},"papermill":{"duration":33.025848,"end_time":"2022-06-25T10:41:51.326769","exception":false,"start_time":"2022-06-25T10:41:18.300921","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LR_START = INIT_LR\nLR_MAX = 1e-3\nLR_MIN = 1e-5\nLR_RAMPUP_EPOCHS = WARMUP_EPO\nLR_SUSTAIN_EPOCHS = 0\nEPOCHS = N_EPOCHS\n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        decay_total_epochs = EPOCHS - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS - 1\n        decay_epoch_index = epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS\n        phase = math.pi * decay_epoch_index / decay_total_epochs\n        cosine_decay = 0.5 * (1 + math.cos(phase))\n        lr = (LR_MAX - LR_MIN) * cosine_decay + LR_MIN\n    return lr\n\nrng = [i for i in range(EPOCHS)]\nlr_y = [lrfn(x) for x in rng]\nplt.figure(figsize=(10, 4))\nplt.plot(rng, lr_y)\nprint(\"Learning rate schedule: {:.3g} to {:.3g} to {:.3g}\". \\\n      format(lr_y[0], max(lr_y), lr_y[-1]))","metadata":{"execution":{"iopub.execute_input":"2022-06-25T10:41:51.44715Z","iopub.status.busy":"2022-06-25T10:41:51.446031Z","iopub.status.idle":"2022-06-25T10:41:51.710935Z","shell.execute_reply":"2022-06-25T10:41:51.710273Z","shell.execute_reply.started":"2022-06-24T20:43:51.738054Z"},"papermill":{"duration":0.32865,"end_time":"2022-06-25T10:41:51.711101","exception":false,"start_time":"2022-06-25T10:41:51.382451","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_callbacks(best_model_file_name):\n    cb_monitor = 'val_loss'\n    cb_mode = 'min'\n    cb_verbose = 1\n\n    checkpoint = tf.keras.callbacks.ModelCheckpoint(\n        best_model_file_name, save_best_only=True,\n        save_weights_only=False, monitor=cb_monitor, mode=cb_mode,\n        verbose=cb_verbose)\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    \n    return [checkpoint, lr_callback]","metadata":{"execution":{"iopub.execute_input":"2022-06-25T10:41:51.83116Z","iopub.status.busy":"2022-06-25T10:41:51.830455Z","iopub.status.idle":"2022-06-25T10:41:51.833566Z","shell.execute_reply":"2022-06-25T10:41:51.833011Z","shell.execute_reply.started":"2022-06-24T20:43:52.031162Z"},"papermill":{"duration":0.066092,"end_time":"2022-06-25T10:41:51.833728","exception":false,"start_time":"2022-06-25T10:41:51.767636","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fit_one_fold(fold_i, best_model_file_name):\n    train_dataset, val_dataset, train_steps, val_steps = make_datasets(fold_i)\n    callbacks = make_callbacks(best_model_file_name)\n\n    history = model.fit(\n        train_dataset, \n        epochs=EPOCHS,\n        verbose=1,\n        callbacks=callbacks,\n        steps_per_epoch=train_steps,\n        validation_data=val_dataset,\n        validation_steps=val_steps)\n    return history","metadata":{"execution":{"iopub.execute_input":"2022-06-25T10:41:51.952986Z","iopub.status.busy":"2022-06-25T10:41:51.95196Z","iopub.status.idle":"2022-06-25T10:41:51.954953Z","shell.execute_reply":"2022-06-25T10:41:51.954414Z","shell.execute_reply.started":"2022-06-24T20:43:52.041365Z"},"papermill":{"duration":0.065349,"end_time":"2022-06-25T10:41:51.955123","exception":false,"start_time":"2022-06-25T10:41:51.889774","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_history(history, title, labels, subplot):\n    plt.subplot(*subplot)\n    plt.title(title)\n    for label in labels:\n        plt.plot(history.history[label], label=label)\n    plt.legend()\n    \ndef plot_fit_result(history):\n    plt.figure(figsize=(12, 4))\n    plot_history(history, \"Loss\", ['loss', 'val_loss'], (1, 2, 1))\n    plot_history(history, \"dice_coef\", ['dice_coef', 'val_dice_coef'], (1, 2, 2))\n    plt.show()","metadata":{"execution":{"iopub.execute_input":"2022-06-25T10:41:52.073959Z","iopub.status.busy":"2022-06-25T10:41:52.073288Z","iopub.status.idle":"2022-06-25T10:41:52.08001Z","shell.execute_reply":"2022-06-25T10:41:52.080827Z","shell.execute_reply.started":"2022-06-24T20:43:52.053621Z"},"papermill":{"duration":0.068999,"end_time":"2022-06-25T10:41:52.08102","exception":false,"start_time":"2022-06-25T10:41:52.012021","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def resize_image_to(image, height, width):\n    resized_image = tf.image.resize(\n        image, [width, height],\n        method=tf.image.ResizeMethod.BILINEAR)\n    return resized_image","metadata":{"execution":{"iopub.execute_input":"2022-06-25T10:41:52.196448Z","iopub.status.busy":"2022-06-25T10:41:52.195709Z","iopub.status.idle":"2022-06-25T10:41:52.199753Z","shell.execute_reply":"2022-06-25T10:41:52.200277Z","shell.execute_reply.started":"2022-06-24T20:43:52.066874Z"},"papermill":{"duration":0.063481,"end_time":"2022-06-25T10:41:52.200465","exception":false,"start_time":"2022-06-25T10:41:52.136984","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def encode_pred(one_pred):\n    curr_pred = one_pred.flatten()\n    \n    prev_pred = np.empty_like(curr_pred)\n    prev_pred[1:] = curr_pred[:-1]\n    prev_pred[0] = 0\n    \n    next_pred = np.empty_like(curr_pred)\n    next_pred[:-1] = curr_pred[1:]\n    next_pred[-1] = 0\n    \n    pixel_no = np.arange(len(curr_pred))\n    start_pixels = pixel_no[(prev_pred == 0) & (curr_pred == 1)]\n    end_pixels = pixel_no[(curr_pred == 1) & (next_pred == 0)]\n    \n    encode_list = []\n    for start_pixel, end_pixel in zip(start_pixels, end_pixels):\n        encode_list.append(str(start_pixel))\n        encode_list.append(str(end_pixel - start_pixel + 1))\n    \n    encoded_pred = ' '.join(encode_list)\n    return encoded_pred","metadata":{"execution":{"iopub.execute_input":"2022-06-25T10:41:52.32391Z","iopub.status.busy":"2022-06-25T10:41:52.323202Z","iopub.status.idle":"2022-06-25T10:41:52.326754Z","shell.execute_reply":"2022-06-25T10:41:52.326125Z","shell.execute_reply.started":"2022-06-24T20:43:52.082604Z"},"papermill":{"duration":0.068292,"end_time":"2022-06-25T10:41:52.326901","exception":false,"start_time":"2022-06-25T10:41:52.258609","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_predictions(raw_pred, height, width):\n    resized_image = resize_image_to(raw_pred, height, width)\n    bin_pred = np.where(resized_image >= 0.5, 1, 0)\n    large_bowel_pred = encode_pred(bin_pred[:, :, 0])\n    small_bowel_pred = encode_pred(bin_pred[:, :, 1])\n    stomach_pred = encode_pred(bin_pred[:, :, 2])\n    return large_bowel_pred, small_bowel_pred, stomach_pred","metadata":{"execution":{"iopub.execute_input":"2022-06-25T10:41:52.447873Z","iopub.status.busy":"2022-06-25T10:41:52.447201Z","iopub.status.idle":"2022-06-25T10:41:52.450094Z","shell.execute_reply":"2022-06-25T10:41:52.449465Z","shell.execute_reply.started":"2022-06-24T20:43:52.095724Z"},"papermill":{"duration":0.066486,"end_time":"2022-06-25T10:41:52.450239","exception":false,"start_time":"2022-06-25T10:41:52.383753","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict_one_fold(fold_i):\n    pred_ds = make_pred_dataset(fold_i)\n    pred_batch_list = []\n    pred_list = []\n    for i, (image_batch, mask_batch, info_batch) in enumerate(pred_ds):\n        if DEBUG and 3 <= i:\n            break\n        print('.', end='', flush=True)\n\n        pred_batch = model(image_batch, training=False)\n        pred_batch_list.append(pred_batch)\n        \n        sample_id_batch, height_batch, width_batch = info_batch\n        for pred, sample_id, height, width in \\\n                zip(pred_batch, sample_id_batch, height_batch, width_batch):\n            sample_id = sample_id.numpy().decode('utf-8')\n            height = height.numpy()\n            width = width.numpy()\n        \n            large_bowel_pred, small_bowel_pred, stomach_pred = \\\n                make_predictions(pred, height, width)\n            pred_list.append([\n                sample_id, large_bowel_pred, small_bowel_pred, \n                stomach_pred, fold_i])\n    print()\n    \n    preds = np.concatenate(pred_batch_list, axis=0)\n    pred_df = pd.DataFrame(\n        pred_list, \n        columns=['id', 'large_bowel', 'small_bowel', 'stomach', 'fold'])\n    return preds, pred_df","metadata":{"execution":{"iopub.execute_input":"2022-06-25T10:41:52.573606Z","iopub.status.busy":"2022-06-25T10:41:52.572818Z","iopub.status.idle":"2022-06-25T10:41:52.576021Z","shell.execute_reply":"2022-06-25T10:41:52.575426Z","shell.execute_reply.started":"2022-06-24T20:43:52.10764Z"},"papermill":{"duration":0.070085,"end_time":"2022-06-25T10:41:52.576194","exception":false,"start_time":"2022-06-25T10:41:52.506109","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def save_binary(name, bin_file, file_name_format):\n    file_name = file_name_format.format(VID, fold_i)\n    joblib.dump(bin_file, file_name)\n    print(\"{0} are saved to {1}.\".format(name, file_name))\n    \ndef save_df(name, df, file_name_format):\n    file_name = file_name_format.format(VID, fold_i)\n    df.to_csv(file_name, index=False)\n    print(\"{0} is saved to {1}.\".format(name, file_name))","metadata":{"execution":{"iopub.execute_input":"2022-06-25T10:41:52.696748Z","iopub.status.busy":"2022-06-25T10:41:52.69602Z","iopub.status.idle":"2022-06-25T10:41:52.697719Z","shell.execute_reply":"2022-06-25T10:41:52.698347Z","shell.execute_reply.started":"2022-06-24T20:43:52.122101Z"},"papermill":{"duration":0.064665,"end_time":"2022-06-25T10:41:52.698524","exception":false,"start_time":"2022-06-25T10:41:52.633859","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for fold_i in FOLD_I_LIST:\n    print(\"####################\")\n    print(\"# Fold {0}\".format(fold_i))\n    model.set_weights(initial_weights)\n    best_model_file_name = \"seg_model_{0}_{1}.hdf5\".format(VID, fold_i)\n    history = fit_one_fold(fold_i, best_model_file_name)\n    plot_fit_result(history)\n    \n    # model.load_weights(best_model_file_name)\n    # preds, pred_df = predict_one_fold(fold_i)\n    \n    # save_binary(\"preds\", preds, \"preds_{0}_{1}.joblib\")\n    # save_df(\"pred_df\", pred_df, \"pred_{0}_{1}.csv\")\n    print()","metadata":{"execution":{"iopub.execute_input":"2022-06-25T10:41:52.813604Z","iopub.status.busy":"2022-06-25T10:41:52.812901Z","iopub.status.idle":"2022-06-25T13:38:14.083085Z","shell.execute_reply":"2022-06-25T13:38:14.080032Z","shell.execute_reply.started":"2022-06-24T20:43:52.138653Z"},"papermill":{"duration":10581.328797,"end_time":"2022-06-25T13:38:14.083307","exception":false,"start_time":"2022-06-25T10:41:52.75451","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rf TfDataAugmentation","metadata":{"execution":{"iopub.execute_input":"2022-06-25T13:38:14.867408Z","iopub.status.busy":"2022-06-25T13:38:14.866371Z","iopub.status.idle":"2022-06-25T13:38:15.729278Z","shell.execute_reply":"2022-06-25T13:38:15.728544Z","shell.execute_reply.started":"2022-06-24T21:28:47.175738Z"},"papermill":{"duration":1.257351,"end_time":"2022-06-25T13:38:15.729427","exception":false,"start_time":"2022-06-25T13:38:14.472076","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}