{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":61446,"databundleVersionId":6962461,"sourceType":"competition"}],"dockerImageVersionId":30635,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# import rasterio\n# from rasterio.plot import show\n# from rasterio.plot import show_hist\n\nfrom tifffile import tifffile\n\nfrom matplotlib import pyplot as plt\nimport tensorflow as tf\n\nimport numpy as np # linear algebra\nfrom numpy.random import default_rng\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport time\nimport os\nimport gc\nimport time\n\nfrom tqdm import tqdm\nimport albumentations as A   # For image data augmentation\n\nfrom glob import glob\nfrom datetime import datetime\n\n\nKIDNEY_1_DENSE_SHAPE = (1303, 912)\nKIDNEY_1_VOI_SHAPE = (1928, 1928)\nKIDNEY_2_SHAPE = (1041, 1511)\nKIDNEY_3_SPARSE_SHAPE = (1706, 1510)\nKIDNEY_5_SHAPE = (1303, 912)\nKIDNEY_6_SHAPE = (1303, 912)\n\nMIN_HEIGHT = 512\nMIN_WIDTH = 512\nCROPPING_SHAPE = (MIN_HEIGHT, MIN_WIDTH)\n\nINPUT_SHAPE = (1, CROPPING_SHAPE[0], CROPPING_SHAPE[1], 1)\nINPUT_SHAPE_VALIDATION = (1, CROPPING_SHAPE[0], CROPPING_SHAPE[1], 1)\nOUTPUT_SHAPE = INPUT_SHAPE\n\n\nTRAINING_SAMPLES_FRACTION = 1. # Set this to 1. when submitting, .001 when just running\nVALIDATION_SAMPLES_FRACTION = 1.  # Set this to 1. when submitting, .125 when just running\nTESTING_SAMPLES_FRACTION = 1.\n\n\ndef read_image(arg_image_path):\n    \"\"\"\n    Read the image and return the numpy array\n    \"\"\"\n    # return rasterio.open(arg_image_path).read()\n    return tifffile.imread(arg_image_path)\n\n\ndef get_num_samples_from_dir(arg_dir):\n    return len(os.listdir(arg_dir))\n\n\ndef convert_2D_target_to_hot_encoded(arg_array, num_classes):\n    arg_array = np.tile(arg_array, reps=num_classes)\n    for id_class in range(0, num_classes):\n        arg_array[arg_array[:, :, :, 0] == id_class] = [0] * (num_classes - 1 - id_class) + [1] + [0] * id_class\n    return arg_array\n","metadata":{"execution":{"iopub.status.busy":"2024-01-16T05:55:27.368152Z","iopub.execute_input":"2024-01-16T05:55:27.369135Z","iopub.status.idle":"2024-01-16T05:55:45.212409Z","shell.execute_reply.started":"2024-01-16T05:55:27.369083Z","shell.execute_reply":"2024-01-16T05:55:45.211069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create the augmentation pipeline:\n\ntransform = A.Compose([\n    A.HorizontalFlip(p=0.5),\n    A.VerticalFlip(p=0.5),\n    A.RandomRotate90(p=0.5),\n    A.ElasticTransform(p=.5, alpha=240, sigma=120 * 0.05, alpha_affine=120 * 0.03),\n    A.GridDistortion(p=.5),\n#     A.OpticalDistortion(distort_limit=2, shift_limit=0.5, p=.5)\n])\n\ndef apply_augmentation(input_image, input_mask=None):\n    if input_mask is None:\n        return transform(image=input_image)['image']\n    else:\n        augmen = transform(image=input_image, mask=input_mask)\n        return augmen['image'], augmen['mask']","metadata":{"execution":{"iopub.status.busy":"2024-01-16T05:55:45.215511Z","iopub.execute_input":"2024-01-16T05:55:45.216456Z","iopub.status.idle":"2024-01-16T05:55:45.226804Z","shell.execute_reply.started":"2024-01-16T05:55:45.216377Z","shell.execute_reply":"2024-01-16T05:55:45.225401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# initialize some setup variables:\nIS_KAGGLE = True\n\nNUM_CLASSES = 2\n\nif IS_KAGGLE:\n    KIDNEY_1_DENSE_PATH = \"/kaggle/input/blood-vessel-segmentation/train/kidney_1_dense\"\n    KIDNEY_1_VOI_PATH = \"/kaggle/input/blood-vessel-segmentation/train/kidney_1_voi\"\n    KIDNEY_2_PATH = \"/kaggle/input/blood-vessel-segmentation/train/kidney_2\"\n    KIDNEY_3_SPARSE_PATH = \"/kaggle/input/blood-vessel-segmentation/train/kidney_3_sparse\"\n    KIDNEY_5_PATH = \"/kaggle/input/blood-vessel-segmentation/test/kidney_5\"\n    KIDNEY_6_PATH = \"/kaggle/input/blood-vessel-segmentation/test/kidney_6\"\nelse:\n    KIDNEY_1_DENSE_PATH = \"kidney_1_dense\"\n    KIDNEY_1_VOI_PATH = \"kidney_1_voi\"\n    KIDNEY_2_PATH = \"kidney_2\"\n    KIDNEY_3_SPARSE_PATH = \"kidney_3_sparse\"\n    KIDNEY_5_PATH = \"kidney_5\"\n    KIDNEY_6_PATH = \"kidney_6\"\n    \nKIDNEY_1_DENSE_IMAGES_PATH = f\"{KIDNEY_1_DENSE_PATH}/images\"\nKIDNEY_1_DENSE_LABELS_PATH = f\"{KIDNEY_1_DENSE_PATH}/labels\"\n\nKIDNEY_1_VOI_IMAGES_PATH = f\"{KIDNEY_1_VOI_PATH}/images\"\nKIDNEY_1_VOI_LABELS_PATH = f\"{KIDNEY_1_VOI_PATH}/labels\"\n\nKIDNEY_2_IMAGES_PATH = f\"{KIDNEY_2_PATH}/images\"\nKIDNEY_2_LABELS_PATH = f\"{KIDNEY_2_PATH}/labels\"\n\nKIDNEY_3_SPARSE_IMAGES_PATH = f\"{KIDNEY_3_SPARSE_PATH}/images\"\nKIDNEY_3_SPARSE_LABELS_PATH = f\"{KIDNEY_3_SPARSE_PATH}/labels\"\n\nSIMPLE_UNET_MODEL_CHECKPOINT = \"simple_unet\"\nOLD_UNET_MODEL_CHECKPOINT = \"simple_unet_2.keras\"\n\nEPOCHS = 3      # Set this to 200 when submitting\nBATCH_SIZE = 40\nPATIENCE = 5\n\nMODEL_REGEX = \"model_\"","metadata":{"execution":{"iopub.status.busy":"2024-01-16T05:55:45.229177Z","iopub.execute_input":"2024-01-16T05:55:45.229962Z","iopub.status.idle":"2024-01-16T05:55:45.254109Z","shell.execute_reply.started":"2024-01-16T05:55:45.229914Z","shell.execute_reply":"2024-01-16T05:55:45.253191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FULL_METAINFO = {\n    'training_metainfo': {\n        0: {\n            'images_path': KIDNEY_1_DENSE_IMAGES_PATH,\n            'general_path': KIDNEY_1_DENSE_PATH,\n            'labels_path': KIDNEY_1_DENSE_LABELS_PATH,\n            'set_name' : 'kidney_1_dense'\n        }\n    },\n    'validation_metainfo': {\n        0: {\n            'images_path' : KIDNEY_3_SPARSE_IMAGES_PATH,\n            'labels_path' : KIDNEY_3_SPARSE_LABELS_PATH,\n            'set_name' : 'kidney_3_sparse'\n        }\n    },\n    'testing_metainfo': {\n        0: {\n            'images_path' : f\"{KIDNEY_5_PATH}/images\",\n            'general_path' : KIDNEY_5_PATH,\n            'labels_path' : f\"{KIDNEY_5_PATH}/labels\",\n            'set_name' : 'kidney_5'\n        },\n        1: {\n            'images_path' : f'{KIDNEY_6_PATH}/images',\n            'general_path' : KIDNEY_6_PATH,\n            'labels_path' : f'{KIDNEY_6_PATH}/labels',\n            'set_name' : 'kidney_6'\n        }\n    }\n}\n\nif IS_KAGGLE:\n    FULL_METAINFO['training_metainfo'][1] = {\n        'images_path' : KIDNEY_1_VOI_IMAGES_PATH,\n        'general_path' : KIDNEY_1_VOI_PATH,\n        'labels_path' : KIDNEY_1_VOI_LABELS_PATH,\n        'set_name' : 'kidney_1_voi'\n    }\n    FULL_METAINFO['training_metainfo'][2] = {\n        'images_path' : KIDNEY_2_IMAGES_PATH,\n        'general_path' : KIDNEY_2_PATH,\n        'labels_path' : KIDNEY_2_LABELS_PATH,\n        'set_name' : 'kidney_2'\n    }\n\nTEST_SETS_PATH = \"/kaggle/input/blood-vessel-segmentation/test\"","metadata":{"execution":{"iopub.status.busy":"2024-01-16T05:55:45.257483Z","iopub.execute_input":"2024-01-16T05:55:45.258411Z","iopub.status.idle":"2024-01-16T05:55:45.272711Z","shell.execute_reply.started":"2024-01-16T05:55:45.258353Z","shell.execute_reply":"2024-01-16T05:55:45.271530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess_image(new_array, is_target=False, is_validation=False):\n    new_array = np.resize(new_array, CROPPING_SHAPE)\n    new_array = new_array.reshape(INPUT_SHAPE)\n    if is_target:\n        new_array = convert_2D_target_to_hot_encoded(new_array, NUM_CLASSES)\n    else:\n        new_array = new_array.astype(float) / 255.0\n#     if is_validation:\n#         new_array = new_array.reshape((1, new_array.shape[0], new_array.shape[1], new_array.shape[2]))\n    return new_array","metadata":{"execution":{"iopub.status.busy":"2024-01-16T05:55:45.274751Z","iopub.execute_input":"2024-01-16T05:55:45.275640Z","iopub.status.idle":"2024-01-16T05:55:45.289261Z","shell.execute_reply.started":"2024-01-16T05:55:45.275594Z","shell.execute_reply":"2024-01-16T05:55:45.287897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_labels_path(arg_path):\n    return os.path.join(arg_path, 'labels')\n\n\ndef get_images_path(arg_path):\n    return os.path.join(arg_path, 'images')\n\n\nclass ImageCollector:\n    \n    \n    @staticmethod\n    def get_sorted_items(arg_path):\n        filenames = [os.path.join(arg_path, filename) for filename in sorted(os.listdir(arg_path))]\n        return filenames\n\n    \n    @staticmethod\n    def preprocess_pair(arg_tuple):\n        \"\"\"\n        Preprocess tuples of form (image array, label array)\n        \"\"\"\n        # here, arg_path must be a general path\n        new_image, new_mask = apply_augmentation(input_image=arg_tuple[0], input_mask=arg_tuple[1])\n        return (\n            preprocess_image(new_image, is_target=False),\n            preprocess_image(new_mask, is_target=True)\n        )\n            \n    \n    @staticmethod\n    def generate_image_label_pairs(arg_path, generate_labels, fraction):\n        if generate_labels:\n            full_set = list(zip(\n                ImageCollector.get_sorted_items(get_images_path(arg_path)), \n                ImageCollector.get_sorted_items(get_labels_path(arg_path))\n            ))\n        else:\n            full_set = ImageCollector.get_sorted_items(get_images_path(arg_path))\n        rng = default_rng()\n        indices = list(rng.choice(len(full_set), int(len(full_set) * fraction), replace=False))\n        full_set = [full_set[index] for index in indices]\n        del rng\n        del indices\n        for tpl in full_set:\n            if generate_labels:\n                yield ImageCollector.preprocess_pair(\n                    (\n                        read_image(tpl[0]), \n                        read_image(tpl[1])\n                    )\n                )\n            else:\n                yield preprocess_image(apply_augmentation(input_image=read_image(tpl), input_mask=None), is_target=False)\n        \n    \n    @staticmethod\n    def generate_images(is_training):\n        # This generator should also generate at the same time the labels (but how does that work?)\n        if is_training:\n            mode_key = 'training_metainfo'\n            fraction = TRAINING_SAMPLES_FRACTION\n            generate_label = True\n        else:\n            mode_key = 'testing_metainfo'\n            fraction = TESTING_SAMPLES_FRACTION\n            generate_label = False\n        for set_key in sorted(FULL_METAINFO[mode_key].keys()):\n            for tpl in ImageCollector.generate_image_label_pairs(FULL_METAINFO[mode_key][set_key]['general_path'], generate_label, fraction):\n                yield tpl\n        del generate_label\n        del mode_key\n        del fraction\n    \n    \n    @staticmethod\n    def generate_training_set():\n        for tpl in ImageCollector.generate_images(is_training=True):\n            yield tpl\n    \n    \n    @staticmethod\n    def generate_testing_set():\n#         for tpl in ImageCollector.generate_images(is_training=False):\n#             yield tpl\n        testing_sets = sorted(os.listdir(TEST_SETS_PATH))\n        for current_set_name in testing_sets:\n            test_set_path = os.path.join(TEST_SETS_PATH, current_set_name)\n            for tpl in ImageCollector.generate_image_label_pairs(test_set_path, generate_labels=False, fraction=TESTING_SAMPLES_FRACTION):\n                yield tpl\n            del test_set_path\n        del testing_sets","metadata":{"execution":{"iopub.status.busy":"2024-01-16T05:55:45.291102Z","iopub.execute_input":"2024-01-16T05:55:45.291900Z","iopub.status.idle":"2024-01-16T05:55:45.312977Z","shell.execute_reply.started":"2024-01-16T05:55:45.291848Z","shell.execute_reply":"2024-01-16T05:55:45.312037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read the validation set:\ndef read_images_as_nparray(arg_path, image_indices=None, is_target=False, is_validation=False):\n    filenames = sorted(os.listdir(arg_path))\n    if image_indices is not None:\n        filenames = [filenames[index] for index in image_indices]\n    result_arr = None\n    for index, filename in tqdm(enumerate(filenames)):\n        new_array = read_image(os.path.join(arg_path, filename))\n        new_array = preprocess_image(new_array, is_target, is_validation)\n        if index == 0:\n            result_arr = new_array\n        else:\n            result_arr = np.concatenate(\n                [result_arr, new_array], \n                axis=0)\n    del filenames\n    return result_arr\n\n\ndef create_validation_set():\n    num_samples = get_num_samples_from_dir(FULL_METAINFO['validation_metainfo'][0]['images_path'])\n    rng = default_rng()\n    indices = list(rng.choice(num_samples, int(num_samples * VALIDATION_SAMPLES_FRACTION), replace=False))\n    del rng\n    return (\n        read_images_as_nparray(FULL_METAINFO['validation_metainfo'][0]['images_path'], indices, is_target=False, is_validation=True),\n        read_images_as_nparray(FULL_METAINFO['validation_metainfo'][0]['labels_path'], indices, is_target=True, is_validation=True)\n    )\n\n\ndef read_val_images_and_labels():\n    rng = default_rng()\n    num_samples = get_num_samples_from_dir(FULL_METAINFO['validation_metainfo'][0]['images_path'])\n    indices = list(rng.choice(num_samples, int(num_samples * VALIDATION_SAMPLES_FRACTION), replace=False))\n    del rng\n    del num_samples\n    image_filenames = sorted(os.listdir(FULL_METAINFO['validation_metainfo'][0]['images_path']))\n    image_filenames = [image_filenames[index] for index in indices]\n    label_filenames = sorted(os.listdir(FULL_METAINFO['validation_metainfo'][0]['labels_path']))\n    label_filenames = [label_filenames[index] for index in indices]\n    valid_set_x = None\n    valid_set_y = None\n    for i, tpl in tqdm(enumerate(zip(image_filenames, label_filenames))):\n        new_image = read_image(os.path.join(FULL_METAINFO['validation_metainfo'][0]['images_path'], tpl[0]))\n        new_label = read_image(os.path.join(FULL_METAINFO['validation_metainfo'][0]['labels_path'], tpl[1]))\n        new_image, new_label = ImageCollector.preprocess_pair(arg_tuple=(new_image, new_label))\n        if i == 0:\n            valid_set_x = new_image\n            valid_set_y = new_label\n        else:\n            valid_set_x = np.concatenate([valid_set_x, new_image], axis=0)\n            valid_set_y = np.concatenate([valid_set_y, new_label], axis=0)\n        del new_image\n        del new_label\n    del indices\n    del image_filenames\n    del label_filenames\n    return valid_set_x, valid_set_y\n\nvalidation_set_x, validation_set_y = read_val_images_and_labels()","metadata":{"execution":{"iopub.status.busy":"2024-01-16T05:55:45.314564Z","iopub.execute_input":"2024-01-16T05:55:45.315314Z","iopub.status.idle":"2024-01-16T06:15:24.467994Z","shell.execute_reply.started":"2024-01-16T05:55:45.315270Z","shell.execute_reply":"2024-01-16T06:15:24.465954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# MobileNetV2:\ninputs = tf.keras.layers.Input(shape=(INPUT_SHAPE[1], INPUT_SHAPE[2], 1), name=\"input_image\")\nencoder = tf.keras.applications.mobilenet_v2.MobileNetV2(input_tensor=inputs, weights=None, include_top=False, alpha=0.35, classifier_activation='softmax')\nskip_connection_1 = encoder.get_layer(\"input_image\").output\nskip_connection_2 = encoder.get_layer(\"block_1_expand_relu\").output\nskip_connection_3 = encoder.get_layer(\"block_3_expand_relu\").output\nskip_connection_4 = encoder.get_layer(\"block_6_expand_relu\").output\nencoder_output = encoder.get_layer(\"block_13_expand_relu\").output\n\n# The rest of the U-Net:\ndef create_expansion_block(input_layer, skip_connection, num_out_channels, upsample_size, \n                           zero_padding_size=None, cropping_size=None):\n    result = tf.keras.layers.UpSampling2D(upsample_size)(input_layer)\n    if cropping_size is not None:\n        result = tf.keras.layers.Cropping2D(cropping_size)(result)\n    if zero_padding_size is not None:\n        result = tf.keras.layers.ZeroPadding2D(zero_padding_size)(result)\n    result = tf.keras.layers.Concatenate()([result, skip_connection])\n    result = tf.keras.layers.SeparableConv2D(filters=num_out_channels, kernel_size=(3, 3), padding='same')(result)\n    result = tf.keras.layers.BatchNormalization()(result)\n    result = tf.keras.layers.Activation('relu6')(result)\n    result = tf.keras.layers.SeparableConv2D(filters=num_out_channels, kernel_size=(3, 3), padding='same')(result)\n    result = tf.keras.layers.BatchNormalization()(result)\n    result = tf.keras.layers.Activation('relu6')(result)\n    return result\n\nexpansion_block_1 = create_expansion_block(\n    encoder_output, \n    skip_connection_4, \n    num_out_channels=128, \n    upsample_size=2,\n)\nexpansion_block_2 = create_expansion_block(\n    expansion_block_1, \n    skip_connection_3, \n    num_out_channels=64, \n    upsample_size=2,\n)\nexpansion_block_3 = create_expansion_block(\n    expansion_block_2, \n    skip_connection_2, \n    num_out_channels=32, \n    upsample_size=2,\n)\nexpansion_block_4 = create_expansion_block(\n    expansion_block_3,\n    skip_connection_1,\n    num_out_channels=16, \n    upsample_size=2,\n)\n\nconv_layer_2 = tf.keras.layers.Conv2D(filters=NUM_CLASSES, kernel_size=(1, 1), padding='same', activation='softmax')(expansion_block_4)\n\nmodel = tf.keras.models.Model(inputs=inputs, outputs=conv_layer_2, name='my_simple_mobile_unet')","metadata":{"execution":{"iopub.status.busy":"2024-01-16T06:15:24.471083Z","iopub.execute_input":"2024-01-16T06:15:24.472105Z","iopub.status.idle":"2024-01-16T06:15:27.003162Z","shell.execute_reply.started":"2024-01-16T06:15:24.472047Z","shell.execute_reply":"2024-01-16T06:15:27.001843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model.summary()","metadata":{"execution":{"iopub.status.busy":"2024-01-16T06:15:27.004836Z","iopub.execute_input":"2024-01-16T06:15:27.005231Z","iopub.status.idle":"2024-01-16T06:15:27.013495Z","shell.execute_reply.started":"2024-01-16T06:15:27.005194Z","shell.execute_reply":"2024-01-16T06:15:27.011393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow.keras.backend as K\n\nclass SurfaceDiceLoss(tf.keras.losses.Loss):\n    \n    def dice_coef(self, y_true, y_pred, smooth=0):\n        y_true_f = K.flatten(y_true)\n        y_pred_f = K.flatten(y_pred)\n        intersection = K.sum(y_true_f * y_pred_f)\n        dice = (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)\n        return dice\n    \n    def call(self, y_true, y_pred):\n        return -dice_coef(u_true, y_pred)\n        \n\nmodel.compile(optimizer='adam',\n              loss=tf.keras.losses.CategoricalCrossentropy(from_logits=False),\n              metrics=['categorical_accuracy']\n             )","metadata":{"execution":{"iopub.status.busy":"2024-01-16T06:15:27.019532Z","iopub.execute_input":"2024-01-16T06:15:27.020226Z","iopub.status.idle":"2024-01-16T06:15:27.052596Z","shell.execute_reply.started":"2024-01-16T06:15:27.019920Z","shell.execute_reply":"2024-01-16T06:15:27.051217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create the training dataset:\ntraining_dataset = tf.data.Dataset.from_generator(\n    ImageCollector.generate_training_set,\n    output_signature=(\n        tf.TensorSpec(shape=INPUT_SHAPE, dtype=tf.dtypes.float32, name='image'),\n        tf.TensorSpec(shape=(1, INPUT_SHAPE[1], INPUT_SHAPE[2], 2), dtype=tf.dtypes.int32, name='label')\n    )\n)","metadata":{"execution":{"iopub.status.busy":"2024-01-16T06:15:27.053976Z","iopub.execute_input":"2024-01-16T06:15:27.054319Z","iopub.status.idle":"2024-01-16T06:15:27.127887Z","shell.execute_reply.started":"2024-01-16T06:15:27.054289Z","shell.execute_reply":"2024-01-16T06:15:27.126531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_save_name():\n    return f\"model_{datetime.today().strftime('%Y_%m_%d_%H_%M_%S')}.keras\"\n\n# Do the training:\nmodel_history = model.fit(x=training_dataset,\n                          epochs=EPOCHS,\n                          verbose=2,\n                          batch_size=BATCH_SIZE,\n                          validation_data=(validation_set_x, validation_set_y),\n                          callbacks=[\n#                               tf.keras.callbacks.LearningRateScheduler(schedule=scheduler, verbose=1),\n                              tf.keras.callbacks.ModelCheckpoint(\n                                  get_save_name(),\n                                  monitor='val_acc',\n                                  verbose=1,\n                                  save_best_only=True,\n                                  save_weights_only=False,\n                                  mode='max',\n                                  save_freq='epoch',\n                                  options=None,\n                                  initial_value_threshold=None\n                              ),\n                              tf.keras.callbacks.EarlyStopping(monitor='val_loss', \n                                                               patience=PATIENCE, \n                                                               baseline=None, \n                                                               restore_best_weights=True)\n                          ]\n                         )","metadata":{"execution":{"iopub.status.busy":"2024-01-16T06:15:27.129501Z","iopub.execute_input":"2024-01-16T06:15:27.129846Z","iopub.status.idle":"2024-01-16T11:49:09.729263Z","shell.execute_reply.started":"2024-01-16T06:15:27.129814Z","shell.execute_reply":"2024-01-16T11:49:09.726370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save_weights('checkpoints/simple_mobile_unet_weights')\nmodel.save('simple_mobile_unet_model.keras')","metadata":{"execution":{"iopub.status.busy":"2024-01-16T11:50:48.239534Z","iopub.execute_input":"2024-01-16T11:50:48.240143Z","iopub.status.idle":"2024-01-16T11:50:49.666181Z","shell.execute_reply.started":"2024-01-16T11:50:48.240097Z","shell.execute_reply":"2024-01-16T11:50:49.664953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Show the evolution plot (fingers crossed that this thing works in submission mode):\n# summarise history for accuracy:\nplt.plot(model_history.history['categorical_accuracy'])\nplt.plot(model_history.history['val_categorical_accuracy'])\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'validation'], loc='upper left')\nplt.show()\n\n# summarise history for loss:\nplt.plot(model_history.history['loss'])\nplt.plot(model_history.history['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'validation'], loc='upper left')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-01-16T11:49:09.734548Z","iopub.execute_input":"2024-01-16T11:49:09.735109Z","iopub.status.idle":"2024-01-16T11:49:10.543017Z","shell.execute_reply.started":"2024-01-16T11:49:09.735048Z","shell.execute_reply":"2024-01-16T11:49:10.541897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Perform the predictions:\ndel training_dataset\ndel validation_set_x\ndel validation_set_y\n\ntesting_set = tf.data.Dataset.from_generator(\n    ImageCollector.generate_testing_set,\n    output_signature=tf.TensorSpec(shape=INPUT_SHAPE, dtype=tf.dtypes.float32, name='image'),\n)","metadata":{"execution":{"iopub.status.busy":"2024-01-16T11:49:10.544870Z","iopub.execute_input":"2024-01-16T11:49:10.545483Z","iopub.status.idle":"2024-01-16T11:49:10.648972Z","shell.execute_reply.started":"2024-01-16T11:49:10.545447Z","shell.execute_reply":"2024-01-16T11:49:10.647761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = model.predict(\n    x = testing_set,\n    batch_size = BATCH_SIZE,\n    verbose = 2,\n    steps = None,\n)","metadata":{"execution":{"iopub.status.busy":"2024-01-16T11:49:10.650680Z","iopub.execute_input":"2024-01-16T11:49:10.651766Z","iopub.status.idle":"2024-01-16T11:49:13.878451Z","shell.execute_reply.started":"2024-01-16T11:49:10.651721Z","shell.execute_reply":"2024-01-16T11:49:13.877105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We know that the images are read in sorted order, first from kidney 5 then for kidney 6:\n\n# ref.: https://www.kaggle.com/stainsby/fast-tested-rle\ndef rle_encode(img):\n    '''\n    img: numpy array, 1 - mask, 0 - background\n    Returns run length as string formated\n    '''\n    pixels = img.T.flatten()\n    pixels = np.pad(pixels, ((1, 1), ))\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\n\n\ndef rle_encode_2(mask):\n    pixel = mask.flatten()\n    pixel = np.concatenate([[0], pixel, [0]])\n    run = np.where(pixel[1:] != pixel[:-1])[0] + 1\n    run[1::2] -= run[::2]\n    rle = ' '.join(str(r) for r in run)\n    if rle == '':\n        rle = '1 0'\n    return rle\n\n\ndef convert_from_one_hot_to_decimal(arg_img, new_shape):\n    result_array = np.argmax(a=arg_img, axis=2)\n#     result_array[result_array == 1] = 255\n    result_array = np.resize(result_array, new_shape)\n    return result_array\n\n\nids = []\ntest_keys = sorted(FULL_METAINFO['testing_metainfo'].keys())\nfor test_set_key in test_keys:\n    filenames = sorted(os.listdir(FULL_METAINFO['testing_metainfo'][test_set_key]['images_path']))\n    original_shape = read_image(os.path.join(FULL_METAINFO['testing_metainfo'][test_set_key]['images_path'], filenames[0])).shape\n    ids += [(f\"{FULL_METAINFO['testing_metainfo'][test_set_key]['set_name']}_{filename[:-4]}\", original_shape) \n            for filename in filenames]\n    del filenames\ndel test_keys\n\n\nSUBMISSION_FILE_NAME = 'submission.csv'\nimport csv\nwith open(SUBMISSION_FILE_NAME, 'w') as csvfile:\n    writer = csv.writer(csvfile, delimiter=',')\n    writer.writerow([\"id\", \"rle\"])\n    for index, tpl in enumerate(ids):\n        current_pred = predictions[index]\n        current_pred = convert_from_one_hot_to_decimal(current_pred, new_shape=(current_pred.shape[0], current_pred.shape[1]))\n        current_pred = np.resize(current_pred, tpl[1])\n        current_pred = current_pred.astype(bool)\n#         current_encoding = rle_encode(current_pred)\n        current_encoding = rle_encode_2(current_pred)\n        del current_pred\n        writer.writerow([tpl[0], current_encoding])\n","metadata":{"execution":{"iopub.status.busy":"2024-01-16T11:49:13.880377Z","iopub.execute_input":"2024-01-16T11:49:13.880790Z","iopub.status.idle":"2024-01-16T11:49:13.960975Z","shell.execute_reply.started":"2024-01-16T11:49:13.880755Z","shell.execute_reply":"2024-01-16T11:49:13.959666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df = pd.read_csv(SUBMISSION_FILE_NAME)\nsubmission_df.head(100)","metadata":{"execution":{"iopub.status.busy":"2024-01-16T11:49:13.962735Z","iopub.execute_input":"2024-01-16T11:49:13.963509Z","iopub.status.idle":"2024-01-16T11:49:14.019003Z","shell.execute_reply.started":"2024-01-16T11:49:13.963420Z","shell.execute_reply":"2024-01-16T11:49:14.017739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(read_image(\"/kaggle/input/blood-vessel-segmentation/test/kidney_5/images/0000.tif\"))","metadata":{"execution":{"iopub.status.busy":"2024-01-16T11:49:14.020998Z","iopub.execute_input":"2024-01-16T11:49:14.021404Z","iopub.status.idle":"2024-01-16T11:49:14.516611Z","shell.execute_reply.started":"2024-01-16T11:49:14.021369Z","shell.execute_reply":"2024-01-16T11:49:14.515391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(read_image(\"/kaggle/input/blood-vessel-segmentation/test/kidney_5/images/0001.tif\"))","metadata":{"execution":{"iopub.status.busy":"2024-01-16T11:49:14.518517Z","iopub.execute_input":"2024-01-16T11:49:14.519239Z","iopub.status.idle":"2024-01-16T11:49:14.943923Z","shell.execute_reply.started":"2024-01-16T11:49:14.519201Z","shell.execute_reply":"2024-01-16T11:49:14.942713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(read_image(\"/kaggle/input/blood-vessel-segmentation/test/kidney_5/images/0002.tif\"))","metadata":{"execution":{"iopub.status.busy":"2024-01-16T11:49:14.945658Z","iopub.execute_input":"2024-01-16T11:49:14.946003Z","iopub.status.idle":"2024-01-16T11:49:15.348451Z","shell.execute_reply.started":"2024-01-16T11:49:14.945972Z","shell.execute_reply":"2024-01-16T11:49:15.347285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(read_image(\"/kaggle/input/blood-vessel-segmentation/test/kidney_6/images/0000.tif\"))","metadata":{"execution":{"iopub.status.busy":"2024-01-16T11:49:15.350182Z","iopub.execute_input":"2024-01-16T11:49:15.350554Z","iopub.status.idle":"2024-01-16T11:49:16.830135Z","shell.execute_reply.started":"2024-01-16T11:49:15.350521Z","shell.execute_reply":"2024-01-16T11:49:16.828883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(read_image(\"/kaggle/input/blood-vessel-segmentation/test/kidney_6/images/0001.tif\"))","metadata":{"execution":{"iopub.status.busy":"2024-01-16T11:49:16.831831Z","iopub.execute_input":"2024-01-16T11:49:16.832219Z","iopub.status.idle":"2024-01-16T11:49:17.299939Z","shell.execute_reply.started":"2024-01-16T11:49:16.832182Z","shell.execute_reply":"2024-01-16T11:49:17.298698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(read_image(\"/kaggle/input/blood-vessel-segmentation/test/kidney_6/images/0002.tif\"))","metadata":{"execution":{"iopub.status.busy":"2024-01-16T11:49:17.301622Z","iopub.execute_input":"2024-01-16T11:49:17.302022Z","iopub.status.idle":"2024-01-16T11:49:17.775092Z","shell.execute_reply.started":"2024-01-16T11:49:17.301977Z","shell.execute_reply":"2024-01-16T11:49:17.773792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df = pd.read_csv(\"/kaggle/input/blood-vessel-segmentation/sample_submission.csv\")\nsubmission_df.head(100)","metadata":{"execution":{"iopub.status.busy":"2024-01-16T11:49:17.776879Z","iopub.execute_input":"2024-01-16T11:49:17.777227Z","iopub.status.idle":"2024-01-16T11:49:17.800610Z","shell.execute_reply.started":"2024-01-16T11:49:17.777196Z","shell.execute_reply":"2024-01-16T11:49:17.799386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df = pd.read_csv(\"/kaggle/input/blood-vessel-segmentation/train_rles.csv\")\nsubmission_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-01-16T11:49:17.802396Z","iopub.execute_input":"2024-01-16T11:49:17.802810Z","iopub.status.idle":"2024-01-16T11:49:18.930595Z","shell.execute_reply.started":"2024-01-16T11:49:17.802775Z","shell.execute_reply":"2024-01-16T11:49:18.929378Z"},"trusted":true},"execution_count":null,"outputs":[]}]}