{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport cv2\nimport tensorflow as tf\nfrom keras import backend as K\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Input, Conv2D, Conv2DTranspose, MaxPooling2D, Dropout, concatenate\nimport matplotlib.pyplot as plt\n\n# !mkdir /kaggle/working/packages\n# !cp -r /kaggle/input/pycocotools/* /kaggle/working/packages\n# os.chdir(\"/kaggle/working/packages/pycocotools-2.0.6/\")\n# !python setup.py install\n# !pip install . --no-index --find-links /kaggle/working/packages/\n# os.chdir(\"/kaggle/working\")\n\n# import base64\n# from pycocotools import _mask as coco_mask\n# import typing as t\n# import zlib","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-29T02:34:42.656191Z","iopub.execute_input":"2023-06-29T02:34:42.656662Z","iopub.status.idle":"2023-06-29T02:34:51.856065Z","shell.execute_reply.started":"2023-06-29T02:34:42.656633Z","shell.execute_reply":"2023-06-29T02:34:51.855082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import data\nBASE_DIR = \"/kaggle/input/hubmap-hacking-the-human-vasculature/\"\n\ntile_metadata = pd.read_csv(BASE_DIR + 'tile_meta.csv')\nwsi_metadata = pd.read_csv(BASE_DIR + 'wsi_meta.csv')\n\ntrain_files = os.listdir(BASE_DIR + \"train\")\ntest_files = os.listdir(BASE_DIR + \"test\")\n\npolygons_df = pd.read_json(\"/kaggle/input/hubmap-hacking-the-human-vasculature/polygons.jsonl\", lines=True)\n\ntraining_examples = polygons_df['id'].to_numpy()","metadata":{"execution":{"iopub.status.busy":"2023-06-29T02:34:51.858025Z","iopub.execute_input":"2023-06-29T02:34:51.859215Z","iopub.status.idle":"2023-06-29T02:34:56.430240Z","shell.execute_reply.started":"2023-06-29T02:34:51.859180Z","shell.execute_reply":"2023-06-29T02:34:56.429103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_empty_masks(IMAGE_WIDTH=512, IMAGE_HEIGHT=512, NUM_CHANNELS=3):\n    glomerulus_mask = np.zeros((IMAGE_WIDTH, IMAGE_HEIGHT, NUM_CHANNELS), dtype=np.uint8)\n    blood_vessel_mask = np.zeros((IMAGE_WIDTH, IMAGE_HEIGHT, NUM_CHANNELS), dtype=np.uint8)\n    unsure_mask = np.zeros((IMAGE_WIDTH, IMAGE_HEIGHT, NUM_CHANNELS), dtype=np.uint8)\n    return  {'glomerulus': glomerulus_mask,'blood_vessel':blood_vessel_mask ,'unsure':unsure_mask}\n\ndef fill_masks_with_annotations(image_id, masks):\n    try: \n        annots = polygons_df.loc[polygons_df[\"id\"] == image_id, 'annotations'].iloc[0]\n    except error as e: \n        print(\"Could not find image please make sure you entered the correct id and that the image has a corresponding annotation\")\n        return masks\n    \n    for annot in annots:\n        annot_type = annot['type']\n        coordinates = annot['coordinates']\n        color = np.random.randint(0, 255, size=3).tolist()  # Generate a unique color for each instance\n        cv2.fillPoly(masks[annot_type], [np.array(coordinates)], color)\n\n    return masks\n\ndef plot_masks(image_id):\n    \n    empty_masks = create_empty_masks()\n    masks = fill_masks_with_annotations(image_id, empty_masks)\n    \n    fig, axs = plt.subplots(1, 3, figsize=(8,8))\n    axs[0].imshow(masks['glomerulus'], cmap='gray')\n    axs[0].set_title('glomerulus mask')\n\n    axs[1].imshow(masks['blood_vessel'], cmap='gray')\n    axs[1].set_title('blood vessel mask')\n\n    axs[2].imshow(masks['unsure'], cmap='gray')\n    axs[2].set_title('unsure mask')\n    \n    plt.tight_layout()\n    plt.show()\n\ndef combine_masks(masks):\n    glomerulus_mask  = masks['glomerulus']\n    blood_vessel_mask = masks['blood_vessel']\n    unsure_mask = masks['unsure']\n\n    combined_image = np.concatenate([glomerulus_mask, blood_vessel_mask, unsure_mask], axis=2)\n    \n    return combined_image\n\ndef get_image(image_id):\n    image_path = BASE_DIR + \"train/\" + image_id + \".tif\"\n    image = imageio.v2.imread(image_path)\n    return image\n\ndef plot_masks_and_original_image(image_id):\n    empty_masks = create_empty_masks()\n    masks = fill_masks_with_annotations(image_id, empty_masks)\n    image = get_image(image_id)\n    mask_overlay_image, _ = get_mask_image_overlay(image_id)\n    \n    fig, axs = plt.subplots(1, 5, figsize=(16,16))\n    axs[0].imshow(masks['glomerulus'], cmap='gray')\n    axs[0].set_title('glomerulus mask')\n\n    axs[1].imshow(masks['blood_vessel'], cmap='gray')\n    axs[1].set_title('blood vessel mask')\n\n    axs[2].imshow(masks['unsure'], cmap='gray')\n    axs[2].set_title('unsure mask')\n    \n    axs[3].imshow(image)\n    axs[3].set_title('Ground Truth')\n    \n    axs[4].imshow(mask_overlay_image)\n    axs[4].set_title(\"mask on top of image\")\n    \n    for ax in axs.flat:\n        ax.set_xticks([]) \n        ax.set_yticks([]) \n        \n    plt.tight_layout()\n    plt.show()\n\ndef get_mask_image_overlay(image_id):    \n    annots = polygons_df.loc[polygons_df[\"id\"] == image_id, 'annotations'].iloc[0]\n    img = get_image(image_id)\n    \n    RED= (255,0,0)\n    GREEN= (0,255,0)\n    BLUE= (0,0,255)\n    color_map = {'blood_vessel':RED, 'glomerulus':BLUE, 'unsure':GREEN}\n    instance_count = {'blood_vessel':0, 'glomerulus':0, 'unsure':0}\n    \n    for annot in annots:\n        color = color_map[annot['type']]\n        instance_count[annot['type']]+=1\n        coords = np.array(annot['coordinates'])\n        cv2.polylines(img, coords, True, color, 3)\n    \n    return img, instance_count\n\ndef draw_mask_image_overlay(image_id):\n    img, instance_count = get_mask_image_overlay(image_id)\n    \n    fig, axs = plt.subplots(figsize=(10,10))\n    \n    print(f\"Blood Vessels (Red) Count: {instance_count['blood_vessel']}\")\n    print(f\"Glomerulus (Blue) Count: {instance_count['glomerulus']}\")\n    print(f\"Unsure (Green) Count: {instance_count['unsure']}\")\n\n    axs.imshow(img)\n    plt.axis('off')\n    plt.show()\n    \ndef load_file_paths(directory_path, examples):\n    file_extension = '.tif'\n#     file_paths = [directory_path + '/' + id + file_extension for id in examples]\n#     file_paths = [directory_path + '/' + id + file_extension + '0' for id in examples]\n    file_paths = [directory_path + '/' + id + file_extension + str(n) for id in examples for n in range(0,4)]\n    return file_paths\n\ndef tf_get_image(path):\n    image = cv2.imread(path)\n    image = image/255\n    image = image.astype(np.float32)\n    return image\n\ndef get_id(string_path):\n    parts = string_path.split(\"/\")\n    file_name = parts[-1]\n    identity = file_name.split(\".\")[0]\n    return identity\n\ndef tf_get_mask(path):\n    image_id = get_id(path)\n\n    mask = np.zeros((512, 512, 1), dtype=np.uint8)\n    annots = polygons_df.loc[polygons_df[\"id\"] == image_id, 'annotations'].iloc[0]\n    for annot in annots:\n            annot_type = annot['type']\n            coordinates = annot['coordinates']\n            if annot_type == 'blood_vessel':\n                cv2.fillPoly(mask, [np.array(coordinates)], (255,))\n    mask = mask/255\n    mask = mask.astype(np.float32)\n    return mask\n    \ndef tf_map_function(path):\n    def preprocess(path):\n        path = path.decode()\n        rotate = path[-1]\n        path = path[:-1]\n\n        image = tf_get_image(path)\n        mask = tf_get_mask(path)\n        \n        if rotate == '0':\n            return image, mask\n        elif rotate == '1':\n            image = np.rot90(image, 1)\n            mask = np.rot90(mask, 1)\n            return image, mask\n        elif rotate == '2':\n            image = np.rot90(image, 2)\n            mask = np.rot90(mask, 2)\n            return image, mask\n        elif rotate == '3':\n            image = np.rot90(image, 3)\n            mask = np.rot90(mask, 3)\n            return image, mask\n\n    image, mask = tf.numpy_function(preprocess, [path], [tf.float32, tf.float32])\n    image.set_shape([512,512,3])\n    mask.set_shape([512,512,1])\n    \n    return image, mask\n\ndef tf_dataset(file_paths, batch_size=16):\n    dataset = tf.data.Dataset.from_tensor_slices(file_paths)\n    dataset = dataset.map(tf_map_function, num_parallel_calls=tf.data.AUTOTUNE)\n    dataset = dataset.batch(batch_size)\n    return dataset ","metadata":{"execution":{"iopub.status.busy":"2023-06-29T02:34:56.431795Z","iopub.execute_input":"2023-06-29T02:34:56.432171Z","iopub.status.idle":"2023-06-29T02:34:56.467014Z","shell.execute_reply.started":"2023-06-29T02:34:56.432137Z","shell.execute_reply":"2023-06-29T02:34:56.465570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def simple_unet_model(IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS):\n#Build the model\n    inputs = Input((IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS))\n    s = inputs\n\n    #Contraction path\n    c1 = Conv2D(16, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(s)\n    c1 = Dropout(0.1)(c1)\n    c1 = Conv2D(16, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c1)\n    p1 = MaxPooling2D((2, 2))(c1)\n    \n    c2 = Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(p1)\n    c2 = Dropout(0.1)(c2)\n    c2 = Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c2)\n    p2 = MaxPooling2D((2, 2))(c2)\n     \n    c3 = Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(p2)\n    c3 = Dropout(0.2)(c3)\n    c3 = Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c3)\n    p3 = MaxPooling2D((2, 2))(c3)\n     \n    c4 = Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(p3)\n    c4 = Dropout(0.2)(c4)\n    c4 = Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c4)\n    p4 = MaxPooling2D(pool_size=(2, 2))(c4)\n     \n    c5 = Conv2D(256, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(p4)\n    c5 = Dropout(0.3)(c5)\n    c5 = Conv2D(256, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c5)\n    \n    #Expansive path \n    u6 = Conv2DTranspose(128, (2, 2), strides=(2, 2), padding='same')(c5)\n    u6 = concatenate([u6, c4])\n    c6 = Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(u6)\n    c6 = Dropout(0.2)(c6)\n    c6 = Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c6)\n     \n    u7 = Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same')(c6)\n    u7 = concatenate([u7, c3])\n    c7 = Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(u7)\n    c7 = Dropout(0.2)(c7)\n    c7 = Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c7)\n     \n    u8 = Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same')(c7)\n    u8 = concatenate([u8, c2])\n    c8 = Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(u8)\n    c8 = Dropout(0.1)(c8)\n    c8 = Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c8)\n     \n    u9 = Conv2DTranspose(16, (2, 2), strides=(2, 2), padding='same')(c8)\n    u9 = concatenate([u9, c1], axis=3)\n    c9 = Conv2D(16, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(u9)\n    c9 = Dropout(0.1)(c9)\n    c9 = Conv2D(16, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c9)\n     \n    outputs = Conv2D(1, (1, 1), activation='sigmoid')(c9)\n     \n    model = Model(inputs=[inputs], outputs=[outputs])\n    \n    return model\n\ndef jaccard_coef(y_true, y_pred):\n    y_true_f = K.flatten(y_true)\n    y_pred_f = K.flatten(y_pred)\n    intersection = K.sum(y_true_f * y_pred_f)\n    return (intersection + 1.0) / (K.sum(y_true_f) + K.sum(y_pred_f) - intersection + 1.0)\n\n\ndef jaccard_loss(y_true, y_pred):\n    return 1-jaccard_coef(y_true, y_pred)","metadata":{"execution":{"iopub.status.busy":"2023-06-29T02:34:56.470812Z","iopub.execute_input":"2023-06-29T02:34:56.473133Z","iopub.status.idle":"2023-06-29T02:34:56.498131Z","shell.execute_reply.started":"2023-06-29T02:34:56.473099Z","shell.execute_reply":"2023-06-29T02:34:56.496952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def encode_binary_mask(mask: np.ndarray) -> t.Text:\n#   \"\"\"Converts a binary mask into OID challenge encoding ascii text.\"\"\"\n\n#   # check input mask --\n#   if mask.dtype != np.bool:\n#     raise ValueError(\n#         \"encode_binary_mask expects a binary mask, received dtype == %s\" %\n#         mask.dtype)\n\n#   mask = np.squeeze(mask)\n#   if len(mask.shape) != 2:\n#     raise ValueError(\n#         \"encode_binary_mask expects a 2d mask, received shape == %s\" %\n#         mask.shape)\n\n#   # convert input mask to expected COCO API input --\n#   mask_to_encode = mask.reshape(mask.shape[0], mask.shape[1], 1)\n#   mask_to_encode = mask_to_encode.astype(np.uint8)\n#   mask_to_encode = np.asfortranarray(mask_to_encode)\n\n#   # RLE encode mask --\n#   encoded_mask = coco_mask.encode(mask_to_encode)[0][\"counts\"]\n\n#   # compress and base64 encoding --\n#   binary_str = zlib.compress(encoded_mask, zlib.Z_BEST_COMPRESSION)\n#   base64_str = base64.b64encode(binary_str)\n#   return base64_str","metadata":{"execution":{"iopub.status.busy":"2023-06-29T02:34:56.501530Z","iopub.execute_input":"2023-06-29T02:34:56.501836Z","iopub.status.idle":"2023-06-29T02:34:56.512536Z","shell.execute_reply.started":"2023-06-29T02:34:56.501805Z","shell.execute_reply":"2023-06-29T02:34:56.511490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_directory_path = '/kaggle/input/hubmap-hacking-the-human-vasculature/train'\ntrain_file_paths = load_file_paths(train_directory_path, training_examples)\nTRAIN_LENGTH = len(train_file_paths)\n\ntrain_ds = tf_dataset(train_file_paths)","metadata":{"execution":{"iopub.status.busy":"2023-06-29T02:34:56.514134Z","iopub.execute_input":"2023-06-29T02:34:56.514864Z","iopub.status.idle":"2023-06-29T02:34:59.630944Z","shell.execute_reply.started":"2023-06-29T02:34:56.514832Z","shell.execute_reply":"2023-06-29T02:34:59.629977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EPOCHS = 50\nBATCH_SIZE=16\nSTEPS_PER_EPOCH = TRAIN_LENGTH//BATCH_SIZE\n\nvalidation_split = 0.2\ntotal_samples = train_ds.cardinality().numpy()\nvalidation_samples = int(validation_split * total_samples)\ntrain_samples = total_samples - validation_samples\n\nvalidation_ds = train_ds.take(validation_samples)\ntrain_ds = train_ds.skip(validation_samples)\n\ncheckpoint_path = \"model_checkpoint-{epoch:04d}.h5\"\ncheckpoint_callback = tf.keras.callbacks.ModelCheckpoint(\n    filepath=checkpoint_path, \n    verbose=1, \n    save_weights_only=True,\n    save_freq='epoch',\n    period = 10)\n\nearlystop_callback = tf.keras.callbacks.EarlyStopping(monitor='val_jaccard_coef', patience=5)\n\nmodel = simple_unet_model(512,512,3)\n\ncheckpoint_path_old = '/kaggle/input/checkpoint/model_checkpoint-0100.h5'\nmodel.load_weights(checkpoint_path_old)\n\nlearning_rate=0.0001\noptimizer = tf.keras.optimizers.Adam(learning_rate)\nloss = tf.keras.losses.BinaryCrossentropy()\n\nmodel.compile(optimizer=optimizer,\n              loss=[jaccard_loss],\n              metrics=['accuracy', jaccard_coef])\n# model.compile(optimizer=optimizer,\n#               loss = tf.keras.losses.BinaryCrossentropy(),\n#               metrics=['accuracy', jaccard_coef])\n\nmodel_history = model.fit(train_ds, epochs=EPOCHS, validation_data=validation_ds, callbacks=[checkpoint_callback])","metadata":{"execution":{"iopub.status.busy":"2023-06-29T02:34:59.632282Z","iopub.execute_input":"2023-06-29T02:34:59.632618Z","iopub.status.idle":"2023-06-29T02:37:57.780110Z","shell.execute_reply.started":"2023-06-29T02:34:59.632586Z","shell.execute_reply":"2023-06-29T02:37:57.778931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(model_history.history['loss'], label='Training Loss')\nplt.plot(model_history.history['val_loss'], label='Validation Loss')\nplt.title('Loss per epoch')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-29T02:37:57.781785Z","iopub.execute_input":"2023-06-29T02:37:57.782173Z","iopub.status.idle":"2023-06-29T02:37:58.097758Z","shell.execute_reply.started":"2023-06-29T02:37:57.782141Z","shell.execute_reply":"2023-06-29T02:37:58.096879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(model_history.history['jaccard_coef'], label='Training IoU')\nplt.plot(model_history.history['val_jaccard_coef'], label='Validation IoU')\nplt.title('IoU Score per epoch')\nplt.xlabel('Epoch')\nplt.ylabel('IoU Score')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-29T02:37:58.099119Z","iopub.execute_input":"2023-06-29T02:37:58.099631Z","iopub.status.idle":"2023-06-29T02:37:58.395785Z","shell.execute_reply.started":"2023-06-29T02:37:58.099593Z","shell.execute_reply":"2023-06-29T02:37:58.394907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(model_history.history['accuracy'], label='Training Accuracy')\nplt.plot(model_history.history['val_accuracy'], label='Validation Accuracy')\nplt.title('Accuracy per epoch')\nplt.xlabel('Epoch')\nplt.ylabel('Accuracy')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-29T02:37:58.399821Z","iopub.execute_input":"2023-06-29T02:37:58.400173Z","iopub.status.idle":"2023-06-29T02:37:58.704986Z","shell.execute_reply.started":"2023-06-29T02:37:58.400131Z","shell.execute_reply":"2023-06-29T02:37:58.703954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_path = \"/kaggle/input/hubmap-hacking-the-human-vasculature/test/\"\n\nfor img_id in os.listdir(test_path):\n    curr_img = cv2.imread(test_path + img_id)\n    curr_img = np.expand_dims(curr_img, axis=0)\n    curr_img_norm = curr_img/255\n    prediction = model.predict(curr_img_norm)\n    prediction_mask = np.where(prediction[0]>np.mean(prediction[0]), 1, 0).astype(np.bool)","metadata":{"execution":{"iopub.status.busy":"2023-06-29T02:37:58.706393Z","iopub.execute_input":"2023-06-29T02:37:58.706727Z","iopub.status.idle":"2023-06-29T02:37:59.462962Z","shell.execute_reply.started":"2023-06-29T02:37:58.706684Z","shell.execute_reply":"2023-06-29T02:37:59.461110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(curr_img[0])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-29T02:37:59.464442Z","iopub.execute_input":"2023-06-29T02:37:59.464777Z","iopub.status.idle":"2023-06-29T02:37:59.786584Z","shell.execute_reply.started":"2023-06-29T02:37:59.464745Z","shell.execute_reply":"2023-06-29T02:37:59.785776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(curr_img_norm[0])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-29T02:37:59.787530Z","iopub.execute_input":"2023-06-29T02:37:59.787864Z","iopub.status.idle":"2023-06-29T02:38:00.125834Z","shell.execute_reply.started":"2023-06-29T02:37:59.787833Z","shell.execute_reply":"2023-06-29T02:38:00.124932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(prediction[0])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-29T02:38:00.127161Z","iopub.execute_input":"2023-06-29T02:38:00.128030Z","iopub.status.idle":"2023-06-29T02:38:00.378833Z","shell.execute_reply.started":"2023-06-29T02:38:00.127999Z","shell.execute_reply":"2023-06-29T02:38:00.377943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(prediction_mask)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-29T02:38:00.380344Z","iopub.execute_input":"2023-06-29T02:38:00.380696Z","iopub.status.idle":"2023-06-29T02:38:00.627998Z","shell.execute_reply.started":"2023-06-29T02:38:00.380665Z","shell.execute_reply":"2023-06-29T02:38:00.626969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def find_optimal_threshold(image, mask, model):\n    thresholds = np.arange(0.1, 1.0, 0.1) \n    \n    best_threshold = None\n    best_iou = 0.0\n    \n    for threshold in thresholds:\n        predictions = model.predict(image)\n        thresholded_predictions = np.where(predictions >= threshold, 1, 0)\n        \n        iou = jaccard_coef(mask, thresholded_predictions.astype(np.float32))\n        \n        if iou > best_iou:\n            best_iou = iou\n            best_threshold = threshold\n    \n    print(f'best threshold:{best_threshold} best iou: {best_iou}')\n    return best_threshold, best_iou\n\ndef get_prediction(image, mask, model, threshold=0.9):\n    predictions = model.predict(image)\n    thresholded_predictions = np.where(predictions >= threshold, 1, 0)\n    \n    num_images = image.shape[0]\n    fig, axs = plt.subplots(num_images, 3, figsize=(10, 5 * num_images))\n    \n    for i in range(num_images):\n        \n        iou = jaccard_coef(mask[i], thresholded_predictions[i].astype(np.float32)).numpy()\n        \n        axs[i, 0].imshow(thresholded_predictions[i], cmap='gray')\n        axs[i, 0].set_title(f'Prediction, IoU: {iou:.3f}')\n        \n        axs[i, 1].imshow(image[i])\n        axs[i, 1].set_title('Image')\n        \n        axs[i, 2].imshow(mask[i], cmap='gray')\n        axs[i, 2].set_title('Mask')\n    \n    plt.tight_layout()\n    plt.subplots_adjust(hspace=0.4) \n    plt.show()\n    \n    return thresholded_predictions\n\nfor image, mask in validation_ds.take(1):\n    pass\n\nbest_threshold, _ = find_optimal_threshold(image,mask,model)    \n\npredictions = get_prediction(image, mask, model, best_threshold)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-29T02:38:48.536589Z","iopub.execute_input":"2023-06-29T02:38:48.537017Z","iopub.status.idle":"2023-06-29T02:39:02.717690Z","shell.execute_reply.started":"2023-06-29T02:38:48.536982Z","shell.execute_reply":"2023-06-29T02:39:02.716446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_path = \"/kaggle/input/hubmap-hacking-the-human-vasculature/test/\"\n# submission = pd.DataFrame()\n\n# ids = []\n# h = []\n# w = []\n# pred_strings = []\n    \n# for img_id in os.listdir(test_path):\n#     curr_img = cv2.imread(test_path + img_id)\n\n#     ## Get id, height, width\n#     height, width, channels = curr_img.shape\n#     ids.append(img_id.split(\".\")[0])\n#     h.append(height)\n#     w.append(width)\n\n    \n#     ## Get prediction_string\n#     encoded_string = '0 1.0 ' + encode_binary_mask(prediction_mask).decode()\n#     pred_strings.append(encoded_string)\n# #     pred_strings.append('0 1.0 eNoLTDAwyrM3yI/PMwcAE94DZA==')\n\n# submission[\"id\"] = ids\n# submission[\"height\"] = h\n# submission[\"width\"] = w\n# submission[\"prediction_string\"] = pred_strings\n# submission.set_index(\"id\", inplace=True)\n# submission.to_csv(\"submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-06-29T02:38:01.477491Z","iopub.status.idle":"2023-06-29T02:38:01.478213Z","shell.execute_reply.started":"2023-06-29T02:38:01.477936Z","shell.execute_reply":"2023-06-29T02:38:01.477959Z"},"trusted":true},"execution_count":null,"outputs":[]}]}