{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":61446,"databundleVersionId":6962461,"sourceType":"competition"}],"dockerImageVersionId":30635,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfrom PIL import Image\nimport imageio\nimport matplotlib.pyplot as plt\nfrom matplotlib.animation import FuncAnimation\nfrom IPython.display import HTML","metadata":{"execution":{"iopub.status.busy":"2024-02-20T17:42:44.635234Z","iopub.execute_input":"2024-02-20T17:42:44.635639Z","iopub.status.idle":"2024-02-20T17:42:44.682600Z","shell.execute_reply.started":"2024-02-20T17:42:44.635607Z","shell.execute_reply":"2024-02-20T17:42:44.681667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input/blood-vessel-segmentation/'):\n    for filename in filenames:\n        file = os.path.join(dirname, filename)\nprint(file)","metadata":{"execution":{"iopub.status.busy":"2024-02-20T17:42:44.684506Z","iopub.execute_input":"2024-02-20T17:42:44.685329Z","iopub.status.idle":"2024-02-20T17:42:52.785811Z","shell.execute_reply.started":"2024-02-20T17:42:44.685290Z","shell.execute_reply":"2024-02-20T17:42:52.784761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfrom PIL import Image\nimport matplotlib.pyplot as plt\n\n# Base directory path\nbase_dir = '/kaggle/input/blood-vessel-segmentation/'\n\n# Example paths for an image and its label\n\nimage_path = 'train/kidney_3_sparse/images/0795.tif'\nlabel_path = 'train/kidney_3_sparse/labels/0795.tif'\n\n# Load the image and label\nimage = Image.open(os.path.join(base_dir, image_path))\nlabel = Image.open(os.path.join(base_dir, label_path))\n\n# Plotting\nplt.figure(figsize=(8, 4))\n\n# Display the image\nplt.subplot(1, 2, 1)\nplt.imshow(image, cmap = 'gray')\nplt.title('kidney_3_sparse - Image')\nplt.axis('off')\n\n# Display the label\nplt.subplot(1, 2, 2)\nplt.imshow(label, cmap = 'gray')\nplt.title('kidney_3_sparse - Label')\nplt.axis('off')\n\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-02-20T17:42:52.787245Z","iopub.execute_input":"2024-02-20T17:42:52.787598Z","iopub.status.idle":"2024-02-20T17:42:53.425108Z","shell.execute_reply.started":"2024-02-20T17:42:52.787569Z","shell.execute_reply":"2024-02-20T17:42:53.424091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfrom PIL import Image\nimport matplotlib.pyplot as plt\n\n# Base directory path\nbase_dir = '/kaggle/input/blood-vessel-segmentation/'\n\n# List of dataset folders\ndatasets = ['kidney_1_dense', 'kidney_1_voi', 'kidney_2']\n\n# Loop through each dataset\nfor dataset in datasets:\n    image_dir = os.path.join(base_dir, 'train', dataset, 'images')\n    label_dir = os.path.join(base_dir, 'train', dataset, 'labels')\n    \n    # Get list of files and select a file from the middle\n    image_files = os.listdir(image_dir)\n    label_files = os.listdir(label_dir)\n\n    # Ensure there's at least one image and label\n    if not image_files or not label_files:\n        continue\n\n    # Calculating middle index\n    middle_index = len(image_files) // 2\n    image_file = image_files[middle_index]\n    label_file = label_files[middle_index]\n\n    # Paths for the image and its label\n    image_path = os.path.join(image_dir, image_file)\n    label_path = os.path.join(label_dir, label_file)\n\n    # Load the image and label\n    image = Image.open(image_path)\n    label = Image.open(label_path)\n\n    # Plotting\n    plt.figure(figsize=(8, 4))\n\n    # Display the image\n    plt.subplot(1, 2, 1)\n    plt.imshow(image, cmap='gray')\n    plt.title(f'{dataset} - Image')\n    plt.axis('off')\n\n    # Display the label\n    plt.subplot(1, 2, 2)\n    plt.imshow(label, cmap = 'gray')\n    plt.title(f'{dataset} - Label')\n    plt.axis('off')\n\n    plt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-02-23T20:55:47.777838Z","iopub.status.idle":"2024-02-23T20:55:47.778186Z","shell.execute_reply.started":"2024-02-23T20:55:47.778017Z","shell.execute_reply":"2024-02-23T20:55:47.778034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set the base directory and the specific dataset\nbase_dir = '/kaggle/input/blood-vessel-segmentation/'\ndataset = 'kidney_1_dense'  # Specific dataset\n\n# Paths to images and labels\nimage_dir = os.path.join(base_dir, f'train/{dataset}/images')\nlabel_dir = os.path.join(base_dir, f'train/{dataset}/labels')\n\n# List of image files and select first 10\nimage_files = sorted(os.listdir(image_dir))[1200:1300]\n\n# Prepare a figure\nfig, ax = plt.subplots(1, 2, figsize=(12, 6))\n\ndef update(frame_index):\n    \"\"\"Update function for the animation\"\"\"\n    image_path = os.path.join(image_dir, image_files[frame_index])\n    label_path = os.path.join(label_dir, image_files[frame_index])\n\n    # Load image and label\n    image = Image.open(image_path)\n    label = Image.open(label_path)\n\n    # Clear axes and display new image and label\n    ax[0].clear()\n    ax[1].clear()\n    ax[0].imshow(image)\n    ax[0].axis('off')\n    ax[1].imshow(label)\n    ax[1].axis('off')\n\n# Create animation\nani = FuncAnimation(fig, update, frames=len(image_files), interval=50, repeat_delay=1000, blit=False)\nplt.close()","metadata":{"execution":{"iopub.status.busy":"2024-02-20T17:42:55.664042Z","iopub.execute_input":"2024-02-20T17:42:55.664299Z","iopub.status.idle":"2024-02-20T17:42:55.716436Z","shell.execute_reply.started":"2024-02-20T17:42:55.664274Z","shell.execute_reply":"2024-02-20T17:42:55.715508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"HTML(ani.to_html5_video())","metadata":{"execution":{"iopub.status.busy":"2024-02-20T17:42:55.717545Z","iopub.execute_input":"2024-02-20T17:42:55.717834Z","iopub.status.idle":"2024-02-20T17:43:21.705065Z","shell.execute_reply.started":"2024-02-20T17:42:55.717807Z","shell.execute_reply":"2024-02-20T17:43:21.704212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ls '/kaggle/input/blood-vessel-segmentation/train/kidney_3_sparse/labels/'","metadata":{"execution":{"iopub.status.busy":"2024-02-20T17:43:21.706134Z","iopub.execute_input":"2024-02-20T17:43:21.706405Z","iopub.status.idle":"2024-02-20T17:43:22.706373Z","shell.execute_reply.started":"2024-02-20T17:43:21.706381Z","shell.execute_reply":"2024-02-20T17:43:22.705228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport plotly.express as px\n\nbase_path = '/kaggle/input/blood-vessel-segmentation/train/' \n\n# Automatically identify dataset folders\ndatasets = [d for d in os.listdir(base_path) if os.path.isdir(os.path.join(base_path, d))]\n\ndata = []\n\nfor dataset in datasets:\n    images_path = os.path.join(base_path, dataset, 'images')\n    labels_path = os.path.join(base_path, dataset, 'labels')\n\n    # Check if 'images' and 'labels' folders exist\n    if os.path.exists(images_path) and os.path.exists(labels_path):\n        # Count TIFF files in each folder\n        num_images = len([f for f in os.listdir(images_path) if f.endswith('.tiff') or f.endswith('.tif')])\n        num_labels = len([f for f in os.listdir(labels_path) if f.endswith('.tiff') or f.endswith('.tif')])\n\n        data.append({'dataset': dataset, 'num_images': num_images, 'num_labels': num_labels})\n\n# Convert to DataFrame\ndf = pd.DataFrame(data)\n\n# Reshape DataFrame for plotting\ndf_long = df.melt(id_vars='dataset', value_vars=['num_images', 'num_labels'],\n                  var_name='Type', value_name='Count')\n\n# Create a grouped bar chart\nfig = px.bar(df_long, x='dataset', y='Count', color='Type', \n             barmode='group', title='Number of Images and Labels per Dataset')\n\n# Update layout\nfig.update_layout(xaxis_title='Dataset', yaxis_title='Count')\nfig.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-02-20T17:43:22.707897Z","iopub.execute_input":"2024-02-20T17:43:22.708193Z","iopub.status.idle":"2024-02-20T17:43:24.917402Z","shell.execute_reply.started":"2024-02-20T17:43:22.708165Z","shell.execute_reply":"2024-02-20T17:43:24.916441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np\nfrom PIL import Image\nimport matplotlib.pyplot as plt\n\n# Base directory path\nbase_dir = '/kaggle/input/blood-vessel-segmentation/train/'\n\n\n\n\n# Dataset folders\ndatasets = ['kidney_1_dense', 'kidney_1_voi', 'kidney_2', 'kidney_3_sparse']\n\n# Create a subplot for each dataset\nfig, axes = plt.subplots(nrows=len(datasets), ncols=1, figsize=(10, 4*len(datasets)))\n\nfor i, dataset in enumerate(datasets):\n    label_dir = os.path.join(base_dir, dataset, 'labels')\n    label_files = os.listdir(label_dir)\n\n    coverage_data = []  # Reset coverage data for each dataset\n\n    for label_file in label_files:\n        label_path = os.path.join(label_dir, label_file)\n        label = np.array(Image.open(label_path))\n\n        # Normalize mask values from [0, 255] to [0, 1]\n        normalized_label = label / 255.0\n        #print(normalized_label)\n        # Assuming binary mask (0 for background, 1 for vessel)\n        #print(np.mean(normalized_label))\n        coverage = np.mean(normalized_label) * 100  # Percentage of area covered by vessels\n        coverage_data.append(coverage)\n\n    # Plotting coverage distribution for the dataset\n    axes[i].hist(coverage_data, bins=20, color='skyblue')\n    axes[i].set_title(f'Distribution of Vessel Coverage in {dataset}')\n    axes[i].set_xlabel('Coverage (%)')\n    axes[i].set_ylabel('Frequency')\n\nplt.tight_layout()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-02-20T17:43:24.921741Z","iopub.execute_input":"2024-02-20T17:43:24.922474Z","iopub.status.idle":"2024-02-20T17:45:12.577629Z","shell.execute_reply.started":"2024-02-20T17:43:24.922442Z","shell.execute_reply":"2024-02-20T17:45:12.576666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. U-Net","metadata":{}},{"cell_type":"markdown","source":"## Load the data","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nfrom tensorflow.keras.preprocessing.image import load_img, img_to_array\n\ndef load_dataset(image_dir, mask_dir, image_size=(256, 256)):\n    images = []\n    masks = []\n    image_filenames = [f for f in os.listdir(image_dir) if f.endswith('.tiff') or f.endswith('.tif')]\n    \n    for image_filename in image_filenames:\n        image_path = os.path.join(image_dir, image_filename)\n        mask_path = os.path.join(mask_dir, image_filename)  \n        \n        image = load_img(image_path, target_size=image_size, color_mode='grayscale')\n        image = img_to_array(image) / 255.0\n        \n        mask = load_img(mask_path, target_size=image_size, color_mode='grayscale')\n        mask = img_to_array(mask) / 255.0\n        \n        images.append(image)\n        masks.append(mask)\n        \n    return np.array(images), np.array(masks)\n","metadata":{"execution":{"iopub.status.busy":"2024-02-20T17:45:12.578932Z","iopub.execute_input":"2024-02-20T17:45:12.579230Z","iopub.status.idle":"2024-02-20T17:45:18.298966Z","shell.execute_reply.started":"2024-02-20T17:45:12.579204Z","shell.execute_reply":"2024-02-20T17:45:18.298100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess_image(path):\n    \n    img = cv2.imread(path, cv2.IMREAD_UNCHANGED)\n    img = np.tile(img[...,None],[1, 1, 3]) \n    img = img.astype('float32') \n    mx = np.max(img)\n    if mx:\n        img/=mx \n        \n    img = np.transpose(img, (2, 0, 1))\n    img_ten = torch.tensor(img)\n    return img_ten","metadata":{"execution":{"iopub.status.busy":"2024-02-20T17:45:18.300090Z","iopub.execute_input":"2024-02-20T17:45:18.300734Z","iopub.status.idle":"2024-02-20T17:45:18.306842Z","shell.execute_reply.started":"2024-02-20T17:45:18.300704Z","shell.execute_reply":"2024-02-20T17:45:18.305871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess_mask(path):\n    \n    msk = cv2.imread(path, cv2.IMREAD_UNCHANGED)\n    msk = msk.astype('float32')\n    msk/=255.0\n    msk_ten = torch.tensor(msk)\n    \n    return msk_ten","metadata":{"execution":{"iopub.status.busy":"2024-02-20T17:45:18.308635Z","iopub.execute_input":"2024-02-20T17:45:18.308990Z","iopub.status.idle":"2024-02-20T17:45:18.341540Z","shell.execute_reply.started":"2024-02-20T17:45:18.308956Z","shell.execute_reply":"2024-02-20T17:45:18.340653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def augment_image(image, mask):\n    \n    image_np = image.permute(1, 2, 0).numpy()\n    mask_np = mask.numpy()\n\n    transform = A.Compose([\n        A.Resize(256,256, interpolation=cv2.INTER_NEAREST),\n        A.HorizontalFlip(p=0.5),\n        A.VerticalFlip(p=0.5),\n        A.ShiftScaleRotate(scale_limit=0.5, rotate_limit=0, shift_limit=0.1, p=1, border_mode=0),\n        A.RandomCrop(height=256, width=256, always_apply=True),\n        A.RandomBrightness(p=1),\n        A.OneOf(\n            [\n                A.Blur(blur_limit=3, p=1),\n                A.MotionBlur(blur_limit=3, p=1),\n            ],\n            p=0.9,\n        ),\n    \n    ])\n\n    augmented = transform(image=image_np, mask=mask_np)\n    augmented_image, augmented_mask = augmented['image'], augmented['mask']\n\n    augmented_image = torch.tensor(augmented_image, dtype=torch.float32).permute(2, 0, 1)\n    augmented_mask = torch.tensor(augmented_mask, dtype=torch.float32)\n\n    return augmented_image, augmented_mask","metadata":{"execution":{"iopub.status.busy":"2024-02-20T17:45:18.342706Z","iopub.execute_input":"2024-02-20T17:45:18.342991Z","iopub.status.idle":"2024-02-20T17:45:18.354386Z","shell.execute_reply.started":"2024-02-20T17:45:18.342968Z","shell.execute_reply":"2024-02-20T17:45:18.353380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data augmentation","metadata":{}},{"cell_type":"code","source":"# from tensorflow.keras.preprocessing.image import ImageDataGenerator\n# def create_augmented_data(images, masks, batch_size):\n#     # Create an instance of ImageDataGenerator for both images and masks\n#     # with the same parameters to ensure the same transformations are applied\n#     data_gen_args = dict(rotation_range=10,\n#                          width_shift_range=0.1,\n#                          height_shift_range=0.1,\n#                          zoom_range=0.2,\n#                          horizontal_flip=True,\n#                          fill_mode='nearest')\n\n#     image_datagen = ImageDataGenerator(**data_gen_args)\n#     mask_datagen = ImageDataGenerator(**data_gen_args)\n\n#     # Provide the same seed and keyword arguments to the flow methods\n#     seed = 1\n#     image_generator = image_datagen.flow(images, batch_size=batch_size, seed=seed)\n#     mask_generator = mask_datagen.flow(masks, batch_size=batch_size, seed=seed)\n\n#     # Combine generators into one which yields image and masks\n#     train_generator = zip(image_generator, mask_generator)\n\n#     return train_generator","metadata":{"execution":{"iopub.status.busy":"2024-02-20T17:45:18.355501Z","iopub.execute_input":"2024-02-20T17:45:18.355754Z","iopub.status.idle":"2024-02-20T17:45:18.367715Z","shell.execute_reply.started":"2024-02-20T17:45:18.355731Z","shell.execute_reply":"2024-02-20T17:45:18.366888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#from tensorflow.keras.preprocessing.image import ImageDataGenerator\n\n\nbase_path = '/kaggle/input/blood-vessel-segmentation/train/'\n\n# Automatically identify dataset folders\ndatasets = [d for d in os.listdir(base_path) if os.path.isdir(os.path.join(base_path, d))]\n\nfor dataset in datasets:\n    images_path = os.path.join(base_path, dataset, 'images')\n    labels_path = os.path.join(base_path, dataset, 'labels')\n\n    if os.path.exists(images_path) and os.path.exists(labels_path):\n        images, masks = load_dataset(images_path, labels_path, image_size=(256, 256))\n        train_generator = create_augmented_data(images, masks, batch_size=32)\n","metadata":{"execution":{"iopub.status.busy":"2024-02-20T17:45:18.368728Z","iopub.execute_input":"2024-02-20T17:45:18.368963Z","iopub.status.idle":"2024-02-20T17:45:49.038034Z","shell.execute_reply.started":"2024-02-20T17:45:18.368936Z","shell.execute_reply":"2024-02-20T17:45:49.035510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"# from tensorflow.keras.models import Model\n# from tensorflow.keras.layers import Input, Conv2D, MaxPooling2D, Conv2DTranspose, concatenate, Dropout, BatchNormalization\n\n# def unet(input_size=(256, 256, 1), num_classes=1):\n#     inputs = Input(input_size)\n    \n#     # Encoding path\n#     c1 = Conv2D(16, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(inputs)\n#     c1 = Dropout(0.1)(c1)\n#     c1 = Conv2D(16, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c1)\n#     p1 = MaxPooling2D((2, 2))(c1)\n    \n#     c2 = Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(p1)\n#     c2 = Dropout(0.1)(c2)\n#     c2 = Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c2)\n#     p2 = MaxPooling2D((2, 2))(c2)\n    \n#     c3 = Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(p2)\n#     c3 = Dropout(0.2)(c3)\n#     c3 = Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c3)\n#     p3 = MaxPooling2D((2, 2))(c3)\n    \n#     c4 = Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(p3)\n#     c4 = Dropout(0.2)(c4)\n#     c4 = Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c4)\n#     p4 = MaxPooling2D(pool_size=(2, 2))(c4)\n    \n#     c5 = Conv2D(256, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(p4)\n#     c5 = Dropout(0.3)(c5)\n#     c5 = Conv2D(256, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c5)\n    \n#     # Decoding path\n#     u6 = Conv2DTranspose(128, (2, 2), strides=(2, 2), padding='same')(c5)\n#     u6 = concatenate([u6, c4])\n#     c6 = Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(u6)\n#     c6 = Dropout(0.2)(c6)\n#     c6 = Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c6)\n    \n#     u7 = Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same')(c6)\n#     u7 = concatenate([u7, c3])\n#     c7 = Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(u7)\n#     c7 = Dropout(0.2)(c7)\n#     c7 = Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c7)\n    \n#     u8 = Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same')(c7)\n#     u8 = concatenate([u8, c2])\n#     c8 = Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(u8)\n#     c8 = Dropout(0.1)(c8)\n#     c8 = Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c8)\n    \n#     u9 = Conv2DTranspose(16, (2, 2), strides=(2, 2), padding='same')(c8)\n#     u9 = concatenate([u9, c1], axis=3)\n#     c9 = Conv2D(16, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(u9)\n#     c9 = Dropout(0.1)(c9)\n#     c9 = Conv2D(16, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c9)\n    \n#     outputs = Conv2D(num_classes, (1, 1), activation='sigmoid')(c9)\n    \n#     model = Model(inputs=[inputs], outputs=[outputs])\n#     return model\n","metadata":{"execution":{"iopub.status.busy":"2024-02-20T17:45:49.038953Z","iopub.status.idle":"2024-02-20T17:45:49.039295Z","shell.execute_reply.started":"2024-02-20T17:45:49.039124Z","shell.execute_reply":"2024-02-20T17:45:49.039139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from tensorflow.keras.models import Model\n# from tensorflow.keras.layers import Input, Conv2D, MaxPooling2D, Conv2DTranspose, concatenate, Dropout, BatchNormalization, Activation\n\n# def unet(input_size=(256, 256, 1), num_classes=1):\n#     inputs = Input(input_size)\n    \n#     def conv_block(input_tensor, num_filters):\n#         # Block with Convolution, Batch Normalization, and ReLU activation\n#         x = Conv2D(num_filters, (3, 3), kernel_initializer='he_normal', padding='same')(input_tensor)\n#         x = BatchNormalization()(x)\n#         x = Activation('relu')(x)\n        \n#         x = Conv2D(num_filters, (3, 3), kernel_initializer='he_normal', padding='same')(x)\n#         x = BatchNormalization()(x)\n#         x = Activation('relu')(x)\n#         return x\n    \n#     def encoder_block(input_tensor, num_filters):\n#         # Encoder block: ConvBlock + MaxPooling\n#         x = conv_block(input_tensor, num_filters)\n#         p = MaxPooling2D((2, 2))(x)\n#         p = Dropout(0.1)(p)  # Adjust dropout rate as needed\n#         return x, p\n    \n#     def decoder_block(input_tensor, concat_tensor, num_filters):\n#         # Decoder block: Upsampling, Concatenation, ConvBlock\n#         x = Conv2DTranspose(num_filters, (2, 2), strides=(2, 2), padding='same')(input_tensor)\n#         x = concatenate([x, concat_tensor], axis=3)\n#         x = Dropout(0.1)(x)  # Adjust dropout rate as needed\n#         x = conv_block(x, num_filters)\n#         return x\n    \n#     # Encoding path\n#     c1, p1 = encoder_block(inputs, 16)\n#     c2, p2 = encoder_block(p1, 32)\n#     c3, p3 = encoder_block(p2, 64)\n#     c4, p4 = encoder_block(p3, 128)\n    \n#     # Bridge\n#     b = conv_block(p4, 256)\n    \n#     # Decoding path\n#     d1 = decoder_block(b, c4, 128)\n#     d2 = decoder_block(d1, c3, 64)\n#     d3 = decoder_block(d2, c2, 32)\n#     d4 = decoder_block(d3, c1, 16)\n    \n#     # Output\n#     outputs = Conv2D(num_classes, (1, 1), activation='sigmoid')(d4)\n    \n#     model = Model(inputs=[inputs], outputs=[outputs])\n#     return model\n","metadata":{"execution":{"iopub.status.busy":"2024-02-20T17:45:49.040836Z","iopub.status.idle":"2024-02-20T17:45:49.041301Z","shell.execute_reply.started":"2024-02-20T17:45:49.041059Z","shell.execute_reply":"2024-02-20T17:45:49.041081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\n\n\nmodel = unet(input_size=(256, 256, 1), num_classes=1)\n\n\n# class F1Score(tf.keras.metrics.Metric):\n#     def __init__(self, name='f1_score', **kwargs):\n#         super(F1Score, self).__init__(name=name, **kwargs)\n#         self.precision_m = tf.keras.metrics.Precision()\n#         self.recall_m = tf.keras.metrics.Recall()\n\n#     def update_state(self, y_true, y_pred, sample_weight=None):\n#         y_pred = tf.round(y_pred)\n#         self.precision_m.update_state(y_true, y_pred, sample_weight)\n#         self.recall_m.update_state(y_true, y_pred, sample_weight)\n\n#     def result(self):\n#         precision = self.precision_m.result()\n#         recall = self.recall_m.result()\n#         return 2 * ((precision * recall) / (precision + recall + tf.keras.backend.epsilon()))\n\n#     def reset_states(self):\n#         self.precision_m.reset_states()\n#         self.recall_m.reset_states()\n\n\n# Compile the model with precision and recall\nmodel.compile(optimizer='adam',\n              loss='binary_crossentropy',\n              metrics=['accuracy'])\n","metadata":{"execution":{"iopub.status.busy":"2024-02-20T17:45:49.043058Z","iopub.status.idle":"2024-02-20T17:45:49.043404Z","shell.execute_reply.started":"2024-02-20T17:45:49.043232Z","shell.execute_reply":"2024-02-20T17:45:49.043248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.model_selection import train_test_split\n\n# # Assuming `images` and `masks` are numpy arrays containing your data.\n# X_train, X_val, y_train, y_val = train_test_split(images, masks, test_size=0.2, random_state=42)\n","metadata":{"execution":{"iopub.status.busy":"2024-02-20T17:45:49.044422Z","iopub.status.idle":"2024-02-20T17:45:49.044773Z","shell.execute_reply.started":"2024-02-20T17:45:49.044605Z","shell.execute_reply":"2024-02-20T17:45:49.044621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# val_datagen = ImageDataGenerator(rescale=1./255)  # Example of minimal preprocessing\n# val_generator = val_datagen.flow(X_val, y_val, batch_size=32)","metadata":{"execution":{"iopub.status.busy":"2024-02-20T17:45:49.046725Z","iopub.status.idle":"2024-02-20T17:45:49.047176Z","shell.execute_reply.started":"2024-02-20T17:45:49.046938Z","shell.execute_reply":"2024-02-20T17:45:49.046961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# epochs = 10\n\n# # Train the model\n# history = model.fit(\n#     train_generator,\n#     steps_per_epoch=np.ceil(len(X_train) / 32),\n#     epochs=10,\n#     validation_data=val_generator,\n#     validation_steps=np.ceil(len(X_val) / 32)\n# )\n","metadata":{"execution":{"iopub.status.busy":"2024-02-20T17:45:49.048609Z","iopub.status.idle":"2024-02-20T17:45:49.048930Z","shell.execute_reply.started":"2024-02-20T17:45:49.048771Z","shell.execute_reply":"2024-02-20T17:45:49.048786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import matplotlib.pyplot as plt\n\n# # Plot training  accuracy values\n# plt.figure(figsize=(12, 5))\n# plt.subplot(1, 2, 1)\n# plt.plot(history.history['accuracy'])\n# plt.plot(history.history['val_accuracy'])\n# plt.title('Model accuracy')\n# plt.ylabel('Accuracy')\n# plt.xlabel('Epoch')\n# plt.legend(['Train', 'Validation'], loc='upper left')\n\n# # Plot training loss values\n# plt.subplot(1, 2, 2)\n# plt.plot(history.history['loss'])\n# plt.plot(history.history['val_loss'])\n# plt.title('Model loss')\n# plt.ylabel('Loss')\n# plt.xlabel('Epoch')\n# plt.legend(['Train', 'Validation'], loc='upper left')\n\n# plt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-02-20T17:45:49.050555Z","iopub.status.idle":"2024-02-20T17:45:49.050889Z","shell.execute_reply.started":"2024-02-20T17:45:49.050725Z","shell.execute_reply":"2024-02-20T17:45:49.050741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def load_test_images(image_dir, image_size=(256, 256)):\n#     images = []\n#     image_filenames = [f for f in os.listdir(image_dir) if f.endswith('.tiff') or f.endswith('.tif')]\n    \n#     for image_filename in image_filenames:\n#         image_path = os.path.join(image_dir, image_filename)\n#         image = load_img(image_path, target_size=image_size, color_mode='grayscale')\n#         image = img_to_array(image) / 255.0  # Normalize to [0, 1]\n#         images.append(image)\n        \n#     return np.array(images)\n","metadata":{"execution":{"iopub.status.busy":"2024-02-20T17:45:49.052403Z","iopub.status.idle":"2024-02-20T17:45:49.052746Z","shell.execute_reply.started":"2024-02-20T17:45:49.052586Z","shell.execute_reply":"2024-02-20T17:45:49.052602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_base_path = '/kaggle/input/blood-vessel-segmentation/test/'  \n\n# # Load test datasets\n# X_test_kidney_5 = load_test_images(os.path.join(test_base_path, 'kidney_5', 'images'), image_size=(256, 256))\n# X_test_kidney_6 = load_test_images(os.path.join(test_base_path, 'kidney_6', 'images'), image_size=(256, 256))\n","metadata":{"execution":{"iopub.status.busy":"2024-02-20T17:45:49.053903Z","iopub.status.idle":"2024-02-20T17:45:49.054204Z","shell.execute_reply.started":"2024-02-20T17:45:49.054052Z","shell.execute_reply":"2024-02-20T17:45:49.054066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# predictions_kidney_5 = model.predict(X_test_kidney_5)\n# predictions_kidney_6 = model.predict(X_test_kidney_6)\n","metadata":{"execution":{"iopub.status.busy":"2024-02-20T17:45:49.056836Z","iopub.status.idle":"2024-02-20T17:45:49.057179Z","shell.execute_reply.started":"2024-02-20T17:45:49.057014Z","shell.execute_reply":"2024-02-20T17:45:49.057030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# threshold = 1 # Example threshold\n# binary_predictions_kidney_5 = (predictions_kidney_5 > threshold).astype(np.uint8)\n# binary_predictions_kidney_6 = (predictions_kidney_6 > threshold).astype(np.uint8)\n","metadata":{"execution":{"iopub.status.busy":"2024-02-20T17:45:49.058450Z","iopub.status.idle":"2024-02-20T17:45:49.058818Z","shell.execute_reply.started":"2024-02-20T17:45:49.058647Z","shell.execute_reply":"2024-02-20T17:45:49.058663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# binary_predictions_kidney_5","metadata":{"execution":{"iopub.status.busy":"2024-02-20T17:45:49.060600Z","iopub.status.idle":"2024-02-20T17:45:49.061049Z","shell.execute_reply.started":"2024-02-20T17:45:49.060814Z","shell.execute_reply":"2024-02-20T17:45:49.060835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import matplotlib.pyplot as plt\n\n# def plot_predictions(images, predictions, num=5):\n#     plt.figure(figsize=(10, num * 4))\n#     for i in range(num):\n#         plt.subplot(num, 2, 2*i+1)\n#         plt.imshow(images[i].squeeze(), cmap='gray')\n#         plt.title(\"Original Image\")\n#         plt.axis('off')\n        \n#         plt.subplot(num, 2, 2*i+2)\n#         plt.imshow(predictions[i].squeeze(), cmap='gray')\n#         plt.title(\"Predicted Mask\")\n#         plt.axis('off')\n#     plt.tight_layout()\n#     plt.show()\n\n# # Plot some predictions for kidney_5 and kidney_6\n# plot_predictions(X_test_kidney_5, binary_predictions_kidney_5, num=3)\n# plot_predictions(X_test_kidney_6, binary_predictions_kidney_6, num=3)\n","metadata":{"execution":{"iopub.status.busy":"2024-02-20T17:45:49.062610Z","iopub.status.idle":"2024-02-20T17:45:49.063055Z","shell.execute_reply.started":"2024-02-20T17:45:49.062819Z","shell.execute_reply":"2024-02-20T17:45:49.062840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import matplotlib.pyplot as plt\n\n# # Example: Visualizing the first raw prediction\n# plt.imshow(predictions_kidney_5[0].squeeze(), cmap='gray')\n# plt.colorbar()\n# plt.title('Raw Prediction')\n# plt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-02-20T17:45:49.064263Z","iopub.status.idle":"2024-02-20T17:45:49.064737Z","shell.execute_reply.started":"2024-02-20T17:45:49.064499Z","shell.execute_reply":"2024-02-20T17:45:49.064523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. DeepLabv3+","metadata":{}},{"cell_type":"code","source":"import torch\nimport torchvision\nfrom torch.utils.data import Dataset\nfrom torchvision import transforms\nfrom albumentations.pytorch import ToTensorV2\nfrom torchvision.transforms import Compose, ToTensor, Normalize, Resize\nimport tifffile as tiff\nimport cv2\nimport torch.nn as nn\nimport albumentations as A\nimport numpy as np\nimport os\nimport time\nimport torch.nn.functional as F\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom torch.utils.data import DataLoader","metadata":{"execution":{"iopub.status.busy":"2024-02-23T20:55:52.729645Z","iopub.execute_input":"2024-02-23T20:55:52.730000Z","iopub.status.idle":"2024-02-23T20:55:52.738688Z","shell.execute_reply.started":"2024-02-23T20:55:52.729971Z","shell.execute_reply":"2024-02-23T20:55:52.737701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# class SegmentationDataset(Dataset):\n#     def __init__(self, image_dir, mask_dir, transforms=None):\n#         self.image_dir = image_dir\n#         self.mask_dir = mask_dir\n#         self.transforms = transforms\n#         self.image_filenames = [f for f in os.listdir(image_dir) if f.endswith('.tiff') or f.endswith('.tif')]\n\n#     def __len__(self):\n#         return len(self.image_filenames)\n    \n#     def __getitem__(self, idx):\n#         #print(f\"Loading item {idx}\")\n#         image_path = os.path.join(self.image_dir, self.image_filenames[idx])\n#         mask_path = os.path.join(self.mask_dir, self.image_filenames[idx])\n        \n#         # Preprocess the image\n#         # Load the image and mask in grayscale\n#         image = cv2.imread(image_path, cv2.IMREAD_GRAYSCALE)\n#         mask = cv2.imread(mask_path, cv2.IMREAD_GRAYSCALE)\n        \n#         # Ensure image and mask are 3D (H, W, C) after loading in grayscale\n#         image = np.expand_dims(image, axis=-1)\n#         mask = np.expand_dims(mask, axis=-1)\n\n#         # Normalize images and masks to [0, 1]\n#         image = image.astype(np.float32) / 255.0\n#         mask = mask.astype(np.float32) / 255.0\n\n#         # Apply transformations if specified\n#         if self.transforms:\n#             augmented = self.transforms(image=image, mask=mask)\n#             image = augmented['image']\n#             mask = augmented['mask']\n\n#         # Convert image and mask to PyTorch tensors\n# #         image = torch.from_numpy(image).permute(2, 0, 1)  # Convert to tensor and CHW format\n# #         mask = torch.from_numpy(mask).permute(2, 0, 1)  # Convert to tensor and CHW format\n\n#         return image, mask","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nclass SegmentationDataset(Dataset):\n    def __init__(self, image_files, mask_files, input_size=(256, 256), augmentation_transforms=None):\n        self.image_files = image_files\n        self.mask_files = mask_files\n        self.input_size = input_size\n        self.augmentation_transforms = augmentation_transforms\n        \n    def __len__(self):\n        return len(self.image_files)\n    \n    def __getitem__(self, idx):\n        image_path = self.image_files[idx]\n        mask_path = self.mask_files[idx]\n\n        image = preprocess_image(image_path)\n        mask = preprocess_mask(mask_path)\n\n        if self.augmentation_transforms:\n            image, mask = self.augmentation_transforms(image, mask)\n\n        return image, mask\n\n","metadata":{"execution":{"iopub.status.busy":"2024-02-23T20:55:55.546704Z","iopub.execute_input":"2024-02-23T20:55:55.547063Z","iopub.status.idle":"2024-02-23T20:55:55.554355Z","shell.execute_reply.started":"2024-02-23T20:55:55.547035Z","shell.execute_reply":"2024-02-23T20:55:55.553372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess_image(path):\n    img = cv2.imread(path, cv2.IMREAD_UNCHANGED)\n    img = img.astype('float32') \n    mx = np.max(img)\n    if mx:\n        img /= mx  # Normalizing the image\n    img = np.expand_dims(img, axis=0)  # Adding channel dimension (C, H, W) format\n    img_ten = torch.tensor(img)\n    return img_ten\n","metadata":{"execution":{"iopub.status.busy":"2024-02-23T20:55:57.883696Z","iopub.execute_input":"2024-02-23T20:55:57.884431Z","iopub.status.idle":"2024-02-23T20:55:57.889946Z","shell.execute_reply.started":"2024-02-23T20:55:57.884399Z","shell.execute_reply":"2024-02-23T20:55:57.889025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess_mask(path):\n    \n    msk = cv2.imread(path, cv2.IMREAD_UNCHANGED)\n    msk = msk.astype('float32')\n    msk/=255.0\n    msk_ten = torch.tensor(msk)\n    \n    return msk_ten","metadata":{"execution":{"iopub.status.busy":"2024-02-23T20:55:59.973399Z","iopub.execute_input":"2024-02-23T20:55:59.973757Z","iopub.status.idle":"2024-02-23T20:55:59.978729Z","shell.execute_reply.started":"2024-02-23T20:55:59.973730Z","shell.execute_reply":"2024-02-23T20:55:59.977804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def augment_image(image, mask):\n    \n    image_np = image.permute(1, 2, 0).numpy()\n    mask_np = mask.numpy()\n\n    transform = A.Compose([\n        A.Resize(256,256, interpolation=cv2.INTER_NEAREST),\n        A.HorizontalFlip(p=0.5),\n        A.VerticalFlip(p=0.5),\n        A.ShiftScaleRotate(scale_limit=0.5, rotate_limit=0, shift_limit=0.1, p=1, border_mode=0),\n        A.RandomCrop(height=256, width=256, always_apply=True),\n        A.RandomBrightness(p=1),\n        A.OneOf(\n            [\n                A.Blur(blur_limit=3, p=1),\n                A.MotionBlur(blur_limit=3, p=1),\n            ],\n            p=0.9,\n        ),\n    \n    ])\n\n    augmented = transform(image=image_np, mask=mask_np)\n    augmented_image, augmented_mask = augmented['image'], augmented['mask']\n\n    augmented_image = torch.tensor(augmented_image, dtype=torch.float32).permute(2, 0, 1)\n    augmented_mask = torch.tensor(augmented_mask, dtype=torch.float32)\n\n    return augmented_image, augmented_mask","metadata":{"execution":{"iopub.status.busy":"2024-02-23T20:56:03.536080Z","iopub.execute_input":"2024-02-23T20:56:03.536778Z","iopub.status.idle":"2024-02-23T20:56:03.545082Z","shell.execute_reply.started":"2024-02-23T20:56:03.536745Z","shell.execute_reply":"2024-02-23T20:56:03.544221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import albumentations as A\n# from albumentations.pytorch import ToTensorV2\n\n# def get_augmentation():\n#     return A.Compose([\n#         A.Resize(256, 256),\n#         A.HorizontalFlip(p=0.5),\n#         A.VerticalFlip(p=0.5),\n       \n#         ToTensorV2(),  # This is important for converting to PyTorch tensors\n#     ])","metadata":{"execution":{"iopub.status.busy":"2024-02-21T08:58:48.373023Z","iopub.execute_input":"2024-02-21T08:58:48.373391Z","iopub.status.idle":"2024-02-21T08:58:48.381539Z","shell.execute_reply.started":"2024-02-21T08:58:48.373358Z","shell.execute_reply":"2024-02-21T08:58:48.380574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_deeplabv3_plus(num_classes, pretrained=True, fine_tune=True):\n    model = torchvision.models.segmentation.deeplabv3_resnet101(pretrained=pretrained)\n    \n    # Adjust the first convolutional layer for grayscale input, if your images are grayscale\n    model.backbone.conv1 = torch.nn.Conv2d(1, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n    print(model.backbone.conv1)\n    model.classifier[4] = torch.nn.Conv2d(256, num_classes, kernel_size=(1, 1))\n    \n    if fine_tune:\n        for parameter in model.parameters():\n            parameter.requires_grad = False\n        # Unfreeze the classifier and the modified first conv layer\n        for parameter in model.classifier.parameters():\n            parameter.requires_grad = True\n        for parameter in model.backbone.conv1.parameters():\n            parameter.requires_grad = True\n    \n    return model\n","metadata":{"execution":{"iopub.status.busy":"2024-02-23T20:56:10.337534Z","iopub.execute_input":"2024-02-23T20:56:10.337911Z","iopub.status.idle":"2024-02-23T20:56:10.345508Z","shell.execute_reply.started":"2024-02-23T20:56:10.337883Z","shell.execute_reply":"2024-02-23T20:56:10.344508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from tqdm import tqdm\n# def train_model(model, dataloader, optimizer, criterion, num_epochs=25):\n#     print(\"Starting training...\")\n#     model.train()  # Set the model to training mode\n#     for epoch in range(num_epochs):\n#         epoch_loss = 0.0\n        \n#         progress_bar = tqdm(enumerate(dataloader), total=len(dataloader), desc=f'Epoch {epoch+1}/{num_epochs}')\n#         for batch_idx, (images, masks) in progress_bar:\n#             images = images.to(device)\n#             masks = masks.to(device)\n#             optimizer.zero_grad()\n            \n#             # Move data to the appropriate device (e.g., GPU if available)\n#             images = images.to(device)\n#             masks = masks.to(device)\n            \n#             outputs = model(images)['out']\n#             loss = criterion(outputs, masks)\n#             loss.backward()\n#             optimizer.step()\n            \n#             epoch_loss += loss.item()\n#             progress_bar.set_postfix({'loss': loss.item()})\n            \n        \n        \n#         print(f'Epoch {epoch+1}/{num_epochs}, Loss: {epoch_loss/len(dataloader)}')\n#     print(\"Training completed.\")","metadata":{"execution":{"iopub.status.busy":"2024-02-23T20:28:32.180617Z","iopub.execute_input":"2024-02-23T20:28:32.181455Z","iopub.status.idle":"2024-02-23T20:28:32.189174Z","shell.execute_reply.started":"2024-02-23T20:28:32.181424Z","shell.execute_reply":"2024-02-23T20:28:32.188281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DiceLoss(torch.nn.Module):\n    def forward(self, inputs, targets, smooth=1e-6):\n        inputs = torch.sigmoid(inputs)  # Ensure inputs are probabilities\n        assert inputs.min() >= 0 and inputs.max() <= 1, \"Probabilities are out of range.\"\n        #assert targets.min() >= 0 and targets.max() <= 1, \"Target masks are out of range.\"\n        # Flatten label and prediction tensors\n        inputs = inputs.view(-1)\n        targets = targets.view(-1)\n        \n        intersection = (inputs * targets).sum()\n        dice = (2.*intersection + smooth)/(inputs.sum() + targets.sum() + smooth)\n        return 1 - dice\n","metadata":{"execution":{"iopub.status.busy":"2024-02-23T20:56:17.072614Z","iopub.execute_input":"2024-02-23T20:56:17.073578Z","iopub.status.idle":"2024-02-23T20:56:17.081299Z","shell.execute_reply.started":"2024-02-23T20:56:17.073528Z","shell.execute_reply":"2024-02-23T20:56:17.080202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def dice_coeff(prediction, target, smooth=1e-6):\n    # Ensure prediction is in probability space if it's in logits space\n    prediction = torch.sigmoid(prediction)\n\n    # Flatten the tensors to calculate intersection and union across the batch\n    prediction_flat = prediction.view(-1)\n    target_flat = target.view(-1)\n    \n    intersection = (prediction_flat * target_flat).sum()\n    union = prediction_flat.sum() + target_flat.sum()\n    \n    dice = (2. * intersection + smooth) / (union + smooth)\n    return dice\n","metadata":{"execution":{"iopub.status.busy":"2024-02-23T21:33:50.771975Z","iopub.execute_input":"2024-02-23T21:33:50.772984Z","iopub.status.idle":"2024-02-23T21:33:50.779827Z","shell.execute_reply.started":"2024-02-23T21:33:50.772935Z","shell.execute_reply":"2024-02-23T21:33:50.778833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class FocalLoss(nn.modules.loss._WeightedLoss):\n\n    def __init__(self, gamma=0, size_average=None, ignore_index=-100,\n                 reduce=None, balance_param=1.0):\n        super(FocalLoss, self).__init__(size_average)\n        self.gamma = gamma\n        self.size_average = size_average\n        self.ignore_index = ignore_index\n        self.balance_param = balance_param\n\n    def forward(self, input, target):\n        \n        assert len(input.shape) == len(target.shape)\n        assert input.size(0) == target.size(0)\n        assert input.size(1) == target.size(1)\n\n        logpt = - F.binary_cross_entropy_with_logits(input, target)\n        pt = torch.exp(logpt)\n\n        focal_loss = -((1 - pt) ** self.gamma) * logpt\n        balanced_focal_loss = self.balance_param * focal_loss\n        return balanced_focal_loss","metadata":{"execution":{"iopub.status.busy":"2024-02-23T21:33:55.291522Z","iopub.execute_input":"2024-02-23T21:33:55.291881Z","iopub.status.idle":"2024-02-23T21:33:55.299734Z","shell.execute_reply.started":"2024-02-23T21:33:55.291852Z","shell.execute_reply":"2024-02-23T21:33:55.298834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\ndef get_dataset_file_paths(base_path, datasets):\n    image_files = []\n    label_files = []\n\n    for dataset in datasets:\n        images_path = os.path.join(base_path, dataset, 'images')\n        labels_path = os.path.join(base_path, dataset, 'labels')\n\n        dataset_image_files = sorted([os.path.join(images_path, f) for f in os.listdir(images_path) if f.endswith('.tif')])\n        dataset_label_files = sorted([os.path.join(labels_path, f) for f in os.listdir(labels_path) if f.endswith('.tif')])\n\n        image_files.extend(dataset_image_files)\n        label_files.extend(dataset_label_files)\n\n    return image_files, label_files\n\nbase_path = '/kaggle/input/blood-vessel-segmentation/train'\ndatasets = ['kidney_1_dense', 'kidney_2']\nimage_files, label_files = get_dataset_file_paths(base_path, datasets)\n","metadata":{"execution":{"iopub.status.busy":"2024-02-23T20:58:05.723646Z","iopub.execute_input":"2024-02-23T20:58:05.724364Z","iopub.status.idle":"2024-02-23T20:58:05.759809Z","shell.execute_reply.started":"2024-02-23T20:58:05.724310Z","shell.execute_reply":"2024-02-23T20:58:05.758867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_image_files, val_image_files, train_mask_files, val_mask_files = train_test_split(\n    image_files, label_files, test_size=0.2, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2024-02-23T20:58:07.624585Z","iopub.execute_input":"2024-02-23T20:58:07.624943Z","iopub.status.idle":"2024-02-23T20:58:07.632866Z","shell.execute_reply.started":"2024-02-23T20:58:07.624916Z","shell.execute_reply":"2024-02-23T20:58:07.632084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = SegmentationDataset(train_image_files, train_mask_files, augmentation_transforms=augment_image)\nval_dataset = SegmentationDataset(val_image_files, val_mask_files, augmentation_transforms=augment_image)","metadata":{"execution":{"iopub.status.busy":"2024-02-23T20:58:10.639109Z","iopub.execute_input":"2024-02-23T20:58:10.639499Z","iopub.status.idle":"2024-02-23T20:58:10.644198Z","shell.execute_reply.started":"2024-02-23T20:58:10.639466Z","shell.execute_reply":"2024-02-23T20:58:10.643373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataloader = DataLoader(train_dataset, batch_size=8, shuffle=True)\nval_dataloader = DataLoader(val_dataset, batch_size=8, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2024-02-23T20:58:16.276205Z","iopub.execute_input":"2024-02-23T20:58:16.276573Z","iopub.status.idle":"2024-02-23T20:58:16.281473Z","shell.execute_reply.started":"2024-02-23T20:58:16.276545Z","shell.execute_reply":"2024-02-23T20:58:16.280478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataloaders = {\n    'training': train_dataloader,\n    'test': val_dataloader\n}","metadata":{"execution":{"iopub.status.busy":"2024-02-23T20:58:17.583225Z","iopub.execute_input":"2024-02-23T20:58:17.584167Z","iopub.status.idle":"2024-02-23T20:58:17.588467Z","shell.execute_reply.started":"2024-02-23T20:58:17.584132Z","shell.execute_reply":"2024-02-23T20:58:17.587398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for batch_idx, (batch_images, batch_masks) in enumerate(train_dataloader):\n    print(\"Batch\", batch_idx + 1)\n    print(\"Image batch shape:\", batch_images.shape)\n    print(\"Mask batch shape:\", batch_masks.shape)\n    \n    for image, mask, image_path, mask_path in zip(batch_images, batch_masks, train_image_files, train_mask_files):\n       \n        image = image.permute((1, 2, 0)).numpy()*255.0\n        image = image.astype('uint8')\n        mask = (mask*255).numpy().astype('uint8')\n        \n        image_filename = os.path.basename(image_path)\n        mask_filename = os.path.basename(mask_path)\n        \n        plt.figure(figsize=(15, 10))\n        \n        plt.subplot(2, 4, 1)\n        plt.imshow(image, cmap='gray')\n        plt.title(f\"Original Image - {image_filename}\")\n        \n        plt.subplot(2, 4, 2)\n        plt.imshow(mask, cmap='gray')\n        plt.title(f\"Mask Image - {mask_filename}\")\n        \n        plt.tight_layout()\n        plt.show()\n    break","metadata":{"execution":{"iopub.status.busy":"2024-02-23T20:58:20.480356Z","iopub.execute_input":"2024-02-23T20:58:20.481034Z","iopub.status.idle":"2024-02-23T20:58:25.175276Z","shell.execute_reply.started":"2024-02-23T20:58:20.480995Z","shell.execute_reply":"2024-02-23T20:58:25.174446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_default_device():\n    \"\"\"Pick GPU if available, else CPU\"\"\"\n    if torch.cuda.is_available():\n        return torch.device('cuda')\n    else:\n        return torch.device('cpu')\ndevice = get_default_device()\ndevice","metadata":{"execution":{"iopub.status.busy":"2024-02-23T20:58:28.573250Z","iopub.execute_input":"2024-02-23T20:58:28.574150Z","iopub.status.idle":"2024-02-23T20:58:28.580924Z","shell.execute_reply.started":"2024-02-23T20:58:28.574114Z","shell.execute_reply":"2024-02-23T20:58:28.580041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\nimport numpy as np\nimport torch\n\ndef train_and_test(model, dataloaders, optimizer, criterion, num_epochs=100, show_images=False):\n    since = time.time()\n    best_loss = float('inf')  # Use 'inf' for initial comparison\n\n    device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n    model.to(device)\n\n    for epoch in range(1, num_epochs + 1):\n        print(f'Epoch {epoch}/{num_epochs}')\n        \n        # Initialize metrics for the current epoch\n        epoch_metrics = {\n            'training_loss': 0.0,\n            'test_loss': 0.0,\n            'training_dice_coeff': [],\n            'test_dice_coeff': []\n        }\n\n        for phase in ['training', 'test']:\n            if phase == 'training':\n                model.train()\n            else:\n                model.eval()\n\n            running_loss = 0.0\n            dice_coeff_list = []  # List to store dice coefficients for each batch\n            \n            # Progress bar setup\n            progress_bar = tqdm(enumerate(dataloaders[phase]), total=len(dataloaders[phase]), desc=f\"{phase.capitalize()} Phase\")\n            \n            for batch_idx, (inputs, masks) in progress_bar:\n                inputs = inputs.to(device)\n                masks = masks.to(device)\n                # Adjust mask shape to match output shape if necessary\n                masks = masks.unsqueeze(1) if masks.ndim == 3 else masks  # Ensure masks have a channel dimension\n                optimizer.zero_grad()\n\n                with torch.set_grad_enabled(phase == 'training'):\n                    outputs = model(inputs)['out']\n                    loss = criterion(outputs, masks)\n\n                    if phase == 'training':\n                        loss.backward()\n                        optimizer.step()\n\n                running_loss += loss.item() * inputs.size(0)\n                # Update dice_coeff_list with the dice coefficient for the current batch\n                dice_value = dice_coeff(outputs, masks)  # Assuming dice_coeff function is corrected as suggested\n                dice_coeff_list.append(dice_value.item())\n\n                # Update progress bar\n                progress_bar.set_postfix(loss=loss.item(), dice_coeff=dice_value.item())\n\n            epoch_loss = running_loss / len(dataloaders[phase].dataset)\n            epoch_dice = np.mean(dice_coeff_list)\n            epoch_metrics[f'{phase}_loss'] = epoch_loss\n            epoch_metrics[f'{phase}_dice_coeff'] = epoch_dice\n\n            print(f\"{phase.capitalize()} Loss: {epoch_loss:.4f}, {phase.capitalize()} Dice Coeff: {epoch_dice:.4f}\")\n        \n        if epoch_metrics['test_loss'] < best_loss:\n            best_loss = epoch_metrics['test_loss']\n            # Consider implementing model checkpointing here to save the best model\n\n        print('-' * 10)\n    \n    print(f'Best test loss: {best_loss:.4f}')\n    print('Training completed.')\n\n    return model\n","metadata":{"execution":{"iopub.status.busy":"2024-02-23T21:34:01.714478Z","iopub.execute_input":"2024-02-23T21:34:01.714842Z","iopub.status.idle":"2024-02-23T21:34:01.728775Z","shell.execute_reply.started":"2024-02-23T21:34:01.714815Z","shell.execute_reply":"2024-02-23T21:34:01.727904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# # Initialize your dataset with the correct augmentation function\n# base_path = '/kaggle/input/blood-vessel-segmentation/train/'\n\n# # Loop through the datasets and initialize a DataLoader for each\n# # for dataset in os.listdir(base_path):\n# dataset='kidney_1_dense'\n# images_path = os.path.join(base_path, dataset, 'images')\n# labels_path = os.path.join(base_path, dataset, 'labels')\n\n# # Check if the paths exist before proceeding\n# if os.path.exists(images_path) and os.path.exists(labels_path):\n#     seg_dataset = SegmentationDataset(\n#         image_dir=images_path,\n#         mask_dir=labels_path,\n#         #image_size=(256, 256),\n#         transforms=get_augmentation()  # defined augmentation function\n#     )\n#     seg_dataloader = DataLoader(seg_dataset, batch_size=20, shuffle=True, num_workers=4, pin_memory=True)\n\n#     # Initialize the model\n#     model = create_deeplabv3_plus(num_classes=1, pretrained=True, fine_tune=True)\n\n#     # Define optimizer and loss function\n#     optimizer = torch.optim.Adam(model.parameters(), lr=1e-4)\n#     criterion = DiceLoss()\n#     device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n#     model = model.to(device)\n\n#     # Train the model\n#     try:\n#         train_model(model, seg_dataloader, optimizer, criterion, num_epochs=15)\n#     except Exception as e:\n#         print(f\"Training failed due to an exception: {e}\")\n\n#     else:\n#         print(f\"Directory not found for dataset: {dataset}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-02-23T20:57:18.977476Z","iopub.execute_input":"2024-02-23T20:57:18.978189Z","iopub.status.idle":"2024-02-23T20:57:18.983127Z","shell.execute_reply.started":"2024-02-23T20:57:18.978158Z","shell.execute_reply":"2024-02-23T20:57:18.982153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from torchvision.transforms.functional import to_pil_image\n# import matplotlib.pyplot as plt\n# import torch\n# from torchvision import transforms\n\n# # Function to load and preprocess a single image\n# def load_and_preprocess_image(image_path):\n#     image = cv2.imread(image_path, cv2.IMREAD_GRAYSCALE)\n#     image = image.astype(np.float32) / 255.0\n#     preprocess = transforms.Compose([\n#         transforms.ToPILImage(),\n#         transforms.Resize((256, 256)),\n#         transforms.ToTensor()])\n    \n#     image_tensor = preprocess(image)\n    \n#     return image_tensor\n\n# # Load and preprocess test images\n# test_images_dir = '/kaggle/input/blood-vessel-segmentation/test/'\n# # Assuming each directory 'kidney_X/images/' contains one or more image files\n# test_image_paths = []\n# for i in [5, 6]:\n#     dir_path = os.path.join(test_images_dir, f\"kidney_{i}/images/\")\n#     for filename in os.listdir(dir_path):\n#         if filename.endswith(\".tif\"):\n#             test_image_paths.append(os.path.join(dir_path, filename))\n\n# test_images = [load_and_preprocess_image(path) for path in test_image_paths]\n# # Convert to a batched tensor\n# test_images_tensor = torch.stack(test_images)\n# test_images_tensor = test_images_tensor.to(device)\n","metadata":{"execution":{"iopub.status.busy":"2024-02-21T10:36:13.646999Z","iopub.execute_input":"2024-02-21T10:36:13.647902Z","iopub.status.idle":"2024-02-21T10:36:13.714751Z","shell.execute_reply.started":"2024-02-21T10:36:13.647869Z","shell.execute_reply":"2024-02-21T10:36:13.713951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = 25\n\ndef train(num_classes=1):  # Set a default value for num_classes if it's always going to be 1\n    model = create_deeplabv3_plus(num_classes=num_classes, pretrained=True, fine_tune=True)\n    optimizer = torch.optim.Adam(model.parameters(), lr=1e-4)\n    criterion = FocalLoss(gamma=2)\n\n    trained_model, train_epoch_losses, test_epoch_losses = train_and_test(\n        model, dataloaders, optimizer, criterion, num_epochs=epochs\n    )\n\n    return trained_model, train_epoch_losses, test_epoch_losses\n\n# Now when you call train, you can specify the number of classes if needed\ntrained_model, train_epoch_losses, test_epoch_losses = train(num_classes=1)\n","metadata":{"execution":{"iopub.status.busy":"2024-02-23T21:34:05.710787Z","iopub.execute_input":"2024-02-23T21:34:05.711505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def visualize_predictions(model, dataloader, device):\n    model.eval()  # Set the model to evaluation mode\n\n    with torch.no_grad():  # No need to track gradients\n        for sample in iter(dataloader):\n            inputs = sample[0].to(device)\n            true_masks = sample[1].to(device)\n\n            # Get model predictions\n            predicted_masks = model(inputs)\n\n            # Convert tensors to numpy arrays for visualization\n            inputs_np = inputs.cpu().numpy()\n            true_masks_np = true_masks.cpu().numpy()\n            predicted_masks_np = predicted_masks.cpu().numpy()\n\n            # Visualize images, true masks, and predicted masks\n            for i in range(inputs_np.shape[0]):  # Loop through batch\n                plt.figure(figsize=(15, 5))\n\n                plt.subplot(1, 3, 1)\n                plt.imshow(inputs_np[i].transpose(1, 2, 0))  # Assuming input is CxHxW\n                plt.title(\"Original Image\")\n\n                plt.subplot(1, 3, 2)\n                plt.imshow(true_masks_np[i].squeeze(), cmap='gray')  # Assuming mask is 1xHxW\n                plt.title(\"True Mask\")\n\n                plt.subplot(1, 3, 3)\n                plt.imshow(predicted_masks_np[i].squeeze(), cmap='gray')  # Adjust if necessary\n                plt.title(\"Predicted Mask\")\n\n                plt.show()\n\n            break  # Remove this to visualize all batches\n\n# After training\nval_dataloader = dataloaders['test']  # Assuming you have a validation dataloader\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nvisualize_predictions(trained_model, val_dataloader, device)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Set the model to evaluation mode\n# model.eval()\n\n# # Disable gradient computation for inference\n# with torch.no_grad():\n#     predictions = model(test_images_tensor)['out']\n\n# # Apply a threshold to get binary mask predictions\n# threshold = 0.0001\n# predicted_masks = (predictions.sigmoid() > threshold).float()\n","metadata":{"execution":{"iopub.status.busy":"2024-02-21T10:43:21.312253Z","iopub.execute_input":"2024-02-21T10:43:21.313201Z","iopub.status.idle":"2024-02-21T10:43:21.337470Z","shell.execute_reply.started":"2024-02-21T10:43:21.313165Z","shell.execute_reply":"2024-02-21T10:43:21.336505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predicted_masks","metadata":{"execution":{"iopub.status.busy":"2024-02-21T10:43:23.469871Z","iopub.execute_input":"2024-02-21T10:43:23.470990Z","iopub.status.idle":"2024-02-21T10:43:23.484035Z","shell.execute_reply.started":"2024-02-21T10:43:23.470944Z","shell.execute_reply":"2024-02-21T10:43:23.483071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to visualize the predictions\ndef visualize_prediction(image_tensor, mask_tensor):\n    image = to_pil_image(image_tensor)\n    mask = to_pil_image(mask_tensor)  \n    plt.figure(figsize=(10, 5))\n    plt.subplot(1, 2, 1)\n    plt.imshow(image, cmap='gray')\n    plt.title(\"Original Image\")\n    plt.subplot(1, 2, 2)\n    plt.imshow(mask, cmap='gray')\n    plt.title(\"Predicted Mask\")\n    plt.show()\n\n# Visualize predictions for the test images\nfor image_tensor, mask_tensor in zip(test_images_tensor, predicted_masks):\n    visualize_prediction(image_tensor, mask_tensor)\n","metadata":{"execution":{"iopub.status.busy":"2024-02-21T10:43:26.094841Z","iopub.execute_input":"2024-02-21T10:43:26.095595Z","iopub.status.idle":"2024-02-21T10:43:28.881089Z","shell.execute_reply.started":"2024-02-21T10:43:26.095561Z","shell.execute_reply":"2024-02-21T10:43:28.880147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}