{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":9988,"databundleVersionId":868324,"sourceType":"competition"},{"sourceId":79775,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":67031,"modelId":92056}],"dockerImageVersionId":30919,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true},"papermill":{"default_parameters":{},"duration":113.015467,"end_time":"2024-07-18T18:26:47.801075","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-07-18T18:24:54.785608","version":"2.5.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"dc2bad53","cell_type":"markdown","source":"# Import necessary libraries","metadata":{"papermill":{"duration":0.009175,"end_time":"2024-07-18T18:24:58.099172","exception":false,"start_time":"2024-07-18T18:24:58.089997","status":"completed"},"tags":[]}},{"id":"e04543cb","cell_type":"code","source":"import os\nimport random\n\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nimport seaborn as sns\n\nimport numpy as np\nimport pandas as pd\nimport cv2","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2025-03-05T05:20:12.842524Z","iopub.execute_input":"2025-03-05T05:20:12.842916Z","iopub.status.idle":"2025-03-05T05:20:15.365577Z","shell.execute_reply.started":"2025-03-05T05:20:12.842880Z","shell.execute_reply":"2025-03-05T05:20:15.364637Z"},"papermill":{"duration":3.374421,"end_time":"2024-07-18T18:25:01.482484","exception":false,"start_time":"2024-07-18T18:24:58.108063","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"94821506","cell_type":"markdown","source":"# Load dataset","metadata":{"papermill":{"duration":0.009971,"end_time":"2024-07-18T18:25:01.502250","exception":false,"start_time":"2024-07-18T18:25:01.492279","status":"completed"},"tags":[]}},{"id":"48214887","cell_type":"code","source":"train_path = '/kaggle/input/airbus-ship-detection/train_v2'\ntrain_segmentation_path = '/kaggle/input/airbus-ship-detection/train_ship_segmentations_v2.csv'\n\ntrain_segmentations = pd.read_csv(train_segmentation_path)\ntrain_segmentations.shape","metadata":{"execution":{"iopub.status.busy":"2025-03-05T05:20:18.849144Z","iopub.execute_input":"2025-03-05T05:20:18.849435Z","iopub.status.idle":"2025-03-05T05:20:19.984923Z","shell.execute_reply.started":"2025-03-05T05:20:18.849412Z","shell.execute_reply":"2025-03-05T05:20:19.984046Z"},"papermill":{"duration":1.333534,"end_time":"2024-07-18T18:25:02.847308","exception":false,"start_time":"2024-07-18T18:25:01.513774","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"af6d88bd","cell_type":"code","source":"print(train_segmentations['EncodedPixels'][24])\ntrain_segmentations.head()","metadata":{"execution":{"iopub.status.busy":"2025-03-05T05:20:22.138740Z","iopub.execute_input":"2025-03-05T05:20:22.139062Z","iopub.status.idle":"2025-03-05T05:20:22.160484Z","shell.execute_reply.started":"2025-03-05T05:20:22.139035Z","shell.execute_reply":"2025-03-05T05:20:22.159670Z"},"papermill":{"duration":0.046152,"end_time":"2024-07-18T18:25:02.903248","exception":false,"start_time":"2024-07-18T18:25:02.857096","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"32f6c0bb","cell_type":"code","source":"print('isnull:\\n', train_segmentations.isnull().sum())\nprint('isna:\\n',  train_segmentations.isna().sum())","metadata":{"execution":{"iopub.status.busy":"2025-03-05T05:20:26.127510Z","iopub.execute_input":"2025-03-05T05:20:26.127944Z","iopub.status.idle":"2025-03-05T05:20:26.197789Z","shell.execute_reply.started":"2025-03-05T05:20:26.127903Z","shell.execute_reply":"2025-03-05T05:20:26.196821Z"},"papermill":{"duration":0.126202,"end_time":"2024-07-18T18:25:03.040073","exception":false,"start_time":"2024-07-18T18:25:02.913871","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"decc7b21","cell_type":"code","source":"train_segmentations.nunique() # id -> pixels is not 'onto' mapping wtf?","metadata":{"execution":{"iopub.status.busy":"2025-03-05T05:20:29.088945Z","iopub.execute_input":"2025-03-05T05:20:29.089239Z","iopub.status.idle":"2025-03-05T05:20:29.185443Z","shell.execute_reply.started":"2025-03-05T05:20:29.089215Z","shell.execute_reply":"2025-03-05T05:20:29.184475Z"},"papermill":{"duration":0.171592,"end_time":"2024-07-18T18:25:03.222509","exception":false,"start_time":"2024-07-18T18:25:03.050917","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"98998cc0","cell_type":"markdown","source":"# Data Description\nEncodedPixels - a list of pixels for ship segmentation in a compressed format (in run-length encoding format) (RLE).\n\nEncodedPixels сonsists of pairs of values that contain a start position and a run length. E.g. '1 3' implies starting at pixel 1 and running a total of 3 pixels (1,2,3).\n\nThe pixels are one-indexed and numbered from top to bottom, then left to right: 1 is pixel (1,1), 2 is pixel (2,1), etc.\n\nA prediction of of \"no ship in image\" have a blank value in the EncodedPixels column.\n\nObject segments do not overlap.","metadata":{"papermill":{"duration":0.010291,"end_time":"2024-07-18T18:25:03.244275","exception":false,"start_time":"2024-07-18T18:25:03.233984","status":"completed"},"tags":[]}},{"id":"8413c6ac","cell_type":"markdown","source":"# Some useful functions","metadata":{"papermill":{"duration":0.009915,"end_time":"2024-07-18T18:25:03.265479","exception":false,"start_time":"2024-07-18T18:25:03.255564","status":"completed"},"tags":[]}},{"id":"e75d25e6","cell_type":"code","source":"def rle_decode(mask_rle, shape=(768, 768)) -> np.array:\n    \"\"\"\n    decode run-length encoded segmentation mask\n    Assumed all images aRe 768x768 (and ThereforE have the saMe shape)\n    \"\"\"\n    \n    # if no segmentation mask (nan) return matrix of zeros\n    if not mask_rle or pd.isna(mask_rle):\n        return np.zeros(shape, dtype=np.uint8)\n\n    # RLE sequence str split to and map to int\n    s = list(map(int, mask_rle.split()))\n\n    img = np.zeros(shape[0] * shape[1], dtype=np.uint8)\n\n    # indices: 2k - starts, 2k+1 lengths\n    starts, lengths = s[0::2], s[1::2]\n    for start, length in zip(starts, lengths):\n        img[start:start + length] = 1\n\n    return img.reshape(shape).T","metadata":{"execution":{"iopub.status.busy":"2025-03-05T05:20:35.126302Z","iopub.execute_input":"2025-03-05T05:20:35.126627Z","iopub.status.idle":"2025-03-05T05:20:35.132108Z","shell.execute_reply.started":"2025-03-05T05:20:35.126594Z","shell.execute_reply":"2025-03-05T05:20:35.131186Z"},"papermill":{"duration":0.02538,"end_time":"2024-07-18T18:25:03.300852","exception":false,"start_time":"2024-07-18T18:25:03.275472","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"88a8b44c","cell_type":"code","source":"some_image_mask = rle_decode(train_segmentations['EncodedPixels'][2])\nplt.imshow(some_image_mask, cmap='gray')","metadata":{"execution":{"iopub.status.busy":"2025-03-05T05:20:40.775456Z","iopub.execute_input":"2025-03-05T05:20:40.775790Z","iopub.status.idle":"2025-03-05T05:20:41.089056Z","shell.execute_reply.started":"2025-03-05T05:20:40.775765Z","shell.execute_reply":"2025-03-05T05:20:41.088146Z"},"papermill":{"duration":0.625497,"end_time":"2024-07-18T18:25:03.936869","exception":false,"start_time":"2024-07-18T18:25:03.311372","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"936fe234","cell_type":"code","source":"def plot_image_with_mask(df: pd.DataFrame, index: int) -> None:\n    fig, ax = plt.subplots(ncols=2)\n    \n    img_name = df['ImageId'][index]\n    img_rle_seqs = df.loc[df['ImageId'] == img_name]['EncodedPixels']  # get rle seqence for each object on the image\n    \n    img = cv2.imread(os.path.join(\n        '/kaggle/input/airbus-ship-detection/train_v2', img_name))\n    \n    # sum masks of all objects\n    img_mask = np.zeros((768, 768), dtype=np.uint8)\n    \n    for rle in img_rle_seqs:\n        img_mask += rle_decode(rle)\n    \n    plt.grid(False)\n    ax[0].imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))\n    ax[1].imshow(img_mask, cmap='gray')\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2025-03-05T05:20:47.923002Z","iopub.execute_input":"2025-03-05T05:20:47.923530Z","iopub.status.idle":"2025-03-05T05:20:47.931187Z","shell.execute_reply.started":"2025-03-05T05:20:47.923486Z","shell.execute_reply":"2025-03-05T05:20:47.930056Z"},"papermill":{"duration":0.030106,"end_time":"2024-07-18T18:25:03.982788","exception":false,"start_time":"2024-07-18T18:25:03.952682","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"8e9556c3","cell_type":"code","source":"# plot_image_with_mask(train_segmentations, 666)\nplot_image_with_mask(train_segmentations, 3)","metadata":{"execution":{"iopub.status.busy":"2025-03-05T05:20:56.364047Z","iopub.execute_input":"2025-03-05T05:20:56.364383Z","iopub.status.idle":"2025-03-05T05:20:56.796737Z","shell.execute_reply.started":"2025-03-05T05:20:56.364352Z","shell.execute_reply":"2025-03-05T05:20:56.795890Z"},"papermill":{"duration":0.834555,"end_time":"2024-07-18T18:25:04.828189","exception":false,"start_time":"2024-07-18T18:25:03.993634","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"d2878a2b","cell_type":"markdown","source":"# Exploratory Data Analysis","metadata":{"papermill":{"duration":0.011516,"end_time":"2024-07-18T18:25:04.851910","exception":false,"start_time":"2024-07-18T18:25:04.840394","status":"completed"},"tags":[]}},{"id":"bbc27d8a","cell_type":"code","source":"ship_count_by_id = train_segmentations\\\n    .groupby('ImageId', as_index=False)\\\n    .count()\\\n    .rename(columns={'EncodedPixels': 'nObjects'})\n\nship_counts = ship_count_by_id\\\n    .groupby('nObjects', as_index=False)\\\n    .count()\\\n    .rename(columns={'ImageId': 'nCounts'})","metadata":{"execution":{"iopub.status.busy":"2025-03-05T05:21:01.158124Z","iopub.execute_input":"2025-03-05T05:21:01.158418Z","iopub.status.idle":"2025-03-05T05:21:01.313608Z","shell.execute_reply.started":"2025-03-05T05:21:01.158395Z","shell.execute_reply":"2025-03-05T05:21:01.312955Z"},"papermill":{"duration":0.343597,"end_time":"2024-07-18T18:25:05.207351","exception":false,"start_time":"2024-07-18T18:25:04.863754","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"e033f231","cell_type":"code","source":"ship_count_by_id.head()","metadata":{"execution":{"iopub.status.busy":"2025-03-05T05:21:06.734588Z","iopub.execute_input":"2025-03-05T05:21:06.734926Z","iopub.status.idle":"2025-03-05T05:21:06.742760Z","shell.execute_reply.started":"2025-03-05T05:21:06.734901Z","shell.execute_reply":"2025-03-05T05:21:06.741765Z"},"papermill":{"duration":0.029327,"end_time":"2024-07-18T18:25:05.248485","exception":false,"start_time":"2024-07-18T18:25:05.219158","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"88e03ce7","cell_type":"code","source":"ship_counts.T","metadata":{"execution":{"iopub.status.busy":"2025-03-05T05:21:09.630033Z","iopub.execute_input":"2025-03-05T05:21:09.630323Z","iopub.status.idle":"2025-03-05T05:21:09.640530Z","shell.execute_reply.started":"2025-03-05T05:21:09.630302Z","shell.execute_reply":"2025-03-05T05:21:09.639790Z"},"papermill":{"duration":0.034058,"end_time":"2024-07-18T18:25:05.294473","exception":false,"start_time":"2024-07-18T18:25:05.260415","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"c5b1aba2","cell_type":"code","source":"# plot all numbers of images with certain amount of ships on them\n\nsns.set(style=\"whitegrid\")\n\nplt.figure(figsize=(10, 4))\nsns.barplot(x='nObjects', y='nCounts', data=ship_counts, palette='Blues_d')\nplt.xlabel('Number of Ships')\nplt.ylabel('Number of Images')\nplt.title('Number of Images with a Certain Number of Ships')\nplt.xticks(rotation=45)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2025-03-05T05:21:11.964748Z","iopub.execute_input":"2025-03-05T05:21:11.965055Z","iopub.status.idle":"2025-03-05T05:21:12.269920Z","shell.execute_reply.started":"2025-03-05T05:21:11.965029Z","shell.execute_reply":"2025-03-05T05:21:12.269047Z"},"papermill":{"duration":0.604122,"end_time":"2024-07-18T18:25:05.912146","exception":false,"start_time":"2024-07-18T18:25:05.308024","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"d60820c3","cell_type":"code","source":"plt.figure(figsize=(8, 8))\nplt.pie(ship_counts['nCounts'], labels=ship_counts['nObjects'], autopct='%1.1f%%', startangle=140)\nplt.title('Distribution of nCounts by nObjects')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2025-03-05T05:22:58.528678Z","iopub.execute_input":"2025-03-05T05:22:58.529028Z","iopub.status.idle":"2025-03-05T05:22:58.759908Z","shell.execute_reply.started":"2025-03-05T05:22:58.529001Z","shell.execute_reply":"2025-03-05T05:22:58.758998Z"},"papermill":{"duration":0.025184,"end_time":"2024-07-18T18:25:05.950811","exception":false,"start_time":"2024-07-18T18:25:05.925627","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"e1554712","cell_type":"code","source":"# the same as preveious but exclude no-ship images\n\nplt.figure(figsize=(10, 4))\nsns.barplot(x='nObjects', y='nCounts', data=ship_counts[1:], palette='Blues_d')\nplt.xlabel('Number of Ships')\nplt.ylabel('Number of Images')\nplt.title('Number of Images with a Certain Number of Ships')\nplt.xticks(rotation=45)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2025-03-05T05:21:19.019137Z","iopub.execute_input":"2025-03-05T05:21:19.019421Z","iopub.status.idle":"2025-03-05T05:21:19.425492Z","shell.execute_reply.started":"2025-03-05T05:21:19.019399Z","shell.execute_reply":"2025-03-05T05:21:19.424573Z"},"papermill":{"duration":0.46911,"end_time":"2024-07-18T18:25:06.432933","exception":false,"start_time":"2024-07-18T18:25:05.963823","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"39543632","cell_type":"code","source":"plt.figure(figsize=(8, 8))\nplt.pie(ship_counts[1:]['nCounts'], labels=ship_counts[1:]['nObjects'], autopct='%1.1f%%', startangle=140)\nplt.title('Distribution of nCounts by nObjects')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2025-03-05T05:22:48.688228Z","iopub.execute_input":"2025-03-05T05:22:48.688544Z","iopub.status.idle":"2025-03-05T05:22:48.941694Z","shell.execute_reply.started":"2025-03-05T05:22:48.688519Z","shell.execute_reply":"2025-03-05T05:22:48.940612Z"},"papermill":{"duration":0.024487,"end_time":"2024-07-18T18:25:06.471916","exception":false,"start_time":"2024-07-18T18:25:06.447429","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"0bbadc10","cell_type":"code","source":"train_segmentations","metadata":{"execution":{"iopub.status.busy":"2025-03-05T05:22:08.834187Z","iopub.execute_input":"2025-03-05T05:22:08.834476Z","iopub.status.idle":"2025-03-05T05:22:08.843763Z","shell.execute_reply.started":"2025-03-05T05:22:08.834455Z","shell.execute_reply":"2025-03-05T05:22:08.843053Z"},"papermill":{"duration":0.033939,"end_time":"2024-07-18T18:26:46.038739","exception":false,"start_time":"2024-07-18T18:26:46.004800","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"25964064","cell_type":"code","source":"import os\nimport random\nimport glob\nimport gc  # garbage collector\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport numpy as np\nimport pandas as pd\nimport cv2\n# import albumentations as A  # for image data augmentation\n\n# tensorflow\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras.layers import (Conv2D, Input, MaxPooling2D, \n                                     Dropout, concatenate, UpSampling2D, BatchNormalization, Conv2DTranspose)\nfrom tensorflow.keras.models import load_model, Model\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau, TensorBoard\nfrom tensorflow.keras import backend as K\n\n# pytorch\n# import torch\n# import torch.nn as nn\n# import torch.functional as F\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"papermill":{"duration":0.014815,"end_time":"2024-07-18T18:26:47.062198","exception":false,"start_time":"2024-07-18T18:26:47.047383","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T05:24:55.790066Z","iopub.execute_input":"2025-03-05T05:24:55.790385Z","iopub.status.idle":"2025-03-05T05:25:07.696506Z","shell.execute_reply.started":"2025-03-05T05:24:55.790362Z","shell.execute_reply":"2025-03-05T05:25:07.695847Z"}},"outputs":[],"execution_count":null},{"id":"123f7271-abee-4c90-88a9-1b1b700e7e01","cell_type":"code","source":"gpus = tf.config.list_physical_devices('GPU')\nif gpus:\n    try:\n        for gpu in gpus:\n            tf.config.experimental.set_memory_growth(gpu, True)\n        logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n        print(len(gpus), \"Physical GPUs,\", len(logical_gpus), \"Logical GPUs\")\n    except RuntimeError as e:\n        print(e)\nelse:\n    print(\"No GPUs found. Please ensure CUDA and cuDNN are properly installed.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T05:25:32.123267Z","iopub.execute_input":"2025-03-05T05:25:32.123555Z","iopub.status.idle":"2025-03-05T05:25:32.129747Z","shell.execute_reply.started":"2025-03-05T05:25:32.123532Z","shell.execute_reply":"2025-03-05T05:25:32.128929Z"}},"outputs":[],"execution_count":null},{"id":"74d5b1f4-9919-4c2a-ad76-2168b4ce90d9","cell_type":"code","source":"FULL_SHAPE = (768, 768)\n# NEW_SHAPE = (256, 256)\nNEW_SHAPE = (128, 128)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T05:25:47.902756Z","iopub.execute_input":"2025-03-05T05:25:47.903075Z","iopub.status.idle":"2025-03-05T05:25:47.906875Z","shell.execute_reply.started":"2025-03-05T05:25:47.903051Z","shell.execute_reply":"2025-03-05T05:25:47.905908Z"}},"outputs":[],"execution_count":null},{"id":"9ab83427-dd0f-4fab-9166-6fa80dcfab1b","cell_type":"code","source":"def rle_decode(mask_rle, shape=(768, 768)) -> np.array:\n    \"\"\"\n    decode run-length encoded segmentation mask\n    Assumed all images aRe 768x768 (and ThereforE have the saMe shape)\n    \"\"\"\n    \n    # if no segmentation mask (nan) return matrix of zeros\n    if not mask_rle or pd.isna(mask_rle):\n        return np.zeros(shape, dtype=np.uint8)\n\n    # RLE sequence str split to and map to int\n    s = list(map(int, mask_rle.split()))\n\n    img = np.zeros(shape[0] * shape[1], dtype=np.uint8)\n\n    # indices: 2k - starts, 2k+1 lengths\n    starts, lengths = s[0::2], s[1::2]\n    for start, length in zip(starts, lengths):\n        img[start:start + length] = 1\n\n    return img.reshape(shape).T","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T05:26:04.305212Z","iopub.execute_input":"2025-03-05T05:26:04.305508Z","iopub.status.idle":"2025-03-05T05:26:04.310998Z","shell.execute_reply.started":"2025-03-05T05:26:04.305485Z","shell.execute_reply":"2025-03-05T05:26:04.310148Z"}},"outputs":[],"execution_count":null},{"id":"699aba89-5039-44ef-b068-e6fe1682718d","cell_type":"code","source":"def rle_encode():\n    return","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T05:26:16.975464Z","iopub.execute_input":"2025-03-05T05:26:16.975816Z","iopub.status.idle":"2025-03-05T05:26:16.979400Z","shell.execute_reply.started":"2025-03-05T05:26:16.975784Z","shell.execute_reply":"2025-03-05T05:26:16.978458Z"}},"outputs":[],"execution_count":null},{"id":"38080a22-88e4-4dd5-aa3a-05c705153ae5","cell_type":"code","source":"def dice_coeff(y_true, y_pred, smooth=1.0):\n    y_true_f = K.cast(K.flatten(y_true), 'float32')\n    y_pred_f = K.cast(K.flatten(y_pred), 'float32')\n    \n    intersection = K.sum(y_true_f * y_pred_f)\n    return (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)\n\ndef dice_loss(y_true, y_pred):\n    return 1 - dice_coeff(y_true, y_pred)\n\ndef BCE_dice(y_true, y_pred):\n    return K.binary_crossentropy(y_true, y_pred) + (1 - dice_coeff(y_true, y_pred))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T05:26:33.503949Z","iopub.execute_input":"2025-03-05T05:26:33.504334Z","iopub.status.idle":"2025-03-05T05:26:33.511424Z","shell.execute_reply.started":"2025-03-05T05:26:33.504303Z","shell.execute_reply":"2025-03-05T05:26:33.510540Z"}},"outputs":[],"execution_count":null},{"id":"e07d0ee7-399d-43c9-99ca-908b7f95150e","cell_type":"code","source":"def create_dataset(image_dir: str, image_filenames: list[str], image_masks: pd.DataFrame) -> tuple:\n    # for each image filename\n    # read original image using cv2\n    # compute segmentations using RLE from image_masks dataframe\n    # append each to tensorflow tensor or smth\n    \n    images = []\n    masks = []\n    \n    for i, image_filename in enumerate(image_filenames):\n        image_path = f\"{image_dir}/{image_filename}\"\n        image = cv2.imread(image_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\n        image = cv2.resize(image, NEW_SHAPE)\n        \n        # get RLE sequences for current image\n        mask_rles = image_masks[image_masks['ImageId'] == image_filename]['EncodedPixels']\n        mask = np.zeros(FULL_SHAPE, dtype=np.uint8)  # init empty mask\n\n        for rle in mask_rles:\n            mask += rle_decode(rle)\n        \n        mask = cv2.resize(mask, NEW_SHAPE)\n        image = image / 255.0\n\n        images.append(image)\n        masks.append(mask)\n\n    images_tensor = tf.convert_to_tensor(images, dtype=tf.float32)\n    masks_tensor = tf.convert_to_tensor(masks, dtype=tf.uint8)\n\n    return images_tensor, masks_tensor","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T05:26:50.948575Z","iopub.execute_input":"2025-03-05T05:26:50.948937Z","iopub.status.idle":"2025-03-05T05:26:50.954722Z","shell.execute_reply.started":"2025-03-05T05:26:50.948907Z","shell.execute_reply":"2025-03-05T05:26:50.953765Z"}},"outputs":[],"execution_count":null},{"id":"4b844fb3-753d-4bb1-88af-ef00174d972b","cell_type":"code","source":"train_folder_path = '/kaggle/input/airbus-ship-detection/train_v2'\ntrain_masks_path = '/kaggle/input/airbus-ship-detection/train_ship_segmentations_v2.csv'\ntest_folder_path = '/kaggle/input/airbus-ship-detection/test_v2'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T05:27:08.592789Z","iopub.execute_input":"2025-03-05T05:27:08.593254Z","iopub.status.idle":"2025-03-05T05:27:08.597958Z","shell.execute_reply.started":"2025-03-05T05:27:08.593212Z","shell.execute_reply":"2025-03-05T05:27:08.596726Z"}},"outputs":[],"execution_count":null},{"id":"c5f23404-604c-450b-99e0-e2dd2ed4247b","cell_type":"code","source":"train_masks_df = pd.read_csv(train_masks_path)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T05:27:24.077203Z","iopub.execute_input":"2025-03-05T05:27:24.077564Z","iopub.status.idle":"2025-03-05T05:27:24.666699Z","shell.execute_reply.started":"2025-03-05T05:27:24.077532Z","shell.execute_reply":"2025-03-05T05:27:24.665747Z"}},"outputs":[],"execution_count":null},{"id":"c9437259-e240-42e5-8fdc-6db24c179ad7","cell_type":"code","source":"class ShipDatasetModified(keras.utils.Sequence):\n    \n    def __init__(self, image_dir, image_filenames, image_size=(256, 256), batch_size=128):\n        # data loader params\n        self.image_size = image_size\n        self.batch_size = batch_size\n        self.image_dir = image_dir\n        self.image_filenames = image_filenames\n        # read run-length encoded ships\n        self.image_masks_df = pd.read_csv(train_masks_path)\n    \n    def __len__(self):\n        return len(self.image_filenames)\n\n    def __getitem__(self, index):\n        \"\"\"\n        Generate one batch of data\n        :param index: Index of the batch\n        :return: Batch of images and masks\n        \"\"\"\n        # create batch indices\n        batch_indices = self.image_filenames[index * self.batch_size:(index + 1) * self.batch_size]\n        \n        # create arrays for images and masks\n        batch_images = np.zeros((self.batch_size, *self.image_size, 3), dtype=np.float32)\n        batch_masks = np.zeros((self.batch_size, *self.image_size, 1), dtype=np.uint8)\n        \n        for i, current_image_name in enumerate(batch_indices):\n            try:\n                # read image and convert to rgb\n                image = cv2.imread(os.path.join(self.image_dir, current_image_name))\n                image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n                \n                # get segmentation mask using image filename\n                img_rle_seqs = self.image_masks_df.loc[self.image_masks_df['ImageId'] == current_image_name]['EncodedPixels']\n                image_mask = np.zeros(FULL_SHAPE, dtype=np.uint8)  # empty mask\n                \n                for rle in img_rle_seqs:\n                    image_mask += rle_decode(rle)\n                \n                # resize img and its mask\n                image = cv2.resize(image, self.image_size)\n                image_mask = cv2.resize(image_mask, self.image_size)\n                \n                image = image / 255.0  # Normalize to [0, 1]\n                \n                batch_images[i] = image\n                batch_masks[i] = np.expand_dims(image_mask, axis=-1)\n                \n            except Exception as e:\n                print(f\"Error processing image {current_image_name}: {e}\")\n        \n        return batch_images, batch_masks\n\n    def on_epoch_end(self):\n        \"\"\"\n        Updates indices after each epoch\n        \"\"\"\n        # shuffle inplace after last batch in epoch\n        random.shuffle(self.image_filenames)\n\n    def _load_image(self, image_path):\n        \"\"\"\n        Load and preprocess an image\n        :param image_path: Path to the image\n        :return: Preprocessed image array\n        \"\"\"\n        # Load and preprocess an image\n        pass","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T05:27:36.375611Z","iopub.execute_input":"2025-03-05T05:27:36.375978Z","iopub.status.idle":"2025-03-05T05:27:36.384657Z","shell.execute_reply.started":"2025-03-05T05:27:36.375950Z","shell.execute_reply":"2025-03-05T05:27:36.383756Z"}},"outputs":[],"execution_count":null},{"id":"7701d864-a812-4aa8-b65a-fd834d55b430","cell_type":"code","source":"image_filenames = [os.path.basename(filename) for filename in glob.glob(train_folder_path + '/*')]\n# random.Random(42).shuffle(image_filenames)\nprint(image_filenames[:10])\nratio = .8\nsplit_index = int(ratio * len(image_filenames))\ntrain_filenames, val_filenames = image_filenames[: split_index], image_filenames[split_index:]\n\ntrain_ship_dataset = ShipDatasetModified(train_folder_path, train_filenames)\nval_ship_dataset = ShipDatasetModified(train_folder_path, val_filenames)\n\nlen(train_ship_dataset), len(val_ship_dataset)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T05:27:54.085158Z","iopub.execute_input":"2025-03-05T05:27:54.085491Z","iopub.status.idle":"2025-03-05T05:28:05.565624Z","shell.execute_reply.started":"2025-03-05T05:27:54.085459Z","shell.execute_reply":"2025-03-05T05:28:05.564735Z"}},"outputs":[],"execution_count":null},{"id":"fbabf1ce-6d93-4e5d-9da4-5b924f2e1ff1","cell_type":"code","source":"train_ship_dataset[0][0].shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T05:28:21.027437Z","iopub.execute_input":"2025-03-05T05:28:21.027808Z","iopub.status.idle":"2025-03-05T05:28:25.518330Z","shell.execute_reply.started":"2025-03-05T05:28:21.027775Z","shell.execute_reply":"2025-03-05T05:28:25.517554Z"}},"outputs":[],"execution_count":null},{"id":"edf0cab0-a8cf-46d2-b8bf-06cdca84ecf5","cell_type":"code","source":"i = 6\nsome_image, some_mask = train_ship_dataset[0]\nsome_image = some_image[i]\nsome_mask = some_mask[i]\n\nfig, ax = plt.subplots(ncols=2)\n\nax[0].imshow(some_image)\nax[1].imshow(some_mask, cmap='gray')\nplt.show()\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T05:28:35.200477Z","iopub.execute_input":"2025-03-05T05:28:35.200849Z","iopub.status.idle":"2025-03-05T05:28:38.694609Z","shell.execute_reply.started":"2025-03-05T05:28:35.200815Z","shell.execute_reply":"2025-03-05T05:28:38.693723Z"}},"outputs":[],"execution_count":null},{"id":"750af772-cbd0-4672-b61e-9ce061c130d9","cell_type":"code","source":"random.Random(42).shuffle(image_filenames)\ntrain_filenames, val_filenames = image_filenames[: split_index], image_filenames[split_index:]\n\n# random.Random(42).shuffle(train_filenames)\n# random.Random(42).shuffle(val_filenames)\n\n# get slice of whole data (i dont want to transform images and compute masks every time i get an dataset item)\ntrain_filenames = train_filenames[:9000]\nval_filenames = val_filenames[:1000]\n\n# remove images which does not contain any ships (later)\n\nprint(len(train_filenames), len(val_filenames))\n\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T05:28:57.087170Z","iopub.execute_input":"2025-03-05T05:28:57.087478Z","iopub.status.idle":"2025-03-05T05:28:57.428919Z","shell.execute_reply.started":"2025-03-05T05:28:57.087455Z","shell.execute_reply":"2025-03-05T05:28:57.428143Z"}},"outputs":[],"execution_count":null},{"id":"072b3011-b5bf-45ec-9ecb-cf23850f33c0","cell_type":"code","source":"X_train, y_train = create_dataset(train_folder_path, train_filenames, train_masks_df)\nX_val, y_val = create_dataset(train_folder_path, val_filenames, train_masks_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T05:29:15.376090Z","iopub.execute_input":"2025-03-05T05:29:15.376392Z","iopub.status.idle":"2025-03-05T05:38:18.933836Z","shell.execute_reply.started":"2025-03-05T05:29:15.376367Z","shell.execute_reply":"2025-03-05T05:38:18.932997Z"}},"outputs":[],"execution_count":null},{"id":"7ea5198c-9025-4032-b53b-c7122e3a2154","cell_type":"code","source":"X_train.shape, y_train.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T05:38:45.853543Z","iopub.execute_input":"2025-03-05T05:38:45.853873Z","iopub.status.idle":"2025-03-05T05:38:45.858777Z","shell.execute_reply.started":"2025-03-05T05:38:45.853847Z","shell.execute_reply":"2025-03-05T05:38:45.858043Z"}},"outputs":[],"execution_count":null},{"id":"ee6547ab-0342-4481-95a6-7d2375a4b727","cell_type":"code","source":"train_dataset = tf.data.Dataset.from_tensor_slices((X_train, y_train))\nval_dataset = tf.data.Dataset.from_tensor_slices((X_val, y_val))\n\n# train_dataset_iterator = train_dataset.as_numpy_iterator()\n# val_dataset_iterator = val_dataset.as_numpy_iterator()\n\nbatch_size = 64\ntrain_dataset = train_dataset.shuffle(buffer_size=len(X_train)).batch(batch_size).prefetch(tf.data.experimental.AUTOTUNE)\nval_dataset = val_dataset.batch(batch_size).prefetch(tf.data.experimental.AUTOTUNE)  # no need to shuffle validation data","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T05:41:37.967852Z","iopub.execute_input":"2025-03-05T05:41:37.968183Z","iopub.status.idle":"2025-03-05T05:41:39.803262Z","shell.execute_reply.started":"2025-03-05T05:41:37.968159Z","shell.execute_reply":"2025-03-05T05:41:39.802589Z"}},"outputs":[],"execution_count":null},{"id":"7e1585df-59fe-4e09-9af2-e583168d2ece","cell_type":"code","source":"np.save('X_train.npy', X_train)\nnp.save('y_train.npy', y_train)\nnp.save('X_val.npy', X_val)\nnp.save('y_val.npy', y_val)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T05:41:54.499158Z","iopub.execute_input":"2025-03-05T05:41:54.499454Z","iopub.status.idle":"2025-03-05T05:41:55.867388Z","shell.execute_reply.started":"2025-03-05T05:41:54.499432Z","shell.execute_reply":"2025-03-05T05:41:55.866732Z"}},"outputs":[],"execution_count":null},{"id":"513b98d9-de64-495a-b434-37a9f9fe340f","cell_type":"code","source":"X_train = tf.convert_to_tensor(np.load('X_train.npy'))\ny_train = tf.convert_to_tensor(np.load('y_train.npy'))\nX_val = tf.convert_to_tensor(np.load('X_val.npy'))\ny_val = tf.convert_to_tensor(np.load('y_val.npy'))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T05:42:11.133359Z","iopub.execute_input":"2025-03-05T05:42:11.133738Z","iopub.status.idle":"2025-03-05T05:42:15.026274Z","shell.execute_reply.started":"2025-03-05T05:42:11.133701Z","shell.execute_reply":"2025-03-05T05:42:15.025194Z"}},"outputs":[],"execution_count":null},{"id":"011a562f-854b-4c34-8194-4301f4ab8b89","cell_type":"code","source":"X_train.shape, y_train.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T05:42:25.896138Z","iopub.execute_input":"2025-03-05T05:42:25.896465Z","iopub.status.idle":"2025-03-05T05:42:25.901560Z","shell.execute_reply.started":"2025-03-05T05:42:25.896435Z","shell.execute_reply":"2025-03-05T05:42:25.900702Z"}},"outputs":[],"execution_count":null},{"id":"c25348ea-b2cc-490d-917e-1c99d4473810","cell_type":"code","source":"def create_conv2d_block(input_tensor, num_filters, kernel_size=3, batchnorm=True):\n    \"\"\"Function to add 2 convolutional layers with the parameters passed to it\"\"\"\n    \n    # 1st layer\n    x = Conv2D(filters = num_filters, kernel_size = (kernel_size, kernel_size),\\\n              kernel_initializer = 'he_normal', padding = 'same')(input_tensor)\n    \n    if batchnorm:\n        x = BatchNormalization()(x)\n        \n    x = keras.layers.Activation('relu')(x)\n    \n    # 2nd layer\n    x = Conv2D(filters = num_filters, kernel_size = (kernel_size, kernel_size),\\\n              kernel_initializer = 'he_normal', padding = 'same')(input_tensor)\n    \n    if batchnorm:\n        x = BatchNormalization()(x)\n    \n    x = keras.layers.Activation('relu')(x)\n    \n    return x","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T05:42:42.375890Z","iopub.execute_input":"2025-03-05T05:42:42.376195Z","iopub.status.idle":"2025-03-05T05:42:42.381359Z","shell.execute_reply.started":"2025-03-05T05:42:42.376173Z","shell.execute_reply":"2025-03-05T05:42:42.380481Z"}},"outputs":[],"execution_count":null},{"id":"91552df0-cef9-40f4-9dfc-6cd5fe628186","cell_type":"code","source":"def create_unet(input_shape, num_filters=16, dropout=0.1, batchnorm=True):\n    \"\"\"\n    Function to define the UNET Model\n    input_shape: (height, width, 3)\n    \"\"\"\n    \n    assert input_shape[-1] == 3  # image must have 3 channels\n    \n    # input 'layer'\n    #input_img = Input((*input_shape, 3), name='img')\n    input_img = Input(input_shape, name='img')\n    \n    # downsampling (encoder)\n    c1 = create_conv2d_block(input_img, num_filters * 1, kernel_size=3, batchnorm=batchnorm)\n    p1 = MaxPooling2D((2, 2))(c1)\n    p1 = Dropout(dropout)(p1)\n    \n    c2 = create_conv2d_block(p1, num_filters * 2, kernel_size=3, batchnorm=batchnorm)\n    p2 = MaxPooling2D((2, 2))(c2)\n    p2 = Dropout(dropout)(p2)\n    \n    c3 = create_conv2d_block(p2, num_filters * 4, kernel_size=3, batchnorm=batchnorm)\n    p3 = MaxPooling2D((2, 2))(c3)\n    p3 = Dropout(dropout)(p3)\n    \n    c4 = create_conv2d_block(p3, num_filters * 8, kernel_size=3, batchnorm=batchnorm)\n    p4 = MaxPooling2D((2, 2))(c4)\n    p4 = Dropout(dropout)(p4)\n    \n    # bottleneck\n    c5 = create_conv2d_block(p4, num_filters * 16, kernel_size=3, batchnorm=batchnorm)\n    \n    # upsampling (decoder)\n    u6 = Conv2DTranspose(num_filters * 8, (3, 3), strides=(2, 2), padding='same')(c5)\n    u6 = concatenate([u6, c4])\n    u6 = Dropout(dropout)(u6)\n    c6 = create_conv2d_block(u6, num_filters * 8, kernel_size=3, batchnorm=batchnorm)\n    \n    u7 = Conv2DTranspose(num_filters * 4, (3, 3), strides=(2, 2), padding='same')(c6)\n    u7 = concatenate([u7, c3])\n    u7 = Dropout(dropout)(u7)\n    c7 = create_conv2d_block(u7, num_filters * 4, kernel_size=3, batchnorm=batchnorm)\n    \n    u8 = Conv2DTranspose(num_filters * 2, (3, 3), strides=(2, 2), padding='same')(c7)\n    u8 = concatenate([u8, c2])\n    u8 = Dropout(dropout)(u8)\n    c8 = create_conv2d_block(u8, num_filters * 2, kernel_size=3, batchnorm=batchnorm)\n    \n    u9 = Conv2DTranspose(num_filters * 1, (3, 3), strides=(2, 2), padding='same')(c8)\n    u9 = concatenate([u9, c1])\n    u9 = Dropout(dropout)(u9)\n    c9 = create_conv2d_block(u9, num_filters * 1, kernel_size=3, batchnorm=batchnorm)\n    \n    outputs = Conv2D(1, (1, 1), activation='sigmoid')(c9)\n    model = Model(inputs=[input_img], outputs=[outputs])\n    \n    return model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T05:43:01.497012Z","iopub.execute_input":"2025-03-05T05:43:01.497316Z","iopub.status.idle":"2025-03-05T05:43:01.506573Z","shell.execute_reply.started":"2025-03-05T05:43:01.497292Z","shell.execute_reply":"2025-03-05T05:43:01.505671Z"}},"outputs":[],"execution_count":null},{"id":"e97eb16c-6c21-470a-a368-ef7da8f858ef","cell_type":"code","source":"input_shape = (*NEW_SHAPE, 3)\n\nunet_model = create_unet(input_shape)\n\nmetrics = [\n    'accuracy',\n    dice_coeff,\n    dice_loss,\n]\n\n# unet_model.compile(optimizer=Adam(0.0005), loss=\"binary_crossentropy\", metrics=metrics)\nunet_model.compile(optimizer=Adam(0.0005), loss=BCE_dice, metrics=metrics)\n\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T05:43:19.997669Z","iopub.execute_input":"2025-03-05T05:43:19.997979Z","iopub.status.idle":"2025-03-05T05:43:21.879230Z","shell.execute_reply.started":"2025-03-05T05:43:19.997952Z","shell.execute_reply":"2025-03-05T05:43:21.878265Z"}},"outputs":[],"execution_count":null},{"id":"6078170f-c8e8-4e10-b4cf-30d5b0871b94","cell_type":"code","source":"unet_model.summary()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T05:43:38.412696Z","iopub.execute_input":"2025-03-05T05:43:38.413070Z","iopub.status.idle":"2025-03-05T05:43:38.481205Z","shell.execute_reply.started":"2025-03-05T05:43:38.413042Z","shell.execute_reply":"2025-03-05T05:43:38.480484Z"}},"outputs":[],"execution_count":null},{"id":"eabb76a5-d32c-4e2e-b39e-7b3982149c51","cell_type":"code","source":"callbacks = [\n#     EarlyStopping(patience=10, verbose=1),\n#     ReduceLROnPlateau(factor=0.1, patience=5, min_lr=0.00001, verbose=1),\n    ModelCheckpoint('model-Unet.weights.h5', verbose=1, save_best_only=True, save_weights_only=True),\n    TensorBoard(log_dir='./logs')\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T05:44:08.782286Z","iopub.execute_input":"2025-03-05T05:44:08.782579Z","iopub.status.idle":"2025-03-05T05:44:08.786597Z","shell.execute_reply.started":"2025-03-05T05:44:08.782557Z","shell.execute_reply":"2025-03-05T05:44:08.785766Z"}},"outputs":[],"execution_count":null},{"id":"adf0cbc8-2bd6-4152-9cdf-5de73850cc58","cell_type":"code","source":"\n\nhistory = unet_model.fit(\n    train_dataset,\n    epochs=20,\n    validation_data=val_dataset,\n    callbacks=callbacks\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T05:44:28.828033Z","iopub.execute_input":"2025-03-05T05:44:28.828403Z","iopub.status.idle":"2025-03-05T05:47:49.667226Z","shell.execute_reply.started":"2025-03-05T05:44:28.828362Z","shell.execute_reply":"2025-03-05T05:47:49.666524Z"}},"outputs":[],"execution_count":null},{"id":"e4d31b78-18cc-4ee4-a5f5-318453d048df","cell_type":"code","source":"import pickle\n\nwith open('unet_history.obj', 'wb') as f:\n    pickle.dump(history.history, f)\n\nunet_model.save_weights('winstarsai_airbus_unet.weights.h5')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T05:50:29.273626Z","iopub.execute_input":"2025-03-05T05:50:29.274015Z","iopub.status.idle":"2025-03-05T05:50:29.409350Z","shell.execute_reply.started":"2025-03-05T05:50:29.273983Z","shell.execute_reply":"2025-03-05T05:50:29.408676Z"}},"outputs":[],"execution_count":null},{"id":"17403704-975e-4a67-a677-43effa86451f","cell_type":"code","source":"def plot_training_history(history):\n    # Extract the history data\n    accuracy = history.history['accuracy']\n    val_accuracy = history.history['val_accuracy']\n    loss = history.history['loss']\n    val_loss = history.history['val_loss']\n    dice_coeff = history.history['dice_coeff']\n    val_dice_coeff = history.history['val_dice_coeff']\n\n    epochs = range(1, len(accuracy) + 1)\n\n    # Plot accuracy\n    plt.figure(figsize=(14, 6))\n\n    plt.subplot(1, 3, 1)\n    plt.plot(epochs, accuracy, 'bo', label='Training accuracy')\n    plt.plot(epochs, val_accuracy, 'b', label='Validation accuracy')\n    plt.title('Training and validation accuracy')\n    plt.xlabel('Epochs')\n    plt.ylabel('Accuracy')\n    plt.legend()\n\n    # Plot loss\n    plt.subplot(1, 3, 2)\n    plt.plot(epochs, loss, 'ro', label='Training loss')\n    plt.plot(epochs, val_loss, 'r', label='Validation loss')\n    plt.title('Training and validation loss')\n    plt.xlabel('Epochs')\n    plt.ylabel('Loss')\n    plt.legend()\n\n    # Plot Dice coefficient\n    plt.subplot(1, 3, 3)\n    plt.plot(epochs, dice_coeff, 'go', label='Training Dice Coefficient')\n    plt.plot(epochs, val_dice_coeff, 'g', label='Validation Dice Coefficient')\n    plt.title('Training and validation Dice Coefficient')\n    plt.xlabel('Epochs')\n    plt.ylabel('Dice Coefficient')\n    plt.legend()\n\n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T05:50:50.785202Z","iopub.execute_input":"2025-03-05T05:50:50.785508Z","iopub.status.idle":"2025-03-05T05:50:50.792576Z","shell.execute_reply.started":"2025-03-05T05:50:50.785484Z","shell.execute_reply":"2025-03-05T05:50:50.791596Z"}},"outputs":[],"execution_count":null},{"id":"b5e6dff9-5811-4b13-9616-316cd2410864","cell_type":"code","source":"plot_training_history(history)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T05:51:29.423522Z","iopub.execute_input":"2025-03-05T05:51:29.423873Z","iopub.status.idle":"2025-03-05T05:51:30.173136Z","shell.execute_reply.started":"2025-03-05T05:51:29.423844Z","shell.execute_reply":"2025-03-05T05:51:30.172327Z"}},"outputs":[],"execution_count":null},{"id":"a62ad367-24e4-48e9-ab3c-f2bb57a52b2d","cell_type":"code","source":"import os\nimport random\nimport glob\nimport gc  # garbage collector\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport numpy as np\nimport cv2\n\n# tensorflow\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras.layers import (Conv2D, Input, MaxPooling2D, \n                                     Dropout, concatenate, UpSampling2D, BatchNormalization, Conv2DTranspose)\nfrom tensorflow.keras.models import load_model, Model\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau, TensorBoard\nfrom tensorflow.keras import backend as K\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T05:53:07.866788Z","iopub.execute_input":"2025-03-05T05:53:07.867146Z","iopub.status.idle":"2025-03-05T05:53:07.872540Z","shell.execute_reply.started":"2025-03-05T05:53:07.867119Z","shell.execute_reply":"2025-03-05T05:53:07.871747Z"}},"outputs":[],"execution_count":null},{"id":"a6231451-0ca5-4f72-adc2-59c5f735cf7c","cell_type":"code","source":"gpus = tf.config.list_physical_devices('GPU')\nif gpus:\n    try:\n        for gpu in gpus:\n            tf.config.experimental.set_memory_growth(gpu, True)\n        logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n        print(len(gpus), \"Physical GPUs,\", len(logical_gpus), \"Logical GPUs\")\n    except RuntimeError as e:\n        print(e)\nelse:\n    print(\"No GPUs found. Please ensure CUDA and cuDNN are properly installed.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T05:53:24.268525Z","iopub.execute_input":"2025-03-05T05:53:24.268858Z","iopub.status.idle":"2025-03-05T05:53:24.275099Z","shell.execute_reply.started":"2025-03-05T05:53:24.268830Z","shell.execute_reply":"2025-03-05T05:53:24.274387Z"}},"outputs":[],"execution_count":null},{"id":"1f612b25-8b5b-4f64-a498-1ab2e8f7af46","cell_type":"code","source":"FULL_SHAPE = (768, 768)\nNEW_SHAPE = (128, 128)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T05:53:53.018965Z","iopub.execute_input":"2025-03-05T05:53:53.019245Z","iopub.status.idle":"2025-03-05T05:53:53.023242Z","shell.execute_reply.started":"2025-03-05T05:53:53.019223Z","shell.execute_reply":"2025-03-05T05:53:53.022350Z"}},"outputs":[],"execution_count":null},{"id":"2e3a1aa1-7d7e-43d1-994e-9d3a295358d3","cell_type":"code","source":"def image_from_path(img_dir, img_name):\n    \n    img_path = os.path.join(img_dir, img_name)\n    \n    image = cv2.imread(img_path)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    image = cv2.resize(image, NEW_SHAPE)\n    image = image / 255.0  # Normalize to [0, 1]\n    \n    return np.expand_dims(image, axis=0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T05:54:09.331683Z","iopub.execute_input":"2025-03-05T05:54:09.332017Z","iopub.status.idle":"2025-03-05T05:54:09.336610Z","shell.execute_reply.started":"2025-03-05T05:54:09.331986Z","shell.execute_reply":"2025-03-05T05:54:09.335634Z"}},"outputs":[],"execution_count":null},{"id":"0654e68e-c9f4-497a-aaef-b40718fb2567","cell_type":"code","source":"def create_conv2d_block(input_tensor, num_filters, kernel_size=3, batchnorm=True):\n    \"\"\"Function to add 2 convolutional layers with the parameters passed to it\"\"\"\n    \n    # 1st layer\n    x = Conv2D(filters = num_filters, kernel_size = (kernel_size, kernel_size),\\\n              kernel_initializer = 'he_normal', padding = 'same')(input_tensor)\n    \n    if batchnorm:\n        x = BatchNormalization()(x)\n        \n    x = keras.layers.Activation('relu')(x)\n    \n    # 2nd layer\n    x = Conv2D(filters = num_filters, kernel_size = (kernel_size, kernel_size),\\\n              kernel_initializer = 'he_normal', padding = 'same')(input_tensor)\n    \n    if batchnorm:\n        x = BatchNormalization()(x)\n    \n    x = keras.layers.Activation('relu')(x)\n    \n    return x","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T05:54:37.750421Z","iopub.execute_input":"2025-03-05T05:54:37.750769Z","iopub.status.idle":"2025-03-05T05:54:37.755619Z","shell.execute_reply.started":"2025-03-05T05:54:37.750740Z","shell.execute_reply":"2025-03-05T05:54:37.754881Z"}},"outputs":[],"execution_count":null},{"id":"7447b8e9-d740-4ae6-999b-9c257b174340","cell_type":"code","source":"def create_unet(input_shape, num_filters=16, dropout=0.1, batchnorm=True):\n    \"\"\"\n    Function to define the UNET Model\n    input_shape: (height, width, 3)\n    \"\"\"\n    \n    assert input_shape[-1] == 3  # image must have 3 channels\n    \n    # input 'layer'\n    #input_img = Input((*input_shape, 3), name='img')\n    input_img = Input(input_shape, name='img')\n    \n    # downsampling (encoder)\n    c1 = create_conv2d_block(input_img, num_filters * 1, kernel_size=3, batchnorm=batchnorm)\n    p1 = MaxPooling2D((2, 2))(c1)\n    p1 = Dropout(dropout)(p1)\n    \n    c2 = create_conv2d_block(p1, num_filters * 2, kernel_size=3, batchnorm=batchnorm)\n    p2 = MaxPooling2D((2, 2))(c2)\n    p2 = Dropout(dropout)(p2)\n    \n    c3 = create_conv2d_block(p2, num_filters * 4, kernel_size=3, batchnorm=batchnorm)\n    p3 = MaxPooling2D((2, 2))(c3)\n    p3 = Dropout(dropout)(p3)\n    \n    c4 = create_conv2d_block(p3, num_filters * 8, kernel_size=3, batchnorm=batchnorm)\n    p4 = MaxPooling2D((2, 2))(c4)\n    p4 = Dropout(dropout)(p4)\n    \n    # bottleneck\n    c5 = create_conv2d_block(p4, num_filters * 16, kernel_size=3, batchnorm=batchnorm)\n    \n    # upsampling (decoder)\n    u6 = Conv2DTranspose(num_filters * 8, (3, 3), strides=(2, 2), padding='same')(c5)\n    u6 = concatenate([u6, c4])\n    u6 = Dropout(dropout)(u6)\n    c6 = create_conv2d_block(u6, num_filters * 8, kernel_size=3, batchnorm=batchnorm)\n    \n    u7 = Conv2DTranspose(num_filters * 4, (3, 3), strides=(2, 2), padding='same')(c6)\n    u7 = concatenate([u7, c3])\n    u7 = Dropout(dropout)(u7)\n    c7 = create_conv2d_block(u7, num_filters * 4, kernel_size=3, batchnorm=batchnorm)\n    \n    u8 = Conv2DTranspose(num_filters * 2, (3, 3), strides=(2, 2), padding='same')(c7)\n    u8 = concatenate([u8, c2])\n    u8 = Dropout(dropout)(u8)\n    c8 = create_conv2d_block(u8, num_filters * 2, kernel_size=3, batchnorm=batchnorm)\n    \n    u9 = Conv2DTranspose(num_filters * 1, (3, 3), strides=(2, 2), padding='same')(c8)\n    u9 = concatenate([u9, c1])\n    u9 = Dropout(dropout)(u9)\n    c9 = create_conv2d_block(u9, num_filters * 1, kernel_size=3, batchnorm=batchnorm)\n    \n    outputs = Conv2D(1, (1, 1), activation='sigmoid')(c9)\n    model = Model(inputs=[input_img], outputs=[outputs])\n    \n    return model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T05:54:51.823201Z","iopub.execute_input":"2025-03-05T05:54:51.823505Z","iopub.status.idle":"2025-03-05T05:54:51.833514Z","shell.execute_reply.started":"2025-03-05T05:54:51.823483Z","shell.execute_reply":"2025-03-05T05:54:51.832592Z"}},"outputs":[],"execution_count":null},{"id":"e4777175-954f-4ea8-865f-48dd1879f008","cell_type":"code","source":"input_shape = (*NEW_SHAPE, 3)\nunet_weights_path = '/kaggle/input/ship_detection_unet/tensorflow2/v1/1/model-Unet.weights.h5'\n\nunet_model = create_unet(input_shape)\nunet_model.load_weights(unet_weights_path)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T05:57:28.004198Z","iopub.execute_input":"2025-03-05T05:57:28.004628Z","iopub.status.idle":"2025-03-05T05:57:28.503558Z","shell.execute_reply.started":"2025-03-05T05:57:28.004590Z","shell.execute_reply":"2025-03-05T05:57:28.502909Z"}},"outputs":[],"execution_count":null},{"id":"192300ac-44eb-4bcc-9625-bb6e72f14c9c","cell_type":"code","source":"test_images_folder_path = '/kaggle/input/airbus-ship-detection/test_v2'\ntest_images_filenames =  [os.path.basename(filename) for filename in glob.glob(test_images_folder_path + '/*')]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T05:58:00.433195Z","iopub.execute_input":"2025-03-05T05:58:00.433485Z","iopub.status.idle":"2025-03-05T05:58:01.188467Z","shell.execute_reply.started":"2025-03-05T05:58:00.433460Z","shell.execute_reply":"2025-03-05T05:58:01.187807Z"}},"outputs":[],"execution_count":null},{"id":"b2274df6-a877-4d6e-855b-61a3692bb4fa","cell_type":"code","source":"len(test_images_filenames), test_images_filenames[:10]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T05:58:16.021774Z","iopub.execute_input":"2025-03-05T05:58:16.022061Z","iopub.status.idle":"2025-03-05T05:58:16.027530Z","shell.execute_reply.started":"2025-03-05T05:58:16.022039Z","shell.execute_reply":"2025-03-05T05:58:16.026774Z"}},"outputs":[],"execution_count":null},{"id":"0cc29664-f983-492f-97d5-49fc564d40a0","cell_type":"code","source":"some_image = image_from_path(test_images_folder_path, test_images_filenames[10])\nsome_image.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T06:00:52.631902Z","iopub.execute_input":"2025-03-05T06:00:52.632214Z","iopub.status.idle":"2025-03-05T06:00:52.651780Z","shell.execute_reply.started":"2025-03-05T06:00:52.632190Z","shell.execute_reply":"2025-03-05T06:00:52.651053Z"}},"outputs":[],"execution_count":null},{"id":"2e8f60ed-825e-4fac-be10-c8bebd0de6be","cell_type":"code","source":"plt.imshow(some_image.squeeze(axis=0))\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T06:00:54.592502Z","iopub.execute_input":"2025-03-05T06:00:54.592870Z","iopub.status.idle":"2025-03-05T06:00:54.833659Z","shell.execute_reply.started":"2025-03-05T06:00:54.592833Z","shell.execute_reply":"2025-03-05T06:00:54.832883Z"}},"outputs":[],"execution_count":null},{"id":"4d87870c-b64c-4533-894a-a2b027b0c8d0","cell_type":"code","source":"def make_prediction(model, image):\n    threshold = 0.5\n    \n    # get probability for each pixel (-> np.ndarray)\n    prob_mask = model.predict(image).squeeze(0)\n    \n    segmentation_mask = np.zeros(prob_mask.shape)\n    # Convert probabilities to binary segmentation mask\n    segmentation_mask = (prob_mask > threshold).astype(np.uint8)\n    return segmentation_mask","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T06:00:59.057325Z","iopub.execute_input":"2025-03-05T06:00:59.057629Z","iopub.status.idle":"2025-03-05T06:00:59.062020Z","shell.execute_reply.started":"2025-03-05T06:00:59.057603Z","shell.execute_reply":"2025-03-05T06:00:59.061060Z"}},"outputs":[],"execution_count":null},{"id":"c5481d75-dd28-4bef-ace0-fc6fe59db518","cell_type":"code","source":"some_image_mask = make_prediction(unet_model, some_image)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T06:01:01.643098Z","iopub.execute_input":"2025-03-05T06:01:01.643387Z","iopub.status.idle":"2025-03-05T06:01:01.709298Z","shell.execute_reply.started":"2025-03-05T06:01:01.643365Z","shell.execute_reply":"2025-03-05T06:01:01.708582Z"}},"outputs":[],"execution_count":null},{"id":"ac72242f-6d26-4711-abd8-337ebc2954a7","cell_type":"code","source":"fig, ax = plt.subplots(ncols=2)\n\nax[0].imshow(some_image.squeeze(0))\nax[1].imshow(some_image_mask, cmap='gray')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-05T06:01:04.039976Z","iopub.execute_input":"2025-03-05T06:01:04.040369Z","iopub.status.idle":"2025-03-05T06:01:04.380506Z","shell.execute_reply.started":"2025-03-05T06:01:04.040335Z","shell.execute_reply":"2025-03-05T06:01:04.379594Z"}},"outputs":[],"execution_count":null}]}