{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nimport cv2","metadata":{"papermill":{"duration":0.191779,"end_time":"2023-02-04T15:44:15.014295","exception":false,"start_time":"2023-02-04T15:44:14.822516","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-05T15:05:27.829702Z","iopub.execute_input":"2023-02-05T15:05:27.830215Z","iopub.status.idle":"2023-02-05T15:05:27.975743Z","shell.execute_reply.started":"2023-02-05T15:05:27.830152Z","shell.execute_reply":"2023-02-05T15:05:27.974787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Analysis","metadata":{"papermill":{"duration":0.010122,"end_time":"2023-02-04T15:44:15.035163","exception":false,"start_time":"2023-02-04T15:44:15.025041","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## Read Data Set","metadata":{"papermill":{"duration":0.01037,"end_time":"2023-02-04T15:44:15.057314","exception":false,"start_time":"2023-02-04T15:44:15.046944","status":"completed"},"tags":[]}},{"cell_type":"code","source":"segmentations = pd.read_csv(\"/kaggle/input/airbus-ship-detection/train_ship_segmentations_v2.csv\")\nsegmentations['EncodedPixels'] = segmentations['EncodedPixels'].astype('string')","metadata":{"papermill":{"duration":1.285796,"end_time":"2023-02-04T15:44:16.353420","exception":false,"start_time":"2023-02-04T15:44:15.067624","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-05T15:05:27.977535Z","iopub.execute_input":"2023-02-05T15:05:27.977980Z","iopub.status.idle":"2023-02-05T15:05:29.442598Z","shell.execute_reply.started":"2023-02-05T15:05:27.977943Z","shell.execute_reply":"2023-02-05T15:05:29.441551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Description","metadata":{"papermill":{"duration":0.014866,"end_time":"2023-02-04T15:44:16.384299","exception":false,"start_time":"2023-02-04T15:44:16.369433","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":" **EncodedPixels** - a list of pixels for ship segmentation in a compressed format (in run-length encoding format). \n \n EncodedPixels сonsists of pairs of values that contain a start position and a run length. \n E.g. '1 3' implies starting at pixel 1 and running a total of 3 pixels (1,2,3).\n \n The pixels are one-indexed and numbered from top to bottom, then left to right: 1 is pixel (1,1), 2 is pixel (2,1), etc. \n \n A prediction of of \"no ship in image\" have a blank value in the EncodedPixels column.\n \n Object segments do not overlap.\n ","metadata":{"papermill":{"duration":0.014723,"end_time":"2023-02-04T15:44:16.413969","exception":false,"start_time":"2023-02-04T15:44:16.399246","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## Feature Extraction","metadata":{"papermill":{"duration":0.014648,"end_time":"2023-02-04T15:44:16.443561","exception":false,"start_time":"2023-02-04T15:44:16.428913","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"Let's add new features:\n* **ImageHeight** - image height\n* **ImageWidth** - image weight\n* **ShipAreaPercentage** - percentage of the ship segmentation area in the image. Measured as a percentage, from 0 to 100%","metadata":{"papermill":{"duration":0.024195,"end_time":"2023-02-04T15:44:16.483874","exception":false,"start_time":"2023-02-04T15:44:16.459679","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def get_train_image(name: str):\n    path = f'/kaggle/input/airbus-ship-detection/train_v2/{name}'\n    return cv2.imread(path)\n\ndef extract_features_from_image(row: pd.Series) -> pd.Series:\n    image = np.zeros((768, 768, 3))# get_train_image(row['ImageId'])\n    row['ImageHeight'], row['ImageWidth'], _ = image.shape\n    return row\n\nsegmentations = segmentations.apply(lambda x: extract_features_from_image(x), axis=1)","metadata":{"papermill":{"duration":386.05566,"end_time":"2023-02-04T15:50:42.554879","exception":false,"start_time":"2023-02-04T15:44:16.499219","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-05T15:05:29.448145Z","iopub.execute_input":"2023-02-05T15:05:29.450639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pixels_number(encoded_pixels: str) -> int:\n    if pd.isna(encoded_pixels):\n        return 0\n    return np.array(encoded_pixels.split()[1::2], dtype=int).sum()\n\nsegmentations['ShipAreaPercentage'] = segmentations.apply(lambda x: pixels_number(x['EncodedPixels']) / (x['ImageHeight'] * x['ImageWidth']) * 100, axis=1)","metadata":{"papermill":{"duration":4.988651,"end_time":"2023-02-04T15:50:47.554650","exception":false,"start_time":"2023-02-04T15:50:42.565999","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Corrupted images","metadata":{"papermill":{"duration":0.011642,"end_time":"2023-02-04T15:50:47.577235","exception":false,"start_time":"2023-02-04T15:50:47.565593","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"6384c3e78.jpg - corrupted image, a `Premature end of JPEG file` error occurs when reading the file.","metadata":{"papermill":{"duration":0.010109,"end_time":"2023-02-04T15:50:47.597909","exception":false,"start_time":"2023-02-04T15:50:47.587800","status":"completed"},"tags":[]}},{"cell_type":"code","source":"corrupted_images = ['6384c3e78.jpg']\nsegmentations[segmentations['ImageId'].isin(corrupted_images)]","metadata":{"papermill":{"duration":0.043475,"end_time":"2023-02-04T15:50:47.651562","exception":false,"start_time":"2023-02-04T15:50:47.608087","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So let's delete this image.","metadata":{"papermill":{"duration":0.010142,"end_time":"2023-02-04T15:50:47.672672","exception":false,"start_time":"2023-02-04T15:50:47.662530","status":"completed"},"tags":[]}},{"cell_type":"code","source":"segmentations = segmentations.drop(segmentations[segmentations['ImageId'].isin(corrupted_images)].index)\nsegmentations[segmentations['ImageId'].isin(corrupted_images)]","metadata":{"papermill":{"duration":0.05404,"end_time":"2023-02-04T15:50:47.737307","exception":false,"start_time":"2023-02-04T15:50:47.683267","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Exploring the data","metadata":{"papermill":{"duration":0.010586,"end_time":"2023-02-04T15:50:47.758924","exception":false,"start_time":"2023-02-04T15:50:47.748338","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print(f'There are {segmentations.shape[0]} rows.')\nsegmentations.head(10)","metadata":{"papermill":{"duration":0.026796,"end_time":"2023-02-04T15:50:47.796223","exception":false,"start_time":"2023-02-04T15:50:47.769427","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_images_number = segmentations['ImageId'].nunique()\nprint(f'There are {train_images_number} train images.')","metadata":{"papermill":{"duration":0.055467,"end_time":"2023-02-04T15:50:47.862679","exception":false,"start_time":"2023-02-04T15:50:47.807212","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Image resolution","metadata":{"papermill":{"duration":0.010527,"end_time":"2023-02-04T15:50:47.884242","exception":false,"start_time":"2023-02-04T15:50:47.873715","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print(segmentations['ImageHeight'].value_counts())\nprint(segmentations['ImageWidth'].value_counts())","metadata":{"papermill":{"duration":0.024814,"end_time":"2023-02-04T15:50:47.919812","exception":false,"start_time":"2023-02-04T15:50:47.894998","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"All images have the same resolution 768 * 768 * 3.","metadata":{"papermill":{"duration":0.010651,"end_time":"2023-02-04T15:50:47.941096","exception":false,"start_time":"2023-02-04T15:50:47.930445","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## Distribution of the number of ships in images","metadata":{"papermill":{"duration":0.010451,"end_time":"2023-02-04T15:50:47.962458","exception":false,"start_time":"2023-02-04T15:50:47.952007","status":"completed"},"tags":[]}},{"cell_type":"code","source":"images_without_ships = segmentations['EncodedPixels'].isna().sum()\nprint(f'There are {images_without_ships} images without ships.')","metadata":{"papermill":{"duration":0.03192,"end_time":"2023-02-04T15:50:48.005310","exception":false,"start_time":"2023-02-04T15:50:47.973390","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"segmentations['ShipCount'] = segmentations.apply(lambda x: 0 if pd.isna(x['EncodedPixels']) else 1, axis=1)\nships_numbers = segmentations[['ImageId','ShipCount']].groupby(['ImageId']).sum()\nprint(ships_numbers.value_counts())\n\nf,ax=plt.subplots(1, 2,figsize=(20,10))\n\nships_numbers.hist(bins = 15, ax=ax[0])\n\ny = ships_numbers.value_counts().values\npercent = 100.*y/y.sum()\nlabels = ['{0} - {1:1.2f} %'.format(i,j) for i,j in zip(ships_numbers.value_counts().index.get_level_values(0), percent)]\nships_numbers.value_counts().plot.pie(labels=None, ax=ax[1])\nax[1].legend(labels, bbox_to_anchor=(1., 1.), fontsize=14)\nax[1].yaxis.set_visible(False)\nax[1].set_title('Distribution number of ships')\n\nplt.show()","metadata":{"papermill":{"duration":3.518932,"end_time":"2023-02-04T15:50:51.534990","exception":false,"start_time":"2023-02-04T15:50:48.016058","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see that 78% of images don't have any ships. Only 14% have one ship. And on the remaining 8% of the images there are from 2 to 15 ships.","metadata":{"papermill":{"duration":0.013436,"end_time":"2023-02-04T15:50:51.561711","exception":false,"start_time":"2023-02-04T15:50:51.548275","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## Ship segmentation area distribution","metadata":{"papermill":{"duration":0.011942,"end_time":"2023-02-04T15:50:51.586232","exception":false,"start_time":"2023-02-04T15:50:51.574290","status":"completed"},"tags":[]}},{"cell_type":"code","source":"f,ax=plt.subplots(1, 3,figsize=(30,10))\n\nsegmentations['ShipAreaPercentage'].hist(bins=20, ax=ax[0])\nsegmentations['ShipAreaPercentage'].plot.box(ax=ax[1])\nax[1].set_ylabel('Ship Area Percentage')\nax[1].set_xlabel('')\nsegmentations['ShipAreaPercentage'].apply(lambda x: x ** 0.5).plot.box(ax=ax[2])\nax[2].set_ylabel('Sqrt(Ship Area Percentage)')\nax[2].set_xlabel('')","metadata":{"papermill":{"duration":0.853372,"end_time":"2023-02-04T15:50:52.451807","exception":false,"start_time":"2023-02-04T15:50:51.598435","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"All ships are very small in the images. Ships occupy no more than 4% of the image. Most of the ships take up less than 0.1% of the image.","metadata":{"papermill":{"duration":0.018063,"end_time":"2023-02-04T15:50:52.488913","exception":false,"start_time":"2023-02-04T15:50:52.470850","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## Display ship segmentation pixels","metadata":{"papermill":{"duration":0.018993,"end_time":"2023-02-04T15:50:52.525992","exception":false,"start_time":"2023-02-04T15:50:52.506999","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def show_image_with_encoded_pixels(image_id: str):\n    rows = segmentations[segmentations['ImageId'] == image_id]\n    if len(rows) == 0:\n        return\n    \n    image = get_train_image(image_id)\n    image_size, _, _ = image.shape\n    ship_count = len(rows)\n    all_ships = np.zeros_like(image)\n\n    ax_rows_number = ship_count + 1\n    f,ax = plt.subplots(ax_rows_number, 3, figsize=(15, 5 * ax_rows_number))\n\n    for i in range(ship_count):\n        image_info = rows.iloc[i]\n\n        encoded_pixels = np.array(image_info['EncodedPixels'].split(), dtype=int)\n        pixels, shift = encoded_pixels[::2], encoded_pixels[1::2]\n        ship = np.zeros_like(image)\n\n        for pixel, shift in zip(pixels, shift):\n            for j in range(shift):\n                cur_pixel = pixel + j - 1\n                ship[cur_pixel % image_size, cur_pixel // image_size] = [255, 255, 255]\n        all_ships += ship\n        \n        ax[i, 0].imshow(image)\n        ax[i, 1].imshow(ship)\n        ax[i, 2].imshow(image * (ship // 255))\n        \n    ax[ship_count, 0].imshow(image)\n    ax[ship_count, 1].imshow(all_ships)\n    ax[ship_count, 2].imshow(image * (all_ships // 255))\n    plt.show()","metadata":{"papermill":{"duration":0.032343,"end_time":"2023-02-04T15:50:52.582753","exception":false,"start_time":"2023-02-04T15:50:52.550410","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_id = '0006c52e8.jpg'\nshow_image_with_encoded_pixels(image_id)\nsegmentations[segmentations['ImageId'] == image_id]","metadata":{"papermill":{"duration":1.173985,"end_time":"2023-02-04T15:50:53.769449","exception":false,"start_time":"2023-02-04T15:50:52.595464","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_id = '00113a75c.jpg'\nshow_image_with_encoded_pixels(image_id)\nsegmentations[segmentations['ImageId'] == image_id]","metadata":{"papermill":{"duration":3.873642,"end_time":"2023-02-04T15:50:57.661478","exception":false,"start_time":"2023-02-04T15:50:53.787836","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_id = '000fd9827.jpg'\nshow_image_with_encoded_pixels(image_id)\nsegmentations[segmentations['ImageId'] == image_id]","metadata":{"papermill":{"duration":1.140126,"end_time":"2023-02-04T15:50:58.830807","exception":false,"start_time":"2023-02-04T15:50:57.690681","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_id = '007534159.jpg'\nshow_image_with_encoded_pixels(image_id)\nsegmentations[segmentations['ImageId'] == image_id]","metadata":{"papermill":{"duration":1.572816,"end_time":"2023-02-04T15:51:00.436683","exception":false,"start_time":"2023-02-04T15:50:58.863867","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"First, we can notice that all the ship segment labels in the training dataset are bounding boxes (rectangles).\nMoreover the ships in the images have a slightly different shape from a rectangular one.\n\nSecondly, there are images in which the ships are side by side, and their segments have a common border.","metadata":{"papermill":{"duration":0.033808,"end_time":"2023-02-04T15:51:00.506742","exception":false,"start_time":"2023-02-04T15:51:00.472934","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## Data analysis results\n\nIt was found that most of the images (78%) do not contain ships.\nAll the ships in the images take up a very small part (approximately 0,1%), the pictures contain a lot of background.\nAlso, target segmentation for labeled pictures consists of bounding boxes (rectangles). Ships do not intersect, but their borders in some images have adjacent borders.","metadata":{"papermill":{"duration":0.034007,"end_time":"2023-02-04T15:51:00.574487","exception":false,"start_time":"2023-02-04T15:51:00.540480","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# Train Unet Model","metadata":{"papermill":{"duration":0.033754,"end_time":"2023-02-04T15:51:00.642084","exception":false,"start_time":"2023-02-04T15:51:00.608330","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import random\nimport os\nimport tensorflow as tf\nfrom tensorflow import keras\n\n# set the random seed:\nRANDOM_SEED = 77\nrandom.seed(RANDOM_SEED)\n\nTRAIN_DIR = '/kaggle/input/airbus-ship-detection/train_v2/'\nTEST_DIR = '/kaggle/input/airbus-ship-detection/test_v2/'","metadata":{"papermill":{"duration":4.666522,"end_time":"2023-02-04T15:51:05.342551","exception":false,"start_time":"2023-02-04T15:51:00.676029","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/airbus-ship-detection/train_ship_segmentations_v2.csv\")\ndf['EncodedPixels'] = df['EncodedPixels'].astype('string')\n\n# Delete corrupted images\nCORRUPTED_IMAGES = ['6384c3e78.jpg']\ndf = df.drop(df[df['ImageId'].isin(CORRUPTED_IMAGES)].index)\n\n# Dataframe that contains the segmentation for each ship in the image. \ninstance_segmentation = df\n\n# Dataframe that contains the segmentation of all ships in the image.\nimage_segmentation = df.groupby(by=['ImageId'])['EncodedPixels'].apply(lambda x: np.nan if pd.isna(x).any() else ' '.join(x)).reset_index()","metadata":{"papermill":{"duration":37.021013,"end_time":"2023-02-04T15:51:42.397723","exception":false,"start_time":"2023-02-04T15:51:05.376710","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Utils","metadata":{"papermill":{"duration":0.033656,"end_time":"2023-02-04T15:51:42.466327","exception":false,"start_time":"2023-02-04T15:51:42.432671","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def rle_to_mask(rle: str, shape=(768, 768)):\n    '''\n    :param rle: run length encoded pixels as string formated\n           shape: (height,width) of array to return \n    :return: numpy 2D array, 1 - mask, 0 - background\n    '''\n    encoded_pixels = np.array(rle.split(), dtype=int)\n    starts = encoded_pixels[::2] - 1\n    ends = starts + encoded_pixels[1::2]\n    img = np.zeros(shape[0] * shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo:hi] = 1\n    return img.reshape(shape).T  # Needed to align to RLE direction\n\ndef mask_to_rle(img, shape=(768, 768)) -> str:\n    \"\"\"\n    :param img: numpy 2D array, 1 - mask, 0 - background\n           shape: (height,width) dimensions of the image \n    :return: run length encoded pixels as string formated\n    \"\"\"\n    img = img.astype('float32')\n    img = cv2.resize(img, shape, interpolation=cv2.INTER_AREA)\n    img = np.stack(np.vectorize(lambda x: 0 if x < 0.1 else 1)(img), axis=1)\n    pixels = img.T.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)","metadata":{"papermill":{"duration":0.047908,"end_time":"2023-02-04T15:51:42.547750","exception":false,"start_time":"2023-02-04T15:51:42.499842","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data preprocessing","metadata":{"papermill":{"duration":0.033678,"end_time":"2023-02-04T15:51:42.615747","exception":false,"start_time":"2023-02-04T15:51:42.582069","status":"completed"},"tags":[]}},{"cell_type":"code","source":"IMAGES_WITHOUT_SHIPS_NUMBER = 25000\n\n# reduce the number of images without ships\nimages_without_ships = image_segmentation[image_segmentation['EncodedPixels'].isna()]['ImageId'].values[:IMAGES_WITHOUT_SHIPS_NUMBER]\nimages_with_ships = image_segmentation[image_segmentation['EncodedPixels'].notna()]['ImageId'].values\nimages_list = np.append(images_without_ships, images_with_ships)\n\n# remove corrupted images\nimages_list = np.array(list(filter(lambda x: x not in CORRUPTED_IMAGES, images_list)))","metadata":{"papermill":{"duration":0.099078,"end_time":"2023-02-04T15:51:42.748707","exception":false,"start_time":"2023-02-04T15:51:42.649629","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"VALIDATION_LENGTH = 2000\nTEST_LENGTH = 2000\nTRAIN_LENGTH = len(images_list) - VALIDATION_LENGTH - TEST_LENGTH\nBATCH_SIZE = 16\nBUFFER_SIZE = 1000\nIMG_SHAPE = (256, 256)\nNUM_CLASSES = 2","metadata":{"papermill":{"duration":0.043166,"end_time":"2023-02-04T15:51:42.826054","exception":false,"start_time":"2023-02-04T15:51:42.782888","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def one_hot(a, num_classes):\n    return np.squeeze(np.eye(num_classes)[a])\n\ndef load_train_image(tensor) -> tuple:\n    path = tf.get_static_value(tensor).decode(\"utf-8\")\n\n    image_id = path.split('/')[-1]\n    input_image = cv2.imread(path)\n    input_image = tf.image.resize(input_image, IMG_SHAPE)\n    input_image = tf.cast(input_image, tf.float32) / 255.0\n\n    encoded_mask = image_segmentation[image_segmentation['ImageId'] == image_id].iloc[0]['EncodedPixels']\n    input_mask = np.zeros(IMG_SHAPE + (1,), dtype=np.int8)\n    if not pd.isna(encoded_mask):\n        input_mask = rle_to_mask(encoded_mask)\n        input_mask = cv2.resize(input_mask, IMG_SHAPE, interpolation=cv2.INTER_AREA)\n        input_mask = np.expand_dims(input_mask, axis=2)\n    one_hot_segmentation_mask = one_hot(input_mask, NUM_CLASSES)\n    input_mask_tensor = tf.convert_to_tensor(one_hot_segmentation_mask, dtype=tf.float32)\n    \n    class_weights = tf.constant([0.0005, 0.9995], tf.float32)\n    sample_weights = tf.gather(class_weights, indices=tf.cast(input_mask_tensor, tf.int32), name='cast_sample_weights')\n\n    return input_image, input_mask_tensor, sample_weights\n\nimages_list = tf.data.Dataset.list_files([f'{TRAIN_DIR}{name}' for name in images_list], shuffle=True)\ntrain_images = images_list.map(lambda x: tf.py_function(load_train_image, [x], [tf.float32, tf.float32]), num_parallel_calls=tf.data.AUTOTUNE)\n\nvalidation_dataset = train_images.take(VALIDATION_LENGTH)\ntest_dataset = train_images.skip(VALIDATION_LENGTH).take(TEST_LENGTH)\ntrain_dataset = train_images.skip(VALIDATION_LENGTH + TEST_LENGTH)\n\ntrain_batches = (\n    train_dataset\n    .repeat()\n    .batch(BATCH_SIZE))\n\nvalidation_batches = validation_dataset.batch(BATCH_SIZE)\n\ntest_batches = test_dataset.batch(BATCH_SIZE)","metadata":{"papermill":{"duration":231.205574,"end_time":"2023-02-04T15:55:34.065265","exception":false,"start_time":"2023-02-04T15:51:42.859691","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## UNet segmentation model","metadata":{"papermill":{"duration":0.033864,"end_time":"2023-02-04T15:55:34.134313","exception":false,"start_time":"2023-02-04T15:55:34.100449","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"The UNet network was chosen as the architecture of the neural network for semantic segmentation, which allows high-precision segmentation of images, highlighting all the ships in the image.\nFor better accuracy and fewer model parameters, we use a MobileNetV2 classification model with trained weights as an encoder.\n","metadata":{"papermill":{"duration":0.03408,"end_time":"2023-02-04T15:55:34.202560","exception":false,"start_time":"2023-02-04T15:55:34.168480","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import keras.backend as K\nimport tensorflow_addons as tfa\n\nclass UNetModel:\n    def __init__(self, input_shape=(128, 128, 3)):\n        self._model = self._build_model(input_shape)\n\n    @property\n    def model(self) -> tf.keras.Model:\n        return self._model\n    \n    def _build_model(self, input_shape, num_classes=NUM_CLASSES) -> tf.keras.Model:\n        inputs = tf.keras.layers.Input(shape=input_shape)\n        \n        filters_list = [16, 32, 64]\n\n        # apply Encoder\n        encoder_outputs = self._encoder(input_shape, filters_list)(inputs)\n        print(f'Encoder output tensors: {encoder_outputs}')\n\n        # apply Decoder and establishing the skip connections\n        x = self._decoder(encoder_outputs, filters_list[::-1])\n\n        # This is the last layers of the model\n        last = self._conv_blocks(num_classes, size=1)(x)\n        outputs = tf.keras.activations.softmax(last)\n\n        return tf.keras.Model(inputs=inputs, outputs=outputs)\n    \n    def _encoder(self, input_shape, filters_list):\n        inputs = tf.keras.layers.Input(shape=input_shape)\n        outputs = []\n\n        model = tf.keras.Sequential()\n        x = model(inputs)\n\n        for filters in filters_list:\n            x = self._conv_blocks(filters=filters, size=3, apply_instance_norm=True)(x)\n            x = self._conv_blocks(filters=filters, size=1, apply_instance_norm=True)(x)\n            outputs.append(x)\n            x = tf.keras.layers.MaxPool2D(pool_size=(2, 2))(x)\n\n        output = self._conv_blocks(filters=128, size=3, apply_batch_norm=True, apply_dropout=False)(x)\n        outputs.append(output)\n\n        # Create the feature extraction model\n        encoder = tf.keras.Model(inputs=inputs, outputs=outputs, name=\"encoder\")\n        encoder.trainable = True\n        return encoder\n    \n    def _decoder(self, encoder_outputs, filters_list):     \n        x = encoder_outputs[-1]\n        for filters, skip, apply_dropout in zip(filters_list, encoder_outputs[-2::-1], [False] * 4):\n            x = self._upsample_block(filters, 3)(x)\n            x = tf.keras.layers.Concatenate()([x, skip])\n            x = self._conv_blocks(filters, size=3, apply_batch_norm=True, apply_dropout=apply_dropout)(x)\n            x = self._conv_blocks(filters, size=1, apply_batch_norm=True)(x)\n        return x\n    \n    def _conv_blocks(self, filters, size, apply_batch_norm=False, apply_instance_norm=False, apply_dropout=False):\n        \"\"\"Downsamples an input. Conv2D => Batchnorm => Dropout => LeakyReLU\n            :param:\n                filters: number of filters\n                size: filter size\n                apply_dropout: If True, adds the dropout layer\n            :return: Downsample Sequential Model\n        \"\"\"\n        initializer = tf.random_normal_initializer(0., 0.02)\n        result = tf.keras.Sequential()\n        result.add(\n          tf.keras.layers.Conv2D(filters, size, strides=1,\n                                 padding='same', use_bias=False,\n                                 kernel_initializer=initializer,))\n        if apply_batch_norm:\n            result.add(tf.keras.layers.BatchNormalization())\n        if apply_instance_norm:\n            result.add(tfa.layers.InstanceNormalization())\n        result.add(tf.keras.layers.Activation(tfa.activations.mish))\n        if apply_dropout:\n            result.add(tf.keras.layers.Dropout(0.55))\n        return result\n    \n    def _upsample_block(self, filters, size, apply_dropout=False):\n        \"\"\"Upsamples an input. Conv2DTranspose => Batchnorm => Dropout => LeakyReLU\n            :param:\n                filters: number of filters\n                size: filter size\n                apply_dropout: If True, adds the dropout layer\n            :return: Upsample Sequential Model\n        \"\"\"\n        initializer = tf.random_normal_initializer(0., 0.02)\n        result = tf.keras.Sequential()\n        result.add(\n          tf.keras.layers.Conv2DTranspose(filters, size, strides=2,\n                                          padding='same',\n                                          kernel_initializer=initializer,\n                                          use_bias=False))\n        result.add(tf.keras.layers.BatchNormalization())\n        if apply_dropout:\n            result.add(tf.keras.layers.Dropout(0.1))\n        result.add(tf.keras.layers.Activation(tfa.activations.mish))\n        return result\n    \n\ndef dice(targets, inputs, smooth=1e-6):\n    axis = [1,2,3]\n    intersection = K.sum(targets * inputs, axis=axis)\n    dice = (2 * intersection + smooth) / (K.sum(targets, axis=axis) + K.sum(inputs, axis=axis) + smooth)\n    return dice\n\ndef bce_loss(targets, inputs, smooth=1e-6):\n    axis = [1,2,3]\n    loss = K.sum(targets * tf.math.log(inputs + smooth) + (1 - targets) * tf.math.log(1 - inputs + smooth), axis=axis)\n    return - loss\n\ndef bce_dice_loss(targets, inputs):\n    return bce_loss(targets, inputs) - tf.math.log(dice(targets, inputs))","metadata":{"papermill":{"duration":0.995281,"end_time":"2023-02-04T15:55:35.232053","exception":false,"start_time":"2023-02-04T15:55:34.236772","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class IoU(tf.keras.metrics.Metric):\n    def __init__(self, num_classes: int, target_class_ids: list, sparse_y_true: bool, sparse_y_pred: bool,\n                 axis: int = -1, name=None, dtype=None):\n        super(IoU, self).__init__(name=name, dtype=dtype)\n        self.num_classes = num_classes\n        self.target_class_ids = target_class_ids\n        self.sparse_y_true = sparse_y_true\n        self.sparse_y_pred = sparse_y_pred\n        self.axis = axis\n\n        # Variable to accumulate the predictions in the confusion matrix.\n        self.total_cm = self.add_weight(\n            'total_confusion_matrix',\n            shape=(num_classes, num_classes),\n            initializer='zeros')\n\n    def update_state(self, y_true, y_pred, sample_weight=None):\n        \"\"\"Accumulates the confusion matrix statistics.\n        Args:\n          y_true: The ground truth values.\n          y_pred: The predicted values.\n          sample_weight: Optional weighting of each example. Defaults to 1. Can be a\n            `Tensor` whose rank is either 0, or the same rank as `y_true`, and must\n            be broadcastable to `y_true`.\n        Returns:\n          Update op.\n        \"\"\"\n        \n        y_true = tf.reshape(y_true, [-1] + list(y_pred.shape[1:]))\n        \n        if not self.sparse_y_true:\n            y_true = tf.argmax(y_true, axis=self.axis)\n        if not self.sparse_y_pred:\n            y_pred = tf.argmax(y_pred, axis=self.axis)\n            \n        y_true = tf.cast(y_true, self._dtype)\n        y_pred = tf.cast(y_pred, self._dtype)\n\n        # Flatten the input if its rank > 1.\n        if y_pred.shape.ndims > 1:\n            y_pred = tf.reshape(y_pred, [-1])\n\n        if y_true.shape.ndims > 1:\n            y_true = tf.reshape(y_true, [-1])\n\n        if sample_weight is not None:\n            sample_weight = tf.reshape(sample_weight, [-1, 128, 128])\n            sample_weight = tf.cast(sample_weight, self._dtype)\n            if sample_weight.shape.ndims > 1:\n                sample_weight = tf.reshape(sample_weight, [-1])\n\n        # Accumulate the prediction to current confusion matrix.\n        current_cm = tf.math.confusion_matrix(y_true, y_pred, self.num_classes, weights=sample_weight, dtype=self._dtype)\n        return self.total_cm.assign_add(current_cm)\n    \n    def reset_state(self):\n        tf.keras.backend.set_value(\n            self.total_cm, np.zeros((self.num_classes, self.num_classes))\n        )\n    \n    def result(self):\n        \"\"\"Compute the intersection-over-union via the confusion matrix.\"\"\"\n        sum_over_row = tf.cast(\n            tf.reduce_sum(self.total_cm, axis=0), dtype=self._dtype)\n        sum_over_col = tf.cast(\n            tf.reduce_sum(self.total_cm, axis=1), dtype=self._dtype)\n        true_positives = tf.cast(\n            tf.linalg.tensor_diag_part(self.total_cm), dtype=self._dtype)\n\n        # sum_over_row + sum_over_col = 2 * true_positives + false_positives + false_negatives.\n        denominator = sum_over_row + sum_over_col - true_positives\n\n        # Only keep the target classes\n        true_positives = tf.gather(true_positives, self.target_class_ids)\n        denominator = tf.gather(denominator, self.target_class_ids)\n\n        # If the denominator is 0, we need to ignore the class.\n        num_valid_entries = tf.reduce_sum(\n            tf.cast(tf.not_equal(denominator, 0), dtype=self._dtype))\n\n        iou = tf.math.divide_no_nan(true_positives, denominator)\n\n        return tf.math.divide_no_nan(\n            tf.reduce_sum(iou, name='mean_iou'), num_valid_entries)\n    \n    def get_config(self):\n        config = {\n            \"num_classes\": self.num_classes,\n            \"target_class_ids\": self.target_class_ids,\n            \"sparse_y_true\": self.sparse_y_true,\n            \"sparse_y_pred\": self.sparse_y_pred,\n            \"axis\": self.axis,\n        }\n        base_config = super().get_config()\n        return dict(list(base_config.items()) + list(config.items()))","metadata":{"papermill":{"duration":0.053895,"end_time":"2023-02-04T15:55:35.320083","exception":false,"start_time":"2023-02-04T15:55:35.266188","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EPOCHS = 15\nSTEPS_PER_EPOCH = TRAIN_LENGTH // BATCH_SIZE\n\noptimizer = tfa.optimizers.RectifiedAdam(\n    learning_rate=0.005,\n    total_steps=EPOCHS * STEPS_PER_EPOCH,\n    warmup_proportion=0.3,\n    min_lr=0.00001,\n)\noptimizer = tfa.optimizers.Lookahead(optimizer)\n\nloss = tf.keras.losses.CategoricalCrossentropy()\nmIoU = IoU(num_classes=2, target_class_ids=[0, 1], sparse_y_true=False, sparse_y_pred=False, name='mean-IoU')\n\nmodel = UNetModel(IMG_SHAPE + (3,)).model\nmodel.compile(optimizer=optimizer, \n              loss=loss, # bce_dice_loss,\n              metrics=[mIoU],)\n\ntrainable_params = np.sum([np.prod(v.get_shape().as_list()) for v in model.trainable_variables])\nprint(f'Trainable params: {trainable_params}')\n\ntf.keras.utils.plot_model(model, show_shapes=True)","metadata":{"papermill":{"duration":2.363101,"end_time":"2023-02-04T15:55:37.716543","exception":false,"start_time":"2023-02-04T15:55:35.353442","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"checkpoint_filepath = '/kaggle/working/checkpoints/model-checkpoint'\nsave_callback = keras.callbacks.ModelCheckpoint(\n    filepath=checkpoint_filepath,\n    monitor='val_mean-IoU',\n    mode='max',\n    save_best_only=True\n)\n\nmodel_history = model.fit(train_batches,\n                          epochs=EPOCHS,\n                          steps_per_epoch=STEPS_PER_EPOCH,\n                          validation_data=validation_batches,\n                          callbacks=[save_callback])\nmodel.load_weights(checkpoint_filepath)","metadata":{"execution":{"iopub.execute_input":"2023-02-04T15:55:37.796599Z","iopub.status.busy":"2023-02-04T15:55:37.796215Z","iopub.status.idle":"2023-02-05T00:10:13.583933Z","shell.execute_reply":"2023-02-05T00:10:13.582972Z"},"papermill":{"duration":29678.925797,"end_time":"2023-02-05T00:10:16.681940","exception":false,"start_time":"2023-02-04T15:55:37.756143","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss = model_history.history['loss']\nval_loss = model_history.history['val_loss']\n\nplt.figure()\nplt.plot(model_history.epoch, loss, 'r', label='Training loss')\nplt.plot(model_history.epoch, val_loss, 'C2', label='Validation loss')\nplt.title('Training and Validation Loss')\nplt.xlabel('Epoch')\nplt.ylabel('Loss Value')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.execute_input":"2023-02-05T00:10:24.078467Z","iopub.status.busy":"2023-02-05T00:10:24.078106Z","iopub.status.idle":"2023-02-05T00:10:24.324621Z","shell.execute_reply":"2023-02-05T00:10:24.323691Z"},"papermill":{"duration":3.776699,"end_time":"2023-02-05T00:10:24.326871","exception":false,"start_time":"2023-02-05T00:10:20.550172","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mIoU = model_history.history['mean-IoU']\nval_mIoU = model_history.history['val_mean-IoU']\n\nplt.figure()\nplt.plot(model_history.epoch, mIoU, 'm', label='Training mean IoU')\nplt.plot(model_history.epoch, val_mIoU, 'y', label='Validation mean IoU')\n\nplt.title('Training and Validation Metrics')\nplt.xlabel('Epoch')\nplt.ylabel('Metric Value')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.execute_input":"2023-02-05T00:10:31.118525Z","iopub.status.busy":"2023-02-05T00:10:31.118161Z","iopub.status.idle":"2023-02-05T00:10:31.896370Z","shell.execute_reply":"2023-02-05T00:10:31.895473Z"},"papermill":{"duration":4.225831,"end_time":"2023-02-05T00:10:31.898498","exception":false,"start_time":"2023-02-05T00:10:27.672667","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict(image):\n    image = np.expand_dims(image, axis=0)\n    pred_mask = model.predict(image)[0].argmax(axis=-1)  \n    return pred_mask","metadata":{"papermill":{"duration":3.172895,"end_time":"2023-02-05T00:10:38.607914","exception":false,"start_time":"2023-02-05T00:10:35.435019","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-05T14:12:14.516701Z","iopub.status.idle":"2023-02-05T14:12:14.517877Z","shell.execute_reply.started":"2023-02-05T14:12:14.517622Z","shell.execute_reply":"2023-02-05T14:12:14.517646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Deep Watershed Transform","metadata":{"papermill":{"duration":3.729929,"end_time":"2023-02-05T00:10:45.657042","exception":false,"start_time":"2023-02-05T00:10:41.927113","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# TODO","metadata":{"papermill":{"duration":3.846115,"end_time":"2023-02-05T00:10:52.644060","exception":false,"start_time":"2023-02-05T00:10:48.797945","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-05T14:12:14.519202Z","iopub.status.idle":"2023-02-05T14:12:14.520034Z","shell.execute_reply.started":"2023-02-05T14:12:14.519781Z","shell.execute_reply":"2023-02-05T14:12:14.519806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Results analysis","metadata":{"papermill":{"duration":3.134931,"end_time":"2023-02-05T00:10:59.407806","exception":false,"start_time":"2023-02-05T00:10:56.272875","status":"completed"},"tags":[]}},{"cell_type":"code","source":"N = 5\n\nf,ax = plt.subplots(N, 3, figsize=(10, 4 * N))\ni = 0\nfor image, mask in test_dataset.take(N):\n    mask = mask.numpy().argmax(axis=-1)\n    ax[i, 0].imshow(image)\n    ax[i, 0].set_title('image')\n    ax[i, 1].imshow(mask)\n    ax[i, 1].set_title('true mask')\n\n    pred_mask = predict(image)\n    ax[i, 2].imshow(pred_mask)\n    ax[i, 2].set_title('predicted mask')\n    i += 1\n\nplt.show()","metadata":{"papermill":{"duration":48.909569,"end_time":"2023-02-05T00:11:51.590225","exception":false,"start_time":"2023-02-05T00:11:02.680656","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-05T14:12:14.521539Z","iopub.status.idle":"2023-02-05T14:12:14.522379Z","shell.execute_reply.started":"2023-02-05T14:12:14.522113Z","shell.execute_reply":"2023-02-05T14:12:14.522152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = model.evaluate(test_batches)\nprint(\"test loss, test mIoU:\", results)","metadata":{"papermill":{"duration":92.661717,"end_time":"2023-02-05T00:13:28.137311","exception":false,"start_time":"2023-02-05T00:11:55.475594","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-05T14:12:14.524090Z","iopub.status.idle":"2023-02-05T14:12:14.524852Z","shell.execute_reply.started":"2023-02-05T14:12:14.524586Z","shell.execute_reply":"2023-02-05T14:12:14.524611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mIoU = IoU(num_classes=2, target_class_ids=[0, 1], sparse_y_true=True, sparse_y_pred=True, name='mean-IoU')\nIoU_results = []\nfor image, true_mask in test_dataset.take(TEST_LENGTH):\n    true_mask = true_mask.numpy().argmax(axis=-1)\n    pred_mask = predict(image)\n    mIoU.update_state(true_mask, pred_mask)\n    \n    iou = IoU(num_classes=2, target_class_ids=[0, 1], sparse_y_true=True, sparse_y_pred=True, name='mean-IoU')\n    iou.update_state(true_mask, pred_mask)\n    IoU_results.append(iou.result())\n\nplt.hist(IoU_results, bins=15)\nprint(mIoU.result())","metadata":{"papermill":{"duration":192.98482,"end_time":"2023-02-05T00:16:44.859519","exception":false,"start_time":"2023-02-05T00:13:31.874699","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-05T14:12:14.526301Z","iopub.status.idle":"2023-02-05T14:12:14.527006Z","shell.execute_reply.started":"2023-02-05T14:12:14.526752Z","shell.execute_reply":"2023-02-05T14:12:14.526775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{"papermill":{"duration":3.468991,"end_time":"2023-02-05T00:16:51.713059","exception":false,"start_time":"2023-02-05T00:16:48.244068","status":"completed"},"tags":[]}},{"cell_type":"code","source":"submission = pd.read_csv(\"/kaggle/input/airbus-ship-detection/sample_submission_v2.csv\")\n\ndef set_model_prediction(row: pd.Series) -> pd.Series:\n    image = cv2.imread(f'{TEST_DIR}{row[\"ImageId\"]}')\n    image = cv2.resize(image, IMG_SHAPE, interpolation=cv2.INTER_AREA)\n    image = image / 255.0\n    pred_mask = predict(image)\n    row['EncodedPixels'] = mask_to_rle(pred_mask)\n    if row['EncodedPixels'] == '':\n        row['EncodedPixels'] = np.nan\n    return row\n\nsubmission = submission.apply(lambda x: set_model_prediction(x), axis=1).set_index(\"ImageId\")\n\nsubmission.to_csv(\"./submission.csv\")\nsubmission","metadata":{"papermill":{"duration":2902.09794,"end_time":"2023-02-05T01:05:16.940706","exception":false,"start_time":"2023-02-05T00:16:54.842766","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-05T14:12:14.528366Z","iopub.status.idle":"2023-02-05T14:12:14.529072Z","shell.execute_reply.started":"2023-02-05T14:12:14.528817Z","shell.execute_reply":"2023-02-05T14:12:14.528841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# How can we try to improve the model\n\n1. Pretrain the encoder for classifying ships. And then do not change the encoder weights when training the segmentation model.\n2. Use Data Augmentation for images with ships\n3. Reduce the number of images without ships when training the model\n4. Use original image shape (`768 * 768 * 3`) when training model","metadata":{"papermill":{"duration":3.200103,"end_time":"2023-02-05T01:05:23.443782","exception":false,"start_time":"2023-02-05T01:05:20.243679","status":"completed"},"tags":[]}}]}