{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# REQUIRED\n\nfrom PIL import Image, ImageDraw, ImageEnhance\nfrom IPython.display import display\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport pandas as pd\nimport numpy as np\nimport ast\nimport os","metadata":{"execution":{"iopub.status.busy":"2021-12-08T13:01:19.574869Z","iopub.execute_input":"2021-12-08T13:01:19.575338Z","iopub.status.idle":"2021-12-08T13:01:24.625143Z","shell.execute_reply.started":"2021-12-08T13:01:19.57522Z","shell.execute_reply":"2021-12-08T13:01:24.624171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# CONSTANTS\n\nINPUT_PATH = str('../input/tensorflow-great-barrier-reef')\nIMAGE_SIZE = tuple((int(1280), int(720))) # width x height","metadata":{"execution":{"iopub.status.busy":"2021-12-08T13:01:24.627211Z","iopub.execute_input":"2021-12-08T13:01:24.627468Z","iopub.status.idle":"2021-12-08T13:01:24.632566Z","shell.execute_reply.started":"2021-12-08T13:01:24.627435Z","shell.execute_reply":"2021-12-08T13:01:24.631535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# GET THE DATA AND DISPLAY ITS STRUCTURE\n\ntrain_dataset = pd.read_csv(INPUT_PATH + '/train.csv')\nprint(train_dataset)\nprint(\"\\n\")\nprint(train_dataset.info())","metadata":{"execution":{"iopub.status.busy":"2021-12-08T13:01:24.633919Z","iopub.execute_input":"2021-12-08T13:01:24.634573Z","iopub.status.idle":"2021-12-08T13:01:24.730424Z","shell.execute_reply.started":"2021-12-08T13:01:24.634517Z","shell.execute_reply":"2021-12-08T13:01:24.729765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# DISPLAY THE DATA\n\nfor video_id in train_dataset['video_id'].unique():\n    print(f'Image ID: {video_id}')\n    print(f'Number of images withou COTS:  {sum(train_dataset[train_dataset[\"video_id\"]==video_id][\"annotations\"] == \"[]\")}')\n    print(f'Number of images with COTS:  {sum(train_dataset[train_dataset[\"video_id\"]==video_id][\"annotations\"] != \"[]\")}')\n    if video_id != 2:\n        print(f'----------------------------------')","metadata":{"execution":{"iopub.status.busy":"2021-12-08T13:01:24.731655Z","iopub.execute_input":"2021-12-08T13:01:24.73269Z","iopub.status.idle":"2021-12-08T13:01:24.758263Z","shell.execute_reply.started":"2021-12-08T13:01:24.73264Z","shell.execute_reply":"2021-12-08T13:01:24.757387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# CONVERT DATA ANNOTATIONS STR to LIST\n\ntrain_dataset['annotations'] = train_dataset['annotations'].apply(ast.literal_eval)","metadata":{"execution":{"iopub.status.busy":"2021-12-08T13:01:24.760213Z","iopub.execute_input":"2021-12-08T13:01:24.760461Z","iopub.status.idle":"2021-12-08T13:01:25.223007Z","shell.execute_reply.started":"2021-12-08T13:01:24.760429Z","shell.execute_reply":"2021-12-08T13:01:25.222292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ADD A COLUMN WITH THE IMAGE PATH\n\ntrain_dataset['image_path'] = INPUT_PATH + '/train_images/video_' + train_dataset['video_id'].astype(str) + '/' + train_dataset['video_frame'].astype(str) + \".jpg\"","metadata":{"execution":{"iopub.status.busy":"2021-12-08T13:01:25.224408Z","iopub.execute_input":"2021-12-08T13:01:25.224672Z","iopub.status.idle":"2021-12-08T13:01:25.278108Z","shell.execute_reply.started":"2021-12-08T13:01:25.22464Z","shell.execute_reply":"2021-12-08T13:01:25.277039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ADD A COLUMN WITH THE NUMBER OF BOXES PER IMAGE\n\ntrain_dataset['num_bboxes'] = train_dataset['annotations'].apply(lambda x: len(x))","metadata":{"execution":{"iopub.status.busy":"2021-12-08T13:01:25.279851Z","iopub.execute_input":"2021-12-08T13:01:25.280077Z","iopub.status.idle":"2021-12-08T13:01:25.295545Z","shell.execute_reply.started":"2021-12-08T13:01:25.280051Z","shell.execute_reply":"2021-12-08T13:01:25.294598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# DISPLAY THE NEW STRUCTURE\n\ntrain_dataset.head(18)","metadata":{"execution":{"iopub.status.busy":"2021-12-08T13:01:25.297014Z","iopub.execute_input":"2021-12-08T13:01:25.297365Z","iopub.status.idle":"2021-12-08T13:01:25.328295Z","shell.execute_reply.started":"2021-12-08T13:01:25.297332Z","shell.execute_reply":"2021-12-08T13:01:25.327187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def draw_grid(draw, strides=8):\n    x_lines = int(IMAGE_SIZE[0] // strides)\n    y_lines = int(IMAGE_SIZE[1] // strides)\n    \n    for line in range(x_lines):\n        shape = tuple((((IMAGE_SIZE[0] // x_lines) * line, 0), ((IMAGE_SIZE[0] // x_lines) * line, IMAGE_SIZE[1])))\n        draw.line(shape, fill=\"black\", width=1)\n        \n    for line in range(y_lines):\n        shape = tuple(((0, (IMAGE_SIZE[1] // y_lines) * line), (IMAGE_SIZE[0], (IMAGE_SIZE[1] // y_lines) * line)))\n        draw.line(shape, fill=\"black\", width=1)\n        \ndef draw_bbox(draw, bbox):\n    x, y, width, height = bbox['x'], bbox['y'], bbox['width'], bbox['height']\n    draw.rectangle([x, y, x + width, y + height], width=2, outline='salmon')\n    draw.text([x, y - 10], 'COTS', width=7, fill='salmon')\n    \ndef draw_bboxes(image_path, bboxes, grid=False):\n    image = Image.open(image_path)\n    image = image.resize(IMAGE_SIZE)\n    draw  = ImageDraw.Draw(image)\n    \n    if grid:\n        draw_grid(draw)\n            \n    for bbox in bboxes:\n        draw_bbox(draw, bbox)\n    \n    return image","metadata":{"execution":{"iopub.status.busy":"2021-12-08T13:01:25.32985Z","iopub.execute_input":"2021-12-08T13:01:25.330164Z","iopub.status.idle":"2021-12-08T13:01:25.340919Z","shell.execute_reply.started":"2021-12-08T13:01:25.330119Z","shell.execute_reply":"2021-12-08T13:01:25.33997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# DISPLAY AN IMAGE\n\nIMAGE_TEST_ID = int(35)\ndisplay(draw_bboxes(train_dataset['image_path'][IMAGE_TEST_ID], train_dataset['annotations'][IMAGE_TEST_ID]))","metadata":{"execution":{"iopub.status.busy":"2021-12-08T13:01:25.342351Z","iopub.execute_input":"2021-12-08T13:01:25.342701Z","iopub.status.idle":"2021-12-08T13:01:25.775246Z","shell.execute_reply.started":"2021-12-08T13:01:25.342605Z","shell.execute_reply":"2021-12-08T13:01:25.774285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(draw_bboxes(train_dataset['image_path'][IMAGE_TEST_ID], train_dataset['annotations'][IMAGE_TEST_ID], grid=True))","metadata":{"execution":{"iopub.status.busy":"2021-12-08T13:01:25.776612Z","iopub.execute_input":"2021-12-08T13:01:25.77688Z","iopub.status.idle":"2021-12-08T13:01:26.133037Z","shell.execute_reply.started":"2021-12-08T13:01:25.776846Z","shell.execute_reply":"2021-12-08T13:01:26.130069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_image(image_path):\n    image = tf.io.read_file(image_path)\n    image = tf.image.decode_jpeg(image)\n    image = tf.cast(image, tf.float32)[..., :3]\n    return ((image / 127.5) - 1) # [-1, 1] (height, width, channels)\n\ndef label_image(labels):\n    label_map = np.zeros((IMAGE_SIZE[1] // 8, IMAGE_SIZE[0] // 8, 1))\n#     label_map = np.reshape(label_map, (IMAGE_SIZE[0] // 8, IMAGE_SIZE[1] // 8, 1))\n    for label in labels:\n        if label['x']:\n            x1, y1, x2, y2 = label['x'] // 8, label['y'] // 8, (label['x'] // 8) + (label['width'] // 8), (label['y'] // 8) + (label['height'] // 8)\n            for y in range((IMAGE_SIZE[0] // 8)):\n                for x in range((IMAGE_SIZE[1] // 8)):\n                    if x >= x1 and x <= x2 and y >= y1 and y <= y2:\n                        label_map[y][x][0] = 1.0\n\n    return tf.cast(tf.convert_to_tensor(label_map), tf.float32) # [0, 1] (height, width, channels)","metadata":{"execution":{"iopub.status.busy":"2021-12-08T13:01:26.134211Z","iopub.execute_input":"2021-12-08T13:01:26.134434Z","iopub.status.idle":"2021-12-08T13:01:26.14319Z","shell.execute_reply.started":"2021-12-08T13:01:26.134407Z","shell.execute_reply":"2021-12-08T13:01:26.142457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# DATASET MAKER\n\n\"\"\"\n\n    In this case, since there is only one possibility (COTS),the tiles\n    in the grid where there a COTS will be activated with a 1 value.\n    \n    Example in a 5x5 matrix where in the rigth side there's a COTS:\n    0 0 0 1 1\n    0 0 0 1 1\n    0 0 1 1 1\n    0 0 0 1 1\n    0 0 0 0 1\n\n\"\"\"\n\ndata = list([])\nfor i, image in enumerate(train_dataset['image_path'][:18]):\n    data.append([load_image(image), label_image(train_dataset['annotations'][i])])\n    \nprint(tf.shape(data[0][0]))\nprint(data[0][1].shape)\nplt.subplots(figsize=(32, 18))\nplt.imshow(data[17][1])\nplt.show()\nplt.subplots(figsize=(32, 18))\nplt.imshow(data[17][0] * 0.5 + 0.5)\nplt.show()\ndisplay(draw_bboxes(train_dataset['image_path'][17], train_dataset['annotations'][17]))","metadata":{"execution":{"iopub.status.busy":"2021-12-08T13:01:26.144518Z","iopub.execute_input":"2021-12-08T13:01:26.145007Z","iopub.status.idle":"2021-12-08T13:01:29.727131Z","shell.execute_reply.started":"2021-12-08T13:01:26.144964Z","shell.execute_reply":"2021-12-08T13:01:29.724831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# THIS MODEL IS BASED ON VGG16 MODEL AND IMPROVE\n\ndef vgg16_residual_head(**kwargs):\n    inputs = tf.keras.layers.Input((IMAGE_SIZE[1], IMAGE_SIZE[0], 3)) # (height, width, channels)\n    \n    x = tf.keras.layers.Conv2D(64, kernel_size=3, padding='same')(inputs)\n    x = tf.keras.layers.Conv2D(64, kernel_size=3, padding='same', strides=2)(x)\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.keras.layers.LeakyReLU(0.05)(x)\n    x = tf.keras.layers.Conv2D(128, kernel_size=3, padding='same')(x)\n    r = x\n    x = tf.keras.layers.Conv2D(128, kernel_size=3, padding='same')(x)\n    x = tf.keras.layers.Add()([r, x])\n    x = tf.keras.layers.Conv2D(128, kernel_size=3, padding='same', strides=2)(x)\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.keras.layers.LeakyReLU(0.05)(x)\n    x = tf.keras.layers.Conv2D(256, kernel_size=3, padding='same')(x)\n    r = x\n    x = tf.keras.layers.Conv2D(256, kernel_size=3, padding='same')(x)\n    x = tf.keras.layers.Add()([r, x])\n    x = tf.keras.layers.Conv2D(256, kernel_size=3, padding='same')(x)\n    x = tf.keras.layers.LeakyReLU(0.05)(x)\n    x = tf.keras.layers.Conv2D(256, kernel_size=3, padding='same', strides=2)(x)\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.keras.layers.LeakyReLU(0.05)(x)\n    \n    outputs = tf.keras.layers.Conv2D(1, kernel_size=1, padding='same', activation='sigmoid')(x)\n    \n    return tf.keras.models.Model(inputs=inputs, outputs=outputs, **kwargs)\n\nmodel = vgg16_residual_head(name='model2_vgg16_head')\nmodel.summary(120)","metadata":{"execution":{"iopub.status.busy":"2021-12-08T13:01:52.071057Z","iopub.execute_input":"2021-12-08T13:01:52.071379Z","iopub.status.idle":"2021-12-08T13:01:52.235827Z","shell.execute_reply.started":"2021-12-08T13:01:52.071348Z","shell.execute_reply":"2021-12-08T13:01:52.23489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = tf.reshape(load_image(train_dataset['image_path'][IMAGE_TEST_ID]), (1, IMAGE_SIZE[1], IMAGE_SIZE[0], 3))\nY = tf.reshape(label_image(train_dataset['annotations'][IMAGE_TEST_ID]), (1, 90, 160, 1))","metadata":{"execution":{"iopub.status.busy":"2021-12-08T13:01:56.760226Z","iopub.execute_input":"2021-12-08T13:01:56.761117Z","iopub.status.idle":"2021-12-08T13:01:56.794792Z","shell.execute_reply.started":"2021-12-08T13:01:56.761068Z","shell.execute_reply":"2021-12-08T13:01:56.793844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimizer = tf.keras.optimizers.Adam(1e-4, epsilon=1e-06)\nloss_obj  = tf.keras.losses.BinaryCrossentropy()","metadata":{"execution":{"iopub.status.busy":"2021-12-08T13:03:47.127493Z","iopub.execute_input":"2021-12-08T13:03:47.127872Z","iopub.status.idle":"2021-12-08T13:03:47.133382Z","shell.execute_reply.started":"2021-12-08T13:03:47.12783Z","shell.execute_reply":"2021-12-08T13:03:47.132509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@tf.function\ndef train_step(x, y):\n    with tf.GradientTape() as tape:\n        pred  = model(x, training=True)\n        mae   = 1.5 * tf.reduce_mean(tf.math.abs((y - pred)))\n        mse   = 1.5 * tf.reduce_mean(tf.math.square((pred - y)))\n        bce   = loss_obj(y, pred)\n        loss  = mae + bce + mse\n        \n        grad = tape.gradient(loss, model.trainable_weights)\n        optimizer.apply_gradients(zip(grad, model.trainable_weights))\n        \n    return loss","metadata":{"execution":{"iopub.status.busy":"2021-12-08T13:40:04.341383Z","iopub.execute_input":"2021-12-08T13:40:04.341733Z","iopub.status.idle":"2021-12-08T13:40:04.35695Z","shell.execute_reply.started":"2021-12-08T13:40:04.34169Z","shell.execute_reply":"2021-12-08T13:40:04.356067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_loss = train_step(X, Y)","metadata":{"execution":{"iopub.status.busy":"2021-12-08T13:40:08.542754Z","iopub.execute_input":"2021-12-08T13:40:08.543331Z","iopub.status.idle":"2021-12-08T13:40:25.170296Z","shell.execute_reply.started":"2021-12-08T13:40:08.543289Z","shell.execute_reply":"2021-12-08T13:40:25.169416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'Loss: {model_loss.numpy()}')","metadata":{"execution":{"iopub.status.busy":"2021-12-08T13:40:34.618804Z","iopub.execute_input":"2021-12-08T13:40:34.619151Z","iopub.status.idle":"2021-12-08T13:40:34.624722Z","shell.execute_reply.started":"2021-12-08T13:40:34.619119Z","shell.execute_reply":"2021-12-08T13:40:34.62407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"limit = int(100)\nfor i, image_path in enumerate(train_dataset['image_path']):\n    X = tf.reshape(load_image(train_dataset['image_path'][IMAGE_TEST_ID]), (1, IMAGE_SIZE[1], IMAGE_SIZE[0], 3))\n    Y = tf.reshape(label_image(train_dataset['annotations'][IMAGE_TEST_ID]), (1, IMAGE_SIZE[1]//8, IMAGE_SIZE[0]//8, 1))\n    loss = train_step(X, Y)\n    print(f'Step: [{(i + 1)}/{limit}] - Loss: {loss.numpy()}')\n    if i == limit:\n        break","metadata":{"execution":{"iopub.status.busy":"2021-12-08T13:09:51.087798Z","iopub.execute_input":"2021-12-08T13:09:51.08835Z","iopub.status.idle":"2021-12-08T13:26:18.325802Z","shell.execute_reply.started":"2021-12-08T13:09:51.088306Z","shell.execute_reply":"2021-12-08T13:26:18.324499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs  = load_image(train_dataset['image_path'][16])\noutputs = model(tf.reshape(inputs, (1, IMAGE_SIZE[1], IMAGE_SIZE[0], 3)))\ntarget  = label_image(train_dataset['annotations'][16])","metadata":{"execution":{"iopub.status.busy":"2021-12-08T13:33:08.1557Z","iopub.execute_input":"2021-12-08T13:33:08.155996Z","iopub.status.idle":"2021-12-08T13:33:10.730426Z","shell.execute_reply.started":"2021-12-08T13:33:08.155966Z","shell.execute_reply":"2021-12-08T13:33:10.729723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Input')\nplt.subplots(figsize=(32, 18))\nplt.imshow(inputs * 0.5 + 0.5)\nplt.show()\nprint('Output')\nplt.subplots(figsize=(32, 18))\nplt.imshow(outputs[0])\nplt.show()\nprint('Target')\nplt.subplots(figsize=(32, 18))\nplt.imshow(target)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-08T13:33:13.192861Z","iopub.execute_input":"2021-12-08T13:33:13.193822Z","iopub.status.idle":"2021-12-08T13:33:16.167054Z","shell.execute_reply.started":"2021-12-08T13:33:13.193771Z","shell.execute_reply":"2021-12-08T13:33:16.166036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}