{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Objective of first fast YOLO inspired network\nThe first network will generate a square where we will be abble to find the real starfish. It means that I need a very good recall without lose too much accuracy...  \nI choose for a fast network to avoid overfitting.  \n**Important to notice: it is not a real fast YOLO implementation but just inspired in YOLO.**","metadata":{}},{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"# Data analysis and processing\nimport pandas as pd\nimport numpy as np\n\n# Tensor processing tool\nimport tensorflow as tf\nimport tensorflow_addons as tfa\nfrom tensorflow import keras","metadata":{"execution":{"iopub.status.busy":"2022-02-06T02:07:10.345020Z","iopub.execute_input":"2022-02-06T02:07:10.345781Z","iopub.status.idle":"2022-02-06T02:07:10.350911Z","shell.execute_reply.started":"2022-02-06T02:07:10.345730Z","shell.execute_reply":"2022-02-06T02:07:10.350216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Help Functions","metadata":{}},{"cell_type":"code","source":"def image_path(video_id, video_frame):\n    return f\"/kaggle/input/tensorflow-great-barrier-reef/train_images/video_{video_id}/{video_frame}.jpg\"","metadata":{"execution":{"iopub.status.busy":"2022-02-06T02:07:10.352662Z","iopub.execute_input":"2022-02-06T02:07:10.353753Z","iopub.status.idle":"2022-02-06T02:07:10.360808Z","shell.execute_reply.started":"2022-02-06T02:07:10.353715Z","shell.execute_reply":"2022-02-06T02:07:10.360189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def cocoincode(cocobboxdict):\n    \"\"\"COCO incode is a np array like ['x' 'y' 'width' 'height']\"\"\"\n    return np.array([cocobboxdict['x'], cocobboxdict['y'], cocobboxdict['width'], cocobboxdict['height']], dtype=np.int32)","metadata":{"execution":{"iopub.status.busy":"2022-02-06T02:07:10.362343Z","iopub.execute_input":"2022-02-06T02:07:10.362937Z","iopub.status.idle":"2022-02-06T02:07:10.369578Z","shell.execute_reply.started":"2022-02-06T02:07:10.362823Z","shell.execute_reply":"2022-02-06T02:07:10.368949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def coco2yolo(cocobbox):\n    \"\"\" Transform a COCO np array like ['x' 'y' 'width' 'height'] in a YOLO np array like ['x' 'y' 'width' 'height']\"\"\"\n    return np.array([(cocobbox[0] + cocobbox[2]//2 + cocobbox[2]%2), (cocobbox[1] + cocobbox[3]//2 + cocobbox[3]%2), cocobbox[2], cocobbox[3]], dtype=np.int32)","metadata":{"execution":{"iopub.status.busy":"2022-02-06T02:07:10.371008Z","iopub.execute_input":"2022-02-06T02:07:10.371349Z","iopub.status.idle":"2022-02-06T02:07:10.377628Z","shell.execute_reply.started":"2022-02-06T02:07:10.371317Z","shell.execute_reply":"2022-02-06T02:07:10.376856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def yolo2yolosquared(yolobbox):\n    \"\"\" Transform a YOLO np array like ['x' 'y' 'width' 'height'] to a YOLO np array like ['x' 'y' 'side']\"\"\"\n    return np.array([yolobbox[0], yolobbox[1], max(yolobbox[2], yolobbox[3])], dtype=np.int32)","metadata":{"execution":{"iopub.status.busy":"2022-02-06T02:07:10.380707Z","iopub.execute_input":"2022-02-06T02:07:10.380936Z","iopub.status.idle":"2022-02-06T02:07:10.386016Z","shell.execute_reply.started":"2022-02-06T02:07:10.380907Z","shell.execute_reply":"2022-02-06T02:07:10.385278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def yolosquared2yolo(yolosquaredbbox):\n    \"\"\" Transform a YOLO np array like ['x' 'y' 'side'] to a YOLO np array like ['x' 'y' 'width' 'height']\"\"\"\n    return np.array([yolosquaredbbox[0], yolosquaredbbox[1], yolosquaredbbox[2], yolosquaredbbox[2]], dtype=np.int32)\n    ","metadata":{"execution":{"iopub.status.busy":"2022-02-06T02:07:10.387689Z","iopub.execute_input":"2022-02-06T02:07:10.388003Z","iopub.status.idle":"2022-02-06T02:07:10.394142Z","shell.execute_reply.started":"2022-02-06T02:07:10.387970Z","shell.execute_reply":"2022-02-06T02:07:10.393435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def yolo2coco(yolobboxes):\n    pass","metadata":{"execution":{"iopub.status.busy":"2022-02-06T02:07:10.395609Z","iopub.execute_input":"2022-02-06T02:07:10.395857Z","iopub.status.idle":"2022-02-06T02:07:10.402200Z","shell.execute_reply.started":"2022-02-06T02:07:10.395827Z","shell.execute_reply":"2022-02-06T02:07:10.401510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.utils import load_img\nfrom tensorflow.keras.utils import img_to_array\n\ndef my_load_image(image_path):\n    return np.array(img_to_array(load_img(image_path))/255, np.float)","metadata":{"execution":{"iopub.status.busy":"2022-02-06T02:07:10.403560Z","iopub.execute_input":"2022-02-06T02:07:10.403880Z","iopub.status.idle":"2022-02-06T02:07:10.410290Z","shell.execute_reply.started":"2022-02-06T02:07:10.403848Z","shell.execute_reply":"2022-02-06T02:07:10.409561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Functions and classes","metadata":{}},{"cell_type":"markdown","source":"## YOLO Squared Transformer","metadata":{}},{"cell_type":"code","source":"class YOLOSquaredTransformer:\n    \n    def __init__(self, S, B, img_width, img_height):\n        self.S = S\n        self.B = B\n        self.img_width = img_width\n        self.img_height = img_height\n    \n        self.l_x = img_width  // S + (1 if img_width  % S != 0 else 0 )\n        self.l_y = img_height // S + (1 if img_height % S != 0 else 0 )\n        \n        self.img_big_side = max([img_width, img_height])\n        \n    def batch_from_cocodictlist(self, cocodictlistbatch):\n        \"\"\"\n        Receive a batch of cocodictlists where labels are 'x_left', 'y_top', 'width' and 'height' creating a numpy batch \n        output in format of YOLO Squared where shape is (batch_size, S, S, B, 4) being ['prob', 'x_center', 'y_center', 'side'] -> normalized\n        \"\"\"\n        batch_size = len(cocodictlistbatch)\n    \n        bboxes_in_square = np.zeros((batch_size, S, S),dtype=np.int32)\n        output = np.zeros((batch_size, S, S, B, 4),dtype=np.float)\n        \n        for batch_pos, cocodictlist in enumerate(cocodictlistbatch):\n            for cocodict in cocodictlist:\n                coco = cocoincode(cocodict)\n                yolo = coco2yolo(coco)\n                yolosquared = yolo2yolosquared(yolo)\n\n                S_x = yolosquared[0] // self.l_x - (1 if yolosquared[0] % self.l_x == 0 else 0 )\n                S_y = yolosquared[1] // self.l_y - (1 if yolosquared[1] % self.l_y == 0 else 0 )\n                \n                if S_x < self.S and S_y < self.S:\n                    n = bboxes_in_square[batch_pos ,S_y, S_x]\n                    if n < B:\n                        output[batch_pos, S_y, S_x, n, 0] = 1\n                        output[batch_pos, S_y, S_x, n, 1] = (yolosquared[0] - S_x*self.l_x)/self.l_x\n                        output[batch_pos, S_y, S_x, n, 2] = (yolosquared[1] - S_y*self.l_y)/self.l_y\n                        output[batch_pos, S_y, S_x, n, 3] = yolosquared[2]/self.img_big_side\n                        bboxes_in_square[batch_pos, S_y, S_x] = n + 1\n        \n        return output\n    \n    def cocolistbatch_from_yolosquaredbatch(self, y, cut_prob):\n        '''\n        Take a batch of tensors representing a yolo output (['prob', 'x_center', 'y_center', 'side'] -> normalized)  whit shape: (batch_size, S, S, B, 4) \n        and transforms it in a list of a list of coco values (['prob', 'x_lefttop', 'y_lefttop', 'width', 'height']) with shape: (batch_size, variable, 4)\n        where prob is bigger than cut_prob.\n        '''\n        pass\n    \n    def iou(self, y_true, y_pred):\n        \"\"\"\n        We only know that tensor have rank 5 and the shape is: (batch_size, S, S, B, 4)\n\n        bbox are: ['prob' 'x' 'y' 'side'] -> normalized\n        \"\"\"\n        side_scale = max([img_width, img_height])\n\n        l_true = y_true[:,:,:,3:4] * self.img_big_side\n        l_pred = y_pred[:,:,:,3:4] * self.img_big_side\n\n        x_true = y_true[:,:,:,1:2] * self.l_x\n        x_pred = y_pred[:,:,:,1:2] * self.l_x\n\n        yc_true = y_true[:,:,:,2:3] * self.l_y\n        yc_pred = y_pred[:,:,:,2:3] * self.l_y\n        \n        dx = tf.math.abs(x_true - x_pred)\n        dy = tf.math.abs(yc_true - yc_pred)\n        \n        l_x = tf.math.maximum(tf.math.maximum(l_true/2 + l_pred/2 + dx, l_true), l_pred)\n        l_y = tf.math.maximum(tf.math.maximum(l_true/2 + l_pred/2 + dy, l_true), l_pred)\n        \n        \n        I_area = tf.nn.relu(l_true + l_pred - l_x) * tf.nn.relu(l_true + l_pred - l_x)\n        \n        U_area = tf.pow(l_true, 2) + tf.pow(l_pred, 2) - I_area\n\n        IoU = I_area/U_area\n        \n        return IoU","metadata":{"execution":{"iopub.status.busy":"2022-02-06T02:07:10.411433Z","iopub.execute_input":"2022-02-06T02:07:10.411794Z","iopub.status.idle":"2022-02-06T02:07:10.432721Z","shell.execute_reply.started":"2022-02-06T02:07:10.411759Z","shell.execute_reply":"2022-02-06T02:07:10.431951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Batch generator","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.utils import Sequence\nfrom json import loads\n\nclass YoloSquaredSequence(Sequence):\n\n    def __init__(self, train_df, batch_size, transformer):\n        self.train_df = train_df\n        self.batch_size = batch_size\n        self.transformer = transformer\n\n    def __len__(self):\n        return len(self.train_df) // self.batch_size\n\n    def __getitem__(self, idx):\n        batch_df = self.train_df[idx*self.batch_size : (idx + 1 )*(self.batch_size)]\n        \n        X = self.get_x(batch_df)\n        \n        y = self.get_y(batch_df)\n\n        return X, y\n    \n    def get_x(self, batch_df):\n        return np.array([my_load_image(image_path) for image_path in batch_df['image_path']])\n    \n    def get_y(self, batch_df):\n        cocodictlistbatch = [loads(annotations.replace(\"'\",'\"')) for annotations in batch_df['annotations']]\n        \n        return self.transformer.batch_from_cocodictlist(cocodictlistbatch)","metadata":{"execution":{"iopub.status.busy":"2022-02-06T02:07:10.433984Z","iopub.execute_input":"2022-02-06T02:07:10.434459Z","iopub.status.idle":"2022-02-06T02:07:10.444212Z","shell.execute_reply.started":"2022-02-06T02:07:10.434426Z","shell.execute_reply":"2022-02-06T02:07:10.443460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loss Function","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.losses import Loss\n\nclass SquaredYoloLoss(Loss):\n    \"\"\"We only know that tensor have rank 5 and the shape is: (batch_size, S, S, B, 4)\"\"\"\n    def __init__(self, lambda_coord, lambda_noobj, transformer, *args, **kwargs):\n        self.lambda_coord = lambda_coord\n        self.lambda_noobj = lambda_noobj\n        self.transformer = transformer\n        \n        super(SquaredYoloLoss, self).__init__(*args, **kwargs)\n        \n    def call(self, y_true, y_pred):\n        # Necessario achar qual objeto é par de qual...\n        \n        \n        # Look at y_true if we have bbox\n        obj_exists = y_true[:,:,:,0:1]\n        obj_noexists = 1 - obj_exists\n        \n        iou = self.transformer.iou(y_true, y_pred)\n        \n        sum_exists_coord = self.lambda_coord * tf.reduce_sum(obj_exists * (tf.pow(y_pred[:,:,:,1:4] - y_true[:,:,:,1:4],2))) + \\\n                           tf.reduce_sum(tf.nn.relu(y_true[:,:,:,3:4] - y_pred[:,:,:,3:4]))\n            \n        sum_predict = tf.reduce_sum( obj_exists * tf.pow(y_pred[:,:,:,0:1] - iou, 2) )\n        \n        sum_noexists = self.lambda_noobj * tf.reduce_sum(obj_noexists * (tf.pow(y_pred[:,:,:,0:1],2)))\n        \n        return sum_exists_coord + sum_predict + sum_noexists","metadata":{"execution":{"iopub.status.busy":"2022-02-06T02:07:10.445367Z","iopub.execute_input":"2022-02-06T02:07:10.445978Z","iopub.status.idle":"2022-02-06T02:07:10.456946Z","shell.execute_reply.started":"2022-02-06T02:07:10.445941Z","shell.execute_reply":"2022-02-06T02:07:10.456267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Metric Function","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.metrics import Metric\n\nclass YOLOSquaredMetric(Metric):\n    def __init__(self, transformer, name='YOLO Recall Squared Metric', **kwargs):\n        super(YOLOSquaredMetric, self).__init__(name=name, **kwargs)\n        self.recall = self.add_weight(name='yoloRecall', initializer='zeros')\n        self.objects = self.add_weight(name='objects', initializer='zeros')\n        self.finds = self.add_weight(name='finds', initializer='zeros')\n        self.transformer = transformer\n        \n    def update_state(self, y_true, y_pred, sample_weight=None):\n        obj_exists = y_true[:,:,:,0:1]\n        iou = self.transformer.iou(y_true, y_pred)\n        \n        self.objects = tf.reduce_sum(obj_exists)\n        \n    def result(self):\n        return self.objects","metadata":{"execution":{"iopub.status.busy":"2022-02-06T02:07:10.458131Z","iopub.execute_input":"2022-02-06T02:07:10.458601Z","iopub.status.idle":"2022-02-06T02:07:10.467236Z","shell.execute_reply.started":"2022-02-06T02:07:10.458565Z","shell.execute_reply":"2022-02-06T02:07:10.466573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Specifications\n\n## Constants - Hyperparameters","metadata":{}},{"cell_type":"code","source":"# grid size SxS:\nS = 7\n\n# bboxes per cell in grid\nB = 2\n\n# batch size\nbatch_size = 40\n\n# image shape\nimg_shape = (720, 1280, 3)\nimg_height = img_shape[0]\nimg_width = img_shape[1]","metadata":{"execution":{"iopub.status.busy":"2022-02-06T02:07:10.468386Z","iopub.execute_input":"2022-02-06T02:07:10.468872Z","iopub.status.idle":"2022-02-06T02:07:10.475353Z","shell.execute_reply.started":"2022-02-06T02:07:10.468778Z","shell.execute_reply":"2022-02-06T02:07:10.474720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preprocessing data to train ","metadata":{}},{"cell_type":"code","source":"def load_train_df():\n    # Load train.csv:\n    complete_train_df = pd.read_csv('/kaggle/input/tensorflow-great-barrier-reef/train.csv')\n    \n    # Only use images where we have starfish\n    complete_train_df = complete_train_df[complete_train_df['annotations'] != '[]']\n    \n    # Create column image_path -> too expensive in memory\n    complete_train_df['image_path'] = complete_train_df.apply(lambda row : image_path(row['video_id'], row['video_frame']), axis=1)\n    \n    #Shuffle dataframe\n    complete_train_df = complete_train_df.sample(frac=1)\n    return complete_train_df\n\ntrain_df = load_train_df()","metadata":{"execution":{"iopub.status.busy":"2022-02-06T02:07:10.478677Z","iopub.execute_input":"2022-02-06T02:07:10.479144Z","iopub.status.idle":"2022-02-06T02:07:10.594796Z","shell.execute_reply.started":"2022-02-06T02:07:10.479097Z","shell.execute_reply":"2022-02-06T02:07:10.594191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model Definition","metadata":{}},{"cell_type":"code","source":"transformer = YOLOSquaredTransformer(S, B, img_width, img_height)","metadata":{"execution":{"iopub.status.busy":"2022-02-06T02:07:10.596837Z","iopub.execute_input":"2022-02-06T02:07:10.597211Z","iopub.status.idle":"2022-02-06T02:07:10.601422Z","shell.execute_reply.started":"2022-02-06T02:07:10.597177Z","shell.execute_reply":"2022-02-06T02:07:10.600687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_sequence = YoloSquaredSequence(train_df, batch_size, transformer)","metadata":{"execution":{"iopub.status.busy":"2022-02-06T02:07:10.602751Z","iopub.execute_input":"2022-02-06T02:07:10.603383Z","iopub.status.idle":"2022-02-06T02:07:10.608710Z","shell.execute_reply.started":"2022-02-06T02:07:10.603346Z","shell.execute_reply":"2022-02-06T02:07:10.607930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = keras.models.Sequential([\n    keras.layers.Conv2D(128, 9, strides=(2,4), activation=\"relu\", padding=\"same\", input_shape=img_shape),\n    keras.layers.MaxPooling2D(2),\n    keras.layers.Conv2D(128, 3, activation=\"relu\", padding=\"same\"),\n    keras.layers.MaxPooling2D(2),\n    keras.layers.Conv2D(128, 1, activation=\"relu\", padding=\"same\"),\n    keras.layers.MaxPooling2D(2),\n    keras.layers.Conv2D(128, 3, activation=\"relu\", padding=\"same\"),\n    keras.layers.MaxPooling2D(2),\n    keras.layers.Conv2D(256, 1, activation=\"relu\", padding=\"same\"),\n    keras.layers.Conv2D(256, 3, activation=\"relu\", padding=\"same\"),\n    keras.layers.MaxPooling2D(2),\n    keras.layers.Flatten(),\n    keras.layers.Dense(S*S*B*4, activation=\"relu\"),\n    keras.layers.Dropout(0.5),\n    keras.layers.Dense(S*S*B*4, activation=\"sigmoid\"),\n    keras.layers.Reshape((S,S,B,4))\n])","metadata":{"execution":{"iopub.status.busy":"2022-02-06T02:07:10.610572Z","iopub.execute_input":"2022-02-06T02:07:10.612273Z","iopub.status.idle":"2022-02-06T02:07:10.701969Z","shell.execute_reply.started":"2022-02-06T02:07:10.612238Z","shell.execute_reply":"2022-02-06T02:07:10.701328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2022-02-06T02:07:10.703227Z","iopub.execute_input":"2022-02-06T02:07:10.703453Z","iopub.status.idle":"2022-02-06T02:07:10.716668Z","shell.execute_reply.started":"2022-02-06T02:07:10.703421Z","shell.execute_reply":"2022-02-06T02:07:10.715999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss_function = SquaredYoloLoss(5, 0.5, transformer)","metadata":{"execution":{"iopub.status.busy":"2022-02-06T02:07:10.717492Z","iopub.execute_input":"2022-02-06T02:07:10.717658Z","iopub.status.idle":"2022-02-06T02:07:10.722988Z","shell.execute_reply.started":"2022-02-06T02:07:10.717637Z","shell.execute_reply":"2022-02-06T02:07:10.722094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"yolo_recall = YOLOSquaredMetric(transformer)","metadata":{"execution":{"iopub.status.busy":"2022-02-06T02:07:10.724560Z","iopub.execute_input":"2022-02-06T02:07:10.725111Z","iopub.status.idle":"2022-02-06T02:07:10.734383Z","shell.execute_reply.started":"2022-02-06T02:07:10.725074Z","shell.execute_reply":"2022-02-06T02:07:10.733697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer=\"rmsprop\",\n    loss=loss_function,\n    metrics=[\"accuracy\",yolo_recall])","metadata":{"execution":{"iopub.status.busy":"2022-02-06T02:07:10.735675Z","iopub.execute_input":"2022-02-06T02:07:10.736333Z","iopub.status.idle":"2022-02-06T02:07:10.746394Z","shell.execute_reply.started":"2022-02-06T02:07:10.736296Z","shell.execute_reply":"2022-02-06T02:07:10.745672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fit(x=batch_sequence,\n    batch_size=batch_size,\n    epochs=30)","metadata":{"execution":{"iopub.status.busy":"2022-02-06T02:07:10.748836Z","iopub.execute_input":"2022-02-06T02:07:10.750272Z","iopub.status.idle":"2022-02-06T02:10:16.278175Z","shell.execute_reply.started":"2022-02-06T02:07:10.750235Z","shell.execute_reply":"2022-02-06T02:10:16.277054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save(\"fastLazyYOLO\")","metadata":{"execution":{"iopub.status.busy":"2022-02-06T02:10:16.279329Z","iopub.status.idle":"2022-02-06T02:10:16.280246Z","shell.execute_reply.started":"2022-02-06T02:10:16.279971Z","shell.execute_reply":"2022-02-06T02:10:16.279996Z"},"trusted":true},"execution_count":null,"outputs":[]}]}