{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# this notebook implements the yolov5 in tensorflow gpu with replicating the process from official repository https://github.com/ultralytics/yolov5 \n\nconverted the pretrained weights to tensorflow format","metadata":{"execution":{"iopub.status.busy":"2022-02-15T03:51:25.205463Z","iopub.execute_input":"2022-02-15T03:51:25.205784Z","iopub.status.idle":"2022-02-15T03:51:25.210587Z","shell.execute_reply.started":"2022-02-15T03:51:25.205753Z","shell.execute_reply":"2022-02-15T03:51:25.209672Z"}}},{"cell_type":"code","source":"import tensorflow as tf\nimport numpy as np\nimport pandas as pd\nimport os,shutil,math , yaml\nimport matplotlib.pyplot as plt\nimport ast\nfrom tqdm import tqdm\nimport argparse\nimport sys, math\nfrom copy import deepcopy\nfrom pathlib import Path\nfrom tensorflow import keras","metadata":{"execution":{"iopub.status.busy":"2022-02-15T07:39:27.776705Z","iopub.execute_input":"2022-02-15T07:39:27.777400Z","iopub.status.idle":"2022-02-15T07:39:27.782767Z","shell.execute_reply.started":"2022-02-15T07:39:27.777358Z","shell.execute_reply":"2022-02-15T07:39:27.781830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from module import Conv2d, Conv, Bottleneck, SPP, SPPF, Focus, BottleneckCSP, C3,Upsample,Concat,Detect, BN\n# from yololoss import YoloLoss\n# from anchorlabel import AnchorLabeler\n# from lrscheduler import LrScheduler\n# from preprocess_data import DataReader","metadata":{"execution":{"iopub.status.busy":"2022-02-15T07:39:27.784852Z","iopub.execute_input":"2022-02-15T07:39:27.785309Z","iopub.status.idle":"2022-02-15T07:39:27.790980Z","shell.execute_reply.started":"2022-02-15T07:39:27.785272Z","shell.execute_reply":"2022-02-15T07:39:27.790153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def auto_select_accelerator():\n    TPU_DETECTED = False\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        print(\"Running on TPU:\", tpu.master())\n        TPU_DETECTED = True\n    except ValueError:\n        strategy = tf.distribute.get_strategy()\n    print(f\"Running on {strategy.num_replicas_in_sync} replicas\")\n\n    return strategy , strategy.num_replicas_in_sync","metadata":{"execution":{"iopub.status.busy":"2022-02-15T07:39:27.795907Z","iopub.execute_input":"2022-02-15T07:39:27.797478Z","iopub.status.idle":"2022-02-15T07:39:27.802900Z","shell.execute_reply.started":"2022-02-15T07:39:27.797451Z","shell.execute_reply":"2022-02-15T07:39:27.802193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def coco2yolo(image_height, image_width, bboxes):\n    \"\"\"\n    coco => [xmin, ymin, w, h]\n    yolo => [xmid, ymid, w, h] (normalized)\n    \"\"\"\n    \n    bboxes = bboxes.copy().astype(float) # otherwise all value will be 0 as voc_pascal dtype is np.int\n    \n    # normolizinig\n    bboxes[..., [0, 2]]= bboxes[..., [0, 2]]/ image_width\n    bboxes[..., [1, 3]]= bboxes[..., [1, 3]]/ image_height\n    \n    # converstion (xmin, ymin) => (xmid, ymid)\n    bboxes[..., [0, 1]] = bboxes[..., [0, 1]] + bboxes[..., [2, 3]]/2\n    \n    return bboxes.tolist()","metadata":{"execution":{"iopub.status.busy":"2022-02-15T07:39:27.804671Z","iopub.execute_input":"2022-02-15T07:39:27.805036Z","iopub.status.idle":"2022-02-15T07:39:27.812564Z","shell.execute_reply.started":"2022-02-15T07:39:27.804984Z","shell.execute_reply":"2022-02-15T07:39:27.811875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_bbox(annots):\n    bboxes = [list(annot.values()) for annot in annots]\n#     print(np.array(bboxes).shape)\n    bboxes = coco2yolo(image_height = 720, image_width = 1280, bboxes = np.array(bboxes))\n    bboxes = [bbox + [0] for bbox in bboxes]\n    return bboxes\n\n\nFOLD = 1\ndf = pd.read_csv('../input/reef-cv-strategy-subsequences-dataframes/cross-validation/train-10folds.csv')\ndf['annotations'] = df['annotations'].apply(lambda x: ast.literal_eval(x))\ndf = df[df['n_annotations'] > 0]\ndf['bboxes'] = df['annotations'].apply(get_bbox)\ntrain_df = df.query('fold != @FOLD')\nval_df = df.query('fold == @FOLD')\n\ntrain_annotations_dict = {}\ntrain_annotations_dict['image_dir'] = train_df['image_path'].values.tolist()\ntrain_annotations_dict['labels'] = train_df['bboxes'].values.tolist()","metadata":{"execution":{"iopub.status.busy":"2022-02-15T07:39:27.814093Z","iopub.execute_input":"2022-02-15T07:39:27.814674Z","iopub.status.idle":"2022-02-15T07:39:28.441644Z","shell.execute_reply.started":"2022-02-15T07:39:27.814638Z","shell.execute_reply":"2022-02-15T07:39:28.440939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Params:\n    warmup_epochs = 2\n    n_epochs = 3\n    batch_size = 4\n    warmup_steps = 500\n    img_size = 640\n    label_smoothing = 0.0\n    num_classes = 1\n    buffer_size = 256\n    momentum = 0.93\n    optimizer = 'adam'\n    init_learning_rate = 1e-4\n    warmup_learning_rate = 1e-5\n    len_train_dataset = 5000\n    box = 0.05\n    obj = 1.0\n    cls = 1.0\n    fliplr  = 0.5\n    flipud  = 0.5\n    hsv_v = 0.4\n    hsv_h = 0.017\n    hsv_s = 0.7\n    degrees = 0.0\n    shear = 0.0\n    scale = 0.9\n    perspective = 0.0\n    translate = 0.1\n    mosaic = 1.0\n    mixup = 0.1\n    weight_dir = '/kaggle/input/yolov5-cv-045/yolov5s6.h5'\n    yaml_dir = '/kaggle/input/yolov5-lib-ds/models/hub/yolov5s6.yaml'","metadata":{"execution":{"iopub.status.busy":"2022-02-15T07:39:28.442898Z","iopub.execute_input":"2022-02-15T07:39:28.443139Z","iopub.status.idle":"2022-02-15T07:39:28.449553Z","shell.execute_reply.started":"2022-02-15T07:39:28.443108Z","shell.execute_reply":"2022-02-15T07:39:28.448895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# preprocessing is being kept same as pytorch version to get similar result after training","metadata":{}},{"cell_type":"code","source":"import cv2\nimport math\nimport random\nimport numpy as np\nimport tensorflow as tf\n# random.seed(1919)\nclass DataReader(object):\n    '''\n    read the image and label from the text information (generated by dataset/prepare_data.py)\n    resize the image, and adjust the label rect if necessary\n    augment the dataset (augment function is defined in dataset/augment_data.py)\n    '''\n    def __init__(self, hyp, annotations_dict, img_size=640, transforms=None, mosaic=False, augment=False, filter_idx=None, test=False):\n#         self.annotations_dir = annotations_dir\n#         self.annotations = self.load_annotations(annotations_dir)\n        self.hyp = hyp\n        self.img_size = img_size  # image_target_size\n        self.transforms = transforms\n        self.mosaic = mosaic\n        self.augment = augment\n        self.test = test\n        self.image_paths = annotations_dict['image_dir']\n        self.labels = annotations_dict['labels']\n        self.n = len(self.image_paths)\n        self.idx = range(len(self.image_paths))\n        if filter_idx is not None:  # filter some samples\n            self.idx = [i for i in self.idx if i in filter_idx]\n            print('filter {} from {}'.format(len(self.idx), len(self.annotations)))\n\n#         for i in self.idx:\n#             image_dir, label = self.parse_annotations(self.annotations[i])\n#             self.images_dir.append(image_dir)\n#             self.labels_ori.append(label)\n\n    def iter(self):\n        for i in self.idx:\n            yield self[i]\n            \n    def __len__(self):\n        return len(self.labels)\n    \n    def __getitem__(self, index):\n#             index = self.indices[index]  # linear, shuffled, or image_weights\n\n            hyp = self.hyp\n            mosaic = self.mosaic and random.random() < hyp.mosaic\n            if mosaic:\n                # Load mosaic\n                \n                img, labels = load_mosaic(self, index)\n                shapes = None\n\n                # MixUp augmentation\n                if random.random() < hyp.mixup:\n                    img, labels = mixup(img, labels, *load_mosaic(self, random.randint(0, self.n - 1)))\n\n            else:\n                # Load image\n                img, (h0, w0), (h, w) = load_image(self, index)\n\n                # Letterbox\n                shape = self.batch_shapes[self.batch[index]] if self.rect else self.img_size  # final letterboxed shape\n                img, ratio, pad = letterbox(img, shape, auto=False, scaleup=self.augment)\n                shapes = (h0, w0), ((h / h0, w / w0), pad)  # for COCO mAP rescaling\n\n                labels = self.labels[index].copy()\n                if labels.size:  # normalized xywh to pixel xyxy format\n                    labels[:, 0:4] = xywhn2xyxy(labels[:, 0:4], ratio[0] * w, ratio[1] * h, padw=pad[0], padh=pad[1])\n\n                if self.augment:\n                    img, labels = random_perspective(img, labels,\n                                                     degrees=hyp.degrees,\n                                                     translate=hyp.translate,\n                                                     scale=hyp.scale,\n                                                     shear=hyp.shear,\n                                                     perspective=hyp.perspective)\n\n            nl = len(labels)  # number of labels\n            if nl:\n                labels[:, 0:4] = xyxy2xywhn(labels[:, 0:4], w=img.shape[1], h=img.shape[0], clip=True, eps=1E-3)\n\n            if self.augment:\n                # Albumentations\n#                 img, labels = self.albumentations(img, labels)\n                nl = len(labels)  # update after albumentations\n\n                # HSV color-space\n                augment_hsv(img, hgain=hyp.hsv_h, sgain=hyp.hsv_s, vgain=hyp.hsv_v)\n\n                # Flip up-down\n                if random.random() < hyp.flipud:\n                    img = np.flipud(img)\n                    if nl:\n                        labels[:, 1] = 1 - labels[:, 1]\n\n                # Flip left-right\n                if random.random() < hyp.fliplr:\n                    img = np.fliplr(img)\n                    if nl:\n                        labels[:, 0] = 1 - labels[:, 0]\n            return img, labels\n                        \n\n                # Cutouts\n                # labels = cutout(img, labels, p=0.5)\n                # nl = len(labels)  # update after cutout\n\n#             labels_out = torch.zeros((nl, 6))\n#             if nl:\n#                 labels_out[:, 1:] = torch.from_numpy(labels)\n\n                \ndef load_image(self, idx):\n    image_path  = self.image_paths[idx]\n    img = cv2.imread(image_path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB).astype(np.float32)\n#     img = img / 255.0\n    return img\n\ndef load_mosaic(self, index):\n    # YOLOv5 4-mosaic loader. Loads 1 image + 3 random images into a 4-image mosaic\n    labels4 = []\n    s = self.img_size\n    mosaic_border = [-self.img_size // 2, -self.img_size // 2]\n    yc, xc = (int(random.uniform(-x, 2 * s + x)) for x in mosaic_border)  # mosaic center x, y\n    indices = [index] + random.choices(self.idx, k=3)  # 3 additional image indices\n    random.shuffle(indices)\n#     print(indices)\n    for i, index in enumerate(indices):\n        # Load image\n        img = load_image(self, index)\n        h, w,_ = img.shape\n#         plt.imshow(img)\n#         plt.show()\n        # place img in img4\n        if i == 0:  # top left\n            img4 = np.full((s * 2, s * 2, img.shape[2]), 114, dtype=np.uint8)  # base image with 4 tiles\n            x1a, y1a, x2a, y2a = max(xc - w, 0), max(yc - h, 0), xc, yc  # xmin, ymin, xmax, ymax (large image)\n            x1b, y1b, x2b, y2b = w - (x2a - x1a), h - (y2a - y1a), w, h  # xmin, ymin, xmax, ymax (small image)\n        elif i == 1:  # top right\n            x1a, y1a, x2a, y2a = xc, max(yc - h, 0), min(xc + w, s * 2), yc\n            x1b, y1b, x2b, y2b = 0, h - (y2a - y1a), min(w, x2a - x1a), h\n        elif i == 2:  # bottom left\n            x1a, y1a, x2a, y2a = max(xc - w, 0), yc, xc, min(s * 2, yc + h)\n            x1b, y1b, x2b, y2b = w - (x2a - x1a), 0, w, min(y2a - y1a, h)\n        elif i == 3:  # bottom right\n            x1a, y1a, x2a, y2a = xc, yc, min(xc + w, s * 2), min(s * 2, yc + h)\n            x1b, y1b, x2b, y2b = 0, 0, min(w, x2a - x1a), min(y2a - y1a, h)\n\n        img4[y1a:y2a, x1a:x2a] = img[y1b:y2b, x1b:x2b]  # img4[ymin:ymax, xmin:xmax]\n        padw = x1a - x1b\n        padh = y1a - y1b\n\n        # Labels\n        labels = np.array(self.labels[index].copy())\n        if labels.size:\n            labels[:, 0:4] = xywhn2xyxy(labels[:, 0:4], w, h, padw, padh)  # normalized xywh to pixel xyxy format\n#             segments = [xyn2xy(x, w, h, padw, padh) for x in segments]\n        labels4.append(labels)\n#         segments4.extend(segments)\n#     plt.imshow(img4)\n#     plt.show()\n    # Concat/clip labels\n    labels4 = np.concatenate(labels4, 0)\n    for x in (labels4[:, 0:4]):\n        np.clip(x, 0, 2 * s, out=x)  # clip when using random_perspective()\n    # img4, labels4 = replicate(img4, labels4)  # replicate\n    \n    # Augment\n#     img4, labels4, segments4 = copy_paste(img4, labels4, segments4, p=self.hyp['copy_paste'])\n    img4, labels4 = random_perspective(img4, labels4,\n                                       degrees = self.hyp.degrees,\n                                       translate=self.hyp.translate,\n                                       scale=self.hyp.scale,\n                                       shear=self.hyp.shear,\n                                       perspective=self.hyp.perspective,\n                                       border= mosaic_border)  # border to remove\n\n    return img4, labels4\n\n\n\ndef random_perspective(img, label=(), degrees=10, translate=.1, scale=.1, shear=10, perspective=0.0, border=(0, 0)):\n    # labels style: pixel, [xyxy, cls]\n    img = img.astype(np.uint8)\n\n    height = img.shape[0] + border[0] * 2  # shape(h,w,c)\n    width = img.shape[1] + border[1] * 2\n\n    # Center\n    C = np.eye(3)\n    C[0, 2] = -img.shape[1] / 2  # x translation (pixels)\n    C[1, 2] = -img.shape[0] / 2  # y translation (pixels)\n\n    # Perspective\n    P = np.eye(3)\n    P[2, 0] = random.uniform(-perspective, perspective)  # x perspective (about y)\n    P[2, 1] = random.uniform(-perspective, perspective)  # y perspective (about x)\n\n    # Rotation and Scale\n    R = np.eye(3)\n    a = random.uniform(-degrees, degrees)\n    # a += random.choice([-180, -90, 0, 90])  # add 90deg rotations to small rotations\n    s = random.uniform(1 - scale, 1 + scale)\n    # s = 2 ** random.uniform(-scale, scale)\n    R[:2] = cv2.getRotationMatrix2D(angle=a, center=(0, 0), scale=s)\n\n    # Shear\n    S = np.eye(3)\n    S[0, 1] = math.tan(random.uniform(-shear, shear) * math.pi / 180)  # x shear (deg)\n    S[1, 0] = math.tan(random.uniform(-shear, shear) * math.pi / 180)  # y shear (deg)\n\n    # Translation\n    T = np.eye(3)\n    T[0, 2] = random.uniform(0.5 - translate, 0.5 + translate) * width  # x translation (pixels)\n    T[1, 2] = random.uniform(0.5 - translate, 0.5 + translate) * height  # y translation (pixels)\n\n    # Combined rotation matrix\n    M = T @ S @ R @ P @ C  # order of operations (right to left) is IMPORTANT\n    if (border[0] != 0) or (border[1] != 0) or (M != np.eye(3)).any():  # image changed\n        if perspective:\n            img = cv2.warpPerspective(img, M, dsize=(width, height), borderValue=(114, 114, 114))\n        else:  # affine\n            img = cv2.warpAffine(img, M[:2], dsize=(width, height), borderValue=(114, 114, 114))\n\n    # Transform label coordinates\n    n = len(label)\n    if n:\n        if np.max(label[:, 0:4]) <= 1.0:  # transfer to pixel level\n            label[:, [0, 2]] = label[:, [0, 2]] * img.shape[1]\n            label[:, [1, 3]] = label[:, [1, 3]] * img.shape[0]\n        # assert np.max(labels[:, 0:4]) > 1, \"don't use norm box coordinates here\"\n        # warp points\n        xy = np.ones((n * 4, 3))\n        xy[:, :2] = label[:, [0, 1, 2, 3, 0, 3, 2, 1]].reshape(n * 4, 2)  # x1y1, x2y2, x1y2, x2y1\n\n        xy = (xy @ M.T)[:, :2].reshape(n, 8)\n\n        # create new boxes\n        x = xy[:, [0, 2, 4, 6]]\n        y = xy[:, [1, 3, 5, 7]]\n        xy = np.concatenate((x.min(1), y.min(1), x.max(1), y.max(1))).reshape(4, n).T\n\n        # reject warped points outside of image\n        xy[:, [0, 2]] = xy[:, [0, 2]].clip(0, width)\n        xy[:, [1, 3]] = xy[:, [1, 3]].clip(0, height)\n        w = xy[:, 2] - xy[:, 0]\n        h = xy[:, 3] - xy[:, 1]\n        area = w * h\n        area0 = (label[:, 2] - label[:, 0]) * (label[:, 3] - label[:, 1])\n        ar = np.maximum(w / (h + 1e-16), h / (w + 1e-16))  # aspect ratio\n        i = (w > 2) & (h > 2) & (area / (area0 * scale + 1e-16) > 0.2) & (ar < 20)\n\n        label = label[i]\n        label[:, 0:4] = xy[i]\n        \n        if label.size == 0:  # in case, all labels is out\n            label = np.array([[0, 0, 0, 0, 0]], np.float32)\n    return img, label\n\n\ndef augment_hsv(img, hgain=0.5, sgain=0.5, vgain=0.5):\n    rand = np.random.uniform(-1, 1, 3) * [hgain, sgain, vgain] + 1\n    hue, sat, val = cv2.split(cv2.cvtColor(img, cv2.COLOR_BGR2HSV))\n    dtype = img.dtype\n\n    x = np.arange(0, 256, dtype=np.int16)\n    lut_hue = ((x * rand[0]) % 180).astype(dtype)\n    lut_sat = np.clip(x * rand[1], 0, 255).astype(dtype)\n    lut_val = np.clip(x * rand[2], 0, 255).astype(dtype)\n\n    img_hsv = cv2.merge((cv2.LUT(hue, lut_hue), cv2.LUT(sat, lut_sat), cv2.LUT(val, lut_val))).astype(dtype)\n    return cv2.cvtColor(img_hsv, cv2.COLOR_HSV2BGR)\n\n\ndef mixup(im, labels, im2, labels2):\n    # Applies MixUp augmentation https://arxiv.org/pdf/1710.09412.pdf\n    r = np.random.beta(32.0, 32.0)  # mixup ratio, alpha=beta=32.0\n    im = (im * r + im2 * (1 - r)).astype(np.uint8)\n    labels = np.concatenate((labels, labels2), 0)\n    return im, labels\n\n\ndef xywhn2xyxy(x, w=640, h=640, padw=0, padh=0):\n    # Convert nx4 boxes from [x, y, w, h] normalized to [x1, y1, x2, y2] where xy1=top-left, xy2=bottom-right\n    y = np.copy(x)\n    y[:, 0] = w * (x[:, 0] - x[:, 2] / 2) + padw  # top left x\n    y[:, 1] = h * (x[:, 1] - x[:, 3] / 2) + padh  # top left y\n    y[:, 2] = w * (x[:, 0] + x[:, 2] / 2) + padw  # bottom right x\n    y[:, 3] = h * (x[:, 1] + x[:, 3] / 2) + padh  # bottom right y\n    return y\n\n\ndef xyxy2xywhn(x, w=640, h=640, clip=False, eps=0.0):\n    # Convert nx4 boxes from [x1, y1, x2, y2] to [x, y, w, h] normalized where xy1=top-left, xy2=bottom-right\n#     if clip:\n#         clip_coords(x, (h - eps, w - eps))  # warning: inplace clip\n    y = np.copy(x)\n    y[:, 0] = ((x[:, 0] + x[:, 2]) / 2) / w  # x center\n    y[:, 1] = ((x[:, 1] + x[:, 3]) / 2) / h  # y center\n    y[:, 2] = (x[:, 2] - x[:, 0]) / w  # width\n    y[:, 3] = (x[:, 3] - x[:, 1]) / h  # height\n    return y\n\ndef letterbox(im, new_shape=(640, 640), color=(114, 114, 114), auto=False, scaleFill=False, scaleup=True, stride=32):\n    # Resize and pad image while meeting stride-multiple constraints\n    shape = im.shape[:2]  # current shape [height, width]\n    if isinstance(new_shape, int):\n        new_shape = (new_shape, new_shape)\n\n    # Scale ratio (new / old)\n    r = min(new_shape[0] / shape[0], new_shape[1] / shape[1])\n    if not scaleup:  # only scale down, do not scale up (for better val mAP)\n        r = min(r, 1.0)\n\n    # Compute padding\n    ratio = r, r  # width, height ratios\n    new_unpad =  int(round(shape[0] * r)), int(round(shape[1] * r))\n    dw, dh = new_shape[1] - new_unpad[0], new_shape[0] - new_unpad[1]  # wh padding\n    if auto:  # minimum rectangle\n        dw, dh = tf.math.floormod(dw, stride), tf.math.floormod(dh, stride)  # wh padding\n    elif scaleFill:  # stretch\n        dw, dh = 0.0, 0.0\n        new_unpad = (new_shape[1], new_shape[0])\n        ratio = new_shape[1] / shape[1], new_shape[0] / shape[0]  # width, height ratios\n\n    dw /= 2  # divide padding into 2 sides\n    dh /= 2\n\n    if shape[::-1] != new_unpad:  # resize\n#         print(new_unpad)\n        im = tf.image.resize(im, size = new_unpad, method='bilinear')\n\n    top, bottom = int(round(dh - 0.1)), int(round(dh + 0.1))\n    left, right = int(round(dw - 0.1)), int(round(dw + 0.1))\n    im = tf.image.resize_with_crop_or_pad(im, new_shape[0], new_shape[1])\n#     im = cv2.copyMakeBorder(im.numpy(), top, bottom, left, right, cv2.BORDER_CONSTANT, value=color)  # add border\n    return im, ratio, (dw, dh)","metadata":{"execution":{"iopub.status.busy":"2022-02-15T07:39:28.452651Z","iopub.execute_input":"2022-02-15T07:39:28.453191Z","iopub.status.idle":"2022-02-15T07:39:28.524552Z","shell.execute_reply.started":"2022-02-15T07:39:28.453093Z","shell.execute_reply":"2022-02-15T07:39:28.523664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#this is the main part of yolo that differs from other detectors where we need to assign true label to each grid of output to the model\n#since tensorflow does not support indexing so we need to assign label to each grid before feeding data to model","metadata":{}},{"cell_type":"code","source":"class AnchorLabeler(object):\n    # transfer the annotated label to model target by anchor encoding, to calculate anchor based loss next step\n    def __init__(self, anchors, grids, img_size=640, assign_method='wh', extend_offset=True, rect_style='rect4', anchor_match_threshold=4.0):  # 4.0 or 0.3\n        self.anchors = anchors  # from yaml.anchors to Detect.anchors, w/h based on grid coordinators\n        self.grids = grids\n        self.img_size = img_size\n        self.assign_method = assign_method\n        self.extend_offset = extend_offset\n        self.rect_style = rect_style\n        self.anchor_match_threshold = anchor_match_threshold\n\n    def encode(self, labels):\n        self.num_scales = self.anchors.shape[0]\n        self.n_anchor_per_scale = self.anchors.shape[1]\n        y_anchor_encode = []\n        gain = tf.ones(5, tf.float32)\n\n        for i in range(self.num_scales):\n            anchor = self.anchors[i]\n            grid_size = tf.cast(self.grids[i], tf.int32)\n            y_true = tf.zeros([grid_size, grid_size, self.n_anchor_per_scale, 6], tf.float32)\n            gain = tf.tensor_scatter_nd_update(gain, [[0], [1], [2], [3]], [grid_size] * 4)\n            scaled_labels = labels * gain  # label coordinator now is the same with anchors\n\n            if labels is not None:\n                gt_wh = scaled_labels[..., 2:4]  # n_gt * 2\n                if self.assign_method == 'wh':\n                    assert self.anchor_match_threshold > 1, 'threshold is totally different for wh and iou assign'\n                    matched_matrix = self.assign_criterion_wh(gt_wh, anchor, self.anchor_match_threshold)\n                elif self.assign_method == 'iou':\n                    assert self.anchor_match_threshold < 1, 'threshold is totally different for wh and iou assign'\n                    matched_matrix = self.assign_criterion_iou(gt_wh, anchor, self.anchor_match_threshold)\n                else:\n                    raise ValueError\n\n                n_gt = tf.shape(gt_wh)[0]\n                assigned_anchor = tf.tile(tf.reshape(tf.range(self.n_anchor_per_scale), (self.n_anchor_per_scale, 1)),\n                                          (1, n_gt))\n\n                assigned_anchor = tf.expand_dims(assigned_anchor[matched_matrix], 1)  # filter\n                assigned_anchor = tf.cast(assigned_anchor, tf.int32)\n\n                assigned_label = tf.tile(tf.expand_dims(scaled_labels, 0), [self.n_anchor_per_scale, 1, 1])\n                assigned_label = assigned_label[matched_matrix]\n\n                if self.extend_offset:\n                    assigned_label, assigned_anchor, grid_offset = self.enrich_pos_by_position(\n                        assigned_label, assigned_anchor, gain, matched_matrix)\n                else:\n                    grid_offset = tf.zeros_like(assigned_label[:, 0:2])\n\n                assigned_grid = tf.cast(assigned_label[..., 0:2] - grid_offset, tf.int32)  # n_matched * 2\n                assigned_grid = tf.clip_by_value(assigned_grid, clip_value_min=0, clip_value_max=grid_size-1)\n                \n                # tensor: grid * grid * 3 * 6, indices（sparse index）: ~n_gt * gr * gr * 3, updates: ~n_gt * 6\n                assigned_indices = tf.concat([assigned_grid[:, 1:2], assigned_grid[:, 0:1], assigned_anchor],\n                                             axis=1)\n\n                xy, wh, clss = tf.split(assigned_label, (2, 2, 1), axis=-1)\n                xy = xy / gain[0] * self.img_size\n                wh = wh / gain[1] * self.img_size\n                obj = tf.ones_like(clss)\n                assigned_updates = tf.concat([xy, wh, obj, clss], axis=-1)\n\n                y_true = tf.tensor_scatter_nd_update(y_true, assigned_indices, assigned_updates)\n            y_anchor_encode.append(y_true)\n        return tuple(y_anchor_encode)  # add a tuple is important here, otherwise raise an error\n\n    def assign_criterion_wh(self, gt_wh, anchors, anchor_threshold):\n        # return: please note that the v5 default anchor_threshold is 4.0, related to the positive sample augment\n        gt_wh = tf.expand_dims(gt_wh, 0)  # => 1 * n_gt * 2\n        anchors = tf.expand_dims(anchors, 1)  # => n_anchor * 1 * 2\n        ratio = gt_wh / anchors  # => n_anchor * n_gt * 2\n        matched_matrix = tf.reduce_max(tf.math.maximum(ratio, 1 / ratio),\n                                       axis=2) < anchor_threshold  # => n_anchor * n_gt\n        return matched_matrix\n\n    def assign_criterion_iou(self, gt_wh, anchors, anchor_threshold):\n        # by IOU, anchor_threshold < 1\n        box_wh = tf.expand_dims(gt_wh, 0)  # => 1 * n_gt * 2\n        box_area = box_wh[..., 0] * box_wh[..., 1]  # => 1 * n_gt\n\n        anchors = tf.cast(anchors, tf.float32)  # => n_anchor * 2\n        anchors = tf.expand_dims(anchors, 1)  # => n_anchor * 1 * 2\n        anchors_area = anchors[..., 0] * anchors[..., 1]  # => n_anchor * 1\n\n        inter = tf.math.minimum(anchors[..., 0], box_wh[..., 0]) * tf.math.minimum(anchors[..., 1],\n                                                                                   box_wh[..., 1])  # n_gt * n_anchor\n        iou = inter / (anchors_area + box_area - inter + 1e-9)\n\n        iou = iou > anchor_threshold\n        return iou\n\n    def enrich_pos_by_position(self, assigned_label, assigned_anchor, gain, matched_matrix, rect_style='rect4'):\n        # using offset to extend more postive result, if x\n        assigned_xy = assigned_label[..., 0:2]  # n_matched * 2\n        offset = tf.constant([[0, 0], [1, 0], [0, 1], [-1, 0], [0, -1]], tf.float32)\n        grid_offset = tf.zeros_like(assigned_xy)\n\n        if rect_style == 'rect2':\n            g = 0.2  # offset\n        elif rect_style == 'rect4':\n            g = 0.5  # offset\n            matched = (assigned_xy % 1. < g) & (assigned_xy > 1.)\n            matched_left = matched[:, 0]\n            matched_up = matched[:, 1]\n            matched = (assigned_xy % 1. > (1 - g)) & (assigned_xy < tf.expand_dims(gain[0:2], 0) - 1.)\n            matched_right = matched[:, 0]\n            matched_down = matched[:, 1]\n\n            assigned_anchor = tf.concat([assigned_anchor, assigned_anchor[matched_left], assigned_anchor[matched_up],\n                                         assigned_anchor[matched_right], assigned_anchor[matched_down]], axis=0)\n            assigned_label = tf.concat([assigned_label, assigned_label[matched_left], assigned_label[matched_up],\n                                        assigned_label[matched_right], assigned_label[matched_down]], axis=0)\n\n            grid_offset = g * tf.concat(\n                [grid_offset, grid_offset[matched_left] + offset[1], grid_offset[matched_up] + offset[2],\n                 grid_offset[matched_right] + offset[3], grid_offset[matched_down] + offset[4]], axis=0)\n\n        return assigned_label, assigned_anchor, grid_offset","metadata":{"execution":{"iopub.status.busy":"2022-02-15T07:39:28.526329Z","iopub.execute_input":"2022-02-15T07:39:28.526686Z","iopub.status.idle":"2022-02-15T07:39:28.559441Z","shell.execute_reply.started":"2022-02-15T07:39:28.526646Z","shell.execute_reply":"2022-02-15T07:39:28.558653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# data pipeline from data_reader (image,label) to tf.data","metadata":{}},{"cell_type":"code","source":"class DataLoader(object):\n\n    def __init__(self, data_reader, anchors, stride, img_size=640, anchor_assign_method='wh',\n                 anchor_positive_augment=True):\n        self.data_reader = data_reader\n        self.anchor_label = AnchorLabeler(anchors,\n                                          grids=img_size / stride,\n                                          img_size=img_size,\n                                          assign_method=anchor_assign_method,\n                                          extend_offset=anchor_positive_augment)\n        self.img_size = img_size\n\n    def __call__(self, batch_size=8, anchor_label=True):\n        dataset = tf.data.Dataset.from_generator(self.data_reader.iter,\n                                                 output_types=(tf.float32, tf.float32),\n                                                 output_shapes=([self.img_size, self.img_size, 3], [None, 5]))\n\n        if anchor_label:  # when train\n            dataset = dataset.map(self.transform, num_parallel_calls=tf.data.experimental.AUTOTUNE)\n        dataset = dataset.batch(batch_size).prefetch(tf.data.experimental.AUTOTUNE)\n        return dataset\n\n    def transform(self, image, label):\n        label_encoder = self.anchor_label.encode(label)\n        return image, label_encoder","metadata":{"execution":{"iopub.status.busy":"2022-02-15T07:39:28.561225Z","iopub.execute_input":"2022-02-15T07:39:28.561565Z","iopub.status.idle":"2022-02-15T07:39:28.572045Z","shell.execute_reply.started":"2022-02-15T07:39:28.561526Z","shell.execute_reply":"2022-02-15T07:39:28.571340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# module for yolo model","metadata":{}},{"cell_type":"code","source":"tf.keras.backend.clear_session()\n\nclass BN(keras.layers.Layer):\n    # TensorFlow BatchNormalization wrapper\n    def __init__(self, w=None):\n        super().__init__()\n        self.bn = keras.layers.BatchNormalization()\n\n    def call(self, inputs):\n        return self.bn(inputs)\n\n\nclass Pad(keras.layers.Layer):\n    def __init__(self, pad):\n        super().__init__()\n        self.pad = tf.constant([[0, 0], [pad, pad], [pad, pad], [0, 0]])\n\n    def call(self, inputs):\n        return tf.pad(inputs, self.pad, mode='constant', constant_values=0)\n    \ndef autopad(k, p=None):  # kernel, padding\n    # Pad to 'same'\n    if p is None:\n        p = k // 2 if isinstance(k, int) else [x // 2 for x in k]  # auto-pad\n        return p\n    else:\n        return p\n\nclass Conv(keras.layers.Layer):\n    # Standard convolution\n    def __init__(self, c1, c2, k=1, s=1, p=None, g=1, act=True, w=None):\n        # ch_in, ch_out, weights, kernel, stride, padding, groups\n        super().__init__()\n        assert g == 1, \"TF v2.2 Conv2D does not support 'groups' argument\"\n        assert isinstance(k, int), \"Convolution with multiple kernels are not allowed.\"\n        # TensorFlow convolution padding is inconsistent with PyTorch (e.g. k=3 s=2 'SAME' padding)\n        # see https://stackoverflow.com/questions/52975843/comparing-conv2d-with-padding-between-tensorflow-and-pytorch\n\n        conv = keras.layers.Conv2D(\n            c2, k, s, 'SAME' if s == 1 else 'VALID', use_bias=False)\n#             kernel_initializer=keras.initializers.Constant(w.conv.weight.permute(2, 3, 1, 0).numpy()),\n#             bias_initializer='zeros' if hasattr(w, 'bn') else keras.initializers.Constant(w.conv.bias.numpy()))\n        self.conv = conv if s == 1 else keras.Sequential([Pad(autopad(k, p)), conv])\n        self.bn = BN()# if hasattr(w, 'bn') else tf.identity\n\n        # YOLOv5 activations\n#         if isinstance(w.act, nn.LeakyReLU):\n#             self.act = (lambda x: keras.activations.relu(x, alpha=0.1)) if act else tf.identity\n#         elif isinstance(w.act, nn.Hardswish):\n#             self.act = (lambda x: x * tf.nn.relu6(x + 3) * 0.166666667) if act else tf.identity\n#         elif isinstance(w.act, (nn.SiLU, SiLU)):\n        self.act = (lambda x: keras.activations.swish(x)) if act else tf.identity\n#         else:\n#             raise Exception(f'no matching TensorFlow activation found for {w.act}')\n\n    def call(self, inputs):\n        return self.act(self.bn(self.conv(inputs)))\n\n\nclass Focus(keras.layers.Layer):\n    # Focus wh information into c-space\n    def __init__(self, c1, c2, k=1, s=1, p=None, g=1, act=True, w=None):\n        # ch_in, ch_out, kernel, stride, padding, groups\n        super().__init__()\n        self.conv = Conv(c1 * 4, c2, k, s, p, g, act)\n\n    def call(self, inputs):  # x(b,w,h,c) -> y(b,w/2,h/2,4c)\n        # inputs = inputs / 255  # normalize 0-255 to 0-1\n        return self.conv(tf.concat([inputs[:, ::2, ::2, :],\n                                    inputs[:, 1::2, ::2, :],\n                                    inputs[:, ::2, 1::2, :],\n                                    inputs[:, 1::2, 1::2, :]], 3))\n\n\nclass Bottleneck(keras.layers.Layer):\n    # Standard bottleneck\n    def __init__(self, c1, c2, shortcut=True, g=1, e=0.5, w=None):  # ch_in, ch_out, shortcut, groups, expansion\n        super().__init__()\n        c_ = int(c2 * e)  # hidden channels\n        self.cv1 = Conv(c1, c_, 1, 1)\n        self.cv2 = Conv(c_, c2, 3, 1, g=g)\n        self.add = shortcut and c1 == c2\n\n    def call(self, inputs):\n        return inputs + self.cv2(self.cv1(inputs)) if self.add else self.cv2(self.cv1(inputs))\n\n\nclass Conv2d(keras.layers.Layer):\n    # Substitution for PyTorch nn.Conv2D\n    def __init__(self, c1, c2, k, s=1, g=1, bias=True, w=None):\n        super().__init__()\n        assert g == 1, \"TF v2.2 Conv2D does not support 'groups' argument\"\n        self.conv = keras.layers.Conv2D(\n            c2, k, s, 'VALID', use_bias=bias,\n)\n\n    def call(self, inputs):\n        return self.conv(inputs)\n\n\nclass BottleneckCSP(keras.layers.Layer):\n    # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\n    def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5, w=None):\n        # ch_in, ch_out, number, shortcut, groups, expansion\n        super().__init__()\n        c_ = int(c2 * e)  # hidden channels\n        self.cv1 = Conv(c1, c_, 1, 1)\n        self.cv2 = Conv2d(c1, c_, 1, 1, bias=False)\n        self.cv3 = Conv2d(c_, c_, 1, 1, bias=False)\n        self.cv4 = Conv(2 * c_, c2, 1, 1, w=w.cv4)\n        self.bn = BN()\n        self.act = lambda x: keras.activations.relu(x, alpha=0.1)\n        self.m = keras.Sequential([Bottleneck(c_, c_, shortcut, g, e=1.0, w=w.m[j]) for j in range(n)])\n\n    def call(self, inputs):\n        y1 = self.cv3(self.m(self.cv1(inputs)))\n        y2 = self.cv2(inputs)\n        return self.cv4(self.act(self.bn(tf.concat((y1, y2), axis=3))))\n\n\nclass C3(keras.layers.Layer):\n    # CSP Bottleneck with 3 convolutions\n    def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5, w=None):\n        # ch_in, ch_out, number, shortcut, groups, expansion\n        super().__init__()\n        c_ = int(c2 * e)  # hidden channels\n        self.cv1 = Conv(c1, c_, 1, 1)\n        self.cv2 = Conv(c1, c_, 1, 1)\n        self.cv3 = Conv(2 * c_, c2, 1, 1)\n        self.m = keras.Sequential([Bottleneck(c_, c_, shortcut, g, e=1.0) for j in range(n)])\n\n    def call(self, inputs):\n        return self.cv3(tf.concat((self.m(self.cv1(inputs)), self.cv2(inputs)), axis=3))\n\n\nclass SPP(keras.layers.Layer):\n    # Spatial pyramid pooling layer used in YOLOv3-SPP\n    def __init__(self, c1, c2, k=(5, 9, 13), w=None):\n        super().__init__()\n        c_ = c1 // 2  # hidden channels\n        self.cv1 = Conv(c1, c_, 1, 1)\n        self.cv2 = Conv(c_ * (len(k) + 1), c2, 1, 1)\n        self.m = [keras.layers.MaxPool2D(pool_size=x, strides=1, padding='SAME') for x in k]\n\n    def call(self, inputs):\n        x = self.cv1(inputs)\n        return self.cv2(tf.concat([x] + [m(x) for m in self.m], 3))\n\n\nclass SPPF(keras.layers.Layer):\n    # Spatial pyramid pooling-Fast layer\n    def __init__(self, c1, c2, k=5, w=None):\n        super().__init__()\n        c_ = c1 // 2  # hidden channels\n        self.cv1 = Conv(c1, c_, 1, 1)\n        self.cv2 = Conv(c_ * 4, c2, 1, 1)\n        self.m = keras.layers.MaxPool2D(pool_size=k, strides=1, padding='SAME')\n\n    def call(self, inputs):\n        x = self.cv1(inputs)\n        y1 = self.m(x)\n        y2 = self.m(y1)\n        return self.cv2(tf.concat([x, y1, y2, self.m(y2)], 3))\n\n\nclass Detect(keras.layers.Layer):\n    def __init__(self, nc=80, anchors=(), ch=(), imgsz=(640, 640), w=None):  # detection layer\n        super().__init__()\n#         self.stride = tf.convert_to_tensor(w.stride.numpy(), dtype=tf.float32)\n        \n        self.nc = nc  # number of classes\n        self.no = nc + 5  # number of outputs per anchor\n        self.nl = len(anchors)  # number of detection layers\n        self.na = len(anchors[0]) // 2  # number of anchors\n        self.grid = [tf.zeros(1)] * self.nl  # init grid\n        if self.nl == 4:\n            self.stride = tf.convert_to_tensor(np.array([8, 16, 32,64], np.float32))\n        else:\n            self.stride = tf.convert_to_tensor(np.array([8, 16, 32], np.float32))\n            \n#         self.anchors = tf.convert_to_tensor(anchors, dtype=tf.float32)\n#         print(anchors)\n        self.anchors = tf.cast(tf.reshape(anchors, [self.nl, -1, 2]), tf.float32)\n        \n        self.anchor_grid = tf.reshape(self.anchors * tf.reshape(self.stride, [self.nl, 1, 1]),\n                                      [self.nl, 1, -1, 1, 2])\n          # fixed here, modify if structure changes\n        \n        self.m = [Conv2d(x, self.no * self.na, 1) for i, x in enumerate(ch)]\n          # set to False after building model\n        self.imgsz = imgsz\n        for i in range(self.nl):\n            ny, nx = self.imgsz[0] // self.stride[i], self.imgsz[1] // self.stride[i]\n            self.grid[i] = self._make_grid(nx, ny)\n\n    def call(self, inputs, training = True):\n        z = []  # inference output\n        x = []\n        for i in range(self.nl):\n            x.append(self.m[i](inputs[i]))\n            # x(bs,20,20,255) to x(bs,3,20,20,85)\n            ny, nx = self.imgsz[0] // self.stride[i], self.imgsz[1] // self.stride[i]\n            x[i] = tf.transpose(tf.reshape(x[i], [-1, ny * nx, self.na, self.no]), [0, 2, 1, 3])\n\n            if not training:  # inference\n                y = tf.sigmoid(x[i])\n                xy = (y[..., 0:2] * 2 - 0.5 + self.grid[i]) * self.stride[i]  # xy\n                wh = (y[..., 2:4] * 2) ** 2 * self.anchor_grid[i]  / self.stride[i]\n                # Normalize xywh to 0-1 to reduce calibration error\n#                 xy /= tf.constant([[self.imgsz[1], self.imgsz[0]]], dtype=tf.float32)\n#                 wh /= tf.constant([[self.imgsz[1], self.imgsz[0]]], dtype=tf.float32)\n                y = tf.concat([xy, wh, y[..., 4:]], -1)\n                z.append(tf.reshape(y, [-1, self.na * ny * nx, self.no]))\n\n        return x if training else (tf.concat(z, 1), x)\n\n    @staticmethod\n    def _make_grid(nx=20, ny=20):\n        # yv, xv = torch.meshgrid([torch.arange(ny), torch.arange(nx)])\n        # return torch.stack((xv, yv), 2).view((1, 1, ny, nx, 2)).float()\n        xv, yv = tf.meshgrid(tf.range(nx), tf.range(ny))\n        return tf.cast(tf.reshape(tf.stack([xv, yv], 2), [1, 1, ny * nx, 2]), dtype=tf.float32)\n\n\nclass Upsample(keras.layers.Layer):\n    def __init__(self, size, scale_factor, mode, w=None):  # warning: all arguments needed including 'w'\n        super().__init__()\n        assert scale_factor == 2, \"scale_factor must be 2\"\n        self.upsample = lambda x: tf.image.resize(x, (x.shape[1] * 2, x.shape[2] * 2), method=mode)\n        # self.upsample = keras.layers.UpSampling2D(size=scale_factor, interpolation=mode)\n        # with default arguments: align_corners=False, half_pixel_centers=False\n        # self.upsample = lambda x: tf.raw_ops.ResizeNearestNeighbor(images=x,\n        #                                                            size=(x.shape[1] * 2, x.shape[2] * 2))\n\n    def call(self, inputs):\n        return self.upsample(inputs)\n\n\nclass Concat(keras.layers.Layer):\n    def __init__(self, dimension=1, w=None):\n        super().__init__()\n        assert dimension == 1, \"convert only NCHW to NHWC concat\"\n        self.d = 3\n\n    def call(self, inputs):\n        return tf.concat(inputs, self.d)","metadata":{"execution":{"iopub.status.busy":"2022-02-15T07:39:28.574127Z","iopub.execute_input":"2022-02-15T07:39:28.574501Z","iopub.status.idle":"2022-02-15T07:39:28.639224Z","shell.execute_reply.started":"2022-02-15T07:39:28.574399Z","shell.execute_reply":"2022-02-15T07:39:28.638508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_divisible(x, divisor):\n    # Returns nearest x divisible by divisor\n    if isinstance(divisor, tf.Tensor):\n        divisor = int(divisor.max())  # to int\n    return math.ceil(x / divisor) * divisor\n\nclass Yolo(object):\n    def __init__(self, yaml_dir, params, img_size, tf_nms = False, training = True):\n#         tf.keras.backend.clear_session()\n        self.tf_nms = tf_nms\n        with open(yaml_dir) as f:\n            yaml_dict = yaml.load(f, Loader=yaml.FullLoader)\n#         print(yaml_dict)\n        yaml_dict['nc'] = 1\n#         yaml_dict['anchors'] =\n        self.params = params\n        self.module_list = self.parse_model(yaml_dict, ch = [3], imgsz = [img_size[0], img_size[1]])\n        module = self.module_list[-1]\n        self.training = training\n#         print(module.f)\n        if isinstance(module, Detect):\n            # transfer the anchors to grid coordinator, 3 * 3 * 2\n            module.anchors /= tf.reshape(module.stride, [-1, 1, 1])\n     \n    def __call__(self, img_size, name='yolo'):\n        x = tf.keras.Input([img_size[0], img_size[1], 3])\n        output = self.forward(x, self.tf_nms)\n        return tf.keras.Model(inputs=x, outputs=output, name=name)\n\n    def forward(self, x, tf_nms = False):\n        y = []\n        for module in self.module_list:\n            if module.f != -1:  # if not from previous layer\n                if isinstance(module.f, int):\n                    x = y[module.f]\n                else:\n                    x = [x if j == -1 else y[j] for j in module.f]\n            if isinstance(module, Detect):\n                x = module(x, self.training)\n            else:\n                x = module(x)\n            y.append(x)\n        \n\n        # Add TensorFlow NMS\n        if tf_nms:\n            boxes = self._xywh2xyxy(x[0][..., :4])\n            probs = x[0][:, :, 4:5]\n            classes = x[0][:, :, 5:]\n            scores = probs * classes\n#             if agnostic_nms:\n#                 nms = AgnosticNMS()((boxes, classes, scores), topk_all, iou_thres, conf_thres)\n#                 return nms, x[1]\n#             else:\n            boxes = tf.expand_dims(boxes, 2)\n            nms = tf.image.combined_non_max_suppression(\n                boxes, scores, 100, 100, 0.5, 0.10, clip_boxes=False)\n            return nms, x[1]\n\n        return x  # output only first tensor [1,6300,85] = [xywh, conf, class0, class1, ...]\n    def parse_model(self, d, ch, imgsz):  # model_dict, input_channels(3)\n#         LOGGER.info(f\"\\n{'':>3}{'from':>18}{'n':>3}{'params':>10}  {'module':<40}{'arguments':<30}\")\n        anchors, nc, gd, gw = d['anchors'], d['nc'], d['depth_multiple'], d['width_multiple']\n        na = (len(anchors[0]) // 2) if isinstance(anchors, list) else anchors  # number of anchors\n        no = na * (nc + 5)  # number of outputs = anchors * (classes + 5)\n#         print(anchors)\n        layers, save, c2 = [], [], ch[-1]  # layers, savelist, ch out\n        for i, (f, n, m, args) in enumerate(d['backbone'] + d['head']):  # from, number, module, args\n            m = m.replace('nn.', '') if isinstance(m, str) else m\n#             print(m)\n            m_str = m\n            m = eval(m) if isinstance(m, str) else m  # eval strings\n            for j, a in enumerate(args):\n                try:\n                    args[j] = eval(a) if isinstance(a, str) else a  # eval strings\n                except NameError:\n                    pass\n\n            n = max(round(n * gd), 1) if n > 1 else n  # depth gain\n    #         print(i, n)\n            if m in [Conv2d, Conv, Bottleneck, SPP, SPPF, Focus, BottleneckCSP, C3]:\n                c1, c2 = ch[f], args[0]\n                c2 = make_divisible(c2 * gw, 8) if c2 != no else c2\n\n                args = [c1, c2, *args[1:]]\n                if m in [BottleneckCSP, C3]:\n                    args.insert(2, n)\n                    n = 1\n            elif m is BN:\n                args = [ch[f]]\n            elif m is Concat:\n                c2 = sum(ch[-1 if x == -1 else x + 1] for x in f)\n            elif m is Detect:\n                args.append([ch[x + 1] for x in f])\n                if isinstance(args[1], int):  # number of anchors\n                    args[1] = [list(range(args[1] * 2))] * len(f)\n                args.append(imgsz)\n            else:\n                c2 = ch[f]\n            tf_m = eval(m) if isinstance(m, str) else m\n#             if i == 0:\n#                 args[-2] = 1\n            m_ = tf.keras.Sequential(*[tf_m(*args) for _ in range(n)]) if n > 1 else tf_m(*args)\n#             else:\n#                 m_ = tf.keras.Sequential(*[tf_m(*args) for _ in range(n)]) if n > 1 else tf_m(*args)\n            m_.i, m_.f = i, f\n            layers.append(m_)\n            ch.append(c2)\n        return layers\n    \n    @staticmethod\n    def _xywh2xyxy(xywh):\n        # Convert nx4 boxes from [x, y, w, h] to [x1, y1, x2, y2] where xy1=top-left, xy2=bottom-right\n        x, y, w, h = tf.split(xywh, num_or_size_splits=4, axis=-1)\n        return tf.concat([x - w / 2, y - h / 2, x + w / 2, y + h / 2], axis=-1)","metadata":{"execution":{"iopub.status.busy":"2022-02-15T07:39:28.640542Z","iopub.execute_input":"2022-02-15T07:39:28.640789Z","iopub.status.idle":"2022-02-15T07:39:28.670484Z","shell.execute_reply.started":"2022-02-15T07:39:28.640755Z","shell.execute_reply":"2022-02-15T07:39:28.669258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def one_cycle(y1=0.0, y2=1.0, steps=100):\n    # lambda function for sinusoidal ramp from y1 to y2 https://arxiv.org/pdf/1812.01187.pdf\n    return lambda x: ((1 - math.cos(x * math.pi / steps)) / 2) * (y2 - y1) + y1\n\nclass LrScheduler(object):\n    def __init__(self, total_steps, params, scheduler_method='cosine'):\n        self.scheduler = Cosine(total_steps, params)\n        self.step_count = 0\n        self.total_steps = total_steps\n\n    def step(self):\n        self.step_count += 1\n        lr = self.scheduler(self.step_count)\n        return lr\n\n    def plot(self):\n        lr = []\n        for i in range(self.total_steps):\n            lr.append(self.step())\n        plt.plot(range(self.total_steps), lr)\n        plt.show()\n\n\n\nclass Cosine(tf.keras.optimizers.schedules.LearningRateSchedule):\n    def __init__(self, total_steps, params):\n        # create the cosine learning rate with linear warmup\n        super(Cosine, self).__init__()\n        self.total_steps = total_steps\n        self.params = params\n\n    def __call__(self, global_step):\n        init_lr = self.params.init_learning_rate\n        warmup_lr = self.params.warmup_learning_rate# if 'warmup_learning_rate' in self.params else 0.0\n        warmup_steps = self.params.warmup_steps\n        assert warmup_steps < self.total_steps, \"warmup {}, total {}\".format(warmup_steps, self.total_steps)\n\n        linear_warmup = warmup_lr + tf.cast(global_step, tf.float32) / warmup_steps * (init_lr - warmup_lr)\n        cosine_learning_rate = init_lr * (\n                    tf.cos(np.pi * (global_step - warmup_steps) / (self.total_steps - warmup_steps)) + 1.0) / 2.0\n        learning_rate = tf.where(global_step < warmup_steps, linear_warmup, cosine_learning_rate)\n        return learning_rate","metadata":{"execution":{"iopub.status.busy":"2022-02-15T07:39:28.671761Z","iopub.execute_input":"2022-02-15T07:39:28.673229Z","iopub.status.idle":"2022-02-15T07:39:28.685991Z","shell.execute_reply.started":"2022-02-15T07:39:28.672938Z","shell.execute_reply":"2022-02-15T07:39:28.685261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class YoloLoss(object):\n    def __init__(self, hyp, anchors, ignore_iou_threshold, num_classes, img_size, label_smoothing=0):\n        self.hyp = hyp\n        self.anchors = anchors\n        self.strides = [8, 16, 32] if len(anchors) == 3 else [8,16,32,64] \n        self.ignore_iou_threshold = ignore_iou_threshold\n        self.num_classes = num_classes\n        self.img_size = img_size\n        self.bce_conf = tf.keras.losses.BinaryCrossentropy(from_logits = True, reduction=tf.keras.losses.Reduction.NONE)\n        self.bce_class = tf.keras.losses.BinaryCrossentropy(from_logits = True, reduction=tf.keras.losses.Reduction.NONE,\n                                                            label_smoothing=label_smoothing)\n\n    def __call__(self, y_true, y_pred):\n        iou_loss_all = obj_loss_all = class_loss_all = 0\n        balance = [4.0, 1.0, 0.4] if len(y_pred) == 3 else [4.0, 1.0, 0.4, 0.1]  # P3-5 or P3-6        \n#         print(len(y_pred))\n        for i, (pred, true) in enumerate(zip(y_pred, y_true)):\n#             print(pred.numpy())\n            # preprocess, true: batch_size * grid * grid * 3 * 6, pred: batch_size * grid * grid * clss+5\n            grid = tf.cast(tf.sqrt(tf.cast(tf.shape(pred)[2], tf.float32)), tf.int32)\n#             true = tf.expand_dims(true, axis= 0)\n#             print(pred.shape)\n            pred = tf.reshape(tf.transpose(pred, [0,2,1,3]), (tf.shape(pred)[0], grid, grid, 3,6))\n            true_box, true_obj, true_class = tf.split(true, (4, 1, -1), axis=-1)\n            pred_box_xy, pred_box_wh, pred_obj,pred_class = tf.split(pred, (2, 2, 1, -1), axis=-1)\n            obj_mask = tf.squeeze(true_obj, -1)\n            if tf.shape(true_class)[-1] == 1 and self.num_classes > 1:\n                true_class = tf.squeeze(tf.one_hot(tf.cast(true_class, tf.dtypes.int32), depth=self.num_classes, axis=-1), -2) \n\n            pxy = tf.nn.sigmoid(pred_box_xy) * 2 - 0.5\n            pwh = (tf.nn.sigmoid(pred_box_wh) * 2) ** 2 * self.anchors[i]\n            pbox = tf.concat([pxy, pwh], axis = -1)  # predicted box\n#             print(pbox.shape, true_box.shape)\n            iou = bbox_iou(pbox, true_box, xyxy=False, ciou = True)  # iou(prediction, target)\n\n            iou_loss = (1.0 - iou) * obj_mask  # iou loss\n            iou_loss_all += tf.reduce_mean(iou_loss) * balance[i]\n#             print(true_obj.shape, iou.shape, pred_obj.shape)\n#             iou = tf.expand_dims(iou, axis= 4)\n#             print(iou)\n#             print(true_obj.shape, iou.shape, pred_obj.shape)\n            iou_weight = tf.cast(tf.clip_by_value(iou, 0, 1) * obj_mask , obj_mask.dtype)\n            obji = tf.nn.weighted_cross_entropy_with_logits(tf.squeeze(true_obj, axis = -1),\n                                                             tf.squeeze(pred_obj, axis = -1), \n                                                            pos_weight = 1) * obj_mask\n#             print(obji.shape)\n#             obji = self.bce_conf(true_obj, pred_obj) * obj_mask\n            obj_loss_all += tf.reduce_mean(obji) * balance[i]  # obj loss\n        iou_loss_all *= self.hyp.box\n        obj_loss_all *= self.hyp.obj\n        class_loss_all *= self.hyp.cls\n\n        return iou_loss_all, obj_loss_all, class_loss_all\n\n\ndef bbox_iou(bbox1, bbox2, xyxy=False, giou=False, diou=False, ciou=False, epsilon=1e-9):\n    assert bbox1.shape == bbox2.shape\n    # giou loss: https://arxiv.org/abs/1902.09630\n    if xyxy:\n        b1x1, b1y1, b1x2, b1y2 = bbox1[..., 0], bbox1[..., 1], bbox1[..., 2], bbox1[..., 3]\n        b2x1, b2y1, b2x2, b2y2 = bbox2[..., 0], bbox2[..., 1], bbox2[..., 2], bbox2[..., 3]\n    else:  # xywh -> xyxy\n        b1x1, b1x2 = bbox1[..., 0] - bbox1[..., 2] / 2, bbox1[..., 0] + bbox1[..., 2] / 2\n        b1y1, b1y2 = bbox1[..., 1] - bbox1[..., 3] / 2, bbox1[..., 1] + bbox1[..., 3] / 2\n        b2x1, b2x2 = bbox2[..., 0] - bbox2[..., 2] / 2, bbox2[..., 0] + bbox2[..., 2] / 2\n        b2y1, b2y2 = bbox2[..., 1] - bbox2[..., 3] / 2, bbox2[..., 1] + bbox2[..., 3] / 2\n\n    # intersection area\n    inter = tf.maximum(tf.minimum(b1x2, b2x2) - tf.maximum(b1x1, b2x1), 0) * \\\n            tf.maximum(tf.minimum(b1y2, b2y2) - tf.maximum(b1y1, b2y1), 0)\n\n    # union area\n    w1, h1 = b1x2 - b1x1 + epsilon, b1y2 - b1y1 + epsilon\n    w2, h2 = b2x2 - b2x1+ epsilon, b2y2 - b2y1 + epsilon\n    union = w1 * h1 + w2 * h2 - inter + epsilon\n\n    # iou\n    iou = inter / union\n\n    if giou or diou or ciou:\n        # enclosing box\n        cw = tf.maximum(b1x2, b2x2) - tf.minimum(b1x1, b2x1)\n        ch = tf.maximum(b1y2, b2y2) - tf.minimum(b1y1, b2y1)\n        if giou:\n            enclose_area = cw * ch + epsilon\n            giou = iou - 1.0 * (enclose_area - union) / enclose_area\n            return tf.clip_by_value(giou, -1, 1)\n        if diou or ciou:\n            c2 = cw ** 2 + ch ** 2 + epsilon\n            rho2 = ((b2x1 + b2x2) - (b1x1 + b1x2)) ** 2 / 4 + ((b2y1 + b2y2) - (b1y1 + b1y2)) ** 2 / 4\n            if diou:\n                return iou - rho2 / c2\n            elif ciou:\n                v = (4 / math.pi ** 2) * tf.pow(tf.atan(w2 / h2) - tf.atan(w1 / h1), 2)\n                alpha = v / (1 - iou + v)\n                return iou - (rho2 / c2 + v * alpha)\n    return tf.clip_by_value(iou, 0, 1)","metadata":{"execution":{"iopub.status.busy":"2022-02-15T07:39:28.689277Z","iopub.execute_input":"2022-02-15T07:39:28.689492Z","iopub.status.idle":"2022-02-15T07:39:28.720997Z","shell.execute_reply.started":"2022-02-15T07:39:28.689469Z","shell.execute_reply":"2022-02-15T07:39:28.720202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Trainer(object):\n    \"\"\" Trainer class that uses the dataset and model to train\n    # Usage\n    data_loader = tf.data.Dataset()\n    trainer = Trainer(params)\n    trainer.train(data_loader)\n    \"\"\"\n    def __init__(self, params):\n        \"\"\" Constructor\n        :param params: dict, with dir and training parameters\n        \"\"\"\n        self.params = params\n#         if os.path.exists(self.params['log_dir']):\n#             shutil.rmtree(self.params['log_dir'])\n#         self.log_writer = tf.summary.create_file_writer(self.params.log_dir)\n        self.global_step = tf.Variable(0, trainable=False, dtype=tf.int64)\n#         nc = Params.nc\n#         nl = de_parallel(model).model[-1].nl  # number of detection layers (to scale hyps)\n#         hyp['box'] *= 3 / nl  # scale to layers\n#         hyp['cls'] *= nc / 80 * 3 / nl  # scale to classes and layers\n#         hyp['obj'] *= (imgsz / 640) ** 2 * 3 / nl  # scale to image size and layers\n        self.build_model()\n        \n\n    def build_model(self):\n        \"\"\" Build the model,\n        define the training strategy and model, loss, optimizer\n        :return:\n        \"\"\"\n        self.strategy, self.replicas = auto_select_accelerator()\n        \n        with self.strategy.scope():\n            self.model = Yolo(yaml_dir=self.params.yaml_dir, img_size = [self.params.img_size, self.params.img_size], params = self.params)\n#             print(self.model.module_list[-1].stride)\n            self.anchors = self.model.module_list[-1].anchors   \n            self.stride = self.model.module_list[-1].stride\n            self.nl = self.model.module_list[-1].nl\n            self.nc = self.model.module_list[-1].nc\n#             self.num_classes = self.model.module_list[-1].num_classes\n            self.loss_sum = tf.keras.metrics.Sum(name='sum', dtype=None)\n            self.loss_fn = YoloLoss(self.params, self.model.module_list[-1].anchors,\n                                    ignore_iou_threshold=0.3,\n                                    num_classes=self.params.num_classes,\n                                    label_smoothing=self.params.label_smoothing,\n                                    img_size=self.params.img_size)\n            if self.params.optimizer == 'sgd':\n                self.optimizer = tf.keras.optimizers.SGD(momentum = self.params.momentum)\n            elif self.params.optimizer == 'adam':\n                self.optimizer = tf.keras.optimizers.Adam()\n        self.params.box *= 3 / self.nl  # scale to layers\n        self.params.cls *= self.nc / 80 * 3 / self.nl  # scale to classes and layers\n        self.params.obj *= (self.params.img_size / 640) ** 2 * 3 / self.nl  # scale to image size and layers\n        \n    def train(self, train_dataset, valid_dataset=None, transfer='scratch'):\n        \"\"\" train function\n        :param train_dataset: train dataset built by tf.data\n        :param valid_dataset: valid dataset build by td.data, optional\n        :param transfer: pretrain\n        :return:\n        \"\"\"\n        steps_per_epoch = int(self.params.len_train_dataset // self.params.batch_size)\n        self.total_steps = int(self.params.n_epochs * steps_per_epoch)\n        self.params.warmup_steps = self.params.warmup_epochs * steps_per_epoch\n# \n        with self.strategy.scope():\n            self.lr_scheduler = LrScheduler(self.total_steps, self.params, scheduler_method='cosine')\n            # => tf.keras.Model\n            self.model = self.model([self.params.img_size, self.params.img_size])\n#             self.model.summary()\n            self.model.load_weights(self.params.weight_dir)\n\n        train_dataset = self.strategy.experimental_distribute_dataset(train_dataset)        \n        \n#         epoch = 0\n        for epoch in range(self.params.n_epochs):\n            for step, (image, target) in enumerate(tqdm(train_dataset, total = steps_per_epoch)):                \n                loss = self.dist_train_step(image, target)\n#                 print('=> Epoch {}, Step {}, Loss {:.5f}'.format(epoch, self.global_step.numpy(), \n#                                                                 self.loss_sum.result() / (self.global_step.numpy())))\n    #             print(iou_loss, conf_loss, class_loss)\n    #             with self.log_writer.as_default():\n    #                 tf.summary.scalar('loss', loss, step=self.global_step)\n    #                 tf.summary.scalar('lr', self.optimizer.lr, step=self.global_step)\n    #             self.log_writer.flush()\n    #             print(steps_per_epoch)\n#                 if (self.global_step.numpy() + 1) % steps_per_epoch == 0:\n            print('=> Epoch {}, Step {}, Loss {:.5f}'.format(epoch, self.global_step.numpy(), \n                                                                self.loss_sum.result() / (self.global_step.numpy())))\n#                     epoch += 1\n            self.model.save_weights(f'/kaggle/working/model epoch {epoch}.h5')\n#             step += 1\n\n    @tf.function\n    def train_step(self, image, target):\n        \n        with tf.GradientTape() as tape:\n            logit = self.model(image, training=True)\n#             print(logit)\n            iou_loss, conf_loss, prob_loss = self.loss_fn(target, logit)\n            total_loss = iou_loss + conf_loss + prob_loss\n#             print(total_loss)\n            total_loss = tf.reduce_sum(total_loss) * self.params.batch_size\n#             total_loss = tf.nn.compute_average_loss(total_loss, global_batch_size = params.batch_size)\n        gradients = tape.gradient(total_loss, self.model.trainable_variables)\n        self.optimizer.apply_gradients(zip(gradients, self.model.trainable_variables))\n        self.loss_sum.update_state(total_loss)\n        lr = self.lr_scheduler.step()\n        self.optimizer.lr.assign(lr)\n        self.global_step.assign_add(1)    \n        return total_loss\n\n    @tf.function\n    def dist_train_step(self, image, target):\n        with self.strategy.scope():\n            loss = self.strategy.run(self.train_step, args=(image, target))\n#             total_loss_mean = self.strategy.reduce(tf.distribute.ReduceOp.MEAN, loss, axis=None)\n            return loss\n\n    def validate(self, valid_dataset):\n        valid_loss = []\n        for step, (image, target) in enumerate(valid_dataset):\n            step_valid_loss = self.valid_step(image, target)\n            valid_loss.append(step_valid_loss)\n        return np.mean(valid_loss)\n\n    def valid_step(self, image, label):\n        logit = self.model(image, training=False)\n        iou_loss, conf_loss, prob_loss = self.loss_fn(label, logit)\n        return iou_loss + conf_loss + prob_loss\n","metadata":{"execution":{"iopub.status.busy":"2022-02-15T07:40:21.292728Z","iopub.execute_input":"2022-02-15T07:40:21.293002Z","iopub.status.idle":"2022-02-15T07:40:21.320443Z","shell.execute_reply.started":"2022-02-15T07:40:21.292974Z","shell.execute_reply":"2022-02-15T07:40:21.319556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainer = Trainer(Params)  \nreader = DataReader(hyp = Params, \n                    annotations_dict = train_annotations_dict,\n                    img_size = Params.img_size,\n                    mosaic = True, \n                    augment = True)\n\ndataloader = DataLoader(reader, trainer.anchors, trainer.stride, Params.img_size)\ntrain_dataset = dataloader(Params.batch_size, anchor_label = True)","metadata":{"execution":{"iopub.status.busy":"2022-02-15T07:40:21.705708Z","iopub.execute_input":"2022-02-15T07:40:21.706037Z","iopub.status.idle":"2022-02-15T07:40:24.364942Z","shell.execute_reply.started":"2022-02-15T07:40:21.705988Z","shell.execute_reply":"2022-02-15T07:40:24.364204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# training the model\ntrainer.train(train_dataset)","metadata":{"execution":{"iopub.status.busy":"2022-02-15T07:40:24.368685Z","iopub.execute_input":"2022-02-15T07:40:24.369287Z","iopub.status.idle":"2022-02-15T07:59:18.457427Z","shell.execute_reply.started":"2022-02-15T07:40:24.369254Z","shell.execute_reply":"2022-02-15T07:59:18.455148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}