{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Submission notebook","metadata":{}},{"cell_type":"code","source":"!cp /kaggle/input/run-1280-yolov5l/best.pt /kaggle/working/best.pt\n!cp /kaggle/input/run-1280-yolov5l/yolov_train3_results/last.pt /kaggle/working/yolov5l_train3_last.pt\n# !cp /kaggle/input/run-1280-yolov5l/yolov5l6_train3_weights/last.pt /kaggle/working/yolov5l6_last.pt\n# !cp /kaggle/input/run-1280-yolov5l/yolov5l6_train5_weights/last.pt /kaggle/working/yolov5l6_last.pt\n!cp /kaggle/input/run-1280-yolov5l/yolov5l6_train6_weights/last.pt /kaggle/working/yolov5l6_last.pt\n# !cp /kaggle/input/run-1280-yolov5l/yolov5s6_train1_weights/best.pt /kaggle/working/yolov5s6_best.pt\n# !cp /kaggle/input/reef-yolor-w6/yoloR_w6_train/train1/best_r.pt /kaggle/working/yolor_w6.pt","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# setup - copy font for offline use\n!mkdir -p /root/.config/Ultralytics\n!cp /kaggle/input/yolov5-font/Arial.ttf /root/.config/Ultralytics/","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install /kaggle/input/pycocotools/pycocotools-2.0.3.tar","metadata":{"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %cd /kaggle/working/\n# !rm -rf yolor\n# %cp -r ../input/hey-yolor/. /kaggle/working/yolor","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Imports","metadata":{}},{"cell_type":"code","source":"# %cd /kaggle/working/yolor\n# from utils.torch_utils import select_device\n# import pycocotools\n# # YoloR methods\n# from yolor.models.models import Darknet\n# from yolor.utils.datasets import letterbox\n# from yolor.utils.general import non_max_suppression, scale_coords","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd /kaggle/working","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport torch\nimport cv2\nimport matplotlib.pyplot as plt\nimport matplotlib \n%matplotlib inline\nimport pandas as pd\nfrom PIL import Image\nimport os\nfrom pathlib import Path\nimport sys\nimport gc\nimport json5\nimport shutil\n\nsys.path.append('../input/tensorflow-great-barrier-reef')\nsys.path.append('../input/heyyolov5')\nsys.path.append('../input/yolov5pip/yolov5-pip')\n\nsys.path.append('../input/weightedboxesfusion')\nfrom ensemble_boxes import *\n\nsys.path.append('../input/isears-tf-clahe')\nimport tf_clahe\n\nnp.random.seed(0)\n\nimport tensorflow as tf\ntf.version\n\ntorch.no_grad()","metadata":{"execution":{"iopub.status.busy":"2022-01-23T11:37:57.919375Z","iopub.execute_input":"2022-01-23T11:37:57.919749Z","iopub.status.idle":"2022-01-23T11:38:00.306451Z","shell.execute_reply.started":"2022-01-23T11:37:57.919655Z","shell.execute_reply":"2022-01-23T11:38:00.305741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Helper Funcs","metadata":{}},{"cell_type":"code","source":"# Memory saving function credit to https://www.kaggle.com/gemartin/load-data-reduce-memory-usage\ndef reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n\n    for col in df.columns:\n        col_type = df[col].dtype\n\n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    del start_mem, end_mem\n    return df\n\ndef load_labels():\n    labels = pd.read_csv(\"/kaggle/input/tensorflow-great-barrier-reef/train.csv\", skipinitialspace=True)\n    labels.drop_duplicates(inplace=True)\n    labels.drop(['sequence'], axis = 1, inplace=True)\n    labels.drop(['sequence_frame'], axis = 1, inplace=True)\n    labels = reduce_mem_usage(labels)\n    return labels\n\ndef get_path(row):\n    row['image_path'] = f'/kaggle/input/tensorflow-great-barrier-reef/train_images/video_{row.video_id}/{row.video_frame}.jpg'\n    return row\n\ndef process_annot(ann_data):\n    yolo_labels = []\n    metric_boxes = []\n    if ann_data and ann_data != '[]':\n        ann_splits = json5.loads(ann_data)\n        for ann_split in ann_splits:\n            row = dict()\n            # load annotation data as json and get values\n            a_data = json5.loads(ann_split)\n            row['xmin'] = int(a_data[\"x\"])\n            row['ymin'] = int(a_data[\"y\"])\n            row['xmax'] = int(a_data[\"x\"]) + int(a_data[\"width\"])\n            row['ymax'] = int(a_data[\"y\"]) + int(a_data[\"height\"])\n            metric_boxes.append(np.array([row['xmin'],row['ymin'],a_data[\"width\"],a_data[\"height\"]]))\n            row['score'] = 1.0\n            row['label'] = 'starfish'\n            yolo_labels.append(row)\n        del ann_splits\n    return yolo_labels, metric_boxes\n \ndef get_sample_imgs(sample_count = 50):\n    labels = load_labels()\n    labels = labels[labels['annotations'].str.len() > 49]\n    labels = labels.apply(get_path, axis=1)\n    return labels.sample(sample_count)","metadata":{"execution":{"iopub.status.busy":"2022-01-23T11:38:03.735885Z","iopub.execute_input":"2022-01-23T11:38:03.736312Z","iopub.status.idle":"2022-01-23T11:38:03.757637Z","shell.execute_reply.started":"2022-01-23T11:38:03.736274Z","shell.execute_reply":"2022-01-23T11:38:03.756694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Image Processing Helpers","metadata":{}},{"cell_type":"code","source":"def plot_images(img1, img2, img3, title='', lbl1='', lbl2='', lbl3 = ''):\n    plt.close('all')\n    fig, ax = plt.subplots(1, 3, figsize=(23,13))\n    ax[0].imshow(img1, cmap = plt.get_cmap(name = 'gray'))\n    ax[0].set_axis_off()\n    ax[0].set_title(lbl1, fontsize=18)\n    ax[1].imshow(img2, cmap = plt.get_cmap(name = 'gray'))\n    ax[1].set_axis_off()\n    ax[1].set_title(lbl2, fontsize=18)\n    ax[2].imshow(img3, cmap = plt.get_cmap(name = 'gray'))\n    ax[2].set_axis_off()\n    ax[2].set_title(lbl3, fontsize=18)\n    fig.suptitle(title, fontsize=22, y=0.81)\n    plt.tight_layout()\n    plt.show()\n\n@tf.function(experimental_compile=True)  # Enable XLA\ndef tf_clahe_gpu(og_img):\n    # channel 1 is grayscale and 3 is RGB\n    # og_img = tf.io.decode_jpeg(tf.io.read_file(img_path), channels=3)\n    correct_img = tf_clahe.clahe(og_img, tile_grid_size=(28, 28), clip_limit=3.3, gpu_optimized=True)\n    return correct_img, og_img\n\ndef tf_enhance_image_gpu(og_img):\n    import torch\n    torch.cuda.empty_cache()\n    corrected_img, og_img = tf_clahe_gpu(tf.constant(og_img))\n    n_img = tf.image.adjust_gamma(corrected_img, 1.2)\n    return n_img.numpy()\n\n# further read for good tips/tricks\n# https://www.kaggle.com/soumya9977/learning-to-sea-underwater-img-enhancement-eda#%F0%9F%8E%AF-Main-Working-Code\n\n# CLAHE (Contrast Limited Adaptive Histogram Equalization)\ndef clhae(img):\n    import cv2\n    clahe = cv2.createCLAHE(clipLimit=3.3, tileGridSize=(27,27))\n    lab = cv2.cvtColor(img, cv2.COLOR_BGR2LAB)  # convert from BGR to LAB color space\n    l, a, b = cv2.split(lab)  # split on 3 different channels\n    l2 = clahe.apply(l)  # apply CLAHE to the L-channel\n    lab = cv2.merge((l2,a,b))  # merge channels\n    img = cv2.cvtColor(lab, cv2.COLOR_LAB2BGR)  # convert from LAB to BGR\n    del lab, l2, l, clahe\n    return img\n\n# auto white balance\n# credit to https://gist.github.com/DavidYKay/9dad6c4ab0d8d7dbf3dc\ndef white_balance(img, percent=1):\n    out_channels = []\n    cumstops = (\n        img.shape[0] * img.shape[1] * percent / 200.0,\n        img.shape[0] * img.shape[1] * (1 - percent / 200.0)\n    )\n    for channel in cv2.split(img):\n        cumhist = np.cumsum(cv2.calcHist([channel], [0], None, [256], (0,256)))\n        low_cut, high_cut = np.searchsorted(cumhist, cumstops)\n        lut = np.concatenate((\n            np.zeros(low_cut),\n            np.around(np.linspace(0, 255, high_cut - low_cut + 1)),\n            255 * np.ones(255 - high_cut)\n        ))\n        out_channels.append(cv2.LUT(channel, lut.astype('uint8')))\n        del cumhist, low_cut, high_cut\n    del cumstops\n    return cv2.merge(out_channels)\n\n# auto gamma correction\n# why 1.2? because, I felt it is better visually\ndef gamma_correction(img, gamma =1.2):\n    igamma = 1.0 / gamma\n    imin, imax = img.min(), img.max()\n    img_c = img.copy()\n    img_c = ((img_c - imin) / (imax - imin)) ** igamma\n    img_c = img_c * (imax - imin) + imin\n    del imin, imax, igamma\n    return img_c.astype(np.uint8)\n\n# combined helper function\n# NOTE : ORDER MATTERS HERE\ndef enhance_image(img):\n    # why in this order? again, visually it gives a lot better results\n    # also, photographer recommend to gamma correct -> white balance -> color correction\n    _img = img.copy()\n    _img = gamma_correction(_img)\n    _img = white_balance(_img)\n    _img = clhae(_img)\n    return _img\n\ndef convert_to_gray(img):\n    return cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n\ndef invert_image(img):\n    # invert\n    return cv2.bitwise_not(img.copy())\n\ndef drawBoundingBoxes(image_data, infer_results):\n    image_data = cv2.cvtColor(image_data, cv2.COLOR_BGR2RGB)\n    pinkish_red = (227, 27, 90)\n    for res in infer_results:\n        left = int(res['xmin'])\n        top = int(res['ymin'])\n        right = int(res['xmax'])\n        bottom = int(res['ymax'])\n        score = res['score']\n        label = res['label'] if res['label'] is not None else 'starfish'\n        im_h, im_w, _ = image_data.shape\n        thick = int((im_h + im_w) // 750) * 2\n        # draw bb\n        cv2.rectangle(image_data,(left, top), (right, bottom), pinkish_red, thick)\n        tf = max(thick - 1, 1)   # font thickness\n        t_size = cv2.getTextSize(label, 0, fontScale=thick / 3, thickness=int(tf))[0]\n        c2 = left + t_size[0], top - t_size[1] + 2\n        # draw text fill\n        cv2.rectangle(image_data, (left, top ), c2, pinkish_red, -1, cv2.LINE_AA)  # filled\n        # draw text\n        cv2.putText(image_data, f'{label} {score}', (left, top - 6), 0, 1e-3 * im_h, (255,255,255), int(thick*0.5))\n        del left, top, right, bottom, score, label, im_h, im_w, tf, t_size, c2\n        \n    return image_data\n\ndef draw_fused_boxes(image, boxes_list, scores_list, labels_list, image_size=(1280, 720)):\n    thickness = 5\n#     image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    thick = int((image_size[0] + image_size[1]) // 750) * 2\n    for i in range(len(boxes_list)):\n        x1 = int(image_size[0] * boxes_list[i][0])\n        y1 = int(image_size[1] * boxes_list[i][1])\n        x2 = int(image_size[0] * boxes_list[i][2])\n        y2 = int(image_size[1] * boxes_list[i][3])\n        lbl = labels_list[i]  \n        score = round(scores_list[i], 2)\n        label = 'starfish'\n        tf = max(thick - 1, 1)   # font thickness\n        t_size = cv2.getTextSize(label, 0, fontScale=thick / 3, thickness=int(tf))[0]\n        c2 = x1 + t_size[0], y1 - t_size[1] + 2\n        cv2.rectangle(image, (x1, y1), (x2, y2), (227, 27, 90), thick, cv2.LINE_AA)\n        # draw text fill\n        cv2.rectangle(image, (x1, y1), c2, (227, 27, 90), -1, cv2.LINE_AA)  # filled\n        # draw text\n        cv2.putText(image, f'{label} {score}', (x1, y1 - 6), 0, 1e-3 * image_size[1], (255,255,255), int(thick*0.5))\n    return image\n\n# best score : 502 with iou_thr=0.45 & skip_box_thr=0.15\n# best score : 513 with iou_thr=0.51 & skip_box_thr=0.11\ndef fuse_boxes(img, boxes_list, scores_list, labels_list, draw_boxes = True,\n               gen_submit_str = True, iou_thr=0.51, skip_box_thr=0.11):\n    if not boxes_list or len(boxes_list) <=0:\n        return dict(\n        img= img,\n        boxes= (),\n        scores= (), \n        labels= (), \n        detect_count= 0,\n        submission_str= '')\n#     weights = [2, 1, 3] # weights/priority to model data\n    # skip_box_thr : We skip boxes with confidence lower than skip_box_thr\n    boxes, scores, labels = weighted_boxes_fusion(boxes_list, scores_list, labels_list, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)\n\n    ann = ' '\n    im_h, im_w = img.shape[:2]\n    metric_boxes = []\n    if gen_submit_str and boxes is not None and len(boxes) > 0:\n        \n        score_sum = np.sum(scores)\n        pred_count = len(boxes)\n        if pred_count >= 1:\n            avg_conf = score_sum/pred_count\n            print(f'[Fusion] Total starfishs : {pred_count}, Average confidence : {avg_conf}')\n        else:\n            print(\"[Fusion] No starfish detected\")\n            \n        for idx, bb in enumerate(boxes):\n            xmin = bb[0]*im_w\n            ymin = bb[1]*im_h\n            xmax = bb[2]*im_w\n            ymax = bb[3]*im_h\n            w = xmax - xmin\n            h = ymax - ymin\n            score = round(scores[idx], 2)\n            ann += f'{score} {int(xmin)} {int(ymin)} {int(w)} {int(h)}'\n            metric_boxes.append(np.array([score,xmin,ymin,w,h]))\n            ann +=' '\n        print(f'fuse {ann}')\n\n        if draw_boxes:\n            img = draw_fused_boxes(img, boxes, scores, labels)\n\n    return dict(\n        img= img,\n        boxes = boxes,\n        scores= scores, \n        labels= labels,\n        metric_boxes=metric_boxes,\n        detect_count = len(boxes),\n        submission_str= ann.strip(' '))\n\ndef free():\n    import gc\n    import torch\n    import matplotlib.pyplot as plt\n    gc.collect()\n    if torch.cuda.is_available():\n        torch.cuda.empty_cache() \n        \ndef tf_save_image(img, fname):\n    path = f'/kaggle/working/{fname}.jpg'\n    enc = tf.image.encode_jpeg(img)\n    tf.io.write_file(tf.constant(path), enc)\n    return path","metadata":{"execution":{"iopub.status.busy":"2022-01-23T11:38:06.108853Z","iopub.execute_input":"2022-01-23T11:38:06.109160Z","iopub.status.idle":"2022-01-23T11:38:06.153482Z","shell.execute_reply.started":"2022-01-23T11:38:06.109124Z","shell.execute_reply":"2022-01-23T11:38:06.152795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model Helpers","metadata":{}},{"cell_type":"code","source":"# load model from yolov5\ndef load_model(model_path = \"/kaggle/working/best.pt\", conf = 0.40, iou = 0.5):\n    from yolov5 import YOLOv5\n\n    if not torch.cuda.is_available():\n        print('using cpu')\n        device = 'cpu'\n    else:\n        device = 'cuda:0'\n        print('using gpu')\n        \n    yolov5 = YOLOv5(model_path, device)\n    print('conf : ', conf, ' iou : ', iou)\n    yolov5.model.conf = conf\n    yolov5.model.iou = iou\n    yolov5.model.max_det = 1000\n    return yolov5\n\n# best score 0.502 with m1_conf=0.39, m1_iou=0.51, m2_conf=0.09, m2_iou=0.69, m3_conf=0.21, m3_iou=0.79\n# new best score 0.513 with m1_conf=0.10, m1_iou=0.51, m2_conf=0.09, m2_iou=0.41, m3_conf=0.10, m3_iou=0.51\ndef load_ensemble(m1_conf=0.10, m1_iou=0.51, m2_conf=0.09, m2_iou=0.41, m3_conf=0.10, m3_iou=0.51, m4_conf = 0.35, m4_iou=0.51):\n    with torch.no_grad():\n        m1 = load_model(conf=m1_conf, iou=m1_iou)\n        m2 = load_model('/kaggle/working/yolov5l_train3_last.pt', conf=m2_conf, iou=m2_iou)\n        m3 = load_model('/kaggle/working/yolov5l6_last.pt', conf=m3_conf, iou=m3_iou)\n\n        return m1, m2, m3\n\n# process prediction results\ndef process_preds(results, im_w = 1280, im_h=720, model_id = ''):\n    default_res = dict(\n        boxes= (),\n        scores= (), \n        labels= (), \n        detect_count= 0)\n    \n    res = results.pandas().xyxy\n    if not res:\n        return default_res\n    rp1 = res[0]\n    if rp1 is None or rp1.empty:\n        return default_res\n\n    del res\n    # normalize values for ensemble box func\n    rp1['xmin_normalized'] = rp1['xmin'] / im_w\n    rp1['ymin_normalized'] = rp1['ymin'] / im_h\n    rp1['xmax_normalized'] = rp1['xmax'] / im_w\n    rp1['ymax_normalized'] = rp1['ymax'] / im_h\n    rp1['w'] = rp1['xmax'] - rp1['xmin']\n    rp1['h'] = rp1['ymax'] - rp1['ymin']\n    bboxes  = rp1[['xmin_normalized','ymin_normalized','xmax_normalized','ymax_normalized']].values.astype(float)\n    scores = rp1.confidence.values\n    labels = rp1['class'].values\n    \n    idxs = np.where(scores >= 0.12)[0]\n    bboxes = bboxes[idxs]\n    scores = scores[idxs]\n    labels = labels[idxs]\n    \n    score_sum = np.sum(scores)\n    pred_count = len(scores)\n    if pred_count >= 1:\n        avg_conf = score_sum/pred_count\n        print(f'[{model_id}] Total starfishs : {pred_count}, Average confidence : {avg_conf}')\n    else:\n        print(f'[{model_id}] No starfish detected')\n    \n    return dict(\n        boxes= bboxes,\n        scores= scores, \n        labels= labels, \n        detect_count= pred_count)\n\n# predict helper\ndef predict(model, img, imgsz = 1280, draw_boxes = True, model_id = '', augment = True):\n    with torch.no_grad():\n        # yolo-pip package has \"predict\" function\n        # ultralytics/yolo has constructor predict\n        predict = getattr(model, \"predict\", None)\n\n        if callable(predict):\n            results = model.predict(img, size=imgsz, augment=augment)\n        else:\n            results = model(img, size=imgsz, augment=augment)\n\n        proc_res = process_preds(results, 1280, 720, model_id=model_id)\n        del results\n        if draw_boxes:\n            img = draw_fused_boxes(img, proc_res['boxes'], proc_res['scores'], proc_res['labels'])\n\n        proc_res['img'] = img\n        return proc_res\n\ndef submission_predict(model1, model2, model3, img, submit_mode = False, fuse_iou_thr=0.51, fuse_skip_box_thr=0.10):\n    free()\n    draw_boxes = not submit_mode\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n\n#     enhanced_image = enhance_image(img)\n\n    enhanced_image = tf_enhance_image_gpu(img)\n    \n    m1_res = predict(model1, img, imgsz=1280, draw_boxes = draw_boxes, model_id = 'Ensemble M1 TTA-False', augment=False)\n    m2_res = predict(model2, enhanced_image, imgsz=1280, draw_boxes = draw_boxes, model_id = 'Ensemble M2 TTA-False', augment=False)\n    m3_res = predict(model3, enhanced_image, imgsz=2048, draw_boxes = draw_boxes, model_id = 'Ensemble M3 TTA-False', augment=False)\n\n    del enhanced_image\n    free()\n\n    fusion_res = fuse_boxes(m2_res['img'], \\\n                [m1_res['boxes'],m2_res['boxes'],m3_res['boxes']], \\\n                [m1_res['scores'],m2_res['scores'],m3_res['scores']], \\\n                [m1_res['labels'],m2_res['labels'],m3_res['labels']], \\\n                            draw_boxes = draw_boxes, iou_thr=fuse_iou_thr, skip_box_thr=fuse_skip_box_thr)\n\n    if submit_mode:\n        return fusion_res['submission_str']\n    \n    del m1_res, m2_res, m3_res, img\n    return  fusion_res\n\ndef submission_predict_perf(model1, model2, model3, img, fuse_iou_thr=0.51, fuse_skip_box_thr=0.10):\n    return submission_predict(model1, model2, model3, img, submit_mode = True, fuse_iou_thr=fuse_iou_thr, fuse_skip_box_thr=fuse_skip_box_thr)","metadata":{"execution":{"iopub.status.busy":"2022-01-23T11:47:04.428759Z","iopub.execute_input":"2022-01-23T11:47:04.429309Z","iopub.status.idle":"2022-01-23T11:47:04.452485Z","shell.execute_reply.started":"2022-01-23T11:47:04.429268Z","shell.execute_reply":"2022-01-23T11:47:04.451477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calc_iou(bboxes1, bboxes2, bbox_mode='xywh'):\n    assert len(bboxes1.shape) == 2 and bboxes1.shape[1] == 4\n    assert len(bboxes2.shape) == 2 and bboxes2.shape[1] == 4\n    \n    bboxes1 = bboxes1.copy()\n    bboxes2 = bboxes2.copy()\n    \n    if bbox_mode == 'xywh':\n        bboxes1[:, 2:] += bboxes1[:, :2]\n        bboxes2[:, 2:] += bboxes2[:, :2]\n\n    x11, y11, x12, y12 = np.split(bboxes1, 4, axis=1)\n    x21, y21, x22, y22 = np.split(bboxes2, 4, axis=1)\n    xA = np.maximum(x11, np.transpose(x21))\n    yA = np.maximum(y11, np.transpose(y21))\n    xB = np.minimum(x12, np.transpose(x22))\n    yB = np.minimum(y12, np.transpose(y22))\n    interArea = np.maximum((xB - xA + 1), 0) * np.maximum((yB - yA + 1), 0)\n    boxAArea = (x12 - x11 + 1) * (y12 - y11 + 1)\n    boxBArea = (x22 - x21 + 1) * (y22 - y21 + 1)\n    iou = interArea / (boxAArea + np.transpose(boxBArea) - interArea)\n    return iou\n\ndef f_beta(tp, fp, fn, beta=2):\n    return (1+beta**2)*tp / ((1+beta**2)*tp + beta**2*fn+fp)\n\ndef calc_is_correct_at_iou_th(gt_bboxes, pred_bboxes, iou_th, verbose=False):\n    gt_bboxes = gt_bboxes.copy()\n    pred_bboxes = pred_bboxes.copy()\n    \n    tp = 0\n    fp = 0\n    for k, pred_bbox in enumerate(pred_bboxes): # fixed in ver.7\n        ious = calc_iou(gt_bboxes, pred_bbox[None, 1:])\n        max_iou = ious.max()\n        if max_iou > iou_th:\n            tp += 1\n            gt_bboxes = np.delete(gt_bboxes, ious.argmax(), axis=0)\n        else:\n            fp += 1\n        if len(gt_bboxes) == 0:\n            fp += len(pred_bboxes) - (k + 1) # fix in ver.7\n            break\n\n    fn = len(gt_bboxes)\n    return tp, fp, fn\n\ndef calc_is_correct(gt_bboxes, pred_bboxes, iou_th=0.5):\n    \"\"\"\n    gt_bboxes: (N, 4) np.array in xywh format\n    pred_bboxes: (N, 5) np.array in conf+xywh format\n    \"\"\"\n    if len(gt_bboxes) == 0 and len(pred_bboxes) == 0:\n        tps, fps, fns = 0, 0, 0\n        return tps, fps, fns\n\n    elif len(gt_bboxes) == 0:\n        tps, fps, fns = 0, len(pred_bboxes)*11, 0\n        return tps, fps, fns\n\n    elif len(pred_bboxes) == 0:\n        tps, fps, fns = 0, 0, len(gt_bboxes)*11\n        return tps, fps, fns\n\n    pred_bboxes = pred_bboxes[pred_bboxes[:,0].argsort()[::-1]] # sort by conf\n\n    tps, fps, fns = 0, 0, 0\n    tp, fp, fn = calc_is_correct_at_iou_th(gt_bboxes, pred_bboxes, iou_th)\n    tps += tp\n    fps += fp\n    fns += fn\n    return tps, fps, fns\n\ndef calc_f2_score(gt_bboxes_list, pred_bboxes_list, verbose=False):\n    \"\"\"\n    gt_bboxes_list: list of (N, 4) np.array in xywh format\n    pred_bboxes_list: list of (N, 5) np.array in conf+xywh format\n    \"\"\"\n    f2s = []\n    for iou_th in np.arange(0.3, 0.85, 0.05):\n        tps, fps, fns = 0, 0, 0\n        for gt_bboxes, pred_bboxes in zip(gt_bboxes_list, pred_bboxes_list):\n            tp, fp, fn = calc_is_correct(gt_bboxes, pred_bboxes, iou_th)\n            tps += tp\n            fps += fp\n            fns += fn\n            if verbose:\n                num_gt = len(gt_bboxes)\n                num_pred = len(pred_bboxes)\n                print(f'num_gt:{num_gt:<3} num_pred:{num_pred:<3} tp:{tp:<3} fp:{fp:<3} fn:{fn:<3}')\n        f2 = f_beta(tps, fps, fns, beta=2)    \n        print(f'f2@ iou: {round(iou_th,2)}, f2 : {round(f2,2)}, f2(with comp difference) : {round(f2-0.18, 2)}')\n        f2s.append(f2)\n        \n    f2mean = np.mean(f2s)\n    print(f'f2 mean is {f2mean}, with competition difference {f2mean-0.178}')\n    return f2mean","metadata":{"execution":{"iopub.status.busy":"2022-01-23T11:39:13.285832Z","iopub.execute_input":"2022-01-23T11:39:13.286125Z","iopub.status.idle":"2022-01-23T11:39:13.308300Z","shell.execute_reply.started":"2022-01-23T11:39:13.286092Z","shell.execute_reply":"2022-01-23T11:39:13.307324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def convert_to_cxywh(bboxes):\n#     predictions = []\n#     detects = []\n    \n#     for i in range(len(bboxes)):\n#         box = bboxes[i]\n#         score = round(scores[i],2)\n#         x_min = int(box[0])\n#         y_min = int(box[1])\n#         x_max = int(box[2])\n#         y_max = int(box[3])\n        \n#         bbox_width = x_max - x_min\n#         bbox_height = y_max - y_min\n#         detects.append([score, x_min, y_min, bbox_width, bbox_height])\n#         predictions.append('{:.2f} {} {} {} {}'.format(score, x_min, y_min, bbox_width, bbox_height))\n#         return detects, ' '.join(predictions)\n    \n# def show_prediction(img, bboxes, scores):\n\n#     for box, score in zip(bboxes, scores):\n#         cv2.rectangle(img, (int(box[0]), int(box[1])), (int(box[2]), int(box[3])), (0,0,255), 2)\n#         cv2.putText(img, f'{score:.2f}', (int(box[0]), int(box[1]-3)), cv2.FONT_HERSHEY_SIMPLEX, 0.4, (0,0,255), 1, cv2.LINE_AA)\n    \n#     img = img[:,:,::-1]\n#     img = Image.fromarray(img).resize((1280, 720))\n#     return img\n\n# def load_yolor_model(model_path, cfg_path, img_size, device):\n#     with torch.no_grad():\n#         model = Darknet(cfg_path).cuda()\n#         model.load_state_dict(torch.load(model_path, map_location=device)['model'])\n#         model.to(device).eval()\n#         model.half()\n\n#         init = torch.zeros((1, 3, img_size, img_size), device=device)\n#         _ = model(init.half()) if device.type != 'cpu' else None \n#         return model\n\n# def predict_img(model, img_in, img_size, conf_thres, iou_thres, device):\n#     with torch.no_grad():\n#         bboxes = []\n#         scores = []\n#         classes = []\n\n#         img = letterbox(img_in, new_shape=img_size, auto_size=64)[0]\n#         img = np.ascontiguousarray(img[:, :, ::-1].transpose(2, 0, 1))\n#         img = (torch.from_numpy(img).to(device).half() / 255.0).unsqueeze(0)\n\n#         pred = model(img)[0]\n\n#         if pred is not None:\n#             pred = non_max_suppression(pred, conf_thres=conf_thres, iou_thres=iou_thres, merge=False, classes=None, agnostic=False)\n\n#         for i, det in enumerate(pred):\n#             if det is not None and len(det):\n#                 det[:, :4] = scale_coords(img.shape[2:], det[:, :4], img_in.shape).round()    \n\n#         det = det.cpu().detach().numpy()\n\n#         bboxes = det[:, :4] \n#         scores = det[:, 4] \n#         classes = det[:, 5]\n        \n#         norm_boxes = []\n#         for i in range(len(bboxes)):\n#             box = bboxes[i]\n#             x_min = round(box[0]/img_in.shape[1],4)\n#             y_min = round(box[1]/img_in.shape[0],4)\n#             x_max = round(box[2]/img_in.shape[1],4)\n#             y_max = round(box[3]/img_in.shape[0],4)\n#             norm_boxes.append([x_min, y_min, x_max, y_max])\n        \n#         return bboxes, scores, classes, norm_boxes","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# device = select_device('0')\n# model = load_yolor_model('./yolor_w6.pt', './yolor/cfg/yolor_w6.cfg', 6400, device)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# img = cv2.imread('../input/tensorflow-great-barrier-reef/train_images/video_0/9651.jpg')\n# # img = cv2.imread('../input/tensorflow-great-barrier-reef/train_images/video_0/9700.jpg')\n# # # img = cv2.imread('../input/tensorflow-great-barrier-reef/train_images/video_0/9674.jpg')\n# img = enhance_image(img)\n# bboxes, scores, classes, norm_boxes = predict_img(model, img, 6400, conf_thres = 0.01, iou_thres = 0.3, device = device)\n# display(show_prediction(img, bboxes, scores))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def timereps(func, reps = 1):\n#     from time import time\n#     start = time()\n#     for i in range(0, reps):\n#         func()\n#     end = time()\n#     return (end - start) / reps","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# del model\n# free()\n# model = load_model('/kaggle/working/yolov5l6_last.pt', conf=0.15, iou=0.51)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# img = cv2.imread('../input/tensorflow-great-barrier-reef/train_images/video_2/5803.jpg')\n# # # # img = cv2.imread('../input/tensorflow-great-barrier-reef/train_images/video_0/19.jpg')\n# # # # img = cv2.imread('../input/tensorflow-great-barrier-reef/train_images/video_0/12.jpg')\n# # # free()\n# # # # model1, model2, model3 = load_ensemble()\n# # # # def infer():\n# # # #     print(submission_predict_perf(model1, model2, model3, img))\n    \n# # # # # timereps(infer)\n# # # # # infer()\n# # # # del model1, model2, model3\n# # # # print('before ', img)\n# # img = enhance_image(img)\n# img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n# pred = predict(model, img, 2048)\n# fusion_res = fuse_boxes(img,[pred['boxes']], [pred['scores']], [pred['labels']], draw_boxes = True, iou_thr=0.3, skip_box_thr=0.05)\n# display(Image.fromarray(fusion_res['img']))\n# # # cv2.normalize(img, img, 0, 255, cv2.NORM_MINMAX)\n\n# # # # print('after ', img)\n# # # pred = predict(model, img, enhance_img=True)\n# # # fusion_res = fuse_boxes(img,[pred['boxes']], [pred['scores']], [pred['labels']], draw_boxes = True)\n# # # # display(Image.fromarray(fusion_res['img']))\n\n# # # # fusion_res['submission_str']\n# # # # print('res ', fusion_res['submission_str'])\n# # # del model\n# # # free()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Run infer on train samples","metadata":{}},{"cell_type":"code","source":"import sys\nsys.path.append('../input/weightedboxesfusion')\nfrom ensemble_boxes import *\n\ndef show_original(img, annotations):\n    og_bb_list, m_boxes = process_annot(annotations)\n    return drawBoundingBoxes(img, og_bb_list), og_bb_list\n\ndef infer_samples(benchmark_mode = False):\n    from tqdm import tqdm\n    # calculate the f2-measure\n    from sklearn.metrics import fbeta_score\n    from sklearn.metrics import f1_score\n    from sklearn.metrics import precision_score\n    from sklearn.metrics import recall_score\n    import matplotlib.pyplot as plt\n    %matplotlib inline\n    free()\n\n    os.chdir('/kaggle/working/')\n\n    m1_sc = 0.0\n    m2_sc = 0.0\n    labels = get_sample_imgs(1000)[:40]\n    print(len(labels.values))\n    \n#     model = load_model('/kaggle/working/yolov5l6_last.pt', conf=0.05, iou=0.3)\n#     model2 = load_model('/kaggle/working/yolov5l6_last.pt', conf=0.15, iou=0.45)\n#     model1 = load_model('/kaggle/working/best.pt', conf=0.12, iou=0.3)\n\n#     model1, model2, model3 = load_ensemble(m1_conf=0.15, m1_iou=0.51, m2_conf=0.09, m2_iou=0.51, m3_conf=0.19, m3_iou=0.51)\n#     model1, model2, model3 = load_ensemble(m1_conf=0.32, m1_iou=0.51, m2_conf=0.09, m2_iou=0.51, m3_conf=0.19, m3_iou=0.51)\n#     model1, model2, model3 = load_ensemble(m1_conf=0.12, m1_iou=0.51, m2_conf=0.09, m2_iou=0.45, m3_conf=0.18, m3_iou=0.51)\n\n    # F2 scores over 3 iterations are : [0.7, 0.6500000000000001, 0.7]\n    model1, model2, model3 = load_ensemble(m1_conf=0.05, m1_iou=0.30, m2_conf=0.09, m2_iou=0.40, m3_conf=0.09, m3_iou=0.40)\n#     model3 = load_model('/kaggle/working/yolov5l6_last.pt', conf=0.12, iou=0.50)\n    if not benchmark_mode:\n        g_ytrue = []\n        m1_ypred = []\n        m2_ypred = []\n    \n    gt_preds = []\n    x_preds = []\n    for i, la in tqdm(labels.iterrows(), total=len(labels.values)):\n        path = la['image_path']\n        annotations = la['annotations']\n        if os.path.exists(path) and os.path.isfile(path):\n            img = cv2.imread(path)\n#             img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n\n            if benchmark_mode:\n                print(submission_predict_perf(model1, model2, model3, img))\n            else:\n                img1, og_bb_list = show_original(img, annotations)\n\n                # 0.497 score with 0.55, 0.40\n                # 0.520 score with fuse_iou_thr=0.50, fuse_skip_box_thr=0.19 [0.725, 0.625, 0.725]\n                # with iou=0.55 & skip_conf=0.40, f2 over 3 iter [0.775, 0.775, 0.8] (2nd run [0.825, 0.925, 0.775])\n                # with 0.45, 0.35 [0.7, 0.825, 0.675]\n                # with 0.45, 0.40 [0.8,0.8,0.7],[0.775, 0.675, 0.650]\n                fusion_res = submission_predict(model1, model2, model3, img, fuse_iou_thr=0.65, fuse_skip_box_thr=0.09)\n\n                og_bb_list, m_boxes = process_annot(annotations)\n                \n                for x in m_boxes:\n                    gt_preds.append(x)\n                    \n                for x in fusion_res['metric_boxes']:\n                    x_preds.append(x)\n                \n                g_ytrue.append(len(og_bb_list))\n                print('Ground Truth ', len(og_bb_list))\n#                 m1_ypred.append(m1_res['detect_count'])\n#                 m1_ypred.append(len(norm_boxes))\n#                 m1_ypred.append(detect_count)\n                m2_ypred.append(fusion_res['detect_count'])\n#                 display(Image.fromarray(fusion_res['img']))\n#                 plot_images(img1, m1_res['img'], fusion_res['img'], 'Image Comparison',\n#                             f'Ground Truth \\n{path}', 'Single Model', 'Ensembled Results')\n                del img, img1, fusion_res, og_bb_list\n    \n                plt.show()\n                \n        del path, annotations\n\n    free()\n    if not benchmark_mode:\n        gt_preds = np.array(gt_preds)\n        x_preds = np.array(x_preds)\n        \n        print('G1 ytrue', g_ytrue)\n#         print('m1 ypred', m1_ypred)\n        print('m2 ypred', m2_ypred)\n\n#         m1_sc = fbeta_score(g_ytrue, m1_ypred, average='micro', beta=2.0)\n        m2_sc = fbeta_score(g_ytrue, m2_ypred, average='micro', beta=2.0)\n#         print('F2 Beta Sore for Model 1 : ', m1_sc)\n        print('F2 Beta Score for Ensemble: ', m2_sc)\n#         del g_ytrue, m1_ypred, m2_ypred\n        calc_f2_score([gt_preds], [x_preds], verbose=False)\n    \n#     del labels, model1, model2, model3\n    return m2_sc\n\n# timereps(infer_samples)\n# infer_samples()\n\n# n_iter = 3\n# f2_scores = []\n\n# for i in range(n_iter):\n#     f2_scores.append(infer_samples())\n    \n# print(f'F2 scores over {n_iter} iterations are : {f2_scores}')","metadata":{"_kg_hide-input":false,"scrolled":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-01-23T11:51:28.378247Z","iopub.execute_input":"2022-01-23T11:51:28.378666Z","iopub.status.idle":"2022-01-23T11:52:03.243560Z","shell.execute_reply.started":"2022-01-23T11:51:28.378626Z","shell.execute_reply":"2022-01-23T11:52:03.242869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Submission","metadata":{}},{"cell_type":"code","source":"# free()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import greatbarrierreef\n\ndef create_submission(disable_make_env = False):\n    from IPython.display import display\n    os.chdir('/kaggle/working/')\n    free()\n\n#     model1, model2, model3 = load_ensemble(m1_conf=0.32, m1_iou=0.51, m2_conf=0.09, m2_iou=0.51, m3_conf=0.19, m3_iou=0.51)\n#     model1, model2, model3 = load_ensemble(m1_conf=0.09, m1_iou=0.51, m2_conf=0.09, m2_iou=0.51, m3_conf=0.12, m3_iou=0.51)\n    \n#     model1, model2, model3 = load_ensemble(m1_conf=0.05, m1_iou=0.3, m2_conf=0.05, m2_iou=0.3, m3_conf=0.05, m3_iou=0.3)\n    \n#     model = load_model('/kaggle/working/yolov5l6_last.pt', conf=0.05, iou=0.3)\n#     model2 = load_model('/kaggle/working/yolov5l6_last.pt', conf=0.15, iou=0.45)\n#     model1 = load_model('/kaggle/working/best.pt', conf=0.12, iou=0.3)\n\n    model1, model2, model3 = load_ensemble(m1_conf=0.05, m1_iou=0.30, m2_conf=0.09, m2_iou=0.40, m3_conf=0.09, m3_iou=0.40)\n\n    if not disable_make_env:\n        env = greatbarrierreef.make_env() \n        \n    iter_test = env.iter_test()\n    for img, pred_df in iter_test:\n        free()\n        \n        fusion_res = submission_predict(model1, model2, model3, img, fuse_iou_thr=0.65, fuse_skip_box_thr=0.09)\n\n        print('predicted annotation ',  fusion_res['submission_str'])\n        pred_df['annotations'] = fusion_res['submission_str']\n        \n        env.predict(pred_df)\n        free()\n    \n#     del model1, model2, model3\n    gc.collect()\n    \n# run create_submission\ncreate_submission()","metadata":{"execution":{"iopub.status.busy":"2022-01-23T11:54:31.767503Z","iopub.execute_input":"2022-01-23T11:54:31.767780Z","iopub.status.idle":"2022-01-23T11:54:39.374657Z","shell.execute_reply.started":"2022-01-23T11:54:31.767751Z","shell.execute_reply":"2022-01-23T11:54:39.373894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.read_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-01-23T11:54:44.502805Z","iopub.execute_input":"2022-01-23T11:54:44.503093Z","iopub.status.idle":"2022-01-23T11:54:44.518990Z","shell.execute_reply.started":"2022-01-23T11:54:44.503060Z","shell.execute_reply":"2022-01-23T11:54:44.518335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# cleanup\n!rm /kaggle/working/best.pt\n!rm /kaggle/working/yolov5l_train3_last.pt\n!rm /kaggle/working/yolov5l6_last.pt\n!rm /kaggle/working/yolov5s6_best.pt","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}