{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## 1. Imports","metadata":{}},{"cell_type":"code","source":"import os\nimport cv2\nimport ast\nimport yaml\nimport torch\nimport shutil\nimport random\nimport numpy as np\nimport pandas as pd\nfrom joblib import Parallel, delayed\nfrom sklearn.model_selection import GroupKFold\n\nfrom tqdm.notebook import tqdm\ntqdm.pandas()\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport sys\nsys.path.append('../input/yolov5ds')\nsys.path.append('../input/hyperparameters-for-yolov5')\nsys.path.append('../input/tensorflow-great-barrier-reef')\n\nimport utils\ntqdm.pandas()\n\n%pip install -q wandb\n%pip install wandb --upgrade\nimport wandb\nwandb.login(key=\"f04c0b8d3b383666c2518b204435adcb3f9532e9\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. Configuration","metadata":{}},{"cell_type":"code","source":"def random_seed(seed):\n    np.random.seed(seed)\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed_all(seed)\n        torch.backends.cudnn.deterministic = True\n        \nrandom_seed(1702)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FOLD = 2\nIMAGE_WIDTH = 1280\nIMAGE_HEIGHT = 720\nREMOVE_NOBBOX = True \nNUM_BACKGROUND_IMG = 983 # ~20% \n\nCWD = '/kaggle/working/'\nIMAGE_DIR = '/kaggle/images' \nLABEL_DIR = '/kaggle/labels' \nROOT_DIR = '/kaggle/input/tensorflow-great-barrier-reef/'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p {IMAGE_DIR}\n!mkdir -p {LABEL_DIR}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. Helpers\n\nI don't write any code in this section.","metadata":{}},{"cell_type":"code","source":"def coco2yolo(image_width, image_height, bboxes):\n    bboxes = bboxes.copy().astype(float) \n    \n    # normalize\n    bboxes[..., [0, 2]] = bboxes[..., [0, 2]] / image_width\n    bboxes[..., [1, 3]] = bboxes[..., [1, 3]] / image_height\n    \n    # gets xmid and ymid \n    bboxes[..., [0, 1]] = bboxes[..., [0, 1]] + bboxes[..., [2, 3]] / 2\n    \n    return bboxes\n\ndef get_bbox(annots):\n    # converts from dictionary to list \n    # formart after converting: [x, y, width, height]\n    \n    bboxes = [list(annot.values()) for annot in annots]\n    return bboxes","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4. Dataset","metadata":{}},{"cell_type":"code","source":"def get_path(row):\n    row['old_image_path'] = f'{ROOT_DIR}/train_images/video_{row.video_id}/{row.video_frame}.jpg'\n    row['image_path'] = f'{IMAGE_DIR}/video_{row.video_id}_{row.video_frame}.jpg'\n    row['label_path'] = f'{LABEL_DIR}/video_{row.video_id}_{row.video_frame}.txt'\n    \n    return row\n\ndf = pd.read_csv(f'{ROOT_DIR}/train.csv')\ndf = df.progress_apply(get_path, axis=1)\ndf['annotations'] = df['annotations'].progress_apply(lambda x: ast.literal_eval(x))\ndf['num_bbox'] = df['annotations'].progress_apply(lambda x: len(x))\ndf['bboxes'] = df.annotations.progress_apply(get_bbox)\n\ndf.head(2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# gets background images\ndf_empty_bbox = df[df[\"num_bbox\"] == 0]\ndf_empty_bbox = df_empty_bbox.sample(frac=1).reset_index(drop=True).iloc[:NUM_BACKGROUND_IMG,]\n\n# gets images with objects\ndf = df[df[\"num_bbox\"] > 0]\n\n# concats background images and image with objects\ndf = pd.concat([df, df_empty_bbox], ignore_index=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# copy from the original path to kaggle/working \n# because the models requires folder that can be written data on\n\ndef make_copy(path):\n    data = path.split('/')\n    filename = data[-1]\n    video_id = data[-2]\n    new_path = os.path.join(IMAGE_DIR, f'{video_id}_{filename}')\n    shutil.copy(path, new_path)\n    return\n\n# using Parrallel for faster copying \nimage_paths = df.old_image_path.tolist()\n_ = Parallel(n_jobs=-1, backend='threading')(delayed(make_copy)(path) for path in tqdm(image_paths))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = df[df.video_id != FOLD]\nvalid_df = df[df.video_id == FOLD]\n\ntrain_df.shape[0], valid_df.shape[0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 5. Files required for YOLOv5\n\nMore details about the requirements can be found [here](https://github.com/ultralytics/yolov5/wiki/Train-Custom-Data).","metadata":{}},{"cell_type":"code","source":"# dataset.yaml for YOLOv5\n\nwith open(os.path.join(CWD, 'train.txt'), 'w') as f:\n    for path in train_df.image_path.tolist():\n        f.write(path + '\\n')\n            \nwith open(os.path.join(CWD, 'val.txt'), 'w') as f:\n    for path in valid_df.image_path.tolist():\n        f.write(path + '\\n')\n\ndata = dict(\n    path  = CWD,\n    train =  os.path.join(CWD, 'train.txt'),\n    val   =  os.path.join(CWD, 'val.txt' ),\n    nc    = 1,\n    names = ['cots'],\n    )\n\nwith open(os.path.join(CWD, 'starfish.yaml'), 'w') as outfile:\n    yaml.dump(data, outfile, default_flow_style=False)\n\nf = open(os.path.join(CWD, 'starfish.yaml'), 'r')\nprint('\\nyaml:')\nprint(f.read())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# labels for YOLOv5\n\nfor row_idx in tqdm(range(df.shape[0])):\n    row = df.iloc[row_idx]\n    bboxes_coco = np.array(row.bboxes).astype(np.float32).copy()\n    num_bbox = row.num_bbox\n    names = ['cots'] * num_bbox\n    labels = [0] * num_bbox\n\n    with open(row.label_path, 'w') as f:\n        if num_bbox < 1:\n            annot = ''\n            f.write(annot)\n            continue\n            \n        bboxes_yolo  = coco2yolo(IMAGE_WIDTH, IMAGE_HEIGHT, bboxes_coco)\n        bboxes_yolo  = np.clip(bboxes_yolo, 0, 1)\n        \n        for bbox_idx in range(len(bboxes_yolo)):\n            label = [str(labels[bbox_idx])]\n            bboxes = list(bboxes_yolo[bbox_idx].astype(str))\n            new_line = (['\\n'] if num_bbox != (bbox_idx + 1) else [''])\n            \n            annot =  label + bboxes + new_line\n            annot = ' '.join(annot)\n            annot = annot.strip(' ')\n            \n            f.write(annot)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 6. Training","metadata":{}},{"cell_type":"code","source":"%cd /kaggle/working\n!rm -r /kaggle/working/yolov5\n!cp -r ../input/yolov5ds /kaggle/working/yolov5\n%cd yolov5\n\n!python train.py --img 1280\\\n--batch 10\\\n--epochs 20\\\n--data /kaggle/working/starfish.yaml\\\n--weights yolov5m.pt --workers 0\\\n--adam\\\n--save-period 1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def voc2yolo(bboxes, image_height=720, image_width=1280):\n    bboxes = bboxes.copy().astype(float) \n    \n    bboxes[..., [0, 2]] = bboxes[..., [0, 2]]/ image_width\n    bboxes[..., [1, 3]] = bboxes[..., [1, 3]]/ image_height\n    \n    w = bboxes[..., 2] - bboxes[..., 0]\n    h = bboxes[..., 3] - bboxes[..., 1]\n    \n    bboxes[..., 0] = bboxes[..., 0] + w/2\n    bboxes[..., 1] = bboxes[..., 1] + h/2\n    bboxes[..., 2] = w\n    bboxes[..., 3] = h\n    \n    return bboxes\n\ndef yolo2coco(bboxes, image_height=720, image_width=1280):\n    bboxes = bboxes.copy().astype(float) \n    \n    bboxes[..., [0, 2]]= bboxes[..., [0, 2]] * image_width\n    bboxes[..., [1, 3]]= bboxes[..., [1, 3]] * image_height\n    \n    bboxes[..., [0, 1]] = bboxes[..., [0, 1]] - bboxes[..., [2, 3]] / 2\n    \n    return bboxes\n\ndef voc2coco(bboxes, image_height=720, image_width=1280):\n    bboxes  = voc2yolo(bboxes, image_height, image_width)\n    bboxes  = yolo2coco(bboxes, image_height, image_width)\n    \n    return bboxes\n\ndef predict(model, img, size=768, augment=False):\n    height, width = img.shape[:2]\n    results = model(img, size=size, augment=augment)  \n    preds   = results.pandas().xyxy[0]\n    bboxes  = preds[['xmin','ymin','xmax','ymax']].values\n    \n    if len(bboxes):\n        bboxes  = voc2coco(bboxes,height,width).astype(int)\n        confs   = preds.confidence.values\n        return bboxes, confs\n    else:\n        return [],[]\n    \ndef format_prediction(bboxes, confs):\n    annot = ''\n    \n    if len(bboxes)>0:\n        for idx in range(len(bboxes)):\n            xmin, ymin, w, h = bboxes[idx]\n            conf             = confs[idx]\n            annot += f'{conf} {xmin} {ymin} {w} {h}'\n            annot +=' '\n            \n        annot = annot.strip(' ')\n        \n    return annot","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calc_iou(bboxes1, bboxes2, bbox_mode='xywh'):\n    assert len(bboxes1.shape) == 2 and bboxes1.shape[1] == 4\n    assert len(bboxes2.shape) == 2 and bboxes2.shape[1] == 4\n    \n    bboxes1 = bboxes1.copy()\n    bboxes2 = bboxes2.copy()\n    \n    if bbox_mode == 'xywh':\n        bboxes1[:, 2:] += bboxes1[:, :2]\n        bboxes2[:, 2:] += bboxes2[:, :2]\n\n    x11, y11, x12, y12 = np.split(bboxes1, 4, axis=1)\n    x21, y21, x22, y22 = np.split(bboxes2, 4, axis=1)\n    xA = np.maximum(x11, np.transpose(x21))\n    yA = np.maximum(y11, np.transpose(y21))\n    xB = np.minimum(x12, np.transpose(x22))\n    yB = np.minimum(y12, np.transpose(y22))\n    interArea = np.maximum((xB - xA + 1), 0) * np.maximum((yB - yA + 1), 0)\n    boxAArea = (x12 - x11 + 1) * (y12 - y11 + 1)\n    boxBArea = (x22 - x21 + 1) * (y22 - y21 + 1)\n    iou = interArea / (boxAArea + np.transpose(boxBArea) - interArea)\n    return iou\n\ndef f_beta(tp, fp, fn, beta=2):\n    return (1+beta**2)*tp / ((1+beta**2)*tp + beta**2*fn+fp)\n\ndef imagewise_f2_score_at_iou_th(gt_bboxes, pred_bboxes, iou_th, verbose=False):\n    gt_bboxes = gt_bboxes.copy()\n    pred_bboxes = pred_bboxes.copy()\n\n    tp = 0\n    fp = 0\n    for k, pred_bbox in enumerate(pred_bboxes): # fixed in ver.7\n        ious = calc_iou(gt_bboxes, pred_bbox[None, 1:])\n        max_iou = ious.max()\n        if max_iou > iou_th:\n            tp += 1\n            gt_bboxes = np.delete(gt_bboxes, ious.argmax(), axis=0)\n        else:\n            fp += 1\n        if len(gt_bboxes) == 0:\n            fp += len(pred_bboxes) - (k + 1) # fix in ver.7\n            break\n\n    fn = len(gt_bboxes)\n    score = f_beta(tp, fp, fn, beta=2)\n    \n    return score\n\n\n\ndef imagewise_f2_score(gt_bboxes, pred_bboxes, verbose=False):\n    \"\"\"\n    gt_bboxes: (N, 4) np.array in xywh format\n    pred_bboxes: (N, 5) np.array in conf+xywh format\n    \"\"\"\n    # v2: add corner case hundling.\n    if len(gt_bboxes) == 0 and len(pred_bboxes) == 0:\n        return 1.0\n    elif len(gt_bboxes) == 0 or len(pred_bboxes) == 0:\n        return 0.0\n    \n    pred_bboxes = pred_bboxes[pred_bboxes[:,0].argsort()[::-1]] # sort by conf\n    \n    scores = []\n    for iou_th in np.arange(0.3, 0.85, 0.05):\n        scores.append(imagewise_f2_score_at_iou_th(gt_bboxes, pred_bboxes, iou_th, verbose))\n    return np.mean(scores)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p /root/.config/Ultralytics\n!cp /kaggle/input/yolov5-font/Arial.ttf /root/.config/Ultralytics/","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IOU = 0.35\nCONF = 0.15\nIMG_SIZE = 1280\nAUGMENT = False\n\nCKPT_PATH = '../input/starfish/epoch19.pt'\nROOT_DIR  = '/kaggle/input/tensorflow-great-barrier-reef/'","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_model(ckpt_path, conf=0.25, iou=0.50):\n    model = torch.hub.load('/kaggle/input/yolov5-lib-ds',\n                           'custom',\n                           path=ckpt_path,\n                           source='local',\n                           force_reload=True)  \n    model.conf = conf  \n    model.iou  = iou  \n    model.classes = None \n    model.multi_label = False  \n    model.max_det = 1000 \n    \n    return model\n\nmodel = load_model(CKPT_PATH, conf=CONF, iou=IOU)\ngt_bboxes_list = []\npred_bboxes_list = []\n\nfor idx in range(valid_df.shape[0]):\n    row = valid_df.iloc[idx]\n    \n    bboxes, confs = predict(model, row[\"old_image_path\"], size=IMG_SIZE, augment=AUGMENT)\n    annot = format_prediction(bboxes, confs)\n    pred_df['annotations'] = annot\n    \n    gt_bboxes_list.append(np.asarray(bbox, dtype=\"float64\"))\n    pred_bboxes_list.append(np.asarray(pred_bboxes.split(\" \"), dtype=\"float64\"))\n    ","metadata":{},"execution_count":null,"outputs":[]}]}