{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Merging public bounding box datasets 👨‍🔬\nA lot of public notebooks / datasets are providing labeled images to crop the dataset. Let's merge them together !<br>\nWe may either want to use only the manual annotations to fine-tune a YOLOv5 model, or use all of them to directly create a cropped dataset\n\n## Sources for this notebook\n* https://www.kaggle.com/bsridatta/happierwhale\n* https://www.kaggle.com/phalanx/whale2-cropped-dataset\n* https://www.kaggle.com/awsaf49/happywhale-boundingbox-yolov5\n* https://www.kaggle.com/yusukesueyoshi/happy-whale-and-dolphin-anotated-in-yolov5-format\n* http://happywhale.theoboyer.fr/data/dataset.csv\n* https://www.kaggle.com/jpbremer/fullbodywhaleannotations","metadata":{}},{"cell_type":"markdown","source":"<div style=\"background-color: #c75a5a; border-radius: 10px; text-align: center; padding: 25px\">\n        Previous versions were wrong ! I fixed the interpretation of the yolo coordinates in the last one\n</div>","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport json\nimport matplotlib.pyplot as plt\nimport cv2\nfrom random import random","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-03-06T10:59:39.375143Z","iopub.execute_input":"2022-03-06T10:59:39.375972Z","iopub.status.idle":"2022-03-06T10:59:39.734392Z","shell.execute_reply.started":"2022-03-06T10:59:39.375875Z","shell.execute_reply":"2022-03-06T10:59:39.733661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_data = {\n    \"image\": [],\n    \"xmin\": [],\n    \"ymin\": [],\n    \"xmax\": [],\n    \"ymax\": [],\n    \"source\": [],\n    \"type\": []\n}\n\ndef submit_annotation(image, x1, y1, x2, y2, source, i_type=\"train\", only_if_new=False):\n    if x1 == x2 or y1 == y2 or (only_if_new and image in total_data[\"image\"]):\n        return\n    #assert x1 >= 0\n    #assert x2 >= 0\n    #assert y1 <= 1\n    #assert y2 <= 1\n    total_data[\"image\"].append(image)\n    total_data[\"xmin\"].append(min(x1, x2))\n    total_data[\"ymin\"].append(min(y1, y2))\n    total_data[\"xmax\"].append(max(x1, x2))\n    total_data[\"ymax\"].append(max(y1, y2))\n    total_data[\"source\"].append(source)\n    total_data[\"type\"].append(i_type)\n    \ndef plot_bbox(image, x1, y1, x2, y2):\n    path = os.path.join(\"../input/happy-whale-and-dolphin/train_images\", image)\n    image = cv2.imread(path)\n    xmin = int(min(x1, x2) * image.shape[1])\n    ymin = int(min(y1, y2) * image.shape[0])\n    xmax = int(max(x1, x2) * image.shape[1])\n    ymax = int(max(y1, y2) * image.shape[0])\n    image = cv2.rectangle(\n        image,\n        (xmin, ymin),\n        (xmax, ymax),\n        (0,255,255),\n        3\n    )\n    plt.imshow(image[:, :, ::-1])\n    plt.show()\n    \ndef plot_bbox_dic(x):\n    plot_bbox(\n        x[\"image\"],\n        x[\"xmin\"],\n        x[\"ymin\"],\n        x[\"xmax\"],\n        x[\"ymax\"]\n    )\n    \ndef dump_dataset():\n    return pd.DataFrame(total_data)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-03-06T10:59:39.736435Z","iopub.execute_input":"2022-03-06T10:59:39.736782Z","iopub.status.idle":"2022-03-06T10:59:39.752823Z","shell.execute_reply.started":"2022-03-06T10:59:39.736732Z","shell.execute_reply":"2022-03-06T10:59:39.751785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = \"../input/extra-happywhale-metadata\"\ndata = pd.read_csv(os.path.join(path, \"train.csv\"), index_col=\"image\")\nTRAIN_IMG_HEIGHT = data[\"img_height\"].to_dict()\nTRAIN_IMG_WIDTH = data[\"img_width\"].to_dict()\ndata = pd.read_csv(os.path.join(path, \"test.csv\"), index_col=\"image\")\nTEST_IMG_HEIGHT = data[\"img_height\"].to_dict()\nTEST_IMG_WIDTH = data[\"img_width\"].to_dict()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-03-06T10:59:39.753973Z","iopub.execute_input":"2022-03-06T10:59:39.754215Z","iopub.status.idle":"2022-03-06T10:59:40.320371Z","shell.execute_reply.started":"2022-03-06T10:59:39.754186Z","shell.execute_reply":"2022-03-06T10:59:40.319673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = \"../input/happierwhale/yolo_annotations/yolo_annotations\"\nfor f in os.listdir(path):\n    with open(os.path.join(path, f), 'r') as file:\n        d = file.read()\n    if len(d):\n        _, x, y, w, h = tuple(map(float, d.split(\"\\n\")[0].split(\" \")))\n        x1 = x - w / 2\n        y1 = y - h / 2\n        x2 = x + w / 2\n        y2 = y + h / 2\n        submit_annotation(f[:-3] + \"jpg\", x1, y1, x2, y2, \"happierwhale\")","metadata":{"execution":{"iopub.status.busy":"2022-03-06T10:59:40.321973Z","iopub.execute_input":"2022-03-06T10:59:40.322314Z","iopub.status.idle":"2022-03-06T10:59:48.401850Z","shell.execute_reply.started":"2022-03-06T10:59:40.322284Z","shell.execute_reply":"2022-03-06T10:59:48.401102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = \"../input/happy-whale-and-dolphin-anotated-in-yolov5-format/whale/\"\nlpath = os.path.join(path, \"train\", \"labels\")\nfor f in os.listdir(lpath):\n    with open(os.path.join(lpath, f), 'r') as file:\n        d = file.read()\n    if len(d):\n        _, x, y, w, h = tuple(map(float, d.split(\"\\n\")[0].split(\" \")))\n        x1 = x - w / 2\n        y1 = y - h / 2\n        x2 = x + w / 2\n        y2 = y + h / 2\n        submit_annotation(f.split(\"_\")[0] + \".jpg\", x1, y1, x2, y2, \"manual_yolo\")\n\nlpath = os.path.join(path, \"valid\", \"labels\")\nfor f in os.listdir(lpath):\n    with open(os.path.join(lpath, f), 'r') as file:\n        d = file.read()\n    if len(d):\n        _, x, y, w, h = tuple(map(float, d.split(\"\\n\")[0].split(\" \")))\n        x1 = x - w / 2\n        y1 = y - h / 2\n        x2 = x + w / 2\n        y2 = y + h / 2\n        submit_annotation(f.split(\"_\")[0] + \".jpg\", x1, y1, x2, y2, \"manual_yolo\")","metadata":{"execution":{"iopub.status.busy":"2022-03-06T10:59:48.403035Z","iopub.execute_input":"2022-03-06T10:59:48.403391Z","iopub.status.idle":"2022-03-06T10:59:49.013494Z","shell.execute_reply.started":"2022-03-06T10:59:48.403363Z","shell.execute_reply":"2022-03-06T10:59:49.012644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = \"../input/happywhales-labelme-segmentation-dataset\"\n\nwith open(os.path.join(path, \"train.json\")) as f:\n    data = json.load(f)\n    \nfor image, annotation in zip(data[\"images\"], data[\"annotations\"]):\n    bbox = annotation[\"bbox\"]\n    x1 = bbox[0] / image[\"width\"]\n    y1 = bbox[1] / image[\"height\"]\n    w = bbox[2] / image[\"width\"]\n    h = bbox[3] / image[\"height\"]\n    \n    x2 = x1 + w\n    y2 = y1 + h\n    \n    x1 = max(x1, 0)\n    y1 = max(y1, 0)\n    x2 = min(x2, 1)\n    y2 = min(y2, 1)\n    \n    submit_annotation(\n        image[\"file_name\"],\n        x1, y1,\n        x2, y2, \n        \"segmentation\"\n    )\n    \nwith open(os.path.join(path, \"val.json\")) as f:\n    data = json.load(f)\n    \nfor image, annotation in zip(data[\"images\"], data[\"annotations\"]):\n    bbox = annotation[\"bbox\"]\n    \n    x1 = bbox[0] / image[\"width\"]\n    y1 = bbox[1] / image[\"height\"]\n    w = bbox[2] / image[\"width\"]\n    h = bbox[3] / image[\"height\"]\n    \n    x2 = x1 + w\n    y2 = y1 + h\n    \n    x1 = max(x1, 0)\n    y1 = max(y1, 0)\n    x2 = min(x2, 1)\n    y2 = min(y2, 1)\n    \n    submit_annotation(\n        image[\"file_name\"],\n        x1, y1,\n        x2, y2, \n        \"segmentation\"\n    )","metadata":{"execution":{"iopub.status.busy":"2022-03-06T10:59:49.015646Z","iopub.execute_input":"2022-03-06T10:59:49.015948Z","iopub.status.idle":"2022-03-06T10:59:49.049588Z","shell.execute_reply.started":"2022-03-06T10:59:49.015908Z","shell.execute_reply":"2022-03-06T10:59:49.048750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.read_csv(\"../input/fullbodywhaleannotations/fullbody_annotations.csv\")\ndata[\"width\"] = data[\"filename\"].map(TRAIN_IMG_WIDTH)\ndata[\"height\"] = data[\"filename\"].map(TRAIN_IMG_HEIGHT)\nfor _, x in data.iterrows():\n    x1 = x[\"x\"] / x[\"width\"]\n    y1 = x[\"y\"] / x[\"height\"]\n    x2 = (x[\"x\"] + x[\"w\"]) / x[\"width\"]\n    y2 = (x[\"y\"] + x[\"h\"]) / x[\"height\"]\n    \n    x1 = max(0, x1)\n    y1 = max(0, y1)\n    x2 = min(1, x2)\n    y2 = min(1, y2)\n    \n    submit_annotation(\n        x[\"filename\"],\n        x1, y1,\n        x2, y2,\n        \"fullbody\",\n        i_type=\"train\" if x[\"filename\"] in TRAIN_IMG_WIDTH else \"test\"\n    )\n","metadata":{"execution":{"iopub.status.busy":"2022-03-06T10:59:49.051188Z","iopub.execute_input":"2022-03-06T10:59:49.051503Z","iopub.status.idle":"2022-03-06T10:59:49.514027Z","shell.execute_reply.started":"2022-03-06T10:59:49.051462Z","shell.execute_reply":"2022-03-06T10:59:49.513413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!curl http://happywhale.theoboyer.fr/data/annotations.csv > annotations.csv\ndata = pd.read_csv(\"annotations.csv\")\ndata = data[data[\"judge_decision\"] == \"accepted\"]\n!rm annotations.csv\n\ndef submit(x):\n    submit_annotation(\n        x[\"image\"],\n        x[\"x1\"], x[\"y1\"],\n        x[\"x2\"], x[\"y2\"],\n        \"crowd_source\",\n        i_type=\"train\",\n        only_if_new=True\n    )\n\n_ = data.apply(submit, axis=1)","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-03-06T10:59:49.515481Z","iopub.execute_input":"2022-03-06T10:59:49.516097Z","iopub.status.idle":"2022-03-06T10:59:52.501697Z","shell.execute_reply.started":"2022-03-06T10:59:49.516051Z","shell.execute_reply":"2022-03-06T10:59:52.500714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = \"../input/happywhale-boundingbox-yolov5/\"\n\ndata = pd.read_csv(os.path.join(path, \"train.csv\"))\ndata = data[~data[\"bbox\"].isna()]\ndata[\"bbox\"] = data[\"bbox\"].map(eval).map(lambda x: x[0])\ndata[\"x1\"] = data[\"bbox\"].apply(lambda x: float(x[0])) / data[\"width\"]\ndata[\"y1\"] = data[\"bbox\"].apply(lambda x: float(x[1])) / data[\"height\"]\ndata[\"x2\"] = data[\"bbox\"].apply(lambda x: float(x[2])) / data[\"width\"]\ndata[\"y2\"] = data[\"bbox\"].apply(lambda x: float(x[3])) / data[\"height\"]\n\ndef submit(x):\n    submit_annotation(\n        x[\"image\"],\n        x[\"x1\"], x[\"y1\"],\n        x[\"x2\"], x[\"y2\"],\n        \"YOLOv5\",\n        i_type=\"train\"\n    )\n\n_ = data.apply(submit, axis=1)\n\ndata = pd.read_csv(os.path.join(path, \"test.csv\"))\ndata = data[~data[\"bbox\"].isna()]\ndata[\"bbox\"] = data[\"bbox\"].map(eval).map(lambda x: x[0])\ndata[\"x1\"] = data[\"bbox\"].apply(lambda x: float(x[0])) / data[\"width\"]\ndata[\"y1\"] = data[\"bbox\"].apply(lambda x: float(x[1])) / data[\"height\"]\ndata[\"x2\"] = data[\"bbox\"].apply(lambda x: float(x[2])) / data[\"width\"]\ndata[\"y2\"] = data[\"bbox\"].apply(lambda x: float(x[3])) / data[\"height\"]\n\ndef submit(x):\n    submit_annotation(\n        x[\"image\"],\n        x[\"x1\"], x[\"y1\"],\n        x[\"x2\"], x[\"y2\"],\n        \"YOLOv5\",\n        i_type=\"test\"\n    )\n\n_ = data.apply(submit, axis=1)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-03-06T10:59:52.503470Z","iopub.execute_input":"2022-03-06T10:59:52.504298Z","iopub.status.idle":"2022-03-06T10:59:57.356319Z","shell.execute_reply.started":"2022-03-06T10:59:52.504262Z","shell.execute_reply":"2022-03-06T10:59:57.355587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = \"../input/whale2-cropped-dataset\"\ndata = pd.read_csv(os.path.join(path, \"train2.csv\"))\ndata = data[~data[\"box\"].isna()]\ndata[\"box\"] = data[\"box\"].apply(lambda x: x.split(\" \"))\ndata[\"width\"] = data[\"image\"].map(TRAIN_IMG_WIDTH)\ndata[\"height\"] = data[\"image\"].map(TRAIN_IMG_HEIGHT)\ndata[\"x1\"] = data[\"box\"].apply(lambda x: float(x[0])) / data[\"width\"]\ndata[\"y1\"] = data[\"box\"].apply(lambda x: float(x[1])) / data[\"height\"]\ndata[\"x2\"] = data[\"box\"].apply(lambda x: float(x[2])) / data[\"width\"]\ndata[\"y2\"] = data[\"box\"].apply(lambda x: float(x[3])) / data[\"height\"]\n\ndef submit(x):\n    submit_annotation(\n        x[\"image\"],\n        x[\"x1\"], x[\"y1\"],\n        x[\"x2\"], x[\"y2\"],\n        \"detic\",\n        i_type=\"train\"\n    )\n\n_ = data.apply(submit, axis=1)\n\ndata = pd.read_csv(os.path.join(path, \"test2.csv\"))\ndata = data[~data[\"box\"].isna()]\ndata[\"box\"] = data[\"box\"].apply(lambda x: x.split(\" \"))\ndata[\"width\"] = data[\"image\"].map(TEST_IMG_WIDTH)\ndata[\"height\"] = data[\"image\"].map(TEST_IMG_HEIGHT)\ndata[\"x1\"] = data[\"box\"].apply(lambda x: float(x[0])) / data[\"width\"]\ndata[\"y1\"] = data[\"box\"].apply(lambda x: float(x[1])) / data[\"height\"]\ndata[\"x2\"] = data[\"box\"].apply(lambda x: float(x[2])) / data[\"width\"]\ndata[\"y2\"] = data[\"box\"].apply(lambda x: float(x[3])) / data[\"height\"]\n\ndef submit(x):\n    submit_annotation(\n        x[\"image\"],\n        x[\"x1\"], x[\"y1\"],\n        x[\"x2\"], x[\"y2\"],\n        \"detic\",\n        i_type=\"test\"\n    )\n\n_ = data.apply(submit, axis=1)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-03-06T10:59:57.358554Z","iopub.execute_input":"2022-03-06T10:59:57.358792Z","iopub.status.idle":"2022-03-06T10:59:59.982589Z","shell.execute_reply.started":"2022-03-06T10:59:57.358765Z","shell.execute_reply":"2022-03-06T10:59:59.981852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# How to solve conflict\nWe will solve conflicts by being as conservative as possible, meaning that if we have to bounding boxes for one image, we will keep the minimum box that includes both of them","metadata":{}},{"cell_type":"code","source":"manual_sources = [\"happierwhale\", \"fullbody\", \"crowd_source\"]","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-03-06T10:59:59.984336Z","iopub.execute_input":"2022-03-06T10:59:59.984642Z","iopub.status.idle":"2022-03-06T10:59:59.989473Z","shell.execute_reply.started":"2022-03-06T10:59:59.984585Z","shell.execute_reply":"2022-03-06T10:59:59.988678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = dump_dataset()\ndata.describe()","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-03-06T10:59:59.990536Z","iopub.execute_input":"2022-03-06T10:59:59.990784Z","iopub.status.idle":"2022-03-06T11:00:00.170182Z","shell.execute_reply.started":"2022-03-06T10:59:59.990756Z","shell.execute_reply":"2022-03-06T11:00:00.169378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"manual_dataset = data[data[\"source\"].isin(manual_sources)]\nmanual_dataset = manual_dataset.groupby(\"image\").agg({\n    \"xmin\": \"min\",\n    \"ymin\": \"min\",\n    \"xmax\": \"max\",\n    \"ymax\": \"max\"\n})\nmanual_dataset[\"image\"] = manual_dataset.index\nmanual_dataset[\"x1\"] = manual_dataset[\"xmin\"]\nmanual_dataset[\"y1\"] = manual_dataset[\"ymin\"]\nmanual_dataset[\"x2\"] = manual_dataset[\"xmax\"]\nmanual_dataset[\"y2\"] = manual_dataset[\"ymax\"]\nmanual_dataset = manual_dataset[[\"image\", \"x1\", \"y1\", \"x2\", \"y2\"]]\nmanual_dataset.to_csv(\"dataset.csv\", index=False)\nmanual_dataset.describe()","metadata":{"_kg_hide-input":false,"_kg_hide-output":false,"execution":{"iopub.status.busy":"2022-03-06T11:00:00.171457Z","iopub.execute_input":"2022-03-06T11:00:00.171749Z","iopub.status.idle":"2022-03-06T11:00:00.278565Z","shell.execute_reply.started":"2022-03-06T11:00:00.171712Z","shell.execute_reply":"2022-03-06T11:00:00.277701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visual random check 👀","metadata":{}},{"cell_type":"code","source":"for _, x in data[data[\"source\"].isin(manual_sources)].sample(15).iterrows():\n    plt.title(x[\"image\"] + \" from \" + x[\"source\"])\n    plot_bbox_dic(x)","metadata":{"execution":{"iopub.status.busy":"2022-03-06T11:00:00.279982Z","iopub.execute_input":"2022-03-06T11:00:00.280246Z","iopub.status.idle":"2022-03-06T11:00:11.595410Z","shell.execute_reply.started":"2022-03-06T11:00:00.280207Z","shell.execute_reply":"2022-03-06T11:00:11.594553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"![](https://upload.wikimedia.org/wikipedia/commons/thumb/e/ea/Thats_all_folks.svg/2560px-Thats_all_folks.svg.png)","metadata":{}}]}