{"cells":[{"metadata":{},"cell_type":"markdown","source":"Copy and edit from the great kernel: https://www.kaggle.com/ekhtiar/eda-find-me-in-the-clouds.    \n\nThis kernel can convert the dataset to coco format and passcalvoc format"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install pycocotools","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nimport os\nimport cv2\n# visualization\nimport matplotlib.pyplot as plt\nfrom matplotlib import patches as patches\n# plotly offline imports\nfrom plotly.offline import download_plotlyjs, init_notebook_mode, iplot\nfrom plotly import subplots\nimport plotly.express as px\nimport plotly.figure_factory as ff\nfrom plotly.graph_objs import *\nfrom plotly.graph_objs.layout import Margin, YAxis, XAxis\ninit_notebook_mode()\n# frequent pattern mining\n\n\nfrom pycocotools.coco import COCO\nfrom pycocotools.mask import encode,decode,area,toBbox\n\nimport json","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_path = '../input/understanding_cloud_organization'\ntrain_csv_path = os.path.join(data_path,'train.csv')\ntrain_image_path = os.path.join(data_path,'train_images')\npd.read_csv(train_csv_path).head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(train_csv_path).fillna(-1)\ntrain_df['ImageId'] = train_df['Image_Label'].apply(lambda x: x.split('_')[0])\ntrain_df['Label'] = train_df['Image_Label'].apply(lambda x: x.split('_')[1])\n# lets create a dict with class id and encoded pixels and group all the defaults per image\ntrain_df['Label_EncodedPixels'] = train_df.apply(lambda row: (row['Label'], row['EncodedPixels']), axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cats_dic={'background':0,'Fish':1, 'Flower':2, 'Gravel':3, 'Sugar':4}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"grouped_EncodedPixels = train_df.groupby('ImageId')['Label_EncodedPixels'].apply(list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def np_resize(img, input_shape):\n    \"\"\"\n    Reshape a numpy array, which is input_shape=(height, width), \n    as opposed to input_shape=(width, height) for cv2\n    \"\"\"\n    height, width = input_shape\n    return cv2.resize(img, (width, height))\n    \ndef mask2rle(img):\n    '''\n    img: numpy array, 1 - mask, 0 - background\n    Returns run length as string formated\n    '''\n    pixels= img.T.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\n\ndef rle2mask(rle, input_shape=(1400,2100)):\n    width, height = input_shape[:2]\n    \n    mask= np.zeros( width*height ).astype(np.uint8)\n    \n    array = np.asarray([int(x) for x in rle.split()])\n    starts = array[0::2]\n    lengths = array[1::2]\n\n    current_position = 0\n    for index, start in enumerate(starts):\n        mask[int(start):int(start+lengths[index])] = 1\n        current_position += lengths[index]\n        \n    return mask.reshape(height, width).T\n\ndef build_masks(rles, input_shape, reshape=None):\n    depth = len(rles)\n    if reshape is None:\n        masks = np.zeros((*input_shape, depth))\n    else:\n        masks = np.zeros((*reshape, depth))\n    \n    for i, rle in enumerate(rles):\n        if type(rle) is str:\n            if reshape is None:\n                masks[:, :, i] = rle2mask(rle, input_shape)\n            else:\n                mask = rle2mask(rle, input_shape)\n                reshaped_mask = np_resize(mask, reshape)\n                masks[:, :, i] = reshaped_mask\n    \n    return masks\n\ndef build_rles(masks, reshape=None):\n    width, height, depth = masks.shape\n    \n    rles = []\n    \n    for i in range(depth):\n        mask = masks[:, :, i]\n        \n        if reshape:\n            mask = mask.astype(np.float32)\n            mask = np_resize(mask, reshape).astype(np.int64)\n        \n        rle = mask2rle(mask)\n        rles.append(rle)\n        \n    return rles","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def mask2polygon(mask):\n    contours, hierarchy = cv2.findContours((mask).astype(np.uint8), cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)\n    # mask_new, contours, hierarchy = cv2.findContours((mask).astype(np.uint8), cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)\n    segmentation = []\n    for contour in contours:\n        contour_list = contour.flatten().tolist()\n        if len(contour_list) > 4:# and cv2.contourArea(contour)>10000\n            segmentation.append(contour_list)\n    return segmentation\n\ndef rlestr2list(rlestr):\n    array = np.asarray([int(x) for x in rlestr.split()])\n    return array\ndef rlestr2rleseg(rlestr):\n    segmentation={\"counts\":rlestr2list(rlestr), \"size\": [1400, 2100]}\n    return segmentation","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def mask2area(mask):\n    return area(encode(mask))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def bounding_box(img):\n    # return max and min of a mask to draw bounding box\n    rows = np.any(img, axis=1)\n    cols = np.any(img, axis=0)\n    rmin, rmax = np.where(rows)[0][[0, -1]]\n    cmin, cmax = np.where(cols)[0][[0, -1]]\n\n    return rmin, rmax, cmin, cmax","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class NpEncoder(json.JSONEncoder):\n    def default(self, obj):\n        if isinstance(obj, np.integer):\n            return int(obj)\n        elif isinstance(obj, np.floating):\n            return float(obj)\n        elif isinstance(obj, np.ndarray):\n            return obj.tolist()\n        else:\n            return super(NpEncoder, self).default(obj)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":""},{"metadata":{},"cell_type":"markdown","source":"convert_to_coco"},{"metadata":{"trusted":true},"cell_type":"code","source":"def convert(grouped_EncodedPixels,categories, json_file=\"test.json\"):\n    \"\"\"\n    json_file : 保存生成的json文件路径\n    \"\"\"\n    json_dict = {\"images\": [], \"type\": \"instances\", \"annotations\": [],\n                 \"categories\": []}\n    bnd_id = 1\n    image_id=0\n    \n    for img_name in grouped_EncodedPixels.index:\n        image_id +=1\n        height, width=1400,2100\n        image = {'file_name': img_name, 'height': height, 'width': width,\n                 'id': image_id}\n#         print(image)\n        json_dict['images'].append(image)\n        rle_lists=grouped_EncodedPixels[img_name]\n        for rle in rle_lists:\n            # 可能需要根据具体格式修改的地方\n            category = rle[0]\n            if category not in categories:\n                new_id = len(categories)\n                categories[category] = new_id\n            category_id = categories[category]\n            rlestr=rle[1]\n            if rlestr!=-1:\n#                 print(category)\n                mask=rle2mask(rlestr)\n                ymin,ymax,xmin,xmax=bounding_box(mask)\n            \n#                 print(xmin, ymin, xmax, ymax)\n                assert(xmax > xmin)\n                assert(ymax > ymin)\n                o_width = abs(xmax - xmin)\n                o_height = abs(ymax - ymin)\n                ann = {'area': o_width*o_height, 'iscrowd':0, 'image_id':\n                       image_id, 'bbox': [xmin, ymin, o_width, o_height],\n                       'category_id': category_id, 'id': bnd_id, 'ignore': 0,\n                       'segmentation':mask2polygon(mask)}\n                json_dict['annotations'].append(ann)\n                bnd_id = bnd_id + 1\n    for cate, cid in categories.items():\n        cat = {'supercategory': 'none', 'id': cid, 'name': cate}\n        json_dict['categories'].append(cat)\n#     print(json_dict)\n    json_fp = open(json_file, 'w',encoding='utf-8')\n    json_str = json.dumps(json_dict,cls=NpEncoder)\n    json_fp.write(json_str)\n    json_fp.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# convert(grouped_EncodedPixels,cats_dic)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"test:"},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\nfrom pycocotools.coco import COCO\nfrom pycocotools.mask import encode,decode,area,toBbox\n\nimport numpy as np\nimport skimage.io as io\nimport matplotlib.pyplot as plt\nimport pylab\npylab.rcParams['figure.figsize'] = (8.0, 10.0)\n\n\nannFile='test.json'\ndef test():\n    coco=COCO(annFile)\n\n    imgIds = coco.getImgIds()\n    imags=coco.loadImgs(imgIds)\n\n    annIds = coco.getAnnIds(imgIds=imgIds)\n    ann = coco.loadAnns(annIds)[0]\n\n    mask=coco.annToMask(ann)\n    rle=coco.annToRLE(ann)\n\n    rle=encode(mask)\n    mask=decode(rle)\n\n    area(rle)\n    toBbox(rle)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# test()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"convert_to voc"},{"metadata":{"trusted":true},"cell_type":"code","source":"import xml.etree.ElementTree as ET","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xmlstr='''<annotation>\n\t<folder>VOC2007</folder>\n\t<filename>000001.jpg</filename>\n\t<source>\n\t\t<database>The VOC2007 Database</database>\n\t\t<annotation>PASCAL VOC2007</annotation>\n\t\t<image>flickr</image>\n\t\t<flickrid>341012865</flickrid>\n\t</source>\n\t<owner>\n\t\t<flickrid>Fried Camels</flickrid>\n\t\t<name>Jinky the Fruit Bat</name>\n\t</owner>\n\t<size>\n\t\t<width>2100</width>\n\t\t<height>1400</height>\n\t\t<depth>3</depth>\n\t</size>\n\t<segmented>0</segmented>\n\t\n</annotation>\n'''\nobjectstr='''\n    <object>\n\t\t<name>person</name>\n\t\t<pose>Left</pose>\n\t\t<truncated>1</truncated>\n\t\t<difficult>0</difficult>\n\t\t<bndbox>\n\t\t\t<xmin>8</xmin>\n\t\t\t<ymin>12</ymin>\n\t\t\t<xmax>352</xmax>\n\t\t\t<ymax>498</ymax>\n\t\t</bndbox>\n\t</object>\n    '''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def convert2voc(grouped_EncodedPixels,categories,base_dir=\".\"):\n\n    json_dict = {\"images\": [], \"type\": \"instances\", \"annotations\": [],\n                 \"categories\": []}\n    bnd_id = 1\n    image_id=0\n    \n    for img_name in grouped_EncodedPixels.index:\n        root = ET.fromstring(xmlstr)\n        root.find('filename').text=img_name\n        rle_lists=grouped_EncodedPixels[img_name]\n        \n        for rle in rle_lists:\n            rlestr=rle[1]\n            if rlestr!=-1:\n                object_el=ET.fromstring(objectstr)\n                object_el.find('name').text=rle[0]\n                mask=rle2mask(rlestr)\n                ymin,ymax,xmin,xmax=bounding_box(mask)\n                assert(xmax > xmin)\n                assert(ymax > ymin)\n                bndbox=object_el.find('bndbox')\n                bndbox.find('ymin').text=str(ymin)\n                bndbox.find('ymax').text=str(ymax)\n                bndbox.find('xmin').text=str(xmin)\n                bndbox.find('xmax').text=str(xmax)\n                root.append(object_el)\n        rough_string = ET.tostring(root, encoding=\"utf-8\", method=\"xml\")\n        filename=img_name.replace(\"jpg\",\"xml\")\n        filename=os.path.join(base_dir,\"Annotations\",filename)\n        with open(filename,'wb') as f:\n            f.write(rough_string)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n!mkdir Annotations\n!mkdir ImageSets\n\n!mkdir ImageSets/Main\n!ls","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base_dir=\".\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# convert2voc(grouped_EncodedPixels,cats_dic)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls Annotations","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_dataset(grouped_EncodedPixels,base_dir=\".\",filename=\"train.txt\"):\n    filename=os.path.join(base_dir,\"ImageSets/Main\",filename)\n    with open(filename,'w') as f:\n        for img_name in grouped_EncodedPixels.index:\n            img_name=img_name.replace(\".jpg\",\"\")\n            f.writelines(img_name)\n            f.writelines(\"\\n\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create_dataset(grouped_EncodedPixels,base_dir)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls ImageSets/Main","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}