{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"markdown","source":"## This code work for transform the official data-csv to COCO format Dataset \n> In the TrainSet（518GB），this code need run 23 hours with mutil-thread。"},{"metadata":{},"cell_type":"markdown","source":""},{"metadata":{"trusted":true},"cell_type":"code","source":"# -*- coding: utf-8 -*-\n\n\nimport os\nimport json\nimport numpy as np\nimport pandas as pd\nimport glob\nimport cv2\nimport os\nimport shutil\nfrom IPython import embed\nfrom sklearn.model_selection import train_test_split\nfrom tqdm import tqdm\nfrom multiprocessing import Pool\nimport multiprocessing\nnp.random.seed(41)\nfrom Dataset import *\n# 0为背景\nfrom multiprocessing.pool import ThreadPool\nimport time\nclasses='/workspace/competation/opendimage/detection/data/data_index/class-descriptions-boxable.csv'\n\nclassid=pd.read_csv(classes)\n\nclasslist=classid['code'].tolist()\nnamelist=classid['name'].tolist()\nclassname_to_id={}\ncode_to_id={}\nfor index,c in enumerate(namelist):\n    classname_to_id[c]=index\n    code_to_id[classid['code'][index]]=index\n    print (c,\":\",classname_to_id[c],':',classid['code'][index])\n\n# classname_to_id = {\"person\": 1}\n\n\nclass Csv2CoCo:\n\n    def __init__(self, image_dir, total_annos):\n        self.images = []\n        self.annotations = []\n        self.categories = []\n        self.img_id = 0\n        self.ann_id = 0\n        self.image_dir = image_dir\n        self.total_annos = total_annos\n\n    def save_coco_json(self, instance, save_path):\n        json.dump(instance, open(save_path, 'w'), ensure_ascii=False, indent=2)  # indent=2 更加美观显示\n\n    # 由txt文件构建COCO\n    def to_coco(self, keys):\n        # pool = multiprocessing.Pool(processes=3)\n        pool = ThreadPool(processes=8)\n\n        self._init_categories()\n        self.keys=keys\n\n\n        # imagekeys=pool.map(self._image,keys)\n\n        start = time.time()\n\n        self.imagekeys=pool.map(self._image,keys)\n        pool.close()\n        pool.join()\n        end = time.time()\n\n        print('\\nimagekey process',len(self.imagekeys),'in\\n:',end-start,'per',len(self.imagekeys)/(end-start),'/sec\\n\\n')\n        # exit(0)\n        # self.image_s=self.imagekeys\n\n\n        for key in tqdm(keys):\n\n            # self.imagekey=self._image(key)\n            #\n            # if self.imagekey==0:\n            #     continue\n            # self.images.append(self.imagekey)\n            shapes = self.total_annos[key]\n            for shape in shapes:\n                bboxi=shape[:-1]\n                label = shape[-1]\n                annotation = self._annotation(bboxi, label)\n\n                # if display:\n                    # print('\\ndraw in ',annotation['bbox'])\n                    # c=annotation['bbox']\n                    # self.img = cv2.rectangle(self.img, (c[0],c[1]), (c[0]+c[2],c[1]+c[3]), (0, 0, 255), 2)\n                    # self.img=cv2.putText(self.img,label,(c[0],c[1]),cv2.FONT_ITALIC,1,(255,0,0),1)\n                    # cv2.imwrite(str(index)+'.jpg',self.img)\n\n                self.annotations.append(annotation)\n                self.ann_id += 1\n            self.img_id += 1\n\n        instance = {}\n        instance['info'] = 'spytensor created'\n        instance['license'] = ['license']\n        instance['images'] = self.imagekeys\n        instance['annotations'] = self.annotations\n        instance['categories'] = self.categories\n        return instance\n\n    # 构建类别\n    def _init_categories(self):\n        for k, v in classname_to_id.items():\n            category = {}\n            category['id'] = v\n            category['name'] = k\n            self.categories.append(category)\n\n    # 构建COCO的image字段\n    def _image(self, path):\n        image = {}\n        # print(path)\n        img = cv2.imread(path)\n        # img=pool.map(cv2.imread,[path])[0]\n\n        # print(img)\n        if type(img)==type(None):\n            return 0\n        self.img=img\n        # image_file=open('path','rb')\n        # my_image = Image(image_file)\n\n        # print(path,'\\nresolution :',my_image.y_resolution,my_image.x_resolution)\n        image['height'] = img.shape[0]\n        image['width'] = img.shape[1]\n        image['id'] = self.img_id\n        image['file_name'] = path\n        self.image=image\n        return image\n\n    # 构建COCO的annotation字段\n    def _annotation(self, shape, label):\n        # label = shape[-1]\n        points = shape[:4]\n        annotation = {}\n        annotation['id'] = self.ann_id\n        annotation['image_id'] = self.img_id\n        annotation['category_id'] = int(classname_to_id[label])\n        # annotation['segmentation'] = self._get_seg(points)\n        annotation['bbox'] = self._get_box(points)\n        annotation['iscrowd'] = 0\n        annotation['area'] = 1.0\n        return annotation\n\n    # COCO的格式： [x1,y1,w,h] 对应COCO的bbox格式\n    def _get_box(self, points):\n        #(xmax.xmin.ymax.ymin)\n        # print('\\npoints',points,'\\n')\n        min_x = int(float(points[0])*self.image['width'])\n        min_y = int(float(points[2])*self.image['height'])\n        max_x = int(float(points[1])*self.image['width'])\n        max_y = int(float(points[3])*self.image['height'])\n        # print(min_x,min_y, max_x-min_x,max_y-min_y)\n        return [min_x,min_y, max_x-min_x,max_y-min_y]\n        #xywh\n    # segmentation\n    # def _get_seg(self, points):\n    #     min_x = points[0]\n    #     min_y = points[2]\n    #     max_x = points[1]\n    #     max_y = points[3]\n    #     h = max_y - min_y\n    #     w = max_x - min_x\n    #     a = []\n    #     a.append([min_x, min_y, min_x, min_y + 0.5 * h, min_x, max_y, min_x + 0.5 * w, max_y, max_x, max_y, max_x,\n    #               max_y - 0.5 * h, max_x, min_y, max_x - 0.5 * w, min_y])\n    #     return a\n\n\n\nif __name__ == '__main__':\n    csv_file = \"/workspace/competation/opendimage/detection/data/data_index/validation-annotations-bbox.csv\"\n    image_dir = \"/workspace/competation/opendimage/detection/data/train\"\n    saved_coco_path = \"/workspace/competation/opendimage/detection/data/\"\n    # 整合csv格式标注文件\n    total_csv_annotations = {}\n    annotations = pd.read_csv(Train).values\n\n    for annotation in tqdm(annotations):\n        # print (annotation)\n        from exif import Image\n        key = annotation[0].split(os.sep)[-1]\n        key=os.path.join(image_dir,key+'.jpg')\n        #XMin,XMax,YMin,YMax\n        value = np.array([[annotation[4]\n                            ,annotation[5]\n                            ,annotation[6]\n                            ,annotation[7]\n                            ,namelist[code_to_id[annotation[2]]]]])\n        if key in total_csv_annotations.keys():\n            total_csv_annotations[key] = np.concatenate((total_csv_annotations[key], value), axis=0)\n        else:\n            total_csv_annotations[key] = value\n    # print(total_csv_annotations)\n    # # 按照键值划分数据\n    total_keys = list(total_csv_annotations.keys())\n    # print(total_keys[3])\n    # print(total_csv_annotations[total_keys[3]])\n    train_keys, val_keys = train_test_split(total_keys, test_size=0.01)\n    print(\"train_n:\", len(train_keys), 'val_n:', len(val_keys))\n    # 创建必须的文件夹\n    if not os.path.exists('%scoco/annotations/' % saved_coco_path):\n        os.makedirs('%scoco/annotations/' % saved_coco_path)\n    if not os.path.exists('%scoco/images/train2017/' % saved_coco_path):\n        os.makedirs('%scoco/images/train2017/' % saved_coco_path)\n    if not os.path.exists('%scoco/images/val2017/' % saved_coco_path):\n        os.makedirs('%scoco/images/val2017/' % saved_coco_path)\n    # 把训练集转化为COCO的json格式\n    l2c_train = Csv2CoCo(image_dir=image_dir, total_annos=total_csv_annotations)\n    train_instance = l2c_train.to_coco(train_keys)\n    l2c_train.save_coco_json(train_instance, '%scoco/annotations/instances_train2017.json' % saved_coco_path)\n\n    for file in train_keys:\n        shutil.copy(file, \"%scoco/images/train2017/\" % saved_coco_path)\n    for file in val_keys:\n        shutil.copy(file, \"%scoco/images/val2017/\" % saved_coco_path)\n\n\n    # 把验证集转化为COCO的json格式\n    l2c_val = Csv2CoCo(image_dir=image_dir, total_annos=total_csv_annotations)\n    val_instance = l2c_val.to_coco(val_keys)\n    l2c_val.save_coco_json(val_instance, '%scoco/annotations/instances_val2017.json' % saved_coco_path)\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}