{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"TENSORFLOW_INSTALL = True\nUSE_ARFAN_CODE = False\nGITHUB_FRESH = False\nEVALUATE = True\nKAGGLE = True\nEVALUATE = True\nVISUAL = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import glob\nglob_list = glob.glob(f'/kaggle/input/mask-rcnn-train-1536-5-5w-0-0001/*.h5')\nWEIGHTS_PATH = glob_list[0] if glob_list else '' ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"WEIGHTS_PATH","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if TENSORFLOW_INSTALL:\n    !conda install tensorflow-gpu==1.14.0 -y\n    !pip install keras==2.1.5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow\nprint(tensorflow.__version__)\nimport keras\nprint(keras.__version__)\nprint(tensorflow.test.is_gpu_available())\nimport os\nimport gc\nimport sys\nimport json\nimport random\nfrom pathlib import Path\n\nimport cv2 # CV2 for image manipulation\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nfrom tqdm import tqdm\n\nfrom imgaug import augmenters as iaa\n\nimport seaborn as sns\nimport matplotlib.image as mpimg\nfrom matplotlib import pyplot as plt\n\nfrom sklearn.model_selection import StratifiedKFold, KFold","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nwith open('/kaggle/input/imaterialist-fashion-2020-fgvc7/label_descriptions.json', 'r') as file:\n    label_desc = json.load(file)\nsample_sub_df = pd.read_csv('/kaggle/input/imaterialist-fashion-2020-fgvc7/sample_submission.csv')\ntrain_df = pd.read_csv('/kaggle/input/imaterialist-fashion-2020-fgvc7/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_classes = len(label_desc['categories'])\nnum_attributes = len(label_desc['attributes'])\nprint(f'Total # of classes: {num_classes}')\nprint(f'Total # of attributes: {num_attributes}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"categories_df = pd.DataFrame(label_desc['categories'])\nattributes_df = pd.DataFrame(label_desc['attributes'])\ncategories_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_df = train_df.groupby('ImageId')['EncodedPixels', 'ClassId', 'AttributesIds'].agg(lambda x: list(x))\nsize_df = train_df.groupby('ImageId')['Height', 'Width'].first()\nimage_df = image_df.join(size_df, on='ImageId')\n\nprint(\"Total images: \", len(image_df))\nimage_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nfrom pathlib import Path\nos.chdir('/kaggle/working/')\nif os.path.exists(\"Mask_RCNN\"):\n    if GITHUB_FRESH and USE_ARFAN_CODE:\n        os.system('rm -rf /kaggle/working/Mask_RCNN')\n        !git clone https://www.github.com/AR-fan/Mask_RCNN.git\n        !rm -rf .git # to prevent an error when the kernel is committed\n        !rm -rf images assets # to prevent displaying images at the bottom of a kernel\nelif USE_ARFAN_CODE:\n    !git clone https://www.github.com/AR-fan/Mask_RCNN.git\n    !rm -rf .git # to prevent an error when the kernel is committed\n    !rm -rf images assets # to prevent displaying images at the bottom of a kernel        \nelse:\n    !git clone https://github.com/matterport/Mask_RCNN    \n    !rm -rf .git # to prevent an error when the kernel is committed\n    !rm -rf images assets # to prevent displaying images at the bottom of a kernel    \n    \nos.chdir('Mask_RCNN')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_DIR = Path('/kaggle/input/imaterialist-fashion-2020-fgvc7')\nROOT_DIR = Path('/kaggle/working')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# sys.path.append(ROOT_DIR/'Mask_RCNN')\nimport sys\nfrom mrcnn.config import Config\nfrom mrcnn import utils\nimport mrcnn.model as modellib\nfrom mrcnn import visualize\nfrom mrcnn.model import log","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class FashionConfig(Config):\n    \"\"\"Configuration for training on the toy shapes dataset.\n    Derives from the base Config class and overrides values specific\n    to the toy shapes dataset.\n    \"\"\"\n    # Give the configuration a recognizable name\n    NAME = \"class\"\n\n    # Train on 1 GPU and 8 images per GPU. We can put multiple images on each\n    # GPU because the images are small. Batch size is 8 (GPUs * images/GPU).\n    GPU_COUNT = 1\n    IMAGES_PER_GPU = 1\n    \n    TOP_DOWN_PYRAMID_SIZE = 256\n    RPN_TRAIN_ANCHORS_PER_IMAGE = 256\n\n    # Number of classes (including background)\n    NUM_CLASSES = 1 + len(categories_df)  # background + 46 classes\n\n    # Use small images for faster training. Set the limits of the small side\n    # the large side, and that determines the image shape.\n    IMAGE_MIN_DIM =  1536\n    IMAGE_MAX_DIM = 1536\n\n    # Length of square anchor side in pixels\n    RPN_ANCHOR_SCALES = (75,187,308,554,1032)# (32, 64, 128, 256, 512)\n\n    # Ratios of anchors at each cell (width/height)\n    # A value of 1 represents a square anchor, and 0.5 is a wide anchor\n    RPN_ANCHOR_RATIOS = [0.7, 0.9, 1.2] # [0.45, 0.9, 1.28, 1.47]\n    \n    # Reduce training ROIs per image because the images are small and have\n    # few objects. Aim to allow ROI sampling to pick 33% positive ROIs.\n    TRAIN_ROIS_PER_IMAGE = 200 # 200-》100\n    \n    RPN_TRAIN_ANCHORS_PER_IMAGE = 256\n    PRE_NMS_LIMIT = 6000 # \n    \n    MASK_POOL_SIZE = 28 # 14->28\n    MASK_SHAPE = [56, 56] # [28, 28] -> [56, 56]\n    \n    POST_NMS_ROIS_TRAINING = 2000\n    POST_NMS_ROIS_INFERENCE = 1000\n\n    DETECTION_MAX_INSTANCES = 74\n    \n\n    MAX_GT_INSTANCES = 74\n\n    # Use a small epoch since the data is simple\n    STEPS_PER_EPOCH = 5000 # 100 1000\n\n    # use small validation steps since the epoch is small\n    VALIDATION_STEPS = 5 # 5 50\n    \n    # Non-maximum suppression threshold for detection\n    DETECTION_NMS_THRESHOLD = 0.3 \n    \n    # Non-max suppression threshold to filter RPN proposals.\n    # You can increase this during training to generate more propsals.\n    RPN_NMS_THRESHOLD = 0.7\n    \nconfig = FashionConfig()\nconfig.display()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# This code partially supports k-fold training, \n# you can specify the fold to train and the total number of folds here\nFOLD = 0\nN_FOLDS = 2\n\nkf = KFold(n_splits=N_FOLDS, random_state=42, shuffle=True)\nsplits = kf.split(image_df) # ideally, this should be multilabel stratification\n\ndef get_fold():    \n    for i, (train_index, valid_index) in enumerate(splits):\n        if i == FOLD:\n            return image_df.iloc[train_index], image_df.iloc[valid_index]\n\ntrain_df, valid_df = get_fold()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import warnings \nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nclass InferenceConfig(FashionConfig):\n    GPU_COUNT = 1\n    IMAGES_PER_GPU = 1\n    DETECTION_MIN_CONFIDENCE = 0.7 \n\ninference_config = InferenceConfig()\n\nmodel = modellib.MaskRCNN(mode='inference', \n                          config=inference_config,\n                          model_dir=ROOT_DIR)\n\nassert WEIGHTS_PATH != '', \"Provide path to trained weights\"\nprint(\"Loading weights from \", WEIGHTS_PATH)\nmodel.load_weights(WEIGHTS_PATH, by_name=True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_unique_array = image_df.index.unique()\nimage_unique_dict = {}\nfor i,name in enumerate(image_unique_array):\n    image_unique_dict[name] = i\nlen(image_unique_dict)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!git clone https://github.com/AR-fan/cocoapi.git \n#    TypeError: object of type <class 'numpy.float64'> cannot be safely interpreted as an integer.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.chdir('./cocoapi/PythonAPI')\n!make","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from pycocotools.coco import COCO\nfrom pycocotools import mask as maskUtils\nfrom pycocotools.cocoeval import COCOeval","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Since the submission system does not permit overlapped masks, we have to fix them\ndef refine_masks(masks):\n    rois = np.zeros((masks.shape[-1], 4))\n    areas = np.sum(masks.reshape(-1, masks.shape[-1]), axis=0) \n    mask_index = np.argsort(areas)\n    union_mask = np.zeros(masks.shape[:-1], dtype=bool)\n    for m in mask_index: \n        masks[:, :, m] = np.logical_and(masks[:, :, m], np.logical_not(union_mask)) \n        union_mask = np.logical_or(masks[:, :, m], union_mask) \n    for m in range(masks.shape[-1]):\n        mask_pos = np.where(masks[:, :, m]==True)\n        if np.any(mask_pos):\n            y1, x1 = np.min(mask_pos, axis=1)\n            y2, x2 = np.max(mask_pos, axis=1)\n            rois[m, :] = [y1, x1, y2, x2]\n    return masks, rois","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"evaluate_train_results = []\nimages = []\nannotations = []\ncategories = []    \nid_temp = -1    \nfashion_annotations_coco_predicts = []\n\nfor i, row in tqdm(valid_df[:10].iterrows(), total=10):    \n# for i, row in tqdm(image_df[:10].iterrows(), total=10):    \n\n    image_path = str(DATA_DIR/'train'/row.name) + '.jpg'\n    image = cv2.imread(image_path)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\n    result = model.detect([image])[0]\n    result['shape'] = image.shape[0:2]\n\n    if result['rois'] is None:\n        continue         \n\n    if result['masks'].shape[-1] > 1:\n        masks, rois = refine_masks(result['masks']) \n    else: \n        masks = result['masks']\n\n    # Loop through detections\n    for k in range(result['rois'].shape[0]):\n        class_id = int(result[\"class_ids\"][k]) - 1 \n        score = result[\"scores\"][k]\n        bbox = np.around(result[\"rois\"][k], 1)\n        mask = masks[:, :, k].astype(np.uint8)\n\n        evaluate_result = {\n            \"image_id\": image_unique_dict[row.name],\n            \"category_id\": class_id,\n            \"bbox\": [bbox[1], bbox[0], bbox[3] - bbox[1], bbox[2] - bbox[0]],\n            \"score\": score,\n            \"segmentation\": maskUtils.encode(np.asfortranarray(mask)) \n#                 \"area\"\n        }\n        evaluate_train_results.append(evaluate_result)            \n\n\n    # train_label\n\n\n    for j in range(len(row['ClassId'])):\n        id_temp = id_temp + 1\n        annotation = {}\n        annotation['id'] = id_temp\n        annotation['image_id'] = image_unique_dict[row.name] # int  \n        annotation['category_id'] = int(row['ClassId'][j]) # numpy.int64 -> int type(train_df.iloc[0]['ClassId'])\n        sub_mask = np.full(row['Height']*row['Width'], 0, dtype=np.uint8)        \n        fashion_rle = [int(x) for x in row[\"EncodedPixels\"][j].split(' ')]\n        for i, start_pixel in enumerate(fashion_rle[::2]):\n            sub_mask[start_pixel: start_pixel+fashion_rle[2*i+1]] = 1      \n        sub_mask = sub_mask.reshape((row['Height'], row['Width']), order='F')\n        rle = maskUtils.encode(sub_mask.astype(np.uint8)) # np.asfortranarray(mask)         \n        annotation['segmentation'] = rle            \n        annotation['iscrowd'] = 0 #  1\n        annotation['area'] = maskUtils.area(rle)\n        annotations.append(annotation)            \n\n        image = {}\n        image['id'] = image_unique_dict[row.name] # int  \n        image['width'] = row['Width']\n        image[\"height\"] = row['Height']\n        image['file_name'] = row.name\n        images.append(image)\n\n\n        fashion_annotations_coco_predict = {\n            \"image_id\": image_unique_dict[row.name],\n            \"category_id\": int(row[\"ClassId\"][j]),\n#                 \"bbox\": [bbox[1], bbox[0], bbox[3] - bbox[1], bbox[2] - bbox[0]],\n            \"score\": 1,\n            \"segmentation\": rle \n#                 \"area\"\n        }\n        fashion_annotations_coco_predicts.append(fashion_annotations_coco_predict)            \n\n\nfor cat in label_desc.get('categories'):\n#     {'id': 0, 'name': 'shirt, blouse', 'supercategory': 'upperbody', 'level': 2}\n#     category_map[cat.get('id')] = cat.get('name')\n    category = {}\n    category['id'] = cat['id']\n    category['name'] = cat['name']\n#     category['supercategory'] = cat['supercategory']    \n    categories.append(category)\n\n\nfashion_annotations = {}\nfashion_annotations['images'] = images\nfashion_annotations['annotations'] = annotations\nfashion_annotations['categories'] = categories             \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# TypeError: Object of type 'bytes' is not JSON serializable\n# https://blog.csdn.net/bear_sun/article/details/79397155\nclass MyEncoder(json.JSONEncoder):\n    def default(self, obj):\n        if isinstance(obj, np.ndarray):\n            return obj.tolist() \n        elif isinstance(obj, bytes):\n            return str(obj, encoding='utf-8');\n        if isinstance(obj, np.integer):\n            return int(obj)\n        elif isinstance(obj, np.floating):\n            return float(obj)\n        elif isinstance(obj, np.ndarray):\n            return obj.tolist()\n        return json.JSONEncoder.default(self, obj)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"! mkdir -p /kaggle/working/kaggle/\n!touch /kaggle/working/kaggle/fashion_annotations.json\nwith open('/kaggle/working/kaggle/fashion_annotations.json',\"w\") as dump_f:\n    json.dump(fashion_annotations,dump_f, cls=MyEncoder) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!mkdir -p /kaggle/working/kaggle/\n!touch /kaggle/working/kaggle/evaluate_train_results.json\nwith open('/kaggle/working/kaggle/evaluate_train_results.json',\"w\") as dump_f2:\n    json.dump(evaluate_train_results,dump_f2, cls=MyEncoder) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# fashion_annotations_coco_predicts\n!mkdir -p /kaggle/working/kaggle/\n!touch /kaggle/working/kaggle/fashion_annotations_coco_predicts.json\nwith open('/kaggle/working/kaggle/fashion_annotations_coco_predicts.json',\"w\") as dump_f3:\n    json.dump(fashion_annotations_coco_predicts,dump_f3, cls=MyEncoder) \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# %%python2\n\n# !pip install matplotlib\n\nfrom pycocotools.coco import COCO\nfrom pycocotools import mask as maskUtils\nfrom pycocotools.cocoeval import COCOeval\n# class FashionCOCO(COCO):\n#     def __init__(self, df):\n#         # TODO df->dict\n\n#         self.dataset,self.anns,self.cats,self.imgs = dict(),dict(),dict(),dict()\n#         self.imgToAnns, self.catToImgs = defaultdict(list), defaultdict(list)            \n#         self.dataset = dataset # dict\n#         self.createIndex()   \n\nannFile = '/kaggle/working/kaggle/fashion_annotations.json'\ncocoGt=COCO(annFile)\nresTrain = '/kaggle/working/kaggle/fashion_annotations_coco_predicts.json'\ncocoDt=cocoGt.loadRes(resTrain)\ncocoEval = COCOeval(cocoGt,cocoDt, iouType='segm')\n# cocoEval.params.imgIds  = imgIds\ncocoEval.evaluate()\ncocoEval.accumulate()\ncocoEval.summarize()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"annFile = '/kaggle/working/kaggle/fashion_annotations.json'\ncocoGt=COCO(annFile)\nresTrain = '/kaggle/working/kaggle/evaluate_train_results.json'\ncocoDt=cocoGt.loadRes(resTrain)\ncocoEval = COCOeval(cocoGt,cocoDt, iouType='segm')\n# cocoEval.params.imgIds  = imgIds\ncocoEval.evaluate()\ncocoEval.accumulate()\ncocoEval.summarize()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def rle_decode(rle_str, mask_shape, mask_dtype):\n    s = rle_str.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    mask = np.zeros(np.prod(mask_shape), dtype=mask_dtype)\n    for lo, hi in zip(starts, ends):\n        mask[lo:hi] = 1\n    return mask.reshape(mask_shape[::-1]).T\n\ndef rle_to_string(runs):\n    return ' '.join(str(x) for x in runs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMAGE_SIZE = 400\n\nif VISUAL:\n    label_names = ['bg']\n    label_names.extend(categories_df['name'].values)\n    for i in range(3):\n\n        image_id = image_df.iloc[i].name\n        image_path = \"{}/train/{}.jpg\".format(DATA_DIR,image_id)\n        print(image_path)\n\n        img = cv2.imread(image_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        img = cv2.resize(img, (IMAGE_SIZE, IMAGE_SIZE), interpolation=cv2.INTER_AREA)  \n        \n        \n        result = model.detect([img])\n        r = result[0]\n\n        if r['masks'].size > 0:\n\n            masks, rois = refine_masks(r['masks'])\n        else:\n            masks, rois = r['masks'], r['rois']\n\n\n        visualize.display_instances(img, rois, masks, r['class_ids'], \n                                    label_names, r['scores'],\n                                    title=image_id, figsize=(12, 12))  \n        \n        \n        \n        ground_truth = image_df.iloc[i]        \n        true_mask = np.zeros((IMAGE_SIZE, IMAGE_SIZE,len(ground_truth['ClassId'])), dtype=np.uint8)\n\n\n        for m, (annotation, label) in enumerate(zip(ground_truth['EncodedPixels'], ground_truth['ClassId'])):\n            sub_mask = np.full(ground_truth['Height']*ground_truth['Width'], 0, dtype=np.uint8)\n            annotation = [int(x) for x in annotation.split(' ')]\n\n            for i, start_pixel in enumerate(annotation[::2]):\n                sub_mask[start_pixel: start_pixel+annotation[2*i+1]] = 1\n\n            sub_mask = sub_mask.reshape((ground_truth['Height'], ground_truth['Width']), order='F')\n\n            sub_mask = cv2.resize(sub_mask.astype('uint8'), \n                                            (img.shape[1], img.shape[0]), interpolation=cv2.INTER_NEAREST)\n            true_mask[:, :, m] = sub_mask\n  \n        true_masks, true_rois = refine_masks(true_mask)       \n    \n        true_class_ids = ground_truth['ClassId']      \n        true_class_ids = np.array(true_class_ids)+1\n        true_scores = np.ones(len(ground_truth['ClassId']))\n        \n        visualize.display_instances(img, true_rois, true_masks, true_class_ids, \n                                    label_names, true_scores,\n                                    title=image_id+\"_true\", figsize=(12, 12))          \n        print(true_class_ids)\n        print(r['class_ids'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}