{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ls","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# setting path for each of the files\nclass_path='/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_detailed_class_info.csv'\nlabels_path='/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_labels.csv'\nImage_train_path='/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_images/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!git clone https://github.com/kbardool/keras-frcnn.git","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv(class_path)\ndf_1 = pd.read_csv(labels_path)\ndf_class = df[\"class\"]\ndf_refine = df_1.drop(\"Target\",axis =1)\nlast = pd.concat([df_refine,df_class],axis = 1 )\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"l = []\nfor i in last['patientId']:\n    l.append('/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_images/' + i+'.jpg')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(l)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del last[\"patientId\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"patientId = pd.Series(l,name='patientId')\npatientId\ndf_n = pd.concat([patientId,last],axis=1)\ndf_n\ndf_n.dropna(inplace= True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_n.to_csv(\"new.txt\",index= None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_n['x-max'] = df_n['x'] + df_n['width']\ndf_n['y-max'] = df_n['y'] + df_n['height']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final = pd.DataFrame(df_n[['patientId','x','x-max','y','y-max','class']])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final.to_csv(\"annotation.txt\",index=None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ls","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"os.remove('new.txt')","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"import shutil\n\ndir_path = '/kaggle/working/keras-frcnn'\n\ntry:\n    shutil.rmtree(dir_path)\nexcept OSError as e:\n    print(\"Error: %s : %s\" % (dir_path, e.strerror))","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nos.mkdir('/kaggle/working/train/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import cv2\nimport os\nimport pydicom\n\ninputdir = '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_images/'\noutdir = '/kaggle/working/train/'\n#os.mkdir(outdir)\n\ntrain_list = [ f for f in  os.listdir(inputdir)]\n\nfor f in train_list:   # remove \"[:10]\" to convert all images \n    ds = pydicom.read_file(inputdir + f) # read dicom image\n    img = ds.pixel_array # get image array\n    cv2.imwrite(outdir + f.replace('.dcm','.jpg'),img) # write png image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from __future__ import division\nfrom __future__ import print_function\nfrom __future__ import absolute_import\nimport random\nimport pprint\nimport sys\nimport time\nimport numpy as np\nfrom optparse import OptionParser\nimport pickle\nimport math\nimport cv2\nimport copy\nfrom matplotlib import pyplot as plt\nimport tensorflow as tf\nimport pandas as pd\nimport os\n\nfrom sklearn.metrics import average_precision_score\n\nfrom keras import backend as K\nfrom keras.optimizers import Adam, SGD, RMSprop\nfrom keras.layers import Flatten, Dense, Input, Conv2D, MaxPooling2D, Dropout\nfrom keras.layers import GlobalAveragePooling2D, GlobalMaxPooling2D, TimeDistributed\nfrom keras.engine.topology import get_source_inputs\nfrom keras.utils import layer_utils\nfrom keras.utils.data_utils import get_file\nfrom keras.objectives import categorical_crossentropy\n\nfrom keras.models import Model\nfrom keras.utils import generic_utils\nfrom keras.engine import Layer, InputSpec\nfrom keras import initializers, regularizers","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Config:\n\n\tdef __init__(self):\n\n\t\t# Print the process or not\n\t\tself.verbose = True\n\n\t\t# Name of base network\n\t\tself.network = 'vgg'\n\n\t\t# Setting for data augmentation\n\t\tself.use_horizontal_flips = False\n\t\tself.use_vertical_flips = False\n\t\tself.rot_90 = False\n\n\t\t# Anchor box scales\n    # Note that if im_size is smaller, anchor_box_scales should be scaled\n    # Original anchor_box_scales in the paper is [128, 256, 512]\n\t\tself.anchor_box_scales = [64, 128, 256] \n\n\t\t# Anchor box ratios\n\t\tself.anchor_box_ratios = [[1, 1], [1./math.sqrt(2), 2./math.sqrt(2)], [2./math.sqrt(2), 1./math.sqrt(2)]]\n\n\t\t# Size to resize the smallest side of the image\n\t\t# Original setting in paper is 600. Set to 300 in here to save training time\n\t\tself.im_size = 300\n\n\t\t# image channel-wise mean to subtract\n\t\tself.img_channel_mean = [103.939, 116.779, 123.68]\n\t\tself.img_scaling_factor = 1.0\n\n\t\t# number of ROIs at once\n\t\tself.num_rois = 4\n\n\t\t# stride at the RPN (this depends on the network configuration)\n\t\tself.rpn_stride = 16\n\n\t\tself.balanced_classes = False\n\n\t\t# scaling the stdev\n\t\tself.std_scaling = 4.0\n\t\tself.classifier_regr_std = [8.0, 8.0, 4.0, 4.0]\n\n\t\t# overlaps for RPN\n\t\tself.rpn_min_overlap = 0.3\n\t\tself.rpn_max_overlap = 0.7\n\n\t\t# overlaps for classifier ROIs\n\t\tself.classifier_min_overlap = 0.1\n\t\tself.classifier_max_overlap = 0.5\n\n\t\t# placeholder for the class mapping, automatically generated by the parser\n\t\tself.class_mapping = None\n\n\t\tself.model_path = None","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_data(input_path):\n\t\"\"\"Parse the data from annotation file\n\t\n\tArgs:\n\t\tinput_path: annotation file path\n\n\tReturns:\n\t\tall_data: list(filepath, width, height, list(bboxes))\n\t\tclasses_count: dict{key:class_name, value:count_num} \n\t\t\te.g. {'Car': 2383, 'Mobile phone': 1108, 'Person': 3745}\n\t\tclass_mapping: dict{key:class_name, value: idx}\n\t\t\te.g. {'Car': 0, 'Mobile phone': 1, 'Person': 2}\n\t\"\"\"\n\tfound_bg = False\n\tall_imgs = {}\n\n\tclasses_count = {}\n\n\tclass_mapping = {}\n\n\tvisualise = True\n\n\ti = 1\n\t\n\twith open(input_path,'r') as f:\n\n\t\tprint('Parsing annotation files')\n\n\t\tfor line in f:\n\n\t\t\t# Print process\n\t\t\tsys.stdout.write('\\r'+'idx=' + str(i))\n\t\t\ti += 1\n\n\t\t\tline_split = line.strip().split(',')\n\n\t\t\t# Make sure the info saved in annotation file matching the format (path_filename, x1, y1, x2, y2, class_name)\n\t\t\t# Note:\n\t\t\t#\tOne path_filename might has several classes (class_name)\n\t\t\t#\tx1, y1, x2, y2 are the pixel value of the origial image, not the ratio value\n\t\t\t#\t(x1, y1) top left coordinates; (x2, y2) bottom right coordinates\n\t\t\t#   x1,y1-------------------\n\t\t\t#\t|\t\t\t\t\t\t|\n\t\t\t#\t|\t\t\t\t\t\t|\n\t\t\t#\t|\t\t\t\t\t\t|\n\t\t\t#\t|\t\t\t\t\t\t|\n\t\t\t#\t---------------------x2,y2\n\n\t\t\t(filename,x1,y1,x2,y2,class_name) = line_split\n\n\t\t\tif class_name not in classes_count:\n\t\t\t\tclasses_count[class_name] = 1\n\t\t\telse:\n\t\t\t\tclasses_count[class_name] += 1\n\n\t\t\tif class_name not in class_mapping:\n\t\t\t\tif class_name == 'bg' and found_bg == False:\n\t\t\t\t\tprint('Found class name with special name bg. Will be treated as a background region (this is usually for hard negative mining).')\n\t\t\t\t\tfound_bg = True\n\t\t\t\tclass_mapping[class_name] = len(class_mapping)\n\n\t\t\tif filename not in all_imgs:\n\t\t\t\tall_imgs[filename] = {}\n\t\t\t\t\n\t\t\t\timg = cv2.imread(filename)\n\t\t\t\t(rows,cols) = img.shape[:2]\n\t\t\t\tall_imgs[filename]['filepath'] = filename\n\t\t\t\tall_imgs[filename]['width'] = cols\n\t\t\t\tall_imgs[filename]['height'] = rows\n\t\t\t\tall_imgs[filename]['bboxes'] = []\n\t\t\t\t# if np.random.randint(0,6) > 0:\n\t\t\t\t# \tall_imgs[filename]['imageset'] = 'trainval'\n\t\t\t\t# else:\n\t\t\t\t# \tall_imgs[filename]['imageset'] = 'test'\n\n\t\t\tall_imgs[filename]['bboxes'].append({'class': class_name, 'x1': int(x1), 'x2': int(x2), 'y1': int(y1), 'y2': int(y2)})\n\n\n\t\tall_data = []\n\t\tfor key in all_imgs:\n\t\t\tall_data.append(all_imgs[key])\n\t\t\n\t\t# make sure the bg class is last in the list\n\t\tif found_bg:\n\t\t\tif class_mapping['bg'] != len(class_mapping) - 1:\n\t\t\t\tkey_to_switch = [key for key in class_mapping.keys() if class_mapping[key] == len(class_mapping)-1][0]\n\t\t\t\tval_to_switch = class_mapping['bg']\n\t\t\t\tclass_mapping['bg'] = len(class_mapping) - 1\n\t\t\t\tclass_mapping[key_to_switch] = val_to_switch\n\t\t\n\t\treturn all_data, classes_count, class_mapping","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class RoiPoolingConv(Layer):\n    '''ROI pooling layer for 2D inputs.\n    See Spatial Pyramid Pooling in Deep Convolutional Networks for Visual Recognition,\n    K. He, X. Zhang, S. Ren, J. Sun\n    # Arguments\n        pool_size: int\n            Size of pooling region to use. pool_size = 7 will result in a 7x7 region.\n        num_rois: number of regions of interest to be used\n    # Input shape\n        list of two 4D tensors [X_img,X_roi] with shape:\n        X_img:\n        `(1, rows, cols, channels)`\n        X_roi:\n        `(1,num_rois,4)` list of rois, with ordering (x,y,w,h)\n    # Output shape\n        3D tensor with shape:\n        `(1, num_rois, channels, pool_size, pool_size)`\n    '''\n    def __init__(self, pool_size, num_rois, **kwargs):\n\n        self.dim_ordering = K.common.image_dim_ordering()\n        self.pool_size = pool_size\n        self.num_rois = num_rois\n\n        super(RoiPoolingConv, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        self.nb_channels = input_shape[0][3]   \n\n    def compute_output_shape(self, input_shape):\n        return None, self.num_rois, self.pool_size, self.pool_size, self.nb_channels\n\n    def call(self, x, mask=None):\n\n        assert(len(x) == 2)\n\n        # x[0] is image with shape (rows, cols, channels)\n        img = x[0]\n\n        # x[1] is roi with shape (num_rois,4) with ordering (x,y,w,h)\n        rois = x[1]\n\n        input_shape = K.shape(img)\n\n        outputs = []\n\n        for roi_idx in range(self.num_rois):\n\n            x = rois[0, roi_idx, 0]\n            y = rois[0, roi_idx, 1]\n            w = rois[0, roi_idx, 2]\n            h = rois[0, roi_idx, 3]\n\n            x = K.cast(x, 'int32')\n            y = K.cast(y, 'int32')\n            w = K.cast(w, 'int32')\n            h = K.cast(h, 'int32')\n\n            # Resized roi of the image to pooling size (7x7)\n            rs = tf.image.resize(img[:, y:y+h, x:x+w, :], (self.pool_size, self.pool_size))\n            outputs.append(rs)\n                \n\n        final_output = K.concatenate(outputs, axis=0)\n\n        # Reshape to (1, num_rois, pool_size, pool_size, nb_channels)\n        # Might be (1, 4, 7, 7, 3)\n        final_output = K.reshape(final_output, (1, self.num_rois, self.pool_size, self.pool_size, self.nb_channels))\n\n        # permute_dimensions is similar to transpose\n        final_output = K.permute_dimensions(final_output, (0, 1, 2, 3, 4))\n\n        return final_output\n    \n    \n    def get_config(self):\n        config = {'pool_size': self.pool_size,\n                  'num_rois': self.num_rois}\n        base_config = super(RoiPoolingConv, self).get_config()\n        return dict(list(base_config.items()) + list(config.items()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_img_output_length(width, height):\n    def get_output_length(input_length):\n        return input_length//16\n\n    return get_output_length(width), get_output_length(height)    \n\ndef nn_base(input_tensor=None, trainable=False):\n\n\n    input_shape = (None, None, 3)\n\n    if input_tensor is None:\n        img_input = Input(shape=input_shape)\n    else:\n        if not K.is_keras_tensor(input_tensor):\n            img_input = Input(tensor=input_tensor, shape=input_shape)\n        else:\n            img_input = input_tensor\n\n    bn_axis = 3\n\n    # Block 1\n    x = Conv2D(64, (3, 3), activation='relu', padding='same', name='block1_conv1')(img_input)\n    x = Conv2D(64, (3, 3), activation='relu', padding='same', name='block1_conv2')(x)\n    x = MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(x)\n\n    # Block 2\n    x = Conv2D(128, (3, 3), activation='relu', padding='same', name='block2_conv1')(x)\n    x = Conv2D(128, (3, 3), activation='relu', padding='same', name='block2_conv2')(x)\n    x = MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(x)\n\n    # Block 3\n    x = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv1')(x)\n    x = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv2')(x)\n    x = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv3')(x)\n    x = MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(x)\n\n    # Block 4\n    x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv1')(x)\n    x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv2')(x)\n    x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv3')(x)\n    x = MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')(x)\n\n    # Block 5\n    x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv1')(x)\n    x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv2')(x)\n    x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv3')(x)\n    # x = MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool')(x)\n\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def rpn_layer(base_layers, num_anchors):\n    \"\"\"Create a rpn layer\n        Step1: Pass through the feature map from base layer to a 3x3 512 channels convolutional layer\n                Keep the padding 'same' to preserve the feature map's size\n        Step2: Pass the step1 to two (1,1) convolutional layer to replace the fully connected layer\n                classification layer: num_anchors (9 in here) channels for 0, 1 sigmoid activation output\n                regression layer: num_anchors*4 (36 in here) channels for computing the regression of bboxes with linear activation\n    Args:\n        base_layers: vgg in here\n        num_anchors: 9 in here\n\n    Returns:\n        [x_class, x_regr, base_layers]\n        x_class: classification for whether it's an object\n        x_regr: bboxes regression\n        base_layers: vgg in here\n    \"\"\"\n    x = Conv2D(512, (3, 3), padding='same', activation='relu', kernel_initializer='normal', name='rpn_conv1')(base_layers)\n\n    x_class = Conv2D(num_anchors, (1, 1), activation='sigmoid', kernel_initializer='uniform', name='rpn_out_class')(x)\n    x_regr = Conv2D(num_anchors * 4, (1, 1), activation='linear', kernel_initializer='zero', name='rpn_out_regress')(x)\n\n    return [x_class, x_regr, base_layers]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def classifier_layer(base_layers, input_rois, num_rois, nb_classes = 4):\n    \"\"\"Create a classifier layer\n    \n    Args:\n        base_layers: vgg\n        input_rois: `(1,num_rois,4)` list of rois, with ordering (x,y,w,h)\n        num_rois: number of rois to be processed in one time (4 in here)\n\n    Returns:\n        list(out_class, out_regr)\n        out_class: classifier layer output\n        out_regr: regression layer output\n    \"\"\"\n\n    input_shape = (num_rois,7,7,512)\n\n    pooling_regions = 7\n\n    # out_roi_pool.shape = (1, num_rois, channels, pool_size, pool_size)\n    # num_rois (4) 7x7 roi pooling\n    out_roi_pool = RoiPoolingConv(pooling_regions, num_rois)([base_layers, input_rois])\n\n    # Flatten the convlutional layer and connected to 2 FC and 2 dropout\n    out = TimeDistributed(Flatten(name='flatten'))(out_roi_pool)\n    out = TimeDistributed(Dense(4096, activation='relu', name='fc1'))(out)\n    out = TimeDistributed(Dropout(0.5))(out)\n    out = TimeDistributed(Dense(4096, activation='relu', name='fc2'))(out)\n    out = TimeDistributed(Dropout(0.5))(out)\n\n    # There are two output layer\n    # out_class: softmax acivation function for classify the class name of the object\n    # out_regr: linear activation function for bboxes coordinates regression\n    out_class = TimeDistributed(Dense(nb_classes, activation='softmax', kernel_initializer='zero'), name='dense_class_{}'.format(nb_classes))(out)\n    # note: no regression target for bg class\n    out_regr = TimeDistributed(Dense(4 * (nb_classes-1), activation='linear', kernel_initializer='zero'), name='dense_regress_{}'.format(nb_classes))(out)\n\n    return [out_class, out_regr]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def union(au, bu, area_intersection):\n\tarea_a = (au[2] - au[0]) * (au[3] - au[1])\n\tarea_b = (bu[2] - bu[0]) * (bu[3] - bu[1])\n\tarea_union = area_a + area_b - area_intersection\n\treturn area_union\n\n\ndef intersection(ai, bi):\n\tx = max(ai[0], bi[0])\n\ty = max(ai[1], bi[1])\n\tw = min(ai[2], bi[2]) - x\n\th = min(ai[3], bi[3]) - y\n\tif w < 0 or h < 0:\n\t\treturn 0\n\treturn w*h\n\n\ndef iou(a, b):\n\t# a and b should be (x1,y1,x2,y2)\n\n\tif a[0] >= a[2] or a[1] >= a[3] or b[0] >= b[2] or b[1] >= b[3]:\n\t\treturn 0.0\n\n\tarea_i = intersection(a, b)\n\tarea_u = union(a, b, area_i)\n\n\treturn float(area_i) / float(area_u + 1e-6)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def calc_rpn(C, img_data, width, height, resized_width, resized_height, img_length_calc_function):\n\t\"\"\"(Important part!) Calculate the rpn for all anchors \n\t\tIf feature map has shape 38x50=1900, there are 1900x9=17100 potential anchors\n\t\n\tArgs:\n\t\tC: config\n\t\timg_data: augmented image data\n\t\twidth: original image width (e.g. 600)\n\t\theight: original image height (e.g. 800)\n\t\tresized_width: resized image width according to C.im_size (e.g. 300)\n\t\tresized_height: resized image height according to C.im_size (e.g. 400)\n\t\timg_length_calc_function: function to calculate final layer's feature map (of base model) size according to input image size\n\n\tReturns:\n\t\ty_rpn_cls: list(num_bboxes, y_is_box_valid + y_rpn_overlap)\n\t\t\ty_is_box_valid: 0 or 1 (0 means the box is invalid, 1 means the box is valid)\n\t\t\ty_rpn_overlap: 0 or 1 (0 means the box is not an object, 1 means the box is an object)\n\t\ty_rpn_regr: list(num_bboxes, 4*y_rpn_overlap + y_rpn_regr)\n\t\t\ty_rpn_regr: x1,y1,x2,y2 bunding boxes coordinates\n\t\"\"\"\n\tdownscale = float(C.rpn_stride) \n\tanchor_sizes = C.anchor_box_scales   # 128, 256, 512\n\tanchor_ratios = C.anchor_box_ratios  # 1:1, 1:2*sqrt(2), 2*sqrt(2):1\n\tnum_anchors = len(anchor_sizes) * len(anchor_ratios) # 3x3=9\n\n\t# calculate the output map size based on the network architecture\n\t(output_width, output_height) = img_length_calc_function(resized_width, resized_height)\n\n\tn_anchratios = len(anchor_ratios)    # 3\n\t\n\t# initialise empty output objectives\n\ty_rpn_overlap = np.zeros((output_height, output_width, num_anchors))\n\ty_is_box_valid = np.zeros((output_height, output_width, num_anchors))\n\ty_rpn_regr = np.zeros((output_height, output_width, num_anchors * 4))\n\n\tnum_bboxes = len(img_data['bboxes'])\n\n\tnum_anchors_for_bbox = np.zeros(num_bboxes).astype(int)\n\tbest_anchor_for_bbox = -1*np.ones((num_bboxes, 4)).astype(int)\n\tbest_iou_for_bbox = np.zeros(num_bboxes).astype(np.float32)\n\tbest_x_for_bbox = np.zeros((num_bboxes, 4)).astype(int)\n\tbest_dx_for_bbox = np.zeros((num_bboxes, 4)).astype(np.float32)\n\n\t# get the GT box coordinates, and resize to account for image resizing\n\tgta = np.zeros((num_bboxes, 4))\n\tfor bbox_num, bbox in enumerate(img_data['bboxes']):\n\t\t# get the GT box coordinates, and resize to account for image resizing\n\t\tgta[bbox_num, 0] = bbox['x1'] * (resized_width / float(width))\n\t\tgta[bbox_num, 1] = bbox['x2'] * (resized_width / float(width))\n\t\tgta[bbox_num, 2] = bbox['y1'] * (resized_height / float(height))\n\t\tgta[bbox_num, 3] = bbox['y2'] * (resized_height / float(height))\n\t\n\t# rpn ground truth\n\n\tfor anchor_size_idx in range(len(anchor_sizes)):\n\t\tfor anchor_ratio_idx in range(n_anchratios):\n\t\t\tanchor_x = anchor_sizes[anchor_size_idx] * anchor_ratios[anchor_ratio_idx][0]\n\t\t\tanchor_y = anchor_sizes[anchor_size_idx] * anchor_ratios[anchor_ratio_idx][1]\t\n\t\t\t\n\t\t\tfor ix in range(output_width):\t\t\t\t\t\n\t\t\t\t# x-coordinates of the current anchor box\t\n\t\t\t\tx1_anc = downscale * (ix + 0.5) - anchor_x / 2\n\t\t\t\tx2_anc = downscale * (ix + 0.5) + anchor_x / 2\t\n\t\t\t\t\n\t\t\t\t# ignore boxes that go across image boundaries\t\t\t\t\t\n\t\t\t\tif x1_anc < 0 or x2_anc > resized_width:\n\t\t\t\t\tcontinue\n\t\t\t\t\t\n\t\t\t\tfor jy in range(output_height):\n\n\t\t\t\t\t# y-coordinates of the current anchor box\n\t\t\t\t\ty1_anc = downscale * (jy + 0.5) - anchor_y / 2\n\t\t\t\t\ty2_anc = downscale * (jy + 0.5) + anchor_y / 2\n\n\t\t\t\t\t# ignore boxes that go across image boundaries\n\t\t\t\t\tif y1_anc < 0 or y2_anc > resized_height:\n\t\t\t\t\t\tcontinue\n\n\t\t\t\t\t# bbox_type indicates whether an anchor should be a target\n\t\t\t\t\t# Initialize with 'negative'\n\t\t\t\t\tbbox_type = 'neg'\n\n\t\t\t\t\t# this is the best IOU for the (x,y) coord and the current anchor\n\t\t\t\t\t# note that this is different from the best IOU for a GT bbox\n\t\t\t\t\tbest_iou_for_loc = 0.0\n\n\t\t\t\t\tfor bbox_num in range(num_bboxes):\n\t\t\t\t\t\t\n\t\t\t\t\t\t# get IOU of the current GT box and the current anchor box\n\t\t\t\t\t\tcurr_iou = iou([gta[bbox_num, 0], gta[bbox_num, 2], gta[bbox_num, 1], gta[bbox_num, 3]], [x1_anc, y1_anc, x2_anc, y2_anc])\n\t\t\t\t\t\t# calculate the regression targets if they will be needed\n\t\t\t\t\t\tif curr_iou > best_iou_for_bbox[bbox_num] or curr_iou > C.rpn_max_overlap:\n\t\t\t\t\t\t\tcx = (gta[bbox_num, 0] + gta[bbox_num, 1]) / 2.0\n\t\t\t\t\t\t\tcy = (gta[bbox_num, 2] + gta[bbox_num, 3]) / 2.0\n\t\t\t\t\t\t\tcxa = (x1_anc + x2_anc)/2.0\n\t\t\t\t\t\t\tcya = (y1_anc + y2_anc)/2.0\n\n\t\t\t\t\t\t\t# x,y are the center point of ground-truth bbox\n\t\t\t\t\t\t\t# xa,ya are the center point of anchor bbox (xa=downscale * (ix + 0.5); ya=downscale * (iy+0.5))\n\t\t\t\t\t\t\t# w,h are the width and height of ground-truth bbox\n\t\t\t\t\t\t\t# wa,ha are the width and height of anchor bboxe\n\t\t\t\t\t\t\t# tx = (x - xa) / wa\n\t\t\t\t\t\t\t# ty = (y - ya) / ha\n\t\t\t\t\t\t\t# tw = log(w / wa)\n\t\t\t\t\t\t\t# th = log(h / ha)\n\t\t\t\t\t\t\ttx = (cx - cxa) / (x2_anc - x1_anc)\n\t\t\t\t\t\t\tty = (cy - cya) / (y2_anc - y1_anc)\n\t\t\t\t\t\t\ttw = np.log((gta[bbox_num, 1] - gta[bbox_num, 0]) / (x2_anc - x1_anc))\n\t\t\t\t\t\t\tth = np.log((gta[bbox_num, 3] - gta[bbox_num, 2]) / (y2_anc - y1_anc))\n\t\t\t\t\t\t\n\t\t\t\t\t\tif img_data['bboxes'][bbox_num]['class'] != 'bg':\n\n\t\t\t\t\t\t\t# all GT boxes should be mapped to an anchor box, so we keep track of which anchor box was best\n\t\t\t\t\t\t\tif curr_iou > best_iou_for_bbox[bbox_num]:\n\t\t\t\t\t\t\t\tbest_anchor_for_bbox[bbox_num] = [jy, ix, anchor_ratio_idx, anchor_size_idx]\n\t\t\t\t\t\t\t\tbest_iou_for_bbox[bbox_num] = curr_iou\n\t\t\t\t\t\t\t\tbest_x_for_bbox[bbox_num,:] = [x1_anc, x2_anc, y1_anc, y2_anc]\n\t\t\t\t\t\t\t\tbest_dx_for_bbox[bbox_num,:] = [tx, ty, tw, th]\n\n\t\t\t\t\t\t\t# we set the anchor to positive if the IOU is >0.7 (it does not matter if there was another better box, it just indicates overlap)\n\t\t\t\t\t\t\tif curr_iou > C.rpn_max_overlap:\n\t\t\t\t\t\t\t\tbbox_type = 'pos'\n\t\t\t\t\t\t\t\tnum_anchors_for_bbox[bbox_num] += 1\n\t\t\t\t\t\t\t\t# we update the regression layer target if this IOU is the best for the current (x,y) and anchor position\n\t\t\t\t\t\t\t\tif curr_iou > best_iou_for_loc:\n\t\t\t\t\t\t\t\t\tbest_iou_for_loc = curr_iou\n\t\t\t\t\t\t\t\t\tbest_regr = (tx, ty, tw, th)\n\n\t\t\t\t\t\t\t# if the IOU is >0.3 and <0.7, it is ambiguous and no included in the objective\n\t\t\t\t\t\t\tif C.rpn_min_overlap < curr_iou < C.rpn_max_overlap:\n\t\t\t\t\t\t\t\t# gray zone between neg and pos\n\t\t\t\t\t\t\t\tif bbox_type != 'pos':\n\t\t\t\t\t\t\t\t\tbbox_type = 'neutral'\n\n\t\t\t\t\t# turn on or off outputs depending on IOUs\n\t\t\t\t\tif bbox_type == 'neg':\n\t\t\t\t\t\ty_is_box_valid[jy, ix, anchor_ratio_idx + n_anchratios * anchor_size_idx] = 1\n\t\t\t\t\t\ty_rpn_overlap[jy, ix, anchor_ratio_idx + n_anchratios * anchor_size_idx] = 0\n\t\t\t\t\telif bbox_type == 'neutral':\n\t\t\t\t\t\ty_is_box_valid[jy, ix, anchor_ratio_idx + n_anchratios * anchor_size_idx] = 0\n\t\t\t\t\t\ty_rpn_overlap[jy, ix, anchor_ratio_idx + n_anchratios * anchor_size_idx] = 0\n\t\t\t\t\telif bbox_type == 'pos':\n\t\t\t\t\t\ty_is_box_valid[jy, ix, anchor_ratio_idx + n_anchratios * anchor_size_idx] = 1\n\t\t\t\t\t\ty_rpn_overlap[jy, ix, anchor_ratio_idx + n_anchratios * anchor_size_idx] = 1\n\t\t\t\t\t\tstart = 4 * (anchor_ratio_idx + n_anchratios * anchor_size_idx)\n\t\t\t\t\t\ty_rpn_regr[jy, ix, start:start+4] = best_regr\n\n\t# we ensure that every bbox has at least one positive RPN region\n\n\tfor idx in range(num_anchors_for_bbox.shape[0]):\n\t\tif num_anchors_for_bbox[idx] == 0:\n\t\t\t# no box with an IOU greater than zero ...\n\t\t\tif best_anchor_for_bbox[idx, 0] == -1:\n\t\t\t\tcontinue\n\t\t\ty_is_box_valid[\n\t\t\t\tbest_anchor_for_bbox[idx,0], best_anchor_for_bbox[idx,1], best_anchor_for_bbox[idx,2] + n_anchratios *\n\t\t\t\tbest_anchor_for_bbox[idx,3]] = 1\n\t\t\ty_rpn_overlap[\n\t\t\t\tbest_anchor_for_bbox[idx,0], best_anchor_for_bbox[idx,1], best_anchor_for_bbox[idx,2] + n_anchratios *\n\t\t\t\tbest_anchor_for_bbox[idx,3]] = 1\n\t\t\tstart = 4 * (best_anchor_for_bbox[idx,2] + n_anchratios * best_anchor_for_bbox[idx,3])\n\t\t\ty_rpn_regr[\n\t\t\t\tbest_anchor_for_bbox[idx,0], best_anchor_for_bbox[idx,1], start:start+4] = best_dx_for_bbox[idx, :]\n\n\ty_rpn_overlap = np.transpose(y_rpn_overlap, (2, 0, 1))\n\ty_rpn_overlap = np.expand_dims(y_rpn_overlap, axis=0)\n\n\ty_is_box_valid = np.transpose(y_is_box_valid, (2, 0, 1))\n\ty_is_box_valid = np.expand_dims(y_is_box_valid, axis=0)\n\n\ty_rpn_regr = np.transpose(y_rpn_regr, (2, 0, 1))\n\ty_rpn_regr = np.expand_dims(y_rpn_regr, axis=0)\n\n\tpos_locs = np.where(np.logical_and(y_rpn_overlap[0, :, :, :] == 1, y_is_box_valid[0, :, :, :] == 1))\n\tneg_locs = np.where(np.logical_and(y_rpn_overlap[0, :, :, :] == 0, y_is_box_valid[0, :, :, :] == 1))\n\n\tnum_pos = len(pos_locs[0])\n\n\t# one issue is that the RPN has many more negative than positive regions, so we turn off some of the negative\n\t# regions. We also limit it to 256 regions.\n\tnum_regions = 256\n\n\tif len(pos_locs[0]) > num_regions/2:\n\t\tval_locs = random.sample(range(len(pos_locs[0])), len(pos_locs[0]) - num_regions/2)\n\t\ty_is_box_valid[0, pos_locs[0][val_locs], pos_locs[1][val_locs], pos_locs[2][val_locs]] = 0\n\t\tnum_pos = num_regions/2\n\n\tif len(neg_locs[0]) + num_pos > num_regions:\n\t\tval_locs = random.sample(range(len(neg_locs[0])), len(neg_locs[0]) - num_pos)\n\t\ty_is_box_valid[0, neg_locs[0][val_locs], neg_locs[1][val_locs], neg_locs[2][val_locs]] = 0\n\n\ty_rpn_cls = np.concatenate([y_is_box_valid, y_rpn_overlap], axis=1)\n\ty_rpn_regr = np.concatenate([np.repeat(y_rpn_overlap, 4, axis=1), y_rpn_regr], axis=1)\n\n\treturn np.copy(y_rpn_cls), np.copy(y_rpn_regr), num_pos","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_new_img_size(width, height, img_min_side=300):\n\tif width <= height:\n\t\tf = float(img_min_side) / width\n\t\tresized_height = int(f * height)\n\t\tresized_width = img_min_side\n\telse:\n\t\tf = float(img_min_side) / height\n\t\tresized_width = int(f * width)\n\t\tresized_height = img_min_side\n\n\treturn resized_width, resized_height\n\ndef augment(img_data, config, augment=True):\n\tassert 'filepath' in img_data\n\tassert 'bboxes' in img_data\n\tassert 'width' in img_data\n\tassert 'height' in img_data\n\n\timg_data_aug = copy.deepcopy(img_data)\n\n\timg = cv2.imread(img_data_aug['filepath'])\n\n\tif augment:\n\t\trows, cols = img.shape[:2]\n\n\t\tif config.use_horizontal_flips and np.random.randint(0, 2) == 0:\n\t\t\timg = cv2.flip(img, 1)\n\t\t\tfor bbox in img_data_aug['bboxes']:\n\t\t\t\tx1 = bbox['x1']\n\t\t\t\tx2 = bbox['x2']\n\t\t\t\tbbox['x2'] = cols - x1\n\t\t\t\tbbox['x1'] = cols - x2\n\n\t\tif config.use_vertical_flips and np.random.randint(0, 2) == 0:\n\t\t\timg = cv2.flip(img, 0)\n\t\t\tfor bbox in img_data_aug['bboxes']:\n\t\t\t\ty1 = bbox['y1']\n\t\t\t\ty2 = bbox['y2']\n\t\t\t\tbbox['y2'] = rows - y1\n\t\t\t\tbbox['y1'] = rows - y2\n\n\t\tif config.rot_90:\n\t\t\tangle = np.random.choice([0,90,180,270],1)[0]\n\t\t\tif angle == 270:\n\t\t\t\timg = np.transpose(img, (1,0,2))\n\t\t\t\timg = cv2.flip(img, 0)\n\t\t\telif angle == 180:\n\t\t\t\timg = cv2.flip(img, -1)\n\t\t\telif angle == 90:\n\t\t\t\timg = np.transpose(img, (1,0,2))\n\t\t\t\timg = cv2.flip(img, 1)\n\t\t\telif angle == 0:\n\t\t\t\tpass\n\n\t\t\tfor bbox in img_data_aug['bboxes']:\n\t\t\t\tx1 = bbox['x1']\n\t\t\t\tx2 = bbox['x2']\n\t\t\t\ty1 = bbox['y1']\n\t\t\t\ty2 = bbox['y2']\n\t\t\t\tif angle == 270:\n\t\t\t\t\tbbox['x1'] = y1\n\t\t\t\t\tbbox['x2'] = y2\n\t\t\t\t\tbbox['y1'] = cols - x2\n\t\t\t\t\tbbox['y2'] = cols - x1\n\t\t\t\telif angle == 180:\n\t\t\t\t\tbbox['x2'] = cols - x1\n\t\t\t\t\tbbox['x1'] = cols - x2\n\t\t\t\t\tbbox['y2'] = rows - y1\n\t\t\t\t\tbbox['y1'] = rows - y2\n\t\t\t\telif angle == 90:\n\t\t\t\t\tbbox['x1'] = rows - y2\n\t\t\t\t\tbbox['x2'] = rows - y1\n\t\t\t\t\tbbox['y1'] = x1\n\t\t\t\t\tbbox['y2'] = x2        \n\t\t\t\telif angle == 0:\n\t\t\t\t\tpass\n\n\timg_data_aug['width'] = img.shape[1]\n\timg_data_aug['height'] = img.shape[0]\n\treturn img_data_aug, img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_anchor_gt(all_img_data, C, img_length_calc_function, mode='train'):\n\t\"\"\" Yield the ground-truth anchors as Y (labels)\n\t\t\n\tArgs:\n\t\tall_img_data: list(filepath, width, height, list(bboxes))\n\t\tC: config\n\t\timg_length_calc_function: function to calculate final layer's feature map (of base model) size according to input image size\n\t\tmode: 'train' or 'test'; 'train' mode need augmentation\n\n\tReturns:\n\t\tx_img: image data after resized and scaling (smallest size = 300px)\n\t\tY: [y_rpn_cls, y_rpn_regr]\n\t\timg_data_aug: augmented image data (original image with augmentation)\n\t\tdebug_img: show image for debug\n\t\tnum_pos: show number of positive anchors for debug\n\t\"\"\"\n\twhile True:\n\n\t\tfor img_data in all_img_data:\n\t\t\ttry:\n\n\t\t\t\t# read in image, and optionally add augmentation\n\n\t\t\t\tif mode == 'train':\n\t\t\t\t\timg_data_aug, x_img = augment(img_data, C, augment=True)\n\t\t\t\telse:\n\t\t\t\t\timg_data_aug, x_img = augment(img_data, C, augment=False)\n\n\t\t\t\t(width, height) = (img_data_aug['width'], img_data_aug['height'])\n\t\t\t\t(rows, cols, _) = x_img.shape\n\n\t\t\t\tassert cols == width\n\t\t\t\tassert rows == height\n\n\t\t\t\t# get image dimensions for resizing\n\t\t\t\t(resized_width, resized_height) = get_new_img_size(width, height, C.im_size)\n\n\t\t\t\t# resize the image so that smalles side is length = 300px\n\t\t\t\tx_img = cv2.resize(x_img, (resized_width, resized_height), interpolation=cv2.INTER_CUBIC)\n\t\t\t\tdebug_img = x_img.copy()\n\n\t\t\t\ttry:\n\t\t\t\t\ty_rpn_cls, y_rpn_regr, num_pos = calc_rpn(C, img_data_aug, width, height, resized_width, resized_height, img_length_calc_function)\n\t\t\t\texcept:\n\t\t\t\t\tcontinue\n\n\t\t\t\t# Zero-center by mean pixel, and preprocess image\n\n\t\t\t\tx_img = x_img[:,:, (2, 1, 0)]  # BGR -> RGB\n\t\t\t\tx_img = x_img.astype(np.float32)\n\t\t\t\tx_img[:, :, 0] -= C.img_channel_mean[0]\n\t\t\t\tx_img[:, :, 1] -= C.img_channel_mean[1]\n\t\t\t\tx_img[:, :, 2] -= C.img_channel_mean[2]\n\t\t\t\tx_img /= C.img_scaling_factor\n\n\t\t\t\tx_img = np.transpose(x_img, (2, 0, 1))\n\t\t\t\tx_img = np.expand_dims(x_img, axis=0)\n\n\t\t\t\ty_rpn_regr[:, y_rpn_regr.shape[1]//2:, :, :] *= C.std_scaling\n\n\t\t\t\tx_img = np.transpose(x_img, (0, 2, 3, 1))\n\t\t\t\ty_rpn_cls = np.transpose(y_rpn_cls, (0, 2, 3, 1))\n\t\t\t\ty_rpn_regr = np.transpose(y_rpn_regr, (0, 2, 3, 1))\n\n\t\t\t\tyield np.copy(x_img), [np.copy(y_rpn_cls), np.copy(y_rpn_regr)], img_data_aug, debug_img, num_pos\n\n\t\t\texcept Exception as e:\n\t\t\t\tprint(e)\n\t\t\t\tcontinue","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lambda_rpn_regr = 1.0\nlambda_rpn_class = 1.0\n\nlambda_cls_regr = 1.0\nlambda_cls_class = 1.0\n\nepsilon = 1e-4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def rpn_loss_regr(num_anchors):\n    \"\"\"Loss function for rpn regression\n    Args:\n        num_anchors: number of anchors (9 in here)\n    Returns:\n        Smooth L1 loss function \n                           0.5*x*x (if x_abs < 1)\n                           x_abx - 0.5 (otherwise)\n    \"\"\"\n    def rpn_loss_regr_fixed_num(y_true, y_pred):\n\n        # x is the difference between true value and predicted vaue\n        x = y_true[:, :, :, 4 * num_anchors:] - y_pred\n\n        # absolute value of x\n        x_abs = K.abs(x)\n\n        # If x_abs <= 1.0, x_bool = 1\n        x_bool = K.cast(K.less_equal(x_abs, 1.0), tf.float32)\n\n        return lambda_rpn_regr * K.sum(\n            y_true[:, :, :, :4 * num_anchors] * (x_bool * (0.5 * x * x) + (1 - x_bool) * (x_abs - 0.5))) / K.sum(epsilon + y_true[:, :, :, :4 * num_anchors])\n\n    return rpn_loss_regr_fixed_num\n\n\ndef rpn_loss_cls(num_anchors):\n    \"\"\"Loss function for rpn classification\n    Args:\n        num_anchors: number of anchors (9 in here)\n        y_true[:, :, :, :9]: [0,1,0,0,0,0,0,1,0] means only the second and the eighth box is valid which contains pos or neg anchor => isValid\n        y_true[:, :, :, 9:]: [0,1,0,0,0,0,0,0,0] means the second box is pos and eighth box is negative\n    Returns:\n        lambda * sum((binary_crossentropy(isValid*y_pred,y_true))) / N\n    \"\"\"\n    def rpn_loss_cls_fixed_num(y_true, y_pred):\n\n            return lambda_rpn_class * K.sum(y_true[:, :, :, :num_anchors] * K.binary_crossentropy(y_pred[:, :, :, :], y_true[:, :, :, num_anchors:])) / K.sum(epsilon + y_true[:, :, :, :num_anchors])\n\n    return rpn_loss_cls_fixed_num\n\n\ndef class_loss_regr(num_classes):\n    \"\"\"Loss function for rpn regression\n    Args:\n        num_anchors: number of anchors (9 in here)\n    Returns:\n        Smooth L1 loss function \n                           0.5*x*x (if x_abs < 1)\n                           x_abx - 0.5 (otherwise)\n    \"\"\"\n    def class_loss_regr_fixed_num(y_true, y_pred):\n        x = y_true[:, :, 4*num_classes:] - y_pred\n        x_abs = K.abs(x)\n        x_bool = K.cast(K.less_equal(x_abs, 1.0), 'float32')\n        return lambda_cls_regr * K.sum(y_true[:, :, :4*num_classes] * (x_bool * (0.5 * x * x) + (1 - x_bool) * (x_abs - 0.5))) / K.sum(epsilon + y_true[:, :, :4*num_classes])\n    return class_loss_regr_fixed_num\n\n\ndef class_loss_cls(y_true, y_pred):\n    return lambda_cls_class * K.mean(categorical_crossentropy(y_true[0, :, :], y_pred[0, :, :]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def non_max_suppression_fast(boxes, probs, overlap_thresh=0.9, max_boxes=300):\n    # code used from here: http://www.pyimagesearch.com/2015/02/16/faster-non-maximum-suppression-python/\n    # if there are no boxes, return an empty list\n\n    # Process explanation:\n    #   Step 1: Sort the probs list\n    #   Step 2: Find the larget prob 'Last' in the list and save it to the pick list\n    #   Step 3: Calculate the IoU with 'Last' box and other boxes in the list. If the IoU is larger than overlap_threshold, delete the box from list\n    #   Step 4: Repeat step 2 and step 3 until there is no item in the probs list \n    if len(boxes) == 0:\n        return []\n\n    # grab the coordinates of the bounding boxes\n    x1 = boxes[:, 0]\n    y1 = boxes[:, 1]\n    x2 = boxes[:, 2]\n    y2 = boxes[:, 3]\n\n    np.testing.assert_array_less(x1, x2)\n    np.testing.assert_array_less(y1, y2)\n\n    # if the bounding boxes integers, convert them to floats --\n    # this is important since we'll be doing a bunch of divisions\n    if boxes.dtype.kind == \"i\":\n        boxes = boxes.astype(\"float\")\n\n    # initialize the list of picked indexes\t\n    pick = []\n\n    # calculate the areas\n    area = (x2 - x1) * (y2 - y1)\n\n    # sort the bounding boxes \n    idxs = np.argsort(probs)\n\n    # keep looping while some indexes still remain in the indexes\n    # list\n    while len(idxs) > 0:\n        # grab the last index in the indexes list and add the\n        # index value to the list of picked indexes\n        last = len(idxs) - 1\n        i = idxs[last]\n        pick.append(i)\n\n        # find the intersection\n\n        xx1_int = np.maximum(x1[i], x1[idxs[:last]])\n        yy1_int = np.maximum(y1[i], y1[idxs[:last]])\n        xx2_int = np.minimum(x2[i], x2[idxs[:last]])\n        yy2_int = np.minimum(y2[i], y2[idxs[:last]])\n\n        ww_int = np.maximum(0, xx2_int - xx1_int)\n        hh_int = np.maximum(0, yy2_int - yy1_int)\n\n        area_int = ww_int * hh_int\n\n        # find the union\n        area_union = area[i] + area[idxs[:last]] - area_int\n\n        # compute the ratio of overlap\n        overlap = area_int/(area_union + 1e-6)\n\n        # delete all indexes from the index list that have\n        idxs = np.delete(idxs, np.concatenate(([last],\n            np.where(overlap > overlap_thresh)[0])))\n\n        if len(pick) >= max_boxes:\n            break\n\n    # return only the bounding boxes that were picked using the integer data type\n    boxes = boxes[pick].astype(\"int\")\n    probs = probs[pick]\n    return boxes, probs\n\ndef apply_regr_np(X, T):\n    \"\"\"Apply regression layer to all anchors in one feature map\n\n    Args:\n        X: shape=(4, 18, 25) the current anchor type for all points in the feature map\n        T: regression layer shape=(4, 18, 25)\n\n    Returns:\n        X: regressed position and size for current anchor\n    \"\"\"\n    try:\n        x = X[0, :, :]\n        y = X[1, :, :]\n        w = X[2, :, :]\n        h = X[3, :, :]\n\n        tx = T[0, :, :]\n        ty = T[1, :, :]\n        tw = T[2, :, :]\n        th = T[3, :, :]\n\n        cx = x + w/2.\n        cy = y + h/2.\n        cx1 = tx * w + cx\n        cy1 = ty * h + cy\n\n        w1 = np.exp(tw.astype(np.float64)) * w\n        h1 = np.exp(th.astype(np.float64)) * h\n        x1 = cx1 - w1/2.\n        y1 = cy1 - h1/2.\n\n        x1 = np.round(x1)\n        y1 = np.round(y1)\n        w1 = np.round(w1)\n        h1 = np.round(h1)\n        return np.stack([x1, y1, w1, h1])\n    except Exception as e:\n        print(e)\n        return X\n    \n\ndef calc_iou(R, img_data, C, class_mapping):\n    \"\"\"Converts from (x1,y1,x2,y2) to (x,y,w,h) format\n\n    Args:\n        R: bboxes, probs\n    \"\"\"\n    bboxes = img_data['bboxes']\n    (width, height) = (img_data['width'], img_data['height'])\n    # get image dimensions for resizing\n    (resized_width, resized_height) = get_new_img_size(width, height, C.im_size)\n\n    gta = np.zeros((len(bboxes), 4))\n\n    for bbox_num, bbox in enumerate(bboxes):\n        # get the GT box coordinates, and resize to account for image resizing\n        # gta[bbox_num, 0] = (40 * (600 / 800)) / 16 = int(round(1.875)) = 2 (x in feature map)\n        gta[bbox_num, 0] = int(round(bbox['x1'] * (resized_width / float(width))/C.rpn_stride))\n        gta[bbox_num, 1] = int(round(bbox['x2'] * (resized_width / float(width))/C.rpn_stride))\n        gta[bbox_num, 2] = int(round(bbox['y1'] * (resized_height / float(height))/C.rpn_stride))\n        gta[bbox_num, 3] = int(round(bbox['y2'] * (resized_height / float(height))/C.rpn_stride))\n\n    x_roi = []\n    y_class_num = []\n    y_class_regr_coords = []\n    y_class_regr_label = []\n    IoUs = [] # for debugging only\n\n    # R.shape[0]: number of bboxes (=300 from non_max_suppression)\n    for ix in range(R.shape[0]):\n        (x1, y1, x2, y2) = R[ix, :]\n        x1 = int(round(x1))\n        y1 = int(round(y1))\n        x2 = int(round(x2))\n        y2 = int(round(y2))\n\n        best_iou = 0.0\n        best_bbox = -1\n        # Iterate through all the ground-truth bboxes to calculate the iou\n        for bbox_num in range(len(bboxes)):\n            curr_iou = iou([gta[bbox_num, 0], gta[bbox_num, 2], gta[bbox_num, 1], gta[bbox_num, 3]], [x1, y1, x2, y2])\n\n            # Find out the corresponding ground-truth bbox_num with larget iou\n            if curr_iou > best_iou:\n                best_iou = curr_iou\n                best_bbox = bbox_num\n\n        if best_iou < C.classifier_min_overlap:\n                continue\n        else:\n            w = x2 - x1\n            h = y2 - y1\n            x_roi.append([x1, y1, w, h])\n            IoUs.append(best_iou)\n\n            if C.classifier_min_overlap <= best_iou < C.classifier_max_overlap:\n                # hard negative example\n                cls_name = 'bg'\n            elif C.classifier_max_overlap <= best_iou:\n                cls_name = bboxes[best_bbox]['class']\n                cxg = (gta[best_bbox, 0] + gta[best_bbox, 1]) / 2.0\n                cyg = (gta[best_bbox, 2] + gta[best_bbox, 3]) / 2.0\n\n                cx = x1 + w / 2.0\n                cy = y1 + h / 2.0\n\n                tx = (cxg - cx) / float(w)\n                ty = (cyg - cy) / float(h)\n                tw = np.log((gta[best_bbox, 1] - gta[best_bbox, 0]) / float(w))\n                th = np.log((gta[best_bbox, 3] - gta[best_bbox, 2]) / float(h))\n            else:\n                print('roi = {}'.format(best_iou))\n                raise RuntimeError\n\n        class_num = class_mapping[cls_name]\n        class_label = len(class_mapping) * [0]\n        class_label[class_num] = 1\n        y_class_num.append(copy.deepcopy(class_label))\n        coords = [0] * 4 * (len(class_mapping) - 1)\n        labels = [0] * 4 * (len(class_mapping) - 1)\n        if cls_name != 'bg':\n            label_pos = 4 * class_num\n            sx, sy, sw, sh = C.classifier_regr_std\n            coords[label_pos:4+label_pos] = [sx*tx, sy*ty, sw*tw, sh*th]\n            labels[label_pos:4+label_pos] = [1, 1, 1, 1]\n            y_class_regr_coords.append(copy.deepcopy(coords))\n            y_class_regr_label.append(copy.deepcopy(labels))\n        else:\n            y_class_regr_coords.append(copy.deepcopy(coords))\n            y_class_regr_label.append(copy.deepcopy(labels))\n\n    if len(x_roi) == 0:\n        return None, None, None, None\n\n    # bboxes that iou > C.classifier_min_overlap for all gt bboxes in 300 non_max_suppression bboxes\n    X = np.array(x_roi)\n    # one hot code for bboxes from above => x_roi (X)\n    Y1 = np.array(y_class_num)\n    # corresponding labels and corresponding gt bboxes\n    Y2 = np.concatenate([np.array(y_class_regr_label),np.array(y_class_regr_coords)],axis=1)\n\n    return np.expand_dims(X, axis=0), np.expand_dims(Y1, axis=0), np.expand_dims(Y2, axis=0), IoUs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def rpn_to_roi(rpn_layer, regr_layer, C, dim_ordering, use_regr=True, max_boxes=300,overlap_thresh=0.9):\n\t\"\"\"Convert rpn layer to roi bboxes\n\n\tArgs: (num_anchors = 9)\n\t\trpn_layer: output layer for rpn classification \n\t\t\tshape (1, feature_map.height, feature_map.width, num_anchors)\n\t\t\tMight be (1, 18, 25, 18) if resized image is 400 width and 300\n\t\tregr_layer: output layer for rpn regression\n\t\t\tshape (1, feature_map.height, feature_map.width, num_anchors)\n\t\t\tMight be (1, 18, 25, 72) if resized image is 400 width and 300\n\t\tC: config\n\t\tuse_regr: Wether to use bboxes regression in rpn\n\t\tmax_boxes: max bboxes number for non-max-suppression (NMS)\n\t\toverlap_thresh: If iou in NMS is larger than this threshold, drop the box\n\n\tReturns:\n\t\tresult: boxes from non-max-suppression (shape=(300, 4))\n\t\t\tboxes: coordinates for bboxes (on the feature map)\n\t\"\"\"\n\tregr_layer = regr_layer / C.std_scaling\n\n\tanchor_sizes = C.anchor_box_scales   # (3 in here)\n\tanchor_ratios = C.anchor_box_ratios  # (3 in here)\n\n\tassert rpn_layer.shape[0] == 1\n\n\t(rows, cols) = rpn_layer.shape[1:3]\n\n\tcurr_layer = 0\n\n\t# A.shape = (4, feature_map.height, feature_map.width, num_anchors) \n\t# Might be (4, 18, 25, 18) if resized image is 400 width and 300\n\t# A is the coordinates for 9 anchors for every point in the feature map \n\t# => all 18x25x9=4050 anchors cooridnates\n\tA = np.zeros((4, rpn_layer.shape[1], rpn_layer.shape[2], rpn_layer.shape[3]))\n\n\tfor anchor_size in anchor_sizes:\n\t\tfor anchor_ratio in anchor_ratios:\n\t\t\t# anchor_x = (128 * 1) / 16 = 8  => width of current anchor\n\t\t\t# anchor_y = (128 * 2) / 16 = 16 => height of current anchor\n\t\t\tanchor_x = (anchor_size * anchor_ratio[0])/C.rpn_stride\n\t\t\tanchor_y = (anchor_size * anchor_ratio[1])/C.rpn_stride\n\t\t\t\n\t\t\t# curr_layer: 0~8 (9 anchors)\n\t\t\t# the Kth anchor of all position in the feature map (9th in total)\n\t\t\tregr = regr_layer[0, :, :, 4 * curr_layer:4 * curr_layer + 4] # shape => (18, 25, 4)\n\t\t\tregr = np.transpose(regr, (2, 0, 1)) # shape => (4, 18, 25)\n\n\t\t\t# Create 18x25 mesh grid\n\t\t\t# For every point in x, there are all the y points and vice versa\n\t\t\t# X.shape = (18, 25)\n\t\t\t# Y.shape = (18, 25)\n\t\t\tX, Y = np.meshgrid(np.arange(cols),np. arange(rows))\n\n\t\t\t# Calculate anchor position and size for each feature map point\n\t\t\tA[0, :, :, curr_layer] = X - anchor_x/2 # Top left x coordinate\n\t\t\tA[1, :, :, curr_layer] = Y - anchor_y/2 # Top left y coordinate\n\t\t\tA[2, :, :, curr_layer] = anchor_x       # width of current anchor\n\t\t\tA[3, :, :, curr_layer] = anchor_y       # height of current anchor\n\n\t\t\t# Apply regression to x, y, w and h if there is rpn regression layer\n\t\t\tif use_regr:\n\t\t\t\tA[:, :, :, curr_layer] = apply_regr_np(A[:, :, :, curr_layer], regr)\n\n\t\t\t# Avoid width and height exceeding 1\n\t\t\tA[2, :, :, curr_layer] = np.maximum(1, A[2, :, :, curr_layer])\n\t\t\tA[3, :, :, curr_layer] = np.maximum(1, A[3, :, :, curr_layer])\n\n\t\t\t# Convert (x, y , w, h) to (x1, y1, x2, y2)\n\t\t\t# x1, y1 is top left coordinate\n\t\t\t# x2, y2 is bottom right coordinate\n\t\t\tA[2, :, :, curr_layer] += A[0, :, :, curr_layer]\n\t\t\tA[3, :, :, curr_layer] += A[1, :, :, curr_layer]\n\n\t\t\t# Avoid bboxes drawn outside the feature map\n\t\t\tA[0, :, :, curr_layer] = np.maximum(0, A[0, :, :, curr_layer])\n\t\t\tA[1, :, :, curr_layer] = np.maximum(0, A[1, :, :, curr_layer])\n\t\t\tA[2, :, :, curr_layer] = np.minimum(cols-1, A[2, :, :, curr_layer])\n\t\t\tA[3, :, :, curr_layer] = np.minimum(rows-1, A[3, :, :, curr_layer])\n\n\t\t\tcurr_layer += 1\n\n\tall_boxes = np.reshape(A.transpose((0, 3, 1, 2)), (4, -1)).transpose((1, 0))  # shape=(4050, 4)\n\tall_probs = rpn_layer.transpose((0, 3, 1, 2)).reshape((-1))                   # shape=(4050,)\n\n\tx1 = all_boxes[:, 0]\n\ty1 = all_boxes[:, 1]\n\tx2 = all_boxes[:, 2]\n\ty2 = all_boxes[:, 3]\n\n\t# Find out the bboxes which is illegal and delete them from bboxes list\n\tidxs = np.where((x1 - x2 >= 0) | (y1 - y2 >= 0))\n\n\tall_boxes = np.delete(all_boxes, idxs, 0)\n\tall_probs = np.delete(all_probs, idxs, 0)\n\n\t# Apply non_max_suppression\n\t# Only extract the bboxes. Don't need rpn probs in the later process\n\tresult = non_max_suppression_fast(all_boxes, all_probs, overlap_thresh=overlap_thresh, max_boxes=max_boxes)[0]\n\n\treturn result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base_path = '/kaggle/working'\n\ntrain_path =  '/kaggle/working/annotation.txt' # Training data (annotation file)\n\nnum_rois = 4 # Number of RoIs to process at once.\n\n# Augmentation flag\nhorizontal_flips = True # Augment with horizontal flips in training. \nvertical_flips = True   # Augment with vertical flips in training. \nrot_90 = True           # Augment with 90 degree rotations in training. \n\noutput_weight_path = os.path.join(base_path, 'model/model_frcnn_vgg.hdf5')\n\nrecord_path = os.path.join(base_path, 'model/record.csv') # Record data (used to save the losses, classification accuracy and mean average precision)\n\nbase_weight_path = os.path.join(base_path, 'model/vgg16_weights_tf_dim_ordering_tf_kernels.h5')\n\nconfig_output_filename = os.path.join(base_path, 'model_vgg_config.pickle')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mkdir model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cd model/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!wget https://github.com/fchollet/deep-learning-models/releases/download/v0.1/vgg16_weights_tf_dim_ordering_tf_kernels.h5","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"!wget https://github.com/fchollet/deep-learning-models/releases/download/v0.1/vgg16_weights_tf_dim_ordering_tf_kernels.h5\nvgg_model.load_weights('model/vgg16_weights_tf_dim_ordering_tf_kernels.h5')\n\n!wget --no-check-certificate -q \"https://docs.google.com/uc?export=download&id=18ptTK4Vbeokqpux8Onr0OmwUP9ipmcYO\" -O cfg/rsna_yolov3.cfg_train","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create the config\nC = Config()\n\nC.use_horizontal_flips = horizontal_flips\nC.use_vertical_flips = vertical_flips\nC.rot_90 = rot_90\n\nC.record_path = record_path\nC.model_path = output_weight_path\nC.num_rois = num_rois\n\nC.base_net_weights = base_weight_path","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#--------------------------------------------------------#\n# This step will spend some time to load the data        #\n#--------------------------------------------------------#\nst = time.time()\ntrain_imgs, classes_count, class_mapping = get_data(train_path)\nprint()\nprint('Spend %0.2f mins to load the data' % ((time.time()-st)/60) )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if 'bg' not in classes_count:\n\tclasses_count['bg'] = 0\n\tclass_mapping['bg'] = len(class_mapping)\n# e.g.\n#    classes_count: {'Car': 2383, 'Mobile phone': 1108, 'Person': 3745, 'bg': 0}\n#    class_mapping: {'Person': 0, 'Car': 1, 'Mobile phone': 2, 'bg': 3}\nC.class_mapping = class_mapping\n\nprint('Training images per class:')\npprint.pprint(classes_count)\nprint('Num classes (including bg) = {}'.format(len(classes_count)))\nprint(class_mapping)\n\n# Save the configuration\nwith open(config_output_filename, 'wb') as config_f:\n\tpickle.dump(C,config_f)\n\tprint('Config has been written to {}, and can be loaded when testing to ensure correct results'.format(config_output_filename))\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}