{"cells":[{"metadata":{},"cell_type":"markdown","source":"Try to solve the problem using Region Proposal Network of Faster R-CNN. Not a good solution and not an efficient implementation, just for understanding RPN. Implement RPN with numpy and keras."},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport time\nimport pydicom\nimport matplotlib.pyplot as plt\nimport cv2\nimport tensorflow.keras as keras\nimport keras.backend as K","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"RPN - utitly functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def rect_convert_x1y1wh_to_x1y1x2y2(rect_x1y1wh):\n    rect = np.array(rect_x1y1wh)\n\n    if rect.ndim == 1:\n        rect = rect.reshape((1, 4))\n\n    rect[:, 2] = rect[:, 0] + rect[:, 2]\n    rect[:, 3] = rect[:, 1] + rect[:, 3]\n\n    if rect.shape[0] == 1:\n        rect = rect.reshape(-1)\n\n    return rect\n\ndef rect_convert_x1y1wh_to_cxcywh(rect_x1y1wh):\n    rect = np.array(rect_x1y1wh)\n\n    if rect.ndim == 1:\n        rect = rect.reshape((1, 4))\n\n    rect[:, 0] = rect[:, 0] + rect[:, 2] // 2\n    rect[:, 1] = rect[:, 1] + rect[:, 3] // 2\n\n    if rect.shape[0] == 1:\n        rect = rect.reshape(-1)\n\n    return rect\n\ndef rect_convert_cxcywh_to_x1y1x2y2(rect_cxcywh):\n    rect = np.array(rect_cxcywh)\n\n    if rect.ndim == 1:\n        rect = rect.reshape((1, 4))\n\n    rect[:, 0] = rect[:, 0] - rect[:, 2] // 2\n    rect[:, 1] = rect[:, 1] - rect[:, 3] // 2\n    rect[:, 2] = rect[:, 0] + rect[:, 2]\n    rect[:, 3] = rect[:, 1] + rect[:, 3]\n\n    if rect.shape[0] == 1:\n        rect = rect.reshape(-1)\n\n    return rect\n\ndef rect_convert_cxcywh_to_x1y1wh(rect_cxcywh):\n    rect = np.array(rect_cxcywh)\n\n    if rect.ndim == 1:\n        rect = rect.reshape((1, 4))\n\n    rect[:, 0] = rect[:, 0] - rect[:, 2] // 2\n    rect[:, 1] = rect[:, 1] - rect[:, 3] // 2\n\n    if rect.shape[0] == 1:\n        rect = rect.reshape(-1)\n\n    return rect\n\ndef rect_convert_x1y1x2y2_to_cxcywh(rect_x1y1x2y2):\n    rect = np.array(rect_x1y1x2y2)\n\n    if rect.ndim == 1:\n        rect = rect.reshape((1, 4))\n\n    rect[:, 2] = rect[:, 2] - rect[:, 0]\n    rect[:, 3] = rect[:, 3] - rect[:, 1]\n    rect[:, 0] = rect[:, 0] + rect[:, 2] // 2\n    rect[:, 1] = rect[:, 1] + rect[:, 3] // 2\n    \n    if rect.shape[0] == 1:\n        rect = rect.reshape(-1)\n\n    return rect\n\ndef rect_convert_x1y1x2y2_to_x1y1wh(rect_x1y1x2y2):\n    rect = np.array(rect_x1y1x2y2)\n\n    if rect.ndim == 1:\n        rect = rect.reshape((1, 4))\n\n    rect[:, 2] = rect[:, 2] - rect[:, 0]\n    rect[:, 3] = rect[:, 3] - rect[:, 1]\n\n    if rect.shape[0] == 1:\n        rect = rect.reshape(-1)\n\n    return rect\n\ndef rect_clip_outside(rect_cxcywh, width, height):\n    rect_x1y1x2y2 = rect_convert_cxcywh_to_x1y1x2y2(rect_cxcywh)\n\n    if rect_x1y1x2y2.ndim == 1:\n        rect_x1y1x2y2 = rect_x1y1x2y2.reshape((1, 4))\n\n    rect_x1y1x2y2[:, 0] = np.maximum(0, rect_x1y1x2y2[:, 0])\n    rect_x1y1x2y2[:, 1] = np.maximum(0, rect_x1y1x2y2[:, 1])\n    rect_x1y1x2y2[:, 2] = np.maximum(0, np.minimum(width, rect_x1y1x2y2[:, 2]))\n    rect_x1y1x2y2[:, 3] = np.maximum(0, np.minimum(height, rect_x1y1x2y2[:, 3]))\n\n    result = rect_convert_x1y1x2y2_to_cxcywh(rect_x1y1x2y2)\n    return result\n\ndef tool_iou(base_rect_x1y1x2y2, compare_rects_x1y1x2y2):\n    base_area = (base_rect_x1y1x2y2[2] - base_rect_x1y1x2y2[0]) * (base_rect_x1y1x2y2[3] - base_rect_x1y1x2y2[1])\n    compare_areas = (compare_rects_x1y1x2y2[:, 2] - compare_rects_x1y1x2y2[:, 0]) * (compare_rects_x1y1x2y2[:, 3] - compare_rects_x1y1x2y2[:, 1])\n\n    intersect_x1 = np.maximum(base_rect_x1y1x2y2[0], compare_rects_x1y1x2y2[:, 0])\n    intersect_y1 = np.maximum(base_rect_x1y1x2y2[1], compare_rects_x1y1x2y2[:, 1])\n    intersect_x2 = np.minimum(base_rect_x1y1x2y2[2], compare_rects_x1y1x2y2[:, 2])\n    intersect_y2 = np.minimum(base_rect_x1y1x2y2[3], compare_rects_x1y1x2y2[:, 3])\n    intersect_w = np.maximum(intersect_x2 - intersect_x1, 0)\n    intersect_h = np.maximum(intersect_y2 - intersect_y1, 0)\n    intersect_area = intersect_w * intersect_h\n    union_area = base_area + compare_areas - intersect_area\n\n    result = intersect_area / union_area\n\n    return result\n\ndef tool_nms(rects_cxcywh, scores, threshold):\n    r = []\n    s = []\n\n    # sore by score\n    indexes = np.argsort(scores)\n\n    complete_bool = np.zeros(indexes.shape, dtype = \"bool\")\n    index = len(indexes) - 1\n    while index >= 0:\n        # get highest score\n        i = indexes[index]\n\n        if not complete_bool[i]:\n            rect = rects_cxcywh[i]\n            r.append(rect)\n            s.append(scores[i])\n            complete_bool[i] = True\n\n            iou_result = None\n            compare_rects = rects_cxcywh[complete_bool == False]\n            if compare_rects.shape[0] > 0:\n                rect_x1y1x2y2 = rect_convert_cxcywh_to_x1y1x2y2(rect)\n                compare_rects_x1y1x2y2 = rect_convert_cxcywh_to_x1y1x2y2(compare_rects)\n                if compare_rects_x1y1x2y2.ndim == 1:\n                    compare_rects_x1y1x2y2 = compare_rects_x1y1x2y2.reshape((1, 4))\n\n                iou_result = tool_iou(rect_x1y1x2y2, compare_rects_x1y1x2y2)\n\n                complete_bool[complete_bool == False] = (iou_result > threshold)\n\n        index -= 1\n\n    return (np.array(r), np.array(s))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"RPN - anchors related functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"ANCHOR_SCALES = [ 64 * 64, 128 * 128, 256 * 256 ] # 512 * 512\nANCHOR_ASPECT_RATIO = [ 1.0, 0.5, 2.0 ]\nANCHOR_NUM = len(ANCHOR_SCALES) * len(ANCHOR_ASPECT_RATIO)\n\ndef generate_anchors(width, height, stride):\n    rows = height // stride\n    cols = width // stride\n\n    scales = len(ANCHOR_SCALES)\n    ratios = len(ANCHOR_ASPECT_RATIO)\n    k = scales * ratios\n\n    anchors = np.zeros((rows, cols, k, 4), dtype = \"int\")\n\n    for r in range(rows):\n        cy = r * stride\n        for c in range(cols):\n            cx = c * stride\n            k_num = 0\n            for scale in ANCHOR_SCALES:\n                for ratio in ANCHOR_ASPECT_RATIO:\n                    w = (int)(np.sqrt(scale * ratio))\n                    h = w // ratio\n                    anchors[r, c, k_num] = [cx, cy, w, h]\n                    k_num += 1\n\n    return anchors\n\ndef generate_anchors_train_target(anchors, gt_boxes, img_width, img_height, positive_threshold = 0.7, negative_threshold = 0.3):\n    gt_cls = np.ones(anchors.shape[:3]).flatten() * (-1)\n    gt_reg = np.zeros((anchors.shape[0] * anchors.shape[1] * anchors.shape[2], 4))\n\n    gt_box_count = len(gt_boxes)\n    positive_anchors_gt = np.zeros((anchors.shape[0] * anchors.shape[1] * anchors.shape[2], 4))\n    positive_anchors_iou = np.zeros((anchors.shape[:3])).flatten()\n    negative_anchors_map = np.zeros((anchors.shape[:3])).flatten()\n\n    anchors_flatten = anchors.reshape((-1, 4))\n\n    # check valid anchors\n    anchors_x1y1x2y2_faltten = rect_convert_cxcywh_to_x1y1x2y2(anchors_flatten)\n    valid_bool = (((anchors_x1y1x2y2_faltten[:, 0] < 0) \\\n                 + (anchors_x1y1x2y2_faltten[:, 1] < 0) \\\n                 + (anchors_x1y1x2y2_faltten[:, 2] > img_height) \\\n                 + (anchors_x1y1x2y2_faltten[:, 3] > img_width)) == False)\n    valid_anchors_count = np.sum(valid_bool)\n    positive_valid_anchors_gt = np.zeros((valid_anchors_count, 4))\n    positive_valid_anchors_iou = np.zeros((valid_anchors_count,))\n    negative_valid_anchors_map = np.zeros((valid_anchors_count,))\n\n    for gt_box in gt_boxes:\n        gt_box_x1y1x2y2 = rect_convert_cxcywh_to_x1y1x2y2(gt_box)\n        iou = tool_iou(gt_box_x1y1x2y2, anchors_x1y1x2y2_faltten[valid_bool])\n        positive_gt_bool = (iou > positive_threshold)\n        positive_gt_count = np.sum(positive_gt_bool)\n        if positive_gt_count > 0:\n            greater_iou_bool = (iou > positive_anchors_iou[valid_bool])\n            positive_valid_anchors_iou[greater_iou_bool] = iou[greater_iou_bool]\n            positive_valid_anchors_gt[greater_iou_bool] = gt_box\n        else:\n            iou_max_index = np.argmax(iou)\n            # just let the anchor box with max iou assign to this gt box\n            # set iou to 100 just to not assign to other gt box\n            positive_valid_anchors_iou[iou_max_index] = 100\n            positive_valid_anchors_gt[iou_max_index] = gt_box\n\n        negative_gt_bool = (iou < negative_threshold)\n        negative_valid_anchors_map[negative_gt_bool] += 1\n\n    positive_anchors_gt[valid_bool] = positive_valid_anchors_gt\n    positive_anchors_iou[valid_bool] = positive_valid_anchors_iou\n    negative_anchors_map[valid_bool] = negative_valid_anchors_map\n\n    # class: positive\n    positive_bool = (positive_anchors_iou > positive_threshold)\n    gt_cls[positive_bool] = 1\n    # class: negative\n    negative_bool = np.logical_and(negative_anchors_map == gt_box_count, np.logical_not(positive_bool))\n    gt_cls[negative_bool] = 0\n\n    # regression coefficient\n    positive_count = np.sum(positive_bool)\n    gt_reg_positive = np.zeros((positive_count, 4))\n    gt_reg_positive[:, 0] = (positive_anchors_gt[positive_bool][:, 0] - anchors_flatten[positive_bool][:, 0]) / anchors_flatten[positive_bool][:, 2]\n    gt_reg_positive[:, 1] = (positive_anchors_gt[positive_bool][:, 1] - anchors_flatten[positive_bool][:, 1]) / anchors_flatten[positive_bool][:, 3]\n    gt_reg_positive[:, 2] = np.log(positive_anchors_gt[positive_bool][:, 2] / anchors_flatten[positive_bool][:, 2])\n    gt_reg_positive[:, 3] = np.log(positive_anchors_gt[positive_bool][:, 3] / anchors_flatten[positive_bool][:, 3])\n    gt_reg[positive_bool] = gt_reg_positive\n\n    gt_cls = gt_cls.reshape(anchors.shape[:3])\n    gt_reg = gt_reg.reshape((anchors.shape[0], anchors.shape[1], -1))\n\n    return (gt_cls, gt_reg)\n\ndef generate_anchors_train_batch_target(anchors, gt_boxes, img_width, img_height, positive_threshold = 0.7, negative_threshold = 0.3):\n    (gt_cls, gt_reg) = generate_anchors_train_target(anchors, gt_boxes, img_width, img_height, positive_threshold, negative_threshold)\n\n    batch_size = 256\n    positive_size_max = 128\n\n    gt_cls_flatten = gt_cls.reshape((-1,))\n    gt_reg_flatten = gt_reg.reshape((-1, 4))\n\n    positive_map_bool = (gt_cls_flatten == 1)\n    negative_map_bool = (gt_cls_flatten == 0)\n\n    positive_count = np.sum(positive_map_bool)\n    negative_count = np.sum(negative_map_bool)\n\n    positive_size = min(positive_size_max, positive_count)\n    negative_size = min(batch_size - positive_size, negative_count)\n    batch_size = positive_size + negative_size\n\n    gt_cls_positive_samples_index = np.random.choice(positive_count, positive_size, replace = False)\n    gt_cls_negative_samples_index = np.random.choice(negative_count, negative_size, replace = False)\n\n    positive_sample_map_bool = positive_map_bool[positive_map_bool]\n    positive_sample_map_bool[:] = False\n    positive_sample_map_bool[gt_cls_positive_samples_index] = True\n    positive_map_bool[positive_map_bool] = positive_sample_map_bool\n\n    negative_sample_map_bool = negative_map_bool[negative_map_bool]\n    negative_sample_map_bool[:] = False\n    negative_sample_map_bool[gt_cls_negative_samples_index] = True\n    negative_map_bool[negative_map_bool] = negative_sample_map_bool\n    \n    gt_cls_mask = np.zeros(gt_cls.shape, dtype = gt_cls.dtype)\n    gt_reg_mask = np.zeros(gt_reg.shape, dtype = gt_reg.dtype)\n\n    gt_cls_mask_flatten = gt_cls_mask.reshape((-1,))\n    gt_reg_mask_flatten = gt_reg_mask.reshape((-1, 4))\n\n    gt_cls_mask_flatten[positive_map_bool] = 1\n    gt_cls_mask_flatten[negative_map_bool] = 1\n\n    gt_reg_mask_flatten[positive_map_bool] = 1\n\n    gt_cls_final = np.concatenate([gt_cls, gt_cls_mask], axis = -1)\n    gt_reg_final = np.concatenate([gt_reg, gt_reg_mask], axis = -1)\n\n    return (gt_cls_final, gt_reg_final)\n\ndef get_roi(anchors, anchors_cls, anchors_reg, img_width, img_height, positive_threshold = 0.5, nms_threshold = 0.7, top_n = 2000):\n    anchor_rows, anchor_cols, anchor_k, = anchors.shape[:3]\n\n    all_anchors = anchors.reshape((-1, 4))\n    all_anchors_cls = anchors_cls.reshape((-1,))\n    all_anchors_reg = anchors_reg.reshape((-1, 4))\n\n    positive_condition = (all_anchors_cls > positive_threshold)\n    positive_anchors = all_anchors[positive_condition]\n    positive_anchors_cls = all_anchors_cls[positive_condition]\n    positive_anchors_reg = all_anchors_reg[positive_condition]\n\n    # convert \"anchor box\" to \"refined box\" using \"regression coefficients\"\n    proposals = np.zeros((positive_anchors.shape), dtype = \"int\")\n    proposals[:, 0] = positive_anchors[:, 2] * positive_anchors_reg[:, 0] + positive_anchors[:, 0]\n    proposals[:, 1] = positive_anchors[:, 3] * positive_anchors_reg[:, 1] + positive_anchors[:, 1]\n    proposals[:, 2] = positive_anchors[:, 2] * np.exp(positive_anchors_reg[:, 2])\n    proposals[:, 3] = positive_anchors[:, 3] * np.exp(positive_anchors_reg[:, 3])\n    \n    # clip outside\n    proposals = rect_clip_outside(proposals, img_width, img_height)\n    if proposals.ndim == 1:\n        proposals = proposals.reshape((1, 4))\n    \n    # remove width <= 0 or height <= 0\n    proposals_valid = np.logical_not(np.logical_or(proposals[:, 2] <= 0, proposals[:, 3] <= 0))\n\n    # TODO: pre_top_n\n\n    if proposals[proposals_valid].shape[0] == 0:\n        return ([], [])\n\n    r, p = tool_nms(proposals[proposals_valid], positive_anchors_cls[proposals_valid], nms_threshold)\n\n    r = r[:top_n]\n    p = p[:top_n]\n\n    return (r, p)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"RPN - loss function"},{"metadata":{"trusted":true},"cell_type":"code","source":"def rpn_loss_cls(y_true, y_pred):\n    # y_true: batch, rows, cols, k * 2\n    # y_pred: batch, rows, cols, k\n    \n    spliter = K.shape(y_true)[-1] // 2\n    # important!!\n    mask = y_true[:, :, :, spliter:]\n    y_true_value = y_true[:, :, :, :spliter]\n\n    mask = K.reshape(mask, [-1])\n    y_true_value = K.reshape(y_true_value, [-1])\n    y_pred_value = K.reshape(y_pred, [-1])\n    \n    LAMBDA = 1.0\n    N_CLS = K.maximum(K.sum(mask), 1e-7) # 256.0\n    \n    return (LAMBDA * (K.sum(mask * K.binary_crossentropy(y_true_value, y_pred_value)) / N_CLS))\n\ndef rpn_loss_reg(y_true, y_pred):\n    # y_true: batch, rows, cols, k*4 * 2\n    # y_pred: batch, rows, cols, k*4\n    \n    def smooth_l1(y_true, y_pred):\n        diff = y_pred - y_true\n        diff_abs = K.abs(diff)\n        diff_abs_less_one_bool = K.less(diff_abs, 1)\n        diff_abs_less_one_mask = K.cast(diff_abs_less_one_bool, \"float32\")\n        \n        return (((diff * diff) * 0.5) * diff_abs_less_one_mask + (diff_abs - 0.5) * (1 - diff_abs_less_one_mask))\n    \n    spliter = K.shape(y_true)[-1] // 2\n    # important!!\n    mask = y_true[:, :, :, spliter:]\n    y_true_value = y_true[:, :, :, :spliter]\n\n    mask = K.reshape(mask, [-1, 4])\n    y_true_value = K.reshape(y_true_value, [-1, 4])\n    y_pred_value = K.reshape(y_pred, [-1, 4])\n    \n    LAMBDA = 1.0 # 10.0\n    N_REG = K.maximum(K.sum(mask), 1e-7) # 2400.0\n    \n    return (LAMBDA * (K.sum(mask * smooth_l1(y_true_value, y_pred_value)) / N_REG))\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"RPN - image preprocessing"},{"metadata":{"trusted":true},"cell_type":"code","source":"IMG_ORI_SIZE = 1024\nIMG_SIZE_MIN = 600\nIMG_SCALE = IMG_ORI_SIZE / IMG_SIZE_MIN\nIMG_MEAN_SUBSTRACTION = [103.939, 116.779, 123.68]\n\ndef preprocess_image(img):\n    img_ori_height, img_ori_width = img.shape[:2]\n\n    img_ori_min = min(img_ori_height, img_ori_width)\n    scale = IMG_SIZE_MIN / img_ori_min\n    img_new_height = (int)(img_ori_height * scale)\n    img_new_width = (int)(img_ori_width * scale)\n\n    scale_img = cv2.resize(img, (img_new_width, img_new_height))\n    \n    # TODO: max limitation\n\n    # mean substraction\n    scale_img = scale_img.astype(\"float\")\n    scale_img -= IMG_MEAN_SUBSTRACTION\n\n    return scale_img","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"RPN - model"},{"metadata":{"trusted":true},"cell_type":"code","source":"BASE_CNN_DOWNSAMPLE_RATIO = 16\nBASE_CNN_TRAINABLE_LAYER_START_INDEX = 0 #7\n\n# base cnn\ndef model_base_cnn():\n    vgg16Model = keras.applications.vgg16.VGG16(include_top = False, weights = None, pooling = None)\n    vgg16Model.load_weights(\"/kaggle/input/keras-pretrained-models/vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5\")\n    \n    # trainable\n    for (i, l) in enumerate(vgg16Model.layers):\n        if i < BASE_CNN_TRAINABLE_LAYER_START_INDEX:\n            l.trainable = False\n        else:\n            l.trainable = True\n            \n    last_conv_layer_output = vgg16Model.get_layer(\"block5_conv3\").output\n    \n    return (vgg16Model.input, last_conv_layer_output)\n\n# rpn core\ndef model_rpn_core(x, anchors_num):\n    x = keras.layers.Conv2D(512, 3, strides = 1, padding = \"same\", \n                            activation = \"relu\", \n                            kernel_initializer = keras.initializers.RandomNormal(mean=0.0, stddev=0.01, seed=None), \n                            name = \"rpn_conv\")(x)\n    x_cls = keras.layers.Conv2D(anchors_num, 1, strides = 1, \n                                activation = \"sigmoid\", \n                                kernel_initializer = keras.initializers.RandomNormal(mean=0.0, stddev=0.01, seed=None), \n                                name = \"rpn_conv_cls\")(x)\n    x_reg = keras.layers.Conv2D(anchors_num * 4, 1, strides = 1, \n                                activation = \"linear\", \n                                kernel_initializer = keras.initializers.RandomNormal(mean=0.0, stddev=0.01, seed=None), \n                                name = \"rpn_conv_reg\")(x)\n\n    return [ x_cls, x_reg ]\n\n# the rpn model\nbase_cnn_input, base_cnn_output = model_base_cnn()\nrpn_output = model_rpn_core(base_cnn_output, ANCHOR_NUM)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rpn_model = keras.models.Model(inputs = base_cnn_input, outputs = rpn_output)\nrpn_model.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Prepare training set"},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_TRAIN_IMG_DIR = \"/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_images\"\n\n# stage_2_train_labels.csv\n# stage_2_detailed_class_info.csv\nDATA_TRAIN_LABEL_PATH = \"/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_labels.csv\"\n\ntrain_labels = pd.read_csv(DATA_TRAIN_LABEL_PATH)\nprint(train_labels.head())\nprint(train_labels.shape)\n\ntrain_labels = train_labels.to_numpy()\nprint(train_labels.shape)\n\ntrain_img_files = os.listdir(DATA_TRAIN_IMG_DIR)\n\ntrain_info_dict = {}\n\nfor f in train_img_files:\n    file_path = os.path.join(DATA_TRAIN_IMG_DIR, f)\n#     dcm_data = pydicom.read_file(file_path)\n#     ori_img = dcm_data.pixel_array\n    key = os.path.splitext(f)[0]\n    train_info_dict[key] = { \"file_path\": file_path, \"img\": None, \"bbox\": [] }\n\nfor record in train_labels:\n    patientId = record[0]\n    target = record[5]\n    if target:\n        x = int(record[1] / IMG_SCALE)\n        y = int(record[2] / IMG_SCALE)\n        width = int(record[3] / IMG_SCALE)\n        height = int(record[4] / IMG_SCALE)\n        cx = x + width // 2\n        cy = y + height // 2\n        #use cx,cy,w,h\n        bbox = (cx, cy, width, height)\n        \n        train_info_dict[patientId][\"bbox\"].append(bbox)\n\n# just train the images that have bbox\ntrain_img_info_keys = [ k for (k, v) in train_info_dict.items() if len(v[\"bbox\"]) > 0 ]\ntrain_img_info_keys = np.array(train_img_info_keys)\n\nprint(\"number of all train image: {}\".format(len(train_info_dict)))\nprint(\"number of train image with bbox: {}\".format(len(train_img_info_keys)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Train RPN"},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN_INIT_LEARNING_RATE = 0.001\n\noptimizer = keras.optimizers.SGD(TRAIN_INIT_LEARNING_RATE, 0.9)\nrpn_model.compile(loss = [rpn_loss_cls, rpn_loss_reg], optimizer = optimizer)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ntrain_ts_start = time.time()\n\nTRAIN_TOTAL_EPOCH_NUM = 20\n\ntrain_img_count = len(train_img_info_keys)\nfor ep in range(TRAIN_TOTAL_EPOCH_NUM):\n    # shuffle the images\n    np.random.shuffle(train_img_info_keys)\n    \n    loss_total = 0\n    loss_cls = 0\n    loss_reg = 0\n    \n    for (i, k) in enumerate(train_img_info_keys):\n        img_file_path = train_info_dict[k][\"file_path\"]\n        bbox = train_info_dict[k][\"bbox\"]\n        \n        dcm_data = pydicom.read_file(file_path)\n        ori_img = dcm_data.pixel_array\n        ori_img = cv2.cvtColor(ori_img, cv2.COLOR_GRAY2RGB)\n        img = preprocess_image(ori_img)\n        \n        anchors = generate_anchors(img.shape[1], img.shape[0], BASE_CNN_DOWNSAMPLE_RATIO)\n        (gt_cls, gt_reg) = generate_anchors_train_batch_target(anchors, bbox, img.shape[1], img.shape[0], 0.5, 0.3)\n        \n        X = np.expand_dims(img, axis = 0)\n        Y = [np.expand_dims(gt_cls, axis = 0), np.expand_dims(gt_reg, axis = 0)]\n        \n        losses = rpn_model.train_on_batch(X, Y)\n        loss_total += losses[0]\n        loss_cls += losses[1]\n        loss_reg += losses[2]\n        \n    print(\"Epoch {}: total loss: {}; cls loss: {}; reg loss: {}\".format(ep + 1, loss_total / train_img_count, loss_cls / train_img_count, loss_reg / train_img_count))\n        \ntrain_ts_end = time.time()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_ts_end - train_ts_start)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"raw","source":"Predict"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nDATA_TEST_IMG_DIR = \"/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_test_images\"\n\ntest_img_files = os.listdir(DATA_TEST_IMG_DIR)\ntest_img_files.sort()\n\ntest_predict_result = []\n\nfor (i, f) in enumerate(test_img_files):\n    file_path = os.path.join(DATA_TEST_IMG_DIR, f)\n    \n    dcm_data = pydicom.read_file(file_path)\n    ori_img = dcm_data.pixel_array\n    ori_img = cv2.cvtColor(ori_img, cv2.COLOR_GRAY2RGB)\n    img = preprocess_image(ori_img)\n    \n    X = np.expand_dims(img, axis = 0)\n    \n    Y = rpn_model.predict(X)\n    Y_cls = Y[0]\n    Y_reg = Y[1]\n    \n    (regions, probs) = get_roi(anchors, Y_cls, Y_reg, img.shape[1], img.shape[0], positive_threshold = 0.85, nms_threshold = 0.3, top_n = 4)\n    \n    predict_str = \"\"\n    if len(regions) > 0:\n        regions_final = rect_convert_cxcywh_to_x1y1wh(regions)\n        if regions_final.ndim == 1:\n            regions_final = regions_final.reshape((1, 4))\n        regions_final = regions_final * IMG_SCALE\n        str_list = []\n        for (r, p) in zip(regions_final, probs):\n            str_list.append(\"{:.2f} {:.0f} {:.0f} {:.0f} {:.0f}\".format(p, r[0], r[1], r[2], r[3]))\n        predict_str = \" \".join(str_list)\n        \n    test_predict_result.append(predict_str)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Output submission file"},{"metadata":{"trusted":true},"cell_type":"code","source":"# patientId,PredictionString\n# 0000a175-0e68-4ca4-b1af-167204a7e0bc,0.5 0 0 100 100 0.5 0 0 100 100\n\n# stage_2_sample_submission.csv\n\ntest_img_file_names = [ os.path.splitext(f)[0] for f in test_img_files ]\n\nresult_data_pd = pd.DataFrame(data = { \"patientId\": test_img_file_names, \"PredictionString\": test_predict_result })\nprint(result_data_pd.head())\nresult_data_pd.to_csv(\"submission.csv\", index = False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"References(paper, article, code):\n\nhttps://arxiv.org/abs/1506.01497\n\nhttps://tryolabs.com/blog/2018/01/18/faster-r-cnn-down-the-rabbit-hole-of-modern-object-detection/\nhttp://www.telesens.co/2018/03/11/object-detection-and-classification-using-r-cnns/#Implementation_Details_Training\n\nhttps://github.com/you359/Keras-FasterRCNN"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}