{"cells":[{"metadata":{},"cell_type":"markdown","source":"If you think it's useful, give me an upvote, thanks."},{"metadata":{},"cell_type":"markdown","source":"CenterNet Paper : https://arxiv.org/pdf/1904.07850.pdf\n\nAuthor repo : https://github.com/xingyizhou/CenterNet\n\nKeras repo : https://github.com/see--/keras-centernet"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport cv2\nfrom tqdm import tqdm_notebook as tqdm\nfrom glob import glob\nfrom math import floor\nfrom PIL import ImageFont, ImageDraw, Image\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.metrics import mean_absolute_error, mean_squared_error\n\nimport keras\nfrom keras.layers import Dense, Activation, Input, Conv2D, BatchNormalization, Add, UpSampling2D, ZeroPadding2D, Lambda\nfrom keras.models import Sequential, Model\nfrom keras.preprocessing import image\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.utils import get_file\nfrom keras.optimizers import Adam\nimport keras.backend as K\nimport tensorflow as tf\nimport gc\nimport os\n\nPATH = '../input/pku-autonomous-driving/'\nos.listdir(PATH)\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/pku-autonomous-driving/train.csv')\ntest = pd.read_csv('../input/pku-autonomous-driving/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# I - Main"},{"metadata":{"trusted":true},"cell_type":"code","source":"# def main():\n#     # 1. set heads of HourglassNet\n#     kwargs = {\n#             'num_stacks': 2,\n#             'cnv_dim': 256,\n#             'inres': (512, 512),\n#             }\n#     heads = {\n#             'car_pose': 6,\n#             'confidence': 1\n#             }\n    \n#     # 2. create model\n#     model = HourglassNetwork(heads=heads, **kwargs)\n#     model.load_weights('../input/centernet-objects-as-points/centernet_weights.hdf5', by_name=True)\n    \n#     # 3. train\n#     train_model(model, epoch=3, batch_size=8)\n    \n#     # 4.predict\n#     predict(model)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# II - Preprocessing\n"},{"metadata":{},"cell_type":"markdown","source":"## 1. mutual conversion from strings to coordinates"},{"metadata":{"trusted":true},"cell_type":"code","source":"def str_to_coords(s, names=['id', 'yaw', 'pitch', 'roll', 'x', 'y', 'z']):\n    coords = []\n    for l in np.array(s.split()).reshape([-1, 7]):\n        coords.append(dict(zip(names, l.astype('float'))))\n        if 'id' in coords[-1]:\n            coords[-1]['id'] = int(coords[-1]['id'])\n    return coords\n\ndef coords_to_str(coords):\n    s = []\n    for c in coords:\n        for n in range(7):\n            s.append(str(c[n]))\n    return ' '.join(s)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 2. convert world coordinates to pixel coordinates"},{"metadata":{"trusted":true},"cell_type":"code","source":"camera_matrix = np.array([[2304.5479, 0,  1686.2379],\n                          [0, 2305.8757, 1354.9849],\n                          [0, 0, 1]], dtype=np.float32)\n\ndef pixel_coords(s):\n    coords = str_to_coords(s)\n    xc = [c['x'] for c in coords]\n    yc = [c['y'] for c in coords]\n    zc = [c['z'] for c in coords]\n    P = np.array(list(zip(xc, yc, zc))).T\n    img_p = np.dot(camera_matrix, P).T\n    img_p[:, 0] /= img_p[:, 2]\n    img_p[:, 1] /= img_p[:, 2]\n    u = img_p[:, 0]\n    v = img_p[:, 1]\n    zc = img_p[:, 2]\n    return u, v","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 3. resize image\n> Input of CenterNet: 512 x 512 x 3 (W, H, 3)"},{"metadata":{"trusted":true},"cell_type":"code","source":"def resize_image(img, input_width = 512, input_height = 512):\n    img = cv2.resize(img, (input_width, input_height))\n    return (img / 255).astype('float32')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 4. create mask images"},{"metadata":{"trusted":true},"cell_type":"code","source":"def CreateMaskImages(imageName):\n\n    trainimage = cv2.imread(PATH  + \"/train_images/\" + imageName + '.jpg')\n    imagemask = cv2.imread(PATH + \"/train_masks/\" + imageName + \".jpg\",0)\n    try:\n        imagemaskinv = cv2.bitwise_not(imagemask)\n        res = cv2.bitwise_and(trainimage,trainimage,mask = imagemaskinv)\n        \n        # cut upper half,because it doesn't contain cars.\n        res = res[res.shape[0] // 2:]\n        return res\n    except:\n        trainimage = trainimage[trainimage.shape[0] // 2:]\n        return trainimage","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 5. Heatmap of center points: 128 x 128 x 1\n> Output of CenterNet: (W/R, H/R, C) - here, R = 4, Classification C = 1. \n>\n>(Center points with gaussian distributions is necessary to reduce the training loss when the model detect the points near the exact center.)"},{"metadata":{"trusted":true},"cell_type":"code","source":"def heatmap(u, v, output_width=128, output_height=128, sigma=1):\n    def get_heatmap(p_x, p_y):\n        X1 = np.linspace(1, output_width, output_width)\n        Y1 = np.linspace(1, output_height, output_height)\n        [X, Y] = np.meshgrid(X1, Y1)\n        X = X - floor(p_x)\n        Y = Y - floor(p_y)\n        D2 = X * X + Y * Y\n        E2 = 2.0 * sigma ** 2\n        Exponent = D2 / E2\n        heatmap = np.exp(-Exponent)\n        heatmap = heatmap[:, :, np.newaxis]\n        return heatmap\n    \n    output = np.zeros((128,128,1))\n    for i in range(len(u)):\n        heatmap = get_heatmap(u[i], v[i])\n        output[:,:] = np.maximum(output[:,:],heatmap[:,:])\n      \n    return output","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 6. Regression (yaw, pitch, roll, x, y, z) of the object pose: 128 x 128 x 6\nRegressing rotation directly is not a good choice, I may try this way later.\n\nhttps://www.kaggle.com/diegojohnson/a-way-to-regress-translation-and-rotation\n\nThe CenterNet paper use this method:\n\nhttps://www.kaggle.com/diegojohnson/a-better-way-to-regress-yaw"},{"metadata":{"trusted":true},"cell_type":"code","source":"def pose(s, u, v):\n    regr = np.zeros([128, 128, 6], dtype='float32')\n    coords = str_to_coords(s)\n    for p_x, p_y, regr_dict in zip(u, v, coords):\n        if p_x >= 0 and p_x < 128 and p_y >= 0 and p_y < 128:\n            regr_dict.pop('id')\n            regr[floor(p_y), floor(p_x)] = [regr_dict[n] for n in regr_dict]\n            \n    # x,y,z devide by 100\n    regr[:,:,-3:] /= 100  \n    return regr","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 7. Example"},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":false},"cell_type":"code","source":"def example(i):\n    fig, axes = plt.subplots(2, 4,figsize=(20,20))\n    plt.subplots_adjust(top=0.5)\n\n    img0 = CreateMaskImages(train['ImageId'][i])\n    img1 = resize_image(img0)\n    axes[0, 0].set_title('Mask Image as Input')\n    axes[0, 0].imshow(img1)\n\n    # Image height: img.shape[0] and v         Image width: img.shape[1] and u\n    u, v = pixel_coords(train['PredictionString'][i])\n    u = u * 128 / img0.shape[1]\n    v = (v - img0.shape[0]) * 128 / img0.shape[0]\n    hm = np.squeeze(heatmap(u,v))\n    axes[0, 1].set_title('Heatmap of Center Points as Output')\n    axes[0, 1].imshow(hm)\n\n    regr = pose(train['PredictionString'][i], u,v)\n    axes[0, 2].set_title('Yaw - Ground Truth')\n    axes[0, 2].imshow(regr[..., 0])\n\n    axes[0, 3].set_title('Pitch - Ground Truth')\n    axes[0, 3].imshow(regr[..., 1])\n\n    axes[1, 0].set_title('Roll - Ground Truth')\n    axes[1, 0].imshow(regr[..., 2])\n\n    axes[1, 1].set_title('X - Ground Truth')\n    axes[1, 1].imshow(regr[..., 3])\n\n    axes[1, 2].set_title('Y - Ground Truth')\n    axes[1, 2].imshow(regr[..., 4])\n\n    axes[1, 3].set_title('Z - Ground Truth')\n    axes[1, 3].imshow(regr[..., 5])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# show example of preprocessed datas, you can input any number you want (<4000)\nexample(8)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 6. Data Generator"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_generator(train, batch_size=3):\n    count=0\n    X = []\n    y1 = []\n    y2 = []\n    while True:\n        for i in range(len(train)):\n            img0 = CreateMaskImages(train['ImageId'][i])\n            img1 = resize_image(img0)\n            X.append(img1)\n            \n            u, v = pixel_coords(train['PredictionString'][i])\n            u = u * 128 / img0.shape[1]\n            v = (v - img0.shape[0]) * 128 / img0.shape[0]\n            hm = heatmap(u,v)\n            y2.append(hm)\n\n            p = pose(train['PredictionString'][i], u, v)\n            y1.append(p)\n\n            count+=1\n            if count == batch_size:\n                X_batch = np.array(X, dtype=np.float32)\n                y1_batch = np.array(y1, dtype=np.float32)\n                y2_batch = np.array(y2, dtype=np.float32)\n                \n                del X, y1, y2\n                gc.collect()\n                \n                count = 0\n                X = []\n                y1 = []\n                y2 = []\n                \n                yield(X_batch, {'car_pose.1.1': y1_batch, 'confidence.1.1': y2_batch})","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# III - Model"},{"metadata":{},"cell_type":"markdown","source":"## 1. backbone - HourglassNet\n\nfrom https://blog.csdn.net/caikw62/article/details/95673125\n![HourglassNet](https://img-blog.csdnimg.cn/20190716171510379.jpg?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NhaWt3NjI=,size_16,color_FFFFFF,t_70)"},{"metadata":{"trusted":true},"cell_type":"code","source":"def HourglassNetwork(heads, num_stacks, cnv_dim=256, inres=(512, 512), weights=False,\n                     dims=[256, 384, 384, 384, 512]):\n    \"\"\"Instantiates the Hourglass architecture.\n    Optionally loads weights pre-trained on COCO.\n    Note that the data format convention used by the model is\n    the one specified in your Keras config at `~/.keras/keras.json`.\n    # Arguments\n      num_stacks: number of hourglass modules.\n      cnv_dim: number of filters after the resolution is decreased.\n      inres: network input shape, should be a multiple of 128.\n      weights: one of `None` (random initialization),\n            'ctdet_coco' (pre-training on COCO for 2D object detection),\n            'hpdet_coco' (pre-training on COCO for human pose detection),\n            or the path to the weights file to be loaded.\n      dims: numbers of channels in the hourglass blocks.\n    # Returns\n      A Keras model instance.\n    # Raises\n      ValueError: in case of invalid argument for `weights`,\n          or invalid input shape.\n    \"\"\"\n    \n    input_layer = Input(shape=(inres[0], inres[1], 3), name='HGInput')\n    inter = pre(input_layer, cnv_dim)\n    prev_inter = None\n    outputs = []\n    for i in range(num_stacks):\n        prev_inter = inter\n        _heads, inter = hourglass_module(heads, inter, cnv_dim, i, dims)\n        if i == 1:\n            outputs.extend(_heads)\n        if i < num_stacks - 1:\n            inter_ = Conv2D(cnv_dim, 1, use_bias=False, name='inter_.%d.0' % i)(prev_inter)\n            inter_ = BatchNormalization(epsilon=1e-5, name='inter_.%d.1' % i)(inter_)\n\n            cnv_ = Conv2D(cnv_dim, 1, use_bias=False, name='cnv_.%d.0' % i)(inter)\n            cnv_ = BatchNormalization(epsilon=1e-5, name='cnv_.%d.1' % i)(cnv_)\n\n            inter = Add(name='inters.%d.inters.add' % i)([inter_, cnv_])\n            inter = Activation('relu', name='inters.%d.inters.relu' % i)(inter)\n            inter = residual(inter, cnv_dim, 'inters.%d' % i)\n\n    model = Model(inputs=input_layer, outputs=outputs)\n\n    # load weights\n    if weights:\n        weights_path = get_file('HourglassNet.hdf5',\n                          'https://github.com/see--/keras-centernet/releases/download/0.1.0/ctdet_coco_hg.hdf5',\n                          cache_subdir='hourglassnet', \n                          file_hash='ce01e92f75b533e3ff8e396c76d55d97ff3ec27e99b1bdac1d7b0d6dcf5d90eb')\n        model.load_weights(weights_path, by_name=True)\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def hourglass_module(heads, bottom, cnv_dim, hgid, dims):\n    # create left features , f1, f2, f4, f8, f16 and f32\n    lfs = left_features(bottom, hgid, dims)\n\n    # create right features, connect with left features\n    rf1 = right_features(lfs, hgid, dims)\n    rf1 = convolution(rf1, 3, cnv_dim, name='cnvs.%d' % hgid)\n\n    # add 1x1 conv with two heads, inter is sent to next stage\n    # head_parts is used for intermediate supervision\n    heads = create_heads(heads, rf1, hgid)\n    return heads, rf1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"![convolution](https://img-blog.csdnimg.cn/20190716171120608.jpg?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NhaWt3NjI=,size_16,color_FFFFFF,t_70)"},{"metadata":{"trusted":true},"cell_type":"code","source":"def convolution(_x, k, out_dim, name, stride=1):\n    padding = (k - 1) // 2\n    _x = ZeroPadding2D(padding=padding, name=name + '.pad')(_x)\n    _x = Conv2D(out_dim, k, strides=stride, use_bias=False, name=name + '.conv')(_x)\n    _x = BatchNormalization(epsilon=1e-5, name=name + '.bn')(_x)\n    _x = Activation('relu', name=name + '.relu')(_x)\n    return _x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"![residual](https://img-blog.csdnimg.cn/20190716171157803.jpg?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NhaWt3NjI=,size_16,color_FFFFFF,t_70)"},{"metadata":{"trusted":true},"cell_type":"code","source":"def residual(_x, out_dim, name, stride=1):\n    shortcut = _x\n    num_channels = K.int_shape(shortcut)[-1]\n    _x = ZeroPadding2D(padding=1, name=name + '.pad1')(_x)\n    _x = Conv2D(out_dim, 3, strides=stride, use_bias=False, name=name + '.conv1')(_x)\n    _x = BatchNormalization(epsilon=1e-5, name=name + '.bn1')(_x)\n    _x = Activation('relu', name=name + '.relu1')(_x)\n\n    _x = Conv2D(out_dim, 3, padding='same', use_bias=False, name=name + '.conv2')(_x)\n    _x = BatchNormalization(epsilon=1e-5, name=name + '.bn2')(_x)\n\n    if num_channels != out_dim or stride != 1:\n        shortcut = Conv2D(out_dim, 1, strides=stride, use_bias=False, name=name + '.shortcut.0')(\n            shortcut)\n        shortcut = BatchNormalization(epsilon=1e-5, name=name + '.shortcut.1')(shortcut)\n\n    _x = Add(name=name + '.add')([_x, shortcut])\n    _x = Activation('relu', name=name + '.relu')(_x)\n    return _x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"![pre](https://img-blog.csdnimg.cn/20190716170729501.jpg)"},{"metadata":{"trusted":true},"cell_type":"code","source":"def pre(_x, num_channels):\n    # front module, input to 1/4 resolution\n    _x = convolution(_x, 7, 128, name='pre.0', stride=2)\n    _x = residual(_x, num_channels, name='pre.1', stride=2)\n    return _x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def left_features(bottom, hgid, dims):\n    # create left half blocks for hourglass module\n    # f1, f2, f4 , f8, f16, f32 : 1, 1/2, 1/4 1/8, 1/16, 1/32 resolution\n    # 5 times reduce/increase: (256, 384, 384, 384, 512)\n    features = [bottom]\n    for kk, nh in enumerate(dims):\n        pow_str = ''\n        for _ in range(kk):\n            pow_str += '.center'\n        _x = residual(features[-1], nh, name='kps.%d%s.down.0' % (hgid, pow_str), stride=2)\n        _x = residual(_x, nh, name='kps.%d%s.down.1' % (hgid, pow_str))\n        features.append(_x)\n    return features","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def connect_left_right(left, right, num_channels, num_channels_next, name):\n    # left: 2 residual modules\n    left = residual(left, num_channels_next, name=name + 'skip.0')\n    left = residual(left, num_channels_next, name=name + 'skip.1')\n\n    # up: 2 times residual & nearest neighbour\n    out = residual(right, num_channels, name=name + 'out.0')\n    out = residual(out, num_channels_next, name=name + 'out.1')\n    out = UpSampling2D(name=name + 'out.upsampleNN')(out)\n    out = Add(name=name + 'out.add')([left, out])\n    return out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def bottleneck_layer(_x, num_channels, hgid):\n    # 4 residual blocks with 512 channels in the middle\n    pow_str = 'center.' * 5\n    _x = residual(_x, num_channels, name='kps.%d.%s0' % (hgid, pow_str))\n    _x = residual(_x, num_channels, name='kps.%d.%s1' % (hgid, pow_str))\n    _x = residual(_x, num_channels, name='kps.%d.%s2' % (hgid, pow_str))\n    _x = residual(_x, num_channels, name='kps.%d.%s3' % (hgid, pow_str))\n    return _x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def right_features(leftfeatures, hgid, dims):\n    rf = bottleneck_layer(leftfeatures[-1], dims[-1], hgid)\n    for kk in reversed(range(len(dims))):\n        pow_str = ''\n        for _ in range(kk):\n            pow_str += 'center.'\n        rf = connect_left_right(leftfeatures[kk], rf, dims[kk], dims[max(kk - 1, 0)], name='kps.%d.%s' % (hgid, pow_str))\n    return rf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_heads(heads, rf1, hgid):\n    _heads = []\n    for head in sorted(heads):\n        num_channels = heads[head]\n        _x = Conv2D(256, 3, use_bias=True, padding='same', name=head + '.%d.0.conv' % hgid)(rf1)\n        _x = Activation('relu', name=head + '.%d.0.relu' % hgid)(_x)\n        if head == 'confidence':\n            _x = Conv2D(num_channels, 1, activation='sigmoid', use_bias=True, name=head + '.%d.1' % hgid)(_x)\n        else:\n            _x = Conv2D(num_channels, 1, use_bias=True, name=head + '.%d.1' % hgid)(_x)\n        _heads.append(_x)\n    return _heads","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 2. Decode"},{"metadata":{"trusted":true},"cell_type":"code","source":"# use maxpooling as nms\ndef _nms(heat, kernel=3):\n    hmax = K.pool2d(heat, (kernel, kernel), padding='same', pool_mode='max')\n    keep = K.cast(K.equal(hmax, heat), K.floatx())\n    return heat * keep","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _ctdet_decode(hm, reg, k=100, output_stride=4):\n    hm = _nms(hm)\n    hm_shape = K.shape(hm)\n    reg_shape = K.shape(reg)\n    batch, width, cat = hm_shape[0], hm_shape[2], hm_shape[3]\n\n    hm_flat = K.reshape(hm, (batch, -1))\n    reg_flat = K.reshape(reg, (reg_shape[0], -1, reg_shape[-1]))\n    \n    def _process_sample(args):\n        _hm, _reg = args\n        _scores, _inds = tf.math.top_k(_hm, k=k, sorted=True)\n        _inds = K.cast(_inds / cat, 'int32')\n        _reg = K.gather(_reg, _inds)\n        \n        # get yaw, pitch, roll, x, y, z from regression\n        yaw =  _reg[..., 0]\n        pitch =  _reg[..., 1]\n        roll =  _reg[..., 2]\n        x =  _reg[..., 3] * 100\n        y =  _reg[..., 4] * 100\n        z =  _reg[..., 5] * 100\n\n        _detection = K.stack([yaw, pitch, roll, x, y, z, _scores], -1)\n        return _detection\n    \n    detections = K.map_fn(_process_sample, [hm_flat, reg_flat], dtype=K.floatx())\n    return detections","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def CtDetDecode(model, hm_index=1, reg_index=0, k=100, output_stride=4):\n    def _decode(args):\n        hm, reg = args\n        return _ctdet_decode(hm, reg, k=k, output_stride=output_stride)\n    output = Lambda(_decode)([model.outputs[i] for i in [hm_index, reg_index]])\n    model = Model(model.input, output)\n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# IV - Train"},{"metadata":{"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"def train_model(model,epoch, batch_size = 4):\n    # 1. choose the layers you want to train\n    n = 0\n    for layer in model.layers:\n        layer.trainable = False\n\n        n += 1\n        if n == 500:\n            break\n    \n    # 2. define loss function\n    def focal_loss(hm_true, hm_pred):\n        pos_mask = tf.cast(tf.equal(hm_true, 1), tf.float32)\n        neg_mask = tf.cast(tf.less(hm_true, 1), tf.float32)\n        neg_weights = tf.pow(1 - hm_true, 4)\n\n        pos_loss = -tf.math.log(tf.clip_by_value(hm_pred, 1e-4, 1)) * tf.pow(1 - hm_pred, 2) * pos_mask\n        neg_loss = -tf.math.log(tf.clip_by_value(1 - hm_pred, 1e-4, 1)) * tf.pow(hm_pred, 2) * neg_weights * neg_mask\n\n        num_pos = tf.reduce_sum(pos_mask)\n        pos_loss = tf.reduce_sum(pos_loss)\n        neg_loss = tf.reduce_sum(neg_loss)\n\n        cls_loss = tf.cond(tf.greater(num_pos, 0), lambda: (pos_loss + neg_loss) / num_pos, lambda: neg_loss)\n        return cls_loss\n    \n    def l1_loss(y_true, y_pred):\n        mask = tf.zeros_like(y_true, dtype=tf.float32)\n        mask = tf.equal(y_true, mask)\n        mask = tf.cast(mask, tf.float32)\n        mask = tf.reduce_sum(mask, axis=-1)\n        \n        one = tf.ones_like(mask)\n        zero = tf.zeros_like(mask)\n        mask = tf.where(mask == 6, x=zero, y=one)\n        mask = tf.tile(tf.expand_dims(mask, axis=-1), (1, 1, 1, 6))\n        \n        total_loss = tf.reduce_sum(tf.abs(y_true - y_pred * mask))\n        reg_loss = total_loss / (tf.reduce_sum(mask) + 1e-4)\n        return reg_loss\n    \n    # 3. compile\n    model.compile(optimizer=Adam(),\n                   loss={'car_pose.1.1':l1_loss, 'confidence.1.1':focal_loss},\n                   loss_weights=[1, 1])\n    \n    # 4. fit\n    history = model.fit_generator(train_generator(train,batch_size=batch_size),\n                                  steps_per_epoch = len(train) // batch_size,\n                                  epochs = epoch\n                                  )\n    \n    model.save_weights('/kaggle/working/centernet_weights.hdf5')\n    \n    plt.title(\"model loss\")\n    plt.ylabel(\"loss\")\n    plt.xlabel(\"epoch\")\n    plt.plot(history.history['loss'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# V - Predict"},{"metadata":{"trusted":true},"cell_type":"code","source":"def TestMaskImages(imageName):\n\n    trainimage = cv2.imread(PATH  + \"/test_images/\" + imageName + '.jpg')\n    imagemask = cv2.imread(PATH + \"/test_masks/\" + imageName + \".jpg\",0)\n    try:\n        imagemaskinv = cv2.bitwise_not(imagemask)\n        res = cv2.bitwise_and(trainimage,trainimage,mask = imagemaskinv)\n        res = res[res.shape[0] // 2:]\n        return res\n    except:\n        trainimage = trainimage[trainimage.shape[0] // 2:]\n        return trainimage","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict(model):\n    modelx = CtDetDecode(model)\n    \n    def pred(i):\n        img = TestMaskImages(test['ImageId'][i])\n        img = resize_image(img)\n        X_batch = img[np.newaxis, :]\n\n        detections = modelx.predict(X_batch)\n        detections = np.squeeze(detections)\n\n\n        submission = []\n        for d in detections:\n            yaw, pitch, roll, x, y, z, score = d\n            if score < 0.3:\n                continue\n            else:\n                submission.append(d)\n\n        Prediction_string = coords_to_str(submission)\n        test['PredictionString'][i] = Prediction_string\n\n    for i in tqdm(range(len(test))):\n        pred(i)\n        \n    test.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 1. set heads of HourglassNet\nkwargs = {\n        'num_stacks': 2,\n        'cnv_dim': 256,\n        'inres': (512, 512),\n        }\nheads = {\n        'car_pose': 6,\n        'confidence': 1\n        }\n\n# 2. create model\nmodel = HourglassNetwork(heads=heads, **kwargs)\nmodel.load_weights('../input/centernet-hg/centernet_weights.hdf5', by_name=True)\n\n# 3. train\n# train_model(model, epoch=3, batch_size=16)\n\n# 4.predict\npredict(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def str_to_coords_test(s, names=['yaw', 'pitch', 'roll', 'x', 'y', 'z', 'confidence']):\n    coords = []\n    for l in np.array(s.split()).reshape([-1, 7]):\n        coords.append(dict(zip(names, l.astype('float'))))\n    return coords\n\ndef pixel_coords_test(s):\n    coords = str_to_coords_test(s)\n    xc = [c['x'] for c in coords]\n    yc = [c['y'] for c in coords]\n    zc = [c['z'] for c in coords]\n    P = np.array(list(zip(xc, yc, zc))).T\n    img_p = np.dot(camera_matrix, P).T\n    img_p[:, 0] /= img_p[:, 2]\n    img_p[:, 1] /= img_p[:, 2]\n    u = img_p[:, 0]\n    v = img_p[:, 1]\n    zc = img_p[:, 2]\n    return u, v ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def predict_example(i):\n    fig, axes = plt.subplots(1, 3,figsize=(20,20))\n    plt.subplots_adjust(top=0.5)\n\n    img0 = TestMaskImages(test['ImageId'][i])\n    img1 = resize_image(img0)\n    axes[0].set_title('Input Image')\n    axes[0].imshow(img1)\n\n    # Output Heatmap\n    X_batch = img1[np.newaxis, :]\n    detections = model.predict(X_batch)\n    axes[1].set_title('Output Heatmap')\n    axes[1].imshow(np.squeeze(detections[1]))\n\n    # Detection Result\n    u, v = pixel_coords_test(test['PredictionString'][i])\n    u = u * 128 / img0.shape[1]\n    v = (v - img0.shape[0]) * 128 / img0.shape[0]\n    print(u, v)\n    regr = pose(test['PredictionString'][i], u, v)\n    axes[2].set_title('Prediction Yaw')\n    axes[2].imshow(regr[..., 0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predict_example(6)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}