{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"[當前參考網站](https://www.kaggle.com/diegojohnson/centernet-objects-as-points/data)","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport cv2\nfrom tqdm import tqdm_notebook as tqdm\nfrom glob import glob\nfrom math import floor\nfrom PIL import ImageFont, ImageDraw, Image\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.metrics import mean_absolute_error, mean_squared_error\n\nimport keras\nfrom keras.layers import Dense, Activation, Input, Conv2D, BatchNormalization, Add, UpSampling2D, ZeroPadding2D, Lambda\nfrom keras.models import Sequential, Model\nfrom keras.preprocessing import image\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.utils import get_file\nfrom keras.optimizers import Adam\nimport keras.backend as K\nimport tensorflow as tf\nimport gc\nimport os","metadata":{"execution":{"iopub.status.busy":"2021-05-20T14:09:27.924314Z","iopub.execute_input":"2021-05-20T14:09:27.924847Z","iopub.status.idle":"2021-05-20T14:09:27.930583Z","shell.execute_reply.started":"2021-05-20T14:09:27.924815Z","shell.execute_reply":"2021-05-20T14:09:27.929816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PATH = \"../input/pku-autonomous-driving\"","metadata":{"execution":{"iopub.status.busy":"2021-05-20T14:09:27.940765Z","iopub.execute_input":"2021-05-20T14:09:27.941393Z","iopub.status.idle":"2021-05-20T14:09:27.94834Z","shell.execute_reply.started":"2021-05-20T14:09:27.941348Z","shell.execute_reply":"2021-05-20T14:09:27.947416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 訓練集和測試集列表 ＆ 需要被排除的瑕疵資料\ntrain = pd.read_csv('../input/pku-autonomous-driving/train.csv')\ntest = pd.read_csv('../input/pku-autonomous-driving/sample_submission.csv')\nbad_list = ['ID_1a5a10365',\n    'ID_1db0533c7',\n    'ID_53c3fe91a',\n    'ID_408f58e9f',\n    'ID_4445ae041',\n    'ID_bb1d991f6',\n    'ID_c44983aeb',\n    'ID_f30ebe4d4']\ntrain = train.loc[~train['ImageId'].isin(bad_list)]","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-05-20T14:09:27.96102Z","iopub.execute_input":"2021-05-20T14:09:27.961342Z","iopub.status.idle":"2021-05-20T14:09:28.006461Z","shell.execute_reply.started":"2021-05-20T14:09:27.961312Z","shell.execute_reply":"2021-05-20T14:09:28.00548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 字串資料 ←→ 帶標籤資料 → 像素座標\ndef str_to_coords(s, names=['id', 'yaw', 'pitch', 'roll', 'x', 'y', 'z']):\n    coords = []\n    for l in np.array(s.split()).reshape([-1, 7]):\n        coords.append(dict(zip(names, l.astype('float'))))\n        if 'id' in coords[-1]:\n            coords[-1]['id'] = int(coords[-1]['id'])\n    return coords\n\ndef coords_to_str(coords):\n    s = []\n    for c in coords:\n        for n in range(7):\n            s.append(str(c[n]))\n    return ' '.join(s)\n\n# 攝影機資料\ncamera_matrix = np.array([[2304.5479, 0,  1686.2379],\n                          [0, 2305.8757, 1354.9849],\n                          [0, 0, 1]], dtype=np.float32)\n\ndef pixel_coords(s):\n    coords = str_to_coords(s)\n    xc = [c['x'] for c in coords]\n    yc = [c['y'] for c in coords]\n    zc = [c['z'] for c in coords]\n    P = np.array(list(zip(xc, yc, zc))).T\n    img_p = np.dot(camera_matrix, P).T\n    img_p[:, 0] /= img_p[:, 2]\n    img_p[:, 1] /= img_p[:, 2]\n    u = img_p[:, 0]\n    v = img_p[:, 1]\n    zc = img_p[:, 2]\n    return u, v","metadata":{"execution":{"iopub.status.busy":"2021-05-20T14:09:28.009843Z","iopub.execute_input":"2021-05-20T14:09:28.01013Z","iopub.status.idle":"2021-05-20T14:09:28.019807Z","shell.execute_reply.started":"2021-05-20T14:09:28.010102Z","shell.execute_reply":"2021-05-20T14:09:28.0188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 重設尺寸：CenterNet 的輸入尺寸 = 512*512*3\ndef resize_image(img, input_width = 512, input_height = 512):\n    img = cv2.resize(img, (input_width, input_height))\n    return (img / 255).astype('float32')","metadata":{"execution":{"iopub.status.busy":"2021-05-20T14:09:28.021494Z","iopub.execute_input":"2021-05-20T14:09:28.021917Z","iopub.status.idle":"2021-05-20T14:09:28.035102Z","shell.execute_reply.started":"2021-05-20T14:09:28.021877Z","shell.execute_reply":"2021-05-20T14:09:28.03396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 有 mask image 的資料要遮一下\ndef CreateMaskImages(imageName):\n    trainimage = cv2.imread(PATH  + \"/train_images/\" + imageName + '.jpg')\n    imagemask = cv2.imread(PATH + \"/train_masks/\" + imageName + \".jpg\",0)\n    try:\n        imagemaskinv = cv2.bitwise_not(imagemask)\n        res = cv2.bitwise_and(trainimage,trainimage,mask = imagemaskinv)\n        \n        # cut upper half,because it doesn't contain cars.\n        res = res[res.shape[0] // 2:]\n        return res\n    except:\n        trainimage = trainimage[trainimage.shape[0] // 2:]\n        return trainimage","metadata":{"execution":{"iopub.status.busy":"2021-05-20T14:09:28.036489Z","iopub.execute_input":"2021-05-20T14:09:28.036968Z","iopub.status.idle":"2021-05-20T14:09:28.047515Z","shell.execute_reply.started":"2021-05-20T14:09:28.036933Z","shell.execute_reply":"2021-05-20T14:09:28.046441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Heatmap = 128*128*1\n# 他說：「模型檢測到精確中心附近的點時，具有高斯分佈的中心點對於減少訓練損失是必要的」\ndef heatmap(u, v, output_width=128, output_height=128, sigma=1):\n    \n    def get_heatmap(p_x, p_y):\n        X1 = np.linspace(1, output_width, output_width)\n        Y1 = np.linspace(1, output_height, output_height)\n        [X, Y] = np.meshgrid(X1, Y1)\n        X = X - floor(p_x)\n        Y = Y - floor(p_y)\n        D2 = X * X + Y * Y\n        E2 = 2.0 * sigma ** 2\n        Exponent = D2 / E2\n        heatmap = np.exp(-Exponent)\n        heatmap = heatmap[:, :, np.newaxis]\n        return heatmap\n    \n    output = np.zeros((128,128,1))\n    for i in range(len(u)):\n        heatmap = get_heatmap(u[i], v[i])\n        output[:,:] = np.maximum(output[:,:],heatmap[:,:])\n      \n    return output","metadata":{"execution":{"iopub.status.busy":"2021-05-20T14:09:28.049104Z","iopub.execute_input":"2021-05-20T14:09:28.049388Z","iopub.status.idle":"2021-05-20T14:09:28.060795Z","shell.execute_reply.started":"2021-05-20T14:09:28.04936Z","shell.execute_reply":"2021-05-20T14:09:28.059436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Regression = 128*128*6\n# 他說： 「直接回歸旋轉可能不是個好方法，之後會試試別的方法」\n# 如果這玩意兒會動的話，可以考慮試試其他方法\ndef pose(s, u, v):\n    regr = np.zeros([128, 128, 6], dtype='float32')\n    coords = str_to_coords(s)\n    for p_x, p_y, regr_dict in zip(u, v, coords):\n        if p_x >= 0 and p_x < 128 and p_y >= 0 and p_y < 128:\n            regr_dict.pop('id')\n            regr[floor(p_y), floor(p_x)] = [regr_dict[n] for n in regr_dict]\n            \n    # x,y,z devide by 100\n    regr[:,:,-3:] /= 100  \n    return regr","metadata":{"execution":{"iopub.status.busy":"2021-05-20T14:09:28.109714Z","iopub.execute_input":"2021-05-20T14:09:28.110064Z","iopub.status.idle":"2021-05-20T14:09:28.116441Z","shell.execute_reply.started":"2021-05-20T14:09:28.11003Z","shell.execute_reply":"2021-05-20T14:09:28.115386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def example(i):\n    fig, axes = plt.subplots(2, 4,figsize=(20,20))\n    plt.subplots_adjust(top=0.5)\n\n    img0 = CreateMaskImages(train['ImageId'][i])\n    img1 = resize_image(img0)\n    axes[0, 0].set_title('Mask Image as Input')\n    axes[0, 0].imshow(img1)\n\n    # Image height: img.shape[0] and v         Image width: img.shape[1] and u\n    u, v = pixel_coords(train['PredictionString'][i])\n    u = u * 128 / img0.shape[1]\n    v = (v - img0.shape[0]) * 128 / img0.shape[0]\n    hm = np.squeeze(heatmap(u,v))\n    axes[0, 1].set_title('Heatmap of Center Points as Output')\n    axes[0, 1].imshow(hm)\n\n    regr = pose(train['PredictionString'][i], u,v)\n    axes[0, 2].set_title('Yaw - Ground Truth')\n    axes[0, 2].imshow(regr[..., 0])\n\n    axes[0, 3].set_title('Pitch - Ground Truth')\n    axes[0, 3].imshow(regr[..., 1])\n\n    axes[1, 0].set_title('Roll - Ground Truth')\n    axes[1, 0].imshow(regr[..., 2])\n\n    axes[1, 1].set_title('X - Ground Truth')\n    axes[1, 1].imshow(regr[..., 3])\n\n    axes[1, 2].set_title('Y - Ground Truth')\n    axes[1, 2].imshow(regr[..., 4])\n\n    axes[1, 3].set_title('Z - Ground Truth')\n    axes[1, 3].imshow(regr[..., 5])","metadata":{"execution":{"iopub.status.busy":"2021-05-20T14:09:28.118009Z","iopub.execute_input":"2021-05-20T14:09:28.118308Z","iopub.status.idle":"2021-05-20T14:09:28.132431Z","shell.execute_reply.started":"2021-05-20T14:09:28.118279Z","shell.execute_reply":"2021-05-20T14:09:28.13137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 來看看前面的程式做了啥 (編號須小於4000)\nexample(2050)","metadata":{"execution":{"iopub.status.busy":"2021-05-20T14:09:28.134322Z","iopub.execute_input":"2021-05-20T14:09:28.134712Z","iopub.status.idle":"2021-05-20T14:09:29.259426Z","shell.execute_reply.started":"2021-05-20T14:09:28.134651Z","shell.execute_reply":"2021-05-20T14:09:29.258386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 看起來很高端的訓練集製造機\ndef train_generator(train, batch_size=3):\n    count=0\n    X = []\n    y1 = []\n    y2 = []\n    while True:\n        for i in range(len(train)):\n            img0 = CreateMaskImages(train['ImageId'][i])\n            img1 = resize_image(img0)\n            X.append(img1)\n            \n            u, v = pixel_coords(train['PredictionString'][i])\n            u = u * 128 / img0.shape[1]\n            v = (v - img0.shape[0]) * 128 / img0.shape[0]\n            hm = heatmap(u,v)\n            y2.append(hm)\n\n            p = pose(train['PredictionString'][i], u, v)\n            y1.append(p)\n\n            count+=1\n            if count == batch_size:\n                X_batch = np.array(X, dtype=np.float32)\n                y1_batch = np.array(y1, dtype=np.float32)\n                y2_batch = np.array(y2, dtype=np.float32)\n                \n                del X, y1, y2\n                gc.collect()\n                \n                count = 0\n                X = []\n                y1 = []\n                y2 = []\n                \n                yield(X_batch, {'car_pose.1.1': y1_batch, 'confidence.1.1': y2_batch})","metadata":{"execution":{"iopub.status.busy":"2021-05-20T14:09:29.261351Z","iopub.execute_input":"2021-05-20T14:09:29.261767Z","iopub.status.idle":"2021-05-20T14:09:29.272774Z","shell.execute_reply.started":"2021-05-20T14:09:29.261728Z","shell.execute_reply":"2021-05-20T14:09:29.271774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"以下是看起來超難超看不懂的 model 製作","metadata":{}},{"cell_type":"markdown","source":"![](https://img-blog.csdnimg.cn/20190716171510379.jpg?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NhaWt3NjI=,size_16,color_FFFFFF,t_70)","metadata":{}},{"cell_type":"code","source":"# 架構：神秘的沙漏網路\n'''\n總之先建個模型的骨架，據說可以從 COCO 下載預訓練權值，數據格式在 ~/.keras/keras.json\n參數說明：\n    1. num_stacks = 沙漏 modules 的數量\n    2. cnv_dim = 降低分辨率後 filters 的數量\n    3. inres = 輸入的形狀，須要是 128 的倍數\n    4. weights = 'None' → 隨機初始化\n                 'ctdet_coco' → 2D 物件偵測用\n                 'hpdet_coco' → 人類姿勢偵測用\n    5. dims = 沙漏 blocks 中 channels 的數量\n回傳：\n    keras 的 model 實例 x1\n'''\ndef HourglassNetwork(heads, num_stacks, cnv_dim=256, inres=(512, 512), weights=False, dims=[256, 384, 384, 384, 512]):\n    input_layer = Input(shape=(inres[0], inres[1], 3), name='HGInput')\n    inter = pre(input_layer, cnv_dim)\n    prev_inter = None\n    outputs = []\n    for i in range(num_stacks):\n        prev_inter = inter\n        _heads, inter = hourglass_module(heads, inter, cnv_dim, i, dims)\n        if i == 1:\n            outputs.extend(_heads)\n        if i < num_stacks - 1:\n            inter_ = Conv2D(cnv_dim, 1, use_bias=False, name='inter_.%d.0' % i)(prev_inter)\n            inter_ = BatchNormalization(epsilon=1e-5, name='inter_.%d.1' % i)(inter_)\n\n            cnv_ = Conv2D(cnv_dim, 1, use_bias=False, name='cnv_.%d.0' % i)(inter)\n            cnv_ = BatchNormalization(epsilon=1e-5, name='cnv_.%d.1' % i)(cnv_)\n\n            inter = Add(name='inters.%d.inters.add' % i)([inter_, cnv_])\n            inter = Activation('relu', name='inters.%d.inters.relu' % i)(inter)\n            inter = residual(inter, cnv_dim, 'inters.%d' % i)\n\n    model = Model(inputs=input_layer, outputs=outputs)\n\n    # load weights\n    if weights:\n        weights_path = get_file('HourglassNet.hdf5',\n                          'https://github.com/see--/keras-centernet/releases/download/0.1.0/ctdet_coco_hg.hdf5',\n                          cache_subdir='hourglassnet', \n                          file_hash='ce01e92f75b533e3ff8e396c76d55d97ff3ec27e99b1bdac1d7b0d6dcf5d90eb')\n        model.load_weights(weights_path, by_name=True)\n\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-05-20T14:09:29.274158Z","iopub.execute_input":"2021-05-20T14:09:29.27448Z","iopub.status.idle":"2021-05-20T14:09:29.292417Z","shell.execute_reply.started":"2021-05-20T14:09:29.274448Z","shell.execute_reply":"2021-05-20T14:09:29.291385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 特徵？\ndef hourglass_module(heads, bottom, cnv_dim, hgid, dims):\n    # create left features , f1, f2, f4, f8, f16 and f32\n    lfs = left_features(bottom, hgid, dims)\n\n    # create right features, connect with left features\n    rf1 = right_features(lfs, hgid, dims)\n    rf1 = convolution(rf1, 3, cnv_dim, name='cnvs.%d' % hgid)\n\n    # add 1x1 conv with two heads, inter is sent to next stage\n    # head_parts is used for intermediate supervision\n    heads = create_heads(heads, rf1, hgid)\n    return heads, rf1","metadata":{"execution":{"iopub.status.busy":"2021-05-20T14:09:29.294063Z","iopub.execute_input":"2021-05-20T14:09:29.294336Z","iopub.status.idle":"2021-05-20T14:09:29.310652Z","shell.execute_reply.started":"2021-05-20T14:09:29.294308Z","shell.execute_reply":"2021-05-20T14:09:29.309891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"![](https://img-blog.csdnimg.cn/20190716171120608.jpg?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NhaWt3NjI=,size_16,color_FFFFFF,t_70)","metadata":{}},{"cell_type":"code","source":"# 卷積塊\ndef convolution(_x, k, out_dim, name, stride=1):\n    padding = (k - 1) // 2\n    _x = ZeroPadding2D(padding=padding, name=name + '.pad')(_x)\n    _x = Conv2D(out_dim, k, strides=stride, use_bias=False, name=name + '.conv')(_x)\n    _x = BatchNormalization(epsilon=1e-5, name=name + '.bn')(_x)\n    _x = Activation('relu', name=name + '.relu')(_x)\n    return _x","metadata":{"execution":{"iopub.status.busy":"2021-05-20T14:09:29.311933Z","iopub.execute_input":"2021-05-20T14:09:29.312341Z","iopub.status.idle":"2021-05-20T14:09:29.324218Z","shell.execute_reply.started":"2021-05-20T14:09:29.3123Z","shell.execute_reply":"2021-05-20T14:09:29.323482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"![](https://img-blog.csdnimg.cn/20190716171157803.jpg?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NhaWt3NjI=,size_16,color_FFFFFF,t_70)","metadata":{}},{"cell_type":"code","source":"# 殘差塊\ndef residual(_x, out_dim, name, stride=1):\n    shortcut = _x\n    num_channels = K.int_shape(shortcut)[-1]\n    _x = ZeroPadding2D(padding=1, name=name + '.pad1')(_x)\n    _x = Conv2D(out_dim, 3, strides=stride, use_bias=False, name=name + '.conv1')(_x)\n    _x = BatchNormalization(epsilon=1e-5, name=name + '.bn1')(_x)\n    _x = Activation('relu', name=name + '.relu1')(_x)\n\n    _x = Conv2D(out_dim, 3, padding='same', use_bias=False, name=name + '.conv2')(_x)\n    _x = BatchNormalization(epsilon=1e-5, name=name + '.bn2')(_x)\n\n    if num_channels != out_dim or stride != 1:\n        shortcut = Conv2D(out_dim, 1, strides=stride, use_bias=False, name=name + '.shortcut.0')(\n            shortcut)\n        shortcut = BatchNormalization(epsilon=1e-5, name=name + '.shortcut.1')(shortcut)\n\n    _x = Add(name=name + '.add')([_x, shortcut])\n    _x = Activation('relu', name=name + '.relu')(_x)\n    return _x","metadata":{"execution":{"iopub.status.busy":"2021-05-20T14:09:29.325566Z","iopub.execute_input":"2021-05-20T14:09:29.325997Z","iopub.status.idle":"2021-05-20T14:09:29.342921Z","shell.execute_reply.started":"2021-05-20T14:09:29.325954Z","shell.execute_reply":"2021-05-20T14:09:29.342243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"![](https://img-blog.csdnimg.cn/20190716170729501.jpg)","metadata":{}},{"cell_type":"code","source":"# 還不知道幹啥用\ndef pre(_x, num_channels):\n    # front module, input to 1/4 resolution\n    _x = convolution(_x, 7, 128, name='pre.0', stride=2)\n    _x = residual(_x, num_channels, name='pre.1', stride=2)\n    return _x","metadata":{"execution":{"iopub.status.busy":"2021-05-20T14:09:29.344004Z","iopub.execute_input":"2021-05-20T14:09:29.344411Z","iopub.status.idle":"2021-05-20T14:09:29.358694Z","shell.execute_reply.started":"2021-05-20T14:09:29.344364Z","shell.execute_reply":"2021-05-20T14:09:29.357883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 建構用的工具函式們\ndef left_features(bottom, hgid, dims):\n    # create left half blocks for hourglass module\n    # f1, f2, f4 , f8, f16, f32 : 1, 1/2, 1/4 1/8, 1/16, 1/32 resolution\n    # 5 times reduce/increase: (256, 384, 384, 384, 512)\n    features = [bottom]\n    for kk, nh in enumerate(dims):\n        pow_str = ''\n        for _ in range(kk):\n            pow_str += '.center'\n        _x = residual(features[-1], nh, name='kps.%d%s.down.0' % (hgid, pow_str), stride=2)\n        _x = residual(_x, nh, name='kps.%d%s.down.1' % (hgid, pow_str))\n        features.append(_x)\n    return features\n\ndef connect_left_right(left, right, num_channels, num_channels_next, name):\n    # left: 2 residual modules\n    left = residual(left, num_channels_next, name=name + 'skip.0')\n    left = residual(left, num_channels_next, name=name + 'skip.1')\n\n    # up: 2 times residual & nearest neighbour\n    out = residual(right, num_channels, name=name + 'out.0')\n    out = residual(out, num_channels_next, name=name + 'out.1')\n    out = UpSampling2D(name=name + 'out.upsampleNN')(out)\n    out = Add(name=name + 'out.add')([left, out])\n    return out\n\ndef bottleneck_layer(_x, num_channels, hgid):\n    # 4 residual blocks with 512 channels in the middle\n    pow_str = 'center.' * 5\n    _x = residual(_x, num_channels, name='kps.%d.%s0' % (hgid, pow_str))\n    _x = residual(_x, num_channels, name='kps.%d.%s1' % (hgid, pow_str))\n    _x = residual(_x, num_channels, name='kps.%d.%s2' % (hgid, pow_str))\n    _x = residual(_x, num_channels, name='kps.%d.%s3' % (hgid, pow_str))\n    return _x\n\ndef right_features(leftfeatures, hgid, dims):\n    rf = bottleneck_layer(leftfeatures[-1], dims[-1], hgid)\n    for kk in reversed(range(len(dims))):\n        pow_str = ''\n        for _ in range(kk):\n            pow_str += 'center.'\n        rf = connect_left_right(leftfeatures[kk], rf, dims[kk], dims[max(kk - 1, 0)], name='kps.%d.%s' % (hgid, pow_str))\n    return rf\n\ndef create_heads(heads, rf1, hgid):\n    _heads = []\n    for head in sorted(heads):\n        num_channels = heads[head]\n        _x = Conv2D(256, 3, use_bias=True, padding='same', name=head + '.%d.0.conv' % hgid)(rf1)\n        _x = Activation('relu', name=head + '.%d.0.relu' % hgid)(_x)\n        if head == 'confidence':\n            _x = Conv2D(num_channels, 1, activation='sigmoid', use_bias=True, name=head + '.%d.1' % hgid)(_x)\n        else:\n            _x = Conv2D(num_channels, 1, use_bias=True, name=head + '.%d.1' % hgid)(_x)\n        _heads.append(_x)\n    return _heads","metadata":{"execution":{"iopub.status.busy":"2021-05-20T14:09:29.359808Z","iopub.execute_input":"2021-05-20T14:09:29.360203Z","iopub.status.idle":"2021-05-20T14:09:29.375361Z","shell.execute_reply.started":"2021-05-20T14:09:29.360162Z","shell.execute_reply":"2021-05-20T14:09:29.374445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 這區是 decode 的部分（別跟我說前面這是 GAN？？？ （前面不會是 encode 吧\n\n# use maxpooling as nms\ndef _nms(heat, kernel=3):\n    hmax = K.pool2d(heat, (kernel, kernel), padding='same', pool_mode='max')\n    keep = K.cast(K.equal(hmax, heat), K.floatx())\n    return heat * keep\n\ndef _ctdet_decode(hm, reg, k=100, output_stride=4):\n    hm = _nms(hm)\n    hm_shape = K.shape(hm)\n    reg_shape = K.shape(reg)\n    batch, width, cat = hm_shape[0], hm_shape[2], hm_shape[3]\n\n    hm_flat = K.reshape(hm, (batch, -1))\n    reg_flat = K.reshape(reg, (reg_shape[0], -1, reg_shape[-1]))\n    \n    def _process_sample(args):\n        _hm, _reg = args\n        _scores, _inds = tf.math.top_k(_hm, k=k, sorted=True)\n        _inds = K.cast(_inds / cat, 'int32')\n        _reg = K.gather(_reg, _inds)\n        \n        # get yaw, pitch, roll, x, y, z from regression\n        yaw =  _reg[..., 0]\n        pitch =  _reg[..., 1]\n        roll =  _reg[..., 2]\n        x =  _reg[..., 3] * 100\n        y =  _reg[..., 4] * 100\n        z =  _reg[..., 5] * 100\n\n        _detection = K.stack([yaw, pitch, roll, x, y, z, _scores], -1)\n        return _detection\n    \n    detections = K.map_fn(_process_sample, [hm_flat, reg_flat], dtype=K.floatx())\n    return detections\n\ndef CtDetDecode(model, hm_index=1, reg_index=0, k=100, output_stride=4):\n    def _decode(args):\n        hm, reg = args\n        return _ctdet_decode(hm, reg, k=k, output_stride=output_stride)\n    output = Lambda(_decode)([model.outputs[i] for i in [hm_index, reg_index]])\n    model = Model(model.input, output)\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-05-20T14:09:29.376395Z","iopub.execute_input":"2021-05-20T14:09:29.376804Z","iopub.status.idle":"2021-05-20T14:09:29.394521Z","shell.execute_reply.started":"2021-05-20T14:09:29.376764Z","shell.execute_reply":"2021-05-20T14:09:29.393435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 居然！train 也寫成函式了！\ndef train_model(model,epoch, batch_size = 4):\n    # 1. choose the layers you want to train\n    n = 0\n    for layer in model.layers:\n        layer.trainable = False\n\n        n += 1\n        if n == 500:\n            break\n    \n    # 2. define loss function\n    def focal_loss(hm_true, hm_pred):\n        pos_mask = tf.cast(tf.equal(hm_true, 1), tf.float32)\n        neg_mask = tf.cast(tf.less(hm_true, 1), tf.float32)\n        neg_weights = tf.pow(1 - hm_true, 4)\n\n        pos_loss = -tf.math.log(tf.clip_by_value(hm_pred, 1e-4, 1)) * tf.pow(1 - hm_pred, 2) * pos_mask\n        neg_loss = -tf.math.log(tf.clip_by_value(1 - hm_pred, 1e-4, 1)) * tf.pow(hm_pred, 2) * neg_weights * neg_mask\n\n        num_pos = tf.reduce_sum(pos_mask)\n        pos_loss = tf.reduce_sum(pos_loss)\n        neg_loss = tf.reduce_sum(neg_loss)\n\n        cls_loss = tf.cond(tf.greater(num_pos, 0), lambda: (pos_loss + neg_loss) / num_pos, lambda: neg_loss)\n        return cls_loss\n    \n    def l1_loss(y_true, y_pred):\n        mask = tf.zeros_like(y_true, dtype=tf.float32)\n        mask = tf.equal(y_true, mask)\n        mask = tf.cast(mask, tf.float32)\n        mask = tf.reduce_sum(mask, axis=-1)\n        \n        one = tf.ones_like(mask)\n        zero = tf.zeros_like(mask)\n        mask = tf.where(mask == 6, x=zero, y=one)\n        mask = tf.tile(tf.expand_dims(mask, axis=-1), (1, 1, 1, 6))\n        \n        total_loss = tf.reduce_sum(tf.abs(y_true - y_pred * mask))\n        reg_loss = total_loss / (tf.reduce_sum(mask) + 1e-4)\n        return reg_loss\n    \n    # 3. compile\n    model.compile(optimizer=Adam(),\n                   loss={'car_pose.1.1':l1_loss, 'confidence.1.1':focal_loss},\n                   loss_weights=[1, 1])\n    \n    # 4. fit\n    history = model.fit_generator(train_generator(train,batch_size=batch_size),\n                                  steps_per_epoch = len(train) // batch_size,\n                                  epochs = epoch\n                                  )\n    \n    model.save_weights('/kaggle/working/centernet_weights.hdf5')\n    \n    plt.title(\"model loss\")\n    plt.ylabel(\"loss\")\n    plt.xlabel(\"epoch\")\n    plt.plot(history.history['loss'])","metadata":{"execution":{"iopub.status.busy":"2021-05-20T14:09:29.39572Z","iopub.execute_input":"2021-05-20T14:09:29.39602Z","iopub.status.idle":"2021-05-20T14:09:29.415697Z","shell.execute_reply.started":"2021-05-20T14:09:29.395991Z","shell.execute_reply":"2021-05-20T14:09:29.41496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 好像是看看有沒有遮罩（有的沒有\ndef TestMaskImages(imageName):\n    trainimage = cv2.imread(PATH  + \"/test_images/\" + imageName + '.jpg')\n    imagemask = cv2.imread(PATH + \"/test_masks/\" + imageName + \".jpg\",0)\n    try:\n        imagemaskinv = cv2.bitwise_not(imagemask)\n        res = cv2.bitwise_and(trainimage,trainimage,mask = imagemaskinv)\n        res = res[res.shape[0] // 2:]\n        return res\n    except:\n        trainimage = trainimage[trainimage.shape[0] // 2:]\n        return trainimage","metadata":{"execution":{"iopub.status.busy":"2021-05-20T14:09:29.416826Z","iopub.execute_input":"2021-05-20T14:09:29.417243Z","iopub.status.idle":"2021-05-20T14:09:29.432465Z","shell.execute_reply.started":"2021-05-20T14:09:29.417213Z","shell.execute_reply":"2021-05-20T14:09:29.431476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 訓練完之後輸出囉\ndef predict(model):\n    modelx = CtDetDecode(model)\n    \n    def pred(i):\n        img = TestMaskImages(test['ImageId'][i])\n        img = resize_image(img)\n        X_batch = img[np.newaxis, :]\n\n        detections = modelx.predict(X_batch)\n        detections = np.squeeze(detections)\n\n\n        submission = []\n        for d in detections:\n            yaw, pitch, roll, x, y, z, score = d\n            if score < 0.3:\n                continue\n            else:\n                submission.append(d)\n\n        Prediction_string = coords_to_str(submission)\n        test['PredictionString'][i] = Prediction_string\n\n    for i in tqdm(range(len(test))):\n        pred(i)\n        \n    test.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-05-20T14:09:29.434659Z","iopub.execute_input":"2021-05-20T14:09:29.434967Z","iopub.status.idle":"2021-05-20T14:09:29.448086Z","shell.execute_reply.started":"2021-05-20T14:09:29.43494Z","shell.execute_reply":"2021-05-20T14:09:29.447224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 1. set heads of HourglassNet\nkwargs = {\n        'num_stacks': 2,\n        'cnv_dim': 256,\n        'inres': (512, 512),\n        }\nheads = {\n        'car_pose': 6,\n        'confidence': 1\n        }\n\n# 2. create model\nmodel = HourglassNetwork(heads=heads, **kwargs)\n# model.load_weights('../input/centernet-hg/centernet_weights.hdf5', by_name=True)\n\n# 3. train\ntrain_model(model, epoch=3, batch_size=16)\n\n# 4.predict\npredict(model)","metadata":{"execution":{"iopub.status.busy":"2021-05-20T14:09:29.449217Z","iopub.execute_input":"2021-05-20T14:09:29.449621Z","iopub.status.idle":"2021-05-20T14:11:45.461617Z","shell.execute_reply.started":"2021-05-20T14:09:29.449592Z","shell.execute_reply":"2021-05-20T14:11:45.458858Z"},"trusted":true},"execution_count":null,"outputs":[]}]}