{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport glob\nfrom keras.models import Model\nfrom keras.layers import *\nfrom keras.utils import get_file\nimport keras.backend as K\nimport numpy as np\nimport tensorflow as tf\nimport pandas as pd\nfrom math import sin, cos\nimport math\nimport cv2\nimport matplotlib.pyplot as plt\nfrom keras.optimizers import Adam, RMSprop, SGD\nfrom keras.callbacks import ModelCheckpoint\nfrom keras.backend.tensorflow_backend import set_session, clear_session\nfrom operator import itemgetter\nfrom scipy.ndimage.filters import maximum_filter\nfrom scipy.spatial.transform import Rotation as R\nfrom tqdm.notebook import tqdm\nfrom skimage.morphology import watershed\nfrom skimage.feature import peak_local_max\nfrom skimage import measure\n\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n#    for filename in filenames:\n#        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model_n(bn_train=False, bn_train2=False):\n    \n    def hourglass_module(heads, bottom, cnv_dim, hgid, dims, input_ref):\n        lfs = left_features(bottom, hgid, dims)\n        rf1 = right_features(lfs, hgid, dims)\n        rf1 = convolution(rf1, 3, cnv_dim, name='cnvs.%d' % hgid)\n        heads = create_heads(heads, rf1, hgid, input_ref)\n        return heads, rf1\n\n    def convolution(_x, k, out_dim, name, stride=1):\n        padding = (k - 1) // 2\n        _x = ZeroPadding2D(padding=padding, name=name + '.pad')(_x)\n        _x = Conv2D(out_dim, k, strides=stride, use_bias=False, name=name + '.conv')(_x)\n        if name[:6]=='cnvs.1' and bn_train2==True:\n            #print(name + '.bn')\n            _x = BatchNormalization(epsilon=1e-5, name=name + '.bn')(_x)\n        else:\n            _x = BatchNormalization(epsilon=1e-5, name=name + '.bn')(_x, training=bn_train)\n        _x = Activation('relu', name=name + '.relu')(_x)\n        return _x\n\n    def residual(_x, out_dim, name, stride=1):\n        shortcut = _x\n        num_channels = K.int_shape(shortcut)[-1]\n        _x = ZeroPadding2D(padding=1, name=name + '.pad1')(_x)\n        _x = Conv2D(out_dim, 3, strides=stride, use_bias=False, name=name + '.conv1')(_x)\n\n        if (name[:9]=='kps.1.out' or  name[:10]=='kps.1.skip') and bn_train2==True:\n            #print(name + '.bn1')\n            _x = BatchNormalization(epsilon=1e-5, name=name + '.bn1')(_x)\n        else:\n            _x = BatchNormalization(epsilon=1e-5, name=name + '.bn1')(_x, training=bn_train)\n        _x = Activation('relu', name=name + '.relu1')(_x)\n\n        _x = Conv2D(out_dim, 3, padding='same', use_bias=False, name=name + '.conv2')(_x)\n        if (name[:9]=='kps.1.out' or  name[:10]=='kps.1.skip') and bn_train2==True:\n            #print(name + '.bn2')\n            _x = BatchNormalization(epsilon=1e-5, name=name + '.bn2')(_x)\n        else:\n            _x = BatchNormalization(epsilon=1e-5, name=name + '.bn2')(_x, training=bn_train)\n\n        if num_channels != out_dim or stride != 1:\n            shortcut = Conv2D(out_dim, 1, strides=stride, use_bias=False, name=name + '.shortcut.0')(\n                shortcut)\n            shortcut = BatchNormalization(epsilon=1e-5, name=name + '.shortcut.bn1')(shortcut, training=bn_train)\n\n        _x = Add(name=name + '.add')([_x, shortcut])\n        _x = Activation('relu', name=name + '.relu')(_x)\n        return _x\n\n    def pre(_x, num_channels):\n        _x = convolution(_x, 7, 128, name='pre.0', stride=2)\n        _x = residual(_x, num_channels, name='pre.1', stride=2)\n        return _x\n\n    def left_features(bottom, hgid, dims):\n        features = [bottom]\n        for kk, nh in enumerate(dims):\n            pow_str = ''\n            for _ in range(kk):\n                pow_str += '.center'\n            _x = residual(features[-1], nh, name='kps.%d%s.down.0' % (hgid, pow_str), stride=2)\n            _x = residual(_x, nh, name='kps.%d%s.down.1' % (hgid, pow_str))\n            features.append(_x)\n        return features\n\n    def connect_left_right(left, right, num_channels, num_channels_next, name):\n        left = residual(left, num_channels_next, name=name + 'skip.0')\n        left = residual(left, num_channels_next, name=name + 'skip.1')\n        out = residual(right, num_channels, name=name + 'out.0')\n        out = residual(out, num_channels_next, name=name + 'out.1')\n        out = UpSampling2D(name=name + 'out.upsampleNN')(out)\n        out = Add(name=name + 'out.add')([left, out])\n        return out\n\n    def bottleneck_layer(_x, num_channels, hgid):\n        pow_str = 'center.' * 5\n        _x = residual(_x, num_channels, name='kps.%d.%s0' % (hgid, pow_str))\n        _x = residual(_x, num_channels, name='kps.%d.%s1' % (hgid, pow_str))\n        _x = residual(_x, num_channels, name='kps.%d.%s2' % (hgid, pow_str))\n        _x = residual(_x, num_channels, name='kps.%d.%s3' % (hgid, pow_str))\n        return _x\n\n    def right_features(leftfeatures, hgid, dims):\n        rf = bottleneck_layer(leftfeatures[-1], dims[-1], hgid)\n        for kk in reversed(range(len(dims))):\n            pow_str = ''\n            for _ in range(kk):\n                pow_str += 'center.'\n            rf = connect_left_right(leftfeatures[kk], rf, dims[kk], dims[max(kk - 1, 0)], name='kps.%d.%s' % (hgid, pow_str))\n        return rf\n    \n    def hm_stop_gradient(x):\n        return K.stop_gradient(K.sigmoid(x))\n\n    def stop_gradient(x):\n        return K.stop_gradient(x)\n\n    def create_heads(heads, rf1, hgid, input_ref):\n\n        _heads = []\n        keys = list(heads.keys())  \n\n        if hgid >= 0:\n            head = keys[0]\n            num_channels = heads[head]\n            _x = Conv2D(256, 3, use_bias=True, padding='same', name=head + '.%d.0.conv' % hgid)(rf1)\n            _x = Activation('relu', name=head + '.%d.0.relu' % hgid)(_x)\n            _x = Conv2D(num_channels, 1, use_bias=True, name=head[0] + '%d' % hgid)(_x)\n            _heads.append(_x)\n\n        hm = Lambda(hm_stop_gradient, name='hm_ref.'+ '%d' % hgid)(_heads[0])\n        hm = Concatenate()([rf1, hm])\n\n        if hgid >= 0:\n            head = keys[1]\n            num_channels = heads[head]\n            _x = Conv2D(256, 3, use_bias=True, padding='same', name=head + '.%d.0.conv' % hgid)(hm)\n            _x = Activation('relu', name=head + '.%d.0.relu' % hgid)(_x)\n            _x = Conv2D(num_channels, 1, use_bias=True, name=head[0] + '%d' % hgid)(_x)\n            _heads.append(_x) \n\n        if hgid >= 0:\n            head = keys[2]\n            num_channels = heads[head]\n            #_x = Lambda(stop_gradient, name=head + '.%d.stop_g' % hgid)(hm)\n            _x = Conv2D(256, 3, use_bias=True, padding='same', name=head + '.%d.0.conv' % hgid)(hm)\n            _x = Activation('relu', name=head + '.%d.0.relu' % hgid)(_x)\n            _x = Conv2D(num_channels, 1, use_bias=True, name=head[0] + '%d' % hgid)(_x)\n            _heads.append(_x)\n\n        if hgid >= 0:\n            head = keys[-1]\n            num_channels = heads[head]\n            head = '%d'%hgid + head\n\n            ref = input_ref#Lambda(get_ref, name='coor_ref')(_heads[-1])\n            ref = Conv2D(16, 1, use_bias=True, padding='same', name=head + '.ref.%d.0.conv' % hgid)(ref)\n            #ref = BatchNormalization(epsilon=1e-5, name=head + '.ref.%d.0.bn' % hgid)(ref)\n            ref = Activation('relu', name=head + '.ref.%d.0.relu' % hgid)(ref)\n            ref = Conv2D(128, 1, use_bias=True, padding='same', name=head + '.ref.%d.1.conv' % hgid)(ref)\n            #ref = BatchNormalization(epsilon=1e-5, name=head + '.ref.%d.1.bn' % hgid)(ref)\n            ref = Activation('relu', name=head + '.ref.%d.1.relu' % hgid)(ref)\n\n            _x = Concatenate()([hm, ref])\n\n            for i in range(3):\n                _x = Conv2D(256*(1+hgid), 3, use_bias=True, padding='same', name=head + '.%d.%d.conv' %(hgid,i))(_x)\n                _x = Activation('relu', name=head + '.%d.%d.relu' %(hgid,i))(_x)\n            _x = Conv2D(num_channels, 1, use_bias=True, name=head[1] + '%d' % hgid)(_x)\n            _heads.append(_x)\n\n        return _heads\n    \n    def HourglassNetwork(heads, num_stacks=2, cnv_dim=256, inres=(512, 512), weights='ctdet_coco',\n                         dims=[256, 384, 384, 384, 512]):\n\n        if not (weights in {'ctdet_coco', 'hpdet_coco', None} or os.path.exists(weights)):\n            raise ValueError('The `weights` argument should be either '\n                             '`None` (random initialization), `ctdet_coco` '\n                             '(pre-trained on COCO), `hpdet_coco` (pre-trained on COCO) '\n                             'or the path to the weights file to be loaded.')\n        input_ref = Input(shape=(inres[2], inres[3], 2), name='HGRef')\n        input_layer = Input(shape=(inres[0], inres[1], 3), name='HGInput')\n        inter = pre(input_layer, cnv_dim)\n        prev_inter = None\n        outputs = []\n        for i in range(num_stacks):\n            prev_inter = inter\n            _heads, inter = hourglass_module(heads, inter, cnv_dim, i, dims, input_ref)\n            outputs.extend(_heads)\n            if i < num_stacks - 1:\n                inter_ = Conv2D(cnv_dim, 1, use_bias=False, name='inter_.%d.0' % i)(prev_inter)\n                inter_ = BatchNormalization(epsilon=1e-5, name='inter_.%d.bn1' % i)(inter_, training=bn_train)\n\n                cnv_ = Conv2D(cnv_dim, 1, use_bias=False, name='cnv_.%d.0' % i)(inter)\n                cnv_ = BatchNormalization(epsilon=1e-5, name='cnv_.%d.bn1' % i)(cnv_, training=bn_train)\n\n                inter = Add(name='inters.%d.inters.add' % i)([inter_, cnv_])\n                inter = Activation('relu', name='inters.%d.inters.relu' % i)(inter)\n                inter = residual(inter, cnv_dim, 'inters.%d' % i)\n\n        model = Model(inputs=[input_layer, input_ref], outputs=outputs)\n        \n        if 0: # I use pretrain when training\n            if weights == 'ctdet_coco':\n                weights_path = get_file(\n                    '%s_hg.hdf5' % weights,\n                    CTDET_COCO_WEIGHTS_PATH,\n                    cache_subdir='models',\n                    file_hash='ce01e92f75b533e3ff8e396c76d55d97ff3ec27e99b1bdac1d7b0d6dcf5d90eb')\n                model.load_weights(weights_path, by_name=True)\n            elif weights == 'hpdet_coco':\n                weights_path = get_file(\n                    '%s_hg.hdf5' % weights,\n                    HPDET_COCO_WEIGHTS_PATH,\n                    cache_subdir='models',\n                    file_hash='5c562ee22dc383080629dae975f269d62de3a41da6fd0c821085fbee183d555d')\n                model.load_weights(weights_path)\n            elif weights is not None:\n                model.load_weights(weights)\n\n        return model\n    \n    heads = {'classes': 34, 'hm_car': 1, 'reg_car': 2, 'dof_car': 8}\n    model = HourglassNetwork(heads, num_stacks=2, inres=(None,None,None,None))\n\n    train_layers = ['kps.1.out', 'kps.1.skip', 'cnvs.1']\n\n    if bn_train2 == False:\n        for layer in model.layers:\n            layer.trainable = False\n            if layer.name[:4]=='1dof':\n                layer.trainable = True\n            if layer.name=='d1':\n                layer.trainable = True          \n    elif bn_train == False:\n        for layer in model.layers:\n            layer.trainable = False\n\n            for tl in train_layers:\n                if layer.name[:len(tl)] == tl :\n                    layer.trainable = True\n\n            if layer.name[:4]=='1dof':\n                layer.trainable = True\n            elif layer.name[:9]=='reg_car.1':\n                layer.trainable = True            \n            elif layer.name[:8]=='hm_car.1':\n                layer.trainable = True        \n            elif layer.name[:9]=='classes.1':\n                layer.trainable = True  \n            elif layer.name[1]=='1':\n                layer.trainable = True\n\n            #if layer.trainable == True:\n                #print(layer.name)\n\n    return model","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def normalize_image(image):\n    mean = [0.40789655, 0.44719303, 0.47026116]\n    std = [0.2886383, 0.27408165, 0.27809834]\n    return (np.float32(image) - mean) / std\n\ndef rotateImage(alpha=0, beta=0, gamma=0, dx=1691.5, dy=0):\n    \n    fx, dx = 2304.5479, dx\n    fy, dy = 2305.8757, dy\n    \n    # Projection 2D -> 3D matrix\n    A1 = np.array([[1/fx,    0, -dx/fx],\n                   [0,    1/fy, -dy/fy],\n                   [0,       0,      1],\n                   [0,       0,      1]])\n    \n    # Rotation matrices around the X, Y, and Z axis\n    RX = np.array([[1,          0,           0, 0],\n                   [0, cos(alpha), -sin(alpha), 0],\n                   [0, sin(alpha),  cos(alpha), 0],\n                   [0,          0,           0, 1]])\n    \n    RY = np.array([[cos(beta),  0,  -sin(beta), 0],\n                   [0,          1,           0, 0],\n                   [sin(beta),  0,   cos(beta), 0],\n                   [0,          0,           0, 1]])\n    \n    RZ = np.array([[cos(gamma), -sin(gamma), 0, 0],\n                   [sin(gamma),  cos(gamma), 0, 0],\n                   [0,          0,           1, 0],\n                   [0,          0,           0, 1]])\n    \n    # Composed rotation matrix with (RX, RY, RZ)\n    Rot = np.dot(RZ, np.dot(RX, RY))\n    \n    # 3D -> 2D matrix\n    A2 = np.array([[fx, 0, dx, 0],\n                   [0, fy, dy, 0],\n                   [0, 0,   1, 0]])\n    # Final transformation matrix\n    trans = np.dot(A2,np.dot(Rot, A1))\n    \n    return trans, Rot\n\ndef sigmoid(x):\n    x = np.clip(x, -50, None)\n    return 1 / (1 + np.exp(-x))\n\ndef get_ref(iw=320, ih=128):\n    ref = np.reshape(np.arange(0, ih*iw), (ih, iw, -1))\n    ref_x = ref % iw\n    ref_y = ref // iw\n    return np.dstack([ref_x, ref_y])\n\ndef postprocess2(pred):\n    yaw,px,py,roll,x,y,z,r = pred\n    pitch = math.atan2(py, px)\n    roll = roll%(np.pi*2)-np.pi\n    return yaw, pitch, roll, x*10, y*10, z*10, r*10\n\ndef get_xy_from_XYz(X, Y, z):\n    x = (X - 1686.2379)*z/2304.5479\n    y = (Y - 1354.9849)*z/2305.8757\n    return x,y\n\ndef get_xyz_from_XYr(X, Y, r):\n    r = np.clip(r, 0, None)\n    a = (X - 1686.2379)/2304.5479\n    b = (Y - 1354.9849)/2305.8757\n    z = r/((a*a + b*b + 1)**.5)\n    return a*z, b*z, z\n\ndef TTA_gen(sub, xo, yo, M, trans=[[0,0,0,0]]):\n    \n    inputs = []\n    input_coor = []\n    \n    ref = np.reshape(np.arange(0, xo*yo), (yo, xo, -1))\n    ref_x = ref % xo\n    ref_y = ref // xo\n    ref = np.dstack([(ref_x-(xo-1)/2)/100, ref_y/100])\n\n    while True:\n        for i in range(len(sub)):    \n            image = cv2.imread('../input/pku-autonomous-driving/test_images/%s.jpg'%sub.iloc[i].ImageId)\n            mask = cv2.imread('../input/pku-autonomous-driving/test_masks/%s.jpg'%sub.iloc[i].ImageId)\n            if mask is not None:\n                image = image*(mask<128)            \n            image = image[1355:,:,::-1]\n            \n            for alpha, beta, gamma, flip in trans:\n                alpha = alpha*np.pi/180.\n                beta  = beta *np.pi/180.\n                gamma = gamma*np.pi/180.                \n                Mat, Rot = rotateImage(alpha, beta, gamma)\n                \n                img = cv2.warpPerspective(image.copy(), np.dot(M,Mat), (xo,yo), flags=cv2.INTER_LINEAR)\n\n                if np.random.random()<flip:        \n                    img = img[:,::-1]                \n\n                img = normalize_image(img/255.)    \n\n                coor = ref[::4, ::4]\n\n                inputs.append(img)\n                input_coor.append(coor)\n\n            tmp_inputs = np.array(inputs)\n            tmp_input_coor = np.array(input_coor)\n            inputs = []\n            input_coor = []\n            \n            yield [tmp_inputs, tmp_input_coor]\n            \ndef decode3(pred, trans, Mi, xs=640, ys=128, w=None, interpolation=cv2.INTER_LANCZOS4, sel=None):\n    hms=0\n    masks=0\n    masks2=0\n    ts=0\n    rs=0\n    \n    msk = np.zeros((ys,xs))\n    msk[9:-8,31:-31]=1\n    msk = cv2.GaussianBlur(msk,(65,19), 21)\n    msk = msk/msk.max()\n    \n    if sel is None:\n        sel = np.arange(len(trans))\n        \n    if w is None:\n        w = np.ones(len(trans))\n        \n    for i in sel:\n        alpha, beta, gamma, flip = trans[i]\n        alpha = alpha*np.pi/180.\n        beta  = beta *np.pi/180.\n        gamma = gamma*np.pi/180.\n\n        Mat, Rot = rotateImage(alpha, beta, gamma, dx=1691.5+2000)\n        Ri = np.linalg.inv(Mat)\n        Roti = np.linalg.inv(Rot)\n        \n        Matrix = np.dot(Mf, np.dot(Ri,Mi))\n                \n        if flip:\n            hm = sigmoid(pred[0][i,:,::-1,0])\n            t = pred[-1][i,:,::-1,-4:]\n            r = pred[-1][i,:,::-1,:4]\n            t[:,:,0] = -t[:,:,0]\n            r[:,:,2:] = -r[:,:,2:]\n        else:\n            hm = sigmoid(pred[0][i,:,:,0])\n            t = pred[-1][i,:,:,-4:]\n            r = pred[-1][i,:,:,:4]\n\n        hm = cv2.warpPerspective(hm, Matrix, (xo//4+sxo*2,yo//4+syo+24), flags=interpolation)\n        t = cv2.warpPerspective(t, Matrix, (xo//4+sxo*2,yo//4+syo+24), flags=interpolation)\n        r = cv2.warpPerspective(r, Matrix, (xo//4+sxo*2,yo//4+syo+24), flags=interpolation)\n        mask = cv2.warpPerspective(msk.copy(), Matrix, (xo//4+sxo*2,yo//4+syo+24))\n        \n        ti = np.dot(Roti, np.reshape(t, (-1,4)).T).T.reshape(t.shape)\n        \n        yaw = r[:,:,0]\n        pitch = np.arctan2(r[:,:,2], r[:,:,1])\n        roll = r[:,:,3]+np.pi\n        rr = np.dstack([-pitch, -yaw, -roll])\n        \n        ri_ = r.copy()\n        y,x = np.where(hm>0.01)\n        if len(y)>0:\n            r1 = R.from_euler('xyz', rr[y,x], degrees=False)\n            r2 = R.from_euler('xyz', [beta, -alpha, -gamma], degrees=False).inv()\n            ri = (r2*r1).as_euler('xyz')*(-1)\n            #ri = ri.reshape(rr.shape)\n            ri_[y,x,0] = ri[:,1]\n            ri_[y,x,1] = np.cos(ri[:,0])\n            ri_[y,x,2] = np.sin(ri[:,0])\n            ri_[y,x,3] = ri[:,2]%(np.pi*2)-np.pi\n\n        mask = mask*w[i]\n        if i==0:\n            mask2 = mask\n        else:\n            mask2 = mask*(hm>0.01)\n\n        rs = ri_*mask2[...,np.newaxis]+rs\n        ts = ti*mask[...,np.newaxis]+ts\n        hms = hm*mask+hms\n        masks = mask+masks\n        masks2 = mask2+masks2\n    hms[masks>0] = hms[masks>0]/masks[masks>0]\n    ts[masks>0] = ts[masks>0]/masks[...,np.newaxis][masks>0]\n    rs[masks2>0] = rs[masks2>0]/masks2[...,np.newaxis][masks2>0]\n    return hms, ts, rs, masks\n\ndef creat_sub(params, thr=0.3, thr2=0.7, r=1, pwr=1, avg=True, optimize=3, kernel=(3,3), recall_=0, verbose=1, apn=10, M=None, n=2021, x_shift=0, norm_mask=False):\n\n    ref = get_ref()\n    sub = pd.read_csv(\"../input/pku-autonomous-driving/sample_submission.csv\")[:n]\n    count=np.zeros(len(sub))\n\n    for idx in tqdm(range(len(sub))):\n        \n        hms = 0\n        dofs = 0\n        masks = 0\n        \n        for trans, w, path, Mi, xs, mw in params:\n\n            pred = np.load(path + '/%d.npz'%idx)\n            preds = [pred[key] for key in pred]\n            hm, tsf, rsf, mask = decode3(preds, trans, Mi, xs=xs, w=w)\n            dof = np.dstack([rsf,tsf])\n                \n            if norm_mask:\n                mask = mask/np.max(mask)\n                \n            mask = mask*mw\n            \n            hms = hms + hm*mask\n            mask = mask[...,np.newaxis]  \n            dofs = dofs + dof*mask\n            masks = masks + mask\n               \n        masks[masks==0] = masks[masks==0] + 1e-7\n        p = hms/masks[:,:,0]\n        dofs = dofs/masks\n        \n        ih,iw = hm.shape\n        reg = get_ref(iw=iw, ih=ih)-[sxo, syo]\n\n        #p = hms\n        local_maxi = p*(p == maximum_filter(p,footprint=np.ones(kernel)))>thr\n        py,px = np.where(local_maxi)\n\n        if avg:\n            markers = measure.label(local_maxi)\n            labels_ws = watershed(-p, markers, mask=p>min(thr,thr2))\n            scores = []\n            pdx = []\n            pdy = []\n            dof_=[]\n            for i in range(1, markers.max()+1):\n                y,x = np.where(labels_ws==i)\n                score = p[y,x]\n                scores.append(score.max())\n                score = score*(score >= min(scores[-1]*r, thr2))\n                score = score**pwr\n                ss = score.sum()\n                dof_.append((dofs[y,x]*score[:,np.newaxis]).sum(0)/ss)\n                pdx.append(((reg[y,x,0])*score).sum()/ss)\n                pdy.append(((reg[y,x,1])*score).sum()/ss)\n        else:\n            scores = p[py,px]\n            pdx = reg[py,px,0]\n            pdy = reg[py,px,1]\n            dof_= dofs[py,px,:]\n\n        output = np.zeros((len(dof_),7))\n\n        for j in range(len(dof_)):        \n            pp = postprocess2(dof_[j])\n            output[j,:6] = pp[:6]\n            output[j, 2] = (output[j,2]+np.pi)%(np.pi*2)-np.pi\n            output[j,-1] = scores[j]\n\n            #if optimize ==3:\n            \n            X,Y,Z = np.dot(np.linalg.inv(M), [pdx[j]*4, pdy[j]*4, 1])\n            X=X/Z+x_shift\n            Y=Y/Z+1355\n            x1,y1,z1 = get_xyz_from_XYr(X, Y, pp[-1])\n            x0,y0    = get_xy_from_XYz(X, Y, pp[-2])\n            if optimize ==3:\n                output[j,3:5] = (x0+x1)/2,(y0+y1)/2\n            else:\n                output[j,3:6] = (x0+x1)/2,(y0+y1)/2, (z1+pp[-2])/2\n                \n        count[idx] = len(dof_)\n        sub.iloc[idx].PredictionString = ' '.join(output.reshape(-1).astype('str'))\n\n    print(thr, thr2, r, pwr, avg, optimize, count.sum())\n    return sub, count\n\nk = np.array([[2304.5479, 0,  1686.2379],\n           [0, 2305.8757, 1354.9849],\n           [0, 0, 1]], dtype=np.float32)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# perspective transform matrix","metadata":{}},{"cell_type":"code","source":"y=1650-1355\nyy=2400-1355\ndx=1000\nyo=512\nxo=2048\n\npts1=np.float32([[0+128,y],[3384-128-1,y],[-dx,yy-1],[3384+dx-1,yy-1]])\npts2=np.float32([[0,0],[xo-1,0],[0,yo-1],[xo-1,yo-1]])\nM3=cv2.getPerspectiveTransform(pts1,pts2)\n\npts1=np.float32([[0+128+dx+1000,y],[3384-128-1+dx+1000,y],[0+1000,yy-1],[3384+dx-1+dx+1000,yy-1]])\npts2=np.float32([[0,0],[(xo-1)/4,0],[0,(yo-1)/4],[(xo-1)/4,(yo-1)/4]])\nMi3=cv2.getPerspectiveTransform(pts2,pts1)\n\ny=1650-1355\nyy=2400-1355\ndx=2000\nyo=512\nxo=2560\npts1=np.float32([[0+128,y],[3384-128-1,y],[-dx,yy-1],[3384+dx-1,yy-1]])\npts2=np.float32([[0,0],[xo-1,0],[0,yo-1],[xo-1,yo-1]])\nM=cv2.getPerspectiveTransform(pts1,pts2)\n\nsyo=128\nsxo=64\npts1=np.float32([[0+128+dx,y],[3384-128-1+dx,y],[0,yy-1],[3384+dx-1+dx,yy-1]])\npts2=np.float32([[0,0],[(xo-1)/4,0],[0,(yo-1)/4],[(xo-1)/4,(yo-1)/4]])\npts3=np.float32([[sxo,syo],[(xo-1)/4+sxo,syo],[sxo,(yo-1)/4+syo],[(xo-1)/4+sxo,(yo-1)/4+syo]])\nMf=cv2.getPerspectiveTransform(pts1, pts3)\nMi2=cv2.getPerspectiveTransform(pts2, pts1)\n\ntrans =[[0,0,0], [0,7,0], [0,-7,0], [0,15,5], [0,-15,-5], [1,0,0], [-2,0,0]]\na = np.zeros((len(trans)*2, 4))\na[:,:3] = trans+trans\na[len(trans):,-1] = 1\ntrans3 = a\n\ntrans =[[0,0,0], [0,5,0], [0,-5,0], [0,10,3], [0,-10,-3], [2,0,0], [-5,0,0]]\na = np.zeros((len(trans)*2, 4))\na[:,:3] = trans+trans\na[len(trans):,-1] = 1\ntrans2 = a","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# predict 10 test images","metadata":{}},{"cell_type":"code","source":"sub = pd.read_csv(\"../input/pku-autonomous-driving/sample_submission.csv\")\nmodel = get_model_n()\n\npath = 'test/v99ap2_n'\nif not os.path.exists(path):\n    os.makedirs(path)\n    \nmodel.load_weights('../input/autonomous-driving-checkpoint/v99ap2_n/001-0.703-0.288-0.154-0.758-0.229-0.164-lr-5.5-b4.h5')                   \ngen_test = TTA_gen(sub, 2560, 512, M, trans2)\n\nfor i in tqdm(range(10)):\n    inputs = next(gen_test)\n    preds = model.predict(inputs, batch_size=len(trans))    \n    np.savez(path + '/%d.npz'%i, *preds[-3:])\n    \npath = 'test/v99ap3_n'\nif not os.path.exists(path):\n    os.makedirs(path)\n    \nmodel.load_weights('../input/autonomous-driving-checkpoint/v99ap3_n/001-0.674-0.287-0.156-0.636-0.229-0.167-lr-5.5-b4.h5')                   \ngen_test = TTA_gen(sub, 2048, 512, M3, trans3)\n\nfor i in tqdm(range(10)):\n    inputs = next(gen_test)\n    preds = model.predict(inputs, batch_size=len(trans))    \n    np.savez(path + '/%d.npz'%i, *preds[-3:])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# show inputs and predict heatmap","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(12,12))\nfor i in range(7):\n    plt.subplot(7,2,i*2+1)\n    img = inputs[0][i]\n    plt.imshow((img-img.min())/(img.max()-img.min()), interpolation='bilinear')\n    plt.subplot(7,2,i*2+2)\n    plt.imshow(sigmoid(preds[-3][i,:,:,0]), interpolation='bilinear')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# blend predictions","metadata":{}},{"cell_type":"code","source":"w=[2,1,1,1,1,1,1,2,1,1,1,1,1,1]\nparams = [[trans3, w, 'test/v99ap3_n', Mi3, 512, 1],\n          [trans2, w, 'test/v99ap2_n', Mi2, 640, 1]]\nsub_df, count = creat_sub(params, 0.3, 0.7, 1, 2, avg=True, optimize=3, kernel=(3,3), M=M, n=10)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json\n# Load a 3D model of a car\nwith open('../input/pku-autonomous-driving/car_models_json/mazida-6-2015.json') as json_file:\n    data = json.load(json_file)\nvertices = np.array(data['vertices'])\nvertices[:, 1] = -vertices[:, 1]\ntriangles = np.array(data['faces']) - 1\n\nim_color = cv2.applyColorMap(np.arange(256).astype('uint8') , cv2.COLORMAP_HSV)[:,0,:]\n\ndef draw_obj(image, vertices, triangles, color):\n    for t in triangles:\n        coord = np.array([vertices[t[0]][:2], vertices[t[1]][:2], vertices[t[2]][:2]], dtype=np.int32)\n        cv2.fillConvexPoly(image, coord, color)\n        #cv2.polylines(image, np.int32([coord]), 1, color)\n        \ndef draw_car(yaw, pitch, roll, x, y, z, overlay, color=(0,0,255)):\n    yaw, pitch, roll = -pitch, -yaw, -roll\n    Rt = np.eye(4)\n    t = np.array([x, y, z])\n    Rt[:3, 3] = t\n    Rt[:3, :3] = euler_to_Rot(yaw, pitch, roll).T\n    Rt = Rt[:3, :]\n    P = np.ones((vertices.shape[0],vertices.shape[1]+1))\n    P[:, :-1] = vertices\n    P = P.T\n    img_cor_points = np.dot(k, np.dot(Rt, P))\n    img_cor_points = img_cor_points.T\n    img_cor_points[:, 0] /= img_cor_points[:, 2]\n    img_cor_points[:, 1] /= img_cor_points[:, 2]\n    draw_obj(overlay, img_cor_points, triangles, color)\n\n    return overlay\n\ndef show_res(idx, pred_df, ht=1355, hb=2710, size=12, img_only=False):\n\n    img = cv2.imread('../input/pku-autonomous-driving/test_images/%s.jpg'%pred_df.iloc[idx].ImageId)[:,:,::-1]\n    mask = cv2.imread('../input/pku-autonomous-driving/test_masks/%s.jpg'%pred_df.iloc[idx].ImageId)\n        \n    if mask is not None:\n        img = img*(mask<128)\n        \n    overlay = np.zeros_like(img)\n    \n    pred_string = pred_df.iloc[idx].PredictionString\n    \n    if not isinstance(pred_string, str) or img_only:\n        plt.figure(figsize=(20,size))\n        plt.imshow(img[ht:hb])\n        return\n    \n    items = pred_string.split(' ')\n    items = np.array(items, dtype='float')\n    yaws, pitches, rolls, xs, ys, zs, scores = [items[i::7] for i in range(7)]   \n\n    for yaw, pitch, roll, x, y, z in zip(yaws, pitches, rolls, xs, ys, zs):\n        color = im_color[np.random.randint(256)].tolist()\n        overlay = draw_car(yaw, pitch, roll, x, y, z, overlay, color)\n    \n    #img[overlay>0] = img[overlay>0]//2 + overlay[overlay>0]//2\n    img = (np.maximum(overlay, img)//2 + img//2)\n    #img = overlay//2 + img//2\n    \n    plt.figure(figsize=(20,size))\n    plt.imshow(img[ht:hb])\n    \n    \ndef euler_to_Rot(yaw, pitch, roll):\n    Y = np.array([[cos(yaw), 0, sin(yaw)],\n                  [0, 1, 0],\n                  [-sin(yaw), 0, cos(yaw)]])\n    P = np.array([[1, 0, 0],\n                  [0, cos(pitch), -sin(pitch)],\n                  [0, sin(pitch), cos(pitch)]])\n    R = np.array([[cos(roll), -sin(roll), 0],\n                  [sin(roll), cos(roll), 0],\n                  [0, 0, 1]])\n    return np.dot(Y, np.dot(P, R))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# show results","metadata":{}},{"cell_type":"code","source":"for i in range(10):\n    show_res(i, sub_df, ht=1355, hb=2710, size=6)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}