{"cells":[{"metadata":{},"cell_type":"markdown","source":"# CenterNet Baseline ++\n\n## How far can we go with the baseline model?\n\nChanges from baseline:\n* Larger input size (2048x768)\n* Longer training (50 epochs)\n* use focal loss\n* use gaussian masks and regression targets\n* Normalize input image\n* Change model scale to 4\n* Change distance threshold to 4\n* Pool output by 3x3 pooling\n\nLB PB --> 0.07\n\nV7\n* Encode Z with sigmoid\nLB PB --> 0.078\n\nV8,V9\n* Add rotation augumentations\nLB PB --> 0.090\n\nV10 (not in the notebook)\n* 2x Effnet model + 2x resnet model ensambling\nLB PB --> 0.10\n\nhttps://www.kaggle.com/outrunner/rotation-augmentation\n\nIt took me about a day to train this model.\n\nThe mAP is about 0.20 evaluated on tito's scripts.\n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport cv2\nfrom tqdm import tqdm_notebook as tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom functools import reduce\nimport os\nfrom scipy.optimize import minimize\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.metrics import mean_absolute_error, mean_squared_error\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import models\nfrom torchvision import transforms, utils\n\nPATH = '../input/pku-autonomous-driving/'\nos.listdir(PATH)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load data"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"train = pd.read_csv(PATH + 'train.csv')\ntest = pd.read_csv(PATH + 'sample_submission.csv')\n\n# From camera.zip\ncamera_matrix = np.array([[2304.5479, 0,  1686.2379],\n                          [0, 2305.8757, 1354.9849],\n                          [0, 0, 1]], dtype=np.float32)\ncamera_matrix_inv = np.linalg.inv(camera_matrix)\n\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**ImageId** column contains names of images:"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def imread(path, fast_mode=False):\n    img = cv2.imread(path)\n    if not fast_mode and img is not None and len(img.shape) == 3:\n        img = np.array(img[:, :, ::-1])\n    return img\n\nimg = imread(PATH + 'train_images/ID_8a6e65317' + '.jpg')\nIMG_SHAPE = img.shape\n\nplt.figure(figsize=(15,8))\nplt.imshow(img);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**PredictionString** column contains pose information about all cars  \n\nFrom the data description:\n> The primary data is images of cars and related pose information. The pose information is formatted as strings, as follows:  \n>\n> `model type, yaw, pitch, roll, x, y, z`  \n>\n> A concrete example with two cars in the photo:  \n>\n> `5 0.5 0.5 0.5 0.0 0.0 0.0 32 0.25 0.25 0.25 0.5 0.4 0.7`  \n\nWe will need a function to extract these values:"},{"metadata":{"trusted":true},"cell_type":"code","source":"def str2coords(s, names=['id', 'yaw', 'pitch', 'roll', 'x', 'y', 'z']):\n    '''\n    Input:\n        s: PredictionString (e.g. from train dataframe)\n        names: array of what to extract from the string\n    Output:\n        list of dicts with keys from `names`\n    '''\n    coords = []\n    for l in np.array(s.split()).reshape([-1, 7]):\n        coords.append(dict(zip(names, l.astype('float'))))\n        if 'id' in coords[-1]:\n            coords[-1]['id'] = int(coords[-1]['id'])\n    return coords","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"inp = train['PredictionString'][0]\nprint('Example input:\\n', inp)\nprint()\nprint('Output:\\n', str2coords(inp))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data distributions"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"lens = [len(str2coords(s)) for s in train['PredictionString']]\n\nplt.figure(figsize=(15,6))\nsns.countplot(lens);\nplt.xlabel('Number of cars in image');","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"plt.figure(figsize=(15,6))\nsns.distplot(reduce(lambda a, b: a + b, [[c['x'] for c in str2coords(s)] for s in train['PredictionString']]), bins=500);\n# sns.distplot([str2coords(s)[0]['x'] for s in train['PredictionString']]);\nplt.xlabel('x')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"plt.figure(figsize=(15,6))\nsns.distplot(reduce(lambda a, b: a + b, [[c['y'] for c in str2coords(s)] for s in train['PredictionString']]), bins=500);\nplt.xlabel('y')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"plt.figure(figsize=(15,6))\nsns.distplot(reduce(lambda a, b: a + b, [[c['z'] for c in str2coords(s)] for s in train['PredictionString']]), bins=500);\nplt.xlabel('z')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"plt.figure(figsize=(15,6))\nsns.distplot(reduce(lambda a, b: a + b, [[c['yaw'] for c in str2coords(s)] for s in train['PredictionString']]));\nplt.xlabel('yaw')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"plt.figure(figsize=(15,6))\nsns.distplot(reduce(lambda a, b: a + b, [[c['pitch'] for c in str2coords(s)] for s in train['PredictionString']]));\nplt.xlabel('pitch')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"I guess, pitch and yaw are mixed up in this dataset. Pitch cannot be that big. That would mean that cars are upside down."},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def rotate(x, angle):\n    x = x + angle\n    x = x - (x + np.pi) // (2 * np.pi) * 2 * np.pi\n    return x\n\nplt.figure(figsize=(15,6))\nsns.distplot(reduce(lambda a, b: a + b, [[rotate(c['roll'], np.pi) for c in str2coords(s)] for s in train['PredictionString']]));\nplt.xlabel('roll rotated by pi')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 2D Visualization"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_img_coords(s):\n    '''\n    Input is a PredictionString (e.g. from train dataframe)\n    Output is two arrays:\n        xs: x coordinates in the image\n        ys: y coordinates in the image\n    '''\n    coords = str2coords(s)\n    xs = [c['x'] for c in coords]\n    ys = [c['y'] for c in coords]\n    zs = [c['z'] for c in coords]\n    P = np.array(list(zip(xs, ys, zs))).T\n    img_p = np.dot(camera_matrix, P).T\n    img_p[:, 0] /= img_p[:, 2]\n    img_p[:, 1] /= img_p[:, 2]\n    img_xs = img_p[:, 0]\n    img_ys = img_p[:, 1]\n    img_zs = img_p[:, 2] # z = Distance from the camera\n    return img_xs, img_ys\n\nplt.figure(figsize=(14,14))\nplt.imshow(imread(PATH + 'train_images/' + train['ImageId'][2217] + '.jpg'))\nplt.scatter(*get_img_coords(train['PredictionString'][2217]), color='red', s=100);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"One point is out of image!"},{"metadata":{},"cell_type":"markdown","source":"Let's look at the distribution of all points. Image is here just for reference."},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"xs, ys = [], []\n\nfor ps in train['PredictionString']:\n    x, y = get_img_coords(ps)\n    xs += list(x)\n    ys += list(y)\n\nplt.figure(figsize=(18,18))\nplt.imshow(imread(PATH + 'train_images/' + train['ImageId'][2217] + '.jpg'), alpha=0.3)\nplt.scatter(xs, ys, color='red', s=10, alpha=0.2);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Many points are outside!"},{"metadata":{},"cell_type":"markdown","source":"Let's look at this distribution \"from the sky\""},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# Cars points\nxs, ys = [], []\nfor ps in train['PredictionString']:\n    coords = str2coords(ps)\n    xs += [c['x'] for c in coords]\n    ys += [c['y'] for c in coords]\n\n# Road points\nroad_width = 6\nroad_xs = [-road_width, road_width, road_width, -road_width, -road_width]\nroad_ys = [0, 0, 500, 500, 0]\n\nplt.figure(figsize=(16,16))\nplt.axes().set_aspect(1)\n# View road\nplt.fill(road_xs, road_ys, alpha=0.2, color='gray')\nplt.plot([road_width/2,road_width/2], [0,500], alpha=0.4, linewidth=4, color='white', ls='--')\nplt.plot([-road_width/2,-road_width/2], [0,500], alpha=0.4, linewidth=4, color='white', ls='--')\n# View cars\nplt.scatter(xs, ys, color='red', s=20, alpha=0.2);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Some points are very far away\n\nScale up"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"plt.figure(figsize=(16,16))\nplt.axes().set_aspect(1)\nplt.xlim(-50,50)\nplt.ylim(0,50)\n\n# View road\nplt.fill(road_xs, road_ys, alpha=0.2, color='gray')\nplt.plot([road_width/2,road_width/2], [0,100], alpha=0.4, linewidth=4, color='white', ls='--')\nplt.plot([-road_width/2,-road_width/2], [0,100], alpha=0.4, linewidth=4, color='white', ls='--')\n# View cars\nplt.scatter(xs, ys, color='red', s=10, alpha=0.1);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3D Visualization\nUsed code from https://www.kaggle.com/zstusnoopy/visualize-the-location-and-3d-bounding-box-of-car, but made it one function"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"from math import sin, cos\n\n# convert euler angle to rotation matrix\ndef euler_to_Rot(yaw, pitch, roll):\n    Y = np.array([[cos(yaw), 0, sin(yaw)],\n                  [0, 1, 0],\n                  [-sin(yaw), 0, cos(yaw)]])\n    P = np.array([[1, 0, 0],\n                  [0, cos(pitch), -sin(pitch)],\n                  [0, sin(pitch), cos(pitch)]])\n    R = np.array([[cos(roll), -sin(roll), 0],\n                  [sin(roll), cos(roll), 0],\n                  [0, 0, 1]])\n    return np.dot(Y, np.dot(P, R))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def draw_line(image, points):\n    color = (255, 0, 0)\n    cv2.line(image, tuple(points[0][:2]), tuple(points[3][:2]), color, 16)\n    cv2.line(image, tuple(points[0][:2]), tuple(points[1][:2]), color, 16)\n    cv2.line(image, tuple(points[1][:2]), tuple(points[2][:2]), color, 16)\n    cv2.line(image, tuple(points[2][:2]), tuple(points[3][:2]), color, 16)\n    return image\n\n\ndef draw_points(image, points):\n    for (p_x, p_y, p_z) in points:\n        cv2.circle(image, (p_x, p_y), int(1000 / p_z), (0, 255, 0), -1)\n#         if p_x > image.shape[1] or p_y > image.shape[0]:\n#             print('Point', p_x, p_y, 'is out of image with shape', image.shape)\n    return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def visualize(img, coords):\n    # You will also need functions from the previous cells\n    x_l = 1.02\n    y_l = 0.80\n    z_l = 2.31\n    \n    img = img.copy()\n    for point in coords:\n        # Get values\n        x, y, z = point['x'], point['y'], point['z']\n        yaw, pitch, roll = -point['pitch'], -point['yaw'], -point['roll']\n        # Math\n        Rt = np.eye(4)\n        t = np.array([x, y, z])\n        Rt[:3, 3] = t\n        Rt[:3, :3] = euler_to_Rot(yaw, pitch, roll).T\n        Rt = Rt[:3, :]\n        P = np.array([[x_l, -y_l, -z_l, 1],\n                      [x_l, -y_l, z_l, 1],\n                      [-x_l, -y_l, z_l, 1],\n                      [-x_l, -y_l, -z_l, 1],\n                      [0, 0, 0, 1]]).T\n        img_cor_points = np.dot(camera_matrix, np.dot(Rt, P))\n        img_cor_points = img_cor_points.T\n        img_cor_points[:, 0] /= img_cor_points[:, 2]\n        img_cor_points[:, 1] /= img_cor_points[:, 2]\n        img_cor_points = img_cor_points.astype(int)\n        # Drawing\n        img = draw_line(img, img_cor_points)\n        img = draw_points(img, img_cor_points[-1:])\n    \n    return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"n_rows = 6\n\nfor idx in range(n_rows):\n    fig, axes = plt.subplots(1, 2, figsize=(20,20))\n    img = imread(PATH + 'train_images/' + train['ImageId'].iloc[idx] + '.jpg')\n    axes[0].imshow(img)\n    img_vis = visualize(img, str2coords(train['PredictionString'].iloc[idx]))\n    axes[1].imshow(img_vis)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Image preprocessing"},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ndef sigmoid(data):\n    return 1 / (1 + np.exp(-data))\n\ndef invsigmoid(data):\n    return np.log(data / (1 -data))\n\ndef rotate(x, angle):\n    x = x + angle\n    x = x - (x + np.pi) // (2 * np.pi) * 2 * np.pi\n    return x\n\ndef _regr_preprocess(regr_dict):\n    for name in ['x', 'y', 'z']:\n        regr_dict[name] = regr_dict[name] / 100\n    regr_dict[\"z\"] = 1/(sigmoid(regr_dict[\"z\"]))-1\n    regr_dict['roll'] = rotate(regr_dict['roll'], np.pi)\n    regr_dict['pitch_sin'] = sin(regr_dict['pitch'])\n    regr_dict['pitch_cos'] = cos(regr_dict['pitch'])\n    regr_dict.pop('pitch')\n    regr_dict.pop('id')\n    return regr_dict\n\ndef _regr_back(regr_dict):\n    regr_dict[\"z\"] = invsigmoid(1/(regr_dict[\"z\"]+1))\n    for name in ['x', 'y', 'z']:\n        regr_dict[name] = regr_dict[name] * 100\n    regr_dict['roll'] = rotate(regr_dict['roll'], -np.pi)\n    \n    pitch_sin = regr_dict['pitch_sin'] / np.sqrt(regr_dict['pitch_sin']**2 + regr_dict['pitch_cos']**2)\n    pitch_cos = regr_dict['pitch_cos'] / np.sqrt(regr_dict['pitch_sin']**2 + regr_dict['pitch_cos']**2)\n    regr_dict['pitch'] = np.arccos(pitch_cos) * np.sign(pitch_sin)\n    return regr_dict\n\ndef preprocess_image(img, flip=False):\n    img = img[img.shape[0] // 2:]\n    bg = np.ones_like(img) * img.mean(1, keepdims=True).astype(img.dtype)\n    bg = bg[:, :img.shape[1] // MARGIN]\n    img = np.concatenate([bg, img, bg], 1)\n    img = cv2.resize(img, (IMG_WIDTH, IMG_HEIGHT))\n    \n    if flip:\n        img = img[:,::-1]\n    return (img / 255).astype('float32')\n\ndef preprocess_bg(img, flip=False):\n    img = img[img.shape[0] // 2:]\n    bg = np.zeros_like(img) * img.mean(1, keepdims=True).astype(img.dtype)\n    bg = bg[:, :img.shape[1] // 8]\n    img = np.concatenate([bg, img, bg], 1)\n    img = cv2.resize(img, (IMG_WIDTH, IMG_HEIGHT))\n    \n    if flip:\n        img = img[:,::-1]\n    return (img).astype('float32')\n\n# from centernet repo\ndef draw_msra_gaussian(heatmap, center, sigma=1):\n  tmp_size = sigma * 3\n  mu_x = int(center[0] + 0.5)\n  mu_y = int(center[1] + 0.5)\n  w, h = heatmap.shape[0], heatmap.shape[1]\n  ul = [int(mu_x - tmp_size), int(mu_y - tmp_size)]\n  br = [int(mu_x + tmp_size + 1), int(mu_y + tmp_size + 1)]\n  if ul[0] >= h or ul[1] >= w or br[0] < 0 or br[1] < 0:\n    return heatmap\n  size = 2 * tmp_size + 1\n  x = np.arange(0, size, 1, np.float32)\n  y = x[:, np.newaxis]\n  x0 = y0 = size // 2\n  g = np.exp(- ((x - x0) ** 2 + (y - y0) ** 2) / (2 * sigma ** 2))\n  g_x = max(0, -ul[0]), min(br[0], h) - ul[0]\n  g_y = max(0, -ul[1]), min(br[1], w) - ul[1]\n  img_x = max(0, ul[0]), min(br[0], h)\n  img_y = max(0, ul[1]), min(br[1], w)\n  heatmap[img_y[0]:img_y[1], img_x[0]:img_x[1]] = np.maximum(\n    heatmap[img_y[0]:img_y[1], img_x[0]:img_x[1]],\n    g[g_y[0]:g_y[1], g_x[0]:g_x[1]])\n  return heatmap\n\ndef draw_dense_reg(regmap, heatmap, center, value, radius=1, is_offset=False):\n  diameter = 2 * radius + 1\n  gaussian = gaussian2D((diameter, diameter), sigma= diameter/6)\n  value = np.array(value, dtype=np.float32).reshape(-1, 1, 1)\n  dim = value.shape[0]\n  reg = np.ones((dim, diameter*2+1, diameter*2+1), dtype=np.float32) * value\n  if is_offset and dim == 2:\n    delta = np.arange(diameter*2+1) - radius\n    reg[0] = reg[0] - delta.reshape(1, -1)\n    reg[1] = reg[1] - delta.reshape(-1, 1)\n  \n  x, y = int(center[0]), int(center[1])\n\n  height, width = heatmap.shape[0:2]\n    \n  left, right = min(x, radius), min(width - x, radius + 1)\n  top, bottom = min(y, radius), min(height - y, radius + 1)\n\n  masked_heatmap = heatmap[y - top:y + bottom, x - left:x + right]\n  masked_regmap = regmap[:, y - top:y + bottom, x - left:x + right]\n  masked_gaussian = gaussian[radius - top:radius + bottom,\n                             radius - left:radius + right]\n  masked_reg = reg[:, radius - top:radius + bottom,\n                      radius - left:radius + right]\n  if min(masked_gaussian.shape) > 0 and min(masked_heatmap.shape) > 0: # TODO debug\n    idx = (masked_gaussian >= masked_heatmap).reshape(\n      1, masked_gaussian.shape[0], masked_gaussian.shape[1])\n    masked_regmap = (1-idx) * masked_regmap + idx * masked_reg\n  regmap[:, y - top:y + bottom, x - left:x + right] = masked_regmap\n  return regmap\n\ndef gaussian2D(shape, sigma=1):\n    m, n = [(ss - 1.) / 2. for ss in shape]\n    y, x = np.ogrid[-m:m+1,-n:n+1]\n\n    h = np.exp(-(x * x + y * y) / (2 * sigma * sigma))\n    h[h < np.finfo(h.dtype).eps * h.max()] = 0\n    return h\n\n# modify for this competition\ndef get_mask_and_regr(img, labels, flip=False, coord=False):\n    mask = np.zeros([IMG_HEIGHT // MODEL_SCALE, IMG_WIDTH // MODEL_SCALE], dtype='float32')\n    regr_names = ['x', 'y', 'z', 'yaw', 'pitch', 'roll']\n    regr = np.zeros([IMG_HEIGHT // MODEL_SCALE, IMG_WIDTH // MODEL_SCALE, 7], dtype='float32')\n    \n    if not coord:\n        coords = str2coords(labels)\n        xs, ys = get_img_coords(labels)\n    else:\n        coords = labels\n        xs, ys = labels2xs(coords)    \n    \n    regr = regr.transpose([2,0,1])\n    \n    for x, y, regr_dict in zip(xs, ys, coords):\n        x, y = y, x\n        x = (x - img.shape[0] // 2) * IMG_HEIGHT / (img.shape[0] // 2) / MODEL_SCALE\n        x = np.round(x).astype('int')\n        y = (y + img.shape[1] // MARGIN) * IMG_WIDTH / (img.shape[1] * 1.5) / MODEL_SCALE\n        y = np.round(y).astype('int')\n        if x >= 0 and x < IMG_HEIGHT // MODEL_SCALE and y >= 0 and y < IMG_WIDTH // MODEL_SCALE:\n            mask = draw_msra_gaussian(mask, [y,x])\n            #mask[x, y] = 1\n            # radius = 30/regr_dict[\"z\"]\n            #print(regr_dict[\"z\"])\n            regr_dict = _regr_preprocess(regr_dict)\n            regrs = [regr_dict[n] for n in sorted(regr_dict)]\n            #print(radius)\n            #radius = int(radius*2)\n            #for i in range(7):\n            regr = draw_dense_reg(regr, mask, [y,x], regrs, 3, True)\n            #regr[x, y] = [regr_dict[n] for n in sorted(regr_dict)]\n    regr = regr.transpose([1,2,0])\n    \n    # create mask in regr as well\n    for i, r in enumerate(regr[0,0,:]):\n        #print(regr.shape)\n        regr[:,:,i] *= (mask > 0.1)\n    \n    if flip:\n        mask = np.array(mask[:,::-1])\n        regr = np.array(regr[:,::-1])\n    return mask, regr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"im_color = cv2.applyColorMap(np.arange(256).astype('uint8') , cv2.COLORMAP_HSV)[:,0,:]\nCV_PI = np.pi\n\ndef rotateImage(img, alpha=0, beta=0, gamma=0):\n    fx, dx = 2304.5479, 1686.2379\n    fy, dy = 2305.8757, 1354.9849\n    # get width and height for ease of use in matrices\n    h, w = img.shape[:2]\n    # Projection 2D -> 3D matrix\n    A1 = np.array([[1/fx, 0, -dx/fx],\n                   [0, 1/fx, -dy/fx],\n                   [0, 0,    1],\n                   [0, 0,    1]])\n    \n    # Rotation matrices around the X, Y, and Z axis\n    RX = np.array([[1,          0,           0, 0],\n             [0, cos(alpha), -sin(alpha), 0],\n             [0, sin(alpha),  cos(alpha), 0],\n             [0,          0,           0, 1]])\n    \n    RY = np.array([[cos(beta), 0, -sin(beta), 0],\n              [0, 1,          0, 0],\n              [sin(beta), 0,  cos(beta), 0],\n              [0, 0,          0, 1]])\n    RZ = np.array([[cos(gamma), -sin(gamma), 0, 0],\n              [sin(gamma),  cos(gamma), 0, 0],\n              [0,          0,           1, 0],\n              [0,          0,           0, 1]])\n    # Composed rotation matrix with (RX, RY, RZ)\n    R = np.dot(RZ, np.dot(RX, RY))\n\n    # 3D -> 2D matrix\n    A2 = np.array([[fx, 0, dx, 0],\n                   [0, fy, dy, 0],\n                   [0, 0,   1, 0]])\n    # Final transformation matrix\n    trans = np.dot(A2,np.dot(R, A1))\n    # Apply matrix transformation\n    return cv2.warpPerspective(img, trans, (w,h), flags=cv2.INTER_LANCZOS4), trans, R\n\ndef draw_car(yaw, pitch, roll, x, y, z, overlay, color=(0,0,255)):\n    yaw, pitch, roll = -pitch, -yaw, -roll\n    Rt = np.eye(4)\n    t = np.array([x, y, z])\n    Rt[:3, 3] = t\n    Rt[:3, :3] = euler_to_Rot(yaw, pitch, roll).T\n    Rt = Rt[:3, :]\n    P = np.ones((vertices.shape[0],vertices.shape[1]+1))\n    P[:, :-1] = vertices\n    P = P.T\n    img_cor_points = np.dot(k, np.dot(Rt, P))\n    img_cor_points = img_cor_points.T\n    img_cor_points[:, 0] /= img_cor_points[:, 2]\n    img_cor_points[:, 1] /= img_cor_points[:, 2]\n    draw_obj(overlay, img_cor_points, triangles, color)\n    return overlay\n\ndef draw_obj(image, vertices, triangles, color):\n    for t in triangles:\n        coord = np.array([vertices[t[0]][:2], vertices[t[1]][:2], vertices[t[2]][:2]], dtype=np.int32)\n#         cv2.fillConvexPoly(image, coord, (0,0,255))\n        cv2.polylines(image, np.int32([coord]), 1, color)\n\ndef euler_to_Rot(yaw, pitch, roll):\n    Y = np.array([[cos(yaw), 0, sin(yaw)],\n                  [0, 1, 0],\n                  [-sin(yaw), 0, cos(yaw)]])\n    P = np.array([[1, 0, 0],\n                  [0, cos(pitch), -sin(pitch)],\n                  [0, sin(pitch), cos(pitch)]])\n    R = np.array([[cos(roll), -sin(roll), 0],\n                  [sin(roll), cos(roll), 0],\n                  [0, 0, 1]])\n    return np.dot(Y, np.dot(P, R))\n\nk = np.array([[2304.5479, 0,  1686.2379],\n           [0, 2305.8757, 1354.9849],\n           [0, 0, 1]], dtype=np.float32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from scipy.spatial.transform import Rotation as R\ndef draw_rot(img, labels, alpha = 0, beta = 0, gamma = 0, show=False):\n    pred_string = labels\n    items = pred_string.split(' ')\n    items = np.array(items, dtype='float')\n    model_types, yaws, pitches, rolls, xs, ys, zs = [items[i::7] for i in range(7)]\n\n    alpha = alpha*CV_PI/180.\n    beta = beta*CV_PI/180.\n    gamma = gamma*CV_PI/180.\n\n    dst, Mat,Rot = rotateImage(img, alpha, beta, gamma)\n    overlay = np.zeros_like(dst)\n\n    for i, (yaw, pitch, roll, x, y, z) in enumerate(zip(yaws, pitches, rolls, xs, ys, zs)):\n\n        x,y,z,_ = np.dot(Rot,[x, y, z, 1])\n\n        r1 = R.from_euler('xyz', [-pitch, -yaw, -roll], degrees=False)\n        r2 = R.from_euler('xyz', [beta, -alpha, -gamma], degrees=False)\n\n        pitch2, yaw2, roll2 = (r2*r1).as_euler('xyz')*(-1)\n\n        color = im_color[np.random.randint(256)].tolist()\n        if show:\n            overlay = draw_car(yaw2, pitch2, roll2, x, y, z, overlay, color)\n\n        #print(\"yaw, pitch, roll, yaw2, pitch2, roll2\", np.array([yaw, pitch, roll, yaw2, pitch2, roll2]))\n        #print(\"xyz\", np.array([x,y,z]))\n        \n        yaws[i] = yaw2\n        pitches[i] = pitch2\n        rolls[i] = roll2\n        xs[i] = x\n        ys[i] = y\n        zs[i] = z\n        #break\n    if show:\n        plt.figure(figsize=(20,20))\n        plt.imshow((dst*(np.sum(overlay, axis=-1)[:,:,np.newaxis]==0)+overlay), interpolation='lanczos')\n    \n    return yaws, pitches, rolls, xs, ys, zs, dst","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def coords2str(coords, names=['yaw', 'pitch', 'roll', 'x', 'y', 'z', 'confidence']):\n    s = []\n    for c in coords:\n        for n in names:\n            s.append(str(c.get(n, 0)))\n    return ' '.join(s)\n\ndef stuff2coords(yaws, pitches, rolls, xs, ys, zs):\n    coords = []\n    names=['yaw', 'pitch', 'roll', 'x', 'y', 'z', 'id']\n    for i, (yaw, pitch, roll, x, y, z) in enumerate(zip(yaws, pitches, rolls, xs, ys, zs)):\n        coords.append(dict(zip(names, [yaw,pitch,roll,x,y,z,1])))\n    return coords\n\ndef str2coords(s, names=['id', 'yaw', 'pitch', 'roll', 'x', 'y', 'z']):\n    '''\n    Input:\n        s: PredictionString (e.g. from train dataframe)\n        names: array of what to extract from the string\n    Output:\n        list of dicts with keys from `names`\n    '''\n    coords = []\n    for l in np.array(s.split()).reshape([-1, 7]):\n        coords.append(dict(zip(names, l.astype('float'))))\n        if 'id' in coords[-1]:\n            coords[-1]['id'] = int(coords[-1]['id'])\n    return coords\n\ndef labels2xs(coords):\n    xs = [c['x'] for c in coords]\n    ys = [c['y'] for c in coords]\n    zs = [c['z'] for c in coords]\n    P = np.array(list(zip(xs, ys, zs))).T\n    img_p = np.dot(camera_matrix, P).T\n    img_p[:, 0] /= img_p[:, 2]\n    img_p[:, 1] /= img_p[:, 2]\n    img_xs = img_p[:, 0]\n    img_ys = img_p[:, 1]\n    img_zs = img_p[:, 2] # z = Distance from the camera\n    return img_xs, img_ys\n\ndef get_img_coords(s):\n    '''\n    Input is a PredictionString (e.g. from train dataframe)\n    Output is two arrays:\n        xs: x coordinates in the image\n        ys: y coordinates in the image\n    '''\n    coords = str2coords(s)\n    xs = [c['x'] for c in coords]\n    ys = [c['y'] for c in coords]\n    zs = [c['z'] for c in coords]\n    P = np.array(list(zip(xs, ys, zs))).T\n    img_p = np.dot(camera_matrix, P).T\n    img_p[:, 0] /= img_p[:, 2]\n    img_p[:, 1] /= img_p[:, 2]\n    img_xs = img_p[:, 0]\n    img_ys = img_p[:, 1]\n    img_zs = img_p[:, 2] # z = Distance from the camera\n    return img_xs, img_ys","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img0 = imread(PATH + 'train_images/' + train['ImageId'][0] + '.jpg')\nIMG_SHAPE = img0.shape\n\n# augument\nyaws, pitches, rolls, xs, ys, zs, dst = draw_rot(img0, train['PredictionString'][0], alpha=1,beta=10)\ncoords = stuff2coords(yaws, pitches, rolls, xs, ys, zs)\n\nimg = preprocess_image(dst)\n\nprint(coords)\nmask, regr = get_mask_and_regr(img0, coords, coord=True)\n\nprint('img.shape', img.shape, 'std:', np.std(img))\nprint('mask.shape', mask.shape, 'std:', np.std(mask))\nprint('regr.shape', regr.shape, 'std:', np.std(regr))\n\nplt.figure(figsize=(16,16))\nplt.title('Processed image')\nplt.imshow(img)\nplt.show()\n\nplt.figure(figsize=(16,16))\nplt.title('Detection Mask')\nplt.imshow(mask)\nplt.show()\n\nplt.figure(figsize=(16,16))\nplt.title('Yaw values')\nplt.imshow(regr[:,:,-2])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img0 = imread(PATH + 'train_images/' + train['ImageId'][0] + '.jpg')\nimg = preprocess_image(img0)\n\nmask, regr = get_mask_and_regr(img0, train['PredictionString'][0])\n\nprint('img.shape', img.shape, 'std:', np.std(img))\nprint('mask.shape', mask.shape, 'std:', np.std(mask))\nprint('regr.shape', regr.shape, 'std:', np.std(regr))\n\nplt.figure(figsize=(16,16))\nplt.title('Processed image')\nplt.imshow(img)\nplt.show()\n\nplt.figure(figsize=(16,16))\nplt.title('Detection Mask')\nplt.imshow(mask)\nplt.show()\n\nplt.figure(figsize=(16,16))\nplt.title('Yaw values')\nplt.imshow(regr[:,:,-2])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Define functions to convert back from 2d map to 3d coordinates and angles"},{"metadata":{},"cell_type":"markdown","source":"Ensure that all the forward and back transformations work consistently"},{"metadata":{},"cell_type":"markdown","source":"# PyTorch Dataset"},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"class CarDataset(Dataset):\n    \"\"\"Car dataset.\"\"\"\n\n    def __init__(self, dataframe, root_dir, training=True, transform=None):\n        self.df = dataframe\n        self.root_dir = root_dir\n        self.transform = transform\n        self.training = training\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        if torch.is_tensor(idx):\n            idx = idx.tolist()\n        \n        # Get image name\n        idx2 = idx\n        idx, labels = self.df.values[idx]\n        img_name = self.root_dir.format(idx)\n        img0 = imread(img_name, True)\n        \n        # Augmentation\n        flip = False\n        alpha = 0\n        beta = 0\n        mask = 0\n        while np.sum(mask) == 0:\n            if self.training:\n                if np.random.rand() < 0.3:\n                    flip = True\n            if self.training:\n                if np.random.rand() < 0.5:\n                    alpha = (np.random.rand()-0.5)*5\n                    beta = (np.random.rand()-0.5)*30\n\n            # augument\n            yaws, pitches, rolls, xs, ys, zs, dst = draw_rot(img0, labels, alpha=alpha, beta=beta)\n            coords = stuff2coords(yaws, pitches, rolls, xs, ys, zs)\n            img = preprocess_image(dst, flip=flip)\n            img = np.rollaxis(img, 2, 0)\n\n            # normalize image\n            norm = True\n            if norm:\n                mean = np.array([[0.485, 0.456, 0.406]]).T\n                std =  np.array([[0.229, 0.224, 0.225]]).T\n                img[0] -= mean[0]\n                img[0] /= std[0]\n                img[1] -= mean[1]\n                img[1] /= std[1]\n                img[2] -= mean[2]\n                img[2] /= std[2]\n\n            # Get mask and regression maps\n            mask, regr = get_mask_and_regr(dst, coords, coord=True, flip=flip)\n            regr = np.rollaxis(regr, 2, 0)\n        \n        return [img, mask, regr]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"train_images_dir = PATH + 'train_images/{}.jpg'\ntest_images_dir = PATH + 'test_images/{}.jpg'\n\ndf_train, df_dev = train_test_split(train, test_size=0.01, random_state=42)\ndf_test = test\n\n# Create dataset objects\ntrain_dataset = CarDataset(df_train, train_images_dir, training=True)\ndev_dataset = CarDataset(df_dev, train_images_dir, training=False)\ntest_dataset = CarDataset(df_test, test_images_dir, training=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Show some generated examples"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"img, mask, regr = train_dataset[0]\n\nplt.figure(figsize=(16,16))\nplt.imshow(np.rollaxis(img, 0, 3))\nplt.show()\n\nplt.figure(figsize=(16,16))\nplt.imshow(mask)\nplt.show()\n\nplt.figure(figsize=(16,16))\nplt.imshow(regr[-2])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"BATCH_SIZE = 2\n\n# Create data generators - they will produce batches\ntrain_loader = DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=4)\ndev_loader = DataLoader(dataset=dev_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4)\ntest_loader = DataLoader(dataset=test_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# PyTorch Model"},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"!pip install efficientnet-pytorch","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from efficientnet_pytorch import EfficientNet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"class double_conv(nn.Module):\n    '''(conv => BN => ReLU) * 2'''\n    def __init__(self, in_ch, out_ch):\n        super(double_conv, self).__init__()\n        self.conv = nn.Sequential(\n            nn.Conv2d(in_ch, out_ch, 3, padding=1),\n            nn.BatchNorm2d(out_ch),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(out_ch, out_ch, 3, padding=1),\n            nn.BatchNorm2d(out_ch),\n            nn.ReLU(inplace=True)\n        )\n\n    def forward(self, x):\n        x = self.conv(x)\n        return x\n\nclass up(nn.Module):\n    def __init__(self, in_ch, out_ch, bilinear=True):\n        super(up, self).__init__()\n\n        #  would be a nice idea if the upsampling could be learned too,\n        #  but my machine do not have enough memory to handle all those weights\n        if bilinear:\n            self.up = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True)\n        else:\n            self.up = nn.ConvTranspose2d(in_ch//2, in_ch//2, 2, stride=2)\n\n        self.conv = double_conv(in_ch, out_ch)\n\n    def forward(self, x1, x2=None):\n        x1 = self.up(x1)\n        \n        # input is CHW\n        diffY = x2.size()[2] - x1.size()[2]\n        diffX = x2.size()[3] - x1.size()[3]\n\n        x1 = F.pad(x1, (diffX // 2, diffX - diffX//2,\n                        diffY // 2, diffY - diffY//2))\n        \n        # for padding issues, see \n        # https://github.com/HaiyongJiang/U-Net-Pytorch-Unstructured-Buggy/commit/0e854509c2cea854e247a9c615f175f76fbb2e3a\n        # https://github.com/xiaopeng-liao/Pytorch-UNet/commit/8ebac70e633bac59fc22bb5195e513d5832fb3bd\n        \n        if x2 is not None:\n            x = torch.cat([x2, x1], dim=1)\n        else:\n            x = x1\n        x = self.conv(x)\n        return x\n\ndef get_mesh(batch_size, shape_x, shape_y):\n    mg_x, mg_y = np.meshgrid(np.linspace(0, 1, shape_y), np.linspace(0, 1, shape_x))\n    mg_x = np.tile(mg_x[None, None, :, :], [batch_size, 1, 1, 1]).astype('float32')\n    mg_y = np.tile(mg_y[None, None, :, :], [batch_size, 1, 1, 1]).astype('float32')\n    mesh = torch.cat([torch.tensor(mg_x).to(device), torch.tensor(mg_y).to(device)], 1)\n    return mesh","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"class MyUNet(nn.Module):\n    '''Mixture of previous classes'''\n    def __init__(self, n_classes):\n        super(MyUNet, self).__init__()\n        self.base_model = EfficientNet.from_pretrained('efficientnet-b0')\n        \n        self.conv0 = double_conv(5, 64)\n        self.conv1 = double_conv(64, 128)\n        self.conv2 = double_conv(128, 512)\n        self.conv3 = double_conv(512, 1024)\n        \n        self.mp = nn.MaxPool2d(2)\n        \n        self.up1 = up(1282 + 1024, 512)\n        self.up2 = up(512 + 512, 256)\n        self.up3 = up(384, 256)\n        self.outc = nn.Conv2d(256, n_classes, 1)\n\n    def forward(self, x):\n        batch_size = x.shape[0]\n        mesh1 = get_mesh(batch_size, x.shape[2], x.shape[3])\n        x0 = torch.cat([x, mesh1], 1)\n        x1 = self.mp(self.conv0(x0))\n        x2 = self.mp(self.conv1(x1))\n        x3 = self.mp(self.conv2(x2))\n        x4 = self.mp(self.conv3(x3))\n        \n        x_center = x[:, :, :, IMG_WIDTH // 8: -IMG_WIDTH // 8]\n        #print(x_center.size())\n        feats = self.base_model.extract_features(x_center)\n        bg = torch.zeros([feats.shape[0], feats.shape[1], feats.shape[2], feats.shape[3] // 8]).to(device)\n        feats = torch.cat([bg, feats, bg], 3)\n        \n        # Add positional info\n        mesh2 = get_mesh(batch_size, feats.shape[2], feats.shape[3])\n        feats = torch.cat([feats, mesh2], 1)\n        \n        x = self.up1(feats, x4)\n        x = self.up2(x, x3)\n        x = self.up3(x, x2)\n        x = self.outc(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# Gets the GPU if there is one, otherwise the cpu\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(device)\n\nn_epochs = 50\n\nmodel = MyUNet(8).to(device)\noptimizer = optim.Adam(model.parameters(), lr=0.0001)\nexp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=max(n_epochs, 10) * len(train_loader) // 3, gamma=0.33)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"# focal loss\ndef neg_loss(pred, gt):\n  ''' Modified focal loss. Exactly the same as CornerNet.\n      Runs faster and costs a little bit more memory\n    Arguments:\n      pred (batch x c x h x w)\n      gt_regr (batch x c x h x w)\n  '''\n  pred = pred.unsqueeze(1).float()\n  gt = gt.unsqueeze(1).float()\n\n  pos_inds = gt.eq(1).float()\n  neg_inds = gt.lt(1).float()\n  neg_weights = torch.pow(1 - gt, 4)\n\n  loss = 0\n\n  pos_loss = torch.log(pred + 1e-12) * torch.pow(1 - pred, 3) * pos_inds\n  neg_loss = torch.log(1 - pred + 1e-12) * torch.pow(pred, 3) * neg_weights * neg_inds\n\n  num_pos  = pos_inds.float().sum()\n  pos_loss = pos_loss.sum()\n  neg_loss = neg_loss.sum()\n\n  if num_pos == 0:\n    loss = loss - neg_loss\n  else:\n    loss = loss - (pos_loss + neg_loss) / num_pos\n  return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def criterion(prediction, mask, regr,weight=0.4, size_average=True):\n    # Binary mask loss\n    pred_mask = torch.sigmoid(prediction[:, 0])\n    mask_loss = neg_loss(pred_mask, mask)\n    \n    # Regression L1 loss\n    pred_regr = prediction[:, 1:]\n    regr_loss = (torch.abs(pred_regr - regr).sum(1) * mask).sum(1).sum(1) / mask.sum(1).sum(1)\n    regr_loss = regr_loss.mean(0)\n  \n    # Sum\n    loss = mask_loss +regr_loss\n    if not size_average:\n        loss *= prediction.shape[0]\n    return loss ,mask_loss , regr_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def train_model(epoch, history=None):\n    model.train()\n    t = tqdm(train_loader)\n    maskloss_acc = []\n    regrloss_acc = []\n    \n    # train loops\n    ####################################################################################\n    for batch_idx, (img_batch, mask_batch, regr_batch) in enumerate(t):\n        img_batch = img_batch.to(device)\n        mask_batch = mask_batch.to(device)\n        regr_batch = regr_batch.to(device)\n        \n        optimizer.zero_grad()\n        output = model(img_batch)\n        loss, mask_loss, regr_loss = criterion(output, mask_batch, regr_batch)\n        if history is not None:\n            history.loc[epoch + batch_idx / len(train_loader), 'train_loss'] = loss.data.cpu().numpy()\n        maskloss_acc.append(mask_loss.data.cpu().numpy())\n        regrloss_acc.append(regr_loss.data.cpu().numpy())\n        loss.backward()\n        \n        t.set_description(f'train_loss (l={loss:.3f})(m={mask_loss:.2f}) (r={regr_loss:.4f}')\n        \n        optimizer.step()\n        \n        exp_lr_scheduler.step()\n    \n        if batch_idx % 200 == 199:\n                print('Train Epoch: {} \\tLR: {:.6f}\\tmaskLoss: {:.6f} \\t regrLoss: {:.6f} '.format(\n                    epoch,\n                    optimizer.state_dict()['param_groups'][0]['lr'],\n                    np.mean(maskloss_acc), np.mean(regrloss_acc)))\n    #####################################################################################\n    \n    print('Train Epoch: {} \\tLR: {:.6f}\\tmaskLoss: {:.6f} \\t regrLoss: {:.6f} '.format(\n                epoch,\n                optimizer.state_dict()['param_groups'][0]['lr'],\n                np.mean(maskloss_acc), np.mean(regrloss_acc)))\n    \n    # save logs\n    log_epoch = {'epoch': epoch+1, 'lr': optimizer.state_dict()['param_groups'][0]['lr'],\n                     'mask_loss': np.mean(maskloss_acc), 'regr_loss': np.mean(regrloss_acc)}\n    logs.append(log_epoch)\n    df = pd.DataFrame(logs)\n    df.to_csv(\"log_output_train.csv\")\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"%%time\nimport gc\n\nhistory = pd.DataFrame()\n\"\"\"\nfor epoch in range(n_epochs):\n    torch.cuda.empty_cache()\n    gc.collect()\n    train_model(epoch, history)\n    torch.save(model.state_dict(), './model{}.pth'.format(str(epoch)))\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history['train_loss'].iloc[100:].plot();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"series = history.dropna()['dev_loss']\nplt.scatter(series.index, series);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visualize predictions"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"img, mask, regr = dev_dataset[0]\n\nplt.figure(figsize=(16,16))\nplt.title('Input image')\nplt.imshow(np.rollaxis(img, 0, 3))\nplt.show()\n\nplt.figure(figsize=(16,16))\nplt.title('Ground truth mask')\nplt.imshow(mask)\nplt.show()\n\noutput = model(torch.tensor(img[None]).to(device))\nlogits = output[0,0].data.cpu().numpy()\n\nplt.figure(figsize=(16,16))\nplt.title('Model predictions')\nplt.imshow(logits)\nplt.show()\n\nplt.figure(figsize=(16,16))\nplt.title('Model predictions thresholded')\nplt.imshow(logits > 0)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"torch.cuda.empty_cache()\ngc.collect()\n\nfor idx in range(8):\n    img, mask, regr = dev_dataset[idx]\n    \n    output = model(torch.tensor(img[None]).to(device)).data.cpu().numpy()\n    coords_pred = extract_coords(output[0])\n    coords_true = extract_coords(np.concatenate([mask[None], regr], 0))\n    \n    img = imread(train_images_dir.format(df_dev['ImageId'].iloc[idx]))\n    \n    fig, axes = plt.subplots(1, 2, figsize=(30,30))\n    axes[0].set_title('Ground truth')\n    axes[0].imshow(visualize(img, coords_true))\n    axes[1].set_title('Prediction')\n    axes[1].imshow(visualize(img, coords_pred))\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Make submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = []\n\ntest_loader = DataLoader(dataset=test_dataset, batch_size=4, shuffle=False, num_workers=4)\n\nmodel.eval()\n\nfor img, _, _ in tqdm(test_loader):\n    with torch.no_grad():\n        output = model(img.to(device))\n    output = output.data.cpu().numpy()\n    for out in output:\n        coords = extract_coords(out)\n        s = coords2str(coords)\n        predictions.append(s)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv(PATH + 'sample_submission.csv')\ntest['PredictionString'] = predictions\ntest.to_csv('predictions.csv', index=False)\ntest.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}