{"cells":[{"metadata":{},"cell_type":"markdown","source":"[[](http://)](http://)# Center-resnext50 Starter\n"},{"metadata":{},"cell_type":"markdown","source":"this kernel was forked from my friend Nirjhor Roy's kernel : https://www.kaggle.com/phoenix9032/center-resnet-starter\n\n\n"},{"metadata":{},"cell_type":"markdown","source":"**ChangeLogs**"},{"metadata":{},"cell_type":"markdown","source":"1. version 1 : 1 epoch took 1+ hours so in version 2 i will try for 8 epoches\n2. for 8 epoches in version 2 i got 0.095 public lb score\n3. in version 3 i will try for 16 epoches!\n4. version 4 : inceptionresnetv2 by replacing resnext50_32x4d\n5. version 5 : densenet201 with radam by replacing adamw \n6. resnext50_32x4d\n7. upto version 14 of this kernel i had implementation error and i apologize for that,after spending few hours i hope i am now able to build center-resnext50 in version 15,please let me know in the comment box if i still have any implementation error or not?\n8. i got error in version 15 because the model couldn't finish training for 10 epoches within 9 hours limit\n9. in version 16 i will only train for 2 epoches"},{"metadata":{},"cell_type":"markdown","source":"**imports**"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport cv2\nfrom tqdm import tqdm_notebook as tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom functools import reduce\nimport os\nfrom sklearn.model_selection import train_test_split\nfrom scipy.optimize import minimize\nfrom tqdm.auto import tqdm as tq\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import models\nfrom torchvision import transforms, utils\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom math import sqrt, acos, pi, sin, cos\nfrom scipy.spatial.transform import Rotation as R\nfrom sklearn.metrics import average_precision_score\nfrom multiprocessing import Pool\n\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.utils.model_zoo as model_zoo\nimport gc\nimport torch\nimport torch.nn as nn\nfrom torch.hub import load_state_dict_from_url\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom math import sqrt, acos, pi, sin, cos\nfrom scipy.spatial.transform import Rotation as R\nfrom sklearn.metrics import average_precision_score\nfrom multiprocessing import Pool\n\n\nPATH = '../input/pku-autonomous-driving/'\nos.listdir(PATH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"## Constants\nSWITCH_LOSS_EPOCH = 5\nprint(torch.__version__) ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load data"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"train = pd.read_csv(PATH + 'train.csv')\ntest = pd.read_csv(PATH + 'sample_submission.csv')\nbad_list = ['ID_1a5a10365',\n'ID_1db0533c7',\n'ID_53c3fe91a',\n'ID_408f58e9f',\n'ID_4445ae041',\n'ID_bb1d991f6',\n'ID_c44983aeb',\n'ID_f30ebe4d4']\ntrain = train.loc[~train['ImageId'].isin(bad_list)]\n# From camera.zip\ncamera_matrix = np.array([[2304.5479, 0,  1686.2379],\n                          [0, 2305.8757, 1354.9849],\n                          [0, 0, 1]], dtype=np.float32)\ncamera_matrix_inv = np.linalg.inv(camera_matrix)\n\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**ImageId** column contains names of images:"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def imread(path, fast_mode=False):\n    img = cv2.imread(path)\n    if not fast_mode and img is not None and len(img.shape) == 3:\n        img = np.array(img[:, :, ::-1])\n    return img\n\nimg = imread(PATH + 'train_images/ID_8a6e65317' + '.jpg')\nIMG_SHAPE = img.shape\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def str2coords(s, names=['id', 'yaw', 'pitch', 'roll', 'x', 'y', 'z']):\n    '''\n    Input:\n        s: PredictionString (e.g. from train dataframe)\n        names: array of what to extract from the string\n    Output:\n        list of dicts with keys from `names`\n    '''\n    coords = []\n    for l in np.array(s.split()).reshape([-1, 7]):\n        coords.append(dict(zip(names, l.astype('float'))))\n        if 'id' in coords[-1]:\n            coords[-1]['id'] = int(coords[-1]['id'])\n    return coords","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"inp = train['PredictionString'][0]\nprint('Example input:\\n', inp)\nprint()\nprint('Output:\\n', str2coords(inp))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Rotate Function"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def rotate(x, angle):\n    x = x + angle\n    x = x - (x + np.pi) // (2 * np.pi) * 2 * np.pi\n    return x\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 2D Visualization"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def get_img_coords(s):\n    '''\n    Input is a PredictionString (e.g. from train dataframe)\n    Output is two arrays:\n        xs: x coordinates in the image\n        ys: y coordinates in the image\n    '''\n    coords = str2coords(s)\n    xs = [c['x'] for c in coords]\n    ys = [c['y'] for c in coords]\n    zs = [c['z'] for c in coords]\n    P = np.array(list(zip(xs, ys, zs))).T\n    img_p = np.dot(camera_matrix, P).T\n    img_p[:, 0] /= img_p[:, 2]\n    img_p[:, 1] /= img_p[:, 2]\n    img_xs = img_p[:, 0]\n    img_ys = img_p[:, 1]\n    img_zs = img_p[:, 2] # z = Distance from the camera\n    return img_xs, img_ys\n\nplt.figure(figsize=(14,14))\nplt.imshow(imread(PATH + 'train_images/' + train['ImageId'][2217] + '.jpg'))\nplt.scatter(*get_img_coords(train['PredictionString'][2217]), color='red', s=100);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"xs, ys = [], []\n\nfor ps in train['PredictionString']:\n    x, y = get_img_coords(ps)\n    xs += list(x)\n    ys += list(y)\n\nplt.figure(figsize=(18,18))\nplt.imshow(imread(PATH + 'train_images/' + train['ImageId'][2217] + '.jpg'), alpha=0.3)\nplt.scatter(xs, ys, color='red', s=10, alpha=0.2);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3D Visualization\nUsed code from https://www.kaggle.com/zstusnoopy/visualize-the-location-and-3d-bounding-box-of-car, but made it one function"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"from math import sin, cos\n\n# convert euler angle to rotation matrix\ndef euler_to_Rot(yaw, pitch, roll):\n    Y = np.array([[cos(yaw), 0, sin(yaw)],\n                  [0, 1, 0],\n                  [-sin(yaw), 0, cos(yaw)]])\n    P = np.array([[1, 0, 0],\n                  [0, cos(pitch), -sin(pitch)],\n                  [0, sin(pitch), cos(pitch)]])\n    R = np.array([[cos(roll), -sin(roll), 0],\n                  [sin(roll), cos(roll), 0],\n                  [0, 0, 1]])\n    return np.dot(Y, np.dot(P, R))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def draw_line(image, points):\n    color = (255, 0, 0)\n    cv2.line(image, tuple(points[0][:2]), tuple(points[3][:2]), color, 16)\n    cv2.line(image, tuple(points[0][:2]), tuple(points[1][:2]), color, 16)\n    cv2.line(image, tuple(points[1][:2]), tuple(points[2][:2]), color, 16)\n    cv2.line(image, tuple(points[2][:2]), tuple(points[3][:2]), color, 16)\n    return image\n\n\ndef draw_points(image, points):\n    for (p_x, p_y, p_z) in points:\n        cv2.circle(image, (p_x, p_y), int(1000 / p_z), (0, 255, 0), -1)\n#         if p_x > image.shape[1] or p_y > image.shape[0]:\n#             print('Point', p_x, p_y, 'is out of image with shape', image.shape)\n    return image","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"def visualize(img, coords):\n    # You will also need functions from the previous cells\n    x_l = 1.02\n    y_l = 0.80\n    z_l = 2.31\n    \n    img = img.copy()\n    for point in coords:\n        # Get values\n        x, y, z = point['x'], point['y'], point['z']\n        yaw, pitch, roll = -point['pitch'], -point['yaw'], -point['roll']\n        # Math\n        Rt = np.eye(4)\n        t = np.array([x, y, z])\n        Rt[:3, 3] = t\n        Rt[:3, :3] = euler_to_Rot(yaw, pitch, roll).T\n        Rt = Rt[:3, :]\n        P = np.array([[x_l, -y_l, -z_l, 1],\n                      [x_l, -y_l, z_l, 1],\n                      [-x_l, -y_l, z_l, 1],\n                      [-x_l, -y_l, -z_l, 1],\n                      [0, 0, 0, 1]]).T\n        img_cor_points = np.dot(camera_matrix, np.dot(Rt, P))\n        img_cor_points = img_cor_points.T\n        img_cor_points[:, 0] /= img_cor_points[:, 2]\n        img_cor_points[:, 1] /= img_cor_points[:, 2]\n        img_cor_points = img_cor_points.astype(int)\n        # Drawing\n        img = draw_line(img, img_cor_points)\n        img = draw_points(img, img_cor_points[-1:])\n    \n    return img","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Image preprocessing"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"IMG_WIDTH = 1536\nIMG_HEIGHT = 512\nMODEL_SCALE = 8\n\ndef _regr_preprocess(regr_dict):\n    for name in ['x', 'y', 'z']:\n        regr_dict[name] = regr_dict[name] / 100\n    regr_dict['roll'] = rotate(regr_dict['roll'], np.pi)\n    regr_dict['pitch_sin'] = sin(regr_dict['pitch'])\n    regr_dict['pitch_cos'] = cos(regr_dict['pitch'])\n    regr_dict.pop('pitch')\n    regr_dict.pop('id')\n    return regr_dict\n\ndef _regr_back(regr_dict):\n    for name in ['x', 'y', 'z']:\n        regr_dict[name] = regr_dict[name] * 100\n    regr_dict['roll'] = rotate(regr_dict['roll'], -np.pi)\n    \n    pitch_sin = regr_dict['pitch_sin'] / np.sqrt(regr_dict['pitch_sin']**2 + regr_dict['pitch_cos']**2)\n    pitch_cos = regr_dict['pitch_cos'] / np.sqrt(regr_dict['pitch_sin']**2 + regr_dict['pitch_cos']**2)\n    regr_dict['pitch'] = np.arccos(pitch_cos) * np.sign(pitch_sin)\n    return regr_dict\n\ndef preprocess_image(img):\n    img = img[img.shape[0] // 2:]\n    bg = np.ones_like(img) * img.mean(1, keepdims=True).astype(img.dtype)\n    bg = bg[:, :img.shape[1] // 4]\n    img = np.concatenate([bg, img, bg], 1)\n    img = cv2.resize(img, (IMG_WIDTH, IMG_HEIGHT))\n    return (img / 255).astype('float32')\n\ndef get_mask_and_regr(img, labels):\n    mask = np.zeros([IMG_HEIGHT // MODEL_SCALE, IMG_WIDTH // MODEL_SCALE], dtype='float32')\n    regr_names = ['x', 'y', 'z', 'yaw', 'pitch', 'roll']\n    regr = np.zeros([IMG_HEIGHT // MODEL_SCALE, IMG_WIDTH // MODEL_SCALE, 7], dtype='float32')\n    coords = str2coords(labels)\n    xs, ys = get_img_coords(labels)\n    for x, y, regr_dict in zip(xs, ys, coords):\n        x, y = y, x\n        x = (x - img.shape[0] // 2) * IMG_HEIGHT / (img.shape[0] // 2) / MODEL_SCALE\n        x = np.round(x).astype('int')\n        y = (y + img.shape[1] // 4) * IMG_WIDTH / (img.shape[1] * 1.5) / MODEL_SCALE\n        y = np.round(y).astype('int')\n        if x >= 0 and x < IMG_HEIGHT // MODEL_SCALE and y >= 0 and y < IMG_WIDTH // MODEL_SCALE:\n            mask[x, y] = 1\n            regr_dict = _regr_preprocess(regr_dict)\n            regr[x, y] = [regr_dict[n] for n in sorted(regr_dict)]\n    return mask, regr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"img0 = imread(PATH + 'train_images/' + train['ImageId'][0] + '.jpg')\nimg = preprocess_image(img0)\n\nmask, regr = get_mask_and_regr(img0, train['PredictionString'][0])\n\nprint('img.shape', img.shape, 'std:', np.std(img))\nprint('mask.shape', mask.shape, 'std:', np.std(mask))\nprint('regr.shape', regr.shape, 'std:', np.std(regr))\n\nplt.figure(figsize=(16,16))\nplt.title('Processed image')\nplt.imshow(img)\nplt.show()\n\nplt.figure(figsize=(16,16))\nplt.title('Detection Mask')\nplt.imshow(mask)\nplt.show()\n\nplt.figure(figsize=(16,16))\nplt.title('Yaw values')\nplt.imshow(regr[:,:,-2])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# PyTorch Dataset"},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"class CarDataset(Dataset):\n    \"\"\"Car dataset.\"\"\"\n\n    def __init__(self, dataframe, root_dir, training=True, transform=None):\n        self.df = dataframe\n        self.root_dir = root_dir\n        self.transform = transform\n        self.training = training\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        if torch.is_tensor(idx):\n            idx = idx.tolist()\n        \n        # Get image name\n        idx, labels = self.df.values[idx]\n        img_name = self.root_dir.format(idx)\n        \n        # Read image\n        img0 = imread(img_name, True)\n        img = preprocess_image(img0)\n        img = np.rollaxis(img, 2, 0)\n        \n        # Get mask and regression maps\n        if self.training:\n            mask, regr = get_mask_and_regr(img0, labels)\n            regr = np.rollaxis(regr, 2, 0)\n        else:\n            mask, regr = 0, 0\n        \n        return [img, mask, regr]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"train_images_dir = PATH + 'train_images/{}.jpg'\ntest_images_dir = PATH + 'test_images/{}.jpg'\n\ndf_train, df_dev = train_test_split(train, test_size=0.08, random_state=63)\ndf_test = test\n\n# Create dataset objects\ntrain_dataset = CarDataset(df_train, train_images_dir)\ndev_dataset = CarDataset(df_dev, train_images_dir)\ntest_dataset = CarDataset(df_test, test_images_dir)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Show some generated examples"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"img, mask, regr = train_dataset[0]\n\nplt.figure(figsize=(16,16))\nplt.imshow(np.rollaxis(img, 0, 3))\nplt.show()\n\nplt.figure(figsize=(16,16))\nplt.imshow(mask)\nplt.show()\n\nplt.figure(figsize=(16,16))\nplt.imshow(regr[-2])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"BATCH_SIZE = 2\n\n# Create data generators - they will produce batches\ntrain_loader = DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=2)\ndev_loader = DataLoader(dataset=dev_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=2)\ntest_loader = DataLoader(dataset=test_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# PyTorch Model"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"class double_conv(nn.Module):\n    '''(conv => BN => ReLU) * 2'''\n    def __init__(self, in_ch, out_ch):\n        super(double_conv, self).__init__()\n        self.conv = nn.Sequential(\n            nn.Conv2d(in_ch, out_ch, 3, padding=1),\n            nn.BatchNorm2d(out_ch),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(out_ch, out_ch, 3, padding=1),\n            nn.BatchNorm2d(out_ch),\n            nn.ReLU(inplace=True)\n        )\n\n    def forward(self, x):\n        x = self.conv(x)\n        return x\n\nclass up(nn.Module):\n    def __init__(self, in_ch, out_ch, bilinear=True):\n        super(up, self).__init__()\n\n        #  would be a nice idea if the upsampling could be learned too,\n        #  but my machine do not have enough memory to handle all those weights\n        if bilinear:\n            self.up = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True)\n        else:\n            self.up = nn.ConvTranspose2d(in_ch//2, in_ch//2, 2, stride=2)\n\n        self.conv = double_conv(in_ch, out_ch)\n\n    def forward(self, x1, x2=None):\n        x1 = self.up(x1)\n        \n        # input is CHW\n        diffY = x2.size()[2] - x1.size()[2]\n        diffX = x2.size()[3] - x1.size()[3]\n\n        x1 = F.pad(x1, (diffX // 2, diffX - diffX//2,\n                        diffY // 2, diffY - diffY//2))\n        \n        # for padding issues, see \n        # https://github.com/HaiyongJiang/U-Net-Pytorch-Unstructured-Buggy/commit/0e854509c2cea854e247a9c615f175f76fbb2e3a\n        # https://github.com/xiaopeng-liao/Pytorch-UNet/commit/8ebac70e633bac59fc22bb5195e513d5832fb3bd\n        \n        if x2 is not None:\n            x = torch.cat([x2, x1], dim=1)\n        else:\n            x = x1\n        x = self.conv(x)\n        return x\n\ndef get_mesh(batch_size, shape_x, shape_y):\n    mg_x, mg_y = np.meshgrid(np.linspace(0, 1, shape_y), np.linspace(0, 1, shape_x))\n    mg_x = np.tile(mg_x[None, None, :, :], [batch_size, 1, 1, 1]).astype('float32')\n    mg_y = np.tile(mg_y[None, None, :, :], [batch_size, 1, 1, 1]).astype('float32')\n    mesh = torch.cat([torch.tensor(mg_x).to(device), torch.tensor(mg_y).to(device)], 1)\n    return mesh","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**ResneXt**"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# https://github.com/pytorch/vision/blob/master/torchvision/models/resnet.py\n\n\n__all__ = ['ResNet', 'resnet18', 'resnet34', 'resnet50', 'resnet101',\n           'resnet152', 'resnext50_32x4d', 'resnext101_32x8d',\n           'wide_resnet50_2', 'wide_resnet101_2']\n\n\nmodel_urls = {\n    'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth',\n    'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth',\n    'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth',\n    'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth',\n    'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth',\n    'resnext50_32x4d': 'https://download.pytorch.org/models/resnext50_32x4d-7cdf4587.pth',\n    'resnext101_32x8d': 'https://download.pytorch.org/models/resnext101_32x8d-8ba56ff5.pth',\n    'wide_resnet50_2': 'https://download.pytorch.org/models/wide_resnet50_2-95faca4d.pth',\n    'wide_resnet101_2': 'https://download.pytorch.org/models/wide_resnet101_2-32ee1156.pth',\n}\n\n\ndef conv3x3(in_planes, out_planes, stride=1, groups=1, dilation=1):\n    \"\"\"3x3 convolution with padding\"\"\"\n    return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,\n                     padding=dilation, groups=groups, bias=False, dilation=dilation)\n\n\ndef conv1x1(in_planes, out_planes, stride=1):\n    \"\"\"1x1 convolution\"\"\"\n    return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False)\n\n\nclass BasicBlock(nn.Module):\n    expansion = 1\n    __constants__ = ['downsample']\n\n    def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1,\n                 base_width=64, dilation=1, norm_layer=None):\n        super(BasicBlock, self).__init__()\n        if norm_layer is None:\n            norm_layer = nn.BatchNorm2d\n        if groups != 1 or base_width != 64:\n            raise ValueError('BasicBlock only supports groups=1 and base_width=64')\n        if dilation > 1:\n            raise NotImplementedError(\"Dilation > 1 not supported in BasicBlock\")\n        # Both self.conv1 and self.downsample layers downsample the input when stride != 1\n        self.conv1 = conv3x3(inplanes, planes, stride)\n        self.bn1 = norm_layer(planes)\n        self.relu = nn.ReLU(inplace=True)\n        self.conv2 = conv3x3(planes, planes)\n        self.bn2 = norm_layer(planes)\n        self.downsample = downsample\n        self.stride = stride\n\n    def forward(self, x):\n        identity = x\n\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n\n        out = self.conv2(out)\n        out = self.bn2(out)\n\n        if self.downsample is not None:\n            identity = self.downsample(x)\n\n        out += identity\n        out = self.relu(out)\n\n        return out\n\n\nclass Bottleneck(nn.Module):\n    expansion = 4\n    __constants__ = ['downsample']\n\n    def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1,\n                 base_width=64, dilation=1, norm_layer=None):\n        super(Bottleneck, self).__init__()\n        if norm_layer is None:\n            norm_layer = nn.BatchNorm2d\n        width = int(planes * (base_width / 64.)) * groups\n        # Both self.conv2 and self.downsample layers downsample the input when stride != 1\n        self.conv1 = conv1x1(inplanes, width)\n        self.bn1 = norm_layer(width)\n        self.conv2 = conv3x3(width, width, stride, groups, dilation)\n        self.bn2 = norm_layer(width)\n        self.conv3 = conv1x1(width, planes * self.expansion)\n        self.bn3 = norm_layer(planes * self.expansion)\n        self.relu = nn.ReLU(inplace=True)\n        self.downsample = downsample\n        self.stride = stride\n\n    def forward(self, x):\n        identity = x\n\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n\n        out = self.conv2(out)\n        out = self.bn2(out)\n        out = self.relu(out)\n\n        out = self.conv3(out)\n        out = self.bn3(out)\n\n        if self.downsample is not None:\n            identity = self.downsample(x)\n\n        out += identity\n        out = self.relu(out)\n\n        return out\n\n\nclass ResNet(nn.Module):\n\n    def __init__(self, block, layers, num_classes=1000, zero_init_residual=False,\n                 groups=1, width_per_group=64, replace_stride_with_dilation=None,\n                 norm_layer=None):\n        super(ResNet, self).__init__()\n        if norm_layer is None:\n            norm_layer = nn.BatchNorm2d\n        self._norm_layer = norm_layer\n\n        self.inplanes = 64\n        self.dilation = 1\n        if replace_stride_with_dilation is None:\n            # each element in the tuple indicates if we should replace\n            # the 2x2 stride with a dilated convolution instead\n            replace_stride_with_dilation = [False, False, False]\n        if len(replace_stride_with_dilation) != 3:\n            raise ValueError(\"replace_stride_with_dilation should be None \"\n                             \"or a 3-element tuple, got {}\".format(replace_stride_with_dilation))\n        self.groups = groups\n        self.base_width = width_per_group\n        self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=7, stride=2, padding=3,\n                               bias=False)\n        self.bn1 = norm_layer(self.inplanes)\n        self.relu = nn.ReLU(inplace=True)\n        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n        self.layer1 = self._make_layer(block, 64, layers[0])\n        self.layer2 = self._make_layer(block, 128, layers[1], stride=2,\n                                       dilate=replace_stride_with_dilation[0])\n        self.layer3 = self._make_layer(block, 256, layers[2], stride=2,\n                                       dilate=replace_stride_with_dilation[1])\n        self.layer4 = self._make_layer(block, 512, layers[3], stride=2,\n                                       dilate=replace_stride_with_dilation[2])\n        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))\n        self.fc = nn.Linear(512 * block.expansion, num_classes)  #herre\n\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')\n            elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)):\n                nn.init.constant_(m.weight, 1)\n                nn.init.constant_(m.bias, 0)\n\n        # Zero-initialize the last BN in each residual branch,\n        # so that the residual branch starts with zeros, and each residual block behaves like an identity.\n        # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677\n        if zero_init_residual:\n            for m in self.modules():\n                if isinstance(m, Bottleneck):\n                    nn.init.constant_(m.bn3.weight, 0)\n                elif isinstance(m, BasicBlock):\n                    nn.init.constant_(m.bn2.weight, 0)\n\n    def _make_layer(self, block, planes, blocks, stride=1, dilate=False):\n        norm_layer = self._norm_layer\n        downsample = None\n        previous_dilation = self.dilation\n        if dilate:\n            self.dilation *= stride\n            stride = 1\n        if stride != 1 or self.inplanes != planes * block.expansion:\n            downsample = nn.Sequential(\n                conv1x1(self.inplanes, planes * block.expansion, stride),\n                norm_layer(planes * block.expansion),\n            )\n\n        layers = []\n        layers.append(block(self.inplanes, planes, stride, downsample, self.groups,\n                            self.base_width, previous_dilation, norm_layer))\n        self.inplanes = planes * block.expansion\n        for _ in range(1, blocks):\n            layers.append(block(self.inplanes, planes, groups=self.groups,\n                                base_width=self.base_width, dilation=self.dilation,\n                                norm_layer=norm_layer))\n\n        return nn.Sequential(*layers)\n\n    '''def _forward_impl(self, x):\n        # See note [TorchScript super()]\n        x = self.conv1(x)\n        x = self.bn1(x)\n        x = self.relu(x)\n        x = self.maxpool(x)\n\n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = self.layer3(x)\n        x = self.layer4(x)\n\n        x = self.avgpool(x)\n        x = torch.flatten(x, 1)\n        x = self.fc(x)\n\n        return x\n\n    def forward(self, x):\n        return self._forward_impl(x)'''\n\n    def forward(self, x):\n        conv1 = F.relu(self.bn1(self.conv1(x)), inplace=True)\n        conv1 = F.max_pool2d(conv1, 3, stride=2, padding=1)\n\n        feats4 = self.layer1(conv1)\n        feats8 = self.layer2(feats4)\n        feats16 = self.layer3(feats8)\n        feats32 = self.layer4(feats16)\n\n        return feats8, feats16, feats32\n\ndef _resnet(arch, block, layers, pretrained, progress, **kwargs):\n    model = ResNet(block, layers, **kwargs)\n    if pretrained:\n        state_dict = load_state_dict_from_url(model_urls[arch],\n                                              progress=progress)\n        model.load_state_dict(state_dict)\n    return model\n\n\ndef resnet18(pretrained=False, progress=True, **kwargs):\n    r\"\"\"ResNet-18 model from\n    `\"Deep Residual Learning for Image Recognition\" <https://arxiv.org/pdf/1512.03385.pdf>`_\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n        progress (bool): If True, displays a progress bar of the download to stderr\n    \"\"\"\n    return _resnet('resnet18', BasicBlock, [2, 2, 2, 2], pretrained, progress,\n                   **kwargs)\n\n\ndef resnet34(pretrained=False, progress=True, **kwargs):\n    r\"\"\"ResNet-34 model from\n    `\"Deep Residual Learning for Image Recognition\" <https://arxiv.org/pdf/1512.03385.pdf>`_\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n        progress (bool): If True, displays a progress bar of the download to stderr\n    \"\"\"\n    return _resnet('resnet34', BasicBlock, [3, 4, 6, 3], pretrained, progress,\n                   **kwargs)\n\n\ndef resnet50(pretrained=False, progress=True, **kwargs):\n    r\"\"\"ResNet-50 model from\n    `\"Deep Residual Learning for Image Recognition\" <https://arxiv.org/pdf/1512.03385.pdf>`_\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n        progress (bool): If True, displays a progress bar of the download to stderr\n    \"\"\"\n    return _resnet('resnet50', Bottleneck, [3, 4, 6, 3], pretrained, progress,\n                   **kwargs)\n\n\ndef resnet101(pretrained=False, progress=True, **kwargs):\n    r\"\"\"ResNet-101 model from\n    `\"Deep Residual Learning for Image Recognition\" <https://arxiv.org/pdf/1512.03385.pdf>`_\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n        progress (bool): If True, displays a progress bar of the download to stderr\n    \"\"\"\n    return _resnet('resnet101', Bottleneck, [3, 4, 23, 3], pretrained, progress,\n                   **kwargs)\n\n\ndef resnet152(pretrained=False, progress=True, **kwargs):\n    r\"\"\"ResNet-152 model from\n    `\"Deep Residual Learning for Image Recognition\" <https://arxiv.org/pdf/1512.03385.pdf>`_\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n        progress (bool): If True, displays a progress bar of the download to stderr\n    \"\"\"\n    return _resnet('resnet152', Bottleneck, [3, 8, 36, 3], pretrained, progress,\n                   **kwargs)\n\n\ndef resnext50_32x4d(pretrained=False, progress=True, **kwargs):\n    r\"\"\"ResNeXt-50 32x4d model from\n    `\"Aggregated Residual Transformation for Deep Neural Networks\" <https://arxiv.org/pdf/1611.05431.pdf>`_\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n        progress (bool): If True, displays a progress bar of the download to stderr\n    \"\"\"\n    kwargs['groups'] = 32\n    kwargs['width_per_group'] = 4\n    return _resnet('resnext50_32x4d', Bottleneck, [3, 4, 6, 3],\n                   pretrained, progress, **kwargs)\n\n\ndef resnext101_32x8d(pretrained=False, progress=True, **kwargs):\n    r\"\"\"ResNeXt-101 32x8d model from\n    `\"Aggregated Residual Transformation for Deep Neural Networks\" <https://arxiv.org/pdf/1611.05431.pdf>`_\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n        progress (bool): If True, displays a progress bar of the download to stderr\n    \"\"\"\n    kwargs['groups'] = 32\n    kwargs['width_per_group'] = 8\n    return _resnet('resnext101_32x8d', Bottleneck, [3, 4, 23, 3],\n                   pretrained, progress, **kwargs)\n\n\ndef wide_resnet50_2(pretrained=False, progress=True, **kwargs):\n    r\"\"\"Wide ResNet-50-2 model from\n    `\"Wide Residual Networks\" <https://arxiv.org/pdf/1605.07146.pdf>`_\n    The model is the same as ResNet except for the bottleneck number of channels\n    which is twice larger in every block. The number of channels in outer 1x1\n    convolutions is the same, e.g. last block in ResNet-50 has 2048-512-2048\n    channels, and in Wide ResNet-50-2 has 2048-1024-2048.\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n        progress (bool): If True, displays a progress bar of the download to stderr\n    \"\"\"\n    kwargs['width_per_group'] = 64 * 2\n    return _resnet('wide_resnet50_2', Bottleneck, [3, 4, 6, 3],\n                   pretrained, progress, **kwargs)\n\n\ndef wide_resnet101_2(pretrained=False, progress=True, **kwargs):\n    r\"\"\"Wide ResNet-101-2 model from\n    `\"Wide Residual Networks\" <https://arxiv.org/pdf/1605.07146.pdf>`_\n    The model is the same as ResNet except for the bottleneck number of channels\n    which is twice larger in every block. The number of channels in outer 1x1\n    convolutions is the same, e.g. last block in ResNet-50 has 2048-512-2048\n    channels, and in Wide ResNet-50-2 has 2048-1024-2048.\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n        progress (bool): If True, displays a progress bar of the download to stderr\n    \"\"\"\n    kwargs['width_per_group'] = 64 * 2\n    return _resnet('wide_resnet101_2', Bottleneck, [3, 4, 23, 3],\n                   pretrained, progress, **kwargs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nbase_model = resnext50_32x4d(pretrained=True)\nbase_model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"class CentResnet(nn.Module):\n    '''Mixture of previous classes'''\n    def __init__(self, n_classes):\n        super(CentResnet, self).__init__()\n        self.base_model = base_model\n        \n        # Lateral layers convert resnet outputs to a common feature size\n        self.lat8 = nn.Conv2d(512, 256, 1)\n        self.lat16 = nn.Conv2d(1024, 256, 1)\n        self.lat32 = nn.Conv2d(2048, 256, 1)\n        self.bn8 = nn.GroupNorm(16, 256)\n        self.bn16 = nn.GroupNorm(16, 256)\n        self.bn32 = nn.GroupNorm(16, 256)\n\n       \n        self.conv0 = double_conv(5, 64)\n        self.conv1 = double_conv(64, 128)\n        self.conv2 = double_conv(128, 512)\n        self.conv3 = double_conv(512, 1024)\n        \n        self.mp = nn.MaxPool2d(2)\n        \n        self.up1 = up(1282 , 512) #+ 1024\n        self.up2 = up(512 + 512, 256)\n        self.outc = nn.Conv2d(256, n_classes, 1)\n        \n    \n    def forward(self, x):\n        batch_size = x.shape[0]\n        mesh1 = get_mesh(batch_size, x.shape[2], x.shape[3])\n        x0 = torch.cat([x, mesh1], 1)\n        x1 = self.mp(self.conv0(x0))\n        x2 = self.mp(self.conv1(x1))\n        x3 = self.mp(self.conv2(x2))\n        x4 = self.mp(self.conv3(x3))\n        \n        #feats = self.base_model.extract_features(x)\n                # Run frontend network\n        feats8, feats16, feats32 = self.base_model(x)\n        lat8 = F.relu(self.bn8(self.lat8(feats8)))\n        lat16 = F.relu(self.bn16(self.lat16(feats16)))\n        lat32 = F.relu(self.bn32(self.lat32(feats32)))\n        \n        # Add positional info\n        mesh2 = get_mesh(batch_size, lat32.shape[2], lat32.shape[3])\n        feats = torch.cat([lat32, mesh2], 1)\n        #print(feats.shape)\n        #print (x4.shape)\n        x = self.up1(feats, x4)\n        x = self.up2(x, x3)\n        x = self.outc(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Gets the GPU if there is one, otherwise the cpu\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(device)\n\nn_epochs = 3\n\nmodel = CentResnet(8).to(device)\noptimizer = optim.AdamW(model.parameters(), lr=0.001)\n#optimizer =  RAdam(model.parameters(), lr = 0.001)\nexp_lr_scheduler = optim.lr_scheduler.CyclicLR(optimizer,base_lr=0.001,cycle_momentum=False,max_lr=0.0013,step_size_up=2000)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def criterion(prediction, mask, regr,weight=0.4, size_average=True):\n    # Binary mask loss\n    pred_mask = torch.sigmoid(prediction[:, 0])\n#     mask_loss = mask * (1 - pred_mask)**2 * torch.log(pred_mask + 1e-12) + (1 - mask) * pred_mask**2 * torch.log(1 - pred_mask + 1e-12)\n    mask_loss = mask * torch.log(pred_mask + 1e-12) + (1 - mask) * torch.log(1 - pred_mask + 1e-12)\n    mask_loss = -mask_loss.mean(0).sum()\n    \n    # Regression L1 loss\n    pred_regr = prediction[:, 1:]\n    regr_loss = (torch.abs(pred_regr - regr).sum(1) * mask).sum(1).sum(1) / mask.sum(1).sum(1)\n    regr_loss = regr_loss.mean(0)\n  \n    # Sum\n    loss = mask_loss +regr_loss\n    if not size_average:\n        loss *= prediction.shape[0]\n    return loss ,mask_loss , regr_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Just for checking the shapes to manage our Unet\ni = 0\nfor batch_idx, (img_batch, mask_batch, regr_batch) in enumerate(tqdm(train_loader)):\n    print(img_batch.shape)\n    print(mask_batch.shape)\n    print(regr_batch.shape)\n    i+=1\n    if i>1:\n        break ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def train(epoch, history=None):\n    model.train()\n    t = tqdm(train_loader)\n    for batch_idx, (img_batch, mask_batch, regr_batch) in enumerate(t):\n        img_batch = img_batch.to(device)\n        mask_batch = mask_batch.to(device)\n        regr_batch = regr_batch.to(device)\n        \n        optimizer.zero_grad()\n        output = model(img_batch)\n        if epoch < SWITCH_LOSS_EPOCH :\n            loss,mask_loss, regr_loss = criterion(output, mask_batch, regr_batch,1)\n        else:\n            loss,mask_loss, regr_loss = criterion(output, mask_batch, regr_batch,0.5)  \n        \n        t.set_description(f'train_loss (l={loss:.3f})(m={mask_loss:.2f}) (r={regr_loss:.4f}')\n        \n        if history is not None:\n            history.loc[epoch + batch_idx / len(train_loader), 'train_loss'] = loss.data.cpu().numpy()\n        \n        loss.backward()\n        \n        optimizer.step()\n        exp_lr_scheduler.step()\n\n    \n    print('Train Epoch: {} \\tLR: {:.6f}\\tLoss: {:.6f}\\tMaskLoss: {:.6f}\\tRegLoss: {:.6f}'.format(\n        epoch,\n        optimizer.state_dict()['param_groups'][0]['lr'],\n        loss.data,\n        mask_loss.data,\n        regr_loss.data))\n\ndef evaluate(epoch, history=None):\n    model.eval()\n    loss = 0\n    valid_loss = 0\n    valid_mask_loss = 0\n    valid_regr_loss = 0\n    with torch.no_grad():\n        for img_batch, mask_batch, regr_batch in dev_loader:\n            img_batch = img_batch.to(device)\n            mask_batch = mask_batch.to(device)\n            regr_batch = regr_batch.to(device)\n\n            output = model(img_batch)\n\n            if epoch < SWITCH_LOSS_EPOCH :\n                loss,mask_loss, regr_loss= criterion(output, mask_batch, regr_batch,1, size_average=False)\n                valid_loss += loss.data\n                valid_mask_loss += mask_loss.data\n                valid_regr_loss += regr_loss.data\n            else :\n                loss,mask_loss, regr_loss = criterion(output, mask_batch, regr_batch,0.5, size_average=False)\n                valid_loss += loss.data\n                valid_mask_loss += mask_loss.data\n                valid_regr_loss += regr_loss.data \n\n    \n    valid_loss /= len(dev_loader.dataset)\n    valid_mask_loss /= len(dev_loader.dataset)\n    valid_regr_loss /= len(dev_loader.dataset)\n    \n    if history is not None:\n        history.loc[epoch, 'dev_loss'] = valid_loss.cpu().numpy()\n        history.loc[epoch, 'mask_loss'] = valid_mask_loss.cpu().numpy()\n        history.loc[epoch, 'regr_loss'] = valid_regr_loss.cpu().numpy()\n\n    \n    print('Dev loss: {:.4f}'.format(valid_loss))\n    #torch.save(model.state_dict(), './validloss_{valid_loss}epoch_{epoch}.pth')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nimport gc\n\nhistory = pd.DataFrame()\n\nfor epoch in range(n_epochs):\n    torch.cuda.empty_cache()\n    gc.collect()\n    train(epoch, history)\n    evaluate(epoch, history)\n    #torch.save(model.state_dict(), './epoch_{epoch}.pth')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"torch.save(model.state_dict(), './resnext50.pth')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history['train_loss'].iloc[100:].plot();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"series1 = history.dropna()['mask_loss']\nplt.plot(series1.index, series1 ,label = 'mask loss');\nseries2 = history.dropna()['regr_loss']\nplt.plot(series2.index, 30*series2,label = 'regr loss');\nseries3 = history.dropna()['dev_loss']\nplt.plot(series3.index, series3,label = 'dev loss');\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"series = history.dropna()['dev_loss']\nplt.scatter(series.index, series);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visualize predictions"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"img, mask, regr = dev_dataset[0]\n\nplt.figure(figsize=(16,16))\nplt.title('Input image')\nplt.imshow(np.rollaxis(img, 0, 3))\nplt.show()\n\nplt.figure(figsize=(16,16))\nplt.title('Ground truth mask')\nplt.imshow(mask)\nplt.show()\n\noutput = model(torch.tensor(img[None]).to(device))\nlogits = output[0,0].data.cpu().numpy()\n\nplt.figure(figsize=(16,16))\nplt.title('Model predictions')\nplt.imshow(logits)\nplt.show()\n\nprint(logits)\nplt.figure(figsize=(16,16))\nplt.title('Model predictions thresholded')\nplt.imshow(logits > -0.5)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Simple test of probabilities\nact = torch.nn.Sigmoid()\nlogtens = torch.from_numpy(logits)\nprobs = act(logtens)\nprobs = probs[probs>0]\nprint(probs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"DISTANCE_THRESH_CLEAR = 2\n\ndef convert_3d_to_2d(x, y, z, fx = 2304.5479, fy = 2305.8757, cx = 1686.2379, cy = 1354.9849):\n    # stolen from https://www.kaggle.com/theshockwaverider/eda-visualization-baseline\n    return x * fx / z + cx, y * fy / z + cy\n\ndef optimize_xy(r, c, x0, y0, z0):\n    def distance_fn(xyz):\n        x, y, z = xyz\n        x, y = convert_3d_to_2d(x, y, z0)\n        y, x = x, y\n        x = (x - IMG_SHAPE[0] // 2) * IMG_HEIGHT / (IMG_SHAPE[0] // 2) / MODEL_SCALE\n        x = np.round(x).astype('int')\n        y = (y + IMG_SHAPE[1] // 4) * IMG_WIDTH / (IMG_SHAPE[1] * 1.5) / MODEL_SCALE\n        y = np.round(y).astype('int')\n        return (x-r)**2 + (y-c)**2\n    \n    res = minimize(distance_fn, [x0, y0, z0], method='Powell')\n    x_new, y_new, z_new = res.x\n    return x_new, y_new, z0\n\ndef clear_duplicates(coords):\n    for c1 in coords:\n        xyz1 = np.array([c1['x'], c1['y'], c1['z']])\n        for c2 in coords:\n            xyz2 = np.array([c2['x'], c2['y'], c2['z']])\n            distance = np.sqrt(((xyz1 - xyz2)**2).sum())\n            if distance < DISTANCE_THRESH_CLEAR:\n                if c1['confidence'] < c2['confidence']:\n                    c1['confidence'] = -1\n    return [c for c in coords if c['confidence'] > 0]\n\ndef extract_coords(prediction):\n    logits = prediction[0]\n    regr_output = prediction[1:]\n    points = np.argwhere(logits > -0.5)\n    col_names = sorted(['x', 'y', 'z', 'yaw', 'pitch_sin', 'pitch_cos', 'roll'])\n    coords = []\n    for r, c in points:\n        regr_dict = dict(zip(col_names, regr_output[:, r, c]))\n        coords.append(_regr_back(regr_dict))\n        coords[-1]['confidence'] = 1 / (1 + np.exp(-logits[r, c]))\n        coords[-1]['x'], coords[-1]['y'], coords[-1]['z'] = optimize_xy(r, c, coords[-1]['x'], coords[-1]['y'], coords[-1]['z'])\n    coords = clear_duplicates(coords)\n    return coords\n\ndef coords2str(coords, names=['yaw', 'pitch', 'roll', 'x', 'y', 'z', 'confidence']):\n    s = []\n    for c in coords:\n        for n in names:\n            s.append(str(c.get(n, 0)))\n    return ' '.join(s)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"torch.cuda.empty_cache()\ngc.collect()\n\nfor idx in range(4):\n    img, mask, regr = dev_dataset[idx]\n    \n    output = model(torch.tensor(img[None]).to(device)).data.cpu().numpy()\n    coords_pred = extract_coords(output[0])\n    coords_true = extract_coords(np.concatenate([mask[None], regr], 0))\n    \n    img = imread(train_images_dir.format(df_dev['ImageId'].iloc[idx]))\n    \n    fig, axes = plt.subplots(1, 2, figsize=(30,30))\n    axes[0].set_title('Ground truth')\n    axes[0].imshow(visualize(img, coords_true))\n    axes[1].set_title('Prediction')\n    axes[1].imshow(visualize(img, coords_pred))\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Make submission "},{"metadata":{"trusted":true},"cell_type":"code","source":"# taken from kernel of @its7171 \nval_preds = []\n\ndev_loader = DataLoader(dataset=dev_dataset, batch_size=2, shuffle=False, num_workers=2)\n\nmodel.eval()\n\nfor img, _, _ in tqdm(dev_loader):\n    with torch.no_grad():\n        output = model(img.to(device))\n    output = output.data.cpu().numpy()\n    for out in output:\n        coords = extract_coords(out)\n        s = coords2str(coords)\n        val_preds.append(s)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_dev['PredictionString'] = val_preds\ndf_dev.head()\ndf_dev.to_csv('val_predictions.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = []\n\ntest_loader = DataLoader(dataset=test_dataset, batch_size=2, shuffle=False, num_workers=2)\n\nmodel.eval()\n\nfor img, _, _ in tqdm(test_loader):\n    with torch.no_grad():\n        output = model(img.to(device))\n    output = output.data.cpu().numpy()\n    for out in output:\n        coords = extract_coords(out)\n        s = coords2str(coords)\n        predictions.append(s)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv(PATH + 'sample_submission.csv')\ntest['PredictionString'] = predictions\ntest.to_csv('predictions.csv', index=False)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef expand_df(df, PredictionStringCols):\n    df = df.dropna().copy()\n    df['NumCars'] = [int((x.count(' ')+1)/7) for x in df['PredictionString']]\n\n    image_id_expanded = [item for item, count in zip(df['ImageId'], df['NumCars']) for i in range(count)]\n    prediction_strings_expanded = df['PredictionString'].str.split(' ',expand = True).values.reshape(-1,7).astype(float)\n    prediction_strings_expanded = prediction_strings_expanded[~np.isnan(prediction_strings_expanded).all(axis=1)]\n    df = pd.DataFrame(\n        {\n            'ImageId': image_id_expanded,\n            PredictionStringCols[0]:prediction_strings_expanded[:,0],\n            PredictionStringCols[1]:prediction_strings_expanded[:,1],\n            PredictionStringCols[2]:prediction_strings_expanded[:,2],\n            PredictionStringCols[3]:prediction_strings_expanded[:,3],\n            PredictionStringCols[4]:prediction_strings_expanded[:,4],\n            PredictionStringCols[5]:prediction_strings_expanded[:,5],\n            PredictionStringCols[6]:prediction_strings_expanded[:,6]\n        })\n    return df\n\ndef str2coords(s, names):\n    coords = []\n    for l in np.array(s.split()).reshape([-1, 7]):\n        coords.append(dict(zip(names, l.astype('float'))))\n    return coords\n\ndef TranslationDistance(p,g, abs_dist = False):\n    dx = p['x'] - g['x']\n    dy = p['y'] - g['y']\n    dz = p['z'] - g['z']\n    diff0 = (g['x']**2 + g['y']**2 + g['z']**2)**0.5\n    diff1 = (dx**2 + dy**2 + dz**2)**0.5\n    if abs_dist:\n        diff = diff1\n    else:\n        diff = diff1/diff0\n    return diff\n\ndef RotationDistance(p, g):\n    true=[ g['pitch'] ,g['yaw'] ,g['roll'] ]\n    pred=[ p['pitch'] ,p['yaw'] ,p['roll'] ]\n    q1 = R.from_euler('xyz', true)\n    q2 = R.from_euler('xyz', pred)\n    diff = R.inv(q2) * q1\n    W = np.clip(diff.as_quat()[-1], -1., 1.)\n    \n    # in the official metrics code:\n    # https://www.kaggle.com/c/pku-autonomous-driving/overview/evaluation\n    #   return Object3D.RadianToDegree( Math.Acos(diff.W) )\n    # this code treat θ and θ+2π differntly.\n    # So this should be fixed as follows.\n    W = (acos(W)*360)/pi\n    if W > 180:\n        W = 360 - W\n    return W","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"thres_tr_list = [0.1, 0.09, 0.08, 0.07, 0.06, 0.05, 0.04, 0.03, 0.02, 0.01]\nthres_ro_list = [50, 45, 40, 35, 30, 25, 20, 15, 10, 5]\n\ndef check_match(idx):\n    keep_gt=False\n    thre_tr_dist = thres_tr_list[idx]\n    thre_ro_dist = thres_ro_list[idx]\n    train_dict = {imgID:str2coords(s, names=['carid_or_score', 'pitch', 'yaw', 'roll', 'x', 'y', 'z']) for imgID,s in zip(train_df['ImageId'],train_df['PredictionString'])}\n    valid_dict = {imgID:str2coords(s, names=['pitch', 'yaw', 'roll', 'x', 'y', 'z', 'carid_or_score']) for imgID,s in zip(valid_df['ImageId'],valid_df['PredictionString'])}\n    result_flg = [] # 1 for TP, 0 for FP\n    scores = []\n    MAX_VAL = 10**10\n    for img_id in valid_dict:\n        for pcar in sorted(valid_dict[img_id], key=lambda x: -x['carid_or_score']):\n            # find nearest GT\n            min_tr_dist = MAX_VAL\n            min_idx = -1\n            for idx, gcar in enumerate(train_dict[img_id]):\n                tr_dist = TranslationDistance(pcar,gcar)\n                if tr_dist < min_tr_dist:\n                    min_tr_dist = tr_dist\n                    min_ro_dist = RotationDistance(pcar,gcar)\n                    min_idx = idx\n                    \n            # set the result\n            if min_tr_dist < thre_tr_dist and min_ro_dist < thre_ro_dist:\n                if not keep_gt:\n                    train_dict[img_id].pop(min_idx)\n                result_flg.append(1)\n            else:\n                result_flg.append(0)\n            scores.append(pcar['carid_or_score'])\n    \n    return result_flg, scores","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#validation_prediction = df_dev\nvalid_df = pd.read_csv('val_predictions.csv')\nexpanded_valid_df = expand_df(valid_df, ['pitch','yaw','roll','x','y','z','Score'])\nvalid_df = valid_df.fillna('')\n\ntrain_df = pd.read_csv('../input/pku-autonomous-driving/train.csv')\ntrain_df = train_df[train_df.ImageId.isin(valid_df.ImageId.unique())]\n# data description page says, The pose information is formatted as\n# model type, yaw, pitch, roll, x, y, z\n# but it doesn't, and it should be\n# model type, pitch, yaw, roll, x, y, z\nexpanded_train_df = expand_df(train_df, ['model_type','pitch','yaw','roll','x','y','z'])\n\nmax_workers = 10\nn_gt = len(expanded_train_df)\nap_list = []\np = Pool(processes=max_workers)\nfor result_flg, scores in p.imap(check_match, range(10)):\n    if np.sum(result_flg) > 0:\n        n_tp = np.sum(result_flg)\n        recall = n_tp/n_gt\n        ap = average_precision_score(result_flg, scores)*recall\n    else:\n        ap = 0\n    ap_list.append(ap)\nmap = np.mean(ap_list)\nprint('map:', map)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}