{"cells":[{"metadata":{"_uuid":"8707337f-04da-4e3c-94a6-80c29233fe09","_cell_guid":"d7401712-27ab-4d8d-99ee-1d5dd08e508a","trusted":true},"cell_type":"markdown","source":"# Changelog\n# All credit to wei hao khoong.\n# ------------------------------------\n# <h4>Version 1</h4>\n# * Used Inception-ResNet-V2 in place of resnext-50.\n# * 6 epochs"},{"metadata":{"_uuid":"5b7afe23-5417-41f9-ad8a-6d3b96515e88","_cell_guid":"37cb6d6f-0dd1-4f5a-a8c8-6961cf2dac95","trusted":true},"cell_type":"markdown","source":"# Experimental Results\n# \n# - 10 epochs: 0.044 public LB score\n# - 8 epochs: 0.047 public LB score"},{"metadata":{"_uuid":"9dac610f-7c43-4bd3-a84e-6225139f7c8d","_cell_guid":"8e0b2284-eaba-4e20-bec6-dde84a35d662","trusted":true},"cell_type":"markdown","source":"# Important Remarks\n# \n# - The public leaderboard is calculated with approximately 9% of the test data\n# - The final results will be based on the other 91%, so the final standings may be very different!\n# - Submit the highest LB from this kernel from final scoring **at your own risk**!!"},{"metadata":{"_uuid":"b8780de2-47ce-4064-af51-dd5c890b59a3","_cell_guid":"ee88f8d3-85c2-44e7-964d-9c1064cb61b4","trusted":true},"cell_type":"markdown","source":"**Progressions** (Original author's comments)"},{"metadata":{"_uuid":"4119f69e-7b4a-400a-8e1f-88dd2db3d577","_cell_guid":"813f78b5-e358-4e55-b81a-7c49efc43dcd","trusted":true},"cell_type":"markdown","source":"1. version 1 : 1 epoch took 1+ hours so in version 2 i will try for 8 epoches\n# 2. for 8 epoches in version 2 i got 0.095 public lb score"},{"metadata":{"_uuid":"d2a12706-6bf7-4033-a947-1d18c6bd5222","_cell_guid":"ae6ec89d-5248-4e26-9eb8-c3a3d8c75b1c","trusted":true},"cell_type":"markdown","source":"# Acknowledgements:\n# \n# All credit belongs to Wei Hao Khoong. Please upvote his kernel [here](https://www.kaggle.com/khoongweihao/autonomous-driving-epoch-on-fire-resnext50)"},{"metadata":{"_uuid":"2c912efb-eabe-4963-9fe3-b34195662de1","_cell_guid":"582f1725-be09-4d59-ada6-e682146afc05","trusted":true},"cell_type":"markdown","source":"**imports**"},{"metadata":{"_uuid":"72975b00-f793-4e06-be3a-95e4a1f9f3d5","_cell_guid":"737a88e2-bae8-4682-afe6-f05b32c6e049","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport cv2\nfrom tqdm import tqdm_notebook as tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom functools import reduce\nimport os\nfrom sklearn.model_selection import train_test_split\nfrom scipy.optimize import minimize\nfrom tqdm.auto import tqdm as tq\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import models\nfrom torchvision import transforms, utils\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom math import sqrt, acos, pi, sin, cos\nfrom scipy.spatial.transform import Rotation as R\nfrom sklearn.metrics import average_precision_score\nfrom multiprocessing import Pool\n\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.utils.model_zoo as model_zoo\nimport gc\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom math import sqrt, acos, pi, sin, cos\nfrom scipy.spatial.transform import Rotation as R\nfrom sklearn.metrics import average_precision_score\nfrom multiprocessing import Pool\n\n\nPATH = '../input/pku-autonomous-driving/'\nos.listdir(PATH)","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"535b420f-88ec-4635-9c9e-30cd34ac02f6","_cell_guid":"e7e42f8d-5859-427f-8ce4-26cfa604575f","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"## Constants\nSWITCH_LOSS_EPOCH = 5\nprint(torch.__version__)","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"40070f7a-7e74-4289-8ec5-cb523e39a0c1","_cell_guid":"7a1db691-a309-4ab6-8855-377a300c37f8","trusted":true},"cell_type":"code","source":"from  keras.applications.inception_resnet_v2 import InceptionResNetV2","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"2f6d1e30-a566-4795-98ff-cb1212b0c7be","_cell_guid":"f32061ff-1ff1-4509-bfd5-51a3e62a47fb","trusted":true},"cell_type":"code","source":"","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"a5355275-876a-497f-95e1-ca3a92f9de3d","_cell_guid":"0ec1a2b2-2c76-470d-8ca4-b5fc4d4d36ac","trusted":true},"cell_type":"markdown","source":"# Load data"},{"metadata":{"_uuid":"4c67e8ba-72f1-44a4-a314-03215618c2f8","_cell_guid":"65543d45-eca5-40c6-95bd-eb953670c6d9","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"train = pd.read_csv(PATH + 'train.csv')\ntest = pd.read_csv(PATH + 'sample_submission.csv')\nbad_list = ['ID_1a5a10365',\n'ID_1db0533c7',\n'ID_53c3fe91a',\n'ID_408f58e9f',\n'ID_4445ae041',\n'ID_bb1d991f6',\n'ID_c44983aeb',\n'ID_f30ebe4d4']\ntrain = train.loc[~train['ImageId'].isin(bad_list)]\n# From camera.zip\ncamera_matrix = np.array([[2304.5479, 0,  1686.2379],\n                          [0, 2305.8757, 1354.9849],\n                          [0, 0, 1]], dtype=np.float32)\ncamera_matrix_inv = np.linalg.inv(camera_matrix)\n\ntrain.head()","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"3a0013de-28ea-4cf8-8429-8055a8ceb763","_cell_guid":"cf927578-658a-431d-aa39-0c852a538e79","trusted":true},"cell_type":"markdown","source":"**ImageId** column contains names of images:"},{"metadata":{"_uuid":"f8973407-becc-4265-85d5-1138bc172749","_cell_guid":"51ce73c9-bc94-4dcc-91c9-d0bab85122d3","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def imread(path, fast_mode=False):\n    img = cv2.imread(path)\n    if not fast_mode and img is not None and len(img.shape) == 3:\n        img = np.array(img[:, :, ::-1])\n    return img\n\nimg = imread(PATH + 'train_images/ID_8a6e65317' + '.jpg')\nIMG_SHAPE = img.shape\n\nplt.figure(figsize=(15,8))\nplt.imshow(img);","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"090c6ec4-bbab-4019-9a46-090b2d511171","_cell_guid":"813d470d-7748-40f0-9bd9-b59dfde8ffd5","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def str2coords(s, names=['id', 'yaw', 'pitch', 'roll', 'x', 'y', 'z']):\n    '''\n    Input:\n        s: PredictionString (e.g. from train dataframe)\n        names: array of what to extract from the string\n    Output:\n        list of dicts with keys from `names`\n    '''\n    coords = []\n    for l in np.array(s.split()).reshape([-1, 7]):\n        coords.append(dict(zip(names, l.astype('float'))))\n        if 'id' in coords[-1]:\n            coords[-1]['id'] = int(coords[-1]['id'])\n    return coords","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"57ed63bd-ab30-4821-87c4-638fd5ced2a5","_cell_guid":"0a25a3e2-550f-40ad-ac7f-a699f5aff42b","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"inp = train['PredictionString'][0]\nprint('Example input:\\n', inp)\nprint()\nprint('Output:\\n', str2coords(inp))","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"038b9dea-1ee5-4798-a02d-f47cc631c5e0","_cell_guid":"a0fa6fa2-af9e-4933-b317-2a6f27f91f97","trusted":true},"cell_type":"markdown","source":"# Data distributions"},{"metadata":{"_uuid":"d687e19b-ef9b-4ce0-a189-fca96b428680","_cell_guid":"9ffc5134-7946-4b7e-b30f-85fa4bb5065c","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def rotate(x, angle):\n    x = x + angle\n    x = x - (x + np.pi) // (2 * np.pi) * 2 * np.pi\n    return x\n\nplt.figure(figsize=(15,6))\nsns.distplot(reduce(lambda a, b: a + b, [[rotate(c['roll'], np.pi) for c in str2coords(s)] for s in train['PredictionString']]));\nplt.xlabel('roll rotated by pi')\nplt.show()","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"da6114dc-05e5-4892-9d6a-b271d539d0bf","_cell_guid":"d3e4a356-5252-460c-a07a-c8cb9ba3b7f6","trusted":true},"cell_type":"markdown","source":"# 2D Visualization"},{"metadata":{"_uuid":"b54c0a4e-0dde-4c83-8f30-6637cbde1abd","_cell_guid":"e83cc119-6a4a-4f37-98bd-054639d94f61","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def get_img_coords(s):\n    '''\n    Input is a PredictionString (e.g. from train dataframe)\n    Output is two arrays:\n        xs: x coordinates in the image\n        ys: y coordinates in the image\n    '''\n    coords = str2coords(s)\n    xs = [c['x'] for c in coords]\n    ys = [c['y'] for c in coords]\n    zs = [c['z'] for c in coords]\n    P = np.array(list(zip(xs, ys, zs))).T\n    img_p = np.dot(camera_matrix, P).T\n    img_p[:, 0] /= img_p[:, 2]\n    img_p[:, 1] /= img_p[:, 2]\n    img_xs = img_p[:, 0]\n    img_ys = img_p[:, 1]\n    img_zs = img_p[:, 2] # z = Distance from the camera\n    return img_xs, img_ys\n\nplt.figure(figsize=(14,14))\nplt.imshow(imread(PATH + 'train_images/' + train['ImageId'][2217] + '.jpg'))\nplt.scatter(*get_img_coords(train['PredictionString'][2217]), color='red', s=100);","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"d52a8891-3ef4-46c5-996f-821e9cdd0dfe","_cell_guid":"3f3221f9-e56e-4c74-afe4-28aa0026c4ff","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"xs, ys = [], []\n\nfor ps in train['PredictionString']:\n    x, y = get_img_coords(ps)\n    xs += list(x)\n    ys += list(y)\n\nplt.figure(figsize=(18,18))\nplt.imshow(imread(PATH + 'train_images/' + train['ImageId'][2217] + '.jpg'), alpha=0.3)\nplt.scatter(xs, ys, color='red', s=10, alpha=0.2);","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"e97b7058-6435-4002-9fed-af7817ffd2f6","_cell_guid":"5fb5ee17-e99b-4edb-a38d-511770167173","trusted":true},"cell_type":"markdown","source":"# 3D Visualization\n# Used code from https://www.kaggle.com/zstusnoopy/visualize-the-location-and-3d-bounding-box-of-car, but made it one function"},{"metadata":{"_uuid":"ab67aadc-5571-46e6-a9c2-dd3401d89a07","_cell_guid":"5ef3c4d4-fb7b-4584-b0a2-a7d9bc374fd2","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"from math import sin, cos\n\n# convert euler angle to rotation matrix\ndef euler_to_Rot(yaw, pitch, roll):\n    Y = np.array([[cos(yaw), 0, sin(yaw)],\n                  [0, 1, 0],\n                  [-sin(yaw), 0, cos(yaw)]])\n    P = np.array([[1, 0, 0],\n                  [0, cos(pitch), -sin(pitch)],\n                  [0, sin(pitch), cos(pitch)]])\n    R = np.array([[cos(roll), -sin(roll), 0],\n                  [sin(roll), cos(roll), 0],\n                  [0, 0, 1]])\n    return np.dot(Y, np.dot(P, R))","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"7a22c98e-0dd2-47a0-aa95-39a69553a7df","_cell_guid":"a6933e20-fe9b-4473-aae4-4352c4f68aee","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def draw_line(image, points):\n    color = (255, 0, 0)\n    cv2.line(image, tuple(points[0][:2]), tuple(points[3][:2]), color, 16)\n    cv2.line(image, tuple(points[0][:2]), tuple(points[1][:2]), color, 16)\n    cv2.line(image, tuple(points[1][:2]), tuple(points[2][:2]), color, 16)\n    cv2.line(image, tuple(points[2][:2]), tuple(points[3][:2]), color, 16)\n    return image\n\n\ndef draw_points(image, points):\n    for (p_x, p_y, p_z) in points:\n        cv2.circle(image, (p_x, p_y), int(1000 / p_z), (0, 255, 0), -1)\n#         if p_x > image.shape[1] or p_y > image.shape[0]:\n#             print('Point', p_x, p_y, 'is out of image with shape', image.shape)\n    return image","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"2d71ecb6-38d9-4a0e-94b7-19f0e73d0f25","_cell_guid":"d88a0cf6-e9c3-48d9-afc6-8b3285213b93","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def visualize(img, coords):\n    # You will also need functions from the previous cells\n    x_l = 1.02\n    y_l = 0.80\n    z_l = 2.31\n    \n    img = img.copy()\n    for point in coords:\n        # Get values\n        x, y, z = point['x'], point['y'], point['z']\n        yaw, pitch, roll = -point['pitch'], -point['yaw'], -point['roll']\n        # Math\n        Rt = np.eye(4)\n        t = np.array([x, y, z])\n        Rt[:3, 3] = t\n        Rt[:3, :3] = euler_to_Rot(yaw, pitch, roll).T\n        Rt = Rt[:3, :]\n        P = np.array([[x_l, -y_l, -z_l, 1],\n                      [x_l, -y_l, z_l, 1],\n                      [-x_l, -y_l, z_l, 1],\n                      [-x_l, -y_l, -z_l, 1],\n                      [0, 0, 0, 1]]).T\n        img_cor_points = np.dot(camera_matrix, np.dot(Rt, P))\n        img_cor_points = img_cor_points.T\n        img_cor_points[:, 0] /= img_cor_points[:, 2]\n        img_cor_points[:, 1] /= img_cor_points[:, 2]\n        img_cor_points = img_cor_points.astype(int)\n        # Drawing\n        img = draw_line(img, img_cor_points)\n        img = draw_points(img, img_cor_points[-1:])\n    \n    return img","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"074d6563-b3ed-4737-8a72-95c44d729d8e","_cell_guid":"050eaf7b-27f6-4f82-8994-28962e4e36e8","trusted":true},"cell_type":"markdown","source":"# Image preprocessing"},{"metadata":{"_uuid":"042c2deb-9165-4b39-a147-637e336441d2","_cell_guid":"90d4eac5-b1c3-4bff-9264-7af237232b0a","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"IMG_WIDTH = 1600\nIMG_HEIGHT = 700\nMODEL_SCALE = 8\n\ndef _regr_preprocess(regr_dict):\n    for name in ['x', 'y', 'z']:\n        regr_dict[name] = regr_dict[name] / 100\n    regr_dict['roll'] = rotate(regr_dict['roll'], np.pi)\n    regr_dict['pitch_sin'] = sin(regr_dict['pitch'])\n    regr_dict['pitch_cos'] = cos(regr_dict['pitch'])\n    regr_dict.pop('pitch')\n    regr_dict.pop('id')\n    return regr_dict\n\ndef _regr_back(regr_dict):\n    for name in ['x', 'y', 'z']:\n        regr_dict[name] = regr_dict[name] * 100\n    regr_dict['roll'] = rotate(regr_dict['roll'], -np.pi)\n    \n    pitch_sin = regr_dict['pitch_sin'] / np.sqrt(regr_dict['pitch_sin']**2 + regr_dict['pitch_cos']**2)\n    pitch_cos = regr_dict['pitch_cos'] / np.sqrt(regr_dict['pitch_sin']**2 + regr_dict['pitch_cos']**2)\n    regr_dict['pitch'] = np.arccos(pitch_cos) * np.sign(pitch_sin)\n    return regr_dict\n\ndef preprocess_image(img):\n    img = img[img.shape[0] // 2:]\n    bg = np.ones_like(img) * img.mean(1, keepdims=True).astype(img.dtype)\n    bg = bg[:, :img.shape[1] // 4]\n    img = np.concatenate([bg, img, bg], 1)\n    img = cv2.resize(img, (IMG_WIDTH, IMG_HEIGHT))\n    return (img / 255).astype('float32')\n\ndef get_mask_and_regr(img, labels):\n    mask = np.zeros([IMG_HEIGHT // MODEL_SCALE, IMG_WIDTH // MODEL_SCALE], dtype='float32')\n    regr_names = ['x', 'y', 'z', 'yaw', 'pitch', 'roll']\n    regr = np.zeros([IMG_HEIGHT // MODEL_SCALE, IMG_WIDTH // MODEL_SCALE, 7], dtype='float32')\n    coords = str2coords(labels)\n    xs, ys = get_img_coords(labels)\n    for x, y, regr_dict in zip(xs, ys, coords):\n        x, y = y, x\n        x = (x - img.shape[0] // 2) * IMG_HEIGHT / (img.shape[0] // 2) / MODEL_SCALE\n        x = np.round(x).astype('int')\n        y = (y + img.shape[1] // 4) * IMG_WIDTH / (img.shape[1] * 1.5) / MODEL_SCALE\n        y = np.round(y).astype('int')\n        if x >= 0 and x < IMG_HEIGHT // MODEL_SCALE and y >= 0 and y < IMG_WIDTH // MODEL_SCALE:\n            mask[x, y] = 1\n            regr_dict = _regr_preprocess(regr_dict)\n            regr[x, y] = [regr_dict[n] for n in sorted(regr_dict)]\n    return mask, regr","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"179dbbfa-1246-43f3-9f1e-3209cd67eefd","_cell_guid":"0dc53ee2-38d8-420d-8cc1-5d1f7f1367d3","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"img0 = imread(PATH + 'train_images/' + train['ImageId'][0] + '.jpg')\nimg = preprocess_image(img0)\n\nmask, regr = get_mask_and_regr(img0, train['PredictionString'][0])\n\nprint('img.shape', img.shape, 'std:', np.std(img))\nprint('mask.shape', mask.shape, 'std:', np.std(mask))\nprint('regr.shape', regr.shape, 'std:', np.std(regr))\n\nplt.figure(figsize=(16,16))\nplt.title('Processed image')\nplt.imshow(img)\nplt.show()\n\nplt.figure(figsize=(16,16))\nplt.title('Detection Mask')\nplt.imshow(mask)\nplt.show()\n\nplt.figure(figsize=(16,16))\nplt.title('Yaw values')\nplt.imshow(regr[:,:,-2])\nplt.show()","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"dcee785f-2373-4ca3-a9cd-81cdb1702295","_cell_guid":"92395f93-4a8d-4aaf-9dea-bd1d8214bd5b","trusted":true},"cell_type":"markdown","source":"# PyTorch Dataset"},{"metadata":{"_uuid":"c02361ef-1869-4025-85e2-33944944101d","_cell_guid":"884936ff-7ad4-48d2-a644-43cf05bd6988","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"class CarDataset(Dataset):\n    \"\"\"Car dataset.\"\"\"\n\n    def __init__(self, dataframe, root_dir, training=True, transform=None):\n        self.df = dataframe\n        self.root_dir = root_dir\n        self.transform = transform\n        self.training = training\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        if torch.is_tensor(idx):\n            idx = idx.tolist()\n        \n        # Get image name\n        idx, labels = self.df.values[idx]\n        img_name = self.root_dir.format(idx)\n        \n        # Read image\n        img0 = imread(img_name, True)\n        img = preprocess_image(img0)\n        img = np.rollaxis(img, 2, 0)\n        \n        # Get mask and regression maps\n        if self.training:\n            mask, regr = get_mask_and_regr(img0, labels)\n            regr = np.rollaxis(regr, 2, 0)\n        else:\n            mask, regr = 0, 0\n        \n        return [img, mask, regr]","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"1c4d4200-cb3c-4d47-8f29-8d8ac445b381","_cell_guid":"4b9c89b2-f40f-4978-9232-27a1b9b76098","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"train_images_dir = PATH + 'train_images/{}.jpg'\ntest_images_dir = PATH + 'test_images/{}.jpg'\n\ndf_train, df_dev = train_test_split(train, test_size=0.1, random_state=63)\ndf_test = test\n\n# Create dataset objects\ntrain_dataset = CarDataset(df_train, train_images_dir)\ndev_dataset = CarDataset(df_dev, train_images_dir)\ntest_dataset = CarDataset(df_test, test_images_dir)","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"68c53f54-4ac5-44ba-8a61-f146eb30f042","_cell_guid":"84b0805b-852a-4ee4-a4dc-5b61765e00c1","trusted":true},"cell_type":"markdown","source":"Show some generated examples"},{"metadata":{"_uuid":"8f627519-867e-4acf-aae0-36a861ddd660","_cell_guid":"b4eefb25-6165-45da-b4ab-5e015a8d992a","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"img, mask, regr = train_dataset[0]\n\nplt.figure(figsize=(16,16))\nplt.imshow(np.rollaxis(img, 0, 3))\nplt.show()\n\nplt.figure(figsize=(16,16))\nplt.imshow(mask)\nplt.show()\n\nplt.figure(figsize=(16,16))\nplt.imshow(regr[-2])\nplt.show()","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"b991fcf9-ad7e-4543-9a9b-c344359baa88","_cell_guid":"8ad645dd-6401-4b8d-9207-413687d2dc38","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"BATCH_SIZE = 1\n\n# Create data generators - they will produce batches\ntrain_loader = DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=0)\ndev_loader = DataLoader(dataset=dev_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=0)\ntest_loader = DataLoader(dataset=test_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4)","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"60c52f44-bc2f-4bf2-8629-d3330fcb0cc2","_cell_guid":"c5614e71-ac74-43d0-8fbe-3efa8d8d0019","trusted":true},"cell_type":"markdown","source":"# PyTorch Model"},{"metadata":{"_uuid":"7fa1a068-c074-4f59-992f-4f691d6b8255","_cell_guid":"4cb1e09d-5e6c-4ea3-91ff-1af769ea115a","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"class double_conv(nn.Module):\n    '''(conv => BN => ReLU) * 2'''\n    def __init__(self, in_ch, out_ch):\n        super(double_conv, self).__init__()\n        self.conv = nn.Sequential(\n            nn.Conv2d(in_ch, out_ch, 3, padding=1),\n            nn.BatchNorm2d(out_ch),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(out_ch, out_ch, 3, padding=1),\n            nn.BatchNorm2d(out_ch),\n            nn.ReLU(inplace=True)\n        )\n\n    def forward(self, x):\n        x = self.conv(x)\n        return x\n\nclass up(nn.Module):\n    def __init__(self, in_ch, out_ch, bilinear=True):\n        super(up, self).__init__()\n\n        #  would be a nice idea if the upsampling could be learned too,\n        #  but my machine do not have enough memory to handle all those weights\n        if bilinear:\n            self.up = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True)\n        else:\n            self.up = nn.ConvTranspose2d(in_ch//2, in_ch//2, 2, stride=2)\n\n        self.conv = double_conv(in_ch, out_ch)\n\n    def forward(self, x1, x2=None):\n        x1 = self.up(x1)\n        \n        # input is CHW\n        diffY = x2.size()[2] - x1.size()[2]\n        diffX = x2.size()[3] - x1.size()[3]\n\n        x1 = F.pad(x1, (diffX // 2, diffX - diffX//2,\n                        diffY // 2, diffY - diffY//2))\n        \n        # for padding issues, see \n        # https://github.com/HaiyongJiang/U-Net-Pytorch-Unstructured-Buggy/commit/0e854509c2cea854e247a9c615f175f76fbb2e3a\n        # https://github.com/xiaopeng-liao/Pytorch-UNet/commit/8ebac70e633bac59fc22bb5195e513d5832fb3bd\n        \n        if x2 is not None:\n            x = torch.cat([x2, x1], dim=1)\n        else:\n            x = x1\n        x = self.conv(x)\n        return x\n\ndef get_mesh(batch_size, shape_x, shape_y):\n    mg_x, mg_y = np.meshgrid(np.linspace(0, 1, shape_y), np.linspace(0, 1, shape_x))\n    mg_x = np.tile(mg_x[None, None, :, :], [batch_size, 1, 1, 1]).astype('float32')\n    mg_y = np.tile(mg_y[None, None, :, :], [batch_size, 1, 1, 1]).astype('float32')\n    mesh = torch.cat([torch.tensor(mg_x).to(device), torch.tensor(mg_y).to(device)], 1)\n    return mesh","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"0fd69cd2-e867-4c89-976f-87f8a5d2433e","_cell_guid":"f06b8d07-943d-4080-b8f8-cdecd824abe4","trusted":true},"cell_type":"markdown","source":"**ResneXt**"},{"metadata":{"_uuid":"d6eb8303-56e4-4f93-911f-2df427627566","_cell_guid":"2335651d-aa8e-4c63-8d47-b7731ddb17f0","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"\n\nmodel_urls = {\n    'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth',\n    'resnext50_32x4d': 'https://download.pytorch.org/models/resnext50_32x4d-7cdf4587.pth',\n    'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth',\n    'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth',\n    'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth',\n    'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth',\n    'densenet201': 'https://download.pytorch.org/models/densenet201-c1103571.pth',\n    'inceptionresnetv2': 'http://data.lip6.fr/cadene/pretrainedmodels/inceptionresnetv2-520b38e4.pth',\n}\n\n\ndef conv3x3(in_planes, out_planes, stride=1):\n    \"\"\"3x3 convolution with padding\"\"\"\n    return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,\n                     padding=1, bias=False)\n\n\ndef conv1x1(in_planes, out_planes, stride=1):\n    \"\"\"1x1 convolution\"\"\"\n    return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False)\n\n\nclass BasicBlock(nn.Module):\n    expansion = 1\n\n    def __init__(self, inplanes, planes, stride=1):\n        super(BasicBlock, self).__init__()\n\n        self.conv1 = conv3x3(inplanes, planes, stride)\n        self.bn1 = nn.GroupNorm(16, planes)\n\n        self.conv2 = conv3x3(planes, planes)\n        self.bn2 = nn.GroupNorm(16, planes)\n\n        if stride != 1 or inplanes != planes:\n            self.downsample = nn.Sequential(\n                conv1x1(inplanes, planes, stride), nn.GroupNorm(16, planes))\n        else:\n            self.downsample = None\n\n\n    def forward(self, x):\n        identity = x\n\n        out = F.relu(self.bn1(self.conv1(x)), inplace=True)\n        out = self.bn2(self.conv2(out))\n\n        if self.downsample is not None:\n            identity = self.downsample(x)\n\n        out += identity\n        out = F.relu(out, inplace=True)\n\n        return out\n\n\nclass Bottleneck(nn.Module):\n    expansion = 4\n\n    def __init__(self, inplanes, planes, stride=1):\n        super(Bottleneck, self).__init__()\n        self.conv1 = conv1x1(inplanes, planes)\n        self.bn1 = nn.GroupNorm(16, planes)\n        self.conv2 = conv3x3(planes, planes, stride)\n        self.bn2 = nn.GroupNorm(16, planes)\n        self.conv3 = conv1x1(planes, planes * self.expansion)\n        self.bn3 = nn.GroupNorm(16, planes * self.expansion)\n\n        if stride != 1 or inplanes != planes * self.expansion:\n            self.downsample = nn.Sequential(\n                conv1x1(inplanes, planes * self.expansion, stride), \n                nn.GroupNorm(16, planes * self.expansion))\n        else:\n            self.downsample = None\n\n    def forward(self, x):\n        identity = x\n\n        out = F.relu(self.bn1(self.conv1(x)), inplace=True)\n        out = F.relu(self.bn2(self.conv2(out)), inplace=True)\n        out = self.bn3(self.conv3(out))\n \n        if self.downsample is not None:\n            identity = self.downsample(x)\n\n        out += identity\n        out = F.relu(out)\n\n        return out\n\n\nclass ResNetFeatures(nn.Module):\n\n    def __init__(self, block, layers, num_classes=1000, zero_init_residual=False):\n        super(ResNetFeatures, self).__init__()\n        self.inplanes = 64\n        self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3,\n                               bias=False)\n        self.bn1 = nn.GroupNorm(16, 64)\n\n        self.layer1 = self._make_layer(block, 64, layers[0])\n        self.layer2 = self._make_layer(block, 128, layers[1], stride=2)\n        self.layer3 = self._make_layer(block, 256, layers[2], stride=2)\n        self.layer4 = self._make_layer(block, 512, layers[3], stride=2)\n\n\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')\n            elif isinstance(m, nn.BatchNorm2d):\n                nn.init.constant_(m.weight, 1)\n                nn.init.constant_(m.bias, 0)\n\n        # Zero-initialize the last BN in each residual branch,\n        # so that the residual branch starts with zeros, and each residual block behaves like an identity.\n        # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677\n        if zero_init_residual:\n            for m in self.modules():\n                if isinstance(m, Bottleneck):\n                    nn.init.constant_(m.bn3.weight, 0)\n                elif isinstance(m, BasicBlock):\n                    nn.init.constant_(m.bn2.weight, 0)\n\n    def _make_layer(self, block, planes, blocks, stride=1):\n\n        layers = []\n        layers.append(block(self.inplanes, planes, stride))\n        self.inplanes = planes * block.expansion\n        for _ in range(1, blocks):\n            layers.append(block(self.inplanes, planes))\n\n        return nn.Sequential(*layers)\n\n\n    def forward(self, x):\n        conv1 = F.relu(self.bn1(self.conv1(x)), inplace=True)\n        conv1 = F.max_pool2d(conv1, 3, stride=2, padding=1)\n\n        feats4 = self.layer1(conv1)\n        feats8 = self.layer2(feats4)\n        feats16 = self.layer3(feats8)\n        feats32 = self.layer4(feats16)\n\n        return feats8, feats16, feats32\n\n\n\ndef resnet18(pretrained=False, **kwargs):\n    \"\"\"Constructs a ResNet-18 model.\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n    \"\"\"\n    model = ResNetFeatures(BasicBlock, [2, 2, 2, 2], **kwargs)\n    if pretrained:\n        _load_pretrained(model, model_zoo.load_url(model_urls['resnet18']))\n    return model\n\n\ndef  resnext50(pretrained=False, **kwargs):\n    \"\"\"Constructs a ResNet-18 model.\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n    \"\"\"\n    model = ResNetFeatures(BasicBlock, [2, 2, 2, 2], **kwargs)\n    if pretrained:\n        _load_pretrained(model, model_zoo.load_url(model_urls['resnext50_32x4d']))\n    return model\n\n\ndef resnet34(pretrained=False, **kwargs):\n    \"\"\"Constructs a ResNet-34 model.\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n    \"\"\"\n    model = ResNetFeatures(BasicBlock, [3, 4, 6, 3], **kwargs)\n    if pretrained:\n        _load_pretrained(model, model_zoo.load_url(model_urls['resnet34']))\n    return model\n\ndef inceptionresnetv2(pretrained=False, **kwargs):\n    \"\"\"Constructs a inceptionresnetv2.\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n    \"\"\"\n    model = ResNetFeatures(BasicBlock, [3, 4, 6, 3], **kwargs)\n    if pretrained:\n        _load_pretrained(model, model_zoo.load_url(model_urls['inceptionresnetv2']))\n    return model\n\ndef densenet201(pretrained=False, **kwargs):\n    \"\"\"Constructs a densenet201 model.\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n    \"\"\"\n    model = ResNetFeatures(BasicBlock, [3, 4, 6, 3], **kwargs)\n    if pretrained:\n        _load_pretrained(model, model_zoo.load_url(model_urls['densenet201']))\n    return model\n\n\ndef _load_pretrained(model, pretrained):\n    model_dict = model.state_dict()\n    pretrained = {k : v for k, v in pretrained.items() if k in model_dict}\n    model_dict.update(pretrained)\n    model.load_state_dict(model_dict)","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"cd2d63bf-aa5a-4018-a990-e341a19ea840","_cell_guid":"24701851-3c22-40c3-a119-c0088a05823b","trusted":true},"cell_type":"markdown","source":"** Inception-ResNet-V2 (2016)**"},{"metadata":{"_uuid":"894d085a-21e3-4f0b-93ee-ede8af01d79e","_cell_guid":"9cb5fc87-5c5c-49a6-8851-4d5dbf60c01b","trusted":true},"cell_type":"markdown","source":"![](https://miro.medium.com/max/3750/1*xpb6QFQ4IknSmxmgai8w-Q.png)"},{"metadata":{"_uuid":"04876811-15ef-47f3-9f9a-603ba04dce17","_cell_guid":"3dc839ac-8128-446c-8cd5-36811ce763df","trusted":true},"cell_type":"code","source":"\nbase_model = inceptionresnetv2(pretrained=False)\nbase_model","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"2f76d8bf-062b-4ce4-a67b-004bb2044fda","_cell_guid":"00fef9cc-8448-47dd-b25b-d7e1a7aaf5a5","trusted":true},"cell_type":"code","source":"","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"d5661a2e-aa60-46b3-96a1-63f058f488f7","_cell_guid":"3f16e5b5-cf48-4730-90e6-96ab98309297","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"class CentResnet(nn.Module):\n    '''Mixture of previous classes'''\n    def __init__(self, n_classes):\n        super(CentResnet, self).__init__()\n        self.base_model = base_model\n        \n        # Lateral layers convert resnet outputs to a common feature size\n        self.lat8 = nn.Conv2d(128, 256, 1)\n        self.lat16 = nn.Conv2d(256, 256, 1)\n        self.lat32 = nn.Conv2d(512, 256, 1)\n        self.bn8 = nn.GroupNorm(16, 256)\n        self.bn16 = nn.GroupNorm(16, 256)\n        self.bn32 = nn.GroupNorm(16, 256)\n\n       \n        self.conv0 = double_conv(5, 64)\n        self.conv1 = double_conv(64, 128)\n        self.conv2 = double_conv(128, 512)\n        self.conv3 = double_conv(512, 1024)\n        \n        self.mp = nn.MaxPool2d(2)\n        \n        self.up1 = up(1282 , 512) #+ 1024\n        self.up2 = up(512 + 512, 256)\n        self.outc = nn.Conv2d(256, n_classes, 1)\n        \n    \n    def forward(self, x):\n        batch_size = x.shape[0]\n        mesh1 = get_mesh(batch_size, x.shape[2], x.shape[3])\n        x0 = torch.cat([x, mesh1], 1)\n        x1 = self.mp(self.conv0(x0))\n        x2 = self.mp(self.conv1(x1))\n        x3 = self.mp(self.conv2(x2))\n        x4 = self.mp(self.conv3(x3))\n        \n        #feats = self.base_model.extract_features(x)\n                # Run frontend network\n        feats8, feats16, feats32 = self.base_model(x)\n        lat8 = F.relu(self.bn8(self.lat8(feats8)))\n        lat16 = F.relu(self.bn16(self.lat16(feats16)))\n        lat32 = F.relu(self.bn32(self.lat32(feats32)))\n        \n        # Add positional info\n        mesh2 = get_mesh(batch_size, lat32.shape[2], lat32.shape[3])\n        feats = torch.cat([lat32, mesh2], 1)\n        #print(feats.shape)\n        #print (x4.shape)\n        x = self.up1(feats, x4)\n        x = self.up2(x, x3)\n        x = self.outc(x)\n        return x","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"a3883658-846b-4234-abc8-6ad455aeecac","_cell_guid":"81b6d252-ab7c-49b8-b207-9ed9b37a5400","trusted":true},"cell_type":"markdown","source":"# Radam"},{"metadata":{"_uuid":"adaa95b4-6d5b-48ef-b1f3-10dc0444094e","_cell_guid":"e34cbf92-def3-4121-93c7-4daa5814d215","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import math\nimport torch\nfrom torch.optim.optimizer import Optimizer, required\n\nclass RAdam(Optimizer):\n\n    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0):\n        defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay)\n        self.buffer = [[None, None, None] for ind in range(10)]\n        super(RAdam, self).__init__(params, defaults)\n\n    def __setstate__(self, state):\n        super(RAdam, self).__setstate__(state)\n\n    def step(self, closure=None):\n\n        loss = None\n        if closure is not None:\n            loss = closure()\n\n        for group in self.param_groups:\n\n            for p in group['params']:\n                if p.grad is None:\n                    continue\n                grad = p.grad.data.float()\n                if grad.is_sparse:\n                    raise RuntimeError('RAdam does not support sparse gradients')\n\n                p_data_fp32 = p.data.float()\n\n                state = self.state[p]\n\n                if len(state) == 0:\n                    state['step'] = 0\n                    state['exp_avg'] = torch.zeros_like(p_data_fp32)\n                    state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)\n                else:\n                    state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)\n                    state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32)\n\n                exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']\n                beta1, beta2 = group['betas']\n\n                exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad)\n                exp_avg.mul_(beta1).add_(1 - beta1, grad)\n\n                state['step'] += 1\n                buffered = self.buffer[int(state['step'] % 10)]\n                if state['step'] == buffered[0]:\n                    N_sma, step_size = buffered[1], buffered[2]\n                else:\n                    buffered[0] = state['step']\n                    beta2_t = beta2 ** state['step']\n                    N_sma_max = 2 / (1 - beta2) - 1\n                    N_sma = N_sma_max - 2 * state['step'] * beta2_t / (1 - beta2_t)\n                    buffered[1] = N_sma\n\n                    # more conservative since it's an approximated value\n                    if N_sma >= 5:\n                        step_size = group['lr'] * math.sqrt((1 - beta2_t) * (N_sma - 4) / (N_sma_max - 4) * (N_sma - 2) / N_sma * N_sma_max / (N_sma_max - 2)) / (1 - beta1 ** state['step'])\n                    else:\n                        step_size = group['lr'] / (1 - beta1 ** state['step'])\n                    buffered[2] = step_size\n\n                if group['weight_decay'] != 0:\n                    p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32)\n\n                # more conservative since it's an approximated value\n                if N_sma >= 5:            \n                    denom = exp_avg_sq.sqrt().add_(group['eps'])\n                    p_data_fp32.addcdiv_(-step_size, exp_avg, denom)\n                else:\n                    p_data_fp32.add_(-step_size, exp_avg)\n\n                p.data.copy_(p_data_fp32)\n\n        return loss","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"8cb38e5f-9130-4d72-b5af-8a0c185689ac","_cell_guid":"dbf60f0f-9df8-4ee5-b275-da8778bf5c71","trusted":true},"cell_type":"code","source":"# Gets the GPU if there is one, otherwise the cpu\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(device)\n\nn_epochs = 9 #original 10, 8, 7\n\nmodel = CentResnet(8).to(device)\noptimizer = optim.AdamW(model.parameters(), lr=0.001)\n#optimizer =  RAdam(model.parameters(), lr = 0.001)\nexp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=max(n_epochs, 10) * 3828 // 3, gamma=0.1)","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"aed75497-283d-472f-ba46-d0f609656cb2","_cell_guid":"eab77964-d32a-48d7-afd8-2c95b2e50f8a","trusted":true},"cell_type":"code","source":"img_batch = torch.randn((1,3,700,1600))","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"88b6dafb-51a3-4d8d-811b-a7841b2c789c","_cell_guid":"cb770992-1b49-4711-a45a-74b1e2d70785","trusted":true},"cell_type":"code","source":"test = model(img_batch.to(device))","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"c9cb959f-0621-4e41-bd81-efb1040d9079","_cell_guid":"5b352e78-69d2-4d2a-9d97-28281eb57f14","trusted":true},"cell_type":"markdown","source":"# Training"},{"metadata":{"_uuid":"c2b51b55-dd06-41a0-990c-d484458a6c04","_cell_guid":"c52944fd-cffe-4fe1-8dfe-cf5ea1756004","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def criterion(prediction, mask, regr,weight=0.4, size_average=True):\n    # Binary mask loss\n    pred_mask = torch.sigmoid(prediction[:, 0])\n#     mask_loss = mask * (1 - pred_mask)**2 * torch.log(pred_mask + 1e-12) + (1 - mask) * pred_mask**2 * torch.log(1 - pred_mask + 1e-12)\n    mask_loss = mask * torch.log(pred_mask + 1e-12) + (1 - mask) * torch.log(1 - pred_mask + 1e-12)\n    mask_loss = -mask_loss.mean(0).sum()\n    \n    # Regression L1 loss\n    pred_regr = prediction[:, 1:]\n    regr_loss = (torch.abs(pred_regr - regr).sum(1) * mask).sum(1).sum(1) / mask.sum(1).sum(1)\n    regr_loss = regr_loss.mean(0)\n  \n    # Sum\n    loss = mask_loss +regr_loss\n    if not size_average:\n        loss *= prediction.shape[0]\n    return loss ,mask_loss , regr_loss","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"7dece8b3-b1f4-44f3-b542-ea7a6e09be0e","_cell_guid":"0f1d6fdc-7b7a-4554-bdf9-535afdd1a56d","trusted":true},"cell_type":"code","source":"## Just for checking the shapes to manage our Unet\ni = 0\nfor batch_idx, (img_batch, mask_batch, regr_batch) in enumerate(tqdm(train_loader)):\n    print(img_batch.shape)\n    print(mask_batch.shape)\n    print(regr_batch.shape)\n    i+=1\n    if i>1:\n        break","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"ff473c33-f9c1-4305-ba8a-b34ab7c97c09","_cell_guid":"51d4c8b1-9aa4-46ea-8c7b-14b4615a4015","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def train(epoch, history=None):\n    model.train()\n    t = tqdm(train_loader)\n    for batch_idx, (img_batch, mask_batch, regr_batch) in enumerate(t):\n        img_batch = img_batch.to(device)\n        mask_batch = mask_batch.to(device)\n        regr_batch = regr_batch.to(device)\n        \n        optimizer.zero_grad()\n        output = model(img_batch)\n        if epoch < SWITCH_LOSS_EPOCH :\n            loss,mask_loss, regr_loss = criterion(output, mask_batch, regr_batch,1)\n        else:\n            loss,mask_loss, regr_loss = criterion(output, mask_batch, regr_batch,0.5)  \n        \n        t.set_description(f'train_loss (l={loss:.3f})(m={mask_loss:.2f}) (r={regr_loss:.4f}')\n        \n        if history is not None:\n            history.loc[epoch + batch_idx / len(train_loader), 'train_loss'] = loss.data.cpu().numpy()\n        \n        loss.backward()\n        \n        optimizer.step()\n        exp_lr_scheduler.step()\n\n    \n    print('Train Epoch: {} \\tLR: {:.6f}\\tLoss: {:.6f}\\tMaskLoss: {:.6f}\\tRegLoss: {:.6f}'.format(\n        epoch,\n        optimizer.state_dict()['param_groups'][0]['lr'],\n        loss.data,\n        mask_loss.data,\n        regr_loss.data))\n\ndef evaluate(epoch, history=None):\n    model.eval()\n    loss = 0\n    valid_loss = 0\n    valid_mask_loss = 0\n    valid_regr_loss = 0\n    with torch.no_grad():\n        for img_batch, mask_batch, regr_batch in dev_loader:\n            img_batch = img_batch.to(device)\n            mask_batch = mask_batch.to(device)\n            regr_batch = regr_batch.to(device)\n\n            output = model(img_batch)\n\n            if epoch < SWITCH_LOSS_EPOCH :\n                loss,mask_loss, regr_loss= criterion(output, mask_batch, regr_batch,1, size_average=False)\n                valid_loss += loss.data\n                valid_mask_loss += mask_loss.data\n                valid_regr_loss += regr_loss.data\n            else :\n                loss,mask_loss, regr_loss = criterion(output, mask_batch, regr_batch,0.5, size_average=False)\n                valid_loss += loss.data\n                valid_mask_loss += mask_loss.data\n                valid_regr_loss += regr_loss.data \n\n    \n    valid_loss /= len(dev_loader.dataset)\n    valid_mask_loss /= len(dev_loader.dataset)\n    valid_regr_loss /= len(dev_loader.dataset)\n    \n    if history is not None:\n        history.loc[epoch, 'dev_loss'] = valid_loss.cpu().numpy()\n        history.loc[epoch, 'mask_loss'] = valid_mask_loss.cpu().numpy()\n        history.loc[epoch, 'regr_loss'] = valid_regr_loss.cpu().numpy()\n\n    \n    print('Dev loss: {:.4f}'.format(valid_loss))","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"fc477bf0-fafa-4ae3-a076-bf6efed6dfac","_cell_guid":"23cd5c61-e6b2-4ee9-b703-0f5291c7ba80","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"import gc\n\nhistory = pd.DataFrame()\n\nfor epoch in range(n_epochs):\n    torch.cuda.empty_cache()\n    gc.collect()\n    train(epoch, history)\n    evaluate(epoch, history)","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"06042151-cd4f-4529-8da2-7e8cdcc36cb4","_cell_guid":"686a468f-ea95-4ea6-b362-99c468329ed3","trusted":true},"cell_type":"code","source":"torch.save(model.state_dict(), './resnext50.pth')","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"409c0945-52db-4667-b3ff-015a0fafcb0a","_cell_guid":"e5740872-e2cb-4f81-9dbd-4509acc5562e","trusted":true},"cell_type":"code","source":"history['train_loss'].iloc[100:].plot();","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"6c43f285-a9fa-4298-a277-a68389ab86b8","_cell_guid":"a60a2974-a06d-42e2-847f-aa89e73117f0","trusted":true},"cell_type":"code","source":"series1 = history.dropna()['mask_loss']\nplt.plot(series1.index, series1 ,label = 'mask loss');\nseries2 = history.dropna()['regr_loss']\nplt.plot(series2.index, 30*series2,label = 'regr loss');\nseries3 = history.dropna()['dev_loss']\nplt.plot(series3.index, series3,label = 'dev loss');\nplt.show()","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"ce96ca28-fd84-4ad4-9428-32aba29f5b3c","_cell_guid":"72565bf1-3b01-4ee7-ab5c-343b79e100c7","trusted":true},"cell_type":"code","source":"series = history.dropna()['dev_loss']\nplt.scatter(series.index, series);","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"3a60d651-f380-4a21-8c09-ea645ef5d2d6","_cell_guid":"635b19f5-6b6c-4ac9-a036-3b85a7ef3a28","trusted":true},"cell_type":"markdown","source":"# Visualize predictions"},{"metadata":{"_uuid":"c31eeae0-c30c-43e4-8cd8-5de02ad4d795","_cell_guid":"265371dd-c913-44d2-bd14-549e1e63392e","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"img, mask, regr = dev_dataset[0]\n\nplt.figure(figsize=(16,16))\nplt.title('Input image')\nplt.imshow(np.rollaxis(img, 0, 3))\nplt.show()\n\nplt.figure(figsize=(16,16))\nplt.title('Ground truth mask')\nplt.imshow(mask)\nplt.show()\n\noutput = model(torch.tensor(img[None]).to(device))\nlogits = output[0,0].data.cpu().numpy()\n\nplt.figure(figsize=(16,16))\nplt.title('Model predictions')\nplt.imshow(logits)\nplt.show()\n\nprint(logits)\nplt.figure(figsize=(16,16))\nplt.title('Model predictions thresholded')\nplt.imshow(logits > 0)\nplt.show()","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"e9c60bb0-e20f-4610-ac63-c8d5ba1ed3ae","_cell_guid":"79ba0d5f-5c8c-44e8-a2cb-5dfad7ddd240","trusted":true},"cell_type":"code","source":"## Simple test of probabilities\nact = torch.nn.Sigmoid()\nlogtens = torch.from_numpy(logits)\nprobs = act(logtens)\nprobs = probs[probs>0]\nprint(probs)","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"01b61848-ab21-4eaf-8609-9cdb671adc65","_cell_guid":"6f048ede-72c9-47c3-bae1-3dcaae1104f7","trusted":true},"cell_type":"code","source":"DISTANCE_THRESH_CLEAR = 2\n\ndef convert_3d_to_2d(x, y, z, fx = 2304.5479, fy = 2305.8757, cx = 1686.2379, cy = 1354.9849):\n    # stolen from https://www.kaggle.com/theshockwaverider/eda-visualization-baseline\n    return x * fx / z + cx, y * fy / z + cy\n\ndef optimize_xy(r, c, x0, y0, z0):\n    def distance_fn(xyz):\n        x, y, z = xyz\n        x, y = convert_3d_to_2d(x, y, z0)\n        y, x = x, y\n        x = (x - IMG_SHAPE[0] // 2) * IMG_HEIGHT / (IMG_SHAPE[0] // 2) / MODEL_SCALE\n        x = np.round(x).astype('int')\n        y = (y + IMG_SHAPE[1] // 4) * IMG_WIDTH / (IMG_SHAPE[1] * 1.5) / MODEL_SCALE\n        y = np.round(y).astype('int')\n        return (x-r)**2 + (y-c)**2\n    \n    res = minimize(distance_fn, [x0, y0, z0], method='Powell')\n    x_new, y_new, z_new = res.x\n    return x_new, y_new, z0\n\ndef clear_duplicates(coords):\n    for c1 in coords:\n        xyz1 = np.array([c1['x'], c1['y'], c1['z']])\n        for c2 in coords:\n            xyz2 = np.array([c2['x'], c2['y'], c2['z']])\n            distance = np.sqrt(((xyz1 - xyz2)**2).sum())\n            if distance < DISTANCE_THRESH_CLEAR:\n                if c1['confidence'] < c2['confidence']:\n                    c1['confidence'] = -1\n    return [c for c in coords if c['confidence'] > 0]\n\ndef extract_coords(prediction):\n    logits = prediction[0]\n    regr_output = prediction[1:]\n    points = np.argwhere(logits > 0)\n    col_names = sorted(['x', 'y', 'z', 'yaw', 'pitch_sin', 'pitch_cos', 'roll'])\n    coords = []\n    for r, c in points:\n        regr_dict = dict(zip(col_names, regr_output[:, r, c]))\n        coords.append(_regr_back(regr_dict))\n        coords[-1]['confidence'] = 1 / (1 + np.exp(-logits[r, c]))\n        coords[-1]['x'], coords[-1]['y'], coords[-1]['z'] = optimize_xy(r, c, coords[-1]['x'], coords[-1]['y'], coords[-1]['z'])\n    coords = clear_duplicates(coords)\n    return coords\n\ndef coords2str(coords, names=['yaw', 'pitch', 'roll', 'x', 'y', 'z', 'confidence']):\n    s = []\n    for c in coords:\n        for n in names:\n            s.append(str(c.get(n, 0)))\n    return ' '.join(s)","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"3deb12dc-aac8-492c-b51a-83d13c7c6417","_cell_guid":"e6599ae2-9e32-44d4-9c5d-920e19fd85ee","trusted":true},"cell_type":"code","source":"torch.cuda.empty_cache()\ngc.collect()\n\nfor idx in range(4):\n    img, mask, regr = dev_dataset[idx]\n    \n    output = model(torch.tensor(img[None]).to(device)).data.cpu().numpy()\n    coords_pred = extract_coords(output[0])\n    coords_true = extract_coords(np.concatenate([mask[None], regr], 0))\n    \n    img = imread(train_images_dir.format(df_dev['ImageId'].iloc[idx]))\n    \n    fig, axes = plt.subplots(1, 2, figsize=(30,30))\n    axes[0].set_title('Ground truth')\n    axes[0].imshow(visualize(img, coords_true))\n    axes[1].set_title('Prediction')\n    axes[1].imshow(visualize(img, coords_pred))\n    plt.show()","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"d507d188-ab2e-40ef-ba6c-6df9ea36bcdf","_cell_guid":"d5dab094-3887-4a0a-8f7e-267d921f0c35","trusted":true},"cell_type":"markdown","source":"# Make submission"},{"metadata":{"_uuid":"b1d49f24-f87c-4212-ae90-affe2f315728","_cell_guid":"2e0e2306-e094-4262-aaa7-4420588bccab","trusted":true},"cell_type":"code","source":"# taken from kernel of @its7171 \nval_preds = []\n\ndev_loader = DataLoader(dataset=dev_dataset, batch_size=4, shuffle=False, num_workers=4)\n\nmodel.eval()\n\nfor img, _, _ in tqdm(dev_loader):\n    with torch.no_grad():\n        output = model(img.to(device))\n    output = output.data.cpu().numpy()\n    for out in output:\n        coords = extract_coords(out)\n        s = coords2str(coords)\n        val_preds.append(s)","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"86592d57-5868-495d-b265-5ca46e851db7","_cell_guid":"12d33073-5109-4508-9cc6-e9d53f84ee2e","trusted":true},"cell_type":"code","source":"df_dev['PredictionString'] = val_preds\ndf_dev.head()\ndf_dev.to_csv('val_predictions.csv', index=False)","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"1e30a860-b535-4d47-9a06-c0bada2e7386","_cell_guid":"9c811a99-417b-4b9b-84dd-689d32141bb1","trusted":true},"cell_type":"code","source":"predictions = []\n\ntest_loader = DataLoader(dataset=test_dataset, batch_size=2, shuffle=False, num_workers=4)\n\nmodel.eval()\n\nfor img, _, _ in tqdm(test_loader):\n    with torch.no_grad():\n        output = model(img.to(device))\n    output = output.data.cpu().numpy()\n    for out in output:\n        coords = extract_coords(out)\n        s = coords2str(coords)\n        predictions.append(s)","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"38d7755d-30a6-4452-8602-82463642f782","_cell_guid":"b770c146-cc46-4223-90c6-ce601371cd14","trusted":true},"cell_type":"code","source":"test = pd.read_csv(PATH + 'sample_submission.csv')\ntest['PredictionString'] = predictions\ntest.to_csv('predictions.csv', index=False)\ntest.head()","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"c6ddb8a0-aa6f-4cfd-bc25-58c2cce894b6","_cell_guid":"441ab6b0-d3da-42a7-9f8e-03e9c012b678","trusted":true},"cell_type":"code","source":"\ndef expand_df(df, PredictionStringCols):\n    df = df.dropna().copy()\n    df['NumCars'] = [int((x.count(' ')+1)/7) for x in df['PredictionString']]\n\n    image_id_expanded = [item for item, count in zip(df['ImageId'], df['NumCars']) for i in range(count)]\n    prediction_strings_expanded = df['PredictionString'].str.split(' ',expand = True).values.reshape(-1,7).astype(float)\n    prediction_strings_expanded = prediction_strings_expanded[~np.isnan(prediction_strings_expanded).all(axis=1)]\n    df = pd.DataFrame(\n        {\n            'ImageId': image_id_expanded,\n            PredictionStringCols[0]:prediction_strings_expanded[:,0],\n            PredictionStringCols[1]:prediction_strings_expanded[:,1],\n            PredictionStringCols[2]:prediction_strings_expanded[:,2],\n            PredictionStringCols[3]:prediction_strings_expanded[:,3],\n            PredictionStringCols[4]:prediction_strings_expanded[:,4],\n            PredictionStringCols[5]:prediction_strings_expanded[:,5],\n            PredictionStringCols[6]:prediction_strings_expanded[:,6]\n        })\n    return df\n\ndef str2coords(s, names):\n    coords = []\n    for l in np.array(s.split()).reshape([-1, 7]):\n        coords.append(dict(zip(names, l.astype('float'))))\n    return coords\n\ndef TranslationDistance(p,g, abs_dist = False):\n    dx = p['x'] - g['x']\n    dy = p['y'] - g['y']\n    dz = p['z'] - g['z']\n    diff0 = (g['x']**2 + g['y']**2 + g['z']**2)**0.5\n    diff1 = (dx**2 + dy**2 + dz**2)**0.5\n    if abs_dist:\n        diff = diff1\n    else:\n        diff = diff1/diff0\n    return diff\n\ndef RotationDistance(p, g):\n    true=[ g['pitch'] ,g['yaw'] ,g['roll'] ]\n    pred=[ p['pitch'] ,p['yaw'] ,p['roll'] ]\n    q1 = R.from_euler('xyz', true)\n    q2 = R.from_euler('xyz', pred)\n    diff = R.inv(q2) * q1\n    W = np.clip(diff.as_quat()[-1], -1., 1.)\n    \n    # in the official metrics code:\n    # https://www.kaggle.com/c/pku-autonomous-driving/overview/evaluation\n    #   return Object3D.RadianToDegree( Math.Acos(diff.W) )\n    # this code treat θ and θ+2π differntly.\n    # So this should be fixed as follows.\n    W = (acos(W)*360)/pi\n    if W > 180:\n        W = 360 - W\n    return W","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"7e027307-2c79-46ad-ac1b-69d2c9cec3b5","_cell_guid":"70174677-7083-433a-937e-88d690be80f4","trusted":true},"cell_type":"code","source":"thres_tr_list = [0.1, 0.09, 0.08, 0.07, 0.06, 0.05, 0.04, 0.03, 0.02, 0.01]\nthres_ro_list = [50, 45, 40, 35, 30, 25, 20, 15, 10, 5]\n\ndef check_match(idx):\n    keep_gt=False\n    thre_tr_dist = thres_tr_list[idx]\n    thre_ro_dist = thres_ro_list[idx]\n    train_dict = {imgID:str2coords(s, names=['carid_or_score', 'pitch', 'yaw', 'roll', 'x', 'y', 'z']) for imgID,s in zip(train_df['ImageId'],train_df['PredictionString'])}\n    valid_dict = {imgID:str2coords(s, names=['pitch', 'yaw', 'roll', 'x', 'y', 'z', 'carid_or_score']) for imgID,s in zip(valid_df['ImageId'],valid_df['PredictionString'])}\n    result_flg = [] # 1 for TP, 0 for FP\n    scores = []\n    MAX_VAL = 10**10\n    for img_id in valid_dict:\n        for pcar in sorted(valid_dict[img_id], key=lambda x: -x['carid_or_score']):\n            # find nearest GT\n            min_tr_dist = MAX_VAL\n            min_idx = -1\n            for idx, gcar in enumerate(train_dict[img_id]):\n                tr_dist = TranslationDistance(pcar,gcar)\n                if tr_dist < min_tr_dist:\n                    min_tr_dist = tr_dist\n                    min_ro_dist = RotationDistance(pcar,gcar)\n                    min_idx = idx\n                    \n            # set the result\n            if min_tr_dist < thre_tr_dist and min_ro_dist < thre_ro_dist:\n                if not keep_gt:\n                    train_dict[img_id].pop(min_idx)\n                result_flg.append(1)\n            else:\n                result_flg.append(0)\n            scores.append(pcar['carid_or_score'])\n    \n    return result_flg, scores","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"aa7d9e06-0191-4e43-88cf-e6be66cf0429","_cell_guid":"68ad0001-cf13-4805-af99-d1fa3de4733d","trusted":true},"cell_type":"code","source":"#validation_prediction = df_dev\nvalid_df = pd.read_csv('val_predictions.csv')\nexpanded_valid_df = expand_df(valid_df, ['pitch','yaw','roll','x','y','z','Score'])\nvalid_df = valid_df.fillna('')\n\ntrain_df = pd.read_csv('../input/pku-autonomous-driving/train.csv')\ntrain_df = train_df[train_df.ImageId.isin(valid_df.ImageId.unique())]\n# data description page says, The pose information is formatted as\n# model type, yaw, pitch, roll, x, y, z\n# but it doesn't, and it should be\n# model type, pitch, yaw, roll, x, y, z\nexpanded_train_df = expand_df(train_df, ['model_type','pitch','yaw','roll','x','y','z'])\n\nmax_workers = 10\nn_gt = len(expanded_train_df)\nap_list = []\np = Pool(processes=max_workers)\nfor result_flg, scores in p.imap(check_match, range(10)):\n    if np.sum(result_flg) > 0:\n        n_tp = np.sum(result_flg)\n        recall = n_tp/n_gt\n        ap = average_precision_score(result_flg, scores)*recall\n    else:\n        ap = 0\n    ap_list.append(ap)\nmap = np.mean(ap_list)\nprint('map:', map)","execution_count":0,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}