{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Overview\n\n- The objectie of this kernel is solely to illustrate the impacts of varying the number of epochs,for version 2 of the forked kernel https://www.kaggle.com/mobassir/center-resnext50-baseline\n- The only changes are to the number of epochs\n- As much GPU hours have been spent to illustrate the results, it will be great if you can **upvote** if you found this helpful in any way, including the original kernel above\n- Thank you and happy new year!","metadata":{}},{"cell_type":"markdown","source":"# Experimental Results\n\n- 10 epochs: 0.044 public LB score\n- 8 epochs: 0.047 public LB score","metadata":{}},{"cell_type":"markdown","source":"# Important Remarks\n\n- The public leaderboard is calculated with approximately 9% of the test data\n- The final results will be based on the other 91%, so the final standings may be very different!\n- Submit the highest LB from this kernel from final scoring **at your own risk**!!","metadata":{}},{"cell_type":"markdown","source":"**Progressions** (Original author's comments)","metadata":{}},{"cell_type":"markdown","source":"1. version 1 : 1 epoch took 1+ hours so in version 2 i will try for 8 epoches\n2. for 8 epoches in version 2 i got 0.095 public lb score","metadata":{}},{"cell_type":"markdown","source":"**imports**","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport cv2\nfrom tqdm import tqdm_notebook as tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom functools import reduce\nimport os\nfrom sklearn.model_selection import train_test_split\nfrom scipy.optimize import minimize\nfrom tqdm.auto import tqdm as tq\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import models\nfrom torchvision import transforms, utils\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom math import sqrt, acos, pi, sin, cos\nfrom scipy.spatial.transform import Rotation as R\nfrom sklearn.metrics import average_precision_score\nfrom multiprocessing import Pool\n\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.utils.model_zoo as model_zoo\nimport gc\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom math import sqrt, acos, pi, sin, cos\nfrom scipy.spatial.transform import Rotation as R\nfrom sklearn.metrics import average_precision_score\nfrom multiprocessing import Pool\n\n\nPATH = '../input/pku-autonomous-driving/'\nos.listdir(PATH)","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2021-12-19T16:12:07.92231Z","iopub.execute_input":"2021-12-19T16:12:07.922542Z","iopub.status.idle":"2021-12-19T16:12:10.555449Z","shell.execute_reply.started":"2021-12-19T16:12:07.922477Z","shell.execute_reply":"2021-12-19T16:12:10.554699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Constants\nSWITCH_LOSS_EPOCH = 5\nprint(torch.__version__) ","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-19T16:12:10.557312Z","iopub.execute_input":"2021-12-19T16:12:10.557564Z","iopub.status.idle":"2021-12-19T16:12:10.562444Z","shell.execute_reply.started":"2021-12-19T16:12:10.557529Z","shell.execute_reply":"2021-12-19T16:12:10.561613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(PATH + 'train.csv')\ntest = pd.read_csv(PATH + 'sample_submission.csv')\nbad_list = ['ID_1a5a10365',\n'ID_1db0533c7',\n'ID_53c3fe91a',\n'ID_408f58e9f',\n'ID_4445ae041',\n'ID_bb1d991f6',\n'ID_c44983aeb',\n'ID_f30ebe4d4']\ntrain = train.loc[~train['ImageId'].isin(bad_list)]\n# From camera.zip\ncamera_matrix = np.array([[2304.5479, 0,  1686.2379],\n                          [0, 2305.8757, 1354.9849],\n                          [0, 0, 1]], dtype=np.float32)\ncamera_matrix_inv = np.linalg.inv(camera_matrix)\n\ntrain.head()","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_kg_hide-input":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","execution":{"iopub.status.busy":"2021-12-19T16:12:10.563689Z","iopub.execute_input":"2021-12-19T16:12:10.564105Z","iopub.status.idle":"2021-12-19T16:12:10.670418Z","shell.execute_reply.started":"2021-12-19T16:12:10.56407Z","shell.execute_reply":"2021-12-19T16:12:10.669521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**ImageId** column contains names of images:","metadata":{}},{"cell_type":"code","source":"def imread(path, fast_mode=False):\n    img = cv2.imread(path)\n    if not fast_mode and img is not None and len(img.shape) == 3:\n        img = np.array(img[:, :, ::-1])\n    return img\n\nimg = imread(PATH + 'train_images/ID_8a6e65317' + '.jpg')\nIMG_SHAPE = img.shape\n\nplt.figure(figsize=(15,8))\nplt.imshow(img);","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-19T16:12:10.6758Z","iopub.execute_input":"2021-12-19T16:12:10.676077Z","iopub.status.idle":"2021-12-19T16:12:12.666159Z","shell.execute_reply.started":"2021-12-19T16:12:10.676042Z","shell.execute_reply":"2021-12-19T16:12:12.663801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def str2coords(s, names=['id', 'yaw', 'pitch', 'roll', 'x', 'y', 'z']):\n    '''\n    Input:\n        s: PredictionString (e.g. from train dataframe)\n        names: array of what to extract from the string\n    Output:\n        list of dicts with keys from `names`\n    '''\n    coords = []\n    for l in np.array(s.split()).reshape([-1, 7]):\n        coords.append(dict(zip(names, l.astype('float'))))\n        if 'id' in coords[-1]:\n            coords[-1]['id'] = int(coords[-1]['id'])\n    return coords","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-19T16:12:12.667058Z","iopub.execute_input":"2021-12-19T16:12:12.667277Z","iopub.status.idle":"2021-12-19T16:12:12.675243Z","shell.execute_reply.started":"2021-12-19T16:12:12.667248Z","shell.execute_reply":"2021-12-19T16:12:12.674578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inp = train['PredictionString'][0]\nprint('Example input:\\n', inp)\nprint()\nprint('Output:\\n', str2coords(inp))","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-19T16:12:12.676785Z","iopub.execute_input":"2021-12-19T16:12:12.677255Z","iopub.status.idle":"2021-12-19T16:12:12.687597Z","shell.execute_reply.started":"2021-12-19T16:12:12.677223Z","shell.execute_reply":"2021-12-19T16:12:12.686781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data distributions","metadata":{}},{"cell_type":"code","source":"def rotate(x, angle):\n    x = x + angle\n    x = x - (x + np.pi) // (2 * np.pi) * 2 * np.pi\n    return x\n\nplt.figure(figsize=(15,6))\nsns.distplot(reduce(lambda a, b: a + b, [[rotate(c['roll'], np.pi) for c in str2coords(s)] for s in train['PredictionString']]));\nplt.xlabel('roll rotated by pi')\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-19T16:12:12.689313Z","iopub.execute_input":"2021-12-19T16:12:12.690012Z","iopub.status.idle":"2021-12-19T16:12:14.485498Z","shell.execute_reply.started":"2021-12-19T16:12:12.689977Z","shell.execute_reply":"2021-12-19T16:12:14.484787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2D Visualization","metadata":{}},{"cell_type":"code","source":"def get_img_coords(s):\n    '''\n    Input is a PredictionString (e.g. from train dataframe)\n    Output is two arrays:\n        xs: x coordinates in the image\n        ys: y coordinates in the image\n    '''\n    coords = str2coords(s)\n    xs = [c['x'] for c in coords]\n    ys = [c['y'] for c in coords]\n    zs = [c['z'] for c in coords]\n    P = np.array(list(zip(xs, ys, zs))).T\n    img_p = np.dot(camera_matrix, P).T\n    img_p[:, 0] /= img_p[:, 2]\n    img_p[:, 1] /= img_p[:, 2]\n    img_xs = img_p[:, 0]\n    img_ys = img_p[:, 1]\n    img_zs = img_p[:, 2] # z = Distance from the camera\n    return img_xs, img_ys\n\nplt.figure(figsize=(14,14))\nplt.imshow(imread(PATH + 'train_images/' + train['ImageId'][2217] + '.jpg'))\nplt.scatter(*get_img_coords(train['PredictionString'][2217]), color='red', s=100);","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-19T16:12:14.487086Z","iopub.execute_input":"2021-12-19T16:12:14.487567Z","iopub.status.idle":"2021-12-19T16:12:16.617139Z","shell.execute_reply.started":"2021-12-19T16:12:14.487529Z","shell.execute_reply":"2021-12-19T16:12:16.616559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xs, ys = [], []\n\nfor ps in train['PredictionString']:\n    x, y = get_img_coords(ps)\n    xs += list(x)\n    ys += list(y)\n\nplt.figure(figsize=(18,18))\nplt.imshow(imread(PATH + 'train_images/' + train['ImageId'][2217] + '.jpg'), alpha=0.3)\nplt.scatter(xs, ys, color='red', s=10, alpha=0.2);","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-19T16:12:16.618236Z","iopub.execute_input":"2021-12-19T16:12:16.618825Z","iopub.status.idle":"2021-12-19T16:12:19.783178Z","shell.execute_reply.started":"2021-12-19T16:12:16.618787Z","shell.execute_reply":"2021-12-19T16:12:19.782559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3D Visualization\nUsed code from https://www.kaggle.com/zstusnoopy/visualize-the-location-and-3d-bounding-box-of-car, but made it one function","metadata":{}},{"cell_type":"code","source":"from math import sin, cos\n\n# convert euler angle to rotation matrix\ndef euler_to_Rot(yaw, pitch, roll):\n    Y = np.array([[cos(yaw), 0, sin(yaw)],\n                  [0, 1, 0],\n                  [-sin(yaw), 0, cos(yaw)]])\n    P = np.array([[1, 0, 0],\n                  [0, cos(pitch), -sin(pitch)],\n                  [0, sin(pitch), cos(pitch)]])\n    R = np.array([[cos(roll), -sin(roll), 0],\n                  [sin(roll), cos(roll), 0],\n                  [0, 0, 1]])\n    return np.dot(Y, np.dot(P, R))","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-19T16:12:19.786336Z","iopub.execute_input":"2021-12-19T16:12:19.78678Z","iopub.status.idle":"2021-12-19T16:12:19.794817Z","shell.execute_reply.started":"2021-12-19T16:12:19.786744Z","shell.execute_reply":"2021-12-19T16:12:19.794092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def draw_line(image, points):\n    color = (255, 0, 0)\n    cv2.line(image, tuple(points[0][:2]), tuple(points[3][:2]), color, 16)\n    cv2.line(image, tuple(points[0][:2]), tuple(points[1][:2]), color, 16)\n    cv2.line(image, tuple(points[1][:2]), tuple(points[2][:2]), color, 16)\n    cv2.line(image, tuple(points[2][:2]), tuple(points[3][:2]), color, 16)\n    return image\n\n\ndef draw_points(image, points):\n    for (p_x, p_y, p_z) in points:\n        cv2.circle(image, (p_x, p_y), int(1000 / p_z), (0, 255, 0), -1)\n#         if p_x > image.shape[1] or p_y > image.shape[0]:\n#             print('Point', p_x, p_y, 'is out of image with shape', image.shape)\n    return image","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-19T16:12:19.796201Z","iopub.execute_input":"2021-12-19T16:12:19.796639Z","iopub.status.idle":"2021-12-19T16:12:19.808529Z","shell.execute_reply.started":"2021-12-19T16:12:19.796594Z","shell.execute_reply":"2021-12-19T16:12:19.807837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def visualize(img, coords):\n    # You will also need functions from the previous cells\n    x_l = 1.02\n    y_l = 0.80\n    z_l = 2.31\n    \n    img = img.copy()\n    for point in coords:\n        # Get values\n        x, y, z = point['x'], point['y'], point['z']\n        yaw, pitch, roll = -point['pitch'], -point['yaw'], -point['roll']\n        # Math\n        Rt = np.eye(4)\n        t = np.array([x, y, z])\n        Rt[:3, 3] = t\n        Rt[:3, :3] = euler_to_Rot(yaw, pitch, roll).T\n        Rt = Rt[:3, :]\n        P = np.array([[x_l, -y_l, -z_l, 1],\n                      [x_l, -y_l, z_l, 1],\n                      [-x_l, -y_l, z_l, 1],\n                      [-x_l, -y_l, -z_l, 1],\n                      [0, 0, 0, 1]]).T\n        img_cor_points = np.dot(camera_matrix, np.dot(Rt, P))\n        img_cor_points = img_cor_points.T\n        img_cor_points[:, 0] /= img_cor_points[:, 2]\n        img_cor_points[:, 1] /= img_cor_points[:, 2]\n        img_cor_points = img_cor_points.astype(int)\n        # Drawing\n        img = draw_line(img, img_cor_points)\n        img = draw_points(img, img_cor_points[-1:])\n    \n    return img","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-19T16:12:19.809945Z","iopub.execute_input":"2021-12-19T16:12:19.810192Z","iopub.status.idle":"2021-12-19T16:12:19.820824Z","shell.execute_reply.started":"2021-12-19T16:12:19.810154Z","shell.execute_reply":"2021-12-19T16:12:19.819847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Image preprocessing","metadata":{}},{"cell_type":"code","source":"IMG_WIDTH = 1600\nIMG_HEIGHT = 700\nMODEL_SCALE = 8\n\ndef _regr_preprocess(regr_dict):\n    for name in ['x', 'y', 'z']:\n        regr_dict[name] = regr_dict[name] / 100\n    regr_dict['roll'] = rotate(regr_dict['roll'], np.pi)\n    regr_dict['pitch_sin'] = sin(regr_dict['pitch'])\n    regr_dict['pitch_cos'] = cos(regr_dict['pitch'])\n    regr_dict.pop('pitch')\n    regr_dict.pop('id')\n    return regr_dict\n\ndef _regr_back(regr_dict):\n    for name in ['x', 'y', 'z']:\n        regr_dict[name] = regr_dict[name] * 100\n    regr_dict['roll'] = rotate(regr_dict['roll'], -np.pi)\n    \n    pitch_sin = regr_dict['pitch_sin'] / np.sqrt(regr_dict['pitch_sin']**2 + regr_dict['pitch_cos']**2)\n    pitch_cos = regr_dict['pitch_cos'] / np.sqrt(regr_dict['pitch_sin']**2 + regr_dict['pitch_cos']**2)\n    regr_dict['pitch'] = np.arccos(pitch_cos) * np.sign(pitch_sin)\n    return regr_dict\n\ndef preprocess_image(img):\n    img = img[img.shape[0] // 2:]\n    bg = np.ones_like(img) * img.mean(1, keepdims=True).astype(img.dtype)\n    bg = bg[:, :img.shape[1] // 4]\n    img = np.concatenate([bg, img, bg], 1)\n    img = cv2.resize(img, (IMG_WIDTH, IMG_HEIGHT))\n    return (img / 255).astype('float32')\n\ndef get_mask_and_regr(img, labels):\n    mask = np.zeros([IMG_HEIGHT // MODEL_SCALE, IMG_WIDTH // MODEL_SCALE], dtype='float32')\n    regr_names = ['x', 'y', 'z', 'yaw', 'pitch', 'roll']\n    regr = np.zeros([IMG_HEIGHT // MODEL_SCALE, IMG_WIDTH // MODEL_SCALE, 7], dtype='float32')\n    coords = str2coords(labels)\n    xs, ys = get_img_coords(labels)\n    for x, y, regr_dict in zip(xs, ys, coords):\n        x, y = y, x\n        x = (x - img.shape[0] // 2) * IMG_HEIGHT / (img.shape[0] // 2) / MODEL_SCALE\n        x = np.round(x).astype('int')\n        y = (y + img.shape[1] // 4) * IMG_WIDTH / (img.shape[1] * 1.5) / MODEL_SCALE\n        y = np.round(y).astype('int')\n        if x >= 0 and x < IMG_HEIGHT // MODEL_SCALE and y >= 0 and y < IMG_WIDTH // MODEL_SCALE:\n            mask[x, y] = 1\n            regr_dict = _regr_preprocess(regr_dict)\n            regr[x, y] = [regr_dict[n] for n in sorted(regr_dict)]\n    return mask, regr","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-19T16:12:19.822292Z","iopub.execute_input":"2021-12-19T16:12:19.822755Z","iopub.status.idle":"2021-12-19T16:12:19.840475Z","shell.execute_reply.started":"2021-12-19T16:12:19.8227Z","shell.execute_reply":"2021-12-19T16:12:19.839581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img0 = imread(PATH + 'train_images/' + train['ImageId'][0] + '.jpg')\nimg = preprocess_image(img0)\n\nmask, regr = get_mask_and_regr(img0, train['PredictionString'][0])\n\nprint('img.shape', img.shape, 'std:', np.std(img))\nprint('mask.shape', mask.shape, 'std:', np.std(mask))\nprint('regr.shape', regr.shape, 'std:', np.std(regr))\n\nplt.figure(figsize=(16,16))\nplt.title('Processed image')\nplt.imshow(img)\nplt.show()\n\nplt.figure(figsize=(16,16))\nplt.title('Detection Mask')\nplt.imshow(mask)\nplt.show()\n\nplt.figure(figsize=(16,16))\nplt.title('Yaw values')\nplt.imshow(regr[:,:,-2])\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-19T16:12:19.841709Z","iopub.execute_input":"2021-12-19T16:12:19.842106Z","iopub.status.idle":"2021-12-19T16:12:21.093352Z","shell.execute_reply.started":"2021-12-19T16:12:19.84207Z","shell.execute_reply":"2021-12-19T16:12:21.092605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# PyTorch Dataset","metadata":{}},{"cell_type":"code","source":"class CarDataset(Dataset):\n    \"\"\"Car dataset.\"\"\"\n\n    def __init__(self, dataframe, root_dir, training=True, transform=None):\n        self.df = dataframe\n        self.root_dir = root_dir\n        self.transform = transform\n        self.training = training\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        if torch.is_tensor(idx):\n            idx = idx.tolist()\n        \n        # Get image name\n        idx, labels = self.df.values[idx]\n        img_name = self.root_dir.format(idx)\n        \n        # Read image\n        img0 = imread(img_name, True)\n        img = preprocess_image(img0)\n        img = np.rollaxis(img, 2, 0)\n        \n        # Get mask and regression maps\n        if self.training:\n            mask, regr = get_mask_and_regr(img0, labels)\n            regr = np.rollaxis(regr, 2, 0)\n        else:\n            mask, regr = 0, 0\n        \n        return [img, mask, regr]","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-12-19T16:12:21.094744Z","iopub.execute_input":"2021-12-19T16:12:21.094987Z","iopub.status.idle":"2021-12-19T16:12:21.104326Z","shell.execute_reply.started":"2021-12-19T16:12:21.094953Z","shell.execute_reply":"2021-12-19T16:12:21.103599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_images_dir = PATH + 'train_images/{}.jpg'\ntest_images_dir = PATH + 'test_images/{}.jpg'\n\ndf_train, df_dev = train_test_split(train, test_size=0.1, random_state=63)\ndf_test = test\n\n# Create dataset objects\ntrain_dataset = CarDataset(df_train, train_images_dir)\ndev_dataset = CarDataset(df_dev, train_images_dir)\ntest_dataset = CarDataset(df_test, test_images_dir)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-19T16:12:21.105417Z","iopub.execute_input":"2021-12-19T16:12:21.105723Z","iopub.status.idle":"2021-12-19T16:12:21.115858Z","shell.execute_reply.started":"2021-12-19T16:12:21.105685Z","shell.execute_reply":"2021-12-19T16:12:21.115179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Show some generated examples","metadata":{}},{"cell_type":"code","source":"img, mask, regr = train_dataset[0]\n\nplt.figure(figsize=(16,16))\nplt.imshow(np.rollaxis(img, 0, 3))\nplt.show()\n\nplt.figure(figsize=(16,16))\nplt.imshow(mask)\nplt.show()\n\nplt.figure(figsize=(16,16))\nplt.imshow(regr[-2])\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-19T16:12:21.117308Z","iopub.execute_input":"2021-12-19T16:12:21.117612Z","iopub.status.idle":"2021-12-19T16:12:22.296356Z","shell.execute_reply.started":"2021-12-19T16:12:21.117577Z","shell.execute_reply":"2021-12-19T16:12:22.295581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 1\n\n# Create data generators - they will produce batches\ntrain_loader = DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=0)\ndev_loader = DataLoader(dataset=dev_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=0)\ntest_loader = DataLoader(dataset=test_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-19T16:12:22.297601Z","iopub.execute_input":"2021-12-19T16:12:22.297858Z","iopub.status.idle":"2021-12-19T16:12:22.305422Z","shell.execute_reply.started":"2021-12-19T16:12:22.297825Z","shell.execute_reply":"2021-12-19T16:12:22.304638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# PyTorch Model","metadata":{}},{"cell_type":"code","source":"class double_conv(nn.Module):\n    '''(conv => BN => ReLU) * 2'''\n    def __init__(self, in_ch, out_ch):\n        super(double_conv, self).__init__()\n        self.conv = nn.Sequential(\n            nn.Conv2d(in_ch, out_ch, 3, padding=1),\n            nn.BatchNorm2d(out_ch),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(out_ch, out_ch, 3, padding=1),\n            nn.BatchNorm2d(out_ch),\n            nn.ReLU(inplace=True)\n        )\n\n    def forward(self, x):\n        x = self.conv(x)\n        return x\n\nclass up(nn.Module):\n    def __init__(self, in_ch, out_ch, bilinear=True):\n        super(up, self).__init__()\n\n        #  would be a nice idea if the upsampling could be learned too,\n        #  but my machine do not have enough memory to handle all those weights\n        if bilinear:\n            self.up = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True)\n        else:\n            self.up = nn.ConvTranspose2d(in_ch//2, in_ch//2, 2, stride=2)\n\n        self.conv = double_conv(in_ch, out_ch)\n\n    def forward(self, x1, x2=None):\n        x1 = self.up(x1)\n        \n        # input is CHW\n        diffY = x2.size()[2] - x1.size()[2]\n        diffX = x2.size()[3] - x1.size()[3]\n\n        x1 = F.pad(x1, (diffX // 2, diffX - diffX//2,\n                        diffY // 2, diffY - diffY//2))\n        \n        # for padding issues, see \n        # https://github.com/HaiyongJiang/U-Net-Pytorch-Unstructured-Buggy/commit/0e854509c2cea854e247a9c615f175f76fbb2e3a\n        # https://github.com/xiaopeng-liao/Pytorch-UNet/commit/8ebac70e633bac59fc22bb5195e513d5832fb3bd\n        \n        if x2 is not None:\n            x = torch.cat([x2, x1], dim=1)\n        else:\n            x = x1\n        x = self.conv(x)\n        return x\n\ndef get_mesh(batch_size, shape_x, shape_y):\n    mg_x, mg_y = np.meshgrid(np.linspace(0, 1, shape_y), np.linspace(0, 1, shape_x))\n    mg_x = np.tile(mg_x[None, None, :, :], [batch_size, 1, 1, 1]).astype('float32')\n    mg_y = np.tile(mg_y[None, None, :, :], [batch_size, 1, 1, 1]).astype('float32')\n    mesh = torch.cat([torch.tensor(mg_x).to(device), torch.tensor(mg_y).to(device)], 1)\n    return mesh","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-19T16:12:22.306892Z","iopub.execute_input":"2021-12-19T16:12:22.307294Z","iopub.status.idle":"2021-12-19T16:12:22.323442Z","shell.execute_reply.started":"2021-12-19T16:12:22.307258Z","shell.execute_reply":"2021-12-19T16:12:22.322776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**ResneXt**","metadata":{}},{"cell_type":"code","source":"\n\nmodel_urls = {\n    'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth',\n    'resnext50_32x4d': 'https://download.pytorch.org/models/resnext50_32x4d-7cdf4587.pth',\n    'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth',\n    'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth',\n    'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth',\n    'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth',\n    'densenet201': 'https://download.pytorch.org/models/densenet201-c1103571.pth',\n    'inceptionresnetv2': 'http://data.lip6.fr/cadene/pretrainedmodels/inceptionresnetv2-520b38e4.pth',\n}\n\n\ndef conv3x3(in_planes, out_planes, stride=1):\n    \"\"\"3x3 convolution with padding\"\"\"\n    return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,\n                     padding=1, bias=False)\n\n\ndef conv1x1(in_planes, out_planes, stride=1):\n    \"\"\"1x1 convolution\"\"\"\n    return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False)\n\n\nclass BasicBlock(nn.Module):\n    expansion = 1\n\n    def __init__(self, inplanes, planes, stride=1):\n        super(BasicBlock, self).__init__()\n\n        self.conv1 = conv3x3(inplanes, planes, stride)\n        self.bn1 = nn.GroupNorm(16, planes)\n\n        self.conv2 = conv3x3(planes, planes)\n        self.bn2 = nn.GroupNorm(16, planes)\n\n        if stride != 1 or inplanes != planes:\n            self.downsample = nn.Sequential(\n                conv1x1(inplanes, planes, stride), nn.GroupNorm(16, planes))\n        else:\n            self.downsample = None\n\n\n    def forward(self, x):\n        identity = x\n\n        out = F.relu(self.bn1(self.conv1(x)), inplace=True)\n        out = self.bn2(self.conv2(out))\n\n        if self.downsample is not None:\n            identity = self.downsample(x)\n\n        out += identity\n        out = F.relu(out, inplace=True)\n\n        return out\n\n\nclass Bottleneck(nn.Module):\n    expansion = 4\n\n    def __init__(self, inplanes, planes, stride=1):\n        super(Bottleneck, self).__init__()\n        self.conv1 = conv1x1(inplanes, planes)\n        self.bn1 = nn.GroupNorm(16, planes)\n        self.conv2 = conv3x3(planes, planes, stride)\n        self.bn2 = nn.GroupNorm(16, planes)\n        self.conv3 = conv1x1(planes, planes * self.expansion)\n        self.bn3 = nn.GroupNorm(16, planes * self.expansion)\n\n        if stride != 1 or inplanes != planes * self.expansion:\n            self.downsample = nn.Sequential(\n                conv1x1(inplanes, planes * self.expansion, stride), \n                nn.GroupNorm(16, planes * self.expansion))\n        else:\n            self.downsample = None\n\n    def forward(self, x):\n        identity = x\n\n        out = F.relu(self.bn1(self.conv1(x)), inplace=True)\n        out = F.relu(self.bn2(self.conv2(out)), inplace=True)\n        out = self.bn3(self.conv3(out))\n \n        if self.downsample is not None:\n            identity = self.downsample(x)\n\n        out += identity\n        out = F.relu(out)\n\n        return out\n\n\nclass ResNetFeatures(nn.Module):\n\n    def __init__(self, block, layers, num_classes=1000, zero_init_residual=False):\n        super(ResNetFeatures, self).__init__()\n        self.inplanes = 64\n        self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3,\n                               bias=False)\n        self.bn1 = nn.GroupNorm(16, 64)\n\n        self.layer1 = self._make_layer(block, 64, layers[0])\n        self.layer2 = self._make_layer(block, 128, layers[1], stride=2)\n        self.layer3 = self._make_layer(block, 256, layers[2], stride=2)\n        self.layer4 = self._make_layer(block, 512, layers[3], stride=2)\n\n\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')\n            elif isinstance(m, nn.BatchNorm2d):\n                nn.init.constant_(m.weight, 1)\n                nn.init.constant_(m.bias, 0)\n\n        # Zero-initialize the last BN in each residual branch,\n        # so that the residual branch starts with zeros, and each residual block behaves like an identity.\n        # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677\n        if zero_init_residual:\n            for m in self.modules():\n                if isinstance(m, Bottleneck):\n                    nn.init.constant_(m.bn3.weight, 0)\n                elif isinstance(m, BasicBlock):\n                    nn.init.constant_(m.bn2.weight, 0)\n\n    def _make_layer(self, block, planes, blocks, stride=1):\n\n        layers = []\n        layers.append(block(self.inplanes, planes, stride))\n        self.inplanes = planes * block.expansion\n        for _ in range(1, blocks):\n            layers.append(block(self.inplanes, planes))\n\n        return nn.Sequential(*layers)\n\n\n    def forward(self, x):\n        conv1 = F.relu(self.bn1(self.conv1(x)), inplace=True)\n        conv1 = F.max_pool2d(conv1, 3, stride=2, padding=1)\n\n        feats4 = self.layer1(conv1)\n        feats8 = self.layer2(feats4)\n        feats16 = self.layer3(feats8)\n        feats32 = self.layer4(feats16)\n\n        return feats8, feats16, feats32\n\n\n\ndef resnet18(pretrained=False, **kwargs):\n    \"\"\"Constructs a ResNet-18 model.\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n    \"\"\"\n    model = ResNetFeatures(BasicBlock, [2, 2, 2, 2], **kwargs)\n    if pretrained:\n        _load_pretrained(model, model_zoo.load_url(model_urls['resnet18']))\n    return model\n\n\ndef  resnext50(pretrained=False, **kwargs):\n    \"\"\"Constructs a ResNet-18 model.\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n    \"\"\"\n    model = ResNetFeatures(BasicBlock, [2, 2, 2, 2], **kwargs)\n    if pretrained:\n        _load_pretrained(model, model_zoo.load_url(model_urls['resnext50_32x4d']))\n    return model\n\n\ndef resnet34(pretrained=False, **kwargs):\n    \"\"\"Constructs a ResNet-34 model.\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n    \"\"\"\n    model = ResNetFeatures(BasicBlock, [3, 4, 6, 3], **kwargs)\n    if pretrained:\n        _load_pretrained(model, model_zoo.load_url(model_urls['resnet34']))\n    return model\n\ndef inceptionresnetv2(pretrained=False, **kwargs):\n    \"\"\"Constructs a inceptionresnetv2.\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n    \"\"\"\n    model = ResNetFeatures(BasicBlock, [3, 4, 6, 3], **kwargs)\n    if pretrained:\n        _load_pretrained(model, model_zoo.load_url(model_urls['inceptionresnetv2']))\n    return model\n\ndef densenet201(pretrained=False, **kwargs):\n    \"\"\"Constructs a densenet201 model.\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n    \"\"\"\n    model = ResNetFeatures(BasicBlock, [3, 4, 6, 3], **kwargs)\n    if pretrained:\n        _load_pretrained(model, model_zoo.load_url(model_urls['densenet201']))\n    return model\n\n\ndef _load_pretrained(model, pretrained):\n    model_dict = model.state_dict()\n    pretrained = {k : v for k, v in pretrained.items() if k in model_dict}\n    model_dict.update(pretrained)\n    model.load_state_dict(model_dict)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-19T16:12:22.32486Z","iopub.execute_input":"2021-12-19T16:12:22.325151Z","iopub.status.idle":"2021-12-19T16:12:22.362167Z","shell.execute_reply.started":"2021-12-19T16:12:22.325118Z","shell.execute_reply":"2021-12-19T16:12:22.361422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Why resnext50 instead of resnet?**","metadata":{}},{"cell_type":"markdown","source":"![](https://pytorch.org/assets/images/resnext.png)","metadata":{}},{"cell_type":"markdown","source":"So a resnext_32*4d represents network with 4 bottleneck [one block in the above diagram] layers, and each layer having cardinality of 32.","metadata":{}},{"cell_type":"markdown","source":"**ResNeXt Architecture Review**","metadata":{}},{"cell_type":"markdown","source":"ResNeXt won 2nd place in ILSVRC 2016 classification task and also showed performance improvements in Coco detection and ImageNet-5k set than their ResNet counter part.\n\nThis is a very simple paper to read which introduces a new term called “cardinality”. The paper simply explains this term and make use of it in ResNet networks and does various ablation studies.\n\nThe paper made several attempts to describe the complexity of Inception networks and why ResNeXt architecture is simple. I m not going to do this here as it would require the reader to understand Inception networks. I will just talk about the architecture here.\n\n![](https://miro.medium.com/max/965/1*mdiQTfovOXKnqzfj727b9Q.png)","metadata":{}},{"cell_type":"markdown","source":"* The above diagram distinguishes between a simple ResNet block and ResNeXt blog.\n* It follows a split-transform-aggregate strategy.\n* The number of paths inside the ResNeXt block is defined as cardinality. In the above diagram C=32\n* All the paths contain the same topology.\n* Instead of having high depth and width, Having high cardinality helps in decreasing validation error.\n* ResNeXt tries embed more subspaces compared to its ResNet counter part.\n* Both the architectures have different width. Layer-1 in ResNet has one conv layer with 64 width, while layer-1 in ResNext has 32 different conv layers with 4 width (32*4 width). Despite the larger overall width in ResNeXt, both the architectures have the same number of parameters(~70k) (ResNet 256*64+3*3*64*64+64*26) (ResNeXt C*(256*d+3*3*d*d+d*256), with C=32 and d=4)","metadata":{}},{"cell_type":"markdown","source":"**Cardinality vs width: with C increasing from 1 to 32, we can clearly see a descrease in top-1 % error rate. Therefore, Increasing the C by decreasing the width has improved the performance of the model.**","metadata":{}},{"cell_type":"markdown","source":"![](https://miro.medium.com/max/614/1*VmsRr6jis0Sg7asAfAT1LQ.png)","metadata":{}},{"cell_type":"markdown","source":"**Increasing Cardinality vs Deeper/Wider:**","metadata":{}},{"cell_type":"markdown","source":"Basically 3 cases were studied. 1) Increasing the number of layers to 200 from 101. 2) Going wider by increasing the bottleneck width. 3) Increasing cardinality by doubling C.\n\nThey have observed that increasing the C gave better performance improvements. below are the results.","metadata":{}},{"cell_type":"markdown","source":"![](https://miro.medium.com/max/638/1*t4CD5LCb-4TtFgXry-Sndg.png)","metadata":{}},{"cell_type":"markdown","source":"**An ensemble of different ResNeXt architecture gave a top-5 error rate of 3.03% thus winning second position in ILSVRC competition.\nThe architecture is simple in design compared to Inception modules.**","metadata":{}},{"cell_type":"markdown","source":"** Inception-ResNet-V2 (2016)**","metadata":{}},{"cell_type":"markdown","source":"![](https://miro.medium.com/max/3750/1*xpb6QFQ4IknSmxmgai8w-Q.png)","metadata":{}},{"cell_type":"code","source":"\nbase_model = resnext50(pretrained=False)\nbase_model","metadata":{"execution":{"iopub.status.busy":"2021-12-19T16:12:22.363217Z","iopub.execute_input":"2021-12-19T16:12:22.363557Z","iopub.status.idle":"2021-12-19T16:12:22.630391Z","shell.execute_reply.started":"2021-12-19T16:12:22.363521Z","shell.execute_reply":"2021-12-19T16:12:22.629646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CentResnet(nn.Module):\n    '''Mixture of previous classes'''\n    def __init__(self, n_classes):\n        super(CentResnet, self).__init__()\n        self.base_model = base_model\n        \n        # Lateral layers convert resnet outputs to a common feature size\n        self.lat8 = nn.Conv2d(128, 256, 1)\n        self.lat16 = nn.Conv2d(256, 256, 1)\n        self.lat32 = nn.Conv2d(512, 256, 1)\n        self.bn8 = nn.GroupNorm(16, 256)\n        self.bn16 = nn.GroupNorm(16, 256)\n        self.bn32 = nn.GroupNorm(16, 256)\n\n       \n        self.conv0 = double_conv(5, 64)\n        self.conv1 = double_conv(64, 128)\n        self.conv2 = double_conv(128, 512)\n        self.conv3 = double_conv(512, 1024)\n        \n        self.mp = nn.MaxPool2d(2)\n        \n        self.up1 = up(1282 , 512) #+ 1024\n        self.up2 = up(512 + 512, 256)\n        self.outc = nn.Conv2d(256, n_classes, 1)\n        \n    \n    def forward(self, x):\n        batch_size = x.shape[0]\n        mesh1 = get_mesh(batch_size, x.shape[2], x.shape[3])\n        x0 = torch.cat([x, mesh1], 1)\n        x1 = self.mp(self.conv0(x0))\n        x2 = self.mp(self.conv1(x1))\n        x3 = self.mp(self.conv2(x2))\n        x4 = self.mp(self.conv3(x3))\n        \n        #feats = self.base_model.extract_features(x)\n                # Run frontend network\n        feats8, feats16, feats32 = self.base_model(x)\n        lat8 = F.relu(self.bn8(self.lat8(feats8)))\n        lat16 = F.relu(self.bn16(self.lat16(feats16)))\n        lat32 = F.relu(self.bn32(self.lat32(feats32)))\n        \n        # Add positional info\n        mesh2 = get_mesh(batch_size, lat32.shape[2], lat32.shape[3])\n        feats = torch.cat([lat32, mesh2], 1)\n        #print(feats.shape)\n        #print (x4.shape)\n        x = self.up1(feats, x4)\n        x = self.up2(x, x3)\n        x = self.outc(x)\n        return x","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-19T16:12:22.631924Z","iopub.execute_input":"2021-12-19T16:12:22.632232Z","iopub.status.idle":"2021-12-19T16:12:22.652585Z","shell.execute_reply.started":"2021-12-19T16:12:22.632191Z","shell.execute_reply":"2021-12-19T16:12:22.651231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Radam","metadata":{}},{"cell_type":"code","source":"import math\nimport torch\nfrom torch.optim.optimizer import Optimizer, required\n\nclass RAdam(Optimizer):\n\n    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0):\n        defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay)\n        self.buffer = [[None, None, None] for ind in range(10)]\n        super(RAdam, self).__init__(params, defaults)\n\n    def __setstate__(self, state):\n        super(RAdam, self).__setstate__(state)\n\n    def step(self, closure=None):\n\n        loss = None\n        if closure is not None:\n            loss = closure()\n\n        for group in self.param_groups:\n\n            for p in group['params']:\n                if p.grad is None:\n                    continue\n                grad = p.grad.data.float()\n                if grad.is_sparse:\n                    raise RuntimeError('RAdam does not support sparse gradients')\n\n                p_data_fp32 = p.data.float()\n\n                state = self.state[p]\n\n                if len(state) == 0:\n                    state['step'] = 0\n                    state['exp_avg'] = torch.zeros_like(p_data_fp32)\n                    state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)\n                else:\n                    state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)\n                    state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32)\n\n                exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']\n                beta1, beta2 = group['betas']\n\n                exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad)\n                exp_avg.mul_(beta1).add_(1 - beta1, grad)\n\n                state['step'] += 1\n                buffered = self.buffer[int(state['step'] % 10)]\n                if state['step'] == buffered[0]:\n                    N_sma, step_size = buffered[1], buffered[2]\n                else:\n                    buffered[0] = state['step']\n                    beta2_t = beta2 ** state['step']\n                    N_sma_max = 2 / (1 - beta2) - 1\n                    N_sma = N_sma_max - 2 * state['step'] * beta2_t / (1 - beta2_t)\n                    buffered[1] = N_sma\n\n                    # more conservative since it's an approximated value\n                    if N_sma >= 5:\n                        step_size = group['lr'] * math.sqrt((1 - beta2_t) * (N_sma - 4) / (N_sma_max - 4) * (N_sma - 2) / N_sma * N_sma_max / (N_sma_max - 2)) / (1 - beta1 ** state['step'])\n                    else:\n                        step_size = group['lr'] / (1 - beta1 ** state['step'])\n                    buffered[2] = step_size\n\n                if group['weight_decay'] != 0:\n                    p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32)\n\n                # more conservative since it's an approximated value\n                if N_sma >= 5:            \n                    denom = exp_avg_sq.sqrt().add_(group['eps'])\n                    p_data_fp32.addcdiv_(-step_size, exp_avg, denom)\n                else:\n                    p_data_fp32.add_(-step_size, exp_avg)\n\n                p.data.copy_(p_data_fp32)\n\n        return loss","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-19T16:12:22.654297Z","iopub.execute_input":"2021-12-19T16:12:22.654549Z","iopub.status.idle":"2021-12-19T16:12:22.686657Z","shell.execute_reply.started":"2021-12-19T16:12:22.654509Z","shell.execute_reply":"2021-12-19T16:12:22.685777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Gets the GPU if there is one, otherwise the cpu\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(device)\n\nn_epochs = 2 #original 10, 8, 7\n\nmodel = CentResnet(8).to(device)\noptimizer = optim.AdamW(model.parameters(), lr=0.001)\n#optimizer =  RAdam(model.parameters(), lr = 0.001)\nexp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=max(n_epochs, 10) * len(train_loader) // 3, gamma=0.1)","metadata":{"execution":{"iopub.status.busy":"2021-12-19T16:12:22.691804Z","iopub.execute_input":"2021-12-19T16:12:22.69451Z","iopub.status.idle":"2021-12-19T16:12:25.888883Z","shell.execute_reply.started":"2021-12-19T16:12:22.694472Z","shell.execute_reply":"2021-12-19T16:12:25.88814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_batch = torch.randn((1,3,700,1600))","metadata":{"execution":{"iopub.status.busy":"2021-12-19T16:12:25.890039Z","iopub.execute_input":"2021-12-19T16:12:25.890307Z","iopub.status.idle":"2021-12-19T16:12:25.922088Z","shell.execute_reply.started":"2021-12-19T16:12:25.890273Z","shell.execute_reply":"2021-12-19T16:12:25.92146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = model(img_batch.to(device))","metadata":{"execution":{"iopub.status.busy":"2021-12-19T16:12:25.923165Z","iopub.execute_input":"2021-12-19T16:12:25.923426Z","iopub.status.idle":"2021-12-19T16:12:30.715466Z","shell.execute_reply.started":"2021-12-19T16:12:25.923391Z","shell.execute_reply":"2021-12-19T16:12:30.714766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"def criterion(prediction, mask, regr,weight=0.4, size_average=True):\n    # Binary mask loss\n    pred_mask = torch.sigmoid(prediction[:, 0])\n#     mask_loss = mask * (1 - pred_mask)**2 * torch.log(pred_mask + 1e-12) + (1 - mask) * pred_mask**2 * torch.log(1 - pred_mask + 1e-12)\n    mask_loss = mask * torch.log(pred_mask + 1e-12) + (1 - mask) * torch.log(1 - pred_mask + 1e-12)\n    mask_loss = -mask_loss.mean(0).sum()\n    \n    # Regression L1 loss\n    pred_regr = prediction[:, 1:]\n    regr_loss = (torch.abs(pred_regr - regr).sum(1) * mask).sum(1).sum(1) / mask.sum(1).sum(1)\n    regr_loss = regr_loss.mean(0)\n  \n    # Sum\n    loss = mask_loss +regr_loss\n    if not size_average:\n        loss *= prediction.shape[0]\n    return loss ,mask_loss , regr_loss","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-19T16:12:30.716619Z","iopub.execute_input":"2021-12-19T16:12:30.716896Z","iopub.status.idle":"2021-12-19T16:12:30.723368Z","shell.execute_reply.started":"2021-12-19T16:12:30.716861Z","shell.execute_reply":"2021-12-19T16:12:30.722679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Just for checking the shapes to manage our Unet\ni = 0\nfor batch_idx, (img_batch, mask_batch, regr_batch) in enumerate(tqdm(train_loader)):\n    print(img_batch.shape)\n    print(mask_batch.shape)\n    print(regr_batch.shape)\n    i+=1\n    if i>1:\n        break ","metadata":{"execution":{"iopub.status.busy":"2021-12-19T16:12:30.728262Z","iopub.execute_input":"2021-12-19T16:12:30.729111Z","iopub.status.idle":"2021-12-19T16:12:31.269687Z","shell.execute_reply.started":"2021-12-19T16:12:30.72907Z","shell.execute_reply":"2021-12-19T16:12:31.268991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(epoch, history=None):\n    model.train()\n    t = tqdm(train_loader)\n    for batch_idx, (img_batch, mask_batch, regr_batch) in enumerate(t):\n        img_batch = img_batch.to(device)\n        mask_batch = mask_batch.to(device)\n        regr_batch = regr_batch.to(device)\n        \n        optimizer.zero_grad()\n        output = model(img_batch)\n        if epoch < SWITCH_LOSS_EPOCH :\n            loss,mask_loss, regr_loss = criterion(output, mask_batch, regr_batch,1)\n        else:\n            loss,mask_loss, regr_loss = criterion(output, mask_batch, regr_batch,0.5)  \n        \n        t.set_description(f'train_loss (l={loss:.3f})(m={mask_loss:.2f}) (r={regr_loss:.4f}')\n        \n        if history is not None:\n            history.loc[epoch + batch_idx / len(train_loader), 'train_loss'] = loss.data.cpu().numpy()\n        \n        loss.backward()\n        \n        optimizer.step()\n        exp_lr_scheduler.step()\n\n    \n    print('Train Epoch: {} \\tLR: {:.6f}\\tLoss: {:.6f}\\tMaskLoss: {:.6f}\\tRegLoss: {:.6f}'.format(\n        epoch,\n        optimizer.state_dict()['param_groups'][0]['lr'],\n        loss.data,\n        mask_loss.data,\n        regr_loss.data))\n\ndef evaluate(epoch, history=None):\n    model.eval()\n    loss = 0\n    valid_loss = 0\n    valid_mask_loss = 0\n    valid_regr_loss = 0\n    with torch.no_grad():\n        for img_batch, mask_batch, regr_batch in dev_loader:\n            img_batch = img_batch.to(device)\n            mask_batch = mask_batch.to(device)\n            regr_batch = regr_batch.to(device)\n\n            output = model(img_batch)\n\n            if epoch < SWITCH_LOSS_EPOCH :\n                loss,mask_loss, regr_loss= criterion(output, mask_batch, regr_batch,1, size_average=False)\n                valid_loss += loss.data\n                valid_mask_loss += mask_loss.data\n                valid_regr_loss += regr_loss.data\n            else :\n                loss,mask_loss, regr_loss = criterion(output, mask_batch, regr_batch,0.5, size_average=False)\n                valid_loss += loss.data\n                valid_mask_loss += mask_loss.data\n                valid_regr_loss += regr_loss.data \n\n    \n    valid_loss /= len(dev_loader.dataset)\n    valid_mask_loss /= len(dev_loader.dataset)\n    valid_regr_loss /= len(dev_loader.dataset)\n    \n    if history is not None:\n        history.loc[epoch, 'dev_loss'] = valid_loss.cpu().numpy()\n        history.loc[epoch, 'mask_loss'] = valid_mask_loss.cpu().numpy()\n        history.loc[epoch, 'regr_loss'] = valid_regr_loss.cpu().numpy()\n\n    \n    print('Dev loss: {:.4f}'.format(valid_loss))","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-19T16:12:31.271292Z","iopub.execute_input":"2021-12-19T16:12:31.271855Z","iopub.status.idle":"2021-12-19T16:12:31.287174Z","shell.execute_reply.started":"2021-12-19T16:12:31.271811Z","shell.execute_reply":"2021-12-19T16:12:31.286423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nimport gc\n\nhistory = pd.DataFrame()\n\nfor epoch in range(n_epochs):\n    torch.cuda.empty_cache()\n    gc.collect()\n    train(epoch, history)\n    evaluate(epoch, history)","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-12-19T16:12:31.288188Z","iopub.execute_input":"2021-12-19T16:12:31.288502Z","iopub.status.idle":"2021-12-19T17:16:23.952866Z","shell.execute_reply.started":"2021-12-19T16:12:31.288468Z","shell.execute_reply":"2021-12-19T17:16:23.951757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.save(model.state_dict(), './resnext50.pth')\n\n\n","metadata":{"execution":{"iopub.status.busy":"2021-12-19T17:17:07.396477Z","iopub.execute_input":"2021-12-19T17:17:07.396726Z","iopub.status.idle":"2021-12-19T17:17:07.415276Z","shell.execute_reply.started":"2021-12-19T17:17:07.396697Z","shell.execute_reply":"2021-12-19T17:17:07.411798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history['train_loss'].iloc[100:].plot();","metadata":{"execution":{"iopub.status.busy":"2021-12-19T17:16:24.334046Z","iopub.status.idle":"2021-12-19T17:16:24.334879Z","shell.execute_reply.started":"2021-12-19T17:16:24.334624Z","shell.execute_reply":"2021-12-19T17:16:24.33465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series1 = history.dropna()['mask_loss']\nplt.plot(series1.index, series1 ,label = 'mask loss');\nseries2 = history.dropna()['regr_loss']\nplt.plot(series2.index, 30*series2,label = 'regr loss');\nseries3 = history.dropna()['dev_loss']\nplt.plot(series3.index, series3,label = 'dev loss');\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-19T17:16:24.336824Z","iopub.status.idle":"2021-12-19T17:16:24.33723Z","shell.execute_reply.started":"2021-12-19T17:16:24.337007Z","shell.execute_reply":"2021-12-19T17:16:24.337029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series = history.dropna()['dev_loss']\nplt.scatter(series.index, series);","metadata":{"execution":{"iopub.status.busy":"2021-12-19T17:16:24.339102Z","iopub.status.idle":"2021-12-19T17:16:24.339525Z","shell.execute_reply.started":"2021-12-19T17:16:24.33928Z","shell.execute_reply":"2021-12-19T17:16:24.339305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualize predictions","metadata":{}},{"cell_type":"code","source":"img, mask, regr = dev_dataset[0]\n\nplt.figure(figsize=(16,16))\nplt.title('Input image')\nplt.imshow(np.rollaxis(img, 0, 3))\nplt.show()\n\nplt.figure(figsize=(16,16))\nplt.title('Ground truth mask')\nplt.imshow(mask)\nplt.show()\n\noutput = model(torch.tensor(img[None]).to(device))\nlogits = output[0,0].data.cpu().numpy()\n\nplt.figure(figsize=(16,16))\nplt.title('Model predictions')\nplt.imshow(logits)\nplt.show()\n\nprint(logits)\nplt.figure(figsize=(16,16))\nplt.title('Model predictions thresholded')\nplt.imshow(logits > 0)\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-19T17:16:24.340638Z","iopub.status.idle":"2021-12-19T17:16:24.343039Z","shell.execute_reply.started":"2021-12-19T17:16:24.342795Z","shell.execute_reply":"2021-12-19T17:16:24.342822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Simple test of probabilities\nact = torch.nn.Sigmoid()\nlogtens = torch.from_numpy(logits)\nprobs = act(logtens)\nprobs = probs[probs>0]\nprint(probs)","metadata":{"execution":{"iopub.status.busy":"2021-12-19T17:16:24.344847Z","iopub.status.idle":"2021-12-19T17:16:24.345502Z","shell.execute_reply.started":"2021-12-19T17:16:24.345268Z","shell.execute_reply":"2021-12-19T17:16:24.345293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DISTANCE_THRESH_CLEAR = 2\n\ndef convert_3d_to_2d(x, y, z, fx = 2304.5479, fy = 2305.8757, cx = 1686.2379, cy = 1354.9849):\n    # stolen from https://www.kaggle.com/theshockwaverider/eda-visualization-baseline\n    return x * fx / z + cx, y * fy / z + cy\n\ndef optimize_xy(r, c, x0, y0, z0):\n    def distance_fn(xyz):\n        x, y, z = xyz\n        x, y = convert_3d_to_2d(x, y, z0)\n        y, x = x, y\n        x = (x - IMG_SHAPE[0] // 2) * IMG_HEIGHT / (IMG_SHAPE[0] // 2) / MODEL_SCALE\n        x = np.round(x).astype('int')\n        y = (y + IMG_SHAPE[1] // 4) * IMG_WIDTH / (IMG_SHAPE[1] * 1.5) / MODEL_SCALE\n        y = np.round(y).astype('int')\n        return (x-r)**2 + (y-c)**2\n    \n    res = minimize(distance_fn, [x0, y0, z0], method='Powell')\n    x_new, y_new, z_new = res.x\n    return x_new, y_new, z0\n\ndef clear_duplicates(coords):\n    for c1 in coords:\n        xyz1 = np.array([c1['x'], c1['y'], c1['z']])\n        for c2 in coords:\n            xyz2 = np.array([c2['x'], c2['y'], c2['z']])\n            distance = np.sqrt(((xyz1 - xyz2)**2).sum())\n            if distance < DISTANCE_THRESH_CLEAR:\n                if c1['confidence'] < c2['confidence']:\n                    c1['confidence'] = -1\n    return [c for c in coords if c['confidence'] > 0]\n\ndef extract_coords(prediction):\n    logits = prediction[0]\n    regr_output = prediction[1:]\n    points = np.argwhere(logits > 0)\n    col_names = sorted(['x', 'y', 'z', 'yaw', 'pitch_sin', 'pitch_cos', 'roll'])\n    coords = []\n    for r, c in points:\n        regr_dict = dict(zip(col_names, regr_output[:, r, c]))\n        coords.append(_regr_back(regr_dict))\n        coords[-1]['confidence'] = 1 / (1 + np.exp(-logits[r, c]))\n        coords[-1]['x'], coords[-1]['y'], coords[-1]['z'] = optimize_xy(r, c, coords[-1]['x'], coords[-1]['y'], coords[-1]['z'])\n    coords = clear_duplicates(coords)\n    return coords\n\ndef coords2str(coords, names=['yaw', 'pitch', 'roll', 'x', 'y', 'z', 'confidence']):\n    s = []\n    for c in coords:\n        for n in names:\n            s.append(str(c.get(n, 0)))\n    return ' '.join(s)","metadata":{"execution":{"iopub.status.busy":"2021-12-19T17:16:24.347694Z","iopub.status.idle":"2021-12-19T17:16:24.348114Z","shell.execute_reply.started":"2021-12-19T17:16:24.347886Z","shell.execute_reply":"2021-12-19T17:16:24.347908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.cuda.empty_cache()\ngc.collect()\n\nfor idx in range(4):\n    img, mask, regr = dev_dataset[idx]\n    \n    output = model(torch.tensor(img[None]).to(device)).data.cpu().numpy()\n    coords_pred = extract_coords(output[0])\n    coords_true = extract_coords(np.concatenate([mask[None], regr], 0))\n    \n    img = imread(train_images_dir.format(df_dev['ImageId'].iloc[idx]))\n    \n    fig, axes = plt.subplots(1, 2, figsize=(30,30))\n    axes[0].set_title('Ground truth')\n    axes[0].imshow(visualize(img, coords_true))\n    axes[1].set_title('Prediction')\n    axes[1].imshow(visualize(img, coords_pred))\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-19T17:16:24.350009Z","iopub.status.idle":"2021-12-19T17:16:24.350413Z","shell.execute_reply.started":"2021-12-19T17:16:24.350199Z","shell.execute_reply":"2021-12-19T17:16:24.350221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Make submission","metadata":{}},{"cell_type":"code","source":"# taken from kernel of @its7171 \nval_preds = []\n\ndev_loader = DataLoader(dataset=dev_dataset, batch_size=4, shuffle=False, num_workers=4)\n\nmodel.eval()\n\nfor img, _, _ in tqdm(dev_loader):\n    with torch.no_grad():\n        output = model(img.to(device))\n    output = output.data.cpu().numpy()\n    for out in output:\n        coords = extract_coords(out)\n        s = coords2str(coords)\n        val_preds.append(s)","metadata":{"execution":{"iopub.status.busy":"2021-12-19T17:16:24.352294Z","iopub.status.idle":"2021-12-19T17:16:24.352689Z","shell.execute_reply.started":"2021-12-19T17:16:24.352472Z","shell.execute_reply":"2021-12-19T17:16:24.352496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_dev['PredictionString'] = val_preds\ndf_dev.head()\ndf_dev.to_csv('val_predictions.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-12-19T17:16:24.354612Z","iopub.status.idle":"2021-12-19T17:16:24.355028Z","shell.execute_reply.started":"2021-12-19T17:16:24.354796Z","shell.execute_reply":"2021-12-19T17:16:24.354823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = []\n\ntest_loader = DataLoader(dataset=test_dataset, batch_size=2, shuffle=False, num_workers=4)\n\nmodel.eval()\n\nfor img, _, _ in tqdm(test_loader):\n    with torch.no_grad():\n        output = model(img.to(device))\n    output = output.data.cpu().numpy()\n    for out in output:\n        coords = extract_coords(out)\n        s = coords2str(coords)\n        predictions.append(s)","metadata":{"execution":{"iopub.status.busy":"2021-12-19T17:16:24.3569Z","iopub.status.idle":"2021-12-19T17:16:24.357308Z","shell.execute_reply.started":"2021-12-19T17:16:24.357093Z","shell.execute_reply":"2021-12-19T17:16:24.357115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv(PATH + 'sample_submission.csv')\ntest['PredictionString'] = predictions\ntest.to_csv('predictions.csv', index=False)\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-19T17:16:24.359205Z","iopub.status.idle":"2021-12-19T17:16:24.359605Z","shell.execute_reply.started":"2021-12-19T17:16:24.359388Z","shell.execute_reply":"2021-12-19T17:16:24.359409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef expand_df(df, PredictionStringCols):\n    df = df.dropna().copy()\n    df['NumCars'] = [int((x.count(' ')+1)/7) for x in df['PredictionString']]\n\n    image_id_expanded = [item for item, count in zip(df['ImageId'], df['NumCars']) for i in range(count)]\n    prediction_strings_expanded = df['PredictionString'].str.split(' ',expand = True).values.reshape(-1,7).astype(float)\n    prediction_strings_expanded = prediction_strings_expanded[~np.isnan(prediction_strings_expanded).all(axis=1)]\n    df = pd.DataFrame(\n        {\n            'ImageId': image_id_expanded,\n            PredictionStringCols[0]:prediction_strings_expanded[:,0],\n            PredictionStringCols[1]:prediction_strings_expanded[:,1],\n            PredictionStringCols[2]:prediction_strings_expanded[:,2],\n            PredictionStringCols[3]:prediction_strings_expanded[:,3],\n            PredictionStringCols[4]:prediction_strings_expanded[:,4],\n            PredictionStringCols[5]:prediction_strings_expanded[:,5],\n            PredictionStringCols[6]:prediction_strings_expanded[:,6]\n        })\n    return df\n\ndef str2coords(s, names):\n    coords = []\n    for l in np.array(s.split()).reshape([-1, 7]):\n        coords.append(dict(zip(names, l.astype('float'))))\n    return coords\n\ndef TranslationDistance(p,g, abs_dist = False):\n    dx = p['x'] - g['x']\n    dy = p['y'] - g['y']\n    dz = p['z'] - g['z']\n    diff0 = (g['x']**2 + g['y']**2 + g['z']**2)**0.5\n    diff1 = (dx**2 + dy**2 + dz**2)**0.5\n    if abs_dist:\n        diff = diff1\n    else:\n        diff = diff1/diff0\n    return diff\n\ndef RotationDistance(p, g):\n    true=[ g['pitch'] ,g['yaw'] ,g['roll'] ]\n    pred=[ p['pitch'] ,p['yaw'] ,p['roll'] ]\n    q1 = R.from_euler('xyz', true)\n    q2 = R.from_euler('xyz', pred)\n    diff = R.inv(q2) * q1\n    W = np.clip(diff.as_quat()[-1], -1., 1.)\n    \n    # in the official metrics code:\n    # https://www.kaggle.com/c/pku-autonomous-driving/overview/evaluation\n    #   return Object3D.RadianToDegree( Math.Acos(diff.W) )\n    # this code treat θ and θ+2π differntly.\n    # So this should be fixed as follows.\n    W = (acos(W)*360)/pi\n    if W > 180:\n        W = 360 - W\n    return W","metadata":{"execution":{"iopub.status.busy":"2021-12-19T17:16:24.36162Z","iopub.status.idle":"2021-12-19T17:16:24.362029Z","shell.execute_reply.started":"2021-12-19T17:16:24.361814Z","shell.execute_reply":"2021-12-19T17:16:24.361835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"thres_tr_list = [0.1, 0.09, 0.08, 0.07, 0.06, 0.05, 0.04, 0.03, 0.02, 0.01]\nthres_ro_list = [50, 45, 40, 35, 30, 25, 20, 15, 10, 5]\n\ndef check_match(idx):\n    keep_gt=False\n    thre_tr_dist = thres_tr_list[idx]\n    thre_ro_dist = thres_ro_list[idx]\n    train_dict = {imgID:str2coords(s, names=['carid_or_score', 'pitch', 'yaw', 'roll', 'x', 'y', 'z']) for imgID,s in zip(train_df['ImageId'],train_df['PredictionString'])}\n    valid_dict = {imgID:str2coords(s, names=['pitch', 'yaw', 'roll', 'x', 'y', 'z', 'carid_or_score']) for imgID,s in zip(valid_df['ImageId'],valid_df['PredictionString'])}\n    result_flg = [] # 1 for TP, 0 for FP\n    scores = []\n    MAX_VAL = 10**10\n    for img_id in valid_dict:\n        for pcar in sorted(valid_dict[img_id], key=lambda x: -x['carid_or_score']):\n            # find nearest GT\n            min_tr_dist = MAX_VAL\n            min_idx = -1\n            for idx, gcar in enumerate(train_dict[img_id]):\n                tr_dist = TranslationDistance(pcar,gcar)\n                if tr_dist < min_tr_dist:\n                    min_tr_dist = tr_dist\n                    min_ro_dist = RotationDistance(pcar,gcar)\n                    min_idx = idx\n                    \n            # set the result\n            if min_tr_dist < thre_tr_dist and min_ro_dist < thre_ro_dist:\n                if not keep_gt:\n                    train_dict[img_id].pop(min_idx)\n                result_flg.append(1)\n            else:\n                result_flg.append(0)\n            scores.append(pcar['carid_or_score'])\n    \n    return result_flg, scores","metadata":{"execution":{"iopub.status.busy":"2021-12-19T17:16:24.363969Z","iopub.status.idle":"2021-12-19T17:16:24.364407Z","shell.execute_reply.started":"2021-12-19T17:16:24.364165Z","shell.execute_reply":"2021-12-19T17:16:24.364187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#validation_prediction = df_dev\nvalid_df = pd.read_csv('val_predictions.csv')\nexpanded_valid_df = expand_df(valid_df, ['pitch','yaw','roll','x','y','z','Score'])\nvalid_df = valid_df.fillna('')\n\ntrain_df = pd.read_csv('../input/pku-autonomous-driving/train.csv')\ntrain_df = train_df[train_df.ImageId.isin(valid_df.ImageId.unique())]\n# data description page says, The pose information is formatted as\n# model type, yaw, pitch, roll, x, y, z\n# but it doesn't, and it should be\n# model type, pitch, yaw, roll, x, y, z\nexpanded_train_df = expand_df(train_df, ['model_type','pitch','yaw','roll','x','y','z'])\n\nmax_workers = 10\nn_gt = len(expanded_train_df)\nap_list = []\np = Pool(processes=max_workers)\nfor result_flg, scores in p.imap(check_match, range(10)):\n    if np.sum(result_flg) > 0:\n        n_tp = np.sum(result_flg)\n        recall = n_tp/n_gt\n        ap = average_precision_score(result_flg, scores)*recall\n    else:\n        ap = 0\n    ap_list.append(ap)\nmap = np.mean(ap_list)\nprint('map:', map)","metadata":{"execution":{"iopub.status.busy":"2021-12-19T17:16:24.366285Z","iopub.status.idle":"2021-12-19T17:16:24.36668Z","shell.execute_reply.started":"2021-12-19T17:16:24.366465Z","shell.execute_reply":"2021-12-19T17:16:24.366487Z"},"trusted":true},"execution_count":null,"outputs":[]}]}