{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\nimport torch\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(device)\n\n\nimport matplotlib.pyplot as plt\nfrom math import sin, cos\nimport pandas as pd\nimport numpy as np\nimport json\nimport cv2\nimport os\nfrom math import sin, cos\n\nPATH = '../input/pku-autonomous-driving/'\ntrain = pd.read_csv(PATH + 'train.csv')\ntest = pd.read_csv(PATH + 'sample_submission.csv')\ncamera_matrix = np.array([[2304.5479, 0,  1686.2379],\n                          [0, 2305.8757, 1354.9849],\n                          [0, 0, 1]], dtype=np.float32)\ncamera_matrix_inv = np.linalg.inv(camera_matrix)\nIMG_SHAPE = (2710, 3384, 3)\n\n# map 3d bounding box into 2d\ncars = [\n        'baojun-310-2017','biaozhi-3008','biaozhi-liangxiang','bieke-yinglang-XT',\n        'biyadi-2x-F0','changanbenben','dongfeng-DS5','feiyate',\n        'fengtian-liangxiang','fengtian-MPV','jilixiongmao-2015','lingmu-aotuo-2009',\n        'lingmu-swift','lingmu-SX4-2012','sikeda-jingrui','fengtian-weichi-2006',\n        '037-CAR02','aodi-a6','baoma-330','baoma-530',\n        'baoshijie-paoche','bentian-fengfan','biaozhi-408','biaozhi-508',\n        'bieke-kaiyue','fute','haima-3','kaidilake-CTS',\n        'leikesasi','mazida-6-2015','MG-GT-2015','oubao',\n        'qiya','rongwei-750','supai-2016','xiandai-suonata',\n        'yiqi-benteng-b50','bieke','biyadi-F3','biyadi-qin',\n        'dazhong','dazhongmaiteng','dihao-EV','dongfeng-xuetielong-C6',\n        'dongnan-V3-lingyue-2011','dongfeng-yulong-naruijie','019-SUV','036-CAR01',\n        'aodi-Q7-SUV','baojun-510','baoma-X5','baoshijie-kayan',\n        'beiqi-huansu-H3','benchi-GLK-300','benchi-ML500','fengtian-puladuo-06',\n        'fengtian-SUV-gai','guangqi-chuanqi-GS4-2015','jianghuai-ruifeng-S3','jili-boyue',\n        'jipu-3','linken-SUV','lufeng-X8','qirui-ruihu',\n        'rongwei-RX5','sanling-oulande','sikeda-SUV','Skoda_Fabia-2011',\n        'xiandai-i25-2016','yingfeinidi-qx80','yingfeinidi-SUV','benchi-SUR',\n        'biyadi-tang','changan-CS35-2012','changan-cs5','changcheng-H6-2016',\n        'dazhong-SUV','dongfeng-fengguang-S560','dongfeng-fengxing-SX6'\n        ]\ncid2name = {}\nfor i in range(len(cars)):\n    cid2name[i] = cars[i]\ndef imread(path, fast_mode=False):\n    img = cv2.imread(path)\n    if not fast_mode and img is not None and len(img.shape) == 3:\n        img = np.array(img[:, :, ::-1])\n    return img\ndef str2coords(s, names=['id', 'yaw', 'pitch', 'roll', 'x', 'y', 'z']):\n    coords = []\n    for l in np.array(s.split()).reshape([-1, 7]):\n        coords.append(dict(zip(names, l.astype('float'))))\n        if 'id' in coords[-1]:\n            coords[-1]['id'] = int(coords[-1]['id'])\n    return coords\ndef rotate(x, angle):\n    x = x + angle\n    x = x - (x + np.pi) // (2 * np.pi) * 2 * np.pi\n    return x\ndef get_img_coords(s):\n    coords = str2coords(s)\n    xs = [c['x'] for c in coords]\n    ys = [c['y'] for c in coords]\n    zs = [c['z'] for c in coords]\n    P = np.array(list(zip(xs, ys, zs))).T\n    img_p = np.dot(camera_matrix, P).T\n    img_p[:, 0] /= img_p[:, 2]\n    img_p[:, 1] /= img_p[:, 2]\n    img_xs = img_p[:, 0]\n    img_ys = img_p[:, 1]\n    img_zs = img_p[:, 2]\n    return img_xs, img_ys\ndef euler_to_Rot(yaw, pitch, roll):\n    Y = np.array([[cos(yaw), 0, sin(yaw)],\n                  [0, 1, 0],\n                  [-sin(yaw), 0, cos(yaw)]])\n    P = np.array([[1, 0, 0],\n                  [0, cos(pitch), -sin(pitch)],\n                  [0, sin(pitch), cos(pitch)]])\n    R = np.array([[cos(roll), -sin(roll), 0],\n                  [sin(roll), cos(roll), 0],\n                  [0, 0, 1]])\n    return np.dot(Y, np.dot(P, R))\ndef draw_obj(image, vertices, triangles, num_for_color):\n    for t in triangles:\n        coord = np.array([vertices[t[0]][:2], vertices[t[1]][:2], vertices[t[2]][:2]], dtype=np.int32)\n#         cv2.fillConvexPoly(image, coord, (0,0,255))\n        place = num_for_color%3\n        color = [0,0,0]\n        color[place] = 255\n        color = tuple(color)\n        cv2.polylines(image, np.int32([coord]), 1, color)\n    return image\ndef draw_bw(image, vertices, triangles, num_for_color):\n    for t in triangles:\n        coord = np.array([vertices[t[0]][:2], vertices[t[1]][:2], vertices[t[2]][:2]], dtype=np.int32)\n        color = [255,255,255]\n        color = tuple(color)\n        cv2.polylines(image, np.int32([coord]), 1, color)\n    return image\ndef visualize(img, coords):\n    img = img.copy()\n    num_for_color = 0\n    masks = np.zeros(8*img.shape[0]*img.shape[1]).reshape(8, img.shape[0], img.shape[1])\n    for point in coords:\n        c_model = cid2name[int(point['id'])] + '.json'\n        with open(PATH+'car_models_json/'+c_model) as json_file:\n            data = json.load(json_file)\n        vertices = np.array(data['vertices'])\n        vertices[:, 1] = -vertices[:, 1]\n        triangles = np.array(data['faces']) - 1\n        x, y, z = point['x'], point['y'], point['z']\n        yaw, pitch, roll = -point['pitch'], -point['yaw'], -point['roll']\n        Rt = np.eye(4)\n        t = np.array([x, y, z])\n        Rt[:3, 3] = t\n        Rt[:3, :3] = euler_to_Rot(yaw, pitch, roll).T\n        Rt = Rt[:3, :]\n        P = np.ones((vertices.shape[0],vertices.shape[1]+1))\n        P[:, :-1] = vertices\n        P = P.T\n        img_cor_points = np.dot(camera_matrix, np.dot(Rt, P))\n        img_cor_points = img_cor_points.T\n        img_cor_points[:, 0] /= img_cor_points[:, 2]\n        img_cor_points[:, 1] /= img_cor_points[:, 2]\n        img_cor_points = img_cor_points.astype(int)\n        # find counters\n        overlay = np.zeros_like(img)\n        overlay = draw_bw(overlay, img_cor_points, triangles, num_for_color)\n        overlay = cv2.cvtColor(overlay, cv2.COLOR_BGR2GRAY)\n        contours, hierarchy = cv2.findContours(overlay, cv2.RETR_CCOMP, cv2.CHAIN_APPROX_SIMPLE)\n        # draw counters\n        for i in range(len(contours)):\n            if hierarchy[0][i][3]==-1:\n                overlay = cv2.drawContours(overlay, contours, i, 255, -1)\n        # for instance segmentation\n        overlay_contours = overlay\n        overlay_contours = cv2.Canny(overlay_contours, 30, 200)\n        kernel = np.ones((8,8),np.uint8)\n        overlay_contours = cv2.dilate(overlay_contours,kernel,iterations = 1)\n        # logits\n        masks[0][overlay!=0] = 1\n        # for IS\n        masks[0][overlay_contours!=0] = 0\n        # x\n        masks[1][overlay!=0] = point['x']/100\n        # y\n        masks[2][overlay!=0] = point['y']/100\n        # z\n        masks[3][overlay!=0] = point['z']/100\n        # yaw\n        masks[4][overlay!=0] = point['yaw']\n        # pitch sin\n        psin = sin(point['pitch'])\n        masks[5][overlay!=0] = psin\n        # pitch cos\n        pcos = cos(point['pitch'])\n        masks[6][overlay!=0] = pcos\n        # roll\n        masks[7][overlay!=0] = rotate(point['roll'],np.pi)\n        \n        #plt.imshow(overlay)\n        #plt.show()\n        img = draw_obj(img, img_cor_points, triangles, num_for_color)\n        num_for_color += 1\n    return img, masks","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2022-11-02T09:27:28.415392Z","iopub.execute_input":"2022-11-02T09:27:28.415727Z","iopub.status.idle":"2022-11-02T09:27:28.971076Z","shell.execute_reply.started":"2022-11-02T09:27:28.415667Z","shell.execute_reply":"2022-11-02T09:27:28.969991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import Dataset, DataLoader\n\nscale = 2\n\nclass CData(Dataset):\n    def __init__(self, dataframe, img_dir, mask_dir, training=True, transform=None):\n        self.df = dataframe\n        self.root_dir = img_dir\n        self.mask_dir = mask_dir\n        self.transform = transform\n        self.training = training\n    def __len__(self):\n        return len(self.df)\n    def __getitem__(self, idx):\n        coord_for_trick = [str2coords(self.df['PredictionString'].iloc[idx])[-(i+1)] for i in range(len(str2coords(self.df['PredictionString'].iloc[idx])))]\n        if torch.is_tensor(idx):\n            idx = idx.tolist()\n        # Get image name\n        idx, labels = self.df.values[idx]\n        img_name = self.root_dir.format(idx)\n        mask_name = self.mask_dir.format(idx)\n        # Read image (size (2710, 3384, 3))\n        img = imread(img_name, True)\n        # Make regr (size (7, 2710, 3384))\n        _, regr = visualize(img, coord_for_trick)\n        # Resize image into (size (3, 2710, 3384))\n        img = img[1430:, :, :]\n        img = cv2.resize(img, (3072//scale, 1280//scale))\n        img = np.rollaxis(img, 2, 0)\n        # (7, 2710, 3384) -> (1280, 3384, 7)\n        regr = regr[:, 1430:, :]\n        regr = np.rollaxis(np.rollaxis(regr, 2, 0), 2, 0)\n        # (1280, 3384, 7) -> (7, 1280, 3072)\n        regr = cv2.resize(regr, (3072//scale, 1280//scale))\n        regr = np.rollaxis(regr, 2, 0)\n        # Get mask and regression maps (size (3, 2710, 3384))\n        mask = imread(mask_name, True)\n        if type(mask)==np.ndarray:\n            mask = mask[1430:, :, :]\n            mask = cv2.resize(mask, (3072//scale, 1280//scale))\n            mask = np.rollaxis(mask, 2, 0)\n        if type(mask)!=np.ndarray:\n            mask = np.zeros(2710*3384*3).reshape(2710,3384,3)\n            mask = mask[1430:, :, :]\n            mask = cv2.resize(mask, (3072//scale, 1280//scale))\n            mask = np.rollaxis(mask, 2, 0)\n        # Cut (2710, 3384) -> (1280, 3384) -> Resize -> (1280, 3072)\n        \n        # To Tensor\n        img = torch.tensor(img, device=device).float()/255\n        mask = torch.tensor(mask, device=device).float()/255\n        regr = torch.tensor(regr, device=device).float()\n        \n        return [img, mask, regr]","metadata":{"execution":{"iopub.status.busy":"2022-11-02T09:27:28.973261Z","iopub.execute_input":"2022-11-02T09:27:28.974006Z","iopub.status.idle":"2022-11-02T09:27:28.999012Z","shell.execute_reply.started":"2022-11-02T09:27:28.973953Z","shell.execute_reply":"2022-11-02T09:27:28.998093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 4\n#dataset loading\n\nfrom sklearn.model_selection import train_test_split\n\ntrain_images_dir = PATH + 'train_images/{}.jpg'\ntrain_masks_dir = PATH + 'train_masks/{}.jpg'\ntest_images_dir = PATH + 'test_images/{}.jpg'\ntest_masks_dir = PATH + 'test_masks/{}.jpg'\n\ndf_train, df_dev = train_test_split(train, test_size=0.1, random_state=42)\ndf_test = test\n\n\ntrain_dataset = CData(df_train, train_images_dir, train_masks_dir, training=True)\ndev_dataset = CData(df_dev, train_images_dir, train_masks_dir, training=False)\ntest_dataset = CData(df_test, test_images_dir, test_masks_dir, training=False)\n\n\ntrain_loader = DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=0)\ndev_loader = DataLoader(dataset=dev_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=0)\ntest_loader = DataLoader(dataset=test_dataset, batch_size=1, shuffle=False, num_workers=0)","metadata":{"execution":{"iopub.status.busy":"2022-11-02T09:27:29.001381Z","iopub.execute_input":"2022-11-02T09:27:29.002298Z","iopub.status.idle":"2022-11-02T09:27:29.816878Z","shell.execute_reply.started":"2022-11-02T09:27:29.002224Z","shell.execute_reply":"2022-11-02T09:27:29.81619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a, b, c = train_dataset[0]\nplt.figure(figsize=(18, 6))\nplt.imshow(np.rollaxis(np.rollaxis(a.detach().cpu().numpy(),2,0),2,0))\nplt.show()\nplt.figure(figsize=(18, 6))\nplt.imshow(np.rollaxis(np.rollaxis(b.detach().cpu().numpy(),2,0),2,0))\nplt.show()\nfor i in range(8):\n    plt.figure(figsize=(18, 6))\n    plt.imshow(c.detach().cpu().numpy()[i])\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-02T09:27:29.81812Z","iopub.execute_input":"2022-11-02T09:27:29.818394Z","iopub.status.idle":"2022-11-02T09:27:37.649555Z","shell.execute_reply.started":"2022-11-02T09:27:29.818343Z","shell.execute_reply":"2022-11-02T09:27:37.648565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nimport torch.nn as nn\n\nclass double_conv(nn.Module):\n    \n    def __init__(self, in_ch, out_ch):\n        super(double_conv, self).__init__()\n        self.conv = nn.Sequential(\n            nn.Conv2d(in_ch, out_ch, 3, padding=1),\n            nn.BatchNorm2d(out_ch),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(out_ch, out_ch, 3, padding=1),\n            nn.BatchNorm2d(out_ch),\n            nn.ReLU(inplace=True))\n\n    def forward(self, x):\n        x = self.conv(x)\n        return x\n\nclass UNet(nn.Module):\n    \n    def __init__(self, n_classes):\n        \n        super(UNet, self).__init__()\n        \n        self.dc0 = double_conv(3, 32)\n        self.mp0 = nn.MaxPool2d(2)\n        self.dc1 = double_conv(32, 64)\n        self.mp1 = nn.MaxPool2d(2)\n        self.dc2 = double_conv(64, 128)\n        self.mp2 = nn.MaxPool2d(2)\n        self.dc3 = double_conv(128, 256)\n        self.mp3 = nn.MaxPool2d(2)\n        self.dc4 = double_conv(256, 256)\n        self.up0 = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True)\n        self.dc5 = double_conv(512, 128)\n        self.up1 = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True)\n        self.dc6 = double_conv(256, 64)\n        self.up2 = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True)\n        self.dc7 = double_conv(128, 32)\n        self.up3 = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True)\n        self.last = nn.Conv2d(64, n_classes, 1)\n\n    def forward(self, x):\n        \n        x0 = self.dc0(x)\n        x1 = self.dc1(self.mp0(x0))\n        x2 = self.dc2(self.mp1(x1))\n        x3 = self.dc3(self.mp2(x2))\n        x = self.dc4(self.mp3(x3))\n        x = torch.cat([x3, self.up0(x)],1)\n        x = self.up1(self.dc5(x))\n        x = torch.cat([x2, x],1)\n        x = self.up2(self.dc6(x))\n        x = torch.cat([x1, x],1)\n        x = self.up3(self.dc7(x))\n        x = torch.cat([x0, x],1)\n        x = self.last(x)\n        \n        return x\n    \ndef criterion(prediction, mask, regr, size_average=True):\n    \n    pred_mask = torch.sigmoid(prediction[:, 0])\n    mask_loss = mask * torch.log(pred_mask + 1e-12) + (1 - mask) * torch.log(1 - pred_mask + 1e-12)\n    mask_loss = -mask_loss.mean(0).sum()\n    \n    pred_regr = prediction[:, 1:]\n    regr_loss = (torch.abs(pred_regr - regr).sum(1) * mask).sum(1).sum(1) / mask.sum(1).sum(1)\n    regr_loss = regr_loss.mean(0)\n    \n    loss = mask_loss + regr_loss\n    if not size_average:\n        loss *= prediction.shape[0]\n    return loss\n\n\ndef train_model(epoch):\n    model.train()\n\n    for batch_idx, (img_batch, mask_batch, regr_batch) in enumerate(tqdm(train_loader)):\n\n        img_batch = img_batch.to(device).float()\n        mask_batch = mask_batch.to(device).float()\n        regr_batch = regr_batch.to(device).float()\n\n        optimizer.zero_grad()\n        output = model(img_batch)\n        loss = criterion(output, regr_batch[:,0], regr_batch[:,1:])\n        loss.backward()\n\n        optimizer.step()\n        \n    \n    print('Train Epoch: {} \\tLR: {:.6f}\\tLoss: {:.6f}'.format(\n        epoch,\n        optimizer.state_dict()['param_groups'][0]['lr'],\n        loss.data))\n    \ndef evaluate_model(epoch):\n    model.eval()\n    loss = 0\n    \n    with torch.no_grad():\n        for img_batch, mask_batch, regr_batch in dev_loader:\n            img_batch = img_batch.to(device).float()\n            mask_batch = mask_batch.to(device).float()\n            regr_batch = regr_batch.to(device).float()\n\n            output = model(img_batch)\n\n            loss += criterion(output, regr_batch[:,0], regr_batch[:,1:], size_average=False).data\n    \n    loss /= len(dev_loader.dataset)\n    print('Dev loss: {:.4f}'.format(loss))\n    \n    return loss","metadata":{"execution":{"iopub.status.busy":"2022-11-02T09:27:37.652772Z","iopub.execute_input":"2022-11-02T09:27:37.653228Z","iopub.status.idle":"2022-11-02T09:27:37.714183Z","shell.execute_reply.started":"2022-11-02T09:27:37.653041Z","shell.execute_reply":"2022-11-02T09:27:37.713445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"import gc\nimport torch.optim as optim\nfrom tqdm import tqdm\n\n\nn_epochs = 30\nmodel = UNet(8).to(device)\noptimizer = optim.Adam(model.parameters(), lr=0.001)\n\n# Training\nfor epoch in range(n_epochs):\n    torch.cuda.empty_cache()\n    gc.collect()\n    train_model(epoch)\n    dev_loss = evaluate_model(epoch)\n    if epoch == 0:\n        best_loss = dev_loss\n    else:\n        if dev_loss <= best_loss:\n            best_loss = dev_loss\n            torch.save(model.state_dict(), './is_unet_v2.pth')\n            print('saved best model')\n\n","metadata":{"execution":{"iopub.status.busy":"2022-11-02T09:27:37.720375Z","iopub.execute_input":"2022-11-02T09:27:37.720804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}