{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"from IPython.display import Image, FileLink\n!pip install lyft-dataset-sdk -q\n!pip install moviepy\n! python ../input/mlcomp/mlcomp/mlcomp/setup.py","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from datetime import datetime\nfrom functools import partial\nimport glob\nfrom multiprocessing import Pool\n\n# Disable multiprocesing for numpy/opencv. We already multiprocess ourselves, this would mean every subprocess produces\n# even more threads which would lead to a lot of context switching, slowing things down a lot.\nimport os\nos.environ[\"OMP_NUM_THREADS\"] = \"1\"\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nimport pandas as pd\nimport cv2\nfrom PIL import Image\nimport numpy as np\nfrom tqdm import tqdm, tqdm_notebook\nimport scipy\nimport scipy.ndimage\nimport scipy.special\nfrom scipy.spatial.transform import Rotation as R\n\nfrom lyft_dataset_sdk.lyftdataset import LyftDataset\nfrom lyft_dataset_sdk.utils.data_classes import LidarPointCloud, Box, Quaternion\nfrom lyft_dataset_sdk.utils.geometry_utils import view_points, transform_matrix\n\nimport time\nfrom lyft_dataset_sdk.utils.map_mask import MapMask\nfrom pathlib import Path\nfrom lyft_dataset_sdk.lyftdataset import LyftDataset,LyftDatasetExplorer","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Lyft SDK requires creating a link to input folders"},{"metadata":{"trusted":true},"cell_type":"code","source":"!ln -s /kaggle/input/3d-object-detection-for-autonomous-vehicles/test_images images\n!ln -s /kaggle/input/3d-object-detection-for-autonomous-vehicles/test_maps maps\n!ln -s /kaggle/input/3d-object-detection-for-autonomous-vehicles/test_lidar lidar","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preprocessing test data"},{"metadata":{"trusted":true},"cell_type":"code","source":"# class LyftTestDataset(LyftDataset):\n#     \"\"\"Database class for Lyft Dataset to help query and retrieve information from the database.\"\"\"\n\n#     def __init__(self, data_path: str, json_path: str, verbose: bool = True, map_resolution: float = 0.1):\n#         \"\"\"Loads database and creates reverse indexes and shortcuts.\n#         Args:\n#             data_path: Path to the tables and data.\n#             json_path: Path to the folder with json files\n#             verbose: Whether to print status messages during load.\n#             map_resolution: Resolution of maps (meters).\n#         \"\"\"\n\n#         self.data_path = Path(data_path).expanduser().absolute()\n#         self.json_path = Path(json_path)\n\n#         self.table_names = [\n#             \"category\",\n#             \"attribute\",\n#             \"sensor\",\n#             \"calibrated_sensor\",\n#             \"ego_pose\",\n#             \"log\",\n#             \"scene\",\n#             \"sample\",\n#             \"sample_data\",\n#             \"map\",\n#         ]\n\n#         start_time = time.time()\n\n#         # Explicitly assign tables to help the IDE determine valid class members.\n#         self.category = self.__load_table__(\"category\")\n#         self.attribute = self.__load_table__(\"attribute\")\n        \n        \n#         self.sensor = self.__load_table__(\"sensor\")\n#         self.calibrated_sensor = self.__load_table__(\"calibrated_sensor\")\n#         self.ego_pose = self.__load_table__(\"ego_pose\")\n#         self.log = self.__load_table__(\"log\")\n#         self.scene = self.__load_table__(\"scene\")\n#         self.sample = self.__load_table__(\"sample\")\n#         self.sample_data = self.__load_table__(\"sample_data\")\n        \n#         self.map = self.__load_table__(\"map\")\n\n#         # Initialize map mask for each map record.\n#         for map_record in self.map:\n#             map_record[\"mask\"] = MapMask(self.data_path / map_record[\"filename\"], resolution=map_resolution)\n\n#         if verbose:\n#             for table in self.table_names:\n#                 print(\"{} {},\".format(len(getattr(self, table)), table))\n#             print(\"Done loading in {:.1f} seconds.\\n======\".format(time.time() - start_time))\n\n#         # Initialize LyftDatasetExplorer class\n#         self.explorer = LyftDatasetExplorer(self)\n#         # Make reverse indexes for common lookups.\n#         self.__make_reverse_index__(verbose)\n        \n#     def __make_reverse_index__(self, verbose: bool) -> None:\n#         \"\"\"De-normalizes database to create reverse indices for common cases.\n#         Args:\n#             verbose: Whether to print outputs.\n#         \"\"\"\n\n#         start_time = time.time()\n#         if verbose:\n#             print(\"Reverse indexing ...\")\n\n#         # Store the mapping from token to table index for each table.\n#         self._token2ind = dict()\n#         for table in self.table_names:\n#             self._token2ind[table] = dict()\n\n#             for ind, member in enumerate(getattr(self, table)):\n#                 self._token2ind[table][member[\"token\"]] = ind\n\n#         # Decorate (adds short-cut) sample_data with sensor information.\n#         for record in self.sample_data:\n#             cs_record = self.get(\"calibrated_sensor\", record[\"calibrated_sensor_token\"])\n#             sensor_record = self.get(\"sensor\", cs_record[\"sensor_token\"])\n#             record[\"sensor_modality\"] = sensor_record[\"modality\"]\n#             record[\"channel\"] = sensor_record[\"channel\"]\n\n#         # Reverse-index samples with sample_data and annotations.\n#         for record in self.sample:\n#             record[\"data\"] = {}\n#             record[\"anns\"] = []\n\n#         for record in self.sample_data:\n#             if record[\"is_key_frame\"]:\n#                 sample_record = self.get(\"sample\", record[\"sample_token\"])\n#                 sample_record[\"data\"][record[\"channel\"]] = record[\"token\"]\n\n#         # Add reverse indices from log records to map records.\n#         if \"log_tokens\" not in self.map[0].keys():\n#             raise Exception(\"Error: log_tokens not in map table. This code is not compatible with the teaser dataset.\")\n#         log_to_map = dict()\n#         for map_record in self.map:\n#             for log_token in map_record[\"log_tokens\"]:\n#                 log_to_map[log_token] = map_record[\"token\"]\n#         for log_record in self.log:\n#             log_record[\"map_token\"] = log_to_map[log_record[\"token\"]]\n\n#         if verbose:\n#             print(\"Done reverse indexing in {:.1f} seconds.\\n======\".format(time.time() - start_time))\n\n\n# level5data = LyftTestDataset(data_path='.', json_path='/kaggle/input/3d-object-detection-for-autonomous-vehicles/test_data', verbose=True)\n# # Our code will generate data, visualization and model checkpoints, they will be persisted to disk in this folder\n# ARTIFACTS_FOLDER = \"./artifacts\"\n# os.makedirs(ARTIFACTS_FOLDER, exist_ok=True)\n# classes = [\"car\", \"motorcycle\", \"bus\", \"bicycle\", \"truck\", \"pedestrian\", \"other_vehicle\", \"animal\", \"emergency_vehicle\"]\n\n\n# sample_sub = pd.read_csv('../input/3d-object-detection-for-autonomous-vehicles/sample_submission.csv')\n\n\n# def create_transformation_matrix_to_voxel_space(shape, voxel_size, offset):\n#     \"\"\"\n#     Constructs a transformation matrix given an output voxel shape such that (0,0,0) ends up in the center.\n#     Voxel_size defines how large every voxel is in world coordinate, (1,1,1) would be the same as Minecraft voxels.\n    \n#     An offset per axis in world coordinates (metric) can be provided, this is useful for Z (up-down) in lidar points.\n#     \"\"\"\n    \n#     shape, voxel_size, offset = np.array(shape), np.array(voxel_size), np.array(offset)\n    \n#     tm = np.eye(4, dtype=np.float32)\n#     translation = shape/2 + offset/voxel_size\n    \n#     tm = tm * np.array(np.hstack((1/voxel_size, [1])))\n#     tm[:3, 3] = np.transpose(translation)\n#     return tm\n\n# def transform_points(points, transf_matrix):\n#     \"\"\"\n#     Transform (3,N) or (4,N) points using transformation matrix.\n#     \"\"\"\n#     if points.shape[0] not in [3,4]:\n#         raise Exception(\"Points input should be (3,N) or (4,N) shape, received {}\".format(points.shape))\n#     return transf_matrix.dot(np.vstack((points[:3, :], np.ones(points.shape[1]))))[:3, :]\n\n\n# def car_to_voxel_coords(points, shape, voxel_size, z_offset=0):\n#     if len(shape) != 3:\n#         raise Exception(\"Voxel volume shape should be 3 dimensions (x,y,z)\")\n        \n#     if len(points.shape) != 2 or points.shape[0] not in [3, 4]:\n#         raise Exception(\"Input points should be (3,N) or (4,N) in shape, found {}\".format(points.shape))\n\n#     tm = create_transformation_matrix_to_voxel_space(shape, voxel_size, (0, 0, z_offset))\n#     p = transform_points(points, tm)\n#     return p\n\n# def create_voxel_pointcloud(points, shape, voxel_size=(0.5,0.5,1), z_offset=0):\n\n#     points_voxel_coords = car_to_voxel_coords(points.copy(), shape, voxel_size, z_offset)\n#     points_voxel_coords = points_voxel_coords[:3].transpose(1,0)\n#     points_voxel_coords = np.int0(points_voxel_coords)\n    \n#     bev = np.zeros(shape, dtype=np.float32)\n#     bev_shape = np.array(shape)\n\n#     within_bounds = (np.all(points_voxel_coords >= 0, axis=1) * np.all(points_voxel_coords < bev_shape, axis=1))\n    \n#     points_voxel_coords = points_voxel_coords[within_bounds]\n#     coord, count = np.unique(points_voxel_coords, axis=0, return_counts=True)\n        \n#     # Note X and Y are flipped:\n#     bev[coord[:,1], coord[:,0], coord[:,2]] = count\n    \n#     return bev\n\n# def normalize_voxel_intensities(bev, max_intensity=16):\n#     return (bev/max_intensity).clip(0,1)\n\n\n\n\n# bev_shape = (336, 336, 3)\n# target_im = np.zeros(bev_shape, dtype=np.uint8)\n\n# def move_boxes_to_car_space(boxes, ego_pose):\n#     \"\"\"\n#     Move boxes from world space to car space.\n#     Note: mutates input boxes.\n#     \"\"\"\n#     translation = -np.array(ego_pose['translation'])\n#     rotation = Quaternion(ego_pose['rotation']).inverse\n    \n#     for box in boxes:\n#         # Bring box to car space\n#         box.translate(translation)\n#         box.rotate(rotation)\n        \n# def scale_boxes(boxes, factor):\n#     \"\"\"\n#     Note: mutates input boxes\n#     \"\"\"\n#     for box in boxes:\n#         box.wlh = box.wlh * factor\n\n# def draw_boxes(im, voxel_size, boxes, classes, z_offset=0.0):\n#     for box in boxes:\n#         # We only care about the bottom corners\n#         corners = box.bottom_corners()\n#         corners_voxel = car_to_voxel_coords(corners, im.shape, voxel_size, z_offset).transpose(1,0)\n#         corners_voxel = corners_voxel[:,:2] # Drop z coord\n\n#         class_color = classes.index(box.name) + 1\n        \n#         if class_color == 0:\n#             raise Exception(\"Unknown class: {}\".format(box.name))\n\n#         cv2.drawContours(im, np.int0([corners_voxel]), 0, (class_color, class_color, class_color), -1)\n\n        \n        \n# def visualize_lidar_of_sample(sample_token, axes_limit=80):\n#     sample = level5data.get(\"sample\", sample_token)\n#     sample_lidar_token = sample[\"data\"][\"LIDAR_TOP\"]\n#     level5data.render_sample_data(sample_lidar_token, axes_limit=axes_limit)\n    \n# # Don't worry about it being mirrored.\n# visualize_lidar_of_sample(sample_sub.loc[0,'Id'])\n\n\n\n\n# # Some hyperparameters we'll need to define for the system\n# voxel_size = (0.4, 0.4, 1.5)\n# z_offset = -2.0\n# bev_shape = (336, 336, 3)\n\n# # We scale down each box so they are more separated when projected into our coarse voxel space.\n# box_scale = 0.8\n\n# NUM_WORKERS = os.cpu_count() * 3\n\n# # \"bev\" stands for birds eye view\n# # test_data_folder = os.path.join(ARTIFACTS_FOLDER, \"bev_test_data\")\n# test_data_folder = '/kaggle/working/artifacts'\n\n\n\n\n\n# def prepare_testing_data_for_scene(sample_token, output_folder=test_data_folder,\n#                                    bev_shape=bev_shape, voxel_size=voxel_size, z_offset=z_offset,\n#                                    box_scale=box_scale):\n#     \"\"\"\n#     Given a sample token (in a scene), output rasterized input volumes in birds-eye-view perspective.\n\n#     \"\"\"\n    \n# #     while sample_token:\n        \n#     sample = level5data.get(\"sample\", sample_token)\n    \n\n#     sample_lidar_token = sample[\"data\"][\"LIDAR_TOP\"]\n#     lidar_data = level5data.get(\"sample_data\", sample_lidar_token)\n#     lidar_filepath = level5data.get_sample_data_path(sample_lidar_token)\n    \n    \n\n#     ego_pose = level5data.get(\"ego_pose\", lidar_data[\"ego_pose_token\"])\n#     calibrated_sensor = level5data.get(\"calibrated_sensor\", lidar_data[\"calibrated_sensor_token\"])\n    \n\n\n#     global_from_car = transform_matrix(ego_pose['translation'],\n#                                        Quaternion(ego_pose['rotation']), inverse=False)\n    \n\n#     car_from_sensor = transform_matrix(calibrated_sensor['translation'], Quaternion(calibrated_sensor['rotation']),\n#                                         inverse=False)\n    \n    \n#     lidar_pointcloud = LidarPointCloud.from_file(lidar_filepath)\n    \n#     lidar_pointcloud.transform(car_from_sensor)\n\n#     bev = create_voxel_pointcloud(lidar_pointcloud.points, bev_shape, voxel_size=voxel_size, z_offset=z_offset)\n#     bev = normalize_voxel_intensities(bev)\n\n#     bev_im = np.round(bev*255).astype(np.uint8)\n\n#     cv2.imwrite(os.path.join(output_folder, \"{}_input.png\".format(sample_token)), bev_im)\n\n    \n    \n# # # to debug..\n# # # doesn't work right now\n# # os.makedirs(test_data_folder, exist_ok=True)\n# # # NUM_WORKERS = 4\n# # pool = Pool(NUM_WORKERS)\n# # for _ in tqdm_notebook(pool.imap(partial(prepare_testing_data_for_scene,\n# #                        output_folder=test_data_folder, bev_shape=bev_shape, voxel_size=voxel_size, z_offset=z_offset, box_scale=box_scale),sample_sub.loc[:,'Id'].values)\n# #                       ,total=len(sample_sub)):\n# #     pass\n\n# # pool.close()\n# # del pool\n\n\n\n\n\n# for token in tqdm_notebook(sample_sub.loc[:,'Id'].values):\n#     prepare_testing_data_for_scene(token)\n\n    \n    \n# # !tar -czf lyft3d_bev_test_data.tar.gz ./artifacts/ \n\n# # !du -h lyft3d_bev_test_data.tar.gz\n# # !rm -r ./artifacts","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"classes = [\"car\", \"motorcycle\", \"bus\", \"bicycle\", \"truck\", \"pedestrian\", \"other_vehicle\", \"animal\", \"emergency_vehicle\"]\ntrain_dataset = LyftDataset(data_path='.', json_path='../input/3d-object-detection-for-autonomous-vehicles/train_data', verbose=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Find the mean height of all categories\nWe can use the mean height instead of blindly using 1.75m for all categories"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset.list_categories()\ndel train_dataset;","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class_heights = {'animal':0.51,'bicycle':1.44,'bus':3.44,'car':1.72,'emergency_vehicle':2.39,'motorcycle':1.59,\n                'other_vehicle':3.23,'pedestrian':1.78,'truck':3.44}\nlevel5data = LyftDataset(data_path='.', json_path='../input/3d-object-detection-for-autonomous-vehicles/test_data', verbose=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def move_boxes_to_car_space(boxes, ego_pose):\n    \"\"\"\n    Move boxes from world space to car space.\n    Note: mutates input boxes.\n    \"\"\"\n    translation = -np.array(ego_pose['translation'])\n    rotation = Quaternion(ego_pose['rotation']).inverse\n    \n    for box in boxes:\n        # Bring box to car space\n        box.translate(translation)\n        box.rotate(rotation)\n        \ndef scale_boxes(boxes, factor):\n    \"\"\"\n    Note: mutates input boxes\n    \"\"\"\n    for box in boxes:\n        box.wlh = box.wlh * factor\n\ndef draw_boxes(im, voxel_size, boxes, classes, z_offset=0.0):\n    for box in boxes:\n        # We only care about the bottom corners\n        corners = box.bottom_corners()\n        corners_voxel = car_to_voxel_coords(corners, im.shape, voxel_size, z_offset).transpose(1,0)\n        corners_voxel = corners_voxel[:,:2] # Drop z coord\n\n        class_color = classes.index(box.name) + 1\n        \n        if class_color == 0:\n            raise Exception(\"Unknown class: {}\".format(box.name))\n\n        cv2.drawContours(im, np.int0([corners_voxel]), 0, (class_color, class_color, class_color), -1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Some hyperparameters we'll need to define for the system\nvoxel_size = (0.4, 0.4, 1.5)\nz_offset = -2.0\nbev_shape = (336, 336, 3)\n\n# We scale down each box so they are more separated when projected into our coarse voxel space.\nbox_scale = 0.8","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"records = [(level5data.get('sample', record['first_sample_token'])['timestamp'], record) for record in level5data.scene]\n\nentries = []\n\nfor start_time, record in sorted(records):\n    start_time = level5data.get('sample', record['first_sample_token'])['timestamp'] / 1000000\n\n    token = record['token']\n    name = record['name']\n    date = datetime.utcfromtimestamp(start_time)\n    host = \"-\".join(record['name'].split(\"-\")[:2])\n    first_sample_token = record[\"first_sample_token\"]\n\n    entries.append((host, name, date, token, first_sample_token))\n            \ndf = pd.DataFrame(entries, columns=[\"host\", \"scene_name\", \"date\", \"scene_token\", \"first_sample_token\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# sample_sub = pd.read_csv('../input/3d-object-detection-for-autonomous-vehicles/sample_submission.csv')\nall_sample_tokens,scene_len = [],[]\nfor sample_token in tqdm_notebook(df.first_sample_token.values):\n    i = 0\n    while sample_token:\n        all_sample_tokens.append(sample_token)\n        sample = level5data.get(\"sample\", sample_token)\n        sample_token = sample[\"next\"]\n        i += 1\n    scene_len.append(i)\n#     print(len(all_sample_tokens[-1]))\n    \nprint('Total number of tokens=',len(all_sample_tokens))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.utils.data\n\ntest_data_folder = '../input/lyft3d-mask-test-data/test_data/test_data'\n\nclass BEVImageDataset(torch.utils.data.Dataset):\n    def __init__(self, sample_token,test_data_folder):\n\n        self.sample_token = sample_token\n        self.test_data_folder = test_data_folder\n\n    def __len__(self):\n        return len(self.sample_token)\n\n    def __getitem__(self, idx):\n        sample_token = self.sample_token[idx]\n        \n#         sample_token = input_filepath.split(\"/\")[-1].replace(\"_input.png\",\"\")\n        \n        input_filepath = os.path.join(test_data_folder,f\"{sample_token}_input.png\")\n\n        map_filepath = os.path.join(test_data_folder,f\"{sample_token}_map.png\")\n        \n        im = cv2.imread(input_filepath, cv2.IMREAD_UNCHANGED)\n        \n        map_im = cv2.imread(map_filepath, cv2.IMREAD_UNCHANGED)\n        print(im.shape,map_im.shape)\n        im = np.concatenate((im, map_im), axis=2)\n#         im = np.array(im).astype(np.float32)/255\n        im = im.astype(np.float32)/255\n        \n        im = torch.from_numpy(im.transpose(2,0,1))\n        \n        return im, sample_token\n\n    \n# input_filepaths = sorted(glob.glob(os.path.join(test_data_folder, \"*_input.png\")))\n# map_filepaths = sorted(glob.glob(os.path.join(test_data_folder, \"*_map.png\")))\n\ntest_dataset = BEVImageDataset(all_sample_tokens,test_data_folder)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Unet Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"# # This implementation was copied from https://github.com/jvanvugt/pytorch-unet, it is MIT licensed.\n\n# class UNet(nn.Module):\n#     def __init__(\n#         self,\n#         in_channels=1,\n#         n_classes=2,\n#         depth=5,\n#         wf=6,\n#         padding=False,\n#         batch_norm=False,\n#         up_mode='upconv',\n#     ):\n#         \"\"\"\n#         Implementation of\n#         U-Net: Convolutional Networks for Biomedical Image Segmentation\n#         (Ronneberger et al., 2015)\n#         https://arxiv.org/abs/1505.04597\n#         Using the default arguments will yield the exact version used\n#         in the original paper\n#         Args:\n#             in_channels (int): number of input channels\n#             n_classes (int): number of output channels\n#             depth (int): depth of the network\n#             wf (int): number of filters in the first layer is 2**wf\n#             padding (bool): if True, apply padding such that the input shape\n#                             is the same as the output.\n#                             This may introduce artifacts\n#             batch_norm (bool): Use BatchNorm after layers with an\n#                                activation function\n#             up_mode (str): one of 'upconv' or 'upsample'.\n#                            'upconv' will use transposed convolutions for\n#                            learned upsampling.\n#                            'upsample' will use bilinear upsampling.\n#         \"\"\"\n#         super(UNet, self).__init__()\n#         assert up_mode in ('upconv', 'upsample')\n#         self.padding = padding\n#         self.depth = depth\n#         prev_channels = in_channels\n#         self.down_path = nn.ModuleList()\n#         for i in range(depth):\n#             self.down_path.append(\n#                 UNetConvBlock(prev_channels, 2 ** (wf + i), padding, batch_norm)\n#             )\n#             prev_channels = 2 ** (wf + i)\n\n#         self.up_path = nn.ModuleList()\n#         for i in reversed(range(depth - 1)):\n#             self.up_path.append(\n#                 UNetUpBlock(prev_channels, 2 ** (wf + i), up_mode, padding, batch_norm)\n#             )\n#             prev_channels = 2 ** (wf + i)\n\n#         self.last = nn.Conv2d(prev_channels, n_classes, kernel_size=1)\n\n#     def forward(self, x):\n#         blocks = []\n#         for i, down in enumerate(self.down_path):\n#             x = down(x)\n#             if i != len(self.down_path) - 1:\n#                 blocks.append(x)\n#                 x = F.max_pool2d(x, 2)\n\n#         for i, up in enumerate(self.up_path):\n#             x = up(x, blocks[-i - 1])\n\n#         return self.last(x)\n\n\n# class UNetConvBlock(nn.Module):\n#     def __init__(self, in_size, out_size, padding, batch_norm):\n#         super(UNetConvBlock, self).__init__()\n#         block = []\n\n#         block.append(nn.Conv2d(in_size, out_size, kernel_size=3, padding=int(padding)))\n#         block.append(nn.ReLU())\n#         if batch_norm:\n#             block.append(nn.BatchNorm2d(out_size))\n\n#         block.append(nn.Conv2d(out_size, out_size, kernel_size=3, padding=int(padding)))\n#         block.append(nn.ReLU())\n#         if batch_norm:\n#             block.append(nn.BatchNorm2d(out_size))\n\n#         self.block = nn.Sequential(*block)\n\n#     def forward(self, x):\n#         out = self.block(x)\n#         return out\n\n\n# class UNetUpBlock(nn.Module):\n#     def __init__(self, in_size, out_size, up_mode, padding, batch_norm):\n#         super(UNetUpBlock, self).__init__()\n#         if up_mode == 'upconv':\n#             self.up = nn.ConvTranspose2d(in_size, out_size, kernel_size=2, stride=2)\n#         elif up_mode == 'upsample':\n#             self.up = nn.Sequential(\n#                 nn.Upsample(mode='bilinear', scale_factor=2),\n#                 nn.Conv2d(in_size, out_size, kernel_size=1),\n#             )\n\n#         self.conv_block = UNetConvBlock(in_size, out_size, padding, batch_norm)\n\n#     def center_crop(self, layer, target_size):\n#         _, _, layer_height, layer_width = layer.size()\n#         diff_y = (layer_height - target_size[0]) // 2\n#         diff_x = (layer_width - target_size[1]) // 2\n#         return layer[\n#             :, :, diff_y : (diff_y + target_size[0]), diff_x : (diff_x + target_size[1])\n#         ]\n\n#     def forward(self, x, bridge):\n#         up = self.up(x)\n#         crop1 = self.center_crop(bridge, up.shape[2:])\n#         out = torch.cat([up, crop1], 1)\n#         out = self.conv_block(out)\n#         return out\n\n# def get_unet_model(in_channels=3, num_output_classes=2):\n#     model = UNet(in_channels=in_channels, n_classes=num_output_classes, wf=5, depth=4, padding=True, up_mode='upsample')\n    \n#     # Optional, for multi GPU training and inference\n#     model = nn.DataParallel(model)\n#     return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# DeepalabV3_Exception"},{"metadata":{"trusted":true},"cell_type":"code","source":"import queue\nimport collections\nimport threading\n\n__all__ = ['FutureResult', 'SlavePipe', 'SyncMaster']\n\n\nclass FutureResult(object):\n    \"\"\"A thread-safe future implementation. Used only as one-to-one pipe.\"\"\"\n\n    def __init__(self):\n        self._result = None\n        self._lock = threading.Lock()\n        self._cond = threading.Condition(self._lock)\n\n    def put(self, result):\n        with self._lock:\n            assert self._result is None, 'Previous result has\\'t been fetched.'\n            self._result = result\n            self._cond.notify()\n\n    def get(self):\n        with self._lock:\n            if self._result is None:\n                self._cond.wait()\n\n            res = self._result\n            self._result = None\n            return res\n\n\n_MasterRegistry = collections.namedtuple('MasterRegistry', ['result'])\n_SlavePipeBase = collections.namedtuple('_SlavePipeBase', ['identifier', 'queue', 'result'])\n\n\nclass SlavePipe(_SlavePipeBase):\n    \"\"\"Pipe for master-slave communication.\"\"\"\n\n    def run_slave(self, msg):\n        self.queue.put((self.identifier, msg))\n        ret = self.result.get()\n        self.queue.put(True)\n        return ret\n\n\nclass SyncMaster(object):\n    def __init__(self, master_callback):\n        \"\"\"\n        Args:\n            master_callback: a callback to be invoked after having collected messages from slave devices.\n        \"\"\"\n        self._master_callback = master_callback\n        self._queue = queue.Queue()\n        self._registry = collections.OrderedDict()\n        self._activated = False\n\n    def __getstate__(self):\n        return {'master_callback': self._master_callback}\n\n    def __setstate__(self, state):\n        self.__init__(state['master_callback'])\n\n    def register_slave(self, identifier):\n        \"\"\"\n        Register an slave device.\n        Args:\n            identifier: an identifier, usually is the device id.\n        Returns: a `SlavePipe` object which can be used to communicate with the master device.\n        \"\"\"\n        if self._activated:\n            assert self._queue.empty(), 'Queue is not clean before next initialization.'\n            self._activated = False\n            self._registry.clear()\n        future = FutureResult()\n        self._registry[identifier] = _MasterRegistry(future)\n        return SlavePipe(identifier, self._queue, future)\n\n    def run_master(self, master_msg):\n        self._activated = True\n\n        intermediates = [(0, master_msg)]\n        for i in range(self.nr_slaves):\n            intermediates.append(self._queue.get())\n\n        results = self._master_callback(intermediates)\n        assert results[0][0] == 0, 'The first result should belongs to the master.'\n\n        for i, res in results:\n            if i == 0:\n                continue\n            self._registry[i].result.put(res)\n\n        for i in range(self.nr_slaves):\n            assert self._queue.get() is True\n\n        return results[0][1]\n\n    @property\n    def nr_slaves(self):\n        return len(self._registry)\n\n\nimport collections\n\nimport torch\nimport torch.nn.functional as F\n\nfrom torch.nn.modules.batchnorm import _BatchNorm\nfrom torch.nn.parallel._functions import ReduceAddCoalesced, Broadcast\n\n# from .comm import SyncMaster\n\n__all__ = ['SynchronizedBatchNorm1d', 'SynchronizedBatchNorm2d', 'SynchronizedBatchNorm3d']\n\n\ndef _sum_ft(tensor):\n    \"\"\"sum over the first and last dimention\"\"\"\n    return tensor.sum(dim=0).sum(dim=-1)\n\n\ndef _unsqueeze_ft(tensor):\n    \"\"\"add new dementions at the front and the tail\"\"\"\n    return tensor.unsqueeze(0).unsqueeze(-1)\n\n\n_ChildMessage = collections.namedtuple('_ChildMessage', ['sum', 'ssum', 'sum_size'])\n_MasterMessage = collections.namedtuple('_MasterMessage', ['sum', 'inv_std'])\n\n\nclass _SynchronizedBatchNorm(_BatchNorm):\n    def __init__(self, num_features, eps=1e-5, momentum=0.1, affine=True):\n        super(_SynchronizedBatchNorm, self).__init__(num_features, eps=eps, momentum=momentum, affine=affine)\n\n        self._sync_master = SyncMaster(self._data_parallel_master)\n\n        self._is_parallel = False\n        self._parallel_id = None\n        self._slave_pipe = None\n\n    def forward(self, input):\n        # If it is not parallel computation or is in evaluation mode, use PyTorch's implementation.\n        if not (self._is_parallel and self.training):\n            return F.batch_norm(\n                input, self.running_mean, self.running_var, self.weight, self.bias,\n                self.training, self.momentum, self.eps)\n\n        # Resize the input to (B, C, -1).\n        input_shape = input.size()\n        input = input.view(input.size(0), self.num_features, -1)\n\n        # Compute the sum and square-sum.\n        sum_size = input.size(0) * input.size(2)\n        input_sum = _sum_ft(input)\n        input_ssum = _sum_ft(input ** 2)\n\n        # Reduce-and-broadcast the statistics.\n        if self._parallel_id == 0:\n            mean, inv_std = self._sync_master.run_master(_ChildMessage(input_sum, input_ssum, sum_size))\n        else:\n            mean, inv_std = self._slave_pipe.run_slave(_ChildMessage(input_sum, input_ssum, sum_size))\n\n        # Compute the output.\n        if self.affine:\n            # MJY:: Fuse the multiplication for speed.\n            output = (input - _unsqueeze_ft(mean)) * _unsqueeze_ft(inv_std * self.weight) + _unsqueeze_ft(self.bias)\n        else:\n            output = (input - _unsqueeze_ft(mean)) * _unsqueeze_ft(inv_std)\n\n        # Reshape it.\n        return output.view(input_shape)\n\n    def __data_parallel_replicate__(self, ctx, copy_id):\n        self._is_parallel = True\n        self._parallel_id = copy_id\n\n        # parallel_id == 0 means master device.\n        if self._parallel_id == 0:\n            ctx.sync_master = self._sync_master\n        else:\n            self._slave_pipe = ctx.sync_master.register_slave(copy_id)\n\n    def _data_parallel_master(self, intermediates):\n        \"\"\"Reduce the sum and square-sum, compute the statistics, and broadcast it.\"\"\"\n\n        # Always using same \"device order\" makes the ReduceAdd operation faster.\n        # Thanks to:: Tete Xiao (http://tetexiao.com/)\n        intermediates = sorted(intermediates, key=lambda i: i[1].sum.get_device())\n\n        to_reduce = [i[1][:2] for i in intermediates]\n        to_reduce = [j for i in to_reduce for j in i]  # flatten\n        target_gpus = [i[1].sum.get_device() for i in intermediates]\n\n        sum_size = sum([i[1].sum_size for i in intermediates])\n        sum_, ssum = ReduceAddCoalesced.apply(target_gpus[0], 2, *to_reduce)\n        mean, inv_std = self._compute_mean_std(sum_, ssum, sum_size)\n\n        broadcasted = Broadcast.apply(target_gpus, mean, inv_std)\n\n        outputs = []\n        for i, rec in enumerate(intermediates):\n            outputs.append((rec[0], _MasterMessage(*broadcasted[i * 2:i * 2 + 2])))\n\n        return outputs\n\n    def _compute_mean_std(self, sum_, ssum, size):\n        \"\"\"Compute the mean and standard-deviation with sum and square-sum. This method\n        also maintains the moving average on the master device.\"\"\"\n        assert size > 1, 'BatchNorm computes unbiased standard-deviation, which requires size > 1.'\n        mean = sum_ / size\n        sumvar = ssum - sum_ * mean\n        unbias_var = sumvar / (size - 1)\n        bias_var = sumvar / size\n\n        self.running_mean = (1 - self.momentum) * self.running_mean + self.momentum * mean.data\n        self.running_var = (1 - self.momentum) * self.running_var + self.momentum * unbias_var.data\n\n        return mean, bias_var.clamp(self.eps) ** -0.5\n\n\nclass SynchronizedBatchNorm1d(_SynchronizedBatchNorm):\n    def _check_input_dim(self, input):\n        if input.dim() != 2 and input.dim() != 3:\n            raise ValueError('expected 2D or 3D input (got {}D input)'\n                             .format(input.dim()))\n        super(SynchronizedBatchNorm1d, self)._check_input_dim(input)\n\n\nclass SynchronizedBatchNorm2d(_SynchronizedBatchNorm):\n    def _check_input_dim(self, input):\n        if input.dim() != 4:\n            raise ValueError('expected 4D input (got {}D input)'\n                             .format(input.dim()))\n        super(SynchronizedBatchNorm2d, self)._check_input_dim(input)\n\n\nclass SynchronizedBatchNorm3d(_SynchronizedBatchNorm):\n    def _check_input_dim(self, input):\n        if input.dim() != 5:\n            raise ValueError('expected 5D input (got {}D input)'\n                             .format(input.dim()))\n        super(SynchronizedBatchNorm3d, self)._check_input_dim(input)\n        \n        \n\n\nimport math\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.utils.model_zoo as model_zoo\n# from modeling.sync_batchnorm.batchnorm import SynchronizedBatchNorm2d\n\nBatchNorm2d = SynchronizedBatchNorm2d\n\nclass SeparableConv2d(nn.Module):\n    def __init__(self, inplanes, planes, kernel_size=3, stride=1, padding=0, dilation=1, bias=False):\n        super(SeparableConv2d, self)._init_()\n\n        self.conv1 = nn.Conv2d(inplanes, inplanes, kernel_size, stride, padding, dilation,\n                               groups=inplanes, bias=bias)\n        self.pointwise = nn.Conv2d(inplanes, planes, 1, 1, 0, 1, 1, bias=bias)\n\n    def forward(self, x):\n        x = self.conv1(x)\n        x = self.pointwise(x)\n        return x\n\n\ndef fixed_padding(inputs, kernel_size, dilation):\n    kernel_size_effective = kernel_size + (kernel_size - 1) * (dilation - 1)\n    pad_total = kernel_size_effective - 1\n    pad_beg = pad_total // 2\n    pad_end = pad_total - pad_beg\n    padded_inputs = F.pad(inputs, (pad_beg, pad_end, pad_beg, pad_end))\n    return padded_inputs\n\n\nclass SeparableConv2d_same(nn.Module):\n    def __init__(self, inplanes, planes, kernel_size=3, stride=1, dilation=1, bias=False):\n        super(SeparableConv2d_same, self).__init__()\n\n        self.conv1 = nn.Conv2d(inplanes, inplanes, kernel_size, stride, 0, dilation,\n                               groups=inplanes, bias=bias)\n        self.pointwise = nn.Conv2d(inplanes, planes, 1, 1, 0, 1, 1, bias=bias)\n\n    def forward(self, x):\n        x = fixed_padding(x, self.conv1.kernel_size[0], dilation=self.conv1.dilation[0])\n        x = self.conv1(x)\n        x = self.pointwise(x)\n        return x\n\n\nclass Block(nn.Module):\n    def __init__(self, inplanes, planes, reps, stride=1, dilation=1, start_with_relu=True, grow_first=True, is_last=False):\n        super(Block, self).__init__()\n\n        if planes != inplanes or stride != 1:\n            self.skip = nn.Conv2d(inplanes, planes, 1, stride=stride, bias=False)\n            self.skipbn = BatchNorm2d(planes)\n        else:\n            self.skip = None\n\n        self.relu = nn.ReLU(inplace=True)\n        rep = []\n\n        filters = inplanes\n        if grow_first:\n            rep.append(self.relu)\n            rep.append(SeparableConv2d_same(inplanes, planes, 3, stride=1, dilation=dilation))\n            rep.append(BatchNorm2d(planes))\n            filters = planes\n\n        for i in range(reps - 1):\n            rep.append(self.relu)\n            rep.append(SeparableConv2d_same(filters, filters, 3, stride=1, dilation=dilation))\n            rep.append(BatchNorm2d(filters))\n\n        if not grow_first:\n            rep.append(self.relu)\n            rep.append(SeparableConv2d_same(inplanes, planes, 3, stride=1, dilation=dilation))\n            rep.append(BatchNorm2d(planes))\n\n        if not start_with_relu:\n            rep = rep[1:]\n\n        if stride != 1:\n            rep.append(SeparableConv2d_same(planes, planes, 3, stride=2))\n\n        if stride == 1 and is_last:\n            rep.append(SeparableConv2d_same(planes, planes, 3, stride=1))\n\n\n        self.rep = nn.Sequential(*rep)\n\n    def forward(self, inp):\n        x = self.rep(inp)\n\n        if self.skip is not None:\n            skip = self.skip(inp)\n            skip = self.skipbn(skip)\n        else:\n            skip = inp\n\n        x += skip\n\n        return x\n\n\nclass Xception(nn.Module):\n    \"\"\"\n    Modified Alighed Xception\n    \"\"\"\n    def __init__(self, inplanes=3, os=16, pretrained=False):\n        super(Xception, self).__init__()\n\n        if os == 16:\n            entry_block3_stride = 2\n            middle_block_dilation = 1\n            exit_block_dilations = (1, 2)\n        elif os == 8:\n            entry_block3_stride = 1\n            middle_block_dilation = 2\n            exit_block_dilations = (2, 4)\n        else:\n            raise NotImplementedError\n\n\n        # Entry flow\n        self.conv1 = nn.Conv2d(inplanes, 32, 3, stride=2, padding=1, bias=False)\n        self.bn1 = BatchNorm2d(32)\n        self.relu = nn.ReLU(inplace=True)\n\n        self.conv2 = nn.Conv2d(32, 64, 3, stride=1, padding=1, bias=False)\n        self.bn2 = BatchNorm2d(64)\n\n        self.block1 = Block(64, 128, reps=2, stride=2, start_with_relu=False)\n        self.block2 = Block(128, 256, reps=2, stride=2, start_with_relu=True, grow_first=True)\n        self.block3 = Block(256, 728, reps=2, stride=entry_block3_stride, start_with_relu=True, grow_first=True,\n                            is_last=True)\n\n        # Middle flow\n        self.block4  = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, start_with_relu=True, grow_first=True)\n        self.block5  = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, start_with_relu=True, grow_first=True)\n        self.block6  = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, start_with_relu=True, grow_first=True)\n        self.block7  = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, start_with_relu=True, grow_first=True)\n        self.block8  = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, start_with_relu=True, grow_first=True)\n        self.block9  = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, start_with_relu=True, grow_first=True)\n        self.block10 = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, start_with_relu=True, grow_first=True)\n        self.block11 = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, start_with_relu=True, grow_first=True)\n        self.block12 = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, start_with_relu=True, grow_first=True)\n        self.block13 = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, start_with_relu=True, grow_first=True)\n        self.block14 = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, start_with_relu=True, grow_first=True)\n        self.block15 = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, start_with_relu=True, grow_first=True)\n        self.block16 = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, start_with_relu=True, grow_first=True)\n        self.block17 = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, start_with_relu=True, grow_first=True)\n        self.block18 = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, start_with_relu=True, grow_first=True)\n        self.block19 = Block(728, 728, reps=3, stride=1, dilation=middle_block_dilation, start_with_relu=True, grow_first=True)\n\n        # Exit flow\n        self.block20 = Block(728, 1024, reps=2, stride=1, dilation=exit_block_dilations[0],\n                             start_with_relu=True, grow_first=False, is_last=True)\n\n        self.conv3 = SeparableConv2d_same(1024, 1536, 3, stride=1, dilation=exit_block_dilations[1])\n        self.bn3 = BatchNorm2d(1536)\n\n        self.conv4 = SeparableConv2d_same(1536, 1536, 3, stride=1, dilation=exit_block_dilations[1])\n        self.bn4 = BatchNorm2d(1536)\n\n        self.conv5 = SeparableConv2d_same(1536, 2048, 3, stride=1, dilation=exit_block_dilations[1])\n        self.bn5 = BatchNorm2d(2048)\n\n        # Init weights\n        self._init_weight()\n\n        # Load pretrained model\n        if pretrained:\n            self._load_xception_pretrained()\n\n    def forward(self, x):\n        # Entry flow\n        x = self.conv1(x)\n        x = self.bn1(x)\n        x = self.relu(x)\n\n        x = self.conv2(x)\n        x = self.bn2(x)\n        x = self.relu(x)\n\n        x = self.block1(x)\n        low_level_feat = x\n        x = self.block2(x)\n        x = self.block3(x)\n\n        # Middle flow\n        x = self.block4(x)\n        x = self.block5(x)\n        x = self.block6(x)\n        x = self.block7(x)\n        x = self.block8(x)\n        x = self.block9(x)\n        x = self.block10(x)\n        x = self.block11(x)\n        x = self.block12(x)\n        x = self.block13(x)\n        x = self.block14(x)\n        x = self.block15(x)\n        x = self.block16(x)\n        x = self.block17(x)\n        x = self.block18(x)\n        x = self.block19(x)\n\n        # Exit flow\n        x = self.block20(x)\n        x = self.conv3(x)\n        x = self.bn3(x)\n        x = self.relu(x)\n\n        x = self.conv4(x)\n        x = self.bn4(x)\n        x = self.relu(x)\n\n        x = self.conv5(x)\n        x = self.bn5(x)\n        x = self.relu(x)\n\n        return x, low_level_feat\n\n    def _init_weight(self):\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n                m.weight.data.normal_(0, math.sqrt(2. / n))\n            elif isinstance(m, BatchNorm2d):\n                m.weight.data.fill_(1)\n                m.bias.data.zero_()\n\n    def _load_xception_pretrained(self):\n        pretrain_dict = model_zoo.load_url('https://uc0112711a29e66e3d2fb148d00e.dl.dropboxusercontent.com/cd/0/get/ArzxK1hYhkVLHHaVB0rGgjFydRENeOMI84-MIpzVlrN4NwOaS8xzfm9hek7ENhW1DLpIFHteA_4jHGcVP9-30mGx0CRZJIg1yBrPe-ytJSGLo8NEhovFBqXvdrevCz6w73c/file#')\n        model_dict = {}\n        state_dict = self.state_dict()\n\n        for k, v in pretrain_dict.items():\n            if k in model_dict:\n                if 'pointwise' in k:\n                    v = v.unsqueeze(-1).unsqueeze(-1)\n                if k.startswith('block11'):\n                    model_dict[k] = v\n                    model_dict[k.replace('block11', 'block12')] = v\n                    model_dict[k.replace('block11', 'block13')] = v\n                    model_dict[k.replace('block11', 'block14')] = v\n                    model_dict[k.replace('block11', 'block15')] = v\n                    model_dict[k.replace('block11', 'block16')] = v\n                    model_dict[k.replace('block11', 'block17')] = v\n                    model_dict[k.replace('block11', 'block18')] = v\n                    model_dict[k.replace('block11', 'block19')] = v\n                elif k.startswith('block12'):\n                    model_dict[k.replace('block12', 'block20')] = v\n                elif k.startswith('bn3'):\n                    model_dict[k] = v\n                    model_dict[k.replace('bn3', 'bn4')] = v\n                elif k.startswith('conv4'):\n                    model_dict[k.replace('conv4', 'conv5')] = v\n                elif k.startswith('bn4'):\n                    model_dict[k.replace('bn4', 'bn5')] = v\n                else:\n                    model_dict[k] = v\n        state_dict.update(model_dict)\n        self.load_state_dict(state_dict)\n\nclass ASPP_module(nn.Module):\n    def __init__(self, inplanes, planes, dilation):\n        super(ASPP_module, self).__init__()\n        if dilation == 1:\n            kernel_size = 1\n            padding = 0\n        else:\n            kernel_size = 3\n            padding = dilation\n        self.atrous_convolution = nn.Conv2d(inplanes, planes, kernel_size=kernel_size,\n                                            stride=1, padding=padding, dilation=dilation, bias=False)\n        self.bn = BatchNorm2d(planes)\n        self.relu = nn.ReLU()\n\n        self._init_weight()\n\n    def forward(self, x):\n        x = self.atrous_convolution(x)\n        x = self.bn(x)\n\n        return self.relu(x)\n\n    def _init_weight(self):\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n                m.weight.data.normal_(0, math.sqrt(2. / n))\n            elif isinstance(m, BatchNorm2d):\n                m.weight.data.fill_(1)\n                m.bias.data.zero_()\n\n\nclass DeepLabv3_plus(nn.Module):\n    def __init__(self, nInputChannels=3, n_classes=21, os=16, pretrained=False, freeze_bn=False, _print=True):\n        if _print:\n            print(\"Constructing DeepLabv3+ model...\")\n            print(\"Backbone: Xception\")\n            print(\"Number of classes: {}\".format(n_classes))\n            print(\"Output stride: {}\".format(os))\n            print(\"Number of Input Channels: {}\".format(nInputChannels))\n        super(DeepLabv3_plus, self).__init__()\n\n        # Atrous Conv\n        self.xception_features = Xception(nInputChannels, os, pretrained)\n\n        # ASPP\n        if os == 16:\n            dilations = [1, 6, 12, 18]\n        elif os == 8:\n            dilations = [1, 12, 24, 36]\n        else:\n            raise NotImplementedError\n\n        self.aspp1 = ASPP_module(2048, 256, dilation=dilations[0])\n        self.aspp2 = ASPP_module(2048, 256, dilation=dilations[1])\n        self.aspp3 = ASPP_module(2048, 256, dilation=dilations[2])\n        self.aspp4 = ASPP_module(2048, 256, dilation=dilations[3])\n\n        self.relu = nn.ReLU()\n\n        self.global_avg_pool = nn.Sequential(nn.AdaptiveAvgPool2d((1, 1)),\n                                             nn.Conv2d(2048, 256, 1, stride=1, bias=False),\n                                             BatchNorm2d(256),\n                                             nn.ReLU())\n\n        self.conv1 = nn.Conv2d(1280, 256, 1, bias=False)\n        self.bn1 = BatchNorm2d(256)\n\n        # adopt [1x1, 48] for channel reduction.\n        self.conv2 = nn.Conv2d(128, 48, 1, bias=False)\n        self.bn2 = BatchNorm2d(48)\n\n        self.last_conv = nn.Sequential(nn.Conv2d(304, 256, kernel_size=3, stride=1, padding=1, bias=False),\n                                       BatchNorm2d(256),\n                                       nn.ReLU(),\n                                       nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1, bias=False),\n                                       BatchNorm2d(256),\n                                       nn.ReLU(),\n                                       nn.Conv2d(256, n_classes, kernel_size=1, stride=1))\n        if freeze_bn:\n            self._freeze_bn()\n\n    def forward(self, input):\n        x, low_level_features = self.xception_features(input)\n        x1 = self.aspp1(x)\n        x2 = self.aspp2(x)\n        x3 = self.aspp3(x)\n        x4 = self.aspp4(x)\n        x5 = self.global_avg_pool(x)\n        x5 = F.interpolate(x5, size=x4.size()[2:], mode='bilinear', align_corners=True)\n\n        x = torch.cat((x1, x2, x3, x4, x5), dim=1)\n\n        x = self.conv1(x)\n        x = self.bn1(x)\n        x = self.relu(x)\n        x = F.interpolate(x, size=(int(math.ceil(input.size()[-2]/4)),\n                                int(math.ceil(input.size()[-1]/4))), mode='bilinear', align_corners=True)\n\n        low_level_features = self.conv2(low_level_features)\n        low_level_features = self.bn2(low_level_features)\n        low_level_features = self.relu(low_level_features)\n\n\n        x = torch.cat((x, low_level_features), dim=1)\n        x = self.last_conv(x)\n        x = F.interpolate(x, size=input.size()[2:], mode='bilinear', align_corners=True)\n\n        return x\n\n    def _freeze_bn(self):\n        for m in self.modules():\n            if isinstance(m, BatchNorm2d):\n                m.eval()\n\n    def _init_weight(self):\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n                m.weight.data.normal_(0, math.sqrt(2. / n))\n            elif isinstance(m, BatchNorm2d):\n                m.weight.data.fill_(1)\n                m.bias.data.zero_()\n\ndef get_1x_lr_params(model):\n    \"\"\"\n    This generator returns all the parameters of the net except for\n    the last classification layer. Note that for each batchnorm layer,\n    requires_grad is set to False in deeplab_resnet.py, therefore this function does not return\n    any batchnorm parameter\n    \"\"\"\n    b = [model.xception_features]\n    for i in range(len(b)):\n        for k in b[i].parameters():\n            if k.requires_grad:\n                yield k\n\n\ndef get_10x_lr_params(model):\n    \"\"\"\n    This generator returns all the parameters for the last layer of the net,\n    which does the classification of pixel into classes\n    \"\"\"\n    b = [model.aspp1, model.aspp2, model.aspp3, model.aspp4, model.conv1, model.conv2, model.last_conv]\n    for j in range(len(b)):\n        for k in b[j].parameters():\n            if k.requires_grad:\n                yield k\n                \ndef DeepLab_model(in_channels=3, num_output_classes=2):\n    model = DeepLabv3_plus(nInputChannels=in_channels, \n                           n_classes=num_output_classes, os=16, \n                           pretrained=False, _print=True)\n    model.eval()\n    # Optional, for multi GPU training and inference\n    model = nn.DataParallel(model)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def box_in_image(box, intrinsic, image_size) -> bool:\n    \"\"\"Check if a box is visible inside an image without accounting for occlusions.\n    Args:\n        box: The box to be checked.\n        intrinsic: <float: 3, 3>. Intrinsic camera matrix.\n        image_size: (width, height)\n        vis_level: One of the enumerations of <BoxVisibility>.\n    Returns: True if visibility condition is satisfied.\n    \"\"\"\n\n    corners_3d = box.corners()\n    corners_img = view_points(corners_3d, intrinsic, normalize=True)[:2, :]\n\n    visible = np.logical_and(corners_img[0, :] > 0, corners_img[0, :] < image_size[0])\n    visible = np.logical_and(visible, corners_img[1, :] < image_size[1])\n    visible = np.logical_and(visible, corners_img[1, :] > 0)\n    visible = np.logical_and(visible, corners_3d[2, :] > 1)\n\n    in_front = corners_3d[2, :] > 0.1  # True if a corner is at least 0.1 meter in front of the camera.\n\n    return any(visible) and all(in_front)\n\nall_pred_fn = []\ndef viz_unet(sample_token,boxes): \n\n    sample = level5data.get(\"sample\", sample_token)\n\n    sample_camera_token = sample[\"data\"][\"CAM_FRONT\"]\n    camera_data = level5data.get(\"sample_data\", sample_camera_token)\n    # camera_filepath = level5data.get_sample_data_path(sample_camera_token)\n\n    ego_pose = level5data.get(\"ego_pose\", camera_data[\"ego_pose_token\"])\n    calibrated_sensor = level5data.get(\"calibrated_sensor\", camera_data[\"calibrated_sensor_token\"])\n    data_path, _, camera_intrinsic = level5data.get_sample_data(sample_camera_token)\n\n\n    data = Image.open(data_path)\n    _, axis = plt.subplots(1, 1, figsize=(9, 9))\n    \n    for i,box in enumerate(boxes):\n\n        # Move box to ego vehicle coord system\n        box.translate(-np.array(ego_pose[\"translation\"]))\n        box.rotate(Quaternion(ego_pose[\"rotation\"]).inverse)\n\n        # Move box to sensor coord system\n        box.translate(-np.array(calibrated_sensor[\"translation\"]))\n        box.rotate(Quaternion(calibrated_sensor[\"rotation\"]).inverse)\n\n        if box_in_image(box,camera_intrinsic,np.array(data).shape):            \n            box.render(axis,camera_intrinsic,normalize=True)\n\n    axis.imshow(data)\n    all_pred_fn.append(f'./cam_viz/cam_preds_{sample_token}.jpg')\n    plt.savefig(all_pred_fn[-1])\n    plt.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# We weigh the loss for the 0 class lower to account for (some of) the big class imbalance.\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nclass_weights = torch.from_numpy(np.array([0.2] + [1.0]*len(classes), dtype=np.float32))\nclass_weights = class_weights.to(device)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loading trained Unet models"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nos.listdir('/kaggle/input')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 16\nmodel1 = DeepLab_model(num_output_classes=len(classes)+1)\n# model1 = get_unet_model(num_output_classes=len(classes)+1)\n\nstate = torch.load('/kaggle/input/lyftunet01/lyft-Unet-epoch-49-loss-0.0.pth')\nmodel1.load_state_dict(state)\nmodel1 = model1.to(device)\nmodel1.eval();\n\n\n# model2 = DeepLab_model(num_output_classes=len(classes)+1)\n# model2 = get_unet_model(num_output_classes=len(classes)+1)\n\n# state = torch.load('/kaggle/input/lyftunet01/lyft-Unet-epoch-48-loss-0.0.pth')\n# model2.load_state_dict(state)\n# model2 = model2.to(device)\n# model2.eval();\n\n\n# model = DeepLab_model(num_output_classes=len(classes)+1)\n# model3 = get_unet_model(num_output_classes=len(classes)+1)\n\n# state = torch.load('/kaggle/input/lyft3d-mask-test-data/unet_checkpoint_epoch_8.pth')\n# model3.load_state_dict(state)\n# model3 = model3.to(device)\n# model3.eval();","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## We can use MLComp to ensemble multiple models easily.\nIt can also be used to TTA. Reduces boiler plate code.\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Model:\n    def __init__(self, models):\n        self.models = models\n    \n    def __call__(self, x):\n        res = []\n        x = x.cuda()\n        with torch.no_grad():\n            for m in self.models:\n                res.append(m(x))\n        res = torch.stack(res)\n        return torch.mean(res, dim=0)\n\nmodel = Model([model1, model2])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def calc_detection_box(prediction_opened,class_probability):\n\n    sample_boxes = []\n    sample_detection_scores = []\n    sample_detection_classes = []\n    \n    contours, hierarchy = cv2.findContours(prediction_opened, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE) \n    \n    for cnt in contours:\n        rect = cv2.minAreaRect(cnt)\n        box = cv2.boxPoints(rect)\n        \n        # Let's take the center pixel value as the confidence value\n        box_center_index = np.int0(np.mean(box, axis=0))\n        \n        for class_index in range(len(classes)):\n            box_center_value = class_probability[class_index+1, box_center_index[1], box_center_index[0]]\n            \n            # Let's remove candidates with very low probability\n            if box_center_value < 0.01:\n                continue\n            \n            box_center_class = classes[class_index]\n\n            box_detection_score = box_center_value\n            sample_detection_classes.append(box_center_class)\n            sample_detection_scores.append(box_detection_score)\n            sample_boxes.append(box)\n            \n    return np.array(sample_boxes),sample_detection_scores,sample_detection_classes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# We perform an opening morphological operation to filter tiny detections\n# Note that this may be problematic for classes that are inherently small (e.g. pedestrians)..\nkernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(3,3))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Test Set Predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\ngc.collect()\ntest_loader = torch.utils.data.DataLoader(test_dataset, batch_size, shuffle=False, num_workers=os.cpu_count()*2)\nprogress_bar = tqdm_notebook(test_loader)\n\n# We quantize to uint8 here to conserve memory. We're allocating >20GB of memory otherwise.\n# predictions = np.zeros((len(test_loader), 1+len(classes), 336, 336), dtype=np.uint8)\n\nsample_tokens = []\nall_losses = []\n\ndetection_boxes = []\ndetection_scores = []\ndetection_classes = []\n\n# Arbitrary threshold in our system to create a binary image to fit boxes around.\nbackground_threshold = 200\n\nwith torch.no_grad():\n    for ii, (X, batch_sample_tokens) in enumerate(progress_bar):\n\n        sample_tokens.extend(batch_sample_tokens)\n        \n        X = X.to(device)  # [N, 1, H, W]\n        prediction = model(X)  # [N, 2, H, W]\n        \n        prediction = F.softmax(prediction, dim=1)\n        \n        prediction_cpu = prediction.cpu().numpy()\n        predictions = np.round(prediction_cpu*255).astype(np.uint8)\n        \n        # Get probabilities for non-background\n        predictions_non_class0 = 255 - predictions[:,0]\n        \n        predictions_opened = np.zeros((predictions_non_class0.shape), dtype=np.uint8)\n\n        for i, p in enumerate(predictions_non_class0):\n            thresholded_p = (p > background_threshold).astype(np.uint8)\n            predictions_opened[i] = cv2.morphologyEx(thresholded_p, cv2.MORPH_OPEN, kernel)\n    \n            sample_boxes,sample_detection_scores,sample_detection_classes = calc_detection_box(predictions_opened[i],\n                                                                                              predictions[i])\n        \n            detection_boxes.append(np.array(sample_boxes))\n            detection_scores.append(sample_detection_scores)\n            detection_classes.append(sample_detection_classes)\n        \n#         # Visualize the first prediction\n#         if ii == 0:\n#             visualize_predictions(X, prediction, apply_softmaxiii=False)\n            ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Total amount of boxes:\", np.sum([len(x) for x in detection_boxes]))\n    \n\n# Visualize the boxes in the first sample\nt = np.zeros_like(predictions_opened[0])\nfor sample_boxes in detection_boxes[0]:\n    box_pix = np.int0(sample_boxes)\n    cv2.drawContours(t,[box_pix],0,(255),2)\nplt.imshow(t)\nplt.show()\n\n# Visualize their probabilities\nplt.hist(detection_scores[0], bins=20)\nplt.xlabel(\"Detection Score\")\nplt.ylabel(\"Count\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Transform predicted boxes back into world space"},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_transformation_matrix_to_voxel_space(shape, voxel_size, offset):\n    \"\"\"\n    Constructs a transformation matrix given an output voxel shape such that (0,0,0) ends up in the center.\n    Voxel_size defines how large every voxel is in world coordinate, (1,1,1) would be the same as Minecraft voxels.\n    \n    An offset per axis in world coordinates (metric) can be provided, this is useful for Z (up-down) in lidar points.\n    \"\"\"\n    \n    shape, voxel_size, offset = np.array(shape), np.array(voxel_size), np.array(offset)\n    \n    tm = np.eye(4, dtype=np.float32)\n    translation = shape/2 + offset/voxel_size\n    \n    tm = tm * np.array(np.hstack((1/voxel_size, [1])))\n    tm[:3, 3] = np.transpose(translation)\n    return tm\n\ndef transform_points(points, transf_matrix):\n    \"\"\"\n    Transform (3,N) or (4,N) points using transformation matrix.\n    \"\"\"\n    if points.shape[0] not in [3,4]:\n        raise Exception(\"Points input should be (3,N) or (4,N) shape, received {}\".format(points.shape))\n    return transf_matrix.dot(np.vstack((points[:3, :], np.ones(points.shape[1]))))[:3, :]\n\n\ndef car_to_voxel_coords(points, shape, voxel_size, z_offset=0):\n    if len(shape) != 3:\n        raise Exception(\"Voxel volume shape should be 3 dimensions (x,y,z)\")\n        \n    if len(points.shape) != 2 or points.shape[0] not in [3, 4]:\n        raise Exception(\"Input points should be (3,N) or (4,N) in shape, found {}\".format(points.shape))\n\n    tm = create_transformation_matrix_to_voxel_space(shape, voxel_size, (0, 0, z_offset))\n    p = transform_points(points, tm)\n    return p\n\ndef create_voxel_pointcloud(points, shape, voxel_size=(0.5,0.5,1), z_offset=0):\n\n    points_voxel_coords = car_to_voxel_coords(points.copy(), shape, voxel_size, z_offset)\n    points_voxel_coords = points_voxel_coords[:3].transpose(1,0)\n    points_voxel_coords = np.int0(points_voxel_coords)\n    \n    bev = np.zeros(shape, dtype=np.float32)\n    bev_shape = np.array(shape)\n\n    within_bounds = (np.all(points_voxel_coords >= 0, axis=1) * np.all(points_voxel_coords < bev_shape, axis=1))\n    \n    points_voxel_coords = points_voxel_coords[within_bounds]\n    coord, count = np.unique(points_voxel_coords, axis=0, return_counts=True)\n        \n    # Note X and Y are flipped:\n    bev[coord[:,1], coord[:,0], coord[:,2]] = count\n    \n    return bev\n\ndef normalize_voxel_intensities(bev, max_intensity=16):\n    return (bev/max_intensity).clip(0,1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.makedirs('./cam_viz',exist_ok=True)\n\nfrom moviepy.editor import ImageSequenceClip \nfrom lyft_dataset_sdk.eval.detection.mAP_evaluation import Box3D, recall_precision\nimport shutil\n\npred_box3ds = []\n\nmax_frames = 128\nvid_count = 0\nprocessed_samples = 0\nfor (sample_token, sample_boxes, sample_detection_scores, sample_detection_class) in tqdm_notebook(zip(sample_tokens, detection_boxes, detection_scores, detection_classes), total=len(sample_tokens)):\n    processed_samples += 1\n    sample_boxes = sample_boxes.reshape(-1, 2) # (N, 4, 2) -> (N*4, 2)\n    sample_boxes = sample_boxes.transpose(1,0) # (N*4, 2) -> (2, N*4)\n\n    # Add Z dimension\n    sample_boxes = np.vstack((sample_boxes, np.zeros(sample_boxes.shape[1]),)) # (2, N*4) -> (3, N*4)\n\n    sample = level5data.get(\"sample\", sample_token)\n    sample_lidar_token = sample[\"data\"][\"LIDAR_TOP\"]\n    lidar_data = level5data.get(\"sample_data\", sample_lidar_token)\n    lidar_filepath = level5data.get_sample_data_path(sample_lidar_token)\n    ego_pose = level5data.get(\"ego_pose\", lidar_data[\"ego_pose_token\"])\n    ego_translation = np.array(ego_pose['translation'])\n\n    global_from_car = transform_matrix(ego_pose['translation'],\n                                       Quaternion(ego_pose['rotation']), inverse=False)\n\n    car_from_voxel = np.linalg.inv(create_transformation_matrix_to_voxel_space(bev_shape, voxel_size, (0, 0, z_offset)))\n\n\n    global_from_voxel = np.dot(global_from_car, car_from_voxel)\n    sample_boxes = transform_points(sample_boxes, global_from_voxel)\n\n    # We don't know at where the boxes are in the scene on the z-axis (up-down), let's assume all of them are at\n    # the same height as the ego vehicle.\n    sample_boxes[2,:] = ego_pose[\"translation\"][2]\n\n\n    # (3, N*4) -> (N, 4, 3)\n    sample_boxes = sample_boxes.transpose(1,0).reshape(-1, 4, 3)\n\n#     box_height = 1.75\n    box_height = np.array([class_heights[cls] for cls in sample_detection_class])\n\n    # Note: Each of these boxes describes the ground corners of a 3D box.\n    # To get the center of the box in 3D, we'll have to add half the height to it.\n    sample_boxes_centers = sample_boxes.mean(axis=1)\n    sample_boxes_centers[:,2] += box_height/2\n\n    # Width and height is arbitrary - we don't know what way the vehicles are pointing from our prediction segmentation\n    # It doesn't matter for evaluation, so no need to worry about that here.\n    # Note: We scaled our targets to be 0.8 the actual size, we need to adjust for that\n    sample_lengths = np.linalg.norm(sample_boxes[:,0,:] - sample_boxes[:,1,:], axis=1) * 1/box_scale\n    sample_widths = np.linalg.norm(sample_boxes[:,1,:] - sample_boxes[:,2,:], axis=1) * 1/box_scale\n    \n    sample_boxes_dimensions = np.zeros_like(sample_boxes_centers) \n    sample_boxes_dimensions[:,0] = sample_widths\n    sample_boxes_dimensions[:,1] = sample_lengths\n    sample_boxes_dimensions[:,2] = box_height\n    \n    temp = []\n    for i in range(len(sample_boxes)):\n        translation = sample_boxes_centers[i]\n        size = sample_boxes_dimensions[i]\n        class_name = sample_detection_class[i]\n        ego_distance = float(np.linalg.norm(ego_translation - translation))\n    \n        \n        # Determine the rotation of the box\n        v = (sample_boxes[i,0] - sample_boxes[i,1])\n        v /= np.linalg.norm(v)\n        r = R.from_dcm([\n            [v[0], -v[1], 0],\n            [v[1],  v[0], 0],\n            [   0,     0, 1],\n        ])\n        quat = r.as_quat()\n        # XYZW -> WXYZ order of elements\n        quat = quat[[3,0,1,2]]\n        \n        detection_score = float(sample_detection_scores[i])\n\n        \n        box3d = Box(\n            token=sample_token,\n            center=list(translation),\n            size=list(size),\n            orientation=Quaternion(quat),\n            name=class_name,\n            score=detection_score\n        )\n        \n        temp.append(box3d)\n        box3d = Box3D(\n            sample_token=sample_token,\n            translation=list(translation),\n            size=list(size),\n            rotation=list(quat),\n            name=class_name,\n            score=detection_score\n        )\n        pred_box3ds.append(box3d)\n        \n#     https://github.com/Zulko/moviepy/issues/903\n    if vid_count < 1:\n        viz_unet(sample_token,temp)\n        if processed_samples==max_frames:\n            os.makedirs('./cam_viz',exist_ok=True)\n            processed_samples = 0\n            vid_count += 1        \n            new_clip = ImageSequenceClip(all_pred_fn,fps=5)\n            all_pred_fn = []\n            new_clip.write_videofile(f\"model_preds_{vid_count}.mp4\") \n            shutil.rmtree('./cam_viz')\n            del new_clip\n            gc.collect()\n            os.makedirs('./cam_viz',exist_ok=True)\n#         os.system('rm -rf ./cam_viz')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!rm -r ./cam_viz/","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Creating Submission File"},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = {}\nfor i in tqdm_notebook(range(len(pred_box3ds))):\n#     yaw = -np.arctan2(pred_box3ds[i].rotation[2], pred_box3ds[i].rotation[0])\n    yaw = 2*np.arccos(pred_box3ds[i].rotation[0]);\n    pred =  str(pred_box3ds[i].score/255) + ' ' + str(pred_box3ds[i].center_x)  + ' '  + \\\n    str(pred_box3ds[i].center_y) + ' '  + str(pred_box3ds[i].center_z) + ' '  + \\\n    str(pred_box3ds[i].width) + ' ' \\\n    + str(pred_box3ds[i].length) + ' '  + str(pred_box3ds[i].height) + ' ' + str(yaw) + ' ' \\\n    + str(pred_box3ds[i].name) + ' ' \n        \n    if pred_box3ds[i].sample_token in sub.keys():     \n        sub[pred_box3ds[i].sample_token] += pred\n    else:\n        sub[pred_box3ds[i].sample_token] = pred        \n    \nsample_sub = pd.read_csv('../input/3d-object-detection-for-autonomous-vehicles/sample_submission.csv')\nfor token in set(sample_sub.Id.values).difference(sub.keys()):\n#     print(token)\n    sub[token] = ''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.DataFrame(list(sub.items()))\nsub.columns = sample_sub.columns\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.tail()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from IPython.display import FileLink\nos.chdir('/kaggle/working')\nsub.to_csv('Lyft-deeplabV3_exception.csv',index=False)\nFileLink('Lyft-deeplabV3_exception.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.chdir('/kaggle/working')\nos.listdir('/kaggle/working')\nFileLink('model_preds_1.mp4')\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"},"widgets":{"application/vnd.jupyter.widget-state+json":{"state":{"00312033bb1645e78614a6a22dc941b1":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":"initial"}},"0696b90d97e44f2cacc68d3b5068da2a":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":"initial"}},"0b3be90793544072b3b14f78555864a1":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"1bc2a07b66784be6a830f7c8c157d2cb":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_48649815d2d84486bc71fa593935e9df","IPY_MODEL_d48e9d143eb24e5f8352088376555d8e"],"layout":"IPY_MODEL_3aa65bd7608049d4ba00ad105a79091a"}},"1e3a207a9b604bc68f40f0f0b47de1b5":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":"initial"}},"29e3fe7cc02d4cde856eb8a87eb7582b":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"385f4fc31b81444cbf53ad668ef7479a":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"3aa65bd7608049d4ba00ad105a79091a":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"463fdcf32ee74e1b828447434e52ec8d":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_b3c912c184754016987420f52d043e4b","placeholder":"​","style":"IPY_MODEL_70e7eb63940e482db6544ab7a37c4867","value":" 3434/3434 [09:54&lt;00:00,  5.78it/s]"}},"48649815d2d84486bc71fa593935e9df":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"IntProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"IntProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"100%","description_tooltip":null,"layout":"IPY_MODEL_29e3fe7cc02d4cde856eb8a87eb7582b","max":27468,"min":0,"orientation":"horizontal","style":"IPY_MODEL_1e3a207a9b604bc68f40f0f0b47de1b5","value":27468}},"6147d329c6304030ae5b1cb03872493a":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_692aaddc32794ec0bf98c703229dd985","IPY_MODEL_463fdcf32ee74e1b828447434e52ec8d"],"layout":"IPY_MODEL_78ba6bc9beb2497c87b9f0515b867fbc"}},"692aaddc32794ec0bf98c703229dd985":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"IntProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"IntProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"100%","description_tooltip":null,"layout":"IPY_MODEL_b219fec1dbcc4ba4bc3cebde30c56e5d","max":3434,"min":0,"orientation":"horizontal","style":"IPY_MODEL_0696b90d97e44f2cacc68d3b5068da2a","value":3434}},"70e7eb63940e482db6544ab7a37c4867":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"7621ac2d34ae4b8b8b2ab68acd6307bb":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"78ba6bc9beb2497c87b9f0515b867fbc":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"81361c3a0f6f4dd18b7d46199bf48e8a":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"IntProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"IntProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"100%","description_tooltip":null,"layout":"IPY_MODEL_bd2f0281ab474513aa49220b44c46506","max":684746,"min":0,"orientation":"horizontal","style":"IPY_MODEL_00312033bb1645e78614a6a22dc941b1","value":684746}},"afb47c7ae7e94849ad40c62d59819300":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_0b3be90793544072b3b14f78555864a1","placeholder":"​","style":"IPY_MODEL_e3a58edb345d4a65a910a2c76eb4c4ee","value":" 684746/684746 [00:10&lt;00:00, 63123.43it/s]"}},"b219fec1dbcc4ba4bc3cebde30c56e5d":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"b3c912c184754016987420f52d043e4b":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"bd2f0281ab474513aa49220b44c46506":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"caacf34d69cf48e88b0caac4030cd2f1":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_81361c3a0f6f4dd18b7d46199bf48e8a","IPY_MODEL_afb47c7ae7e94849ad40c62d59819300"],"layout":"IPY_MODEL_7621ac2d34ae4b8b8b2ab68acd6307bb"}},"d2136c951743454bab59ff71590be6da":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"d48e9d143eb24e5f8352088376555d8e":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_385f4fc31b81444cbf53ad668ef7479a","placeholder":"​","style":"IPY_MODEL_d2136c951743454bab59ff71590be6da","value":" 27468/27468 [05:51&lt;00:00, 78.21it/s]"}},"e3a58edb345d4a65a910a2c76eb4c4ee":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}}},"version_major":2,"version_minor":0}}},"nbformat":4,"nbformat_minor":1}