{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport os\n\nprint(os.listdir('/kaggle/input/3d-object-detection-for-autonomous-vehicles'))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install pyquaternion","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import json\nimport os.path\n\nfrom matplotlib import pyplot as plt\nimport numpy as np\nimport pandas as pd\nfrom pyquaternion import Quaternion\n\nfrom matplotlib import pyplot\nfrom mpl_toolkits.mplot3d import Axes3D\nimport random\nimport itertools\nfrom skimage.morphology import convex_hull_image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Table:\n    def __init__(self, data):\n        self.data = data\n        self.index = {x['token']: x for x in data}\n\n\nDATA_ROOT = '/kaggle/input/3d-object-detection-for-autonomous-vehicles/'\n\n\ndef load_table(name, root=os.path.join(DATA_ROOT, 'train_data')):\n    with open(os.path.join(root, name), 'rb') as f:\n        return Table(json.load(f))\n\n    \nscene = load_table('scene.json')\nsample = load_table('sample.json')\nsample_data = load_table('sample_data.json')\nego_pose = load_table('ego_pose.json')\ncalibrated_sensor = load_table('calibrated_sensor.json')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(os.path.join(DATA_ROOT, 'train.csv')).set_index('Id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def rotate_points(points, rotation, inverse=False):\n    assert points.shape[1] == 3\n    q = Quaternion(rotation)\n    if inverse:\n        q = q.inverse\n    return np.dot(q.rotation_matrix, points.T).T\n    \ndef apply_pose(points, cs, inverse=False):\n    \"\"\" Translate (lidar) points to vehicle coordinates, given a calibrated sensor.\n    \"\"\"\n    points = rotate_points(points, cs['rotation'])\n    points = points + np.array(cs['translation'])\n    return points\n\ndef inverse_apply_pose(points, cs):\n    \"\"\" Reverse of apply_pose (we'll need it later).\n    \"\"\"\n    points = points - np.array(cs['translation']) \n    points = rotate_points(points, np.array(cs['rotation']), inverse=True)\n    return points\n\ndef get_annotations(token):\n    annotations = np.array(train_df.loc[token].PredictionString.split()).reshape(-1, 8)\n    return annotations, {\n        'point': annotations[:, :3].astype(np.float32),\n        'wlh': annotations[:, 3:6].astype(np.float32),\n        'rotation': annotations[:, 6].astype(np.float32),\n        'cls': np.array(annotations[:, 7]),\n    }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import copy\n\nimport math\n\ndef rotate(origin, point, angle):\n    ox, oy, _ = origin\n    px, py, pz = point\n\n    qx = ox + math.cos(angle) * (px - ox) - math.sin(angle) * (py - oy)\n    qy = oy + math.sin(angle) * (px - ox) + math.cos(angle) * (py - oy)\n    return [qx, qy, pz]\n\n\ndef make_box_coords(center, wlh, rotation, ep):\n\n    planar_wlh = copy.deepcopy(wlh)\n    planar_wlh = planar_wlh[[1,0,2]]\n\n    bottom_center = copy.deepcopy(center)\n    bottom_center[-1] = bottom_center[-1] - planar_wlh[-1] / 2\n\n    bottom_points = []\n    bottom_points.append(bottom_center + planar_wlh * [1, 1, 0] / 2)\n    bottom_points.append(bottom_center + planar_wlh * [-1, -1, 0] / 2)\n    bottom_points.append(bottom_center + planar_wlh * [1, -1, 0] / 2)\n    bottom_points.append(bottom_center + planar_wlh * [-1, 1, 0] / 2)\n    bottom_points = np.array(bottom_points)\n\n    rotated_bottom_points = []\n    for point in bottom_points:\n        rotated_bottom_points.append(rotate(bottom_center, point, rotation))\n\n    rotated_bottom_points = np.array(rotated_bottom_points)\n    rotated_top_points = rotated_bottom_points + planar_wlh * [0,0,1]\n\n    box_points = np.concatenate([rotated_bottom_points, rotated_top_points], axis=0)\n\n    box_points = inverse_apply_pose(box_points, ep)\n    \n    return box_points","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_sample_data(sample_token):\n    lidars = []\n    for x in sample_data.data:\n        if x['sample_token'] == sample_token and 'lidar' in x['filename']:\n            lidars.append(x)\n\n    lidars_data = [\n        # here, sorry\n        np.fromfile(os.path.join(DATA_ROOT, x['filename'].replace('lidar/', 'train_lidar/')), dtype=np.float32)\n        .reshape(-1, 5)[:, :3] for x in lidars]\n\n\n    all_points = []\n    all_colors = []\n    for points, lidar in zip(lidars_data, lidars):\n        cs = calibrated_sensor.index[lidar['calibrated_sensor_token']]\n        points = apply_pose(points, cs)\n        all_points.append(points)\n    all_points = np.concatenate(all_points)\n\n\n    ego_pose_token, = {x['ego_pose_token'] for x in lidars}\n    ep = ego_pose.index[ego_pose_token]\n    all_annotations, annotations = get_annotations(sample_token)\n    car_centers = annotations['point'][annotations['cls'] == 'car']\n    car_wlhs = annotations['wlh'][annotations['cls'] == 'car']\n    car_rotations = annotations['rotation'][annotations['cls'] == 'car']\n    \n    # print(annotations)\n\n    all_boxes = []\n    for k in range(len(car_centers)):\n        center = car_centers[k]\n        wlh = car_wlhs[k]\n        rotation = car_rotations[k]\n\n        box_coords = make_box_coords(center, wlh, rotation, ep)\n        all_boxes.append(box_coords)\n\n    all_boxes = np.array(all_boxes)    \n\n    car_centers = inverse_apply_pose(car_centers, ep)\n    \n    return all_annotations, all_points, all_boxes, car_centers\n\n\ndef get_sample_raster(all_points, all_boxes): \n    x_bounds = np.linspace(-100, 100, 1001)\n    y_bounds = np.linspace(-100, 100, 1001)\n    z_bounds = np.linspace(-10, 10, 101)\n\n    sample_hist = np.histogramdd(all_points[:], [x_bounds, y_bounds, z_bounds])[0]\n    sample_mask = np.zeros((len(x_bounds)-1, len(y_bounds)-1, len(z_bounds)-1))\n\n\n\n    for box in all_boxes:\n        x_min, y_min, z_min = box.min(axis=0)\n        x_max, y_max, z_max = box.max(axis=0)\n\n        x_box_bound_cnt = int(1001 / 200 * (x_max - x_min))\n        y_box_bound_cnt = int(1001 / 200 * (y_max - y_min))\n        z_box_bound_cnt = int(101 / 20 * (z_max - z_min))\n\n        box_hist = np.histogramdd(box, [np.linspace(x_min, x_max, x_box_bound_cnt),\n                                        np.linspace(y_min, y_max, y_box_bound_cnt),\n                                        np.linspace(z_min, z_max, z_box_bound_cnt)])[0]\n\n        box_mask = convex_hull_image(box_hist)\n\n\n        x_start_idx = np.where(x_bounds > x_min)[0][0]\n        y_start_idx = np.where(y_bounds > y_min)[0][0]\n        z_start_idx = np.where(z_bounds > z_min)[0][0]\n\n\n        x_cnt = min(sample_mask.shape[0] - x_start_idx - 1, x_box_bound_cnt - 1)\n        y_cnt = min(sample_mask.shape[1] - y_start_idx - 1, y_box_bound_cnt - 1)\n        z_cnt = min(sample_mask.shape[2] - z_start_idx - 1, z_box_bound_cnt - 1)\n\n        sample_mask[x_start_idx:x_start_idx+x_cnt,\n                   y_start_idx:y_start_idx+y_cnt,\n                   z_start_idx:z_start_idx+z_cnt] = sample_mask[x_start_idx:x_start_idx+x_cnt,\n                                                                           y_start_idx:y_start_idx+y_cnt,\n                                                                           z_start_idx:z_start_idx+z_cnt] + box_mask[:x_cnt, :y_cnt, :z_cnt]\n\n    return sample_hist, sample_mask, (x_bounds, y_bounds, z_bounds)\n\n\ndef get_crop_positive(sample_hist, sample_mask, bounds, car_centers, crop_size=(64, 64, 32)):\n    \n    half_x_size = crop_size[0] // 2\n    half_y_size = crop_size[1] // 2\n    half_z_size = crop_size[2] // 2\n    \n    (x_bounds, y_bounds, z_bounds) = bounds\n    if len(car_centers) > 0:\n        idx = np.random.choice(range(len(car_centers)))\n        x_center, y_center, z_center = car_centers[idx]\n    else:\n        x_center, y_center = np.random.randint(-30, 30, 2)\n        z_center = np.random.randint(-10, 10)\n\n    x_center, y_center, z_center = [x_center, y_center, z_center] + np.random.randint(-3, 3, 3)\n\n    x_center = min(x_center, 100 - np.abs(x_bounds[-1] - x_bounds[-2]) * half_x_size - 1)\n    x_center = max(x_center, -100 + np.abs(x_bounds[-1] - x_bounds[-2]) * half_x_size + 1)\n\n    y_center = min(y_center, 100 - np.abs(y_bounds[-1] - y_bounds[-2]) * half_y_size - 1)\n    y_center = max(y_center, -100 + np.abs(y_bounds[-1] - y_bounds[-2]) * half_y_size + 1)\n\n    z_center = min(z_center, 10 - np.abs(z_bounds[-1] - z_bounds[-2]) * half_z_size - 1)\n    z_center = max(z_center, -10 + np.abs(z_bounds[-1] - z_bounds[-2]) * half_z_size + 1)\n\n\n\n\n    x_center_idx = np.where(x_bounds > x_center)[0][0]\n    y_center_idx = np.where(y_bounds > y_center)[0][0]\n    z_center_idx = np.where(z_bounds > z_center)[0][0]\n\n        \n    crop_hist = sample_hist[x_center_idx-half_x_size:x_center_idx+half_x_size,\n                            y_center_idx-half_y_size:y_center_idx+half_y_size,\n                            z_center_idx-half_z_size:z_center_idx+half_z_size]\n    \n    crop_mask = sample_mask[x_center_idx-half_x_size:x_center_idx+half_x_size,\n                            y_center_idx-half_y_size:y_center_idx+half_y_size,\n                            z_center_idx-half_z_size:z_center_idx+half_z_size]\n\n    return crop_hist, crop_mask > 0\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_token = train_df.reset_index()['Id'].values[35]\n\nall_annotations, all_points, all_boxes, car_centers = get_sample_data(sample_token)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Full raster image for one sample"},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_hist, sample_mask, bounds = get_sample_raster(all_points, all_boxes)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Crop from this image"},{"metadata":{"trusted":true},"cell_type":"code","source":"crop_hist, crop_mask = get_crop_positive(sample_hist, sample_mask, bounds, car_centers, crop_size=(128,128,64))\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Source point cloud for full sample"},{"metadata":{"trusted":true},"cell_type":"code","source":"wandb.init(entity=\"wandb\", project=\"lyft\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_points.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"boxes_coords = np.concatenate(all_boxes, axis=0)\n\nplt.figure(figsize=(25,15))\nplt.scatter(all_points[:20000, 0], all_points[:20000, 1],s=[0.1]*len(all_points[:20000]))\n#plt.scatter(car_centers[nearest_idxs, 0], car_centers[nearest_idxs, 1],s=[15]*len(nearest_idxs),color='r')\nplt.scatter(boxes_coords[:, 0], boxes_coords[:, 1],s=[15]*len(boxes_coords),color='r')\nwandb.log({\"bounding_boxes\": plt})\nplt.savefig('bounding_boxes.png')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import cv2\nbb = cv2.imread(\"bounding_boxes.png\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"wandb.log({\"bounding_box\": [wandb.Image(bb)]})","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Logging in wandb"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install wandb -qq","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import wandb\nfrom wandb.keras import WandbCallback","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_points.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"wandb.init(entity=\"wandb\", project=\"lyft\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# center_x center_y center_z width length height yaw class_name\n'''\n\"points\": np.array([[0.4, 1, 1.3], [1, 1, 1], [1.2, 1, 1.2]]),\n\"boxes\": np.array(\n    [\n        {\n            \"x\": 0,\n            \"y\": 0,\n            \"height\": 3,\n            \"width\": 2,\n            \"depth\": 1,\n        },\n        {\n            \"x\": 0.3,\n            \"y\": 0,\n            \"height\": 0.2,\n            \"width\": 0.2,\n            \"depth\": 4,\n        }\n    ]\n'''\nprint('------------------------------------- Array Shape: ', all_annotations.shape)\nprint('\\n\\n------------------------------------- Array: ', all_annotations)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"center_x = all_annotations[:,0]\ncenter_y = all_annotations[:,1]\ncenter_z = all_annotations[:,2]\nheight = all_annotations[:,5]\nwidth = all_annotations[:,3]\nlength = all_annotations[:,4]\nyaw = all_annotations[:,6]\nclass_name =all_annotations[:,7]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(center_z)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"boxes = []\nfor i in range(len(center_x)):\n    a_dict = {\n                \"x\": float(center_x[i]),\n                \"y\": float(center_y[i]),\n                \"height\": float(height[i]),\n                \"width\": float(width[i]),\n                \"depth\": float(length[i])\n            }\n    boxes.append(a_dict)\nboxes = np.array(boxes)\ncenter = np.array([1, 1, 1])\nprint('Points:',type(all_points),'\\n',all_points[:5])\nprint('\\nBoxes:',type(all_points),'\\n',boxes[:5])\nprint('\\nCenters:',type(center),'\\n',center)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"wandb.log({\"point_cloud\": wandb.Object3D(all_points)})\n'''\nwandb.log(\n    {\n        \"point_scene\": wandb.Object3D(\n            {\n                \"type\": \"scene/v1\",\n                \"points\": all_points,\n                \"boxes\": boxes,\n                \"center\": [1, 1, 1]\n            }\n        )\n    }\n)\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ann_idx = 1\n\ncenter_point = car_centers[ann_idx]\nx_min = center_point[0] - 5\nx_max = center_point[0] + 5\ny_min = center_point[1] - 5\ny_max = center_point[1] + 5\nz_min= center_point[2] - 5\nz_max = center_point[2] + 5\n\n\narea_mask = (all_points[:,0] > x_min) * (all_points[:, 0] < x_max) * (all_points[:, 1] > y_min) * (all_points[:, 1] < y_max) * (all_points[:, 2] > z_min) * (all_points[:, 2] < z_max)\narea_mask = np.where(area_mask)[0]\n\n\nfig = pyplot.figure(figsize=(25,15))\nax = Axes3D(fig)\nax.scatter(all_points[area_mask, 0], all_points[area_mask, 1], all_points[area_mask, 2])\n\n#ax.scatter([center_point[0]], [center_point[1]], [center_point[2]], color='k', s=[100])\nax.scatter(all_boxes[ann_idx][:, 0], all_boxes[ann_idx][:, 1], all_boxes[ann_idx][:, 2], color='r', s=[100])\n\n\npyplot.show()\npyplot.savefig('bounding_coords.png')\nbc = cv2.imread(\"bounding_coords.png\")\nwandb.log({\"bounding_coords\": [wandb.Image(bc)]})","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Rastered crop of a small region"},{"metadata":{},"cell_type":"markdown","source":"Show only top projection of 3d image"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(1, 2, figsize=(20,20))\naxes[0].imshow(crop_hist.sum(axis=-1))\naxes[1].imshow(crop_mask.sum(axis=-1))\n    \nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Neural net"},{"metadata":{},"cell_type":"markdown","source":"### Create generator for nn"},{"metadata":{"trusted":true},"cell_type":"code","source":"tokens = train_df.reset_index()['Id'].values\n\ndef generator(tokens, crop_size, batch_size):\n    while True:\n        sample_token = np.random.choice(tokens)\n        all_annotations, all_points, all_boxes, car_centers = get_sample_data(sample_token)\n        sample_hist, sample_mask, bounds = get_sample_raster(all_points, all_boxes)\n        \n        x_batch = []\n        y_batch = []\n        for _ in range(batch_size):\n            crop_hist, crop_mask = get_crop_positive(sample_hist, sample_mask, bounds, car_centers, crop_size=(128, 128, 64))\n            crop_hist, crop_mask = get_crop_positive(sample_hist, sample_mask, bounds, car_centers, crop_size=crop_size)\n\n            x_batch.append(crop_hist)\n            y_batch.append(crop_mask)\n        \n        x_batch = np.array(x_batch)\n        y_batch = np.array(y_batch)\n        \n        x_batch = np.expand_dims(x_batch, axis=1)\n        y_batch = np.expand_dims(y_batch, axis=1)\n        \n        yield x_batch , y_batch","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Train and validation generators"},{"metadata":{"trusted":true},"cell_type":"code","source":"points_train = 10000\npoints_val = 5000\ntrain_loader = generator(tokens[:points_train], (64,64,32), 16)\nval_loader = generator(tokens[15000:], (64,64,32), 16)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for x_batch, y_batch in train_loader:\n    break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"i = 0\nfig, axes = plt.subplots(1, 2, figsize=(20,20))\naxes[0].imshow(x_batch[i].sum(axis=(0, -1)))\naxes[1].imshow(y_batch[i].sum(axis=(0, -1)))\n    \nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Simple 3d Unet"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras import backend as K\nfrom keras.engine import Input, Model\nfrom keras.layers import Conv3D, MaxPooling3D, UpSampling3D, Activation, BatchNormalization, PReLU, Deconvolution3D\nfrom keras.optimizers import Adam\nimport keras\nfrom keras.models import Model, Sequential\nfrom keras.callbacks import ModelCheckpoint","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Configure the sweep – specify the parameters to search through, the search strategy, the optimization metric et all.\nsweep_config = {\n    'method': 'random', #grid, random\n    'metric': {\n      'name': 'accuracy',\n      'goal': 'maximize'   \n    },\n    'parameters': {\n        'epochs': {\n            'values': [2, 10, 20]\n        },\n        'train_size': {\n            'values': [10, 100, 500, 2000, 5000, 10000, 15000]\n        },\n        'base_filters': {\n            'values': [8, 16, 32]\n        },\n        'weight_decay': {\n            'values': [1e-5, 1e-4, 1e-3]\n        },\n        'learning_rate': {\n            'values': [0.01, 0.001, 0.0001, 0.0003, 0.00001, 0.00003]\n        },\n        'optimizer': {\n            'values': ['adam', 'nadam', 'sgd', 'rmsprop']\n        },\n        'batch_norm': {\n            'values': [False, True]\n        },\n        'depth': {\n            'values': [1, 2, 4, 6, 8]\n        }\n    }\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Initialize a new sweep\n# Arguments:\n#     – sweep_config: the sweep config dictionary defined above\n#     – entity: Set the username for the sweep\n#     – project: Set the project name for the sweep\nsweep_id = wandb.sweep(sweep_config, entity=\"wandb\", project=\"lyft\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# The sweep calls this function with each set of hyperparameters\ndef train():\n    # Default values for hyper-parameters we're going to sweep over\n    config_defaults = {\n        'epochs': 10,\n        'train_size': 5000,\n        'learning_rate': 0.00001,\n        'weight_decay': 1e-5,\n        'activation': 'relu',\n        'optimizer': 'adam',\n        'batch_norm': False,\n        'base_filters': 32,\n        'depth': 4,\n        'seed': 42\n    }\n\n    # Initilize a new wandb run\n    wandb.init(config=config_defaults)\n    \n    # Config is a variable that holds and saves hyperparameters and inputs\n    config = wandb.config\n    \n    train_loader = generator(tokens[:config.train_size], (64,64,32), 16)\n    val_loader = generator(tokens[15000:], (64,64,32), 16)\n    \n    K.set_image_data_format(\"channels_first\")\n\n    try:\n        from keras.engine import merge\n    except ImportError:\n        from keras.layers.merge import concatenate\n\n\n    def unet_model_3d(input_shape, pool_size=(2, 2, 2), n_labels=1, initial_learning_rate=config.learning_rate, deconvolution=False,\n                      depth=config.depth, n_base_filters=config.base_filters,\n                      batch_normalization=config.batch_norm, activation_name=\"sigmoid\"):\n\n        inputs = Input(input_shape)\n        current_layer = inputs\n        levels = list()\n\n        # add levels with max pooling\n        for layer_depth in range(depth):\n            layer1 = create_convolution_block(input_layer=current_layer, n_filters=n_base_filters*(2**layer_depth),\n                                              batch_normalization=batch_normalization)\n            layer2 = create_convolution_block(input_layer=layer1, n_filters=n_base_filters*(2**layer_depth)*2,\n                                              batch_normalization=batch_normalization)\n            if layer_depth < depth - 1:\n                current_layer = MaxPooling3D(pool_size=pool_size)(layer2)\n                levels.append([layer1, layer2, current_layer])\n            else:\n                current_layer = layer2\n                levels.append([layer1, layer2])\n\n        # add levels with up-convolution or up-sampling\n        for layer_depth in range(depth-2, -1, -1):\n            up_convolution = get_up_convolution(pool_size=pool_size, deconvolution=deconvolution,\n                                                n_filters=current_layer._keras_shape[1])(current_layer)\n            concat = concatenate([up_convolution, levels[layer_depth][1]], axis=1)\n            current_layer = create_convolution_block(n_filters=levels[layer_depth][1]._keras_shape[1],\n                                                     input_layer=concat, batch_normalization=batch_normalization)\n            current_layer = create_convolution_block(n_filters=levels[layer_depth][1]._keras_shape[1],\n                                                     input_layer=current_layer,\n                                                     batch_normalization=batch_normalization)\n\n        final_convolution = Conv3D(n_labels, (1, 1, 1))(current_layer)\n        act = Activation(activation_name)(final_convolution)\n        model = Model(inputs=inputs, outputs=act)\n\n        return model\n\n\n    def create_convolution_block(input_layer, n_filters, batch_normalization=False, kernel=(3, 3, 3), activation=None,\n                                 padding='same', strides=(1, 1, 1), instance_normalization=False):\n\n        layer = Conv3D(n_filters, kernel, padding=padding, strides=strides)(input_layer)\n        if batch_normalization:\n            layer = BatchNormalization(axis=1)(layer)\n        elif instance_normalization:\n            from keras_contrib.layers.normalization import InstanceNormalization\n\n            layer = InstanceNormalization(axis=1)(layer)\n        if activation is None:\n            return Activation('relu')(layer)\n        else:\n            return activation()(layer)\n\n\n    def compute_level_output_shape(n_filters, depth, pool_size, image_shape):\n        output_image_shape = np.asarray(np.divide(image_shape, np.power(pool_size, depth)), dtype=np.int32).tolist()\n        return tuple([None, n_filters] + output_image_shape)\n\n\n    def get_up_convolution(n_filters, pool_size, kernel_size=(2, 2, 2), strides=(2, 2, 2),\n                           deconvolution=False):\n        if deconvolution:\n            return Deconvolution3D(filters=n_filters, kernel_size=kernel_size,\n                                   strides=strides)\n        else:\n            return UpSampling3D(size=pool_size)\n    \n    # Log graphs\n    wandb.log({\"bounding_box\": [wandb.Image(bb)]})\n    wandb.log({\"bounding_coords\": [wandb.Image(bc)]})\n    \n    model = unet_model_3d((1, 64,64,32))\n    # Define the optimizer\n    if config.optimizer=='sgd':\n      optimizer = keras.optimizers.SGD(lr=config.learning_rate, decay=config.weight_decay, momentum=0.9, nesterov=True)\n    elif config.optimizer=='rmsprop':\n      optimizer = keras.optimizers.RMSprop(lr=config.learning_rate, decay=config.weight_decay)\n    elif config.optimizer=='adam':\n      optimizer = keras.optimizers.Adam(lr=config.learning_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=config.weight_decay)\n    elif config.optimizer=='nadam':\n      optimizer = keras.optimizers.Nadam(lr=config.learning_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=config.weight_decay)\n\n    model.compile(loss='binary_crossentropy',\n            optimizer=optimizer,\n            metrics=['acc'])\n    \n    model.fit_generator(generator=train_loader, steps_per_epoch=100,\n        epochs=config.epochs,\n        verbose=1,\n        callbacks=[WandbCallback(validation_data=(val_loader)), keras.callbacks.ModelCheckpoint('%d.h5', monitor='val_loss', verbose=0, save_best_only=False, save_weights_only=True, mode='auto', period=1)],\n        validation_data=val_loader,\n        validation_steps=50,\n        class_weight=None,\n        max_queue_size=10,\n        use_multiprocessing=False,\n        shuffle=True,\n        initial_epoch=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Initialize a new sweep\n# Arguments:\n#     – sweep_id: the sweep_id to run - this was returned above by wandb.sweep()\n#     – function: function that defines your model architecture and trains it\n\nwandb.agent(sweep_id, train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# If loading pretrained weights\n'''\n!wget 'https://vk.com/doc77582890_516136970?hash=83735ce0a1fe5fe100&dl=078b1ac75312cff898' -O  weights.h5\nmodel.load_weights('weights.h5')\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for x_batch, y_batch in val_loader:\n    break \n    \npred = model.predict(x_batch)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Vizualize predictions in all 3 projections"},{"metadata":{"trusted":true},"cell_type":"code","source":"i = 3\nfig, axes = plt.subplots(1, 4, figsize=(35,20))\naxes[0].imshow(x_batch[i].sum(axis=(0, -1)))\naxes[1].imshow(y_batch[i].sum(axis=(0, -1)))\naxes[2].imshow((pred[i]  ).sum(axis=(0, -1)))\naxes[3].imshow((pred[i] > 0.5 ).sum(axis=(0, -1)))\nplt.show()\n\nfig, axes = plt.subplots(1, 4, figsize=(35,20))\naxes[0].imshow(x_batch[i].sum(axis=(0, 1)).T)\naxes[1].imshow(y_batch[i].sum(axis=(0, 1)).T)\naxes[2].imshow((pred[i]  ).sum(axis=(0, 1)).T)\naxes[3].imshow((pred[i] > 0.5 ).sum(axis=(0, 1)).T)\nplt.show()\n\nfig, axes = plt.subplots(1, 4, figsize=(35,20))\naxes[0].imshow(x_batch[i].sum(axis=(0, 2)).T)\naxes[1].imshow(y_batch[i].sum(axis=(0, 2)).T)\naxes[2].imshow((pred[i]  ).sum(axis=(0, 2)).T)\naxes[3].imshow((pred[i] > 0.5 ).sum(axis=(0, 2)).T)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_boxes[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}