{"cells":[{"metadata":{},"cell_type":"markdown","source":"I would like to share my approach for the lyft motion predction competition. I am new to Kaggle and machine learning so every comment/suggestion is welcome.\n\nThe model was trained for a small part of the data due to lack of hardware (between 1M-2M out of the ~20M samples, I don't have the exact number due to numerous subsequent runs in Kaggle notebooks/Colab)."},{"metadata":{},"cell_type":"markdown","source":"**SETUP**"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nimport numpy as np\nimport PIL\nimport imageio\nfrom tensorflow.keras import Model\nimport tensorflow as tf\nimport tensorflow.keras as keras \nfrom tensorflow.keras.layers import Input, Conv2D, Conv3D, Reshape, TimeDistributed,  Lambda,Dense, Concatenate, Flatten,Dropout\nfrom tensorflow.keras.layers import GlobalMaxPooling2D,  GlobalAveragePooling2D, MaxPooling2D\nfrom tensorflow.keras.layers import  GlobalMaxPooling3D, GlobalAveragePooling3D\nfrom tensorflow.keras import layers\nimport random\nfrom tensorflow.keras.utils import plot_model\nimport gc\nfrom tensorflow.keras.preprocessing.image import array_to_img\n \nDIR_INPUT = \"../input/lyft-motion-prediction-autonomous-vehicles\"\nos.environ[\"L5KIT_DATA_FOLDER\"] = DIR_INPUT\nSINGLE_MODE_SUBMISSION = f\"{DIR_INPUT}/single_mode_sample_submission.csv\"\nMULTI_MODE_SUBMISSION = f\"{DIR_INPUT}/multi_mode_sample_submission.csv\"\n\nfrom l5kit.data import LocalDataManager, ChunkedDataset\nfrom l5kit.dataset import AgentDataset, EgoDataset\nfrom l5kit.evaluation import write_pred_csv\nfrom l5kit.rasterization import build_rasterizer\nfrom l5kit.configs import load_config_data\nfrom l5kit.visualization import draw_trajectory, TARGET_POINTS_COLOR\nfrom l5kit.geometry import transform_points\nimport random\n\nprint('loaded modules')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**CREATE AGENT DATASET**"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"cfg = {\n    'format_version': 4,\n    'model_params': {\n        'history_num_frames': 9,\n        'history_step_size': 1,\n        'history_delta_time': 0.1,\n        'future_num_frames': 50,\n        'future_step_size': 1,\n        'future_delta_time': 0.1\n    },\n    \n        'raster_params': {\n        'raster_size': [256,200],\n        'pixel_size':[0.5,0.5], #[0.5,0.5],\n        'ego_center': [0.2, 0.5], \n        'map_type': 'py_semantic',\n        'satellite_map_key': 'aerial_map.png',\n        'semantic_map_key': 'semantic_map/semantic_map.pb',\n        'dataset_meta_key': 'meta.json',\n        'filter_agents_threshold': 0.5,\n        'disable_traffic_light_faces': False\n\n    },\n    \n    'train_data_loader': {\n        'key': 'scenes/train.zarr',\n        'batch_size': 32,\n        'shuffle': True,\n        'num_workers': 4\n    },\n    \n     'val_data_loader': {\n        'key': 'scenes/validate.zarr',\n        'batch_size':8,\n        'shuffle': True,\n        'num_workers': 4\n    },\n    \n    \n    'test_data_loader': {\n        'key': 'scenes/test.zarr',\n        'batch_size': 8,\n        'shuffle': False,\n        'num_workers': 4\n    },\n    \n    \n    \n}\n\ndm = LocalDataManager()\ndataset_path = dm.require(cfg[\"train_data_loader\"][\"key\"])\nchunked_dataset = ChunkedDataset(dataset_path)\n# open the dataset\nchunked_dataset.open(cached=False)\nprint(chunked_dataset)\n\nval_dataset_path = dm.require(cfg[\"val_data_loader\"][\"key\"])\nval_chunked_dataset = ChunkedDataset(val_dataset_path)\nval_chunked_dataset.open(cached=False)\nprint('validation dataset', val_chunked_dataset)\n\ntrain_cfg = cfg[\"train_data_loader\"]\nval_cfg=cfg[\"val_data_loader\"]\n\n# Rasterizer\ndm = LocalDataManager(None)\nrasterizer = build_rasterizer(cfg, dm)\n\n\ntrain_zarr = ChunkedDataset(dm.require(train_cfg[\"key\"])).open(cached=False)\ntrain_dataset = AgentDataset(cfg, train_zarr, rasterizer,min_frame_future=10)\n\nval_zarr = ChunkedDataset(dm.require(val_cfg[\"key\"])).open(cached=False)\nval_dataset = AgentDataset(cfg, val_zarr, rasterizer,min_frame_future=10)\n\n\nprint('Dataset created')\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**VISUALIZING ONE RANDOM EXAMPLE**"},{"metadata":{"trusted":true},"cell_type":"code","source":"index=random.randint(0,len(train_dataset))\ndata=train_dataset[index]\nprint('history_availabilities',np.sum(data['history_availabilities']))\nprint('target_availabilities', np.sum(data['target_availabilities']) )\nim = data[\"image\"].transpose(1, 2, 0)\nim = train_dataset.rasterizer.to_rgb(im)\nimpred=im\ndpi = 80\nheight, width, depth = im.shape\nfigsize = 3*width / float(dpi), 3*height / float(dpi)\nhist_positions_pixels = transform_points(transform_points(data[\"history_positions\"],data[\"world_from_agent\"]), data['world_to_image'])\ntarget_positions_pixels = transform_points(transform_points(data[\"target_positions\"],data[\"world_from_agent\"]), data['world_to_image'])\nfig = plt.figure(figsize=figsize)\ncenter_in_pixels = np.asarray(cfg[\"raster_params\"][\"ego_center\"]) * cfg[\"raster_params\"][\"raster_size\"]\ndraw_trajectory(im, hist_positions_pixels,  [250,250, 38],radius=1)\ndraw_trajectory(im, target_positions_pixels, [8, 253, 8] ,radius=1)\nplt.imshow(im)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**DATA INPUT PIPELINE**\n\nIn order to use 3D convolutions we have to carefully preprocess the data. The train_dataset returns an array of dimension (2*(history_num_frames+1)+3,height,width). The first history_num_frames+1 entries in the first axis correspond to agent frames, followed by history_num_frames+1 entries for the Ego frames, while the last 3 are the RGB channels of the semantic map. I will divide these 3 kind of inputs in the dataset generator.\n\nTo return the data in a tensorflow friendly format I create generator both for the training and validation dataset and use the from_generator method of tf.data.Dataset."},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size=32\n\nnum_time_frames=cfg[\"model_params\"][\"history_num_frames\"]+1\n#The generator \ndef gen():\n    while True:\n        label=random.randint(0,len(train_dataset)-1)\n        data=train_dataset[label]\n        agent_frames=data['image'][0:num_time_frames,:,:]\n        Ego_frames=data['image'][num_time_frames:((num_time_frames+1)*2-2),:,:]\n        semantic_map=data['image'].transpose(1,2,0)[:,:,((num_time_frames+1)*2-2):]\n        history_positions=data['history_positions']\n\n        yield (Ego_frames, agent_frames, semantic_map,data['target_availabilities'],data['target_positions']),\n        \n\n#The generator \ndef val_gen():\n    while True:\n        label=random.randint(0,len(val_dataset)-1)\n        data=val_dataset[label]\n        agent_frames=data['image'][0:num_time_frames,:,:]\n        Ego_frames=data['image'][num_time_frames:((num_time_frames+1)*2-2),:,:]\n        semantic_map=data['image'].transpose(1,2,0)[:,:,((num_time_frames+1)*2-2):]\n        \n        yield (Ego_frames, agent_frames, semantic_map,data['target_availabilities'],data['target_positions']),\n        \ntrain_ds = tf.data.Dataset.from_generator(gen,((tf.float32, tf.float32,tf.float32,tf.float32, tf.float32),))\nval_ds = tf.data.Dataset.from_generator(val_gen,((tf.float32, tf.float32,tf.float32,tf.float32, tf.float32),))\n\n  \ndef configure_for_performance(ds):\n  ds = ds.batch(batch_size)\n  ds = ds.prefetch(1)\n  return ds\n\ntrain_ds=configure_for_performance(train_ds)\nval_ds=val_ds.batch(8)\n\n\n#Let's visualize one example\n\nfor element in val_ds.take(1):\n    ego=element[0][0]\n    agent=element[0][1]\n    sem_map=element[0][2]\n\nfig=plt.figure()\nplt.imshow(sem_map[0])\nplt.title('Semantic map')\n\nfig, axs=plt.subplots(nrows=10, ncols=2, figsize=(10,50))\nfor i in range(10):\n    axs[i,0].imshow(ego[0,i,:,:])\n    axs[i,0].set_title('Ego_frame_{}'.format(i))    \n    axs[i,1].imshow(agent[0,i,:,:])\n    axs[i,1].set_title('Agent_frame_{}'.format(i))\nfig.tight_layout()\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**MODEL ARCHITECTURE**\n\nLet's go to the core of the model architecture. I will make use of 3D convolutions, the Convolutional Block Attention module from the paper\n\nhttps://arxiv.org/abs/1807.06521\n\nand a 3D version of it that I implemented.\n\nDue to lack of hardware, to reduce the memory consuption of the model I first reduce the dimension of the semantic map, agent and Ego frames by a factor of 8 through the subsequent application of 2D strided convolutions mixed with CBAM blocks. The outputs of these are reshaped into tensors of shape (num_history_frames, img_heigth/8, img_width/8, num_channels) and then summed togheter.\n\nWe thus have, for each time frame, a 2D image with num_channels channels containing information about the semantic maps, the Ego, and the other agents.\n\nWe then feed this data to a series of (strided) 3D convolutions and 3D CBAM blocks. The output of the 3D convolutional part is fed, togheter with the output of 2D convolutions acting only on the semantic map, to a series of fully connected layers which give us the embedded representation of the input data.\n\nThe embedding vector is passed to the 3 prediction branches which are responsible of giving us the 3 predicted trajectories. Each prediction branch is conditioned on the output of the previous ones in the hope to make the predictions more diverse. So the first prediction branch receives as input the embedding vector, the second branch receives as input both the embedding vector and the output of the first prediction branch, while the third receives the embedding vector and the outputs of the other two branches. The predicted trajectory is given by a cumulative sum of the output of each prediction branch to reduce the magnitude of the weights the last layers need to learn. This helps stabilizing training.\n\nLastly, the output of each prediction branch togheter with the embedding vector is passed to two stacked fully connected layers that will return us the probability of each trajectory."},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"#Model architecture\n\nACT=tf.keras.layers.ELU(alpha=0.1)\nnum_halvings=3\nembedding_dim=2048\n\ndef build_3DMLP(filter_size):\n    inputs=Input(shape=filter_size)\n    dense=Dense(filter_size, activation=ACT)(inputs)\n    dense=Dense(filter_size//2, activation=ACT)(dense)\n    dense=Dense(filter_size, activation='sigmoid')(dense)\n    dense=Reshape((1,1,1,filter_size))(dense)\n    MLP=Model(inputs=inputs, outputs=dense)\n    return MLP\n\ndef build_MLP(filter_size):\n    inputs=Input(shape=filter_size)\n    dense=Dense(filter_size, activation=ACT)(inputs)\n    dense=Dense(filter_size//2, activation=ACT)(dense)\n    dense=Dense(filter_size, activation='sigmoid')(dense)\n    dense=Reshape((1,1,filter_size))(dense)\n    MLP=Model(inputs=inputs, outputs=dense)\n    return MLP\n\n\ndef CBAM(inputs, filter_size):\n    Avg=GlobalAveragePooling2D()(inputs)\n    Max=GlobalMaxPooling2D()(inputs)\n    MLP=build_MLP(filter_size)\n    Max=MLP(Max)\n    Avg=MLP(Avg)\n    channel_attention=keras.activations.sigmoid(Avg+Max)\n    channel_conditioned=tf.math.multiply(inputs, channel_attention)\n    channel_average=Lambda(lambda x: keras.backend.mean(x, axis=-1,  keepdims=True))(channel_conditioned)\n    channel_max=Lambda(lambda x: keras.backend.max(x, axis=-1,  keepdims=True))(channel_conditioned)\n    spatial_reduction=Concatenate(axis=-1)([channel_average, channel_max])\n    spatial_attention=Conv2D(filters=filter_size, kernel_size=11,padding='same', activation='sigmoid')(spatial_reduction)\n    spatial_conditioned=tf.math.multiply(channel_conditioned, spatial_attention)\n    \n    return spatial_conditioned\n\ndef CBAM_3D(inputs, filter_size, kernel=(3,3,3)):\n    Avg=GlobalAveragePooling3D()(inputs)\n    Max=GlobalMaxPooling3D()(inputs)\n    MLP=build_3DMLP(filter_size)\n    Max=MLP(Max)\n    Avg=MLP(Avg)\n    channel_attention=keras.activations.sigmoid(Avg+Max)\n    channel_conditioned=tf.math.multiply(inputs, channel_attention)\n    channel_average=Lambda(lambda x: keras.backend.mean(x, axis=-1,  keepdims=True))(channel_conditioned)\n    channel_max=Lambda(lambda x: keras.backend.max(x, axis=-1,  keepdims=True))(channel_conditioned)\n    spatial_reduction=Concatenate(axis=-1)([channel_average, channel_max])\n    spatial_attention=Conv3D(filters=filter_size, kernel_size=kernel,padding='same', activation='sigmoid')(spatial_reduction)\n    spatial_conditioned=tf.math.multiply(channel_conditioned, spatial_attention)\n    \n    return spatial_conditioned\n\n\ndef pred_branch1(embedding, name=None):\n    decoded=Dense(100)(embedding)\n    decoded=Reshape([50,2])(decoded)\n    trajectory=Lambda(lambda x:tf.math.cumsum(x,axis=1))(decoded)\n\n    return trajectory, decoded\n\ndef pred_branch2(embedding,in1, name=None):\n    in1=Flatten()(in1)\n    in1=Dense(256, activation=ACT)(in1)\n    in1=Dropout(0.2)(in1)\n    embedding=Concatenate(axis=-1)([in1,embedding])\n    embedding=Dense(embedding_dim, activation=ACT)(embedding)\n    embedding=Dropout(0.5)(embedding)\n    decoded=Dense(100)(embedding)\n    decoded=Reshape([50,2])(decoded)\n    trajectory=Lambda(lambda x:tf.math.cumsum(x,axis=1))(decoded)\n\n    return trajectory, decoded\n\ndef pred_branch3(embedding,in1, in2, name=None):\n    in1=Flatten()(in1)\n    in1=Dense(256, activation=ACT)(in1)\n    in1=Dropout(0.2)(in1)\n    in2=Flatten()(in2)\n    in2=Dense(256, activation=ACT)(in2)\n    in2=Dropout(0.2)(in2)\n    embedding=Concatenate(axis=-1)([in1,in2, embedding])\n    embedding=Dense(embedding_dim, activation=ACT)(embedding)\n    embedding=Dropout(0.5)(embedding)\n    decoded=Dense(100)(embedding)\n    decoded=Reshape([50,2])(decoded)\n    trajectory=Lambda(lambda x:tf.math.cumsum(x,axis=1))(decoded)\n\n    return trajectory,decoded\n\n\n\n\n\navailaibilities_shape=50\nmap_spatial_shape=cfg['raster_params']['raster_size']\nhistory_frames=cfg['model_params']['history_num_frames']\ntime_frames=history_frames+1\n\n#Number of halvings from original image shape\ndef build_model():\n# INPUTS    \n    \n    Ego_input=Input(shape=[time_frames,map_spatial_shape[0],map_spatial_shape[1]], name='Ego_input')\n    agent_input=Input(shape=[time_frames,map_spatial_shape[0],map_spatial_shape[1]], name='agent_input')\n    map_input=Input(shape=[map_spatial_shape[0],map_spatial_shape[1],3], name='map_input')\n    \n   \n#MAP COMBINATION AND DIMENSION REDUCTION \n    analize_map=map_input\n\n    analize_map=Conv2D(filters=32, kernel_size=7, padding='same', activation=ACT)(analize_map)\n    \n    attention=CBAM(analize_map,32)\n    analize_map=Conv2D(filters=32, kernel_size=3, activation=ACT, padding='same')(analize_map+attention)\n\n    for i in range(num_halvings):\n        analize_map=Conv2D(filters=32*(2**(i)), kernel_size=3, activation=ACT, strides=2, padding='same')(analize_map)\n        attention=CBAM(analize_map,32*(2**(i)))\n        analize_map=Conv2D(filters=32*(2**(i)), kernel_size=3, activation=ACT, padding='same')(analize_map+attention)\n\n    map_embedding=Conv2D(filters=64, kernel_size=3, activation=ACT)(analize_map)\n    attention=CBAM(map_embedding,64)\n    map_embedding=MaxPooling2D(pool_size=4)(map_embedding+attention)\n    map_embedding=Flatten()(map_embedding)\n    map_embegging=Dense(1024, activation=ACT)(map_embedding)\n    map_embedding=Dropout(0.5)(map_embedding)\n\n    analize_map=Conv2D(filters=64, kernel_size=1, activation=ACT)(analize_map)\n\n\n   \n    analize_map=Reshape([1,map_spatial_shape[0]//(2**num_halvings),map_spatial_shape[1]//(2**num_halvings),64], name='map_reshape')(analize_map)\n    \n    analize_Ego=Reshape([time_frames,map_spatial_shape[0],map_spatial_shape[1],1], name='Ego_reshape')(Ego_input)\n    \n    for i in range(num_halvings):\n        analize_Ego=TimeDistributed(Conv2D(filters=16, kernel_size=3, padding='same', strides=(2,2), activation=ACT))(analize_Ego)\n    \n    analize_Ego=TimeDistributed(Conv2D(filters=64, kernel_size=1, activation=ACT, name='analized_Ego'))(analize_Ego)\n                                                             \n    analize_agent=Reshape([time_frames,map_spatial_shape[0],map_spatial_shape[1],1], name='agent_reshape')(agent_input)\n    \n    for i in range(num_halvings):\n        analize_agent=TimeDistributed(Conv2D(filters=16, kernel_size=3, padding='same', strides=(2,2), activation=ACT))(analize_agent)\n    \n    analize_agent=TimeDistributed(Conv2D(filters=64, kernel_size=1, activation=ACT, name='analized_agent'))(analize_agent)\n\n    frame=Lambda(lambda x: tf.math.add(x[0],x[1]))([analize_Ego, analize_agent])\n    frame=Lambda(lambda x: tf.math.add(x[0],x[1]))([frame, analize_map])\n    #eventually try to add here a time distributed conv2D layer\n     \n#MAP ENCODER    \n    kernel=3\n    kernel_shape = (kernel,kernel,kernel)\n    \n    skip=frame    \n    skip=frame    \n    attention=CBAM_3D(frame,64, kernel=(5,7,7))\n    frame=Conv3D(filters=64, kernel_size=kernel_shape, padding='same', activation=ACT)(frame+attention) \n    frame=Conv3D(filters=128, kernel_size=kernel_shape, padding='same', strides=(2,2,2), activation=ACT)(frame+skip) \n    skip=frame\n    attention=CBAM_3D(frame,128,kernel=(3,7,7))\n    frame=Conv3D(filters=128, kernel_size=kernel_shape, padding='same', activation=ACT)(frame+attention) \n    frame=Conv3D(filters=256, kernel_size=kernel_shape, padding='same', strides=(2,2,2), activation=ACT)(frame+skip) \n    skip=frame\n    attention=CBAM_3D(frame,256,kernel=(3,7,7))\n    frame=Conv3D(filters=256, kernel_size=kernel_shape, padding='same', activation=ACT)(frame+attention) \n    frame=Conv3D(filters=512, kernel_size=kernel_shape, padding='same', strides=(2,2,2), activation=ACT)(frame+skip) \n    attention=CBAM_3D(frame,512,kernel=(2,4,4))\n    frame=Conv3D(filters=1024, kernel_size=(2,3,3), padding='same', strides=(2,2,2), activation=ACT)(frame+attention) \n\n    \n#EMBEDDING       \n    embedding=Flatten()(frame)\n    embedding=Dense(2048, activation=ACT)(embedding)\n    embedding=Dropout(0.5)(embedding)    \n    embedding=Concatenate(axis=-1)([embedding, map_embedding])\n    embedding=Dense(4096, activation=ACT)(embedding)\n    embedding=Dropout(0.5)(embedding)\n    embedding=Dense(embedding_dim, activation=ACT)(embedding)\n    embedding=Dropout(0.5)(embedding)\n    pred1,decoded1=pred_branch1(embedding, name='pred1')\n    pred2,decoded2=pred_branch2(embedding,decoded1, name='pred2')\n    pred3,decoded3=pred_branch3(embedding,decoded1,decoded2, name='pred3')\n    \n    analize_pred1=Flatten()(decoded1)\n    analize_pred1=Dense(256, activation=ACT)(analize_pred1)\n    analize_pred1=Dropout(0.2)(analize_pred1)\n    analize_pred2=Flatten()(decoded2)\n    analize_pred2=Dense(256, activation=ACT)(analize_pred2)\n    analize_pred2=Dropout(0.2)(analize_pred2)\n    analize_pred3=Flatten()(decoded3)\n    analize_pred3=Dense(256, activation=ACT)(analize_pred3)\n    analize_pred3=Dropout(0.2)(analize_pred3)\n    confidence_infos=Concatenate(axis=-1)([analize_pred1, analize_pred2, analize_pred3, embedding])\n    confidence_infos=Dense(4096, activation=ACT)(confidence_infos)\n    confidence_infos=Dropout(0.5)(confidence_infos)\n    confidences=Dense(3, activation='softmax', name='confidences')(confidence_infos)\n    \n    model=Model(inputs=[Ego_input,agent_input,map_input], outputs=[pred1, pred2, pred3, confidences])\n    \n    return model\n\nlyft_model=build_model()\nlyft_model.summary()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_model(lyft_model, to_file='model.png',show_shapes=True, show_layer_names=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**LOSS AND TRAINING PIPELINE**\n\nIn order to use the multi loglikelihood to score our predictions we need to build a loss function that combines the different outputs. I implemented this in Keras by wrapping the model into another model and use the add.loss method that allows to compute the loss using the outputs of intermediate layers."},{"metadata":{"trusted":true},"cell_type":"code","source":"continue_training=True\nif continue_training:\n    lyft_model.load_weights('../input/lyft-weights/nobnweights.h5')\n    print('loaded weights')\n \n \ndef multi_log_likelihood(gt, pred1,pred2,pred3,confidences):\n      confidence1,confidence2,confidence3=tf.unstack(confidences,3, axis=-1)\n      pred1x,pred1y=tf.unstack(pred1,2,axis=-1)\n      pred2x,pred2y=tf.unstack(pred2,2,axis=-1)\n      pred3x,pred3y=tf.unstack(pred3,2,axis=-1)\n      gtx,gty=tf.unstack(gt,2,axis=-1)\n      print(gtx.shape)\n      err1=-(1/2)*(tf.math.reduce_sum(tf.math.square(gtx-pred1x),axis=-1)+tf.math.reduce_sum(tf.math.square(gty-pred1y),axis=-1,name='err1'))\n      err2=-(1/2)*(tf.math.reduce_sum(tf.math.square(gtx-pred2x),axis=-1)+tf.math.reduce_sum(tf.math.square(gty-pred2y),axis=-1))\n      err3=-(1/2)*(tf.math.reduce_sum(tf.math.square(gtx-pred3x),axis=-1)+tf.math.reduce_sum(tf.math.square(gty-pred3y),axis=-1))\n      max12=tf.math.maximum(err1,err2)\n      mx=tf.math.maximum(err3,max12)\n      exp1=tf.math.exp(err1-mx)\n      exp2=tf.math.exp(err2-mx)\n      exp3=tf.math.exp(err3-mx)\n      return -mx-tf.math.log(confidence1*exp1+confidence2*exp2+confidence3*exp3)\n    \n    \ndef build_wrapper_model(model):\n    Ego_input=Input(shape=[time_frames,map_spatial_shape[0],map_spatial_shape[1]], name='Ego_input')\n    agent_input=Input(shape=[time_frames,map_spatial_shape[0],map_spatial_shape[1]], name='agent_input')\n    map_input=Input(shape=[map_spatial_shape[0],map_spatial_shape[1],3], name='map_input')\n    availabilities_mask_input=Input(shape=availaibilities_shape, name='availabilities_mask_input')\n    targets=Input(shape=(availaibilities_shape,2), name='targets')\n \n    pred1,pred2,pred3,confidences=model([Ego_input,agent_input,map_input])\n    mask_availabilities=Reshape((availaibilities_shape,1))(availabilities_mask_input)\n    pred1=Lambda(lambda x: tf.math.multiply(x[0], x[1]), name='pred1')([pred1, mask_availabilities])\n    pred2=Lambda(lambda x: tf.math.multiply(x[0], x[1]), name='pred2')([pred2, mask_availabilities])\n    pred3=Lambda(lambda x: tf.math.multiply(x[0], x[1]), name='pred3')([pred3, mask_availabilities])\n    wrapper_model=Model(inputs=[Ego_input,agent_input,map_input, availabilities_mask_input, targets], outputs=[pred1, pred2, pred3, confidences])\n    wrapper_model.add_loss(multi_log_likelihood(targets,pred1,pred2,pred3,confidences))\n \n    return wrapper_model\n\nwrapper=build_wrapper_model(lyft_model)\n\ncall=[tf.keras.callbacks.ModelCheckpoint(\n    'checkweights.h5',\n    monitor=\"val_loss\",\n    verbose=0,\n    save_best_only=False,\n    save_weights_only=True,\n    mode=\"auto\",\n    save_freq=\"epoch\",\n    options=None),\n      \n       tf.keras.callbacks.CSVLogger('log.csv')] \nOptimizer=tf.keras.optimizers.Nadam(learning_rate=1e-6)\n\nwrapper.compile(optimizer=Optimizer, experimental_steps_per_execution=10)\n \n \nhistory = wrapper.fit(train_ds,validation_data=val_ds, epochs=10, steps_per_epoch=500, callbacks=call, validation_steps=25, verbose=1)\n#ATTENTION: choose if to save weights\nsave_weights=True\n#save weights\nif save_weights:\n    lyft_model.save_weights('Lyft_weights.h5')\n    print('weights saved')\n\n   \n#summarize history for loss\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.ylabel('model loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='best')\nplt.savefig('history.png')\nplt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}