{"cells":[{"metadata":{},"cell_type":"markdown","source":"Majority of the code is taken from [here](https://www.analyticsvidhya.com/blog/2021/01/implementation-of-attention-mechanism-for-caption-generation-on-transformers-using-tensorflow/)"},{"metadata":{"trusted":true},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import collections\nfrom glob import glob\nimport re\nimport random\nimport string\nimport time\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom tqdm.notebook import tqdm\n\nimport tensorflow as tf\nfrom keras.callbacks import ModelCheckpoint\nfrom keras.models import Model\nfrom keras.layers import Input\nfrom keras.layers import Dense, BatchNormalization\nfrom keras.layers import LSTM\nfrom keras.layers import Embedding\nfrom keras.layers import Dropout\nfrom keras.layers.merge import add\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.preprocessing.image import load_img, img_to_array\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.utils import to_categorical\nfrom keras.utils import plot_model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data Loading & Preprocessing"},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN_LABELS_PATH = \"../input/bms-molecular-translation/train_labels.csv\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train_labels = pd.read_csv(TRAIN_LABELS_PATH, index_col=0)\ndf_train_labels.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train_labels.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_path_to_caption = collections.defaultdict(list)\n\nfor idx, path in enumerate(df_train_labels.index):\n  caption = f\"<start> {df_train_labels['InChI'].iloc[idx]}  <end>\"\n  image_path = \"../input/bms-molecular-translation/train/{}/{}/{}/{}.png\".format(path[0], path[1], path[2], path)\n  image_path_to_caption[image_path].append(caption)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_paths = list(image_path_to_caption.keys())\nrandom.shuffle(image_paths)\n\n# taking approx 1/50 of all images\ntrain_image_paths = image_paths[:50000]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_captions = []\nimg_name_vector = []\n\nfor image_path in train_image_paths:\n  caption_list = image_path_to_caption[image_path]\n  train_captions.extend(caption_list)\n  img_name_vector.extend([image_path] * len(caption_list))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_image(image_path):\n    img = tf.io.read_file(image_path)\n    img = tf.image.decode_jpeg(img, channels=3)\n    img = tf.image.resize(img, (224, 224))\n    img = tf.keras.applications.mobilenet_v2.preprocess_input(img)\n    return img, image_path","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model Definition"},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"image_model = tf.keras.applications.MobileNetV2(include_top=False, weights=\"imagenet\")\nnew_input = image_model.input\nhidden_layer = image_model.layers[-1].output\nimage_features_extract_model = tf.keras.Model(new_input, hidden_layer)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_path = \"../input/bms-molecular-translation/train/0/0/0/\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!mkdir features","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"encode_train = sorted(set(img_name_vector))\nimage_dataset = tf.data.Dataset.from_tensor_slices(encode_train)\nimage_dataset = image_dataset.map(load_image, num_parallel_calls=tf.data.AUTOTUNE).batch(16)\n\nfor img, path in tqdm(image_dataset):\n  batch_features = image_features_extract_model(img)\n  batch_features = tf.reshape(batch_features, (batch_features.shape[0], -1, batch_features.shape[3]))\n\n  for bf, p in zip(batch_features, path):\n    path_of_feature =\"features/\"+ p.numpy().decode(\"utf-8\")[len(data_path):-4]\n    np.save(path_of_feature, bf.numpy())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# max caption length\ndef calc_max_length(tensor):\n    return max(len(t) for t in tensor)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"top_k = 5000\ntokenizer = tf.keras.preprocessing.text.Tokenizer(num_words=top_k,\n                                                  oov_token=\"<unk>\",\n                                                  filters=\"!#$%&*+.-;?@[]^`{}~ \")\ntokenizer.fit_on_texts(train_captions)\ntokenizer.word_index['<pad>'] = 0\ntokenizer.index_word[0] = '<pad>'\n# Create the tokenized vectors\ntrain_seqs = tokenizer.texts_to_sequences(train_captions)\n# Pad each vector to the max_length of InChI\ncap_vector = tf.keras.preprocessing.sequence.pad_sequences(train_seqs, padding='post')\nmax_length = calc_max_length(train_seqs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_name_train, img_name_val, cap_train, cap_val = train_test_split(img_name_vector,cap_vector, test_size=0.2, random_state=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 64\nBUFFER_SIZE = 1000\nnum_steps = len(img_name_train) // BATCH_SIZE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def map_func(img_name, cap):\n  img_tensor = np.load('features/' + img_name.decode('utf-8')[len(data_path):-4]+'.npy')\n  return img_tensor, cap","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset = tf.data.Dataset.from_tensor_slices((img_name_train, cap_train))\ndataset = dataset.map(lambda item1, item2: tf.numpy_function(\n          map_func, [item1, item2], [tf.float32, tf.int32]),\n          num_parallel_calls=tf.data.AUTOTUNE)\ndataset = dataset.shuffle(BUFFER_SIZE).batch(BATCH_SIZE)\ndataset = dataset.prefetch(buffer_size=tf.data.AUTOTUNE)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Positional Encoding"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_angles(pos, i, d_model):\n   angle_rates = 1 / np.power(10000, (2 * (i//2)) / np.float32(d_model))\n   return pos * angle_rates\n\ndef positional_encoding_1d(position, d_model):\n   angle_rads = get_angles(np.arange(position)[:, np.newaxis],\n                           np.arange(d_model)[np.newaxis, :],\n                           d_model)\n\n   angle_rads[:, 0::2] = np.sin(angle_rads[:, 0::2])\n   angle_rads[:, 1::2] = np.cos(angle_rads[:, 1::2])\n   pos_encoding = angle_rads[np.newaxis, ...]\n   return tf.cast(pos_encoding, dtype=tf.float32)\n\ndef positional_encoding_2d(row,col,d_model):\n   assert d_model % 2 == 0\n   row_pos = np.repeat(np.arange(row),col)[:,np.newaxis]\n   col_pos = np.repeat(np.expand_dims(np.arange(col),0),row,axis=0).reshape(-1,1)\n\n   angle_rads_row = get_angles(row_pos,np.arange(d_model//2)[np.newaxis,:],d_model//2)\n   angle_rads_col = get_angles(col_pos,np.arange(d_model//2)[np.newaxis,:],d_model//2)\n\n   angle_rads_row[:, 0::2] = np.sin(angle_rads_row[:, 0::2])\n   angle_rads_row[:, 1::2] = np.cos(angle_rads_row[:, 1::2])\n   angle_rads_col[:, 0::2] = np.sin(angle_rads_col[:, 0::2])\n   angle_rads_col[:, 1::2] = np.cos(angle_rads_col[:, 1::2])\n   pos_encoding = np.concatenate([angle_rads_row,angle_rads_col],axis=1)[np.newaxis, ...]\n   return tf.cast(pos_encoding, dtype=tf.float32)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Multi-Head Attention"},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_padding_mask(seq):\n   seq = tf.cast(tf.math.equal(seq, 0), tf.float32)\n   return seq[:, tf.newaxis, tf.newaxis, :]  # (batch_size, 1, 1, seq_len)\n\ndef create_look_ahead_mask(size):\n   mask = 1 - tf.linalg.band_part(tf.ones((size, size)), -1, 0)\n   return mask  # (seq_len, seq_len)\n\ndef scaled_dot_product_attention(q, k, v, mask):\n   matmul_qk = tf.matmul(q, k, transpose_b=True)  # (..., seq_len_q, seq_len_k)\n   dk = tf.cast(tf.shape(k)[-1], tf.float32)\n   scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)\n\n   if mask is not None:\n      scaled_attention_logits += (mask * -1e9) \n\n   attention_weights = tf.nn.softmax(scaled_attention_logits, axis=-1) \n   output = tf.matmul(attention_weights, v)  # (..., seq_len_q, depth_v)\n\n   return output, attention_weights","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MultiHeadAttention(tf.keras.layers.Layer):\n   def __init__(self, d_model, num_heads):\n      super(MultiHeadAttention, self).__init__()\n      self.num_heads = num_heads\n      self.d_model = d_model\n      assert d_model % self.num_heads == 0\n      self.depth = d_model // self.num_heads\n      self.wq = tf.keras.layers.Dense(d_model)\n      self.wk = tf.keras.layers.Dense(d_model)\n      self.wv = tf.keras.layers.Dense(d_model)\n      self.dense = tf.keras.layers.Dense(d_model)\n\n   def split_heads(self, x, batch_size):\n      x = tf.reshape(x, (batch_size, -1, self.num_heads, self.depth))\n      return tf.transpose(x, perm=[0, 2, 1, 3])\n\n   def call(self, v, k, q, mask=None):\n      batch_size = tf.shape(q)[0]\n      q = self.wq(q)  # (batch_size, seq_len, d_model)\n      k = self.wk(k)  # (batch_size, seq_len, d_model)\n      v = self.wv(v)  # (batch_size, seq_len, d_model)\n\n      q = self.split_heads(q, batch_size)  # (batch_size, num_heads, seq_len_q, depth)\n      k = self.split_heads(k, batch_size)  # (batch_size, num_heads, seq_len_k, depth)\n      v = self.split_heads(v, batch_size)  # (batch_size, num_heads, seq_len_v, depth)\n\n      scaled_attention, attention_weights = scaled_dot_product_attention(q, k, v, mask)\n      scaled_attention = tf.transpose(scaled_attention, perm=[0, 2, 1, 3])  # (batch_size, seq_len_q,      num_heads, depth)\n\n      concat_attention = tf.reshape(scaled_attention,\n                                 (batch_size, -1, self.d_model))  # (batch_size, seq_len_q, d_model)\n\n      output = self.dense(concat_attention)  # (batch_size, seq_len_q, d_model)\n      return output, attention_weights","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def point_wise_feed_forward_network(d_model, dff):\n   return tf.keras.Sequential([\n                tf.keras.layers.Dense(dff, activation='relu'),\n                tf.keras.layers.Dense(d_model)])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Encoder Layer"},{"metadata":{"trusted":true},"cell_type":"code","source":"class EncoderLayer(tf.keras.layers.Layer):\n   def __init__(self, d_model, num_heads, dff, rate=0.1):\n      super(EncoderLayer, self).__init__()\n      self.mha = MultiHeadAttention(d_model, num_heads)\n      self.ffn = point_wise_feed_forward_network(d_model, dff)\n\n      self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n      self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n\n      self.dropout1 = tf.keras.layers.Dropout(rate)\n      self.dropout2 = tf.keras.layers.Dropout(rate)\n\n\n   def call(self, x, training, mask=None):\n      attn_output, _ = self.mha(x, x, x, mask)  # (batch_size, input_seq_len, d_model)\n      attn_output = self.dropout1(attn_output, training=training)\n      out1 = self.layernorm1(x + attn_output)  # (batch_size, input_seq_len, d_model)\n\n      ffn_output = self.ffn(out1)  # (batch_size, input_seq_len, d_model)\n      ffn_output = self.dropout2(ffn_output, training=training)\n      out2 = self.layernorm2(out1 + ffn_output)  # (batch_size, input_seq_len, d_model)\n      return out2","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Decoder Layer"},{"metadata":{"trusted":true},"cell_type":"code","source":"class DecoderLayer(tf.keras.layers.Layer):\n   def __init__(self, d_model, num_heads, dff, rate=0.1):\n      super(DecoderLayer, self).__init__()\n      self.mha1 = MultiHeadAttention(d_model, num_heads)\n      self.mha2 = MultiHeadAttention(d_model, num_heads)\n\n      self.ffn = point_wise_feed_forward_network(d_model, dff)\n\n      self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n      self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n      self.layernorm3 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n\n      self.dropout1 = tf.keras.layers.Dropout(rate)\n      self.dropout2 = tf.keras.layers.Dropout(rate)\n      self.dropout3 = tf.keras.layers.Dropout(rate)\n\n   def call(self, x, enc_output, training,look_ahead_mask=None, padding_mask=None):\n      attn1, attn_weights_block1 = self.mha1(x, x, x, look_ahead_mask)  # (batch_size, target_seq_len, d_model)\n      attn1 = self.dropout1(attn1, training=training)\n      out1 = self.layernorm1(attn1 + x)\n\n      attn2, attn_weights_block2 = self.mha2(enc_output, enc_output, out1, padding_mask) \n      attn2 = self.dropout2(attn2, training=training)\n      out2 = self.layernorm2(attn2 + out1)  # (batch_size, target_seq_len, d_model)\n\n      ffn_output = self.ffn(out2)  # (batch_size, target_seq_len, d_model)\n      ffn_output = self.dropout3(ffn_output, training=training)\n      out3 = self.layernorm3(ffn_output + out2)  # (batch_size, target_seq_len, d_model)\n\n      return out3, attn_weights_block1, attn_weights_block2","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Encoder"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Encoder(tf.keras.layers.Layer):\n   def __init__(self, num_layers, d_model, num_heads, dff, row_size,col_size,rate=0.1):\n      super(Encoder, self).__init__()\n      self.d_model = d_model\n      self.num_layers = num_layers\n\n      self.embedding = tf.keras.layers.Dense(self.d_model,activation='relu')\n      self.pos_encoding = positional_encoding_2d(row_size,col_size,self.d_model)\n\n      self.enc_layers = [EncoderLayer(d_model, num_heads, dff, rate) for _ in range(num_layers)]\n      self.dropout = tf.keras.layers.Dropout(rate)\n\n   def call(self, x, training, mask=None):\n      seq_len = tf.shape(x)[1]\n      x = self.embedding(x)  # (batch_size, input_seq_len(H*W), d_model)\n      x += self.pos_encoding[:, :seq_len, :]\n      x = self.dropout(x, training=training)\n\n      for i in range(self.num_layers):\n         x = self.enc_layers[i](x, training, mask)\n\n      return x  # (batch_size, input_seq_len, d_model)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Decoder"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Decoder(tf.keras.layers.Layer):\n   def __init__(self, num_layers,d_model,num_heads,dff, target_vocab_size, maximum_position_encoding,   rate=0.1):\n      super(Decoder, self).__init__()\n      self.d_model = d_model\n      self.num_layers = num_layers\n\n      self.embedding = tf.keras.layers.Embedding(target_vocab_size, d_model)\n      self.pos_encoding = positional_encoding_1d(maximum_position_encoding, d_model)\n\n      self.dec_layers = [DecoderLayer(d_model, num_heads, dff, rate)\n                         for _ in range(num_layers)]\n      self.dropout = tf.keras.layers.Dropout(rate)\n\n   def call(self, x, enc_output, training,look_ahead_mask=None, padding_mask=None):\n      seq_len = tf.shape(x)[1]\n      attention_weights = {}\n\n      x = self.embedding(x)  # (batch_size, target_seq_len, d_model)\n      x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n      x += self.pos_encoding[:, :seq_len, :]\n      x = self.dropout(x, training=training)\n\n      for i in range(self.num_layers):\n         x, block1, block2 = self.dec_layers[i](x, enc_output, training,\n                                            look_ahead_mask, padding_mask)\n         \n         attention_weights['decoder_layer{}_block1'.format(i+1)] = block1\n         attention_weights['decoder_layer{}_block2'.format(i+1)] = block2\n\n      return x, attention_weights","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Transformer"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Transformer(tf.keras.Model):\n   def __init__(self, num_layers, d_model, num_heads, dff,row_size,col_size,\n              target_vocab_size,max_pos_encoding, rate=0.1):\n      super(Transformer, self).__init__()\n      self.encoder = Encoder(num_layers, d_model, num_heads, dff,row_size,col_size, rate)\n      self.decoder = Decoder(num_layers, d_model, num_heads, dff,\n                          target_vocab_size,max_pos_encoding, rate)\n      self.final_layer = tf.keras.layers.Dense(target_vocab_size)\n\n   def call(self, inp, tar, training,look_ahead_mask=None,dec_padding_mask=None,enc_padding_mask=None   ):\n      enc_output = self.encoder(inp, training, enc_padding_mask)  # (batch_size, inp_seq_len, d_model      )\n      dec_output, attention_weights = self.decoder(\n      tar, enc_output, training, look_ahead_mask, dec_padding_mask)\n      final_output = self.final_layer(dec_output)  # (batch_size, tar_seq_len, target_vocab_size)\n      return final_output, attention_weights","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Hyperparams"},{"metadata":{"trusted":true},"cell_type":"code","source":"num_layer = 4\nepochs = 20\nd_model = 512\ndff = 2048\nnum_heads = 8\nrow_size = 8\ncol_size = 8\ntarget_vocab_size = top_k + 1\ndropout_rate = 0.1 ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Callbacks & Optimizer"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CustomSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n   def __init__(self, d_model, warmup_steps=4000):\n      super(CustomSchedule, self).__init__()\n      self.d_model = d_model\n      self.d_model = tf.cast(self.d_model, tf.float32)\n      self.warmup_steps = warmup_steps\n\n   def __call__(self, step):\n      arg1 = tf.math.rsqrt(step)\n      arg2 = step * (self.warmup_steps ** -1.5)\n      return tf.math.rsqrt(self.d_model) * tf.math.minimum(arg1, arg2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learning_rate = CustomSchedule(d_model)\noptimizer = tf.keras.optimizers.Adam(learning_rate, beta_1=0.9, beta_2=0.98, epsilon=1e-9)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Loss Function & Metric"},{"metadata":{"trusted":true},"cell_type":"code","source":"loss_object = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True, reduction='none')\n\ndef loss_function(real, pred):\n    mask = tf.math.logical_not(tf.math.equal(real, 0))\n    loss_ = loss_object(real, pred)\n    mask = tf.cast(mask, dtype=loss_.dtype)\n    loss_ *= mask\n    return tf.reduce_sum(loss_)/tf.reduce_sum(mask)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_loss = tf.keras.metrics.Mean(name='train_loss')\ntrain_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy')\ntransformer = Transformer(num_layer, d_model,num_heads, dff, \n                          row_size, col_size, target_vocab_size, \n                          max_pos_encoding=target_vocab_size, rate=dropout_rate)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_masks_decoder(tar):\n   look_ahead_mask = create_look_ahead_mask(tf.shape(tar)[1])\n   dec_target_padding_mask = create_padding_mask(tar)\n   combined_mask = tf.maximum(dec_target_padding_mask, look_ahead_mask)\n   return combined_mask","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"@tf.function\ndef train_step(img_tensor, tar):\n   tar_inp = tar[:, :-1]\n   tar_real = tar[:, 1:]\n   dec_mask = create_masks_decoder(tar_inp)\n   with tf.GradientTape() as tape:\n      predictions, _ = transformer(img_tensor, tar_inp,True, dec_mask)\n      loss = loss_function(tar_real, predictions)\n\n   gradients = tape.gradient(loss, transformer.trainable_variables)   \n   optimizer.apply_gradients(zip(gradients, transformer.trainable_variables))\n   train_loss(loss)\n   train_accuracy(tar_real, predictions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for epoch in tqdm(range(epochs)):\n   start = time.time()\n   train_loss.reset_states()\n   train_accuracy.reset_states()\n   for (batch, (img_tensor, tar)) in enumerate(dataset):\n      train_step(img_tensor, tar)\n      if batch % 50 == 0:\n         print ('Epoch: {} - Batch: {} - Loss: {:.4f} - Accuracy: {:.4f}'.format(\n             epoch+1, batch, train_loss.result(), train_accuracy.result()))\n\n   print ('\\n\\nEPOCH: {} - LOSS: {:.4f} - ACC: {:.4f}'.format(epoch+1, train_loss.result(), train_accuracy.result()))\n   print ('TIME: {} secs\\n'.format(time.time() - start))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Evaluation"},{"metadata":{"trusted":true},"cell_type":"code","source":"def evaluate(image):\n   temp_input = tf.expand_dims(load_image(image)[0], 0)\n   img_tensor_val = image_features_extract_model(temp_input)\n   img_tensor_val = tf.reshape(img_tensor_val, (img_tensor_val.shape[0], -1, img_tensor_val.shape[3]))\n   start_token = tokenizer.word_index['<start>']\n   end_token = tokenizer.word_index['<end>']\n   decoder_input = [start_token]\n   output = tf.expand_dims(decoder_input, 0) #tokens\n   result = [] #word list\n\n   for i in range(max_length):\n      dec_mask = create_masks_decoder(output)\n      predictions, _ = transformer(img_tensor_val,output,False,dec_mask)\n      predictions = predictions[: ,-1:, :]  # (batch_size, 1, vocab_size)\n      predicted_id = tf.cast(tf.argmax(predictions, axis=-1), tf.int32)\n      if predicted_id == end_token:\n         return result, tf.squeeze(output, axis=0)\n      result.append(tokenizer.index_word[int(predicted_id)])\n      output = tf.concat([output, predicted_id], axis=-1)\n\n   return result, tf.squeeze(output, axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(15):\n    rid = np.random.randint(0, len(img_name_val))\n    image = img_name_val[rid]\n    real_caption = ' '.join([tokenizer.index_word[i] for i in cap_val[rid] if i not in [0]])\n    caption, result = evaluate(image)\n\n    print (\"Actual Caption\")\n    print(\"=\"*30)\n    print(real_caption, end=\"\\n\\n\")\n\n    print (\"Predicted Caption\")\n    print(\"=\"*30)\n    print(caption)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}