{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Starter\nThis problem could be considered as an image captioning problem. One way to solve is to extract features from images and then generate captiones using GANS. In this starter, the Captions Generated using Encoder and Decoder. The reference is tensorflow tutorial for Image Captioning.\n\nUseful papers ,implementations and notes will be in the Discussion soon. The link will be added in the comments.\n\nHope it will be useful.\n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport cv2\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN_LABELS_PATH = \"../input/bms-molecular-translation/train_labels.csv\"\n\ndf_train_labels = pd.read_csv(TRAIN_LABELS_PATH, index_col=0)\ndf_train_labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ref: https://www.kaggle.com/ihelon/molecular-translation-exploratory-data-analysis \ndef convert_image_id_2_path(image_id: str) -> str:\n    return \"../input/bms-molecular-translation/train/{}/{}/{}/{}.png\".format(\n        image_id[0], image_id[1], image_id[2], image_id \n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#ref: https://www.kaggle.com/ihelon/molecular-translation-exploratory-data-analysis\ndef visualize_train_batch(image_ids, labels):\n    plt.figure(figsize=(16, 12))\n    \n    for ind, (image_id, label) in enumerate(zip(image_ids, labels)):\n        plt.subplot(3, 3, ind + 1)\n        image = cv2.imread(convert_image_id_2_path(image_id))\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\n        plt.imshow(image)\n#         print(f\"{ind}: {label}\")\n        plt.title(f\"{label[:30]}...\", fontsize=10)\n        plt.axis(\"off\")\n    \n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#ref: https://www.kaggle.com/ihelon/molecular-translation-exploratory-data-analysis\ndef visualize_train_image(image_id, label):\n    plt.figure(figsize=(10, 8))\n    \n    image = cv2.imread(convert_image_id_2_path(image_id))\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    plt.imshow(image)\n    plt.title(f\"{label}\", fontsize=14)\n    plt.axis(\"off\")\n    \n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_row = df_train_labels.sample(5)\nfor i in range(5):\n    visualize_train_image(\n        sample_row.index[i], sample_row[\"InChI\"][i]\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(df_train_labels[\"InChI\"].unique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.utils import shuffle\nimport collections\nimport random\nimport re\nimport numpy as np\nimport os\nimport time\nimport json\nfrom glob import glob\nfrom PIL import Image\nimport pickle","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_path_to_caption = collections.defaultdict(list)\nfor idx,path in enumerate(df_train_labels.index):\n  caption = df_train_labels['InChI'].iloc[idx]\n  image_path = convert_image_id_2_path(path)\n  image_path_to_caption[image_path].append(caption)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_paths = list(image_path_to_caption.keys())\nrandom.shuffle(image_paths)\n# Let us take just first 6000 images for training now \ntrain_image_paths = image_paths[:6000]\nprint(len(train_image_paths))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_captions = []\nimg_name_vector = []\n\nfor image_path in train_image_paths:\n  caption_list = image_path_to_caption[image_path]\n  train_captions.extend(caption_list)\n  img_name_vector.extend([image_path] * len(caption_list))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_captions[0])\nImage.open(img_name_vector[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_image(image_path):\n    img = tf.io.read_file(image_path)\n    img = tf.image.decode_jpeg(img, channels=3)\n    img = tf.image.resize(img, (224, 224))\n    img = tf.keras.applications.mobilenet_v2.preprocess_input(img)\n    return img, image_path","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# I will use MobileNetv2 for feature extraction, ImageNet weights is a really bad choice because it is not trained on text features (Wait for my starting points)\nimage_model = tf.keras.applications.MobileNetV2(include_top=False,\n                                                weights='imagenet')\nnew_input = image_model.input\nhidden_layer = image_model.layers[-1].output\n\nimage_features_extract_model = tf.keras.Model(new_input, hidden_layer)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Data_path = '../input/bms-molecular-translation/train/0/0/0/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# i will save the features in this directory \n!mkdir features","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Extract features\nencode_train = sorted(set(img_name_vector))\nimage_dataset = tf.data.Dataset.from_tensor_slices(encode_train)\nimage_dataset = image_dataset.map(\n  load_image, num_parallel_calls=tf.data.AUTOTUNE).batch(16)\nfor img, path in image_dataset:\n  batch_features = image_features_extract_model(img)\n  batch_features = tf.reshape(batch_features,\n                              (batch_features.shape[0], -1, batch_features.shape[3]))\n\n  for bf, p in zip(batch_features, path):\n    path_of_feature ='features/'+ p.numpy().decode(\"utf-8\")[len(Data_path):-4]\n    np.save(path_of_feature, bf.numpy())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Find the maximum length of InChI \ndef calc_max_length(tensor):\n    return max(len(t) for t in tensor)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"top_k = 5000\ntokenizer = tf.keras.preprocessing.text.Tokenizer(num_words=top_k,\n                                                  oov_token=\"<unk>\",\n                                                  filters='!\"#$%&*+.-;?@[]^`{}~ ')\ntokenizer.fit_on_texts(train_captions)\ntokenizer.word_index['<pad>'] = 0\ntokenizer.index_word[0] = '<pad>'\n# Create the tokenized vectors\ntrain_seqs = tokenizer.texts_to_sequences(train_captions)\n# Pad each vector to the max_length of InChI\ncap_vector = tf.keras.preprocessing.sequence.pad_sequences(train_seqs, padding='post')\nmax_length = calc_max_length(train_seqs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_to_cap_vector = collections.defaultdict(list)\nfor img, cap in zip(img_name_vector, cap_vector):\n  img_to_cap_vector[img].append(cap)\n\n# Create training and validation sets\nimg_keys = list(img_to_cap_vector.keys())\nrandom.shuffle(img_keys)\n\nslice_index = int(len(img_keys)*0.8)\nimg_name_train_keys, img_name_val_keys = img_keys[:slice_index], img_keys[slice_index:]\n\nimg_name_train = []\ncap_train = []\nfor imgt in img_name_train_keys:\n  capt_len = len(img_to_cap_vector[imgt])\n  img_name_train.extend([imgt] * capt_len)\n  cap_train.extend(img_to_cap_vector[imgt])\n\nimg_name_val = []\ncap_val = []\nfor imgv in img_name_val_keys:\n  capv_len = len(img_to_cap_vector[imgv])\n  img_name_val.extend([imgv] * capv_len)\n  cap_val.extend(img_to_cap_vector[imgv])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CFG:\n    BATCH_SIZE = 64\n    BUFFER_SIZE = 1000\n    embedding_dim = 256\n    units = 512\n    vocab_size = top_k + 1\n    num_steps = len(img_name_train) // BATCH_SIZE\n    features_shape = 2048\n    attention_features_shape = 64","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load the Features\ndef map_func(img_name, cap):\n  img_tensor = np.load('features/' + img_name.decode('utf-8')[len(Data_path):-4]+'.npy')\n  return img_tensor, cap","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset = tf.data.Dataset.from_tensor_slices((img_name_train, cap_train))\ndataset = dataset.map(lambda item1, item2: tf.numpy_function(\n          map_func, [item1, item2], [tf.float32, tf.int32]),\n          num_parallel_calls=tf.data.AUTOTUNE)\ndataset = dataset.shuffle(CFG.BUFFER_SIZE).batch(CFG.BATCH_SIZE)\ndataset = dataset.prefetch(buffer_size=tf.data.AUTOTUNE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class BahdanauAttention(tf.keras.Model):\n  def __init__(self, units):\n    super(BahdanauAttention, self).__init__()\n    self.W1 = tf.keras.layers.Dense(units)\n    self.W2 = tf.keras.layers.Dense(units)\n    self.V = tf.keras.layers.Dense(1)\n\n  def call(self, features, hidden):\n    hidden_with_time_axis = tf.expand_dims(hidden, 1)\n    attention_hidden_layer = (tf.nn.tanh(self.W1(features) +\n                                         self.W2(hidden_with_time_axis)))\n    score = self.V(attention_hidden_layer)\n    attention_weights = tf.nn.softmax(score, axis=1)\n    context_vector = attention_weights * features\n    context_vector = tf.reduce_sum(context_vector, axis=1)\n    return context_vector, attention_weights","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CNN_Encoder(tf.keras.Model):\n    def __init__(self, embedding_dim):\n        super(CNN_Encoder, self).__init__()\n        self.fc = tf.keras.layers.Dense(embedding_dim)\n    def call(self, x):\n        x = self.fc(x)\n        x = tf.nn.relu(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class RNN_Decoder(tf.keras.Model):\n  def __init__(self, embedding_dim, units, vocab_size):\n    super(RNN_Decoder, self).__init__()\n    self.units = units\n\n    self.embedding = tf.keras.layers.Embedding(vocab_size,embedding_dim)\n    self.gru = tf.keras.layers.GRU(self.units,\n                                   return_sequences=True,\n                                   return_state=True,\n                                   recurrent_initializer='glorot_uniform')\n    self.fc1 = tf.keras.layers.Dense(self.units)\n    self.fc2 = tf.keras.layers.Dense(vocab_size)\n\n    self.attention = BahdanauAttention(self.units)\n\n  def call(self, x, features, hidden):\n    context_vector, attention_weights = self.attention(features, hidden)\n    x = self.embedding(x)\n    x = tf.concat([tf.expand_dims(context_vector, 1), x], axis=-1)\n    output, state = self.gru(x)\n    x = self.fc1(output)\n    x = tf.reshape(x, (-1, x.shape[2]))\n    x = self.fc2(x)\n    return x, state, attention_weights\n\n  def reset_state(self, batch_size):\n    return tf.zeros((batch_size, self.units))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"encoder = CNN_Encoder(CFG.embedding_dim)\ndecoder = RNN_Decoder(CFG.embedding_dim, CFG.units, CFG.vocab_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"optimizer = tf.keras.optimizers.Adam()\nloss_object = tf.keras.losses.SparseCategoricalCrossentropy(\n    from_logits=True, reduction='none')\n\ndef loss_function(real, pred):\n  mask = tf.math.logical_not(tf.math.equal(real, 0))\n  loss_ = loss_object(real, pred)\n\n  mask = tf.cast(mask, dtype=loss_.dtype)\n  loss_ *= mask\n\n  return tf.reduce_mean(loss_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"checkpoint_path = \"./checkpoints/train\"\nckpt = tf.train.Checkpoint(encoder=encoder,\n                           decoder=decoder,\n                           optimizer = optimizer)\nckpt_manager = tf.train.CheckpointManager(ckpt, checkpoint_path, max_to_keep=5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"loss_plot = []\nstart_epoch = 0\nif ckpt_manager.latest_checkpoint:\n  start_epoch = int(ckpt_manager.latest_checkpoint.split('-')[-1])\n  ckpt.restore(ckpt_manager.latest_checkpoint)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"@tf.function\ndef train_step(img_tensor, target):\n  loss = 0\n  hidden = decoder.reset_state(batch_size=target.shape[0])\n  dec_input = tf.expand_dims([tokenizer.word_index['<unk>']] * target.shape[0], 1)\n  with tf.GradientTape() as tape:\n      features = encoder(img_tensor)\n      for i in range(1, target.shape[1]):\n          predictions, hidden, _ = decoder(dec_input, features, hidden)\n          loss += loss_function(target[:, i], predictions)\n          dec_input = tf.expand_dims(target[:, i], 1)\n  total_loss = (loss / int(target.shape[1]))\n  trainable_variables = encoder.trainable_variables + decoder.trainable_variables\n  gradients = tape.gradient(loss, trainable_variables)\n  optimizer.apply_gradients(zip(gradients, trainable_variables))\n  return loss, total_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"EPOCHS = 20\n\nfor epoch in range(start_epoch, EPOCHS):\n    start = time.time()\n    total_loss = 0\n    \n    for (batch, (img_tensor, target)) in enumerate(dataset):\n        batch_loss, t_loss = train_step(img_tensor, target)\n        total_loss += t_loss\n        if batch % 100 == 0:\n            print ('Epoch {} Batch {} Loss {:.4f}'.format(\n              epoch + 1, batch, batch_loss.numpy() / int(target.shape[1])))\n            \n    # Let us save the loss info to visualize\n    loss_plot.append(total_loss / CFG.num_steps)\n    if epoch % 5 == 0:\n      ckpt_manager.save()\n    print ('Epoch {} Loss {:.6f}'.format(epoch + 1,total_loss/CFG.num_steps))\n    print ('Time taken for 1 epoch {} sec\\n'.format(time.time() - start))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(loss_plot)\nplt.xlabel('Epochs')\nplt.ylabel('Loss')\nplt.title('Loss Plot')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def evaluate(image):\n    attention_plot = np.zeros((max_length, CFG.attention_features_shape))\n    hidden = decoder.reset_state(batch_size=1)\n    temp_input = tf.expand_dims(load_image(image)[0], 0)\n    img_tensor_val = image_features_extract_model(temp_input)\n    img_tensor_val = tf.reshape(img_tensor_val, (img_tensor_val.shape[0], -1, img_tensor_val.shape[3]))\n    features = encoder(img_tensor_val)\n    dec_input = tf.expand_dims([tokenizer.word_index['<unk>']], 0)\n    result = []\n    for i in range(max_length):\n        predictions, hidden, attention_weights = decoder(dec_input, features, hidden)\n        #attention_plot[i] = tf.reshape(attention_weights, (-1, )).numpy()\n        predicted_id = tf.random.categorical(predictions, 1)[0][0].numpy()\n        result.append(tokenizer.index_word[predicted_id])\n        if tokenizer.index_word[predicted_id] == '<end>':\n            return result, attention_plot\n        dec_input = tf.expand_dims([predicted_id], 0)\n    attention_plot = attention_plot[:len(result), :]\n    return result, attention_plot","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_attention(image, result, attention_plot):\n    temp_image = np.array(Image.open(image))\n    fig = plt.figure(figsize=(10, 10))\n    len_result = len(result)\n    for l in range(len_result):\n        temp_att = np.resize(attention_plot[l], (8, 8))\n        ax = fig.add_subplot(len_result//2, len_result//2, l+1)\n        ax.set_title(result[l])\n        img = ax.imshow(temp_image)\n        ax.imshow(temp_att, cmap='gray', alpha=0.6, extent=img.get_extent())\n    plt.tight_layout()\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rid = np.random.randint(0, len(img_name_val))\nimage = img_name_val[rid]\nreal_caption = ' '.join([tokenizer.index_word[i] for i in cap_val[rid] if i not in [0]])\nresult, attention_plot = evaluate(image)\nprint ('Real Caption:', real_caption)\nprint ('Prediction Caption:', ' '.join(result))\n#plot_attention(image, result, attention_plot)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}