{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\n# import shutil\n# shutil.rmtree(\"/kaggle/working/wikiimagescaptions/train_images\")\n# import os\n# os.remove('/kaggle/working/captions_en.txt')","metadata":{"execution":{"iopub.status.busy":"2021-11-22T08:55:31.430703Z","iopub.execute_input":"2021-11-22T08:55:31.430996Z","iopub.status.idle":"2021-11-22T08:55:31.638623Z","shell.execute_reply.started":"2021-11-22T08:55:31.430967Z","shell.execute_reply":"2021-11-22T08:55:31.637533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cp -r ../input/wikiimagescaptions ./","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:37:52.911540Z","iopub.execute_input":"2021-11-23T02:37:52.911809Z","iopub.status.idle":"2021-11-23T02:38:09.059327Z","shell.execute_reply.started":"2021-11-23T02:37:52.911736Z","shell.execute_reply":"2021-11-23T02:38:09.058369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport gc\nimport cv2\nimport time\nimport requests\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom pathlib import Path\nimport matplotlib.pyplot as plt\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nsns.set()\nseed = 1234\nimport datatable as dt\nnp.random.seed(seed)\nfrom datatable import dt, fread\nfrom PIL import Image \nfrom datatable import dt, f, by, g, join, sort, update, ifelse\n\n%config IPCompleter.use_jedi = False\nimport os\nprint(os.listdir('/kaggle/input'))\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-23T04:45:19.083044Z","iopub.execute_input":"2021-11-23T04:45:19.083695Z","iopub.status.idle":"2021-11-23T04:45:19.106779Z","shell.execute_reply.started":"2021-11-23T04:45:19.083653Z","shell.execute_reply":"2021-11-23T04:45:19.105860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/wikipedia-image-caption/train-00000-of-00005.tsv', sep='\\t', usecols = ['image_url', 'page_title', 'language', 'caption_alt_text_description'])\ndf","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:38:20.726574Z","iopub.execute_input":"2021-11-23T02:38:20.727430Z","iopub.status.idle":"2021-11-23T02:42:54.066464Z","shell.execute_reply.started":"2021-11-23T02:38:20.727379Z","shell.execute_reply":"2021-11-23T02:42:54.065743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['language'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-11-22T14:13:39.340498Z","iopub.execute_input":"2021-11-22T14:13:39.340896Z","iopub.status.idle":"2021-11-22T14:13:40.506846Z","shell.execute_reply.started":"2021-11-22T14:13:39.340848Z","shell.execute_reply":"2021-11-22T14:13:40.506038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df[df.language=='en']\ndf.dropna(inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:42:54.068213Z","iopub.execute_input":"2021-11-23T02:42:54.068543Z","iopub.status.idle":"2021-11-23T02:42:55.938738Z","shell.execute_reply.started":"2021-11-23T02:42:54.068506Z","shell.execute_reply":"2021-11-23T02:42:55.936401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.shape","metadata":{"execution":{"iopub.status.busy":"2021-11-22T09:01:02.173823Z","iopub.execute_input":"2021-11-22T09:01:02.174508Z","iopub.status.idle":"2021-11-22T09:01:02.190338Z","shell.execute_reply.started":"2021-11-22T09:01:02.174468Z","shell.execute_reply":"2021-11-22T09:01:02.187952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"caption_df = df['caption_alt_text_description'].values","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:42:55.940061Z","iopub.execute_input":"2021-11-23T02:42:55.940674Z","iopub.status.idle":"2021-11-23T02:42:55.945504Z","shell.execute_reply.started":"2021-11-23T02:42:55.940622Z","shell.execute_reply":"2021-11-23T02:42:55.944738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_dir = '/kaggle/working/wikiimagescaptions/train_images/'\ncaptions_txt = '/kaggle/working/captions_en.txt'\nimage_names = os.listdir(image_dir)","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:42:55.947707Z","iopub.execute_input":"2021-11-23T02:42:55.948839Z","iopub.status.idle":"2021-11-23T02:42:55.959636Z","shell.execute_reply.started":"2021-11-23T02:42:55.948795Z","shell.execute_reply":"2021-11-23T02:42:55.958887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(image_names)","metadata":{"execution":{"iopub.status.busy":"2021-11-22T14:13:42.496486Z","iopub.execute_input":"2021-11-22T14:13:42.497226Z","iopub.status.idle":"2021-11-22T14:13:42.505703Z","shell.execute_reply.started":"2021-11-22T14:13:42.497187Z","shell.execute_reply":"2021-11-22T14:13:42.505118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_captions = []\nfor name in image_names:\n    img_captions.append(caption_df[int(name[:-4])])","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:42:55.961069Z","iopub.execute_input":"2021-11-23T02:42:55.961363Z","iopub.status.idle":"2021-11-23T02:42:55.970016Z","shell.execute_reply.started":"2021-11-23T02:42:55.961328Z","shell.execute_reply":"2021-11-23T02:42:55.969244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_captions[:5]","metadata":{"execution":{"iopub.status.busy":"2021-11-22T14:13:42.524409Z","iopub.execute_input":"2021-11-22T14:13:42.524685Z","iopub.status.idle":"2021-11-22T14:13:42.53694Z","shell.execute_reply.started":"2021-11-22T14:13:42.524643Z","shell.execute_reply":"2021-11-22T14:13:42.536137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(img_captions)","metadata":{"execution":{"iopub.status.busy":"2021-11-22T14:13:42.538112Z","iopub.execute_input":"2021-11-22T14:13:42.538341Z","iopub.status.idle":"2021-11-22T14:13:42.550369Z","shell.execute_reply.started":"2021-11-22T14:13:42.538313Z","shell.execute_reply":"2021-11-22T14:13:42.549482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def save_data(df, num):\n#     captions_dict = {}\n#     for i in tqdm(range(num)) : \n#         img_name = '_'.join(df.iloc[i,2].replace(',',' ').split())\n#         desc = df.iloc[i, 3]\n#         captions_dict[img_name] = desc\n#     return captions_dict\n\n# captions_dict = save_data(df,112904)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import cv2\n# for path_i in training_image_paths:\n#     img = cv2.imread(path_i)\n#     try:\n#         if len(img.shape)!=3:\n#             print(path_i)\n#     except:\n#         print(path_i)","metadata":{"execution":{"iopub.status.busy":"2021-11-22T09:01:02.315977Z","iopub.execute_input":"2021-11-22T09:01:02.317006Z","iopub.status.idle":"2021-11-22T09:01:02.38437Z","shell.execute_reply.started":"2021-11-22T09:01:02.316969Z","shell.execute_reply":"2021-11-22T09:01:02.38322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport tensorflow as tf\n\ndef load_image(image_path):\n    img = tf.io.read_file(image_path)\n    img = tf.image.decode_jpeg(img, channels=3)\n    img = tf.image.resize(img, (299, 299))\n    img = tf.keras.applications.inception_v3.preprocess_input(img)\n    return img, image_path\n    \nimage_model = tf.keras.applications.InceptionV3(include_top=False,\n                                                weights='imagenet')\nnew_input = image_model.input\nhidden_layer = image_model.layers[-1].output\n\nimage_features_extract_model = tf.keras.Model(new_input, hidden_layer)\n\nfrom tqdm import tqdm\n\ntraining_image_names = [name[:-4] for name in os.listdir(image_dir)]\ntraining_image_paths = [image_dir + name +'.jpg' for name in training_image_names]\n\n# Get unique images\nencode_train = sorted(set(training_image_paths))\n\n# Feel free to change batch_size according to your system configuration\nimage_dataset = tf.data.Dataset.from_tensor_slices(encode_train)\nimage_dataset = image_dataset.map(\n  load_image, num_parallel_calls=tf.data.experimental.AUTOTUNE).batch(16)\n\nfor img, path in tqdm(image_dataset):\n#     if not img:\n#         continue\n    batch_features = image_features_extract_model(img)\n    batch_features = tf.reshape(batch_features,\n                              (batch_features.shape[0], -1, batch_features.shape[3]))\n\n    for bf, p in zip(batch_features, path):\n        path_of_feature = p.numpy().decode(\"utf-8\")\n        np.save(path_of_feature, bf.numpy())","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2021-11-23T02:42:55.971359Z","iopub.execute_input":"2021-11-23T02:42:55.971791Z","iopub.status.idle":"2021-11-23T02:44:01.114354Z","shell.execute_reply.started":"2021-11-23T02:42:55.971753Z","shell.execute_reply":"2021-11-23T02:44:01.113600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#--------------------------------------------------\ndef captions_dict(lines):\n    dict = {}\n    print(len(lines))\n  \n  # Make a List of each line in the file\n#     lines = text.split ('\\n')\n    for ind in range(len(lines)):\n#         print(line.split('\\t'))\n        dict [str(ind)] = [lines[ind]]\n    return (dict)\n\nimage_dict = captions_dict(img_captions)","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:44:01.115683Z","iopub.execute_input":"2021-11-23T02:44:01.116223Z","iopub.status.idle":"2021-11-23T02:44:01.134900Z","shell.execute_reply.started":"2021-11-23T02:44:01.116182Z","shell.execute_reply":"2021-11-23T02:44:01.134182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# caption_file = \"/kaggle/working/captions_fr.txt\"\n# captions_txt = '/kaggle/working/captions.txt'\ndef load_captions (filename):\n    with open(filename, \"r\") as fp:\n        # Read all text in the file\n        text = fp.read()\n        return (text)\n\n#--------------------------------------------------\n# Each photo has a unique identifier, which is the file name of the image .jpg file\n# Create a dictionary of photo identifiers (without the .jpg) to captions. Each photo identifier maps to\n# a list of one or more textual descriptions.\n#\n# {\"image_name_1\" : [\"caption 1\", \"caption 2\", \"caption 3\"],\n#  \"image_name_2\" : [\"caption 4\", \"caption 5\"]}\n\n\n# doc = load_captions(caption_file)\n# image_dict = captions_dict(img_captions)","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2021-11-23T04:28:54.538567Z","iopub.execute_input":"2021-11-23T04:28:54.538830Z","iopub.status.idle":"2021-11-23T04:28:54.543693Z","shell.execute_reply.started":"2021-11-23T04:28:54.538802Z","shell.execute_reply":"2021-11-23T04:28:54.542992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nimport re\n\n#--------------------------------------------------\n# Clean the captions data\n#    Convert all words to lowercase.\n#    Remove all punctuation.\n#    Remove all words that are one character or less in length (e.g. ‘a’).\n#    Remove all words with numbers in them.\n#--------------------------------------------------\ndef captions_clean (image_dict):\n  # <key> is the image_name, which can be ignored\n    for key, captions in image_dict.items():\n    \n        # Loop through each caption for this image\n        for i, caption in enumerate (captions):\n      \n            # Convert the caption to lowercase, and then remove all special characters from it\n            caption_nopunct = re.sub(r\"[^a-zA-Z0-9]+\", ' ', caption.lower())\n      \n            # Split the caption into separate words, and collect all words which are more than \n            # one character and which contain only alphabets (ie. discard words with mixed alpha-numerics)\n            clean_words = [word for word in caption_nopunct.split() if ((len(word) > 1) and (word.isalpha()))]\n      \n            # Join those words into a string\n            caption_new = ' '.join(clean_words)\n      \n            # Replace the old caption in the captions list with this new cleaned caption\n            captions[i] = caption_new\n      \ndef add_token (captions):\n    for i, caption in enumerate (captions):\n        captions[i] = 'startseq ' + caption + ' endseq'\n    return (captions)\n\n#--------------------------------------------------\n# Given a set of training, validation or testing image names, return a dictionary\n# containing the corresponding subset from the full dictionary of images with captions\n#\n# This returned subset has the same structure as the full dictionary\n# {\"image_name_1\" : [\"caption 1\", \"caption 2\", \"caption 3\"],\n#  \"image_name_2\" : [\"caption 4\", \"caption 5\"]}\n#--------------------------------------------------\ndef subset_data_dict (image_dict, image_names):\n    dict = { image_name:add_token(captions) for image_name,captions in image_dict.items() if image_name in image_names}\n    return (dict)\n\n#--------------------------------------------------\n# Flat list of all captions\n#--------------------------------------------------\ndef all_captions (data_dict):\n    return ([caption for key, captions in data_dict.items() for caption in captions])\n\n#--------------------------------------------------\n# Calculate the word-length of the caption with the most words\n#--------------------------------------------------\ndef max_caption_length(captions):\n    return max(len(caption.split()) for caption in captions)\n\n#--------------------------------------------------\n# Fit a Keras tokenizer given caption descriptions\n# The tokenizer uses the captions to learn a mapping from words to numeric word indices\n#\n# Later, this tokenizer will be used to encode the captions as numbers\n#--------------------------------------------------\ndef create_tokenizer(data_dict):\n    captions = all_captions(data_dict)\n    max_caption_words = max_caption_length(captions)\n  \n  # Initialise a Keras Tokenizer\n    tokenizer = Tokenizer()\n  \n  # Fit it on the captions so that it prepares a vocabulary of all words\n    tokenizer.fit_on_texts(captions)\n  \n  # Get the size of the vocabulary\n    vocab_size = len(tokenizer.word_index) + 1\n\n    return (tokenizer, vocab_size, max_caption_words)\n\n#--------------------------------------------------\n# Extend a list of text indices to a given fixed length\n#--------------------------------------------------\ndef pad_text (text, max_length): \n    text = pad_sequences([text], maxlen=max_length, padding='post')[0]\n\n    return (text)\n\ncaptions_clean (image_dict)\ntraining_dict = subset_data_dict (image_dict, training_image_names)\n\n# Prepare tokenizer\ntokenizer, vocab_size, max_caption_words = create_tokenizer(training_dict)","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:44:01.136571Z","iopub.execute_input":"2021-11-23T02:44:01.137028Z","iopub.status.idle":"2021-11-23T02:44:01.531002Z","shell.execute_reply.started":"2021-11-23T02:44:01.136988Z","shell.execute_reply":"2021-11-23T02:44:01.530323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def data_prep(data_dict, tokenizer, max_length, vocab_size):\n    X, y = list(), list()\n\n  # For each image and list of captions\n    for image_name, captions in data_dict.items():\n        image_name = image_dir + image_name + '.jpg'\n\n    # For each caption in the list of captions\n        for caption in captions:\n\n      # Convert the caption words into a list of word indices\n            word_idxs = tokenizer.texts_to_sequences([caption])[0]\n\n      # Pad the input text to the same fixed length\n            pad_idxs = pad_text(word_idxs, max_length)\n          \n            X.append(image_name)\n            y.append(pad_idxs)\n\n    return np.array(X), np.array(y)\n    return X, y\n\ntrain_X, train_y = data_prep(training_dict, tokenizer, max_caption_words, vocab_size)","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:44:01.532169Z","iopub.execute_input":"2021-11-23T02:44:01.532411Z","iopub.status.idle":"2021-11-23T02:44:01.828971Z","shell.execute_reply.started":"2021-11-23T02:44:01.532380Z","shell.execute_reply":"2021-11-23T02:44:01.828180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 4\nBUFFER_SIZE = 10\n\n# Load the numpy files\ndef map_func(img_name, cap):\n    img_tensor = np.load(img_name.decode('utf-8')+'.npy')\n    return img_tensor, cap\n\ndataset = tf.data.Dataset.from_tensor_slices((train_X, train_y))\n\n# Use map to load the numpy files in parallel\ndataset = dataset.map(lambda item1, item2: tf.numpy_function(map_func, [item1, item2], [tf.float32, tf.int32]),num_parallel_calls=tf.data.experimental.AUTOTUNE)\n\n# Shuffle and batch\ndataset = dataset.shuffle(BUFFER_SIZE).batch(BATCH_SIZE)\ndataset = dataset.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:44:01.831799Z","iopub.execute_input":"2021-11-23T02:44:01.832333Z","iopub.status.idle":"2021-11-23T02:44:01.874488Z","shell.execute_reply.started":"2021-11-23T02:44:01.832292Z","shell.execute_reply":"2021-11-23T02:44:01.873858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class BahdanauAttention(tf.keras.Model):\n    def __init__(self, units):\n        super(BahdanauAttention, self).__init__()\n        self.W1 = tf.keras.layers.Dense(units)\n        self.W2 = tf.keras.layers.Dense(units)\n        self.V = tf.keras.layers.Dense(1)\n\n    def call(self, features, hidden):\n        # features(CNN_encoder output) shape == (batch_size, 64, embedding_dim)\n\n        # hidden shape == (batch_size, hidden_size)\n        # hidden_with_time_axis shape == (batch_size, 1, hidden_size)\n        hidden_with_time_axis = tf.expand_dims(hidden, 1)\n\n        # attention_hidden_layer shape == (batch_size, 64, units)\n        attention_hidden_layer = (tf.nn.tanh(self.W1(features) +\n                                             self.W2(hidden_with_time_axis)))\n\n        # score shape == (batch_size, 64, 1)\n        # This gives you an unnormalized score for each image feature.\n        score = self.V(attention_hidden_layer)\n\n        # attention_weights shape == (batch_size, 64, 1)\n        attention_weights = tf.nn.softmax(score, axis=1)\n\n        # context_vector shape after sum == (batch_size, hidden_size)\n        context_vector = attention_weights * features\n        context_vector = tf.reduce_sum(context_vector, axis=1)\n\n        return context_vector, attention_weights\n\nclass CNN_Encoder(tf.keras.Model):\n    # Since you have already extracted the features and dumped it\n    # This encoder passes those features through a Fully connected layer\n    def __init__(self, embedding_dim):\n        super(CNN_Encoder, self).__init__()\n        # shape after fc == (batch_size, 64, embedding_dim)\n        self.fc = tf.keras.layers.Dense(embedding_dim)\n\n    def call(self, x):\n        x = self.fc(x)\n        x = tf.nn.relu(x)\n        return x\n\nclass RNN_Decoder(tf.keras.Model):\n    def __init__(self, embedding_dim, units, vocab_size):\n        super(RNN_Decoder, self).__init__()\n        self.units = units\n\n        self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n        self.gru = tf.keras.layers.GRU(self.units,\n                                       return_sequences=True,\n                                       return_state=True,\n                                       recurrent_initializer='glorot_uniform')\n        self.fc1 = tf.keras.layers.Dense(self.units)\n        self.fc2 = tf.keras.layers.Dense(vocab_size)\n\n        self.attention = BahdanauAttention(self.units)\n\n    def call(self, x, features, hidden):\n        # defining attention as a separate model\n        context_vector, attention_weights = self.attention(features, hidden)\n\n        # x shape after passing through embedding == (batch_size, 1, embedding_dim)\n        x = self.embedding(x)\n\n        # x shape after concatenation == (batch_size, 1, embedding_dim + hidden_size)\n        x = tf.concat([tf.expand_dims(context_vector, 1), x], axis=-1)\n\n        # passing the concatenated vector to the GRU\n        output, state = self.gru(x)\n\n        # shape == (batch_size, max_length, hidden_size)\n        x = self.fc1(output)\n\n        # x shape == (batch_size * max_length, hidden_size)\n        x = tf.reshape(x, (-1, x.shape[2]))\n\n        # output shape == (batch_size * max_length, vocab)\n        x = self.fc2(x)\n\n        return x, state, attention_weights\n\n    def reset_state(self, batch_size):\n        return tf.zeros((batch_size, self.units))","metadata":{"execution":{"iopub.status.busy":"2021-11-23T02:44:01.875529Z","iopub.execute_input":"2021-11-23T02:44:01.875753Z","iopub.status.idle":"2021-11-23T02:44:01.893264Z","shell.execute_reply.started":"2021-11-23T02:44:01.875720Z","shell.execute_reply":"2021-11-23T02:44:01.892468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embedding_dim = 256\nunits = 512\nvocab_size = vocab_size\nnum_steps = len(train_X) // BATCH_SIZE\n# Shape of the vector extracted from InceptionV3 is (64, 2048)\n# These two variables represent that vector shape\nfeatures_shape = 2048\nattention_features_shape = 64\n\nencoder = CNN_Encoder(embedding_dim)\ndecoder = RNN_Decoder(embedding_dim, units, vocab_size)\n\noptimizer = tf.keras.optimizers.Adam()\nloss_object = tf.keras.losses.SparseCategoricalCrossentropy(\n    from_logits=True, reduction='none')\n\ndef loss_function(real, pred):\n    mask = tf.math.logical_not(tf.math.equal(real, 0))\n    loss_ = loss_object(real, pred)\n\n    mask = tf.cast(mask, dtype=loss_.dtype)\n    loss_ *= mask\n\n    return tf.reduce_mean(loss_)\n\nloss_plot = []\n@tf.function\ndef train_step(img_tensor, target):\n    loss = 0\n\n    # initializing the hidden state for each batch\n    # because the captions are not related from image to image\n    hidden = decoder.reset_state(batch_size=target.shape[0])\n\n    dec_input = tf.expand_dims([tokenizer.word_index['startseq']] * target.shape[0], 1)\n\n    with tf.GradientTape() as tape:\n        features = encoder(img_tensor)\n\n        for i in range(1, target.shape[1]):\n          # passing the features through the decoder\n            predictions, hidden, _ = decoder(dec_input, features, hidden)\n\n            loss += loss_function(target[:, i], predictions)\n\n            # using teacher forcing\n            dec_input = tf.expand_dims(target[:, i], 1)\n\n    total_loss = (loss / int(target.shape[1]))\n\n    trainable_variables = encoder.trainable_variables + decoder.trainable_variables\n\n    gradients = tape.gradient(loss, trainable_variables)\n\n    optimizer.apply_gradients(zip(gradients, trainable_variables))\n\n    return loss, total_loss\n\nimport time\nstart_epoch = 0\nEPOCHS = 20\n\nfor epoch in range(start_epoch, EPOCHS):\n    start = time.time()\n    total_loss = 0\n\n    for (batch, (img_tensor, target)) in enumerate(dataset):\n        batch_loss, t_loss = train_step(img_tensor, target)\n        total_loss += t_loss\n\n        if batch % 100 == 0:\n            average_batch_loss = batch_loss.numpy()/int(target.shape[1])\n            print(f'Epoch {epoch+1} Batch {batch} Loss {average_batch_loss:.4f}')\n    # storing the epoch end loss value to plot later\n    loss_plot.append(total_loss / num_steps)\n\n    print(f'Epoch {epoch+1} Loss {total_loss/num_steps:.6f}')\n    print(f'Time taken for 1 epoch {time.time()-start:.2f} sec\\n')","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2021-11-23T02:44:01.894719Z","iopub.execute_input":"2021-11-23T02:44:01.895235Z","iopub.status.idle":"2021-11-23T04:12:49.576379Z","shell.execute_reply.started":"2021-11-23T02:44:01.895198Z","shell.execute_reply":"2021-11-23T04:12:49.574166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2021-11-23T04:12:49.578872Z","iopub.execute_input":"2021-11-23T04:12:49.579337Z","iopub.status.idle":"2021-11-23T04:12:50.467080Z","shell.execute_reply.started":"2021-11-23T04:12:49.579297Z","shell.execute_reply":"2021-11-23T04:12:50.466359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_t = df_t[df_t.language == 'fr']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_t.dropna(inplace=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_img_dir = '/kaggle/working/test_images_en/'\nt_captions_txt = '/kaggle/working/test_en.txt'\nif not os.path.exists(test_img_dir):\n    os.makedirs(test_img_dir)\n    \nimport urllib\nimport PIL.Image\n\ndef save_data(df):\n    txt = open(t_captions_txt,'w')\n    for i in range(8000,8100) : \n        link = df.iloc[i, 1]\n        desc = df.iloc[i, 3]\n        try:\n            with urllib.request.urlopen(link) as url:\n                with open(test_img_dir+'/'+str(i)+'.jpg', 'wb') as f:\n                    f.write(url.read())\n            txt.write(str(i)+'\\t'+desc+'\\n')\n        except Exception as e:\n            print('Exception occured :',e)\n    txt.close()\n    \nsave_data(df)","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2021-11-23T04:26:21.584048Z","iopub.execute_input":"2021-11-23T04:26:21.584345Z","iopub.status.idle":"2021-11-23T04:27:28.700632Z","shell.execute_reply.started":"2021-11-23T04:26:21.584314Z","shell.execute_reply":"2021-11-23T04:27:28.699813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def t_captions_dict(text):\n    lines = text.split('\\n')\n    dict = {}\n    for line in lines:\n#         print(line.split('\\t'))\n        if len(line.split('\\t')) !=2:\n            continue\n        ind, caption  = line.split('\\t')\n        dict [str(ind)] = [caption]\n    return (dict)","metadata":{"execution":{"iopub.status.busy":"2021-11-23T04:39:31.549224Z","iopub.execute_input":"2021-11-23T04:39:31.549883Z","iopub.status.idle":"2021-11-23T04:39:31.554711Z","shell.execute_reply.started":"2021-11-23T04:39:31.549846Z","shell.execute_reply":"2021-11-23T04:39:31.553988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import IPython\nfrom IPython.display import display\ndef evaluate(image, max_length):\n    attention_plot = np.zeros((max_length, attention_features_shape))\n\n    hidden = decoder.reset_state(batch_size=1)\n\n    temp_input = tf.expand_dims(load_image(image)[0], 0)\n    img_tensor_val = image_features_extract_model(temp_input)\n    img_tensor_val = tf.reshape(img_tensor_val, (img_tensor_val.shape[0],\n                                                 -1,\n                                                 img_tensor_val.shape[3]))\n\n    features = encoder(img_tensor_val)\n\n    dec_input = tf.expand_dims([tokenizer.word_index['startseq']], 0)\n    result = []\n\n    for i in range(max_length):\n        predictions, hidden, attention_weights = decoder(dec_input,\n                                                         features,\n                                                         hidden)\n\n        attention_plot[i] = tf.reshape(attention_weights, (-1, )).numpy()\n\n        predicted_id = tf.random.categorical(predictions, 1)[0][0].numpy()\n        result.append(tokenizer.index_word[predicted_id])\n\n        if tokenizer.index_word[predicted_id] == 'endseq':\n            return result, attention_plot\n\n        dec_input = tf.expand_dims([predicted_id], 0)\n\n    attention_plot = attention_plot[:len(result), :]\n    return result, attention_plot\n  \ndef check_test(test_image_names, image_dict, image_dir, max_caption_words):\n    # captions on the validation set\n#     rid = np.random.randint(0, len(test_image_names))\n    for rid in range(50):\n        image_name = test_image_names[rid]\n        real_caption = image_dict[image_name]\n\n        image_path = image_dir + image_name + '.jpg'\n        for i in range(5):\n            result, attention_plot = evaluate(image_path, max_caption_words)\n            print('Prediction Caption:', ' '.join(result))\n        #from IPython.display import Image, display\n        #display(Image(image_path))\n        print('Real Caption:', real_caption)\n        dis_image = Image.open(image_path).resize((300,300))\n        display(dis_image)\n\n# test_image_name_file = \"dataset/Flickr_8k.testImages.txt\"\ntest_image_names = [name[:-4] for name in os.listdir(test_img_dir)]#subset_image_name (test_image_name_file)\n# image_dir = \"dataset/Flicker8k_Dataset/\"\ndoc_t = load_captions(t_captions_txt)\ntest_image_dict = t_captions_dict(doc_t)\ncheck_test(list(test_image_names), test_image_dict, test_img_dir, max_caption_words)","metadata":{"execution":{"iopub.status.busy":"2021-11-23T04:54:59.551270Z","iopub.execute_input":"2021-11-23T04:54:59.551594Z","iopub.status.idle":"2021-11-23T04:56:02.594575Z","shell.execute_reply.started":"2021-11-23T04:54:59.551560Z","shell.execute_reply":"2021-11-23T04:56:02.593948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_image_dict","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import urllib.request as urll\n# import PIL.Image\n# from tqdm import tqdm\n\"\"\"\nfrom PIL import Image\nimport requests\nfrom io import BytesIO\n\ndef save_data(df, num1, num2):\n    txt = open(captions_txt,'w')\n    for i in tqdm(range(num1, num2)): \n        link = df.iloc[i, 1]\n        img_name = '_'.join(df.iloc[i,2].replace(',',' ').split())\n        desc = df.iloc[i, 3]\n        try:\n            response = requests.get(link)\n            img = Image.open(BytesIO(response.content))\n            img_low = img.resize((300,300)).convert('RGB')\n            img_low.save(os.path.join(train_img_dir,img_name+'.jpg'))\n            txt.write(img_name+'\\t\\t'+desc+'\\n')\n        except Exception as e:\n            print(\"Exception occured:\",e)\n    txt.close()\n\nsave_data(df, 3100,10000)\n\"\"\"\n\n# def save_data(df, num):\n#     txt = open(captions_txt,'w')\n#     for i in tqdm(range(num)): \n#         link = df.iloc[i, 1]\n#         img_name = '_'.join(df.iloc[i,2].replace(',',' ').split())\n#         desc = df.iloc[i, 3]\n# #         print(link, )\n#         try:\n#             with urll.urlopen(link) as url:\n#                 with open(os.path.join(train_img_dir,img_name+'.jpg'), 'wb') as f:\n#                     f.write(url.read())\n#             img = cv2.imread(os.path.join(train_img_dir,img_name+'.jpg'))\n#             img = cv2.resize(img,(300,300))\n#             cv2.imwrite(os.path.join(train_img_dir,img_name+'.jpg'),img)\n#             txt.write(img_name+'\\t\\t'+desc+'\\n')\n#         except Exception as e:\n#             print(\"Exception occured:\",e)\n#     txt.close()\n\n# save_data(df, 3100)\n\n# def save_data(df, num):\n#     txt = open(captions_txt,'w')\n#     for i in range(num) : \n#         link = df.iloc[i, 1]\n#         desc = df.iloc[i, 3]\n#         try:\n#             with urllib.request.urlopen(link) as url:\n#                 with open(train_img_dir+'/'+str(i)+'.jpg', 'wb') as f:\n#                     f.write(url.read())\n#             txt.write(str(i)+'\\t'+desc+'\\n')\n#         except Exception as e:\n#             print('Exception occured :',e)\n#     txt.close()","metadata":{"scrolled":true,"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n        \n# def load_images(links):\n#     images = []\n#     ind = 0\n#     for link in links:\n#         URL = link\n#         try:\n#             with urllib.request.urlopen(URL) as url:\n#                 with open(train_img_dir'/'+ind+'.jpg', 'wb') as f:\n#                     f.write(url.read())\n\n#             img = PIL.Image.open('./temp.jpg')\n#             img = np.asarray(img)\n#             images.append(img)\n#             ind+=1\n#         except Exception as e:\n#             print('Exception occured',e, ind)\n#             continue\n#     return images\n\n# def display_images(images, title=None): \n#     f, ax = plt.subplots(4,5, figsize=(18,12))\n#     if title:\n#         f.suptitle(title, fontsize = 30)\n\n#     for i, image_id in enumerate(images):\n#         ax[i//5, i%5].imshow(image_id) \n   \n#         ax[i//5, i%5].axis('off')\n\n#     plt.show() \n\n# def get_links(df, num):\n#     return df[df.language=='en']['image_url'][:num].values\n\n# def get_captions(df, num):\n#     return df[df.language=='en']['caption_alt_text_description'][:num].values\n","metadata":{},"execution_count":null,"outputs":[]}]}