{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Load Libraries","metadata":{}},{"cell_type":"code","source":"import string\nimport numpy as np\nimport PIL.Image\n\nfrom os import listdir\nfrom pickle import dump, load\n\nfrom numpy import array\nfrom numpy import argmax\n\nfrom keras.applications.vgg16 import VGG16, preprocess_input\nfrom keras.preprocessing.image import load_img, img_to_array\nfrom keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.utils import to_categorical\nfrom keras.layers.merge import add\nfrom keras.models import Model, load_model\nfrom keras.layers import Input, Dense, LSTM, Embedding, Dropout\nfrom keras.callbacks import ModelCheckpoint\n\nfrom nltk.translate.bleu_score import corpus_bleu","metadata":{"execution":{"iopub.status.busy":"2022-04-27T06:19:50.649557Z","iopub.execute_input":"2022-04-27T06:19:50.649833Z","iopub.status.idle":"2022-04-27T06:19:53.223077Z","shell.execute_reply.started":"2022-04-27T06:19:50.649784Z","shell.execute_reply":"2022-04-27T06:19:53.222146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. Prepare Photo Data","metadata":{}},{"cell_type":"markdown","source":"## VGG16 Model\n\n### About VGG:\n\n* Visual Geometry Group from University of Oxford developed VGG model \n* VGG model won the ImageNet competition in 2014\n* Published as a conference paper at ICLR 2015: \n    https://arxiv.org/pdf/1409.1556.pdf\n* Visual Geometry Group overview: http://www.robots.ox.ac.uk/~vgg/research/very_deep/\n\n\n### About VGG16 Model:\n\n* 3×3 filters in all convolutional layers\n* 16 Layers Model\n* Layer Configurations: https://gist.githubusercontent.com/ksimonyan/211839e770f7b538e2d8/raw/ded9363bd93ec0c770134f4e387d8aaaaa2407ce/VGG_ILSVRC_16_layers_deploy.prototxt\n\n### Applications\n\n* Given an image, find object name in the image.\n* It can detect any one of 1000 images.\n* It takes input image of size 224 * 224 * 3 (RGB image) i.e 224 * 224 pixel image with 3 channels","metadata":{}},{"cell_type":"code","source":"# Extract features from each photo in the directory\ndef extract_features(directory):\n    \n    # Loading the model\n    model = VGG16()\n\n    # Removing the last layer from the loaded model as we require only the features not the classification \n    model.layers.pop()\n    model = Model(inputs=model.inputs, outputs=model.layers[-1].output)\n    \n    # Summarizing the model \n    print(model.summary())\n\n    # Extracting features from each photo and storing it in a dictionary \n    features = dict()\n\n    for name in listdir(directory):\n\n        # Defining the path of the image \n        filename = directory + '/' + name\n        \n        # Loading an image and converting it into size 224 * 224\n        image = load_img(filename, target_size=(224, 224))\n        \n        # Converting the image pixels into a numpy array\n        image = img_to_array(image)\n        \n        # Reshaping data for the model\n        image = image.reshape((1, image.shape[0], image.shape[1], image.shape[2]))\n\n        \n        # Preprocessing the images for the VGG model\n        # The preprocess_input function is meant to adequate your image to the format the model requires.\n        image = preprocess_input(image)\n\n        # Getting features of an image\n        feature = model.predict(image, verbose=0)\n        \n        # Getting the image name\n        image_id = name.split('.')[0]\n\n        # Storing the feature corresponding to the image in the dictionary\n        features[image_id] = feature\n        \n        # print('>%s' % name)\n        \n    return features","metadata":{"execution":{"iopub.status.busy":"2022-04-27T06:20:57.678956Z","iopub.execute_input":"2022-04-27T06:20:57.679253Z","iopub.status.idle":"2022-04-27T06:20:57.68711Z","shell.execute_reply.started":"2022-04-27T06:20:57.679201Z","shell.execute_reply":"2022-04-27T06:20:57.686389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"  # Loading the model\nmodel = VGG16()\n\n    # Removing the last layer from the loaded model as we require only the features not the classification \nmodel.layers.pop()\nmodel = Model(inputs=model.inputs, outputs=model.layers[-1].output)\nprint(model.summary())\n    ","metadata":{"execution":{"iopub.status.busy":"2022-04-27T06:21:04.876155Z","iopub.execute_input":"2022-04-27T06:21:04.876499Z","iopub.status.idle":"2022-04-27T06:21:42.736925Z","shell.execute_reply.started":"2022-04-27T06:21:04.876448Z","shell.execute_reply":"2022-04-27T06:21:42.736144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Defining the directory we are using\ndirectory = '../input/flicker8k-dataset/flickr8k_dataset/Flicker8k_Dataset'\n\n# Extracting features from all the images\nfeatures = extract_features(directory)\n\nprint('Extracted Features: ', len(features))\n\n# Dumping the features in a pickle file for further use\ndump(features, open('flick_features.pkl', 'wb'))","metadata":{"execution":{"iopub.status.busy":"2022-04-27T06:21:42.74057Z","iopub.execute_input":"2022-04-27T06:21:42.740798Z","iopub.status.idle":"2022-04-27T06:24:28.197093Z","shell.execute_reply.started":"2022-04-27T06:21:42.740753Z","shell.execute_reply":"2022-04-27T06:24:28.196355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle\nwith open('./flick_features.pkl','rb') as f:\n  features_dict=pickle.load(f)","metadata":{"execution":{"iopub.status.busy":"2022-04-26T14:19:08.955388Z","iopub.execute_input":"2022-04-26T14:19:08.955657Z","iopub.status.idle":"2022-04-26T14:19:09.106924Z","shell.execute_reply.started":"2022-04-26T14:19:08.955613Z","shell.execute_reply":"2022-04-26T14:19:09.106195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features_list=[]\nfor i in features_dict.values():\n  features_list.append(i)\nfeatures_array=np.array( features_list)","metadata":{"execution":{"iopub.status.busy":"2022-04-26T14:24:26.050511Z","iopub.execute_input":"2022-04-26T14:24:26.050833Z","iopub.status.idle":"2022-04-26T14:24:26.152671Z","shell.execute_reply.started":"2022-04-26T14:24:26.050761Z","shell.execute_reply":"2022-04-26T14:24:26.151872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features_array.shape","metadata":{"execution":{"iopub.status.busy":"2022-04-26T14:24:29.274307Z","iopub.execute_input":"2022-04-26T14:24:29.274579Z","iopub.status.idle":"2022-04-26T14:24:29.280246Z","shell.execute_reply.started":"2022-04-26T14:24:29.274527Z","shell.execute_reply":"2022-04-26T14:24:29.279234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"listdir('../input')","metadata":{"execution":{"iopub.status.busy":"2021-11-25T19:42:05.963245Z","iopub.execute_input":"2021-11-25T19:42:05.963541Z","iopub.status.idle":"2021-11-25T19:42:05.975611Z","shell.execute_reply.started":"2021-11-25T19:42:05.963486Z","shell.execute_reply":"2021-11-25T19:42:05.974148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Preparing Text Data","metadata":{}},{"cell_type":"code","source":"# Loading the file containg all the descriptions into memory\n\ndef load_doc(filename):\n    # Opening the file as read only\n    file = open(filename, 'r')\n\n    # Reading all text and storing it.\n    text = file.read()\n\n    # Closing the file\n    file.close()\n    \n    return text\n","metadata":{"execution":{"iopub.status.busy":"2022-04-27T06:24:28.198462Z","iopub.execute_input":"2022-04-27T06:24:28.198752Z","iopub.status.idle":"2022-04-27T06:24:28.204581Z","shell.execute_reply.started":"2022-04-27T06:24:28.198706Z","shell.execute_reply":"2022-04-27T06:24:28.203636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Function to return a dictionary of photo identifiers to the descriptions","metadata":{}},{"cell_type":"code","source":"def photo_to_description_mapping(descriptions):\n    \n    # Dictionary to store the mapping of photo identifiers to descriptions\n    description_mapping = dict()\n    \n    # Iterating through each line of the descriptions\n    for line in descriptions.split('\\n'):\n        \n        # Splitting the lines by white space\n        words = line.split()\n        \n        # Skipping the lines with length less than 2\n        if len(line)<2:\n            continue\n            \n        # The first word is the image_id and the rest are the part of the description of that image\n        image_id, image_description = words[0], words[1:]\n        \n        # Retaining only the name of the image and removing the extension from it\n        image_id = image_id.split('.')[0]\n        \n        # Image_descriptions contains comma separated words of the description, hence, converting it back to string\n        image_description = ' '.join(image_description)\n        \n        # There are multiple descriptions per image, \n        # hence, corresponding to every image identifier in the dictionary, there is a list of description\n        # if the list does not exist then we need to create it\n        \n        if image_id not in description_mapping:\n            description_mapping[image_id] = list()\n            \n        # Now storing the descriptions in the mapping\n        description_mapping[image_id].append(image_description)\n    \n    return description_mapping","metadata":{"execution":{"iopub.status.busy":"2022-04-27T06:24:28.206071Z","iopub.execute_input":"2022-04-27T06:24:28.206618Z","iopub.status.idle":"2022-04-27T06:24:28.215585Z","shell.execute_reply.started":"2022-04-27T06:24:28.206568Z","shell.execute_reply":"2022-04-27T06:24:28.214986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Function to clean the descriptions in the following ways:\n*     Convert all words to lowercase.\n*    Remove all punctuation.\n*     Remove all words that are one character or less in length (e.g. ‘a’).\n*     Remove all words with numbers in them.","metadata":{}},{"cell_type":"code","source":"def clean_descriptions(description_mapping):\n    \n    # Preapring a translation table for removing all the punctuation\n    table = str.maketrans('','', string.punctuation)\n    \n    # Traversing through the mapping we created\n    for key, descriptions in description_mapping.items():\n        for i in range(len(descriptions)):\n            description = descriptions[i]\n            description = description.split()\n            \n            # Converting all the words to lower case\n            description = [word.lower() for word in description]\n            \n            # Removing the punctuation using the translation table we made\n            description = [word.translate(table) for word in description]\n            \n            # Removing the words with length =1\n            description = [word for word in description if len(word)>1]\n            \n            # Removing all words with number in them\n            description = [word for word in description if word.isalpha()]\n            \n            # Converting the description back to string and overwriting in the descriptions list\n            descriptions[i] = ' '.join(description)","metadata":{"execution":{"iopub.status.busy":"2022-04-27T06:24:28.217394Z","iopub.execute_input":"2022-04-27T06:24:28.217866Z","iopub.status.idle":"2022-04-27T06:24:28.228813Z","shell.execute_reply.started":"2022-04-27T06:24:28.217678Z","shell.execute_reply":"2022-04-27T06:24:28.227988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Ideally, we want a vocabulary that is both expressive and as small as possible. A smaller vocabulary will result in a smaller model that will train faster.\n\nFor reference, we can transform the clean descriptions into a set and print its size to get an idea of the size of our dataset vocabulary.\n\nSets are highly optimized, don't contain any duplicate values. There implementation is based on hash table. Hence we get a vocabulary that is both expressive and small.","metadata":{}},{"cell_type":"code","source":"# Converting the loaded descriptions into a vocabulary of words\n\ndef to_vocabulary(descriptions):\n    \n    # Build a list of all description strings\n    all_desc = set()\n    \n    for key in descriptions.keys():\n        [all_desc.update(d.split()) for d in descriptions[key]]\n    \n    return all_desc","metadata":{"execution":{"iopub.status.busy":"2022-04-27T06:24:28.230158Z","iopub.execute_input":"2022-04-27T06:24:28.230639Z","iopub.status.idle":"2022-04-27T06:24:28.241665Z","shell.execute_reply.started":"2022-04-27T06:24:28.230458Z","shell.execute_reply":"2022-04-27T06:24:28.240934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# save descriptions to file, one per line\ndef save_descriptions(descriptions, filename):\n    lines = list()\n    for key, desc_list in descriptions.items():\n        for desc in desc_list:\n            lines.append(key + ' ' + desc)\n    data = '\\n'.join(lines)\n    file = open(filename, 'w')\n    file.write(data)\n    file.close()","metadata":{"execution":{"iopub.status.busy":"2022-04-27T06:24:28.243934Z","iopub.execute_input":"2022-04-27T06:24:28.244518Z","iopub.status.idle":"2022-04-27T06:24:28.254185Z","shell.execute_reply.started":"2022-04-27T06:24:28.244223Z","shell.execute_reply":"2022-04-27T06:24:28.25341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filename = '../input/flickr8k_text/Flickr8k.token.txt'\n\n# Loading descriptions\ndoc = load_doc(filename)\n\n# Parsing descriptions\ndescriptions = photo_to_description_mapping(doc)\nprint('Loaded: %d ' % len(descriptions))\n\n# Cleaning the descriptions\nclean_descriptions(descriptions)\n\n# Summarizing the vocabulary\nvocabulary = to_vocabulary(descriptions)\nprint('Vocabulary Size: %d' % len(vocabulary))\n\n# Saving to the file\nsave_descriptions(descriptions, 'descriptions.txt')","metadata":{"execution":{"iopub.status.busy":"2022-04-27T06:24:28.255949Z","iopub.execute_input":"2022-04-27T06:24:28.256479Z","iopub.status.idle":"2022-04-27T06:24:28.551713Z","shell.execute_reply.started":"2022-04-27T06:24:28.256253Z","shell.execute_reply":"2022-04-27T06:24:28.550414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Developing Deep Learning Model","metadata":{}},{"cell_type":"markdown","source":"## A.) Loading the data","metadata":{}},{"cell_type":"code","source":"# Function for loading a file into memory and returning text from it\ndef load_file(filename):\n    file = open(filename, 'r')\n    text = file.read()\n    file.close()\n    return text\n\n# Function for loading a pre-defined list of photo identifiers\ndef load_photo_identifiers(filename):\n    \n    # Loading the file containing the list of photo identifier\n    file = load_file(filename)\n    \n    # Creating a list for storing the identifiers\n    photos = list()\n    \n    # Traversing the file one line at a time\n    for line in file.split('\\n'):\n        if len(line) < 1:\n            continue\n        \n        # Image name contains the extension as well but we need just the name\n        identifier = line.split('.')[0]\n        \n        # Adding it to the list of photos\n        photos.append(identifier)\n        \n    # Returning the set of photos created\n    return set(photos)\n\n\n# loading the cleaned descriptions that we created earlier\n# we will only be loading the descriptions of the images that we will use for training\n# hence we need to pass the set of train photos that the above function will be returning\n\ndef load_clean_descriptions(filename, photos):\n    \n    #loading the cleaned description file\n    file = load_file(filename)\n    \n    #creating a dictionary of descripitions for storing the photo to description mapping of train images\n    descriptions = dict()\n    \n    #traversing the file line by line\n    for line in file.split('\\n'):\n        # splitting the line at white spaces\n        words = line.split()\n        \n        # the first word will be the image name and the rest will be the description of that particular image\n        image_id, image_description = words[0], words[1:]\n        \n        # we want to load only those description which corresponds to the set of photos we provided as argument\n        if image_id in photos:\n            #creating list of description if needed\n            if image_id not in descriptions:\n                descriptions[image_id] = list()\n            \n            #the model we will develop will generate a caption given a photo, \n            #and the caption will be generated one word at a time. \n            #The sequence of previously generated words will be provided as input. \n            #Therefore, we will need a ‘first word’ to kick-off the generation process \n            #and a ‘last word‘ to signal the end of the caption.\n            #we will use 'startseq' and 'endseq' for this purpose\n            #also we have to convert image description back to string\n            \n            desc = 'startseq ' + ' '.join(image_description) + ' endseq'\n            descriptions[image_id].append(desc)\n            \n    return descriptions\n\n# function to load the photo features created using the VGG16 model\ndef load_photo_features(filename, photos):\n    \n    #this will load the entire features\n    all_features = load(open(filename, 'rb'))\n    \n    #we are interested in loading the features of the required photos only\n    features = {k: all_features[k] for k in photos}\n    \n    return features","metadata":{"execution":{"iopub.status.busy":"2022-04-26T14:25:06.254336Z","iopub.execute_input":"2022-04-26T14:25:06.254609Z","iopub.status.idle":"2022-04-26T14:25:06.265019Z","shell.execute_reply.started":"2022-04-26T14:25:06.254559Z","shell.execute_reply":"2022-04-26T14:25:06.264077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filename = '../input/flickr8k_text/Flickr_8k.trainImages.txt'\n\n#load the image index\ntrain = load_photo_identifiers(filename)\nprint('Dataset: ',len(train))\n\n#load the captions of images\ntrain_descriptions = load_clean_descriptions('descriptions.txt', train)\nprint('Descriptions: train=', len(train_descriptions))\n\n#load image features\ntrain_features = load_photo_features('features.pkl', train)\nprint('Photos: train=', len(train_features))","metadata":{"execution":{"iopub.status.busy":"2022-04-26T13:28:07.086094Z","iopub.status.idle":"2022-04-26T13:28:07.086692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The description text will need to be encoded to numbers before it can be presented to the model as in input or compared to the model’s predictions.\n\nThe first step in encoding the data is to create a consistent mapping from words to unique integer values. Keras provides the Tokenizer class that can learn this mapping from the loaded description data.\n\nBelow defines the to_lines() to convert the dictionary of descriptions into a list of strings and the create_tokenizer() function that will fit a Tokenizer given the loaded photo description text.","metadata":{}},{"cell_type":"code","source":"# convert a dictionary of clean descriptions to a list of descriptions\ndef to_lines(descriptions):\n    all_desc = list()\n    for key in descriptions.keys():\n        [all_desc.append(d) for d in descriptions[key]]\n    return all_desc\n\n# Given the descriptions, fit a tokenizer\n\n# TOKENIZER CLASS:\n# This class allows to vectorize a text corpus, \n# by turning each text into either a sequence of integers \n# (each integer being the index of a token in a dictionary) \n# or, into a vector where the coefficient for each token could be binary, based on word count, based on tf-idf...\n\ndef create_tokenizer(descriptions):\n    lines = to_lines(descriptions)\n    tokenizer = Tokenizer()\n    tokenizer.fit_on_texts(lines)\n    return tokenizer","metadata":{"execution":{"iopub.status.busy":"2022-04-26T14:27:57.691933Z","iopub.execute_input":"2022-04-26T14:27:57.692265Z","iopub.status.idle":"2022-04-26T14:27:57.699018Z","shell.execute_reply.started":"2022-04-26T14:27:57.692216Z","shell.execute_reply":"2022-04-26T14:27:57.69802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer = create_tokenizer(train_descriptions)\n\nvocab_size = len(tokenizer.word_index) + 1\nprint('Vocabulary Size: ', vocab_size)","metadata":{"execution":{"iopub.status.busy":"2022-04-26T14:28:03.225763Z","iopub.execute_input":"2022-04-26T14:28:03.226053Z","iopub.status.idle":"2022-04-26T14:28:03.246221Z","shell.execute_reply.started":"2022-04-26T14:28:03.226001Z","shell.execute_reply":"2022-04-26T14:28:03.245183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### We can now encode the text.\nEach description will be split into words. The model will be provided one word and the photo and generate the next word. Then the first two words of the description will be provided to the model as input with the image to generate the next word. This is how the model will be trained.","metadata":{}},{"cell_type":"code","source":"#calculated the length of description with most words\ndef max_lengthTEMP(descriptions):\n    lines = to_lines(descriptions)\n    return max(len(d.split()) for d in lines)","metadata":{"execution":{"iopub.status.busy":"2022-04-26T14:15:05.714054Z","iopub.execute_input":"2022-04-26T14:15:05.71436Z","iopub.status.idle":"2022-04-26T14:15:05.721195Z","shell.execute_reply.started":"2022-04-26T14:15:05.714306Z","shell.execute_reply":"2022-04-26T14:15:05.720152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## B.) Defining the Model","metadata":{}},{"cell_type":"markdown","source":"The model is in three parts:\n\n1. **Photo Feature Extractor**: This is a 16-layer VGG model pre-trained on the ImageNet dataset. We have pre-processed the photos with the VGG model (without the output layer) and will use the extracted features predicted by this model as input.\n\n2. **Sequence Processor**: This is a word embedding layer for handling the text input, followed by a Long Short-Term Memory (LSTM) recurrent neural network layer.\n\n3. **Decoder**: Both the feature extractor and sequence processor output a fixed-length vector. These are merged together and processed by a Dense layer to make a final prediction.\nThe Photo Feature Extractor model expects input photo features to be a vector of 4,096 elements. These are processed by a Dense layer to produce a 256 element representation of the photo.\n\n\nThe Sequence Processor model expects input sequences with a pre-defined length (34 words) which are fed into an Embedding layer that uses a mask to ignore padded values. This is followed by an LSTM layer with 256 memory units.\n\nBoth the input models produce a 256 element vector. Further, both input models use regularization in the form of 50% dropout. This is to reduce overfitting the training dataset, as this model configuration learns very fast.\n\nThe Decoder model merges the vectors from both input models using an addition operation. This is then fed to a Dense 256 neuron layer and then to a final output Dense layer that makes a softmax prediction over the entire output vocabulary for the next word in the sequence.\n\n","metadata":{}},{"cell_type":"code","source":"#the below function loop forever with a while loop and within this, \n#loop over each image in the image directory. \n#For each image filename, we can load the image and \n#create all of the input-output sequence pairs from the image’s description.\n\n#data generator, intended to be used in a call to model.fit_generator()\ndef data_generator(descriptions, photos, tokenizer, max_length):\n    while 1:\n        for key, description_list in descriptions.items():\n            #retrieve photo features\n            photo = photos[key][0]\n            input_image, input_sequence, output_word = create_sequences(tokenizer, max_length, description_list, photo)\n            yield [[input_image, input_sequence], output_word]\n\n            \n#we are calling the create_sequence() function to create \n#a batch worth of data for a single photo rather than an entire dataset. \n#This means that we must update the create_sequences() function \n#to delete the “iterate over all descriptions” for-loop.            \n#Updated create sequence function for data_generator\ndef create_sequences(tokenizer, max_length, desc_list, photo):\n    X1, X2, y = list(), list(), list()\n    # walk through each description for the image\n    for desc in desc_list:\n        # encode the sequence\n        seq = tokenizer.texts_to_sequences([desc])[0]\n        # split one sequence into multiple X,y pairs\n        for i in range(1, len(seq)):\n            # split into input and output pair\n            in_seq, out_seq = seq[:i], seq[i]\n            # pad input sequence\n            in_seq = pad_sequences([in_seq], maxlen=max_length)[0]\n            # encode output sequence\n            out_seq = to_categorical([out_seq], num_classes=vocab_size)[0]\n            # store\n            X1.append(photo)\n            X2.append(in_seq)\n            y.append(out_seq)\n    return array(X1), array(X2), array(y)","metadata":{"execution":{"iopub.status.busy":"2021-11-25T19:42:03.407092Z","iopub.status.idle":"2021-11-25T19:42:03.407528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.utils import plot_model\n# define the captioning model\ndef define_model(vocab_size, max_length):\n    \n    # feature extractor model\n    inputs1 = Input(shape=(4096,))\n    fe1 = Dropout(0.5)(inputs1)\n    fe2 = Dense(256, activation='relu')(fe1)\n\n    # sequence model\n    inputs2 = Input(shape=(max_length,))\n    se1 = Embedding(vocab_size, 256, mask_zero=True)(inputs2)\n    se2 = Dropout(0.5)(se1)\n    se3 = LSTM(256)(se2)\n\n    # decoder model\n    decoder1 = add([fe2, se3])\n    decoder2 = Dense(256, activation='relu')(decoder1)\n    outputs = Dense(vocab_size, activation='softmax')(decoder2)\n    \n    # tie it together [image, seq] [word]\n    model = Model(inputs=[inputs1, inputs2], outputs=outputs)\n    model.compile(loss='categorical_crossentropy', optimizer='adam')\n    \n    # summarize model\n    print(model.summary())\n    plot_model(model, to_file='model.png', show_shapes=True)\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2021-11-25T19:42:03.417789Z","iopub.status.idle":"2021-11-25T19:42:03.418231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfilename = '../input/flickr8k_text/Flickr_8k.trainImages.txt'\ntrain = load_photo_identifiers(filename)\n# Returning the set of photos created\n    #return set(photos)\nprint('Dataset: ', len(train))\n#descriptions[image_id].append(desccription)\ntrain_descriptions = load_clean_descriptions('descriptions.txt', train)\nprint('Descriptions: train=', len(train_descriptions))\n\n#dict\ntrain_features = load_photo_features('features.pkl', train)\nprint('Photos: train=', len(train_features))\n\n#dict\ntokenizer = create_tokenizer(train_descriptions)\n\nvocab_size = len(tokenizer.word_index) + 1\nprint('Vocabulary Size:', vocab_size)\nmax_length = max_lengthTEMP(train_descriptions)\nprint('Description Length: ', max_length)\n\nmodel = define_model(vocab_size, max_length)\nepochs = 3\nsteps = len(train_descriptions)\nfor i in range(epochs):\n    generator = data_generator(train_descriptions, train_features, tokenizer, max_length)\n    model.fit_generator(generator, epochs=1, steps_per_epoch=steps, verbose=1)\n    model.save('model_' + str(i) + '.h5')","metadata":{"execution":{"iopub.status.busy":"2021-11-25T19:42:03.430977Z","iopub.status.idle":"2021-11-25T19:42:03.431634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. Evaluate the model","metadata":{}},{"cell_type":"code","source":"#this function maps an integer to a word\ndef word_for_id(integer, tokenizer):\n    for word, index in tokenizer.word_index.items():\n        if index == integer:\n            return word\n    return None\n\n#The function below generates a textual description given a trained model, \n#and a given prepared photo as input. It calls the function word_for_id() \n#in order to map an integer prediction back to a word.\ndef generate_desc(model, tokenizer, photo):\n    #start tge generation process\n    in_text = 'startseq'\n    #iterating over the max_length since the maximum length of the description can be that only\n    for i in range(max_length):\n        #integer ncoding input sequence\n        sequence = tokenizer.texts_to_sequences([in_text])[0]\n        #padding the input\n        sequence = pad_sequences([sequence], maxlen=34)\n        #predicting next word\n        #the predict function will return probability\n        prob = model.predict([photo,sequence], verbose=0)\n        #converting the probability to integer\n        prob = argmax(prob)\n        #calling the word_for_id function in order to map integer to word\n        word = word_for_id(prob, tokenizer)\n        #breaking if word cannot be mapped\n        if word is None:\n            break\n        #appending as input\n        in_text += ' ' + word\n        #break if end is predicted\n        if word == 'endseq':\n            break\n    return in_text\n\n#the below function evaluates the skill of the model\ndef evaluate_model(model, descriptions, photos, tokenizer, max_length):\n    actual, predicted = list(), list()\n    for key, desc_list in descriptions.items():\n        prediction = generate_desc(model, tokenizer, photos[key], max_length)\n        actual_desc = [d.split() for d in desc_list]\n        actual.append(actual_desc)\n        predicted.append(prediction.split())\n\n    print('BLEU-1: ', corpus_bleu(actual, predicted, weights=(1.0, 0, 0, 0)))\n    print('BLEU-2: ', corpus_bleu(actual, predicted, weights=(0.5, 0.5, 0, 0)))\n    print('BLEU-3: ', corpus_bleu(actual, predicted, weights=(0.3, 0.3, 0.3, 0)))\n    print('BLEU-4: ', corpus_bleu(actual, predicted, weights=(0.25, 0.25, 0.25, 0.25)))\n    \ndef max_length(descriptions):\n    lines = to_lines(descriptions)\n    return max(len(d.split()) for d in lines)","metadata":{"execution":{"iopub.status.busy":"2021-11-25T19:42:03.572474Z","iopub.execute_input":"2021-11-25T19:42:03.572734Z","iopub.status.idle":"2021-11-25T19:42:03.584986Z","shell.execute_reply.started":"2021-11-25T19:42:03.572687Z","shell.execute_reply":"2021-11-25T19:42:03.583314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filename = '../input/flickr8k_text/Flickr_8k.trainImages.txt'\ntrain = load_photo_identifiers(filename)\nprint('Dataset: ', len(train))\ntrain_descriptions = load_clean_descriptions('descriptions.txt', train)\nprint('Descriptions: train=', len(train_descriptions))\ntokenizer = create_tokenizer(train_descriptions)\nvocab_size = len(tokenizer.word_index) + 1\nprint('Vocabulary Size: ', vocab_size)\nmax_length = max_lengthTEMP(train_descriptions)\nprint('Description Length: ,', max_length)\n\nfilename = '../input/flickr8k_text/Flickr_8k.testImages.txt'\ntest = load_photo_identifiers(filename)\nprint('Dataset: ', len(test))\ntest_descriptions = load_clean_descriptions('descriptions.txt', test)\nprint('Descriptions: test=', len(test_descriptions))\ntest_features = load_photo_features('features.pkl', test)\nprint('Photos: test=', len(test_features))\n\nfilename = 'model_12.h5'\nmodel = load_model(filename)\nevaluate_model(model, test_descriptions, test_features, tokenizer, max_length)","metadata":{"execution":{"iopub.status.busy":"2021-11-25T19:42:03.591545Z","iopub.execute_input":"2021-11-25T19:42:03.592045Z","iopub.status.idle":"2021-11-25T19:42:03.666446Z","shell.execute_reply.started":"2021-11-25T19:42:03.591843Z","shell.execute_reply":"2021-11-25T19:42:03.609201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5. Generate new descriptions","metadata":{}},{"cell_type":"code","source":"filename = '../input/flickr8k_text/Flickr_8k.trainImages.txt'\ntrain = load_photo_identifiers(filename)\nprint('Dataset: ', len(train))\ntrain_descriptions = load_clean_descriptions('descriptions.txt', train)\nprint('Descriptions: train=', len(train_descriptions))\ntokenizer = create_tokenizer(train_descriptions)\ndump(tokenizer, open('tokenizer.pkl', 'wb'))","metadata":{"execution":{"iopub.status.busy":"2022-04-26T14:20:33.023275Z","iopub.execute_input":"2022-04-26T14:20:33.02356Z","iopub.status.idle":"2022-04-26T14:20:33.296278Z","shell.execute_reply.started":"2022-04-26T14:20:33.023509Z","shell.execute_reply":"2022-04-26T14:20:33.29501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def extract_features(filename):\n    model = VGG16()\n    model.layers.pop()\n    model = Model(inputs=model.inputs, outputs=model.layers[-1].output)\n    image = load_img(filename, target_size=(224, 224))\n    image = img_to_array(image)\n    image = image.reshape((1, image.shape[0], image.shape[1], image.shape[2]))\n    image = preprocess_input(image)\n    feature = model.predict(image, verbose=0)\n    return feature\n#this function maps an integer to a word\ndef word_for_id(integer, tokenizer):\n    for word, index in tokenizer.word_index.items():\n        if index == integer:\n            return word\n    return None\n\ndef generate_desc(model, tokenizer, photo, max_length):\n    in_text = 'startseq'\n    for i in range(max_length):\n        sequence = tokenizer.texts_to_sequences([in_text])[0]\n        sequence = pad_sequences([sequence], maxlen=max_length)\n        yhat = model.predict([photo,sequence], verbose=0)\n        yhat = argmax(yhat)\n        word = word_for_id(yhat, tokenizer)\n        if word is None:\n            break\n        in_text += ' ' + word\n        if word == 'endseq':\n            break\n    return in_text\n\ntokenizer = load(open('tokenizer.pkl', 'rb'))\nmax_length = 34","metadata":{"execution":{"iopub.status.busy":"2021-11-25T19:42:03.611354Z","iopub.status.idle":"2021-11-25T19:42:03.612168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"listdir(\"../input/flickr8k_dataset/Flicker8k_Dataset/2281075738_230892b241.jpg\")","metadata":{"execution":{"iopub.status.busy":"2021-11-25T19:42:03.613149Z","iopub.status.idle":"2021-11-25T19:42:03.613807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = load_model('model_19.h5')\npath = '../input/flickr8k_dataset/Flicker8k_Dataset/111537222_07e56d5a30.jpg'\nphoto = extract_features(path)\n\ndescription = generate_desc(model, tokenizer, photo, max_length)\nprint(description)","metadata":{"execution":{"iopub.status.busy":"2021-11-25T19:42:03.614614Z","iopub.status.idle":"2021-11-25T19:42:03.615284Z"},"trusted":true},"execution_count":null,"outputs":[]}]}