{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nos.environ[\"TF_CPP_MIN_LOG_LEVEL\"] = \"2\"\n\nimport base64\nimport io\nimport json\nimport numpy as np\nimport tensorflow as tf\nimport time\nimport IPython\nimport PIL\nfrom io import BytesIO\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport gc\n\nfrom kaggle_secrets import UserSecretsClient\n\n\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print(\"Device:\", tpu.master())\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept:\n    strategy = tf.distribute.get_strategy()\n\nAUTOTUNE = tf.data.experimental.AUTOTUNE\nprint(\"Number of replicas:\", strategy.num_replicas_in_sync)\nprint(tf.__version__)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-24T22:08:47.371934Z","iopub.execute_input":"2021-11-24T22:08:47.372468Z","iopub.status.idle":"2021-11-24T22:08:52.732159Z","shell.execute_reply.started":"2021-11-24T22:08:47.372331Z","shell.execute_reply":"2021-11-24T22:08:52.731461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = \"../input/wikipedia-image-caption/image_data_test/image_pixels/test_image_pixels_part-00000.csv\"\ncolumns=['image_url', 'b64_bytes', 'metadata_url']\n\ndf = pd.read_csv(path, sep=\"\\t\", header=None) \n\n#For now, only read the first test file to check","metadata":{"execution":{"iopub.status.busy":"2021-11-24T22:09:31.504241Z","iopub.execute_input":"2021-11-24T22:09:31.504752Z","iopub.status.idle":"2021-11-24T22:09:40.483537Z","shell.execute_reply.started":"2021-11-24T22:09:31.504707Z","shell.execute_reply":"2021-11-24T22:09:40.482514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.columns = columns\ndf","metadata":{"execution":{"iopub.status.busy":"2021-11-24T22:10:08.449476Z","iopub.execute_input":"2021-11-24T22:10:08.450583Z","iopub.status.idle":"2021-11-24T22:10:08.494503Z","shell.execute_reply.started":"2021-11-24T22:10:08.450512Z","shell.execute_reply":"2021-11-24T22:10:08.49338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def convert_b64_bytes_to_240x240(b64_byte):\n    #resize the image. Convert to numpy\n    b64EncodedImage = b64_byte\n    b64DecodedImage = base64.b64decode(b64EncodedImage)\n    PILImage = PIL.Image.open(io.BytesIO(b64DecodedImage)).convert('RGB')\n    image_np = np.array(PILImage)\n    \n    #resize to 240x240\n    resized = tf.image.resize(image_np, [240,240])\n    resized = resized.numpy()\n    resized = resized.astype(np.uint8)\n    \n    #convert back to b64 for saving into dataframe\n    pil_img = PIL.Image.fromarray(resized)\n    buff = BytesIO()\n    pil_img.save(buff, format=\"PNG\")\n    finalB64ImageString = base64.b64encode(buff.getvalue()).decode(\"utf-8\")\n    \n    return finalB64ImageString","metadata":{"execution":{"iopub.status.busy":"2021-11-24T22:10:24.733214Z","iopub.execute_input":"2021-11-24T22:10:24.733496Z","iopub.status.idle":"2021-11-24T22:10:24.741538Z","shell.execute_reply.started":"2021-11-24T22:10:24.733467Z","shell.execute_reply":"2021-11-24T22:10:24.740661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"b64_byte = df['b64_bytes'][0]\nb64EncodedImage = b64_byte\nb64DecodedImage = base64.b64decode(b64EncodedImage)\nPILImage = PIL.Image.open(io.BytesIO(b64DecodedImage)).convert('RGB')\nimage_np = np.array(PILImage)","metadata":{"execution":{"iopub.status.busy":"2021-11-24T22:10:27.495737Z","iopub.execute_input":"2021-11-24T22:10:27.496348Z","iopub.status.idle":"2021-11-24T22:10:27.536937Z","shell.execute_reply.started":"2021-11-24T22:10:27.496294Z","shell.execute_reply":"2021-11-24T22:10:27.53603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(image_np)","metadata":{"execution":{"iopub.status.busy":"2021-11-24T22:11:10.929658Z","iopub.execute_input":"2021-11-24T22:11:10.930728Z","iopub.status.idle":"2021-11-24T22:11:11.223184Z","shell.execute_reply.started":"2021-11-24T22:11:10.930675Z","shell.execute_reply":"2021-11-24T22:11:11.222356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['b64_bytes'] = [convert_b64_bytes_to_240x240(b64Image) for b64Image in df['b64_bytes']]","metadata":{"execution":{"iopub.status.busy":"2021-11-24T22:11:14.785401Z","iopub.execute_input":"2021-11-24T22:11:14.787554Z","iopub.status.idle":"2021-11-24T22:14:57.590843Z","shell.execute_reply.started":"2021-11-24T22:11:14.78748Z","shell.execute_reply":"2021-11-24T22:14:57.589816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"b64_byte = df['b64_bytes'][0]\nb64EncodedImage = b64_byte\nb64DecodedImage = base64.b64decode(b64EncodedImage)\nPILImage = PIL.Image.open(io.BytesIO(b64DecodedImage)).convert('RGB')\nimage_np = np.array(PILImage)","metadata":{"execution":{"iopub.status.busy":"2021-11-24T22:15:40.660227Z","iopub.execute_input":"2021-11-24T22:15:40.660786Z","iopub.status.idle":"2021-11-24T22:15:40.672166Z","shell.execute_reply.started":"2021-11-24T22:15:40.660742Z","shell.execute_reply":"2021-11-24T22:15:40.67113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(image_np)","metadata":{"execution":{"iopub.status.busy":"2021-11-24T22:15:45.270122Z","iopub.execute_input":"2021-11-24T22:15:45.271478Z","iopub.status.idle":"2021-11-24T22:15:45.477445Z","shell.execute_reply.started":"2021-11-24T22:15:45.271385Z","shell.execute_reply":"2021-11-24T22:15:45.47681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['b64_bytes'] = [b64image.replace(\"/\", \"_\") for b64image in df['b64_bytes']]\ndf['b64_bytes'] = [b64image.replace(\"+\", \"-\") for b64image in df['b64_bytes']]","metadata":{"execution":{"iopub.status.busy":"2021-11-24T22:15:47.627128Z","iopub.execute_input":"2021-11-24T22:15:47.627754Z","iopub.status.idle":"2021-11-24T22:15:50.184336Z","shell.execute_reply.started":"2021-11-24T22:15:47.627713Z","shell.execute_reply":"2021-11-24T22:15:50.183371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Get the tokenizer from training set\nDS_PATH0 = \"../input/wikipediacaptionsubseten\"\n\ntrainDf = pd.read_csv(DS_PATH0 + \"/wikipedia-train-dataframe-en.csv\") #chunksize=10000\n\ntrainDf['image'] = [b64image.replace(\"/\", \"_\") for b64image in trainDf['image']]\ntrainDf['image'] = [b64image.replace(\"+\", \"-\") for b64image in trainDf['image']]\n\n# Tokenizing the Caption Title/Description Text\nimport nltk\nimport re\nfrom nltk.stem import WordNetLemmatizer\nimport unicodedata\n\nlemma = WordNetLemmatizer()\n\n\ndef strip_accents(text):\n    try:\n        text = unicode(text, 'utf-8')\n    except NameError: # unicode is a default on python 3 \n        pass\n\n    text = unicodedata.normalize('NFD', text)\\\n           .encode('ascii', 'ignore')\\\n           .decode(\"utf-8\")\n\n    return str(text)\n\ndef cleanElement(element):\n    # clean punctuation\n    element = re.sub(r'[^\\w\\s]' ,'', element)\n    \n    # tokenize the words\n    element = element.split()\n    \n    # convert to lower case\n    element = [token.lower() for token in element]\n    \n    # lemmatization\n    element = [lemma.lemmatize(token) for token in element]\n    \n    # join whole token\n    element = ' '.join(element)\n    \n    # remove leading and trailing whitespace\n    element = element.strip()\n    \n    return element\n\ndef clean(theCaption):\n    title_description_list = theCaption.split('[SEP]')\n    \n    if(len(title_description_list) == 2):\n        title = theCaption.split('[SEP]')[0]\n        description = theCaption.split('[SEP]')[1]\n    else:\n        title = theCaption.split('[SEP]')[0]\n        description = \"\"\n    \n    title = cleanElement(title)\n    description = cleanElement(description)\n    \n    if(description == None):\n        descripton = \"\"\n    \n    cleanedCaption = title + ' [SEP] ' + description\n    \n    #strip accented characters\n    cleanedCaption = strip_accents(cleanedCaption)\n    \n    cleanedCaption = cleanedCaption.strip()\n    \n    cleanedCaption = 'startseq ' + cleanedCaption + ' endseq'\n    \n    return cleanedCaption\n\ntrainDf['caption_title_and_reference_description'] = [clean(caption) for caption in trainDf['caption_title_and_reference_description']]\n\ndef max_length(listOfCaptions):\n    return max(len(caption.split()) for caption in listOfCaptions)\n\nfrom tensorflow.keras.preprocessing.text import Tokenizer\n\ndef create_tokenizer(listOfCaptions):\n    tokenizer = Tokenizer()\n    tokenizer.fit_on_texts(listOfCaptions)\n    return tokenizer\n\ntokenizer = create_tokenizer(trainDf['caption_title_and_reference_description'])\nvocab_size = len(tokenizer.word_index) + 1\n\nfrom keras.preprocessing.sequence import pad_sequences\nfrom tensorflow.keras.utils import to_categorical\nimport numpy as np\n\nmax_len = max_length(trainDf['caption_title_and_reference_description'])","metadata":{"execution":{"iopub.status.busy":"2021-11-24T22:15:52.789973Z","iopub.execute_input":"2021-11-24T22:15:52.790317Z","iopub.status.idle":"2021-11-24T22:17:23.586263Z","shell.execute_reply.started":"2021-11-24T22:15:52.790276Z","shell.execute_reply":"2021-11-24T22:17:23.585054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_len","metadata":{"execution":{"iopub.status.busy":"2021-11-24T22:19:09.105047Z","iopub.execute_input":"2021-11-24T22:19:09.106554Z","iopub.status.idle":"2021-11-24T22:19:09.112942Z","shell.execute_reply.started":"2021-11-24T22:19:09.106495Z","shell.execute_reply":"2021-11-24T22:19:09.11198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Redefine model and load weights\n\ntransfer_model = tf.keras.applications.EfficientNetB1(include_top=False, weights=\"imagenet\", input_shape=(240,240,3))\ntransfer_model.trainable = False\n\n#flatten_layer = tf.keras.layers.Flatten()\nglobal_average_layer = tf.keras.layers.GlobalAveragePooling2D()\n\ndef decode(img_str):\n    img = tf.io.decode_base64(img_str)\n    img = tf.image.decode_png(img, channels=3)\n    # if you need to squeeze your input range to [0,1] or [-1,1] do it here\n    return img\n\nfrom keras.models import Model\nfrom keras.layers import Input , Dense , LSTM , Embedding , Dropout\nfrom keras.layers.merge import add\nfrom keras.callbacks import EarlyStopping\n\ndef define_Model(vocab_size , max_length):\n    # feature extractor model\n    inputs1 = tf.keras.Input(shape = (1,),dtype=\"string\", name='inputs1')\n    fe1 = tf.keras.layers.Lambda(lambda img : tf.map_fn(lambda im : decode(im[0]), img, dtype=\"uint8\"))(inputs1) #decoding b64 to 240x240x3 array\n    fe2 = transfer_model(fe1, training=False) #transfer model EfficientNetB1\n    fe3 = global_average_layer(fe2) #Average over the spatial 8x8 spatial locations to a single 1280-element feature vector\n    fe4 = tf.keras.layers.Dropout(0.2)(fe3)\n    fe5 = Dense(256 , activation = 'relu')(fe4)\n    \n    # sequence model\n    inputs2 = tf.keras.Input(shape=(max_length,), name='inputs2')\n    se1 = Embedding(vocab_size,512,mask_zero=True )(inputs2) # mask_zero = ignore padding\n    se2 = Dropout(0.3)(se1)\n    se3 = LSTM(512 , return_sequences=True)(se2)\n    se4 = Dropout(0.3)(se3)\n    se5 = LSTM(256)(se4)\n    \n    \n    #decoder Model\n    decoder1 = add([fe5 , se5])\n    decoder2 = Dense(256 , activation='relu')(decoder1)\n    decoder3 = Dense(512 , activation='relu')(decoder2)\n    outputs = Dense(vocab_size , activation='softmax')(decoder3)\n    \n    # combine both image and text\n    model = Model(inputs=[inputs1 , inputs2] , outputs=outputs)\n    model.compile(loss='categorical_crossentropy' , optimizer = 'adam')\n    \n    # summary\n    print(model.summary())\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2021-11-24T22:19:11.245047Z","iopub.execute_input":"2021-11-24T22:19:11.245373Z","iopub.status.idle":"2021-11-24T22:19:14.444709Z","shell.execute_reply.started":"2021-11-24T22:19:11.245336Z","shell.execute_reply":"2021-11-24T22:19:14.443644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.preprocessing.sequence import pad_sequences\nfrom tensorflow.keras.utils import to_categorical\nimport numpy as np\n\ndef word_for_id(integer , tokenizer):\n    for word,index in tokenizer.word_index.items():\n        if index == integer:\n            return word\n\n\ndef generate_desc(model , tokenizer , b64photo , max_length):\n    b64photo = np.array(b64photo).reshape(1,)\n    input_text = 'startseq'\n    \n    for i in range(max_length):\n        \n        sequence = tokenizer.texts_to_sequences([input_text])[0]\n        sequence = pad_sequences([sequence] , maxlen=max_length)\n        \n        # predict the next word\n        next_word_id = model.predict([b64photo,sequence],verbose = 0)\n        \n        # get highest probality word from list of words\n        next_word_id = np.argmax(next_word_id)\n        \n        # get word from id\n        word = word_for_id(next_word_id , tokenizer)\n        \n        if word is None:\n            break\n            \n        # update input text\n        input_text += ' '+ word\n        \n        if word == 'endseq':\n            break\n            \n    return input_text","metadata":{"execution":{"iopub.status.busy":"2021-11-24T22:19:17.586031Z","iopub.execute_input":"2021-11-24T22:19:17.58708Z","iopub.status.idle":"2021-11-24T22:19:17.598002Z","shell.execute_reply.started":"2021-11-24T22:19:17.587027Z","shell.execute_reply":"2021-11-24T22:19:17.596829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"saved_weights = '../input/wikipedia-image-caption-proper-training/model_2.h5'\nmodel = define_Model(vocab_size , max_len)\nmodel.load_weights(saved_weights)","metadata":{"execution":{"iopub.status.busy":"2021-11-24T22:19:59.556514Z","iopub.execute_input":"2021-11-24T22:19:59.556944Z","iopub.status.idle":"2021-11-24T22:20:07.330843Z","shell.execute_reply.started":"2021-11-24T22:19:59.556909Z","shell.execute_reply":"2021-11-24T22:20:07.32983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#The captions for the test set 'test_image_pixels_part-00000.csv'\ngeneratedCaptions = [generate_desc(model , tokenizer , b64photo , max_len) for b64photo in df['b64_bytes']]","metadata":{"execution":{"iopub.status.busy":"2021-11-24T22:57:50.011698Z","iopub.execute_input":"2021-11-24T22:57:50.012004Z","iopub.status.idle":"2021-11-24T22:59:20.844761Z","shell.execute_reply.started":"2021-11-24T22:57:50.011973Z","shell.execute_reply":"2021-11-24T22:59:20.843526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check one at a time on test\nb64photo = df['b64_bytes'][3]\ngenerate_desc(model , tokenizer , b64photo , max_len)","metadata":{"execution":{"iopub.status.busy":"2021-11-24T22:20:51.960965Z","iopub.execute_input":"2021-11-24T22:20:51.961254Z","iopub.status.idle":"2021-11-24T22:20:57.594841Z","shell.execute_reply.started":"2021-11-24T22:20:51.961225Z","shell.execute_reply":"2021-11-24T22:20:57.593861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#compare with the image\nb64_byte = df['b64_bytes'][3]\nb64_byte = b64_byte.replace(\"_\", \"/\") \nb64_byte = b64_byte.replace(\"-\", \"+\") \nb64EncodedImage = b64_byte\nb64DecodedImage = base64.b64decode(b64EncodedImage)\nPILImage = PIL.Image.open(io.BytesIO(b64DecodedImage)).convert('RGB')\nimage_np = np.array(PILImage)\nplt.imshow(image_np)","metadata":{"execution":{"iopub.status.busy":"2021-11-24T22:20:59.663447Z","iopub.execute_input":"2021-11-24T22:20:59.664706Z","iopub.status.idle":"2021-11-24T22:20:59.865095Z","shell.execute_reply.started":"2021-11-24T22:20:59.664662Z","shell.execute_reply":"2021-11-24T22:20:59.864114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nos.chdir(r'/kaggle/working')","metadata":{"execution":{"iopub.status.busy":"2021-11-24T23:02:08.706558Z","iopub.execute_input":"2021-11-24T23:02:08.707483Z","iopub.status.idle":"2021-11-24T23:02:08.711836Z","shell.execute_reply.started":"2021-11-24T23:02:08.707423Z","shell.execute_reply":"2021-11-24T23:02:08.711152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('generatedCaptions0.txt', 'w') as filehandle:\n    for caption in generatedCaptions:\n        filehandle.write('%s\\n' % caption)","metadata":{"execution":{"iopub.status.busy":"2021-11-24T23:02:10.54632Z","iopub.execute_input":"2021-11-24T23:02:10.547192Z","iopub.status.idle":"2021-11-24T23:02:10.552368Z","shell.execute_reply.started":"2021-11-24T23:02:10.547136Z","shell.execute_reply":"2021-11-24T23:02:10.551677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check generated captions on training set\nb64photo = trainDf['image'][90]\ngenerate_desc(model , tokenizer , b64photo , max_len)","metadata":{"execution":{"iopub.status.busy":"2021-11-24T23:11:11.376401Z","iopub.execute_input":"2021-11-24T23:11:11.377244Z","iopub.status.idle":"2021-11-24T23:11:13.848467Z","shell.execute_reply.started":"2021-11-24T23:11:11.377206Z","shell.execute_reply":"2021-11-24T23:11:13.847253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainDf['caption_title_and_reference_description'][90]","metadata":{"execution":{"iopub.status.busy":"2021-11-24T23:11:16.631755Z","iopub.execute_input":"2021-11-24T23:11:16.632101Z","iopub.status.idle":"2021-11-24T23:11:16.63867Z","shell.execute_reply.started":"2021-11-24T23:11:16.632066Z","shell.execute_reply":"2021-11-24T23:11:16.637827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"b64_byte = trainDf['image'][90]\nb64_byte = b64_byte.replace(\"_\", \"/\") \nb64_byte = b64_byte.replace(\"-\", \"+\") \nb64EncodedImage = b64_byte\nb64DecodedImage = base64.b64decode(b64EncodedImage)\nPILImage = PIL.Image.open(io.BytesIO(b64DecodedImage)).convert('RGB')\nimage_np = np.array(PILImage)\nplt.imshow(image_np)","metadata":{"execution":{"iopub.status.busy":"2021-11-24T23:11:21.987575Z","iopub.execute_input":"2021-11-24T23:11:21.987959Z","iopub.status.idle":"2021-11-24T23:11:22.17478Z","shell.execute_reply.started":"2021-11-24T23:11:21.987921Z","shell.execute_reply":"2021-11-24T23:11:22.17363Z"},"trusted":true},"execution_count":null,"outputs":[]}]}