{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"## Try efficientnet in addition to InceptionV3 of the original example.\n!pip install -q tensorflow==2.2-rc4 # fix TPU memory issue\n!pip install -q efficientnet\n\nN_VOCABS = 20000 # all vocabs of flickr30k is around 18k, so we choose them all -- if training loss does not work well, change to 5K","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-09-17T09:06:45.897502Z","iopub.execute_input":"2021-09-17T09:06:45.897828Z","iopub.status.idle":"2021-09-17T09:08:02.938032Z","shell.execute_reply.started":"2021-09-17T09:06:45.897797Z","shell.execute_reply":"2021-09-17T09:08:02.937109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"USE_PREVIOUS_SAVE = True","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-09-17T09:08:02.941775Z","iopub.execute_input":"2021-09-17T09:08:02.942011Z","iopub.status.idle":"2021-09-17T09:08:02.948552Z","shell.execute_reply.started":"2021-09-17T09:08:02.941984Z","shell.execute_reply":"2021-09-17T09:08:02.947660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\n\n# You'll generate plots of attention in order to see which parts of an image\n# our model focuses on during captioning\nimport matplotlib.pyplot as plt\n\n# Scikit-learn includes many helpful utilities\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.utils import shuffle\n\nimport re\nimport numpy as np\nimport os\nimport time\nimport json\nimport gc\nfrom glob import glob\nfrom PIL import Image\nimport pickle\nimport pandas as pd","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:08:02.949696Z","iopub.execute_input":"2021-09-17T09:08:02.950217Z","iopub.status.idle":"2021-09-17T09:08:02.958202Z","shell.execute_reply.started":"2021-09-17T09:08:02.950189Z","shell.execute_reply":"2021-09-17T09:08:02.956952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from kaggle_datasets import KaggleDatasets\nimport efficientnet.tfkeras as efn \nfrom tokenizers import ByteLevelBPETokenizer","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-09-17T09:08:02.960435Z","iopub.execute_input":"2021-09-17T09:08:02.960709Z","iopub.status.idle":"2021-09-17T09:08:03.219856Z","shell.execute_reply.started":"2021-09-17T09:08:02.960676Z","shell.execute_reply":"2021-09-17T09:08:03.219134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-09-17T09:08:03.220945Z","iopub.execute_input":"2021-09-17T09:08:03.221563Z","iopub.status.idle":"2021-09-17T09:08:03.231077Z","shell.execute_reply.started":"2021-09-17T09:08:03.221525Z","shell.execute_reply":"2021-09-17T09:08:03.230376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LOCAL_FLICKR_PATH = '/kaggle/input/flickr-image-dataset/flickr30k_images/'\nannotation_file = LOCAL_FLICKR_PATH + 'results.csv'\nLOCAL_IMG_PATH = LOCAL_FLICKR_PATH + 'flickr30k_images/'\n\n!ls {LOCAL_IMG_PATH} | wc","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:10:38.688183Z","iopub.execute_input":"2021-09-17T09:10:38.688816Z","iopub.status.idle":"2021-09-17T09:10:39.964812Z","shell.execute_reply.started":"2021-09-17T09:10:38.688776Z","shell.execute_reply":"2021-09-17T09:10:39.963903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n## This steps will take around 25 minutes offline ...\nif strategy.num_replicas_in_sync == 8:\n#     GCS_DS_PATH_FLICKR = KaggleDatasets().get_gcs_path('flickr8k-sau') # 2gb # 5 mins\n    GCS_DS_PATH = KaggleDatasets().get_gcs_path('flickr-image-dataset') # 8gb # 20-25 mins\n    print('yeah')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-09-17T09:10:56.144790Z","iopub.execute_input":"2021-09-17T09:10:56.145127Z","iopub.status.idle":"2021-09-17T09:10:56.160540Z","shell.execute_reply.started":"2021-09-17T09:10:56.145092Z","shell.execute_reply":"2021-09-17T09:10:56.156830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if strategy.num_replicas_in_sync == 8:\n    # print(GCS_DS_PATH_FLICKR)\n    # !gsutil ls $GCS_DS_PATH_FLICKR\n\n    print(GCS_DS_PATH)\n    !gsutil ls $GCS_DS_PATH\n    \n    FLICKR_PATH = GCS_DS_PATH + '/flickr30k_images/'\n    IMG_PATH = FLICKR_PATH + 'flickr30k_images/'\n    # less than 10sec\n    !gsutil ls {IMG_PATH} | wc\nelse: \n    FLICKR_PATH = LOCAL_FLICKR_PATH\n    IMG_PATH = LOCAL_IMG_PATH","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-09-17T09:11:05.147139Z","iopub.execute_input":"2021-09-17T09:11:05.147416Z","iopub.status.idle":"2021-09-17T09:11:05.154104Z","shell.execute_reply.started":"2021-09-17T09:11:05.147388Z","shell.execute_reply":"2021-09-17T09:11:05.153335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# convert the loaded descriptions into a vocabulary of words\ndef to_vocabulary(descriptions):\n    # build a list of all description strings\n    all_desc = set()\n    for key in descriptions.keys():\n        [all_desc.update(d.split()) for d in descriptions[key]]\n    return all_desc\n\n# summarize vocabulary\nvocabulary = to_vocabulary(descriptions)\nprint('Vocabulary Size: %d' % len(vocabulary))","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-09-17T06:47:23.985565Z","iopub.execute_input":"2021-09-17T06:47:23.986115Z","iopub.status.idle":"2021-09-17T06:47:24.046272Z","shell.execute_reply.started":"2021-09-17T06:47:23.986068Z","shell.execute_reply":"2021-09-17T06:47:24.045419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(annotation_file, delimiter='|') # a trick learned from other kernel\nprint(df.shape)\nprint(df.columns[2], df.columns[2] == ' comment') # wtf?\ndf[' comment'].values[0]\ndf.head(6)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-09-17T09:11:24.416035Z","iopub.execute_input":"2021-09-17T09:11:24.416314Z","iopub.status.idle":"2021-09-17T09:11:24.803960Z","shell.execute_reply.started":"2021-09-17T09:11:24.416288Z","shell.execute_reply":"2021-09-17T09:11:24.803079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm, tqdm_notebook\ntqdm.pandas()\nSTART_TOKEN = '<start> '\nEND_TOKEN = ' <end>'\n\ntokenizer = ByteLevelBPETokenizer(lowercase=True)\ntokenizer","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-09-17T09:11:33.362036Z","iopub.execute_input":"2021-09-17T09:11:33.362481Z","iopub.status.idle":"2021-09-17T09:11:33.394271Z","shell.execute_reply.started":"2021-09-17T09:11:33.362440Z","shell.execute_reply":"2021-09-17T09:11:33.393413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_start_end(text):\n    return START_TOKEN + str(text) + END_TOKEN\n\ndf['comment'] = df[' comment'].progress_apply(add_start_end)\ndf.comment.values[:6]","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-09-17T09:11:41.504662Z","iopub.execute_input":"2021-09-17T09:11:41.504965Z","iopub.status.idle":"2021-09-17T09:11:41.796686Z","shell.execute_reply.started":"2021-09-17T09:11:41.504934Z","shell.execute_reply":"2021-09-17T09:11:41.795416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Don't need to do all_captions_dict anymore thanks to xhlulu \"how to use Dataset\" instead of DataGen\n## https://www.kaggle.com/xhlulu/plant-pathology-very-concise-tpu-efficientnet\n## If preparing captions_dict this will take 13 minutes!!!\n\n# all_captions_dict = {} # for data generator : dict of list of all captions\nfull_img_name_list = [] # include gs path\n# img_name_list = [] # only image name, maybe for easier future reference\n\nfor ii in tqdm_notebook(range(len(df))):\n    full_image_path = IMG_PATH + df.image_name.values[ii]\n    full_img_name_list.append(full_image_path)\n                        \n#     captions = df[df['image_name']==name].comment.values\n#     all_captions_dict[name] = captions\n\n# len(all_captions_dict), len(full_img_name_list)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-09-17T09:11:52.521953Z","iopub.execute_input":"2021-09-17T09:11:52.522229Z","iopub.status.idle":"2021-09-17T09:11:53.991646Z","shell.execute_reply.started":"2021-09-17T09:11:52.522202Z","shell.execute_reply":"2021-09-17T09:11:53.990807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_captions_list = list(df.comment.values)\nprint(len(all_captions_list), all_captions_list[:5])\nprint(full_img_name_list[:3])","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-09-17T09:12:00.163763Z","iopub.execute_input":"2021-09-17T09:12:00.164020Z","iopub.status.idle":"2021-09-17T09:12:00.176462Z","shell.execute_reply.started":"2021-09-17T09:12:00.163995Z","shell.execute_reply":"2021-09-17T09:12:00.175765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\ngc.collect()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-09-17T09:12:07.171466Z","iopub.execute_input":"2021-09-17T09:12:07.172108Z","iopub.status.idle":"2021-09-17T09:12:07.570496Z","shell.execute_reply.started":"2021-09-17T09:12:07.172074Z","shell.execute_reply":"2021-09-17T09:12:07.569676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from nltk.stem import PorterStemmer\nps = PorterStemmer()\nfrom nltk.stem.lancaster import LancasterStemmer\nlc = LancasterStemmer()\nfrom nltk.stem import SnowballStemmer\nsb = SnowballStemmer(\"english\")\n\nfrom gensim.models import KeyedVectors\nimport gensim\ndef build_matrix(word_index, embedding_index, vec_dim):\n    \n    num_unk = 0\n    \n    emb_mean, emb_std = -0.0033470048, 0.109855264\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (len(word_index) + 1,vec_dim))\n#     embedding_matrix = np.zeros((len(word_index) + 1, vec_dim))\n    for word, i in word_index.items():\n        known = False\n        for candidate in [word, word.lower(), word.upper(), word.capitalize(), \n                          ps.stem(word), lc.stem(word), sb.stem(word) ]:\n            if candidate in embedding_index:\n                embedding_matrix[i] = embedding_index[candidate]\n                known = True\n                break\n        if known == False: num_unk += 1\n    \n    print('number of unknown words is ', num_unk)\n    return embedding_matrix","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-09-17T09:12:20.736747Z","iopub.execute_input":"2021-09-17T09:12:20.737010Z","iopub.status.idle":"2021-09-17T09:12:20.862580Z","shell.execute_reply.started":"2021-09-17T09:12:20.736983Z","shell.execute_reply":"2021-09-17T09:12:20.861802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nEMBEDDING_FILES = [\n    '../input/gensim-embeddings-dataset/crawl-300d-2M.gensim',\n    '../input/gensim-embeddings-dataset/glove.840B.300d.gensim'\n]\nglove_model = gensim.models.KeyedVectors.load(EMBEDDING_FILES[1], mmap='r')\ngensim_words = glove_model.index2word\nprint(len(gensim_words), gensim_words[:20])\n# How to use\nprint(glove_model['the'].shape)\n'the' in glove_model","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-09-17T09:12:36.770839Z","iopub.execute_input":"2021-09-17T09:12:36.771168Z","iopub.status.idle":"2021-09-17T09:13:07.830265Z","shell.execute_reply.started":"2021-09-17T09:12:36.771135Z","shell.execute_reply":"2021-09-17T09:13:07.829478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Find the maximum length of any caption in our dataset\ndef calc_max_length(tensor):\n    return max(len(t) for t in tensor)","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:13:07.834680Z","iopub.execute_input":"2021-09-17T09:13:07.835127Z","iopub.status.idle":"2021-09-17T09:13:07.844146Z","shell.execute_reply.started":"2021-09-17T09:13:07.835091Z","shell.execute_reply":"2021-09-17T09:13:07.843452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Choose the top_k words from the vocabulary\ntop_k = N_VOCABS \ntokenizer = tf.keras.preprocessing.text.Tokenizer(num_words=top_k,\n                                                  oov_token=\"<unk>\",\n                                                  filters='!\"#$%&()*+.,-/:;=?@[\\]^_`{|}~ ') # note 'a'\ntokenizer.fit_on_texts(all_captions_list)\ntrain_seqs = tokenizer.texts_to_sequences(all_captions_list)\n\ntokenizer.word_index['<pad>'] = 0\ntokenizer.index_word[0] = '<pad>'","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-09-17T09:13:07.845364Z","iopub.execute_input":"2021-09-17T09:13:07.845849Z","iopub.status.idle":"2021-09-17T09:13:13.348212Z","shell.execute_reply.started":"2021-09-17T09:13:07.845801Z","shell.execute_reply":"2021-09-17T09:13:13.347177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make list from dict\ntokenizer.index2word = [tokenizer.index_word[ii] for ii in range(len(tokenizer.word_index)) ] \nprint(tokenizer.index2word[:20]) # see top-20 most frequent words\nprint(tokenizer.index2word[-20:]) # these all come to <unk>\nlen(tokenizer.index2word)","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:13:13.350337Z","iopub.execute_input":"2021-09-17T09:13:13.350872Z","iopub.status.idle":"2021-09-17T09:13:13.363979Z","shell.execute_reply.started":"2021-09-17T09:13:13.350828Z","shell.execute_reply":"2021-09-17T09:13:13.363134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(tokenizer.index_word.get(2000, tokenizer.word_index['<end>']))\nprint(tokenizer.index_word.get(19999, tokenizer.word_index['<end>']))\nprint(tokenizer.word_index['<end>'])","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:13:17.170613Z","iopub.execute_input":"2021-09-17T09:13:17.171493Z","iopub.status.idle":"2021-09-17T09:13:17.177005Z","shell.execute_reply.started":"2021-09-17T09:13:17.171456Z","shell.execute_reply":"2021-09-17T09:13:17.176123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len_cap = np.array([len(text.split()) for text in all_captions_list])\nprint(len_cap.mean(), len_cap.std(), len_cap.max(), len_cap.min())\nmax_seq_len = int(np.percentile(len_cap,99.9))","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:13:24.782188Z","iopub.execute_input":"2021-09-17T09:13:24.782451Z","iopub.status.idle":"2021-09-17T09:13:24.962205Z","shell.execute_reply.started":"2021-09-17T09:13:24.782422Z","shell.execute_reply":"2021-09-17T09:13:24.961458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Create the tokenized vectors\ntrain_seqs = tokenizer.texts_to_sequences(all_captions_list)\n\n# Pad each vector to the max_length of the captions\n# If you do not provide a max_length value, pad_sequences calculates it automatically\ncap_vector = tf.keras.preprocessing.sequence.pad_sequences(train_seqs, padding='post', maxlen = max_seq_len, truncating='post')\n\n# Calculates the max_length, which is used to store the attention weights\nmax_length = calc_max_length(train_seqs) # TF2.1 official calculation --> strange to me, should base on cap_vector","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:13:31.031688Z","iopub.execute_input":"2021-09-17T09:13:31.032402Z","iopub.status.idle":"2021-09-17T09:13:34.381348Z","shell.execute_reply.started":"2021-09-17T09:13:31.032367Z","shell.execute_reply":"2021-09-17T09:13:34.380565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lenx = np.array([len(x) for x in cap_vector])\nprint(lenx.min(), lenx.mean(), cap_vector[0])\nprint(max_length)\nmax_length = max_seq_len\nprint(max_length)","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:13:37.399511Z","iopub.execute_input":"2021-09-17T09:13:37.400273Z","iopub.status.idle":"2021-09-17T09:13:37.484209Z","shell.execute_reply.started":"2021-09-17T09:13:37.400235Z","shell.execute_reply":"2021-09-17T09:13:37.483440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import KFold, GroupKFold\n# Create training and validation sets using an train_test_split --> Here not use, avoid leakage of the same name, using GroupKFolds\n# img_name_train, img_name_val, cap_train, cap_val = train_test_split(full_img_name_list,\n#                                                                     cap_vector,\n#                                                                     test_size=0.2,\n#                                                                     random_state=0)\n\n# 2.5% valid = 3975 captions = 795 images\nkf = GroupKFold(n_splits=40).split(X=full_img_name_list, groups=full_img_name_list)\n\nfor ind, (tr, val) in enumerate(kf):\n    img_name_train = np.array(full_img_name_list)[tr] # np.array make indexing possible\n    img_name_val = np.array(full_img_name_list)[val]\n    \n    cap_train =  cap_vector[tr]\n    cap_val =  cap_vector[val]\n    break","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:13:43.966771Z","iopub.execute_input":"2021-09-17T09:13:43.967048Z","iopub.status.idle":"2021-09-17T09:13:44.592519Z","shell.execute_reply.started":"2021-09-17T09:13:43.967020Z","shell.execute_reply":"2021-09-17T09:13:44.591776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(img_name_train[:6],'\\n')\nprint(cap_train[:6],'\\n')\nlen(img_name_train), len(cap_train), len(img_name_val), len(cap_val)","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:13:51.107170Z","iopub.execute_input":"2021-09-17T09:13:51.108092Z","iopub.status.idle":"2021-09-17T09:13:51.117594Z","shell.execute_reply.started":"2021-09-17T09:13:51.108033Z","shell.execute_reply":"2021-09-17T09:13:51.116678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_size = (299, 299,3)\nAUTO = tf.data.experimental.AUTOTUNE\n\ndef decode_image(filename, label=None, image_size=(target_size[0],target_size[1])):\n    means = [0.485, 0.456, 0.406]\n    stds = [0.229, 0.224, 0.225]\n    \n    bits = tf.io.read_file(filename)\n    image = tf.image.decode_jpeg(bits, channels=3)\n    \n#     image = (tf.cast(image, tf.float32) / 127.5) - 1\n    image = (tf.cast(image, tf.float32) / 255.0)\n    image = (image - means) / stds # for qubvel EfficientNet\n    \n    image = tf.image.resize(image, image_size)\n    \n    if label is None:\n        return image\n    else:\n        return image, label\n\ndef data_augment(image, label=None):\n    image = tf.image.random_flip_left_right(image)\n#     image = tf.image.random_flip_up_down(image)\n    \n    if label is None:\n        return image\n    else:\n        return image, label","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:14:00.678265Z","iopub.execute_input":"2021-09-17T09:14:00.678534Z","iopub.status.idle":"2021-09-17T09:14:00.687574Z","shell.execute_reply.started":"2021-09-17T09:14:00.678507Z","shell.execute_reply":"2021-09-17T09:14:00.685684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Feel free to change these parameters according to your system's configuration\nLR = 3e-4\nBATCH_SIZE = 64 * strategy.num_replicas_in_sync\nif strategy.num_replicas_in_sync == 1:\n    BATCH_SIZE = 1\n\nBUFFER_SIZE = 1000\nembedding_dim = 300 #embedding_matrix.shape[1] # 300 for Glove\nunits = 512\nvocab_size = top_k + 1 # <unk>\n\n## OLD VERSION, in this new version, this shape will be determined automatically\n# Shape of the vector extracted from InceptionV3 is (64, 2048)\n# These two variables represent that vector shape\n# features_shape = 2048\n# attention_features_shape = bf.shape[0] # 64 for InceptionV3, 100 for B1\n\nattention_features_shape = 100\nattention_viz_dim = 10 # 8 for inceptionV3","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:14:07.572015Z","iopub.execute_input":"2021-09-17T09:14:07.572599Z","iopub.status.idle":"2021-09-17T09:14:07.577831Z","shell.execute_reply.started":"2021-09-17T09:14:07.572563Z","shell.execute_reply":"2021-09-17T09:14:07.577047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class BahdanauAttention(tf.keras.Model):\n  def __init__(self, units):\n    super(BahdanauAttention, self).__init__()\n    self.W1 = tf.keras.layers.Dense(units)\n    self.W2 = tf.keras.layers.Dense(units)\n    self.V = tf.keras.layers.Dense(1)\n\n  def call(self, features, hidden):\n    # features(CNN_encoder output) shape == (batch_size, 64, embedding_dim)\n\n    # hidden shape == (batch_size, hidden_size)\n    # hidden_with_time_axis shape == (batch_size, 1, hidden_size)\n    hidden_with_time_axis = tf.expand_dims(hidden, 1)\n\n    # score shape == (batch_size, 64, hidden_size)\n    score = tf.nn.tanh(self.W1(features) + self.W2(hidden_with_time_axis))\n\n    # attention_weights shape == (batch_size, 64, 1)\n    # you get 1 at the last axis because you are applying score to self.V\n    attention_weights = tf.nn.softmax(self.V(score), axis=1)\n\n    # context_vector shape after sum == (batch_size, hidden_size)\n    context_vector = attention_weights * features\n    context_vector = tf.reduce_sum(context_vector, axis=1)\n\n    return context_vector, attention_weights","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:14:14.634664Z","iopub.execute_input":"2021-09-17T09:14:14.635156Z","iopub.status.idle":"2021-09-17T09:14:14.642331Z","shell.execute_reply.started":"2021-09-17T09:14:14.635123Z","shell.execute_reply":"2021-09-17T09:14:14.641520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CNN_Encoder(tf.keras.Model):\n    # Since you have already extracted the features and dumped it using pickle\n    # This encoder passes those features through a Fully connected layer\n    def __init__(self, embedding_dim):\n        super(CNN_Encoder, self).__init__()\n        \n        \n        self.cnn0 = efn.EfficientNetB3(weights='noisy-student', \n                                      input_shape=target_size, include_top=False)\n        \n        \n        # e.g. layers[-1].output = TensorShape([None, 10, 10, 1536]) for B3 (not global pooling)\n        self.cnn = tf.keras.Model(self.cnn0.input, self.cnn0.layers[-1].output) \n        self.cnn.trainable = False\n        \n        # shape after fc == (batch_size, attention_features_shape, embedding_dim) >> this is my mistake, should be hidden instead of embedding_dim\n        self.fc = tf.keras.layers.Dense(embedding_dim)\n        \n    # here, x is img-tensor of target_size\n    def call(self, x):\n        x = self.cnn(x) # 4D\n        x = tf.reshape(x, (x.shape[0], -1, x.shape[3]) ) # 3D\n        x = self.fc(x)\n        x = tf.nn.relu(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:14:23.350521Z","iopub.execute_input":"2021-09-17T09:14:23.352760Z","iopub.status.idle":"2021-09-17T09:14:23.360006Z","shell.execute_reply.started":"2021-09-17T09:14:23.352705Z","shell.execute_reply":"2021-09-17T09:14:23.359219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class RNN_Decoder(tf.keras.Model):\n  def __init__(self, embedding_matrix, units, vocab_size):\n    super(RNN_Decoder, self).__init__()\n    self.units = units\n    \n    self.vocab_size = embedding_matrix.shape[0]\n    \n    # new interface of pretrained embedding weights : https://github.com/tensorflow/tensorflow/issues/31086\n    # see also : https://stackoverflow.com/questions/55770009/how-to-use-a-pre-trained-embedding-matrix-in-tensorflow-2-0-rnn-as-initial-weigh\n    self.embedding = tf.keras.layers.Embedding(self.vocab_size, embedding_matrix.shape[1], \n                                               embeddings_initializer=tf.keras.initializers.Constant(embedding_matrix), \n                                               trainable=False,\n                                               mask_zero=True)\n    self.gru = tf.keras.layers.GRU(self.units,\n                                   return_sequences=True,\n                                   return_state=True,\n                                   recurrent_initializer='glorot_uniform')\n    self.fc1 = tf.keras.layers.Dense(self.units)\n    self.fc2 = tf.keras.layers.Dense(vocab_size)\n\n    self.attention = BahdanauAttention(self.units)\n  \n  # x=sequence of words\n  # features=image's extracted features \n  # hidden=GRU's hidden unit\n  def call(self, x, features, hidden):\n    \n    context_vector, attention_weights = self.attention(features, hidden)\n\n    # x shape after passing through embedding == (batch_size, 1, embedding_dim)\n    x = self.embedding(x)\n\n    # x shape after concatenation == (batch_size, 1, embedding_dim + hidden_size)\n    x = tf.concat([tf.expand_dims(context_vector, 1), x], axis=-1)\n\n    # passing the concatenated vector to the GRU\n    output, state = self.gru(x)\n\n    # shape == (batch_size, max_length, hidden_size)\n    x = self.fc1(output)\n\n    # x shape == (batch_size * max_length, hidden_size)\n    x = tf.reshape(x, (-1, x.shape[2]))\n\n    # output shape == (batch_size * max_length, vocab)\n    x = self.fc2(x)\n\n    return x, state, attention_weights\n\n  def reset_state(self, batch_size):\n    return tf.zeros((batch_size, self.units))","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:14:33.794800Z","iopub.execute_input":"2021-09-17T09:14:33.795618Z","iopub.status.idle":"2021-09-17T09:14:33.808700Z","shell.execute_reply.started":"2021-09-17T09:14:33.795568Z","shell.execute_reply":"2021-09-17T09:14:33.807831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    # tf.keras.backend.clear_session()\n    embedding_matrix = build_matrix(tokenizer.word_index, glove_model, embedding_dim)\n    print(embedding_matrix.shape) # if not use stop-stem trick, num of unknowns is 495 (vs. current 287)\n    \n    encoder = CNN_Encoder(embedding_dim)\n    decoder = RNN_Decoder(embedding_matrix, units, vocab_size)\n    \n    optimizer = tf.keras.optimizers.Adam(learning_rate=LR)\n    loss_object = tf.keras.losses.SparseCategoricalCrossentropy(\n        from_logits=True, reduction='none') \n    # Set reduction to `none` so we can do the reduction afterwards and divide by\n    # global batch size.\n\n    def loss_function(real, pred):\n        mask = tf.math.logical_not(tf.math.equal(real, 0))\n        loss_ = loss_object(real, pred)\n\n        mask = tf.cast(mask, dtype=loss_.dtype)\n        loss_ *= mask\n        \n        # About why we use `tf.nn.compute_average_loss`, please check this tutorial\n        # https://www.tensorflow.org/tutorials/distribute/custom_training#define_the_loss_function\n#         loss_ = tf.reduce_mean(loss_)\n        loss_ = tf.nn.compute_average_loss(loss_, global_batch_size=BATCH_SIZE)\n        \n        return loss_","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:14:41.213981Z","iopub.execute_input":"2021-09-17T09:14:41.214933Z","iopub.status.idle":"2021-09-17T09:14:58.252579Z","shell.execute_reply.started":"2021-09-17T09:14:41.214873Z","shell.execute_reply":"2021-09-17T09:14:58.251690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    checkpoint_path = \"./checkpoints/train\"\n    ckpt = tf.train.Checkpoint(encoder=encoder,\n                           decoder=decoder,\n                           optimizer = optimizer)\n    ckpt_manager = tf.train.CheckpointManager(ckpt, checkpoint_path, max_to_keep=5)\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:14:58.254207Z","iopub.execute_input":"2021-09-17T09:14:58.254449Z","iopub.status.idle":"2021-09-17T09:14:58.677494Z","shell.execute_reply.started":"2021-09-17T09:14:58.254415Z","shell.execute_reply":"2021-09-17T09:14:58.676769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_training_dataset():\n    train_dataset = (\n        tf.data.Dataset\n        .from_tensor_slices((img_name_train, cap_train))\n        .map(decode_image, num_parallel_calls=AUTO)\n        .cache()\n        .map(data_augment, num_parallel_calls=AUTO)\n        .repeat() # Maybe not repeat in custom training (so when and how??) <-- the current version is bug because it repeat indefinitely\n        .shuffle(BATCH_SIZE*8, reshuffle_each_iteration=True)\n        .batch(BATCH_SIZE, drop_remainder=False)\n        .prefetch(AUTO)\n    )\n    return strategy.experimental_distribute_dataset(train_dataset)\n\n\n# if use keras.model.fit, no need for repeat and drop_remainder\nvalid_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((img_name_val, cap_val))\n    .map(decode_image, num_parallel_calls=AUTO)\n#     .repeat()\n    .batch(BATCH_SIZE, drop_remainder=True)\n    .cache()\n    .prefetch(AUTO)\n)\n\nvalid_dist_dataset = strategy.experimental_distribute_dataset(valid_dataset)\nwith strategy.scope():\n    @tf.function\n    def train_step(img_tensor, target):\n        loss = 0\n\n        # initializing the hidden state for each batch\n        # because the captions are not related from image to image\n        hidden = decoder.reset_state(batch_size=target.shape[0])\n\n        dec_input = tf.expand_dims([tokenizer.word_index['<start>']] * target.shape[0], 1)\n\n        with tf.GradientTape() as tape:\n            features = encoder(img_tensor)\n\n            for i in range(1, target.shape[1]):\n                # passing the features through the decoder\n                predictions, hidden, _ = decoder(dec_input, features, hidden)\n\n                loss += loss_function(target[:, i], predictions)\n\n                # using teacher forcing\n                dec_input = tf.expand_dims(target[:, i], 1)\n\n        total_loss = (loss / int(target.shape[1]))\n\n        trainable_variables = encoder.trainable_variables + decoder.trainable_variables\n\n        gradients = tape.gradient(loss, trainable_variables)\n\n        optimizer.apply_gradients(zip(gradients, trainable_variables))\n\n        return loss, total_loss\n    \n    @tf.function\n    def distributed_train_step(inputs):\n\n        (images, labels) = inputs\n#         loss = strategy.experimental_run_v2(train_step, args=(images, labels))\n        loss = strategy.run(train_step, args=(images, labels))\n        \n        return loss\nwith strategy.scope():\n    valid_loss = tf.keras.metrics.Sum()\n    \n    @tf.function \n    def val_step(img_tensor, target, teacher_forcing=True):\n        # Non-teacher-forcing val_loss is too complicated at the moment\n        loss = 0\n#         print(target.shape) # (batch, 47) >> strange that we get None\n        batch = target.shape[0] # BATCH_SIZE//strategy.num_replicas_in_sync #\n        hidden = decoder.reset_state(batch_size= batch)\n#         print(hidden.shape) # (batch,512)\n        \n        dec_input = tf.expand_dims([tokenizer.word_index['<start>']] * batch, 1)\n      #   print(dec_input.shape) # (BATCH_SIZE, 1)\n        features = encoder(img_tensor)\n      #   print(features.shape) # (BATCH_SIZE, IMG_FEAT_LEN, ENCODER_HID) = 64 100 256\n        for i in range(1, target.shape[1]):\n            predictions, hidden, _ = decoder(dec_input, features, hidden)\n            loss += loss_function(target[:, i], predictions)\n\n            # using teacher forcing\n            dec_input = tf.expand_dims(target[:, i], 1)\n\n        avg_loss = (loss / int(target.shape[1]))\n        return loss, avg_loss\n    \n\n    @tf.function\n    def cal_val_loss(val_dataset):\n        # target.shape = (64,49) = (Per Replica BATCH_SIZE?, SEQ_LEN)\n        val_num_steps = len(img_name_val) // BATCH_SIZE\n        valid_data_iter = iter(val_dataset)\n        valid_loss.reset_states()\n        \n        total_loss = 0.0\n        for ii in tf.range(val_num_steps):\n            _, per_replica_val_loss = strategy.run(val_step, args=next(valid_data_iter))\n            t_loss = strategy.reduce(tf.distribute.ReduceOp.SUM, per_replica_val_loss, axis=None)\n            total_loss += t_loss\n#             print(total_loss)\n            \n        valid_loss.update_state(total_loss/val_num_steps)\n#         tf.print('val loss',valid_loss.result().numpy())\n#             tf.print(total_loss)\n#         tf.print ('Valid Loss -- %4f' % (total_loss.eval()/val_num_steps) )\n        return total_loss/val_num_steps\n    ","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:15:06.891648Z","iopub.execute_input":"2021-09-17T09:15:06.892201Z","iopub.status.idle":"2021-09-17T09:15:07.024190Z","shell.execute_reply.started":"2021-09-17T09:15:06.892167Z","shell.execute_reply":"2021-09-17T09:15:07.023493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if USE_PREVIOUS_SAVE: # \n#     print('Use prev. save weights, so make this cell error')\n#     %%time\n\nwith strategy.scope():\n    loss_plot = []\n    val_loss_plot = []\n    EPOCHS = 20 # 1st epoch takes 1hour, after that with cache power, it's just 3-4 mins /epoch\n    best_val_loss = 100\n    start_epoch = 0\n    num_steps = len(img_name_train) // (BATCH_SIZE)\n    start = time.time()\n    total_loss = 0\n    epoch = 0\n    train_dist_dataset = get_training_dataset()\n    \n    if USE_PREVIOUS_SAVE: # \n        print('Use prev. save weights, so run for few epochs')\n        EPOCHS,num_steps = 1,1\n        \n    num_steps_accum = num_steps\n    print(num_steps, BATCH_SIZE, num_steps*BATCH_SIZE)\n    \n    for (batch, inputs) in tqdm_notebook(enumerate(train_dist_dataset)): # by .repeat() this will indefinitely run\n            \n        if batch >= num_steps_accum:\n            epoch += 1\n            print('end of epoch ', epoch)\n            \n            loss_plot.append(total_loss / num_steps_accum)    \n            print ('Epoch {} Loss {:.6f}'.format(epoch,\n                                         total_loss/num_steps_accum))\n            print ('Time taken for 1 epoch {} sec\\n'.format(time.time() - start))\n            \n            if num_steps_accum > num_steps*EPOCHS:\n                print('end of training!!')\n                break\n\n            num_steps_accum += num_steps\n            print('next numsteps ', num_steps_accum)\n\n                \n        # unsupported operand type(s) for +=: 'int' and 'PerReplica'\n        _, per_replica_train_loss = distributed_train_step(inputs)\n        t_loss = strategy.reduce(tf.distribute.ReduceOp.SUM, per_replica_train_loss,\n                         axis=None)\n            \n        total_loss += t_loss\n            \n        if batch % 50 == 0:\n            print ('Epoch {} Batch {} Loss {:.4f}'.format(epoch + 1, batch, t_loss.numpy() ))\n\n            val_loss = cal_val_loss(valid_dist_dataset)\n            val_loss_plot.append(val_loss)\n            \n            print('val result', val_loss.numpy())\n            if val_loss.numpy() < best_val_loss:\n                print('update best val loss from %.4f to %.4f' % (best_val_loss, val_loss.numpy()))\n                best_val_loss = val_loss.numpy()\n                encoder.save_weights('encoder_best.h5')\n                decoder.save_weights('decoder_best.h5')\n#                 ckpt_manager.save()","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:15:20.724318Z","iopub.execute_input":"2021-09-17T09:15:20.724575Z","iopub.status.idle":"2021-09-17T09:19:19.408495Z","shell.execute_reply.started":"2021-09-17T09:15:20.724547Z","shell.execute_reply":"2021-09-17T09:19:19.407608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if USE_PREVIOUS_SAVE:\n    %%time\n\nprint(total_loss, t_loss)\n\nplt.plot(loss_plot)\nplt.xlabel('Epochs')\nplt.ylabel('Train Loss')\nplt.title('Loss Plot')\nplt.show()\n\n# plt.plot(loss_plot)\nplt.plot(val_loss_plot)\nplt.xlabel('Epochs')\nplt.ylabel('Val Loss')\nplt.title('Loss Plot')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:26:16.324197Z","iopub.execute_input":"2021-09-17T09:26:16.325006Z","iopub.status.idle":"2021-09-17T09:26:16.333287Z","shell.execute_reply.started":"2021-09-17T09:26:16.324967Z","shell.execute_reply":"2021-09-17T09:26:16.331980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if USE_PREVIOUS_SAVE:\n    '''\n    ## build construct input_layer, otherwise there is no input_layer and we cannot load weights\n    encoder.build(input_shape = (BATCH_SIZE,299,299,3))\n\n    #>> I don't know how to use model.build with multiple inputs\n    #>> So, I have to use functional API, and manually specify input tensor\n    # >> still error\n    decoder_layer = RNN_Decoder(embedding_matrix, units, vocab_size)\n    inp1 = tf.keras.layers.Input(shape=(1,))\n    inp2 = tf.keras.layers.Input(shape=(attention_features_shape,embedding_dim,))\n    inp3 = tf.keras.layers.Input(shape=(units,))\n    decoder_out = decoder_layer(inp1,inp2,inp3)\n    decoder = tf.keras.Model(inputs=[inp1,inp2,inp3],outputs=decoder_out)\n    '''\n    PATH = '/kaggle/input/image-caption-tf21-v12/'\n    with strategy.scope():\n        try:\n            encoder.load_weights(PATH+'encoder_best.h5')\n            decoder.load_weights(PATH+'decoder_best.h5') \n            # trick still fails due to layer mismatched when call(), have to construct with functional API exactly like subclass\n#             decoder.layers[-1].load_weights(PATH+'decoder_best.h5') # trick to load into layers,see decoder.summary()\n            print(1)\n        except:\n            encoder.load_weights(PATH+'encoder.h5')\n            decoder.load_weights(PATH+'decoder.h5')\n#             decoder.layers[-1].load_weights(PATH+'decoder.h5')\n            print(2)","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:26:18.992801Z","iopub.execute_input":"2021-09-17T09:26:18.993486Z","iopub.status.idle":"2021-09-17T09:26:23.331946Z","shell.execute_reply.started":"2021-09-17T09:26:18.993439Z","shell.execute_reply":"2021-09-17T09:26:23.331086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoder.save_weights('encoder.h5')\ndecoder.save_weights('decoder.h5')\n!ls -sh","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:26:23.333591Z","iopub.execute_input":"2021-09-17T09:26:23.334369Z","iopub.status.idle":"2021-09-17T09:26:25.015549Z","shell.execute_reply.started":"2021-09-17T09:26:23.334330Z","shell.execute_reply":"2021-09-17T09:26:25.014734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_image(image,figsize=None,title=None):\n    \n    if figsize is not None:\n        fig = plt.figure(figsize=figsize)\n        \n    if image.ndim == 2:\n        plt.imshow(image,cmap='gray')\n    else:\n        plt.imshow(image)\n        \n    if title is not None:\n        plt.title(title)\n        \ndef show_Nimages(imgs,scale=1):\n\n    N=len(imgs)\n    fig = plt.figure(figsize=(25/scale, 16/scale))\n    for i, img in enumerate(imgs):\n        ax = fig.add_subplot(1, N, i + 1, xticks=[], yticks=[])\n        show_image(img)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:26:25.017172Z","iopub.execute_input":"2021-09-17T09:26:25.017409Z","iopub.status.idle":"2021-09-17T09:26:25.028754Z","shell.execute_reply.started":"2021-09-17T09:26:25.017378Z","shell.execute_reply":"2021-09-17T09:26:25.025012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluate(image):\n    attention_plot = np.zeros((max_length, attention_features_shape))\n    \n    try:\n        hidden = decoder.reset_state(batch_size=1)\n    except:\n        hidden = decoder.layers[-1].reset_state(batch_size=1)\n        \n    img_tensor_val = tf.expand_dims(decode_image(image), 0)\n#     print(img_tensor_val.shape)\n    features = encoder(img_tensor_val)\n#     print(features.shape)\n    dec_input = tf.expand_dims([tokenizer.word_index['<start>']], 0)\n    result = []\n\n    for i in range(max_length):\n        predictions, hidden, attention_weights = decoder(dec_input, features, hidden)\n\n        attention_plot[i] = tf.reshape(attention_weights, (-1, )).numpy()\n\n        predicted_id = tf.random.categorical(predictions, 1)[0][0].numpy()\n        result.append(tokenizer.index_word[predicted_id])\n\n        if tokenizer.index_word[predicted_id] == '<end>':\n            return result, attention_plot\n\n        dec_input = tf.expand_dims([predicted_id], 0)\n\n    attention_plot = attention_plot[:len(result), :]\n    return result, attention_plot\ndef plot_attention(image, result, attention_plot):\n    \n    bits = tf.io.read_file(image)\n    image = tf.image.decode_jpeg(bits, channels=3)\n    \n    temp_image = np.array(image)\n\n    fig = plt.figure(figsize=(10, 10))\n\n    len_result = len(result)\n    for l in range(len_result):\n        temp_att = np.resize(attention_plot[l], (attention_viz_dim, attention_viz_dim))\n        ax = fig.add_subplot(len_result//2, len_result//2, l+1)\n        ax.set_title(result[l])\n        img = ax.imshow(temp_image)\n        ax.imshow(temp_att, cmap='gray', alpha=0.6, extent=img.get_extent())\n\n    plt.tight_layout()\n    plt.show()\n    \n    return temp_image\ndef print_all_captions(img_list, caps, rid):\n    orig = img_list[rid]\n    for rr in range(rid-5, rid+5):\n        image_name = img_list[rr]\n        if image_name == orig:\n            real_caption = ' '.join([tokenizer.index_word[i] for i in caps[rr] if i not in [0]])\n            print ('Real Caption:', real_caption)\n    return 0\n# captions on the train set\nimgs = []\nfor ii in range(6):\n    rid = np.random.randint(0, len(img_name_train))\n    print_all_captions(img_name_train,cap_train,rid)\n    image = img_name_train[rid]\n    result, attention_plot = evaluate(image)\n    print ('Prediction Caption:', ' '.join(result))\n    img = plot_attention(image, result, attention_plot)\n    imgs.append(img)\n    if (ii+1) %2 == 0:\n        show_Nimages(imgs)\n        imgs = []","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:26:25.031970Z","iopub.execute_input":"2021-09-17T09:26:25.032471Z","iopub.status.idle":"2021-09-17T09:26:26.109137Z","shell.execute_reply.started":"2021-09-17T09:26:25.032437Z","shell.execute_reply":"2021-09-17T09:26:26.107692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# captions on the validation set\nimgs = []\nfor ii in range(6):\n    rid = np.random.randint(0, len(img_name_val))\n    print_all_captions(img_name_val,cap_val,rid)\n    image = img_name_val[rid]\n    result, attention_plot = evaluate(image)\n    print ('Prediction Caption:', ' '.join(result))\n    img = plot_attention(image, result, attention_plot)\n    imgs.append(img)\n    if (ii+1) %2 == 0:\n        show_Nimages(imgs)\n        imgs = []","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:29:42.775848Z","iopub.execute_input":"2021-09-17T09:29:42.776145Z","iopub.status.idle":"2021-09-17T09:29:54.634981Z","shell.execute_reply.started":"2021-09-17T09:29:42.776114Z","shell.execute_reply":"2021-09-17T09:29:54.634171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import gc\n# del dataset\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:30:01.232773Z","iopub.execute_input":"2021-09-17T09:30:01.233043Z","iopub.status.idle":"2021-09-17T09:30:02.215916Z","shell.execute_reply.started":"2021-09-17T09:30:01.233013Z","shell.execute_reply":"2021-09-17T09:30:02.215148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def gen_cap(image):\n    attention_plot = np.zeros((max_length, attention_features_shape))\n    hidden = decoder.reset_state(batch_size=1)\n    img_tensor_val = tf.expand_dims(decode_image(image), 0)\n    features = encoder(img_tensor_val)\n    dec_input = tf.expand_dims([tokenizer.word_index['<start>']], 0)\n    result = []\n\n    for i in range(max_length):\n        predictions, hidden, attention_weights = decoder(dec_input, features, hidden)\n        predicted_id = tf.random.categorical(predictions, 1)[0][0].numpy()\n        word = tokenizer.index_word.get(predicted_id, tokenizer.word_index['<end>'])\n        result.append(word)\n        if word == '<end>':\n            return result\n\n        dec_input = tf.expand_dims([predicted_id], 0)\n\n    return result","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:30:05.253752Z","iopub.execute_input":"2021-09-17T09:30:05.254382Z","iopub.status.idle":"2021-09-17T09:30:05.261960Z","shell.execute_reply.started":"2021-09-17T09:30:05.254342Z","shell.execute_reply":"2021-09-17T09:30:05.260775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(img_name_train),len(img_name_val)\nSTART = 120000\nEND = 150000","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:30:08.199509Z","iopub.execute_input":"2021-09-17T09:30:08.199786Z","iopub.status.idle":"2021-09-17T09:30:08.205684Z","shell.execute_reply.started":"2021-09-17T09:30:08.199757Z","shell.execute_reply":"2021-09-17T09:30:08.204981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# captions on the validation set\nimgs = []\nreal_caps, pred_caps = [], []\nfor rid in tqdm_notebook(range(START, END)): # 100 captions / 1:05 >> 10000 caps / 110mins >> 30,000 / 330mins+30min(preparing) = 6hours\n    image = img_name_train[rid]\n    result = gen_cap(image)\n    \n    real_caps.append(' '.join([tokenizer.index_word[i] for i in cap_train[rid] if i not in [0]]))\n    pred_caps.append(' '.join(result))","metadata":{"execution":{"iopub.status.busy":"2021-09-17T09:30:09.421363Z","iopub.execute_input":"2021-09-17T09:30:09.421646Z","iopub.status.idle":"2021-09-17T11:22:16.864331Z","shell.execute_reply.started":"2021-09-17T09:30:09.421619Z","shell.execute_reply":"2021-09-17T11:22:16.863650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# real_caps, pred_caps\nnp.savetxt('real_caps.txt', real_caps, fmt='%s')\nnp.savetxt('pred_caps.txt', pred_caps, fmt='%s')","metadata":{"execution":{"iopub.status.busy":"2021-09-17T11:22:16.865964Z","iopub.execute_input":"2021-09-17T11:22:16.866660Z","iopub.status.idle":"2021-09-17T11:22:17.118143Z","shell.execute_reply.started":"2021-09-17T11:22:16.866621Z","shell.execute_reply":"2021-09-17T11:22:17.117350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cat real_caps.txt | head\n!cat pred_caps.txt | head","metadata":{"execution":{"iopub.status.busy":"2021-09-17T11:22:17.119311Z","iopub.execute_input":"2021-09-17T11:22:17.119919Z","iopub.status.idle":"2021-09-17T11:22:18.645757Z","shell.execute_reply.started":"2021-09-17T11:22:17.119879Z","shell.execute_reply":"2021-09-17T11:22:18.644941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_image(image,figsize=None,title=None):\n    \n    if figsize is not None:\n        fig = plt.figure(figsize=figsize)\n        \n    if image.ndim == 2:\n        plt.imshow(image,cmap='gray')\n    else:\n        plt.imshow(image)\n        \n    if title is not None:\n        plt.title(title)","metadata":{"execution":{"iopub.status.busy":"2021-09-17T11:23:02.265940Z","iopub.execute_input":"2021-09-17T11:23:02.266537Z","iopub.status.idle":"2021-09-17T11:23:02.274763Z","shell.execute_reply.started":"2021-09-17T11:23:02.266496Z","shell.execute_reply":"2021-09-17T11:23:02.273962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import PIL # We will import the packages at \"use-time (just for this kernel)\n\nPIL.Image.open(\"../input/sample-img/Parisgesch1.JPG\")","metadata":{"execution":{"iopub.status.busy":"2021-09-17T11:24:48.139544Z","iopub.execute_input":"2021-09-17T11:24:48.140330Z","iopub.status.idle":"2021-09-17T11:24:48.532805Z","shell.execute_reply.started":"2021-09-17T11:24:48.140283Z","shell.execute_reply":"2021-09-17T11:24:48.531905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gen_cap(\"../input/sample-img/Parisgesch1.JPG\")","metadata":{"execution":{"iopub.status.busy":"2021-09-17T11:23:30.862910Z","iopub.execute_input":"2021-09-17T11:23:30.865570Z","iopub.status.idle":"2021-09-17T11:23:31.140836Z","shell.execute_reply.started":"2021-09-17T11:23:30.865522Z","shell.execute_reply":"2021-09-17T11:23:31.140157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It's not perfect i know :P","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}