{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Product Catptioning with H&M data\nIn this notebook, I use the model trained in https://www.kaggle.com/code/mohammedobeidat/h-m-product-captioning to generate captions for some products which then I will use in a demo I worked on before that shows the results of a recommender system built in 4 different approaches.<br>\nDemo and documentation can be found here: https://share.streamlit.io/mnobeidat13/handm-recommender-system/main","metadata":{}},{"cell_type":"markdown","source":"The trained model is saved in https://www.kaggle.com/datasets/mohammedobeidat/hm-product-captioning-keras-model<br>\nThis model is a subclassed model and it needs to be built before loading model weights.","metadata":{}},{"cell_type":"code","source":"import os\nimport re\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.applications import efficientnet\nfrom tensorflow.keras.layers import TextVectorization\nfrom sklearn.model_selection import train_test_split\nimport random\nimport warnings\n\nwarnings.filterwarnings('ignore')\n\ndef sample_from_dict(d, sample=5):\n    keys = random.sample(list(d), sample)\n    values = [d[k] for k in keys]\n    return dict(zip(keys, values))\n\n\nseed = 111\ntf.random.set_seed(seed)","metadata":{"id":"x4PfHLiTosIn","execution":{"iopub.status.busy":"2022-07-15T15:18:54.799400Z","iopub.execute_input":"2022-07-15T15:18:54.799848Z","iopub.status.idle":"2022-07-15T15:18:54.809737Z","shell.execute_reply.started":"2022-07-15T15:18:54.799813Z","shell.execute_reply":"2022-07-15T15:18:54.808948Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Desired image dimensions\nIMAGE_SIZE = (299, 299)\n\n# Vocabulary size\nVOCAB_SIZE = 100000\n\n# Fixed length allowed for any sequence\nSEQ_LENGTH = 10\n\n# Dimension for the image embeddings and token embeddings\nEMBED_DIM = 512\n\n# Per-layer units in the feed-forward network\nFF_DIM = 512\n\n# Other training parameters\nBATCH_SIZE = 64\nEPOCHS = 2\nAUTOTUNE = tf.data.AUTOTUNE","metadata":{"id":"-dFJBmMNosIp","execution":{"iopub.status.busy":"2022-07-15T15:18:54.839379Z","iopub.execute_input":"2022-07-15T15:18:54.840075Z","iopub.status.idle":"2022-07-15T15:18:54.846792Z","shell.execute_reply.started":"2022-07-15T15:18:54.840042Z","shell.execute_reply":"2022-07-15T15:18:54.846009Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.read_csv('../input/hm-image-path-with-desc/article_with_paths_and_desc.csv', dtype=str)\ndata.rename({'article_id':'id', 'detail_desc':'desc', 'path':'path'}, inplace=True, axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T15:23:09.985511Z","iopub.execute_input":"2022-07-15T15:23:09.985927Z","iopub.status.idle":"2022-07-15T15:23:10.439532Z","shell.execute_reply.started":"2022-07-15T15:23:09.985896Z","shell.execute_reply":"2022-07-15T15:23:10.438334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T15:23:10.832041Z","iopub.execute_input":"2022-07-15T15:23:10.832464Z","iopub.status.idle":"2022-07-15T15:23:10.845565Z","shell.execute_reply.started":"2022-07-15T15:23:10.832431Z","shell.execute_reply":"2022-07-15T15:23:10.844601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def map_list(item):\n    item = \"<start> \" + item.strip() + \" <end>\"\n    return [item]\n\ndata_dict = pd.Series(data.desc.map(map_list).values,index=data.path).to_dict()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T15:23:12.913028Z","iopub.execute_input":"2022-07-15T15:23:12.913508Z","iopub.status.idle":"2022-07-15T15:23:13.175826Z","shell.execute_reply.started":"2022-07-15T15:23:12.913471Z","shell.execute_reply":"2022-07-15T15:23:13.174338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preparing the dataset","metadata":{"id":"NN1OdJMxosIq"}},{"cell_type":"code","source":"def train_val_split(caption_data, train_size=0.8, shuffle=True):\n\n    # 1. Get the list of all image names\n    all_images = list(caption_data.keys())\n\n    # 2. Shuffle if necessary\n    if shuffle:\n        np.random.shuffle(all_images)\n\n    # 3. Split into training and validation sets\n    train_size = int(len(caption_data) * train_size)\n\n    training_data = {\n        img_name: caption_data[img_name] for img_name in all_images[:train_size]\n    }\n    validation_data = {\n        img_name: caption_data[img_name] for img_name in all_images[train_size:]\n    }\n\n    # 4. Return the splits\n    return training_data, validation_data\n\n\ncaptions_mapping, text_data = data_dict, data.desc.values\n\n# Split the dataset into training and validation sets\ntrain_data, valid_data = train_val_split(captions_mapping)\n\nprint(\"Number of training samples: \", len(train_data))\nprint(\"Number of validation samples: \", len(valid_data))","metadata":{"id":"48SX4BdKosIq","execution":{"iopub.status.busy":"2022-07-15T15:23:14.282683Z","iopub.execute_input":"2022-07-15T15:23:14.283559Z","iopub.status.idle":"2022-07-15T15:23:14.395055Z","shell.execute_reply.started":"2022-07-15T15:23:14.283511Z","shell.execute_reply":"2022-07-15T15:23:14.393873Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Vectorizing the text data","metadata":{"id":"xlA7B3lMosIr"}},{"cell_type":"code","source":"\ndef custom_standardization(input_string):\n    lowercase = tf.strings.lower(input_string)\n    return tf.strings.regex_replace(lowercase, \"[%s]\" % re.escape(strip_chars), \"\")\n\n\nstrip_chars = \"!\\\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~\"\nstrip_chars = strip_chars.replace(\"<\", \"\")\nstrip_chars = strip_chars.replace(\">\", \"\")\n\nvectorization = TextVectorization(\n    max_tokens=VOCAB_SIZE,\n    output_mode=\"int\",\n    output_sequence_length=SEQ_LENGTH,\n    standardize=custom_standardization,\n)\nvectorization.adapt(text_data)\n\n# Data augmentation for image data\nimage_augmentation = keras.Sequential(\n    [\n        layers.RandomFlip(\"horizontal\"),\n        layers.RandomRotation(0.2),\n        layers.RandomContrast(0.3),\n    ]\n)\n","metadata":{"id":"m31Ik5eEosIs","execution":{"iopub.status.busy":"2022-07-15T15:23:15.493960Z","iopub.execute_input":"2022-07-15T15:23:15.494447Z","iopub.status.idle":"2022-07-15T15:23:19.399611Z","shell.execute_reply.started":"2022-07-15T15:23:15.494401Z","shell.execute_reply":"2022-07-15T15:23:19.398442Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Building a `tf.data.Dataset` pipeline for training","metadata":{"id":"54yq7rniosIt"}},{"cell_type":"code","source":"\ndef decode_and_resize(img_path):\n    img = tf.io.read_file(img_path)\n    img = tf.image.decode_jpeg(img, channels=3)\n    img = tf.image.resize(img, IMAGE_SIZE)\n    img = tf.image.convert_image_dtype(img, tf.float32)\n    return img\n\n\ndef process_input(img_path, captions):\n    return decode_and_resize(img_path), vectorization(captions)\n\n\ndef make_dataset(images, captions):\n    dataset = tf.data.Dataset.from_tensor_slices((images, captions))\n    dataset = dataset.shuffle(len(images))\n    dataset = dataset.map(process_input, num_parallel_calls=AUTOTUNE)\n    dataset = dataset.batch(BATCH_SIZE).prefetch(AUTOTUNE)\n\n    return dataset\n\n\n# Pass the list of images and the list of corresponding captions\ntrain_dataset = make_dataset(list(train_data.keys()), list(train_data.values()))\n\nvalid_dataset = make_dataset(list(valid_data.keys()), list(valid_data.values()))","metadata":{"id":"f8yE8MnsosIt","execution":{"iopub.status.busy":"2022-07-15T15:23:19.401300Z","iopub.execute_input":"2022-07-15T15:23:19.403222Z","iopub.status.idle":"2022-07-15T15:23:21.473510Z","shell.execute_reply.started":"2022-07-15T15:23:19.403185Z","shell.execute_reply":"2022-07-15T15:23:21.472442Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ex = next(train_dataset.take(1).as_numpy_iterator())\n\nex[0].shape, ex[1].shape","metadata":{"execution":{"iopub.status.busy":"2022-07-15T15:25:10.974590Z","iopub.execute_input":"2022-07-15T15:25:10.975086Z","iopub.status.idle":"2022-07-15T15:25:11.683588Z","shell.execute_reply.started":"2022-07-15T15:25:10.975053Z","shell.execute_reply":"2022-07-15T15:25:11.682402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Building the model","metadata":{"id":"x6LZtHfJosIt"}},{"cell_type":"code","source":"def get_cnn_model():\n    base_model = efficientnet.EfficientNetB0(\n        input_shape=(*IMAGE_SIZE, 3), include_top=False, weights=\"imagenet\",\n    )\n    # We freeze our feature extractor\n    base_model.trainable = False\n    base_model_out = base_model.output\n    base_model_out = layers.Reshape((-1, base_model_out.shape[-1]))(base_model_out)\n    cnn_model = keras.models.Model(base_model.input, base_model_out)\n    return cnn_model\n\n\nclass TransformerEncoderBlock(layers.Layer):\n    def __init__(self, embed_dim, dense_dim, num_heads, **kwargs):\n        super().__init__(**kwargs)\n        self.embed_dim = embed_dim\n        self.dense_dim = dense_dim\n        self.num_heads = num_heads\n        self.attention_1 = layers.MultiHeadAttention(\n            num_heads=num_heads, key_dim=embed_dim, dropout=0.0\n        )\n        self.layernorm_1 = layers.LayerNormalization()\n        self.layernorm_2 = layers.LayerNormalization()\n        self.dense_1 = layers.Dense(embed_dim, activation=\"relu\")\n\n    def call(self, inputs, training, mask=None):\n        inputs = self.layernorm_1(inputs)\n        inputs = self.dense_1(inputs)\n\n        attention_output_1 = self.attention_1(\n            query=inputs,\n            value=inputs,\n            key=inputs,\n            attention_mask=None,\n            training=training,\n        )\n        out_1 = self.layernorm_2(inputs + attention_output_1)\n        return out_1\n\n\nclass PositionalEmbedding(layers.Layer):\n    def __init__(self, sequence_length, vocab_size, embed_dim, **kwargs):\n        super().__init__(**kwargs)\n        self.token_embeddings = layers.Embedding(\n            input_dim=vocab_size, output_dim=embed_dim\n        )\n        self.position_embeddings = layers.Embedding(\n            input_dim=sequence_length, output_dim=embed_dim\n        )\n        self.sequence_length = sequence_length\n        self.vocab_size = vocab_size\n        self.embed_dim = embed_dim\n        self.embed_scale = tf.math.sqrt(tf.cast(embed_dim, tf.float32))\n\n    def call(self, inputs):\n        length = tf.shape(inputs)[-1]\n        positions = tf.range(start=0, limit=length, delta=1)\n        embedded_tokens = self.token_embeddings(inputs)\n        embedded_tokens = embedded_tokens * self.embed_scale\n        embedded_positions = self.position_embeddings(positions)\n        return embedded_tokens + embedded_positions\n\n    def compute_mask(self, inputs, mask=None):\n        return tf.math.not_equal(inputs, 0)\n\n\nclass TransformerDecoderBlock(layers.Layer):\n    def __init__(self, embed_dim, ff_dim, num_heads, **kwargs):\n        super().__init__(**kwargs)\n        self.embed_dim = embed_dim\n        self.ff_dim = ff_dim\n        self.num_heads = num_heads\n        self.attention_1 = layers.MultiHeadAttention(\n            num_heads=num_heads, key_dim=embed_dim, dropout=0.1\n        )\n        self.attention_2 = layers.MultiHeadAttention(\n            num_heads=num_heads, key_dim=embed_dim, dropout=0.1\n        )\n        self.ffn_layer_1 = layers.Dense(ff_dim, activation=\"relu\")\n        self.ffn_layer_2 = layers.Dense(embed_dim)\n\n        self.layernorm_1 = layers.LayerNormalization()\n        self.layernorm_2 = layers.LayerNormalization()\n        self.layernorm_3 = layers.LayerNormalization()\n\n        self.embedding = PositionalEmbedding(\n            embed_dim=EMBED_DIM, sequence_length=SEQ_LENGTH, vocab_size=VOCAB_SIZE\n        )\n        self.out = layers.Dense(VOCAB_SIZE, activation=\"softmax\")\n\n        self.dropout_1 = layers.Dropout(0.3)\n        self.dropout_2 = layers.Dropout(0.5)\n        self.supports_masking = True\n\n    def call(self, inputs, encoder_outputs, training, mask=None):\n        inputs = self.embedding(inputs)\n        causal_mask = self.get_causal_attention_mask(inputs)\n\n        if mask is not None:\n            padding_mask = tf.cast(mask[:, :, tf.newaxis], dtype=tf.int32)\n            combined_mask = tf.cast(mask[:, tf.newaxis, :], dtype=tf.int32)\n            combined_mask = tf.minimum(combined_mask, causal_mask)\n\n        attention_output_1 = self.attention_1(\n            query=inputs,\n            value=inputs,\n            key=inputs,\n            attention_mask=combined_mask,\n            training=training,\n        )\n        out_1 = self.layernorm_1(inputs + attention_output_1)\n\n        attention_output_2 = self.attention_2(\n            query=out_1,\n            value=encoder_outputs,\n            key=encoder_outputs,\n            attention_mask=padding_mask,\n            training=training,\n        )\n        out_2 = self.layernorm_2(out_1 + attention_output_2)\n\n        ffn_out = self.ffn_layer_1(out_2)\n        ffn_out = self.dropout_1(ffn_out, training=training)\n        ffn_out = self.ffn_layer_2(ffn_out)\n\n        ffn_out = self.layernorm_3(ffn_out + out_2, training=training)\n        ffn_out = self.dropout_2(ffn_out, training=training)\n        preds = self.out(ffn_out)\n        return preds\n\n    def get_causal_attention_mask(self, inputs):\n        input_shape = tf.shape(inputs)\n        batch_size, sequence_length = input_shape[0], input_shape[1]\n        i = tf.range(sequence_length)[:, tf.newaxis]\n        j = tf.range(sequence_length)\n        mask = tf.cast(i >= j, dtype=\"int32\")\n        mask = tf.reshape(mask, (1, input_shape[1], input_shape[1]))\n        mult = tf.concat(\n            [tf.expand_dims(batch_size, -1), tf.constant([1, 1], dtype=tf.int32)],\n            axis=0,\n        )\n        return tf.tile(mask, mult)\n\n\nclass ImageCaptioningModel(keras.Model):\n    def __init__(\n        self, cnn_model, encoder, decoder, num_captions_per_image=1, image_aug=None,\n    ):\n        super().__init__()\n        self.cnn_model = cnn_model\n        self.encoder = encoder\n        self.decoder = decoder\n        self.loss_tracker = keras.metrics.Mean(name=\"loss\")\n        self.acc_tracker = keras.metrics.Mean(name=\"accuracy\")\n        self.num_captions_per_image = num_captions_per_image\n        self.image_aug = image_aug\n\n    def calculate_loss(self, y_true, y_pred, mask):\n        loss = self.loss(y_true, y_pred)\n        mask = tf.cast(mask, dtype=loss.dtype)\n        loss *= mask\n        return tf.reduce_sum(loss) / tf.reduce_sum(mask)\n\n    def calculate_accuracy(self, y_true, y_pred, mask):\n        accuracy = tf.equal(y_true, tf.argmax(y_pred, axis=2))\n        accuracy = tf.math.logical_and(mask, accuracy)\n        accuracy = tf.cast(accuracy, dtype=tf.float32)\n        mask = tf.cast(mask, dtype=tf.float32)\n        return tf.reduce_sum(accuracy) / tf.reduce_sum(mask)\n\n    def _compute_caption_loss_and_acc(self, img_embed, batch_seq, training=True):\n        encoder_out = self.encoder(img_embed, training=training)\n        batch_seq_inp = batch_seq[:, :-1]\n        batch_seq_true = batch_seq[:, 1:]\n        mask = tf.math.not_equal(batch_seq_true, 0)\n        batch_seq_pred = self.decoder(\n            batch_seq_inp, encoder_out, training=training, mask=mask\n        )\n        loss = self.calculate_loss(batch_seq_true, batch_seq_pred, mask)\n        acc = self.calculate_accuracy(batch_seq_true, batch_seq_pred, mask)\n        return loss, acc\n\n    def train_step(self, batch_data):\n        batch_img, batch_seq = batch_data\n        batch_loss = 0\n        batch_acc = 0\n\n        if self.image_aug:\n            batch_img = self.image_aug(batch_img)\n\n        # 1. Get image embeddings\n        img_embed = self.cnn_model(batch_img)\n\n        # 2. Pass each of the five captions one by one to the decoder\n        # along with the encoder outputs and compute the loss as well as accuracy\n        # for each caption.\n        for i in range(self.num_captions_per_image):\n            with tf.GradientTape() as tape:\n                loss, acc = self._compute_caption_loss_and_acc(\n                    img_embed, batch_seq[:, i, :], training=True\n                )\n\n                # 3. Update loss and accuracy\n                batch_loss += loss\n                batch_acc += acc\n\n            # 4. Get the list of all the trainable weights\n            train_vars = (\n                self.encoder.trainable_variables + self.decoder.trainable_variables\n            )\n\n            # 5. Get the gradients\n            grads = tape.gradient(loss, train_vars)\n\n            # 6. Update the trainable weights\n            self.optimizer.apply_gradients(zip(grads, train_vars))\n\n        # 7. Update the trackers\n        batch_acc /= float(self.num_captions_per_image)\n        self.loss_tracker.update_state(batch_loss)\n        self.acc_tracker.update_state(batch_acc)\n\n        # 8. Return the loss and accuracy values\n        return {\"loss\": self.loss_tracker.result(), \"acc\": self.acc_tracker.result()}\n\n    def test_step(self, batch_data):\n        batch_img, batch_seq = batch_data\n        batch_loss = 0\n        batch_acc = 0\n\n        # 1. Get image embeddings\n        img_embed = self.cnn_model(batch_img)\n\n        # 2. Pass each of the five captions one by one to the decoder\n        # along with the encoder outputs and compute the loss as well as accuracy\n        # for each caption.\n        for i in range(self.num_captions_per_image):\n            loss, acc = self._compute_caption_loss_and_acc(\n                img_embed, batch_seq[:, i, :], training=False\n            )\n\n            # 3. Update batch loss and batch accuracy\n            batch_loss += loss\n            batch_acc += acc\n\n        batch_acc /= float(self.num_captions_per_image)\n\n        # 4. Update the trackers\n        self.loss_tracker.update_state(batch_loss)\n        self.acc_tracker.update_state(batch_acc)\n\n        # 5. Return the loss and accuracy values\n        return {\"loss\": self.loss_tracker.result(), \"acc\": self.acc_tracker.result()}\n\n    @property\n    def metrics(self):\n        # We need to list our metrics here so the `reset_states()` can be\n        # called automatically.\n        return [self.loss_tracker, self.acc_tracker]","metadata":{"id":"VvMHO8euosIu","execution":{"iopub.status.busy":"2022-07-15T15:23:22.503474Z","iopub.execute_input":"2022-07-15T15:23:22.504301Z","iopub.status.idle":"2022-07-15T15:23:22.557190Z","shell.execute_reply.started":"2022-07-15T15:23:22.504225Z","shell.execute_reply":"2022-07-15T15:23:22.556155Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnn_model = get_cnn_model()\nencoder = TransformerEncoderBlock(embed_dim=EMBED_DIM, dense_dim=FF_DIM, num_heads=1)\ndecoder = TransformerDecoderBlock(embed_dim=EMBED_DIM, ff_dim=FF_DIM, num_heads=2)\n\nnew_model = ImageCaptioningModel(\n    cnn_model=cnn_model, encoder=encoder, decoder=decoder, image_aug=image_augmentation,\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T15:23:24.279880Z","iopub.execute_input":"2022-07-15T15:23:24.280600Z","iopub.status.idle":"2022-07-15T15:23:26.105319Z","shell.execute_reply.started":"2022-07-15T15:23:24.280566Z","shell.execute_reply":"2022-07-15T15:23:26.104084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model training","metadata":{"id":"Ble5uiLCosIv"}},{"cell_type":"code","source":"# Define the loss function\ncross_entropy = keras.losses.SparseCategoricalCrossentropy(\n    from_logits=False, reduction=\"none\"\n)\n\n# EarlyStopping criteria\nearly_stopping = keras.callbacks.EarlyStopping(patience=3, restore_best_weights=True)\n\n\n# Learning Rate Scheduler for the optimizer\nclass LRSchedule(keras.optimizers.schedules.LearningRateSchedule):\n    def __init__(self, post_warmup_learning_rate, warmup_steps):\n        super().__init__()\n        self.post_warmup_learning_rate = post_warmup_learning_rate\n        self.warmup_steps = warmup_steps\n\n    def __call__(self, step):\n        global_step = tf.cast(step, tf.float32)\n        warmup_steps = tf.cast(self.warmup_steps, tf.float32)\n        warmup_progress = global_step / warmup_steps\n        warmup_learning_rate = self.post_warmup_learning_rate * warmup_progress\n        return tf.cond(\n            global_step < warmup_steps,\n            lambda: warmup_learning_rate,\n            lambda: self.post_warmup_learning_rate,\n        )\n\n\n# Create a learning rate schedule\nnum_train_steps = len(train_dataset) * EPOCHS\nnum_warmup_steps = num_train_steps // 15\nlr_schedule = LRSchedule(post_warmup_learning_rate=1e-4, warmup_steps=num_warmup_steps)\n\nnew_model.compile(optimizer=keras.optimizers.Adam(lr_schedule), loss=cross_entropy)\nnew_model.built = True\n\n# Fit the model\nhistory = new_model.train_on_batch(\n    ex[0], ex[1])","metadata":{"id":"ahdvIzfaosIv","execution":{"iopub.status.busy":"2022-07-15T15:25:59.523836Z","iopub.execute_input":"2022-07-15T15:25:59.524242Z","iopub.status.idle":"2022-07-15T15:26:12.154670Z","shell.execute_reply.started":"2022-07-15T15:25:59.524212Z","shell.execute_reply":"2022-07-15T15:26:12.153605Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_model.load_weights('../input/hm-product-captioning-keras-model/model_weights.h5')","metadata":{"execution":{"iopub.status.busy":"2022-07-15T15:26:15.953253Z","iopub.execute_input":"2022-07-15T15:26:15.954120Z","iopub.status.idle":"2022-07-15T15:26:21.361288Z","shell.execute_reply.started":"2022-07-15T15:26:15.954082Z","shell.execute_reply":"2022-07-15T15:26:21.359766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Check sample predictions","metadata":{"id":"CUOnH3R8osIv"}},{"cell_type":"code","source":"f = open('../input/handm-images-demo-names/images.txt', 'r')\nnames = [line[:-6] for line in f]","metadata":{"execution":{"iopub.status.busy":"2022-07-15T15:38:41.895694Z","iopub.execute_input":"2022-07-15T15:38:41.896197Z","iopub.status.idle":"2022-07-15T15:38:41.907840Z","shell.execute_reply.started":"2022-07-15T15:38:41.896148Z","shell.execute_reply":"2022-07-15T15:38:41.905964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = data[data.id.isin(names)][:100]","metadata":{"execution":{"iopub.status.busy":"2022-07-15T15:40:07.175794Z","iopub.execute_input":"2022-07-15T15:40:07.176307Z","iopub.status.idle":"2022-07-15T15:40:07.203561Z","shell.execute_reply.started":"2022-07-15T15:40:07.176246Z","shell.execute_reply":"2022-07-15T15:40:07.202544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vocab = vectorization.get_vocabulary()\nindex_lookup = dict(zip(range(len(vocab)), vocab))\nmax_decoded_sentence_length = SEQ_LENGTH - 1\nvalid_images = list(valid_data.keys())\n\ndef generate_caption(model, sample_img):\n\n    sample_img = decode_and_resize(sample_img)\n    img = sample_img.numpy().clip(0, 255).astype(np.uint8)\n    img = tf.expand_dims(sample_img, 0)\n    img = model.cnn_model(img)\n    encoded_img = model.encoder(img, training=False)\n    decoded_caption = \"<start> \"\n    for i in range(max_decoded_sentence_length):\n        tokenized_caption = vectorization([decoded_caption])[:, :-1]\n        mask = tf.math.not_equal(tokenized_caption, 0)\n        predictions = model.decoder(\n            tokenized_caption, encoded_img, training=False, mask=mask\n        )\n        sampled_token_index = np.argmax(predictions[0, i, :])\n        sampled_token = index_lookup[sampled_token_index]\n        if sampled_token == \" <end>\":\n            break\n        decoded_caption += \" \" + sampled_token\n\n    decoded_caption = decoded_caption.replace(\"<start> \", \"\")\n    decoded_caption = decoded_caption.replace(\" <end>\", \"\").strip()\n    \n    return decoded_caption\n","metadata":{"execution":{"iopub.status.busy":"2022-07-15T15:52:32.752731Z","iopub.execute_input":"2022-07-15T15:52:32.753276Z","iopub.status.idle":"2022-07-15T15:52:32.786080Z","shell.execute_reply.started":"2022-07-15T15:52:32.753228Z","shell.execute_reply":"2022-07-15T15:52:32.784632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\nimport tensorflow_hub as hub","metadata":{"execution":{"iopub.status.busy":"2022-07-15T15:56:33.682670Z","iopub.execute_input":"2022-07-15T15:56:33.683088Z","iopub.status.idle":"2022-07-15T15:56:34.133486Z","shell.execute_reply.started":"2022-07-15T15:56:33.683057Z","shell.execute_reply":"2022-07-15T15:56:34.132559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"captions = []\nfor row in tqdm(test.iterrows()):\n    captions.append(generate_caption(new_model, row[1]['path']))","metadata":{"execution":{"iopub.status.busy":"2022-07-15T15:55:30.242812Z","iopub.execute_input":"2022-07-15T15:55:30.243308Z","iopub.status.idle":"2022-07-15T15:55:38.457631Z","shell.execute_reply.started":"2022-07-15T15:55:30.243263Z","shell.execute_reply":"2022-07-15T15:55:38.456554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"module_url = \"https://tfhub.dev/google/universal-sentence-encoder/4\"\nmodel = hub.load(module_url)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T15:56:41.841624Z","iopub.execute_input":"2022-07-15T15:56:41.842006Z","iopub.status.idle":"2022-07-15T15:56:59.598511Z","shell.execute_reply.started":"2022-07-15T15:56:41.841976Z","shell.execute_reply":"2022-07-15T15:56:59.593691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['caption'] = captions\ncap_embeds = model(captions).numpy()\ndesc_embeds = model(test.desc.values).numpy()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T16:03:26.182129Z","iopub.execute_input":"2022-07-15T16:03:26.182562Z","iopub.status.idle":"2022-07-15T16:03:26.209777Z","shell.execute_reply.started":"2022-07-15T16:03:26.182528Z","shell.execute_reply":"2022-07-15T16:03:26.208723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"similarity = []\n\nfor i, j in zip(cap_embeds, desc_embeds):\n    similarity.append(np.dot(i, j))","metadata":{"execution":{"iopub.status.busy":"2022-07-15T16:06:10.814243Z","iopub.execute_input":"2022-07-15T16:06:10.814800Z","iopub.status.idle":"2022-07-15T16:06:10.821477Z","shell.execute_reply.started":"2022-07-15T16:06:10.814763Z","shell.execute_reply":"2022-07-15T16:06:10.820579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['similarity'] = similarity","metadata":{"execution":{"iopub.status.busy":"2022-07-15T16:08:29.580495Z","iopub.execute_input":"2022-07-15T16:08:29.581034Z","iopub.status.idle":"2022-07-15T16:08:29.589742Z","shell.execute_reply.started":"2022-07-15T16:08:29.580998Z","shell.execute_reply":"2022-07-15T16:08:29.588453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test","metadata":{"execution":{"iopub.status.busy":"2022-07-15T16:08:32.350486Z","iopub.execute_input":"2022-07-15T16:08:32.350938Z","iopub.status.idle":"2022-07-15T16:08:32.368301Z","shell.execute_reply.started":"2022-07-15T16:08:32.350905Z","shell.execute_reply":"2022-07-15T16:08:32.367429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.to_csv('caption_desc_embeds.csv')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'Average similarity score: {test.mean().similarity}')","metadata":{"execution":{"iopub.status.busy":"2022-07-15T16:20:42.140712Z","iopub.execute_input":"2022-07-15T16:20:42.141120Z","iopub.status.idle":"2022-07-15T16:20:42.153504Z","shell.execute_reply.started":"2022-07-15T16:20:42.141069Z","shell.execute_reply":"2022-07-15T16:20:42.152341Z"},"trusted":true},"execution_count":null,"outputs":[]}]}