{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":29705,"databundleVersionId":2662435,"sourceType":"competition"}],"dockerImageVersionId":30787,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-11-05T02:34:43.037421Z","iopub.execute_input":"2024-11-05T02:34:43.038221Z","iopub.status.idle":"2024-11-05T02:34:43.04849Z","shell.execute_reply.started":"2024-11-05T02:34:43.03818Z","shell.execute_reply":"2024-11-05T02:34:43.047521Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n# 1. Introduction\nIn this notebook, we'll build an image captioning model that generates textual descriptions for images. The model consists of:\n\n* An Encoder: A pre-trained CNN (e.g., ResNet50) that extracts image features.\n* A Decoder: An LSTM network that generates captions based on the encoded image features.\n  \nDue to resource constraints in the Kaggle environment, we'll use a small subset of the data for training and demonstration purposes.","metadata":{}},{"cell_type":"markdown","source":"# 2. Setup and Configuration\nFirst, let's install and import the necessary libraries.","metadata":{}},{"cell_type":"code","source":"# Install necessary libraries\n!pip install --quiet torchvision transformers\n# Install langdetect if not already installed\n!pip install langdetect\n# !pip install torchtext\n\n\n# Import libraries\nimport os\nimport re\nimport time\nimport random\nimport requests\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nfrom io import BytesIO\nfrom collections import Counter\nfrom langdetect import detect, LangDetectException\n# from torchtext.vocab import GloVe\n\nimport torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nimport torchvision.models as models\nfrom torch.nn.utils.rnn import pack_padded_sequence\nfrom torch.utils.data import Dataset, DataLoader\n\n# For text processing\nfrom nltk.tokenize import word_tokenize\nfrom nltk.translate.bleu_score import sentence_bleu\n\n# Set device configuration\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f'Using device: {device}')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-05T02:34:43.050366Z","iopub.execute_input":"2024-11-05T02:34:43.050994Z","iopub.status.idle":"2024-11-05T02:35:06.236439Z","shell.execute_reply.started":"2024-11-05T02:34:43.05095Z","shell.execute_reply":"2024-11-05T02:35:06.234714Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 3. Data Preparation\n\n## 3.1 Load Training Data\nWe'll load a small subset of the training data provided in the competition.","metadata":{}},{"cell_type":"code","source":"# Load a small subset of the training data\ntrain_file = '/kaggle/input/wikipedia-image-caption/train-00000-of-00005.tsv'\ntrain_df = pd.read_csv(train_file, sep='\\t', nrows=30000)\n\n# Filter for English captions\ntrain_df = train_df[train_df['language'] == 'en'].reset_index(drop=True)\n\n# Combine text fields to create captions\ntrain_df['caption'] = train_df['caption_reference_description'].fillna('') + ' ' + \\\n                      train_df['caption_alt_text_description'].fillna('') + ' ' + \\\n                      train_df['context_page_description'].fillna('') + ' ' + \\\n                      train_df['context_section_description'].fillna('') + ' ' + \\\n                      train_df['caption_attribution_description'].fillna('') + ' ' + \\\n                      train_df['page_title'].fillna('')\n\n# Keep necessary columns\ntrain_df = train_df[['image_url', 'caption']]\n\n# Drop rows with empty captions\ntrain_df = train_df[train_df['caption'].str.strip() != ''].reset_index(drop=True)\n\ndef is_english(text):\n    try:\n        return detect(text) == 'en'\n    except LangDetectException:\n        return False\n\n# Apply the function to filter captions\ntrain_df = train_df[train_df['caption'].apply(is_english)].reset_index(drop=True)\n\nvalid_extensions = ('.jpg', '.jpeg', '.png')\ndef is_valid_image_url(url):\n    return url.lower().endswith(valid_extensions)\n    \ntrain_df = train_df[train_df['image_url'].apply(is_valid_image_url)].reset_index(drop=True)\n\nprint(f'Total samples: {len(train_df)}')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-05T02:35:06.23917Z","iopub.execute_input":"2024-11-05T02:35:06.239683Z","iopub.status.idle":"2024-11-05T02:35:08.32385Z","shell.execute_reply.started":"2024-11-05T02:35:06.23963Z","shell.execute_reply":"2024-11-05T02:35:08.32284Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3.2 Download Images\nWe'll download images using their URLs. To avoid overloading the session, we'll limit the number of images.","metadata":{}},{"cell_type":"code","source":"# Limit the number of images to download\ntrain_df = train_df.head(2000)  # Adjust as needed\nprint(f'Number of samples after limiting: {len(train_df)}')\n\n# Function to download an image\ndef download_image(url):\n    try:\n        headers = {'User-Agent': 'Mozilla/5.0'}\n        response = requests.get(url, headers=headers, timeout=5)\n        img = Image.open(BytesIO(response.content)).convert('RGB')\n        return img\n    except Exception as e:\n        print(f\"Error downloading image {url}: {e}\")\n        return None\n\n# Download images and store them in a list\nimages = []\ncaptions = []\nfor idx, row in train_df.iterrows():\n    img = download_image(row['image_url'])\n    if img is not None:\n        images.append(img)\n        captions.append(row['caption'])\n    else:\n        print(f\"Skipping image at index {idx}\")\n\nprint(f\"Number of images downloaded: {len(images)}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-05T02:35:08.325786Z","iopub.execute_input":"2024-11-05T02:35:08.326299Z","iopub.status.idle":"2024-11-05T02:35:41.909097Z","shell.execute_reply.started":"2024-11-05T02:35:08.326214Z","shell.execute_reply":"2024-11-05T02:35:41.908113Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3.3 Preprocess Captions\nWe'll preprocess the captions by:\n\n* Converting to lowercase.\n* Tokenizing.\n* Building a vocabulary.\n* Converting words to indices.","metadata":{}},{"cell_type":"code","source":"import unicodedata\nfrom nltk.tokenize import TweetTokenizer\n\ntokenizer = TweetTokenizer()\n\ndef clean_caption(caption):\n    # Normalize accented characters\n    caption = unicodedata.normalize('NFKD', caption).encode('ascii', 'ignore').decode('utf-8', 'ignore')\n    caption = caption.lower()\n    tokens = tokenizer.tokenize(caption)\n    return tokens\n\n\n# Build vocabulary\nall_tokens = []\nfor caption in captions:\n    tokens = clean_caption(caption)\n    all_tokens.extend(tokens)\n\n# Count word frequencies\nword_counts = Counter(all_tokens)\n\n# Create word to index mapping\nthreshold = 2  # Minimum word frequency\nwords = [word for word, cnt in word_counts.items() if cnt >= threshold]\nword2idx = {word: idx+1 for idx, word in enumerate(words)}  # Start indices from 1\nword2idx['<pad>'] = 0  # Add padding token\nword2idx['<start>'] = len(word2idx)\nword2idx['<end>'] = len(word2idx)\nword2idx['<unk>'] = len(word2idx)\n\nprint(\"Sample word2idx mappings:\")\nfor word, idx in list(word2idx.items())[:10]:\n    print(f\"{word}: {idx}\")\n\n# Create index to word mapping\nidx2word = {idx: word for word, idx in word2idx.items()}\n\nvocab_size = len(word2idx)\nprint(f'Vocabulary size: {vocab_size}')\n\n# Convert captions to sequences of word indices\nmax_caption_length = 20  # Adjust as needed\n\ndef caption_to_indices(caption, word2idx, max_length):\n    tokens = clean_caption(caption)\n    tokens = ['<start>'] + tokens + ['<end>']\n    tokens = tokens[:max_length]  # Truncate if necessary\n    indices = [word2idx.get(word, word2idx['<unk>']) for word in tokens]\n    # Pad with zeros\n    if len(indices) < max_length:\n        indices += [word2idx['<pad>']] * (max_length - len(indices))\n    return indices\n\ncaption_indices = [caption_to_indices(c, word2idx, max_caption_length) for c in captions]\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-05T02:35:41.911987Z","iopub.execute_input":"2024-11-05T02:35:41.912697Z","iopub.status.idle":"2024-11-05T02:35:42.40005Z","shell.execute_reply.started":"2024-11-05T02:35:41.912656Z","shell.execute_reply":"2024-11-05T02:35:42.399028Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 4. Dataset and DataLoader\nWe'll create a custom Dataset class to handle our images and captions.","metadata":{}},{"cell_type":"code","source":"class ImageCaptionDataset(Dataset):\n    def __init__(self, images, captions, caption_indices):\n        self.images = images\n        self.captions = captions\n        self.caption_indices = caption_indices\n        self.transform = transforms.Compose([\n            transforms.Resize((224, 224)),\n            transforms.ToTensor(),\n            transforms.Normalize(\n                mean=[0.485, 0.456, 0.406],  # ImageNet means\n                std=[0.229, 0.224, 0.225]    # ImageNet stds\n            )\n        ])\n\n    def __len__(self):\n        return len(self.images)\n\n    def __getitem__(self, idx):\n        img = self.images[idx]\n        img = self.transform(img)\n        caption = torch.tensor(self.caption_indices[idx], dtype=torch.long)\n        return img, caption\n\n# Create dataset and dataloader\ndataset = ImageCaptionDataset(images, captions, caption_indices)\n\n# Split into training and validation sets\ntrain_size = int(0.8 * len(dataset))\nval_size = len(dataset) - train_size\n\ntrain_dataset, val_dataset = torch.utils.data.random_split(dataset, [train_size, val_size])\n\n# DataLoaders\nbatch_size = 32\n\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=2)\nval_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False, num_workers=2)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-05T02:35:42.40111Z","iopub.execute_input":"2024-11-05T02:35:42.401434Z","iopub.status.idle":"2024-11-05T02:35:42.415875Z","shell.execute_reply.started":"2024-11-05T02:35:42.401401Z","shell.execute_reply":"2024-11-05T02:35:42.414835Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 5. Model Architecture\nWe'll build an image captioning model consisting of:\n\n* Encoder: A CNN that extracts image features.\n* Decoder: An LSTM that generates captions based on the image features.\n\n## 5.1 Encoder (CNN)\nWe'll use a pre-trained ResNet50 model as the encoder.","metadata":{}},{"cell_type":"code","source":"class EncoderCNN(nn.Module):\n    def __init__(self, encoded_image_size=14):\n        super(EncoderCNN, self).__init__()\n        self.enc_image_size = encoded_image_size\n\n        resnet = models.resnet50(pretrained=True)\n        # Remove the fully connected layer\n        modules = list(resnet.children())[:-2]\n        self.resnet = nn.Sequential(*modules)\n        self.adaptive_pool = nn.AdaptiveAvgPool2d((encoded_image_size, encoded_image_size))\n\n        self.fine_tune()\n\n    def forward(self, images):\n        # Shape: (batch_size, 3, 224, 224)\n        features = self.resnet(images)  # Shape: (batch_size, 2048, H/32, W/32)\n        features = self.adaptive_pool(features)  # Shape: (batch_size, 2048, enc_image_size, enc_image_size)\n        features = features.permute(0, 2, 3, 1)  # Shape: (batch_size, enc_image_size, enc_image_size, 2048)\n        return features\n\n    def fine_tune(self, fine_tune=True):\n        for p in self.resnet.parameters():\n            p.requires_grad = False  # Freeze parameters\n\n        # Allow fine-tuning of later layers\n        for c in list(self.resnet.children())[5:]:\n            for p in c.parameters():\n                p.requires_grad = fine_tune\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-05T02:35:42.417167Z","iopub.execute_input":"2024-11-05T02:35:42.417492Z","iopub.status.idle":"2024-11-05T02:35:42.426873Z","shell.execute_reply.started":"2024-11-05T02:35:42.417459Z","shell.execute_reply":"2024-11-05T02:35:42.425797Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 5.1.1 Define an Attention Mechanism\nWe'll create a separate Attention class to compute attention weights between the encoder outputs and the decoder hidden state.","metadata":{}},{"cell_type":"code","source":"class Attention(nn.Module):\n    def __init__(self, encoder_dim, decoder_dim, attention_dim):\n        super(Attention, self).__init__()\n        self.encoder_att = nn.Linear(encoder_dim, attention_dim)  # Linear layer to transform encoder output\n        self.decoder_att = nn.Linear(decoder_dim, attention_dim)  # Linear layer to transform decoder hidden state\n        self.full_att = nn.Linear(attention_dim, 1)  # Linear layer to compute attention scores\n        self.relu = nn.ReLU()\n        self.softmax = nn.Softmax(dim=1)  # Softmax layer to compute weights\n\n    def forward(self, encoder_out, decoder_hidden):\n        # encoder_out: (batch_size, num_pixels, encoder_dim)\n        # decoder_hidden: (batch_size, decoder_dim)\n        \n        att1 = self.encoder_att(encoder_out)  # (batch_size, num_pixels, attention_dim)\n        att2 = self.decoder_att(decoder_hidden).unsqueeze(1)  # (batch_size, 1, attention_dim)\n        att = self.full_att(self.relu(att1 + att2)).squeeze(2)  # (batch_size, num_pixels)\n        alpha = self.softmax(att)  # (batch_size, num_pixels)\n        attention_weighted_encoding = (encoder_out * alpha.unsqueeze(2)).sum(dim=1)  # (batch_size, encoder_dim)\n        \n        return attention_weighted_encoding, alpha\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-05T02:35:42.429875Z","iopub.execute_input":"2024-11-05T02:35:42.43018Z","iopub.status.idle":"2024-11-05T02:35:42.441254Z","shell.execute_reply.started":"2024-11-05T02:35:42.430148Z","shell.execute_reply":"2024-11-05T02:35:42.440504Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 5.2 Decoder (LSTM)","metadata":{}},{"cell_type":"code","source":"class DecoderRNN(nn.Module):\n    def __init__(self, embed_size, hidden_size, vocab_size, encoder_dim=2048, attention_dim=256, dropout=0.5):\n        super(DecoderRNN, self).__init__()\n        self.encoder_dim = encoder_dim\n        self.attention_dim = attention_dim\n        self.embed_size = embed_size\n        self.hidden_size = hidden_size\n        self.vocab_size = vocab_size\n\n        self.attention = Attention(encoder_dim, hidden_size, attention_dim)\n\n        self.embedding = nn.Embedding(vocab_size, embed_size)\n        self.dropout = nn.Dropout(dropout)\n        self.decode_step = nn.LSTMCell(embed_size + encoder_dim, hidden_size)\n        self.init_h = nn.Linear(encoder_dim, hidden_size)\n        self.init_c = nn.Linear(encoder_dim, hidden_size)\n        self.f_beta = nn.Linear(hidden_size, encoder_dim)\n        self.sigmoid = nn.Sigmoid()  # <-- Add this line\n        self.fc = nn.Linear(hidden_size, vocab_size)\n        self.init_weights()\n\n    def init_weights(self):\n        \"\"\"Initialize embedding and fully connected weights.\"\"\"\n        self.embedding.weight.data.uniform_(-0.1, 0.1)\n        self.fc.weight.data.uniform_(-0.1, 0.1)\n        self.fc.bias.data.fill_(0)\n\n    def forward(self, encoder_out, captions):\n        batch_size = encoder_out.size(0)\n        vocab_size = self.vocab_size\n\n        # Flatten image\n        encoder_out = encoder_out.view(batch_size, -1, self.encoder_dim)  # (batch_size, num_pixels, encoder_dim)\n        num_pixels = encoder_out.size(1)\n\n        # Embedding\n        embeddings = self.embedding(captions)  # (batch_size, max_length, embed_size)\n\n        # Initialize LSTM state\n        h, c = self.init_hidden_state(encoder_out)  # (batch_size, hidden_size)\n\n        # Create tensors to hold word prediction scores and alphas\n        predictions = torch.zeros(batch_size, captions.size(1), vocab_size).to(device)\n        alphas = torch.zeros(batch_size, captions.size(1), num_pixels).to(device)\n\n        for t in range(captions.size(1)):\n            attention_weighted_encoding, alpha = self.attention(encoder_out, h)\n            gate = self.sigmoid(self.f_beta(h))  # (batch_size, encoder_dim)\n            attention_weighted_encoding = gate * attention_weighted_encoding\n\n            lstm_input = torch.cat([embeddings[:, t, :], attention_weighted_encoding], dim=1)\n            h, c = self.decode_step(lstm_input, (h, c))\n\n            preds = self.fc(self.dropout(h))  # (batch_size, vocab_size)\n            predictions[:, t, :] = preds\n            alphas[:, t, :] = alpha\n\n        return predictions\n\n    def init_hidden_state(self, encoder_out):\n        \"\"\"Initialize the hidden states.\"\"\"\n        mean_encoder_out = encoder_out.mean(dim=1)\n        h = self.init_h(mean_encoder_out)  # (batch_size, hidden_size)\n        c = self.init_c(mean_encoder_out)\n        return h, c\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-05T02:35:42.44253Z","iopub.execute_input":"2024-11-05T02:35:42.442854Z","iopub.status.idle":"2024-11-05T02:35:42.459444Z","shell.execute_reply.started":"2024-11-05T02:35:42.442821Z","shell.execute_reply":"2024-11-05T02:35:42.458545Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 5.3 Combined Model\nWe combine the encoder and decoder.","metadata":{}},{"cell_type":"code","source":"# Hyperparameters\nembed_size = 256\nhidden_size = 512\nnum_epochs = 500\nlearning_rate = 1e-3\n\nencoder = EncoderCNN().to(device)\ndecoder = DecoderRNN(embed_size, hidden_size, vocab_size).to(device)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-05T02:35:42.460634Z","iopub.execute_input":"2024-11-05T02:35:42.461207Z","iopub.status.idle":"2024-11-05T02:35:43.084256Z","shell.execute_reply.started":"2024-11-05T02:35:42.461172Z","shell.execute_reply":"2024-11-05T02:35:43.083393Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 6. Training the Model\n\n## 6.1 Loss Function and Optimizer\nWe'll use Cross-Entropy Loss and Adam optimizer.","metadata":{}},{"cell_type":"code","source":"criterion = nn.CrossEntropyLoss(ignore_index=word2idx['<pad>'])\nparams = list(decoder.parameters()) + list(encoder.parameters())\noptimizer = torch.optim.Adam(params, lr=learning_rate)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-05T02:35:43.088665Z","iopub.execute_input":"2024-11-05T02:35:43.088959Z","iopub.status.idle":"2024-11-05T02:35:43.099472Z","shell.execute_reply.started":"2024-11-05T02:35:43.088929Z","shell.execute_reply":"2024-11-05T02:35:43.098537Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 6.2 Training Loop","metadata":{}},{"cell_type":"code","source":"total_step = len(train_loader)\nprint_every = max(1, num_epochs // 4) if num_epochs >= 4 else 1  # Adjust as needed\nbest_val_loss = float('inf')\n\nfor epoch in range(num_epochs):\n    # Training\n    encoder.train()\n    decoder.train()\n    total_train_loss = 0\n\n    for i, (images, captions) in enumerate(train_loader):\n        images = images.to(device)\n        captions = captions.to(device)\n\n        # Forward pass\n        features = encoder(images)\n        outputs = decoder(features, captions)\n\n        # Compute loss\n        targets = captions[:, 1:]  # Shifted by one\n        outputs = outputs[:, :-1, :]  # Align outputs with targets\n        outputs = outputs.reshape(-1, outputs.size(2))\n        targets = targets.reshape(-1)\n        loss = criterion(outputs, targets)\n\n        # Backward and optimize\n        decoder.zero_grad()\n        encoder.zero_grad()\n        loss.backward()\n        optimizer.step()\n\n        total_train_loss += loss.item()\n\n        if (i + 1) % print_every == 0:\n            print(f'Epoch [{epoch + 1}/{num_epochs}], Step [{i + 1}/{total_step}], '\n                  f'Loss: {loss.item():.4f}')\n\n    avg_train_loss = total_train_loss / len(train_loader)\n    print(f'Epoch [{epoch + 1}/{num_epochs}], Average Training Loss: {avg_train_loss:.4f}')\n\n    # Validation\n    encoder.eval()\n    decoder.eval()\n    total_val_loss = 0\n\n    with torch.no_grad():\n        for images, captions in val_loader:\n            images = images.to(device)\n            captions = captions.to(device)\n\n            # Forward pass\n            features = encoder(images)\n            outputs = decoder(features, captions)\n\n            # Compute loss\n            targets = captions[:, 1:]  # Shifted by one\n            outputs = outputs[:, :-1, :]  # Align outputs with targets\n            outputs = outputs.reshape(-1, outputs.size(2))\n            targets = targets.reshape(-1)\n            loss = criterion(outputs, targets)\n\n            total_val_loss += loss.item()\n\n    avg_val_loss = total_val_loss / len(val_loader)\n    print(f'Epoch [{epoch + 1}/{num_epochs}], Validation Loss: {avg_val_loss:.4f}')\n\n    # Save the model if validation loss has decreased\n    if avg_val_loss < best_val_loss:\n        best_val_loss = avg_val_loss\n        torch.save({\n            'encoder_state_dict': encoder.state_dict(),\n            'decoder_state_dict': decoder.state_dict(),\n            'optimizer_state_dict': optimizer.state_dict(),\n            'epoch': epoch,\n            'loss': avg_val_loss\n        }, 'best_model.pth')\n        print('Model saved.\\n')\n    else:\n        print('\\n')\n\nprint('Training complete.')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-05T02:35:43.10119Z","iopub.execute_input":"2024-11-05T02:35:43.101643Z","iopub.status.idle":"2024-11-05T02:36:03.664386Z","shell.execute_reply.started":"2024-11-05T02:35:43.101609Z","shell.execute_reply":"2024-11-05T02:36:03.663163Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 7. Generating Captions\nWe'll write a function to generate captions for an image.","metadata":{}},{"cell_type":"code","source":"def generate_caption(encoder, decoder, image, word2idx, idx2word, max_length=20):\n    encoder.eval()\n    decoder.eval()\n    \n    with torch.no_grad():\n        image = image.unsqueeze(0)  # Add batch dimension\n        image = image.to(device)\n        encoder_out = encoder(image)  # (1, enc_image_size, enc_image_size, encoder_dim)\n        \n        # Flatten encoding\n        encoder_dim = encoder_out.size(-1)\n        encoder_out = encoder_out.view(1, -1, encoder_dim)  # (1, num_pixels, encoder_dim)\n        num_pixels = encoder_out.size(1)\n        \n        # Initialize LSTM state\n        h, c = decoder.init_hidden_state(encoder_out)  # (1, hidden_size)\n        \n        # Initialize caption generation process\n        generated_indices = []\n        alphas = []\n        words = []\n        \n        # Start token\n        inputs = torch.tensor([word2idx['<start>']]).to(device)\n        \n        for _ in range(max_length):\n            embeddings = decoder.embedding(inputs).unsqueeze(0)  # (1, 1, embed_size)\n            attention_weighted_encoding, alpha = decoder.attention(encoder_out, h)  # (1, encoder_dim), (1, num_pixels)\n            gate = decoder.sigmoid(decoder.f_beta(h))  # (1, encoder_dim)\n            attention_weighted_encoding = gate * attention_weighted_encoding  # (1, encoder_dim)\n            \n            lstm_input = torch.cat([embeddings.squeeze(1), attention_weighted_encoding], dim=1)  # (1, embed_size + encoder_dim)\n            h, c = decoder.decode_step(lstm_input, (h, c))  # (1, hidden_size)\n            \n            preds = decoder.fc(h)  # (1, vocab_size)\n            _, predicted = preds.max(1)  # (1)\n            predicted_item = predicted.item()\n            generated_indices.append(predicted_item)\n            inputs = predicted  # Prepare input for next iteration\n            \n            # Break if end token is generated\n            if predicted_item == word2idx['<end>']:\n                break\n                \n        # Convert indices to words\n        generated_caption = [idx2word.get(idx, '<unk>') for idx in generated_indices]\n        \n    return generated_caption\n\n\n# Select a random image from the validation set\nsample_image, _ = val_dataset[random.randint(0, len(val_dataset)-1)]\nsample_image = sample_image.to(device)\n\n# Generate caption\ngenerated_caption = generate_caption(encoder, decoder, sample_image, word2idx, idx2word)\nprint('Generated Caption:', ' '.join(generated_caption))\n\n# Display the image\nimport matplotlib.pyplot as plt\n\n# Convert image tensor to NumPy array and transpose dimensions\nimage_np = sample_image.cpu().numpy().transpose(1, 2, 0)\n# Undo normalization\nimage_np = image_np * [0.229, 0.224, 0.225] + [0.485, 0.456, 0.406]\nimage_np = np.clip(image_np, 0, 1)\n\nplt.imshow(image_np)\nplt.title(' '.join(generated_caption))\nplt.axis('off')\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-05T02:36:03.667136Z","iopub.execute_input":"2024-11-05T02:36:03.667533Z","iopub.status.idle":"2024-11-05T02:36:04.046039Z","shell.execute_reply.started":"2024-11-05T02:36:03.667492Z","shell.execute_reply":"2024-11-05T02:36:04.045064Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"ddd","metadata":{}},{"cell_type":"code","source":"class TestDataset(Dataset):\n    def __init__(self, image_paths, transform=None):\n        self.image_paths = image_paths  # List of image file paths or URLs\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.image_paths)\n\n    def __getitem__(self, idx):\n        image_id = self.image_paths[idx].split('/')[-1].split('.')[0]  # Extract image ID from file name\n        image = Image.open(self.image_paths[idx]).convert('RGB')\n        if self.transform:\n            image = self.transform(image)\n        return image_id, image\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-05T02:36:04.047328Z","iopub.execute_input":"2024-11-05T02:36:04.047669Z","iopub.status.idle":"2024-11-05T02:36:04.054431Z","shell.execute_reply.started":"2024-11-05T02:36:04.047635Z","shell.execute_reply":"2024-11-05T02:36:04.053523Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load the test data\ntest_df = pd.read_csv('/kaggle/input/wikipedia-image-caption/test.tsv', sep='\\t')#[:100]\nprint(test_df.head())\nprint(test_df.columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-05T02:36:04.055629Z","iopub.execute_input":"2024-11-05T02:36:04.055912Z","iopub.status.idle":"2024-11-05T02:36:04.194531Z","shell.execute_reply.started":"2024-11-05T02:36:04.055881Z","shell.execute_reply":"2024-11-05T02:36:04.193518Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"captions_df = pd.read_csv('/kaggle/input/wikipedia-image-caption/test_caption_list.csv')#[:100]\nprint(captions_df.head())\nprint(captions_df.columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-05T02:36:04.195807Z","iopub.execute_input":"2024-11-05T02:36:04.196561Z","iopub.status.idle":"2024-11-05T02:36:04.368836Z","shell.execute_reply.started":"2024-11-05T02:36:04.196513Z","shell.execute_reply":"2024-11-05T02:36:04.36789Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from torchvision import transforms\n\nmean = [0.485, 0.456, 0.406]  # Standard ImageNet means\nstd = [0.229, 0.224, 0.225]   # Standard ImageNet stds\n\ntransform_test = transforms.Compose([\n    transforms.Resize((224, 224)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean, std)  # Use the same mean and std as during training\n])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-05T02:36:04.370292Z","iopub.execute_input":"2024-11-05T02:36:04.370999Z","iopub.status.idle":"2024-11-05T02:36:04.376908Z","shell.execute_reply.started":"2024-11-05T02:36:04.370942Z","shell.execute_reply":"2024-11-05T02:36:04.375965Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def download_image(self, url):\n    headers = {'User-Agent': 'Mozilla/5.0'}\n    try:\n        response = requests.get(url, headers=headers, timeout=10)\n        if 'image' not in response.headers.get('Content-Type', ''):\n            print(f\"URL does not contain an image: {url}\")\n            return None\n        img = Image.open(BytesIO(response.content)).convert('RGB')\n        # Check image size\n        if img.size[0] * img.size[1] > 89_478_485:  # Pillow's default limit\n            print(f\"Image {url} is too large, resizing.\")\n            img.thumbnail((1024, 1024), Image.ANTIALIAS)\n        return img\n    except Exception as e:\n        print(f\"Error downloading image {url}: {e}\")\n        return None\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-05T02:36:04.37823Z","iopub.execute_input":"2024-11-05T02:36:04.378623Z","iopub.status.idle":"2024-11-05T02:36:04.390068Z","shell.execute_reply.started":"2024-11-05T02:36:04.37858Z","shell.execute_reply":"2024-11-05T02:36:04.389318Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Update TestDataset\nclass TestDataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.df)\n\n    def download_image(self, url):\n        headers = {'User-Agent': 'Mozilla/5.0'}\n        try:\n            response = requests.get(url, headers=headers, timeout=10)\n            if 'image' not in response.headers.get('Content-Type', ''):\n                print(f\"URL does not contain an image: {url}\")\n                return None\n            img = Image.open(BytesIO(response.content)).convert('RGB')\n            # Check image size and resize if necessary\n            if img.size[0] * img.size[1] > 89_478_485:\n                print(f\"Image {url} is too large ({img.size}), resizing.\")\n                img.thumbnail((1024, 1024), Image.ANTIALIAS)\n            return img\n        except Exception as e:\n            print(f\"Error downloading image {url}: {e}\")\n            return None\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        image_id = row['id']\n        image_url = row['image_url']\n        image = self.download_image(image_url)\n        if image is None:\n            # Create a blank image if download fails\n            image = Image.new('RGB', (224, 224), color='white')\n            print(f\"Using blank image for id {image_id}\")\n        if self.transform:\n            image = self.transform(image)\n        return image_id, image","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-05T02:36:04.391146Z","iopub.execute_input":"2024-11-05T02:36:04.391466Z","iopub.status.idle":"2024-11-05T02:36:04.402338Z","shell.execute_reply.started":"2024-11-05T02:36:04.391431Z","shell.execute_reply":"2024-11-05T02:36:04.401495Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create the test dataset and DataLoader\ntest_dataset = TestDataset(test_df, transform=transform_test)\ntest_loader = DataLoader(test_dataset, batch_size=1, shuffle=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-05T02:36:04.403483Z","iopub.execute_input":"2024-11-05T02:36:04.403849Z","iopub.status.idle":"2024-11-05T02:36:04.422032Z","shell.execute_reply.started":"2024-11-05T02:36:04.403807Z","shell.execute_reply":"2024-11-05T02:36:04.421077Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load your trained encoder model\nencoder = EncoderCNN().to(device)\n\n# Load the trained weights\ncheckpoint = torch.load('best_model.pth', map_location=device)\nencoder.load_state_dict(checkpoint['encoder_state_dict'])\nencoder.eval()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-05T02:36:04.423189Z","iopub.execute_input":"2024-11-05T02:36:04.423537Z","iopub.status.idle":"2024-11-05T02:36:05.278422Z","shell.execute_reply.started":"2024-11-05T02:36:04.423502Z","shell.execute_reply":"2024-11-05T02:36:05.277469Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"image_ids = []\nimage_embeddings = []\n\nwith torch.no_grad():\n    for image_id, image in test_loader:\n        image_id = image_id.item()\n        image = image.to(device)\n        encoder_out = encoder(image)  # Shape: (batch_size, encoder_dim, H, W)\n        # Apply global average pooling\n        encoder_out = torch.nn.functional.adaptive_avg_pool2d(encoder_out, (1, 1))\n        # Flatten the output\n        encoder_out = encoder_out.view(encoder_out.size(0), -1)  # Shape: (batch_size, encoder_dim)\n        image_embedding = encoder_out.cpu()\n        image_ids.append(image_id)\n        image_embeddings.append(image_embedding)\n\n# After collecting all image embeddings\nimage_embeddings = torch.cat(image_embeddings, dim=0)  # Assuming embeddings are 2D tensors\n\n# Convert to NumPy array\nimage_embeddings_np = image_embeddings.numpy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-05T02:52:23.929725Z","iopub.execute_input":"2024-11-05T02:52:23.930726Z","iopub.status.idle":"2024-11-05T02:52:57.281606Z","shell.execute_reply.started":"2024-11-05T02:52:23.930682Z","shell.execute_reply":"2024-11-05T02:52:57.280529Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install transformers\n\nfrom transformers import AutoTokenizer, AutoModel","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-05T02:48:01.018004Z","iopub.status.idle":"2024-11-05T02:48:01.018549Z","shell.execute_reply.started":"2024-11-05T02:48:01.018256Z","shell.execute_reply":"2024-11-05T02:48:01.018302Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load a pre-trained language model (e.g., bert-base-uncased)\ntokenizer = AutoTokenizer.from_pretrained('bert-base-uncased')\ntext_model = AutoModel.from_pretrained('bert-base-uncased').to(device)\ntext_model.eval()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-05T02:49:01.468362Z","iopub.execute_input":"2024-11-05T02:49:01.468757Z","iopub.status.idle":"2024-11-05T02:49:02.070315Z","shell.execute_reply.started":"2024-11-05T02:49:01.46872Z","shell.execute_reply":"2024-11-05T02:49:02.069417Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"caption_texts = captions_df['caption_title_and_reference_description'].tolist()\ncaption_embeddings = []\n\nwith torch.no_grad():\n    for caption in caption_texts:\n        inputs = tokenizer(caption, return_tensors='pt', truncation=True, padding=True).to(device)\n        outputs = text_model(**inputs)\n        embedding = outputs.last_hidden_state.mean(dim=1).cpu()  # Shape: (1, embedding_dim)\n        caption_embeddings.append(embedding)\n\n# Concatenate all caption embeddings into a single tensor\ncaption_embeddings = torch.cat(caption_embeddings, dim=0)  # Shape: (num_captions, embedding_dim)\ncaption_embeddings_np = caption_embeddings.numpy()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-05T02:49:05.780216Z","iopub.execute_input":"2024-11-05T02:49:05.780678Z","iopub.status.idle":"2024-11-05T02:49:06.647019Z","shell.execute_reply.started":"2024-11-05T02:49:05.780633Z","shell.execute_reply":"2024-11-05T02:49:06.646223Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics.pairwise import cosine_similarity\n\nresults = []\n\n# Optionally, normalize embeddings for cosine similarity\nfrom sklearn.preprocessing import normalize\nimage_embeddings_np = normalize(image_embeddings_np, axis=1)\ncaption_embeddings_np = normalize(caption_embeddings_np, axis=1)\n\nfor idx, image_embedding in enumerate(image_embeddings_np):\n    # Compute cosine similarities between the image embedding and all caption embeddings\n    similarities = cosine_similarity([image_embedding], caption_embeddings_np)\n    best_match_idx = np.argmax(similarities)\n    best_caption = caption_texts[best_match_idx]\n    image_id = image_ids[idx]\n    results.append({'id': image_id, 'caption_title_and_reference_description': best_caption})\n    \n    # Optional: Print progress\n    if idx % 100 == 0:\n        print(f\"Processed {idx} images.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-05T02:53:12.160923Z","iopub.execute_input":"2024-11-05T02:53:12.161776Z","iopub.status.idle":"2024-11-05T02:53:12.231793Z","shell.execute_reply.started":"2024-11-05T02:53:12.161733Z","shell.execute_reply":"2024-11-05T02:53:12.230531Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission_df = pd.DataFrame(results)\nsubmission_df = submission_df[['id', 'caption_title_and_reference_description']]\nsubmission_df.to_csv('submission.csv', index=False)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-05T02:36:07.249185Z","iopub.status.idle":"2024-11-05T02:36:07.249576Z","shell.execute_reply.started":"2024-11-05T02:36:07.249389Z","shell.execute_reply":"2024-11-05T02:36:07.249414Z"}},"outputs":[],"execution_count":null}]}