{"cells":[{"metadata":{},"cell_type":"markdown","source":"Hello All, welcome to my first public kernel :) \n\nI'm trying out a CNN-LSTM approach by combining @afajohn 's insightful [notebook](https://www.kaggle.com/afajohn/data-loading-starter/comments) and https://github.com/sauravraghuvanshi/Udacity-Computer-Vision-Nanodegree-Program/blob/master/project_2_image_captioning_project/model.py\n\nGoal is to get it up and running like the following:\n\n![](https://raw.githubusercontent.com/yunjey/pytorch-tutorial/master/tutorials/03-advanced/image_captioning/png/model.png)\n\nIt's not complete yet and I'll be updating it, excited to get feedback from the knowledgable public here!"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import sys\nimport math\nimport cv2\nimport numpy as np\nimport pandas as pd \n\n\nimport seaborn as sns\n\nimport matplotlib.pyplot as plt\nimport torch\nimport torch.nn as nn\nimport torchvision.models as models\nfrom torchvision import transforms\n\nfrom torch.utils.data import Dataset, DataLoader, sampler\nfrom sklearn.feature_extraction.text import CountVectorizer\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_labels = pd.read_csv(\"../input/bms-molecular-translation/train_labels.csv\")\nsample_sub = pd.read_csv(\"../input/bms-molecular-translation/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_labels['molecule'] = train_labels.InChI.apply(lambda x: x[9:])\ncvec = CountVectorizer(analyzer='char', binary=True, lowercase=False)\ncvec.fit(train_labels['molecule'])\n\nTRAIN_BASE_PATH = \"../input/bms-molecular-translation/train\"\nTEST_BASE_PATH = \"../input/bms-molecular-translation/test\"\n\nBATCH_SIZE = 8\nVOCAB_SIZE = len(cvec.vocabulary_)\nMAX_LABEL_LEN = train_labels.molecule.apply(lambda x: len(x)).max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MoleculeDataset(Dataset):\n    def __init__(self, df, dset='train'):\n        super(MoleculeDataset, self).__init__\n        self.df = df\n        self.dset = dset\n    \n    def __getitem__(self, index):\n        imname = self.df.image_id.iloc[index]\n        if self.dset == 'train' or self.dset=='val':\n            basepath = TRAIN_BASE_PATH\n        else:\n            basepath = TEST_BASE_PATH\n            \n        impath = f\"{basepath}/{imname[0]}/{imname[1]}/{imname[2]}/{imname}.png\"\n        \n        image = cv2.imread(impath).astype(np.float32)\n        image = cv2.resize(image, (224,224))\n        \n        if self.dset == 'train' or self.dset=='val':\n            label = self.df[\"molecule\"].iloc[index]\n            \n            label_tensor = torch.zeros((MAX_LABEL_LEN, VOCAB_SIZE))\n            for char_ix, char in enumerate(label):\n                vocab_ix = cvec.vocabulary_.get(char)\n                label_tensor[char_ix, vocab_ix] = 1\n            return image, label, label_tensor\n        else:\n            return image\n        \n    \n    def __len__(self):\n        return self.df.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 64          # batch size\nvocab_size = 36\nembed_size = 500           # dimensionality of image and word embeddings\nhidden_size = 512          # number of features in hidden state of the RNN decoder\nnum_epochs = 3             # number of training epochs\nsave_every = 1             # determines frequency of saving model weights\nprint_every = 100          # determines window for printing average loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_loader(labels,\n               mode='train',\n               batch_size=1,\n               num_workers=0):\n    \n    assert mode in ['train', 'test'], \"mode must be one of 'train' or 'test'.\"\n\n    dataset = MoleculeDataset(labels)\n    \n    data_loader = DataLoader(dataset=dataset,\n                                      batch_size=batch_size,\n                                      shuffle=True,\n                                      num_workers=num_workers)\n\n    return data_loader","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class EncoderCNN(nn.Module):\n    def __init__(self, embed_size):\n        super(EncoderCNN, self).__init__()\n        resnet = models.resnet50(pretrained=True)\n        for param in resnet.parameters():\n            param.requires_grad_(False)\n        \n        modules = list(resnet.children())[:-1]\n        self.resnet = nn.Sequential(*modules)\n        self.embed = nn.Linear(resnet.fc.in_features, embed_size)\n\n    def forward(self, images):\n        features = self.resnet(images)\n        features = features.view(features.size(0), -1)\n        features = self.embed(features)\n        return features\n    \n\nclass DecoderRNN(nn.Module):\n    def __init__(self, embed_size, hidden_size, vocab_size):\n        ''' Initialize the layers of this model.'''\n        super().__init__()\n    \n        # Keep track of hidden_size for initialization of hidden state\n        self.hidden_size = hidden_size\n        self.device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n        self.lstm = nn.LSTM(input_size=embed_size, \\\n                            hidden_size=hidden_size, # LSTM hidden units \n                            num_layers=1, # number of LSTM layer\n                            bias=True, # use bias weights b_ih and b_hh\n                            batch_first=True,  # input & output will have batch size as 1st dimension\n                            dropout=0, # Not applying dropout \n                            bidirectional=False, # unidirectional LSTM\n                           )\n\n        self.linear = nn.Linear(hidden_size, vocab_size)                     \n\n        \n    def init_hidden(self, batch_size):\n        \"\"\" At the start of training, we need to initialize a hidden state;\n        there will be none because the hidden state is formed based on previously seen data.\n        So, this function defines a hidden state with all zeroes\n        The axes semantics are (num_layers, batch_size, hidden_dim)\n        \"\"\"\n        return (torch.zeros((1, batch_size, self.hidden_size), device=self.device), \\\n                torch.zeros((1, batch_size, self.hidden_size), device=self.device))\n               \n\n    def forward(self, features):\n        \"\"\" Define the feedforward behavior of the model \"\"\"\n        \n        \n        # Initialize the hidden state\n        self.batch_size = features.shape[0] # features is of shape (batch_size, embed_size)\n#         print(f'batch_size: {self.batch_size}')\n        self.hidden = self.init_hidden(self.batch_size) \n\n        lstm_out, self.hidden = self.lstm(features.unsqueeze(1), self.hidden) # lstm_out shape : (batch_size, MAX_LABEL_LEN, hidden_size)\n#         print(f'lstm_out: {lstm_out.shape}')\n#         print(f'hidden: {self.hidden[0].shape}')\n        outputs = self.linear(lstm_out) # outputs shape : (batch_size, MAX_LABEL_LEN, vocab_size)\n#         print(f'outputs: {outputs.shape}')\n        return outputs\n\n    ## Greedy search \n    def sample(self, inputs):\n        \" accepts pre-processed image tensor (inputs) and returns predicted sentence (list of tensor ids of length max_len) \"\n        \n        \n        output = []\n        batch_size = inputs.shape[0] # batch_size is 1 at inference, inputs shape : (1, 1, embed_size)\n        hidden = self.init_hidden(batch_size) # Get initial hidden state of the LSTM\n    \n        while True:\n            lstm_out, hidden = self.lstm(inputs, hidden) # lstm_out shape : (1, 1, hidden_size)\n            outputs = self.linear(lstm_out)  # outputs shape : (1, 1, vocab_size)\n            outputs = outputs.squeeze(1) # outputs shape : (1, vocab_size)\n            _, max_indice = torch.max(outputs, dim=1) # predict the most likely next word, max_indice shape : (1)\n            \n            output.append(max_indice.cpu().numpy()[0].item()) # storing the word predicted\n            \n            if (max_indice == 1):\n                # We predicted the <end> word, so there is no further prediction to do\n                break\n            \n            ## Prepare to embed the last predicted word to be the new input of the lstm\n            inputs = self.word_embeddings(max_indice) # inputs shape : (1, embed_size)\n            inputs = inputs.unsqueeze(1) # inputs shape : (1, 1, embed_size)\n            \n        return output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"encoder = EncoderCNN(embed_size)\ndecoder = DecoderRNN(embed_size, hidden_size, vocab_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x = encoder(torch.rand((2,3,288,288)))\nprint(x.shape)\ndecoder(x).shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# transform_train = transforms.Compose([ \n#     transforms.Resize(256),                          # smaller edge of image resized to 256\n#     transforms.RandomCrop(224),                      # get 224x224 crop from random location\n#     transforms.RandomHorizontalFlip(),               # horizontally flip image with probability=0.5\n#     transforms.ToTensor(),                           # convert the PIL Image to a tensor\n#     transforms.Normalize((0.485, 0.456, 0.406),      # normalize image for pre-trained model\n#                          (0.229, 0.224, 0.225))])\n\n# Build data loader.\ndata_loader = get_loader(labels=train_labels,\n                         mode='train',\n                         batch_size=batch_size)\n\n# The size of the vocabulary.\n\n# Initialize the encoder and decoder. \nencoder = EncoderCNN(embed_size)\ndecoder = DecoderRNN(embed_size, hidden_size, vocab_size)\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nencoder.to(device)\ndecoder.to(device)\n\n# Define the loss function. \ncriterion = #TODO\n\nparams = list(decoder.parameters()) + list(encoder.embed.parameters())\n\noptimizer = torch.optim.Adam(params = params, lr = 0.001)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nfor epoch in range(1, num_epochs+1):\n    \n    for i_step in range(1, 100+1):\n\n           \n        \n        # Obtain the batch.\n        images, _, captions = next(iter(data_loader))\n        images = images.permute(0, 3, 1, 2)\n        \n#         print(images.shape)\n#         print(captions.shape)\n        # Move batch of images and captions to GPU if CUDA is available.\n        images = images.to(device)\n        captions = captions.to(device)\n        \n        # Zero the gradients.\n        decoder.zero_grad()\n        encoder.zero_grad()\n        \n        # Pass the inputs through the CNN-RNN model.\n        features = encoder(images)\n        outputs = decoder(features)\n        # Calculate the batch loss.\n        print(f'outputs.view(-1, vocab_size): {outputs.view(-1, vocab_size).flatten().shape}')\n        print(f'captions.view(-1): {captions[:,1,:].squeeze(1).flatten().shape}')\n        loss = criterion(outputs.view(-1, vocab_size), captions)\n        \n        # Backward pass.\n        loss.backward()\n        \n        # Update the parameters in the optimizer.\n        optimizer.step()\n            \n        # Get training statistics.\n        stats = 'Epoch [%d/%d], Step [%d/%d], Loss: %.4f, Perplexity: %5.4f' % (epoch, num_epochs, i_step, total_step, loss.item(), np.exp(loss.item()))\n        \n        \n        # Print training statistics (on different line).\n        if i_step % print_every == 0:\n            print('\\r' + stats)\n            \n    # Save the weights.\n    if epoch % save_every == 0:\n        torch.save(decoder.state_dict(), os.path.join('./models', 'decoder-%d.pkl' % epoch))\n        torch.save(encoder.state_dict(), os.path.join('./models', 'encoder-%d.pkl' % epoch))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}