{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\n# import numpy as np # linear algebra\n# import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#import matplotlib.pyplot as plt\nimport os, random, math\nimport pandas as pd\nimport numpy as np\nfrom typing import Any, Callable, cast, Dict, List, Optional, Tuple\n\n# !conda install -c conda-forge rdkit\n# from rdkit import Chem\n\nimport seaborn as sns\nfrom matplotlib import pyplot as plt\n\nimport PIL\nfrom PIL import Image\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.nn.utils.rnn import pad_sequence\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader,Dataset\n\nimport torchvision\nfrom torchvision import transforms\nimport torchvision.models as models\nimport torchvision.transforms as T\nfrom torchvision.transforms.transforms import Compose, Normalize, Resize, ToTensor, RandomHorizontalFlip, RandomCrop","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EncoderCNN(nn.Module):\n    def __init__(self):\n        super(EncoderCNN, self).__init__()\n        resnet = models.resnet34(pretrained= False)\n        resnet.conv1 = nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3,bias=False) # kernel_size=7, stride=2, padding=3\n        modules = list(resnet.children())[:-3] # use the most effective part for image mapping\n        self.resnet = nn.Sequential(*modules)\n\n    def forward(self, images):\n        features = self.resnet(images)\n        return features\n\n# https://github.com/wzlxjtu/PositionalEncoding2D/blob/master/positionalembedding2d.py\nclass PositionEncode2D(nn.Module):\n    def __init__(self, dim, width, height):\n        super().__init__()\n        assert (dim % 4 == 0)\n        self.width = width\n        self.height = height\n\n        dim = dim // 2\n        d = torch.exp(torch.arange(0., dim, 2) * -(math.log(10000.0) / dim))\n        position_w = torch.arange(0., width).unsqueeze(1)\n        position_h = torch.arange(0., height).unsqueeze(1)\n        pos = torch.zeros(1, dim*2, height, width)\n\n        pos[0, 0:dim:2, :, :] = torch.sin(position_w * d).transpose(0, 1).unsqueeze(1).repeat(1, 1, height, 1)\n        pos[0, 1:dim:2, :, :] = torch.cos(position_w * d).transpose(0, 1).unsqueeze(1).repeat(1, 1, height, 1)\n        pos[0, dim + 0::2, :, :] = torch.sin(position_h * d).transpose(0, 1).unsqueeze(2).repeat(1, 1, 1, width)\n        pos[0, dim + 1::2, :, :] = torch.cos(position_h * d).transpose(0, 1).unsqueeze(2).repeat(1, 1, 1, width)\n        self.register_buffer('pos', pos)\n\n    def forward(self, x):\n        batch_size, C, H, W = x.shape\n        x = x + self.pos[:, :, :H, :W]\n        return x\n\n# https://pytorch.org/tutorials/beginner/transformer_tutorial.html\nclass TransformerModel(nn.Module):\n    def __init__(self, hidden = 768,nhead = 4, nlayers = 3, dropout=0.1):\n        super(TransformerModel, self).__init__()\n        nhead = hidden // 64\n        self.positionencode2D = PositionEncode2D(dim = hidden , width = 40, height = 24)\n        self.encoder_layer = nn.TransformerEncoderLayer(d_model = hidden, nhead = nhead)\n        self.transformer_encoder = nn.TransformerEncoder(self.encoder_layer, num_layers = nlayers)\n        \n        self.fc_out = nn.Linear(hidden, 1)\n        self.relu = nn.ReLU()\n        self.src_mask = None\n\n    def forward(self, src):\n        \n        # enc_src = [src_len, hid dim]\n        src = self.positionencode2D(src) # [batch, src_len]\n        src = src.view(src.size(0),src.size(1),-1)\n        src = src.permute(2,0,1).contiguous()\n        \n        output = self.transformer_encoder(src)\n        output = output.permute(1,0,2).contiguous()\n        return output\n    \n\nclass Attention(nn.Module):\n    def __init__(self, encoder_dim,decoder_dim,attention_dim):\n        super(Attention, self).__init__()\n        \n        self.attention_dim = attention_dim\n        self.W = nn.Linear(decoder_dim,attention_dim)\n        self.U = nn.Linear(encoder_dim,attention_dim)\n        self.A = nn.Linear(attention_dim,1)\n\n    def forward(self, features, hidden_state):\n        u_hs = self.U(features)     #(batch_size,num_layers_pixels,attention_dim) \n        w_ah = self.W(hidden_state) #(batch_size,attention_dim)\n        combined_states = torch.tanh(u_hs + w_ah.unsqueeze(1)) #(batch_size,num_layers_pixels,attemtion_dim)\n        \n        attention_scores = self.A(combined_states)         #(batch_size,num_layers_pixels,1)\n        attention_scores = attention_scores.squeeze(2)     #(batch_size,num_layers_pixels)\n        \n        # to combine features and hidden_state with features as a weight, and then give features\n        alpha = F.softmax(attention_scores,dim=1)          #(batch_size,num_layers_pixels)\n        # Multiply after increasing the dimension, different pixels have different weights\n        # but the same pixel has the same weight for N dimensions\n        attention_weights = features * alpha.unsqueeze(2)  #(batch_size,num_layers_pixels,features_dim)\n        # Sum X again. This is the final matrix regrading weight, but not the weight\n        attention_weights = attention_weights.sum(dim=1)   #(batch_size,num_layers_pixels)\n        return alpha,attention_weights","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}