{"cells":[{"metadata":{},"cell_type":"markdown","source":"I had few minutes today to get into competition and peek at the data. \n\n### Data Loader\n\nI used torch dataloading utility to iterate over images and corresponding labels. Right now I am **NOT** using `torchvision.transforms` for resizing images in a batch into standardized size. It is embedded in loader.\n\nChoose the batch you want to visualize also you can change batch size.\n\n### Label Representation\n\nEach label is a series of characters. I use a `sklearn` `CountVectorizer` to build a vocabulary of unique characters that make up our labels. I make sure I strip `InChI=1S/` from all labels which is the same for all instances. Uniquness of the labels start after that sequence. Later on, each label can be represesnted as a binary 2D `torch.Tensor` with shape `(MAX_LABEL_LENGTH, VOCAB_SIZE)`. `c`'th element of the label is character in the vocabulary with an index `c_ix`. `c`'th row's `char_ix` column is set to 1.\n\n\n[Label Tensor](#intLink)\n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\n\nimport cv2\nimport matplotlib.pyplot as plt\n\nimport seaborn as sns\n\nfrom sklearn.feature_extraction.text import CountVectorizer\n\nfrom tqdm import tqdm\n\npd.set_option('display.max_columns', 500)\npd.set_option('display.max_row', 500)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_labels = pd.read_csv(\"../input/bms-molecular-translation/train_labels.csv\")\nsample_sub = pd.read_csv(\"../input/bms-molecular-translation/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_labels['molecule'] = train_labels.InChI.apply(lambda x: x[9:])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_labels.shape[0], len(train_labels.molecule.unique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cvec = CountVectorizer(analyzer='char', binary=True, lowercase=False)\ncvec.fit(train_labels['molecule'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN_BASE_PATH = \"../input/bms-molecular-translation/train\"\nTEST_BASE_PATH = \"../input/bms-molecular-translation/test\"\n\nBATCH_SIZE = 64\nVOCAB_SIZE = len(cvec.vocabulary_)\nMAX_LABEL_LEN = train_labels.molecule.apply(lambda x: len(x)).max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MoleculeDataset(Dataset):\n    def __init__(self, df, dset='train'):\n        super(MoleculeDataset, self).__init__\n        self.df = df\n        self.dset = dset\n    \n    def __getitem__(self, index):\n        imname = self.df.image_id.iloc[index]\n        if self.dset == 'train' or self.dset=='val':\n            basepath = TRAIN_BASE_PATH\n        else:\n            basepath = TEST_BASE_PATH\n            \n        impath = f\"{basepath}/{imname[0]}/{imname[1]}/{imname[2]}/{imname}.png\"\n        \n        image = cv2.imread(impath)\n        image = cv2.resize(image, (288,288))\n        \n        if self.dset == 'train' or self.dset=='val':\n            label = self.df[\"molecule\"].iloc[index]\n            \n            label_tensor = torch.zeros((MAX_LABEL_LEN, VOCAB_SIZE))\n            for char_ix, char in enumerate(label):\n                vocab_ix = cvec.vocabulary_.get(char)\n                label_tensor[char_ix, vocab_ix] = 1\n            return image, label, label_tensor\n        else:\n            return image\n        \n    \n    def __len__(self):\n        return self.df.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mol_train = MoleculeDataset(train_labels)\nmol_test = MoleculeDataset(sample_sub, \"test\")\n\ntrainloader = DataLoader(mol_train, batch_size=BATCH_SIZE, shuffle=False, num_workers=4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"VISUALIZE_BATCH = 10","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for b_ix, batch in enumerate(trainloader):\n    if b_ix == VISUALIZE_BATCH:\n        break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_tensors = []\nfig, ax = plt.subplots(nrows=BATCH_SIZE, figsize=(60, 30))\nfor i, (img, label, label_tensor) in enumerate(zip(batch[0], batch[1], batch[2])):\n    ax[i].imshow(img)\n    ax[i].set_title(label)\n    label_tensors.append(label_tensor)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"\n### Train Label Stats"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(12,6))\nsns.distplot(train_labels.InChI.apply(lambda x: len(x)), axlabel='Label Length', ax=ax)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### A Label Tensor\n\n<div id=\"intLink\">\n</div>"},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.DataFrame(label_tensor.numpy(), \n             columns = list(dict(sorted(cvec.vocabulary_.items(), key=lambda item: item[1])).keys()))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Image Stats"},{"metadata":{"trusted":true},"cell_type":"code","source":"class ImageDataset(Dataset):\n    def __init__(self, df, dset='train'):\n        super(ImageDataset, self).__init__\n        self.df = df\n        self.dset = dset\n    \n    def __getitem__(self, index):\n        imname = self.df.image_id.iloc[index]\n        if self.dset == 'train' or self.dset=='val':\n            basepath = TRAIN_BASE_PATH\n        else:\n            basepath = TEST_BASE_PATH\n            \n        impath = f\"{basepath}/{imname[0]}/{imname[1]}/{imname[2]}/{imname}.png\"\n        \n        image = cv2.imread(impath)\n        return np.array([image[:,:,0].std(), image[:,:,1].std(), image[:,:,2].std()])/255, np.array([image[:,:,0].mean(), image[:,:,1].mean(), image[:,:,2].mean()])/255\n        \n    def __len__(self):\n        return self.df.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mol_train = ImageDataset(train_labels)\ntrainloader = DataLoader(mol_train, batch_size=64, shuffle=False, num_workers=4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_means = []\nbatch_stds = []\n\nmeans = []\nstds = []\n\nfor ix, batch in tqdm(enumerate(trainloader), total=len(mol_train)//64):\n    batch_means.append(batch[1].numpy().mean(axis=0))\n    batch_stds.append(batch[0].numpy().mean(axis=0))\n    if ix % 100:\n        means.append(np.vstack(batch_means).mean(axis=0))\n        stds.append(np.vstack(batch_stds).mean(axis=0))\n        \n        batch_means = []\n        batch_stds = []\n#     if ix == 500:\n#         break\n\nprint(np.vstack(means).mean(axis=0))\nprint(np.vstack(stds).mean(axis=0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}