{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"markdown","source":"# How to use Convolution Encoder with Apache MXNet\n\nThis kernel shows hot to use convolutions to do NLP.\nI am using [GluonNLP](http://gluon-nlp.mxnet.io/) library on top of [Apache MXNet](https://mxnet.incubator.apache.org/) to run the code.\nUnfortunately, by default GluonNLP is not installed, and you have to manually add it if you want to try the kernel out. It also turns out that Kernel's MXNet doesn't support GPU, so it takes about 3.5 hours to run the example using CPU only."},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import io\nimport warnings\nfrom gluonnlp.embedding import TokenEmbedding\nfrom itertools import takewhile, repeat\nfrom types import SimpleNamespace","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"197322b6a832aa274ebf333020e02e49176a5639"},"cell_type":"markdown","source":"# Custom Embedding\n\nIncluded embeddings in GluonNLP are optimized to work with .npz embedding files, so loading pure text file embedding might be tricky.\nThis code optimizes loading performance by preallocating memory."},{"metadata":{"trusted":true,"_uuid":"9417d9fdebeaf91022d81c11ba370c106e4db660"},"cell_type":"code","source":"class CustomEmbedding(TokenEmbedding):\n    \"\"\"This embedding has an optimized version of loading embedding from a text file\"\"\"\n    UNK_IDX = 0\n\n    def __init__(self, text_file_path, dims, **kwargs):\n        super(CustomEmbedding, self).__init__(**kwargs)\n\n        self._dims = dims\n        embeddings_count = self._rawincount(text_file_path)\n        self._load_embedding_txt_custom(text_file_path, ' ', embeddings_count, encoding='utf8')\n\n    def _rawincount(self, filename):\n        f = open(filename, 'rb')\n        bufgen = takewhile(lambda x: x, (f.raw.read(1024 * 1024) for _ in repeat(None)))\n        return sum(buf.count(b'\\n') for buf in bufgen)\n\n    def _load_embedding_txt_custom(self, pretrained_file_path, elem_delim,\n                            embeddings_count, encoding='utf8'):\n        \"\"\"Load embedding vectors from a pre-trained token embedding file.\n\n        For every unknown token, if its representation `self.unknown_token` is encountered in the\n        pre-trained token embedding file, index 0 of `self.idx_to_vec` maps to the pre-trained token\n        embedding vector loaded from the file; otherwise, index 0 of `self.idx_to_vec` maps to the\n        text embedding vector initialized by `self._init_unknown_vec`.\n\n        If a token is encountered multiple times in the pre-trained text embedding file, only the\n        first-encountered token embedding vector will be loaded and the rest will be skipped.\n        \"\"\"\n\n        index = 0\n        vec_len = None\n        all_elems = nd.zeros(shape=(embeddings_count, self._dims))\n        loaded_unknown_vec = None\n\n        for line_num, line in CustomEmbedding._get_lines(pretrained_file_path, encoding):\n\n            token, vector = CustomEmbedding._parse_embedding_line(line, elem_delim, self._dims)\n\n            assert token and len(vector) == self._dims, \\\n                'line {} in {}: unexpected data format.'.format(line_num, pretrained_file_path)\n\n            if token == self.unknown_token and loaded_unknown_vec is None:\n                loaded_unknown_vec = vector\n\n            elif token in self._token_to_idx:\n                warnings.warn('line {} in {}: duplicate embedding found for '\n                              'token \"{}\". Skipped.'.format(line_num, pretrained_file_path,\n                                                            token))\n\n            elif len(vector) == 1 and line_num == 0:\n                warnings.warn('line {} in {}: skipped likely header line.'\n                              .format(line_num, pretrained_file_path))\n            else:\n                if not vec_len:\n                    vec_len = len(vector)\n\n                    if self.unknown_token:\n                        # Reserve a vector slot for the unknown token at the very beggining\n                        # because the unknown token index is 0.\n                        all_elems[index] = [0] * vec_len\n                        index = index + 1\n                else:\n                    assert len(vector) == vec_len, \\\n                        'line {} in {}: found vector of inconsistent dimension for token ' \\\n                        '\"{}\". expected dim: {}, found: {}'.format(line_num,\n                                                                   pretrained_file_path,\n                                                                   token, vec_len, len(vector))\n                all_elems[index] = vector\n                index = index + 1\n\n                self._idx_to_token.append(token)\n                self._token_to_idx[token] = len(self._idx_to_token) - 1\n\n        self._idx_to_vec = all_elems\n\n        if self.unknown_token:\n            if loaded_unknown_vec is None:\n                self._idx_to_vec[CustomEmbedding.UNK_IDX] = self._init_unknown_vec(shape=vec_len)\n            else:\n                self._idx_to_vec[CustomEmbedding.UNK_IDX] = nd.array(loaded_unknown_vec)\n\n    @staticmethod\n    def _get_lines(pretrained_file_path, encoding):\n        with io.open(pretrained_file_path, 'rb') as f:\n            for line_num, line in enumerate(f):\n                try:\n                    line = line.decode(encoding)\n                except ValueError:\n                    warnings.warn('line {} in {}: failed to decode. Skipping.'\n                                  .format(line_num, pretrained_file_path))\n                    continue\n\n                yield line_num, line\n\n    @staticmethod\n    def _parse_embedding_line(s, elem_delim, dims):\n        i = 0\n\n        token = ''\n        vector = [None] * dims\n        index = 0\n\n        while True:\n            j = s.find(elem_delim, i)\n            if j < 0:\n                if i < len(s):\n                    vector[index - 1] = float(s[i:])\n                break\n\n            if index == 0:\n                token = s[i:j]\n            else:\n                vector[index - 1] = float(s[i:j])\n\n            index = index + 1\n            i = j + 1\n        return token, vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f368a66139d2d2ea5d00ecf8881a6109b1442078"},"cell_type":"code","source":"import csv\nimport multiprocessing as mp\nfrom gluonnlp import Vocab, data\nfrom mxnet.gluon.data import ArrayDataset, SimpleDataset\nfrom nltk import word_tokenize\n\n\ndef get_sub_segment_from_list(dataset, indices):\n    return ArrayDataset([dataset[i] for i in indices])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6a4a758872b8f9a6b4181bcb43a3dfea9d880457"},"cell_type":"markdown","source":"# Custom dataset\n\nApache MXNet uses Datasets to deal with data.\nThe class below allows to wrap original data into custom dataset for later loading with DataLoader"},{"metadata":{"trusted":true,"_uuid":"c2e250838b4fc6de48c0581842f5373226eba104"},"cell_type":"code","source":"class QuoraDataset(ArrayDataset):\n    \"\"\"This dataset provides access to Quora insincere data competition\"\"\"\n\n    def __init__(self, segment, root_dir=\"../input/\"):\n        self._root_dir = root_dir\n        self._segment = segment\n        self._segments = {\n            'train': 'train.csv',\n            'test': 'test.csv'\n        }\n\n        super(QuoraDataset, self).__init__(self._read_data())\n\n    def _read_data(self):\n        file_path = os.path.join(self._root_dir, self._segments[self._segment])\n        with open(file_path, mode='r', encoding='utf-8', newline='') as f:\n            reader = csv.reader(f, delimiter=',', quotechar='\"')\n            # ignore 1st line - which is header\n            data = [(i,) + tuple(row) for i, row in enumerate(reader) if i > 0]\n\n        return data\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5f068c7ba85d89803935b6dcfe2facba13a5f609"},"cell_type":"markdown","source":"# Vocab Provider\n\nWe build our vocab in a separate class and use it later for replacing tokens with indices"},{"metadata":{"trusted":true,"_uuid":"9b8b655d0754c9b0929009eae87caaa95187dc9d"},"cell_type":"code","source":"class VocabProvider:\n    \"\"\"Provide word-level vocab based on datasets\"\"\"\n\n    def __init__(self, datasets):\n        self._datasets = datasets\n\n    def get_vocab(self):\n        all_words = [word for dataset in self._datasets for item in dataset for word in item[2]]\n        vocab = Vocab(data.count_tokens(all_words), min_freq=3)\n        return vocab\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"40cb7a64cbbd780e008f40aed40c5538929410be"},"cell_type":"markdown","source":"# Data tokenizer\n\nWe use NLTK to do simple tokenization with an ability to run it asynchronously\n"},{"metadata":{"trusted":true,"_uuid":"2a43f6482dad6f63b00971ce0ef7f2697c7e01fd"},"cell_type":"code","source":"class DataTokenizer:\n    \"\"\"Run tokenizer on all cores\"\"\"\n\n    def __init__(self, run_async=True):\n        self._run_async = run_async\n\n    def _tokenize(self, item):\n        tokenized_item = [word.lower() for word in word_tokenize(item[2])]\n\n        if len(item) == 4:\n            return item[0], item[1], tokenized_item, float(item[3])\n        else:\n            return item[0], item[1], tokenized_item\n\n    def __call__(self, dataset):\n        if self._run_async:\n            with mp.Pool() as pool:\n                tokenized_dataset = SimpleDataset(pool.map(self._tokenize, dataset))\n\n            return tokenized_dataset\n        else:\n            return SimpleDataset([self._tokenize(item) for item in dataset])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"83fcb9c77cf18f9ebf9fded58e00bef36a23da33"},"cell_type":"markdown","source":"# Data transformation\n\nDataTransformer class replaces tokens with indices from the Vocab. It also cuts very long sentences to the maximum limit.\n"},{"metadata":{"trusted":true,"_uuid":"71c39a7ecc8898c4945ebe85c889e5f31df14d7e"},"cell_type":"code","source":"class DataTransformer:\n    \"\"\"Data transformer cuts max length of string, but does not pad it.\n    GluonNLP can effectively work with variable sequence length\"\"\"\n\n    def __init__(self, word_vocab, max_length=70):\n        self._word_vocab = word_vocab\n        self._max_length = max_length\n\n    def __call__(self, *items):\n        indices = self._word_vocab[items[2][:self._max_length]]\n\n        if len(items) == 4:\n            return items[0], indices, items[3]\n        else:\n            return items[0], indices","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2e88d7ccd6552dea2731c4dfcce30e27a3ce6962"},"cell_type":"code","source":"from gluonnlp.model import ConvolutionalEncoder\nfrom mxnet.gluon import HybridBlock\nfrom mxnet.gluon.nn import Dense, Dropout, HybridSequential, Embedding, Conv2D\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2d44055a8750e7761f35bde391815ce9e0d582c7"},"cell_type":"markdown","source":"# Model\n\nModel consists of an Embedding, Convolutional Encoder and Dense layer.\nConolutionalEncoder is the heart of the model - it applies specified number of convolutions of specified filter size.\nThe model is already implemented in GluonNLP, we don't need to manually create it"},{"metadata":{"trusted":true,"_uuid":"0fb0530cd1205bc69e81621a766d4c0c8e948f20"},"cell_type":"code","source":"class QuoraModel(HybridBlock):\n    def __init__(self, word_vocab_size, dropout=0.1, embedding_size=300, prefix=None, params=None):\n        super(QuoraModel, self).__init__(prefix=prefix, params=params)\n        self._embedding_size = embedding_size\n\n        with self.name_scope():\n            self.embedding = Embedding(input_dim=word_vocab_size, output_dim=self._embedding_size)\n            self.encoder = ConvolutionalEncoder(embed_size=self._embedding_size,\n                                                num_filters=(128, 128, 128, 128,),\n                                                ngram_filter_sizes=(1, 2, 3, 5,),\n                                                conv_layer_activation='tanh',\n                                                num_highway=None,\n                                                output_size=None)\n            self.output = HybridSequential()\n\n            with self.output.name_scope():\n                self.output.add(Dropout(dropout))\n                self.output.add(Dense(units=1))\n\n    def hybrid_forward(self, F, data):\n        embedded = self.embedding(data)\n        embedded = embedded.transpose(axes=(1, 0, 2))\n        encoded = self.encoder(embedded)\n        result = self.output(encoded)\n        return result\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b8edfd35afdd606439f0f0ca29e27a6b95ef5ff6"},"cell_type":"code","source":"import os\nimport time\nimport numpy as np\nimport pandas as pd\nimport mxnet as mx\nfrom gluonnlp.data import FixedBucketSampler\nfrom gluonnlp.data.batchify import Tuple, Pad, Stack\nfrom mxnet.gluon import Trainer\nfrom mxnet.gluon.data import DataLoader\nfrom mxnet import nd, autograd\nfrom sklearn.metrics import f1_score\nfrom sklearn.model_selection import train_test_split\n\n\ndef get_args():\n    args = SimpleNamespace()\n    args.gpu = None\n    args.learning_rate = 0.001\n    args.epochs = 3\n    args.batch_size = 512\n\n    return args\n\n\ndef find_best_f1(outputs, labels):\n    tmp = [0, 0, 0]  # idx, cur, max\n    threshold = 0\n\n    for tmp[0] in np.arange(0.1, 0.501, 0.01):\n        tmp[1] = f1_score(labels.asnumpy(), outputs.asnumpy() > tmp[0])\n        if tmp[1] > tmp[2]:\n            threshold = tmp[0]\n            tmp[2] = tmp[1]\n\n    return tmp[2], threshold\n\n\ndef run_training(net, trainer, train_dataloader, val_dataloader, epochs, context):\n    loss_fn = mx.gluon.loss.SigmoidBinaryCrossEntropyLoss(from_sigmoid=False)\n    print(\"Start training for {} epochs: {}\".format(epochs, time.ctime()))\n\n    for e in range(epochs):\n        train_loss = 0.\n        total_items = 0\n        val_outputs = []\n        val_l = []\n\n        for i, (q_idx, (word_data, word_valid_lengths), label) in enumerate(train_dataloader):\n            items_per_iteration = word_data.shape[0]\n            total_items += items_per_iteration\n\n            word_data = word_data.as_in_context(context)\n            label = label.astype('float32').as_in_context(context)\n\n            with autograd.record():\n                out = net(word_data)\n                loss = loss_fn(out, label)\n\n            loss.backward()\n            trainer.step(1)\n            train_loss += loss.mean().asscalar()\n\n        for i, (q_idx, (word_data, word_valid_lengths), label) in enumerate(val_dataloader):\n            word_data = word_data.as_in_context(context)\n            label = label.astype('float32').as_in_context(context)\n\n            out = net(word_data)\n            val_outputs.append(out.sigmoid())\n            val_l.append(label.reshape(shape=(label.shape[0], 1)))\n\n        val_outputs = mx.nd.concat(*val_outputs, dim=0)\n        val_l = mx.nd.concat(*val_l, dim=0)\n\n        val_f1, threshold = find_best_f1(val_outputs, val_l)\n        print(\"Epoch {}. Current Loss: {:.5f}. Val F1: {:.3f}, Val Threshold: {:.3f}, {}\"\n              .format(e, train_loss / total_items, val_f1, threshold, time.ctime()))\n\n    return net, threshold\n\n\ndef run_evaluation(net, dataloader, threshold, context):\n    print(\"Start predicting\")\n    outputs = []\n    result = []\n\n    for i, (q_idx, (word_data, word_valid_lengths)) in enumerate(dataloader):\n        word_data = word_data.as_in_context(context)\n\n        out = net(word_data)\n        outputs.append((q_idx, out.sigmoid() > threshold))\n\n    for batch in outputs:\n        result.extend([(int(q_idx.asscalar()), int(pred.asscalar()))\n                       for q_idx, pred in zip(batch[0], batch[1])])\n\n    return result\n\n\ndef load_and_process_dataset(dataset, word_vocab, path=None, async=True):\n    tokenizer = DataTokenizer(async)\n    transformer = DataTransformer(word_vocab)\n    lazy_trasform = True if not path else False\n    processed_dataset = tokenizer(dataset).transform(transformer, lazy=lazy_trasform)\n\n    return processed_dataset\n\n\ndef load_vocab(dataset):\n    tokenizer = DataTokenizer()\n    tokenized_dataset = tokenizer(dataset)\n    vocab_provider = VocabProvider([tokenized_dataset])\n    word_vocab = vocab_provider.get_vocab()\n\n    return word_vocab\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"de8d8b5236ce4bd9f6d89ba81d85cf3b8670e5b6"},"cell_type":"markdown","source":"# Run training\n"},{"metadata":{"trusted":true,"_uuid":"7297a50e40e854b6eb9842266297d4533cd79398"},"cell_type":"code","source":"args = get_args()\ncontext = mx.cpu(0) if args.gpu is None else mx.gpu(args.gpu)\n\nprint('Script started. {}'.format(time.ctime()))\ndataset = QuoraDataset('train')\nword_vocab = load_vocab(dataset)\nglove = CustomEmbedding('../input/embeddings/glove.840B.300d/glove.840B.300d.txt', 300)\nword_vocab.set_embedding(glove)\n\nmodel = QuoraModel(len(word_vocab))\nmodel.initialize(mx.init.Xavier(magnitude=2.24), ctx=context)\nmodel.embedding.weight.set_data(word_vocab.embedding.idx_to_vec)\nmodel.embedding.collect_params().setattr('grad_req', 'null')\nmodel.hybridize(static_alloc=True)\n\ntrain_indices, dev_indices = train_test_split(range(len(dataset)), train_size=0.9)\n\ntrain_dataset = load_and_process_dataset(get_sub_segment_from_list(dataset, train_indices),\n                                         word_vocab)\n\ndev_dataset = load_and_process_dataset(get_sub_segment_from_list(dataset, dev_indices),\n                                       word_vocab)\n\nbatchify_fn = Tuple(Stack(),\n                    Pad(axis=0, pad_val=word_vocab[word_vocab.padding_token], ret_length=True),\n                    Stack())\n\ntrain_sampler = FixedBucketSampler(lengths=[len(item[1]) for item in train_dataset],\n                                   batch_size=args.batch_size,\n                                   shuffle=True)\n\ndev_sampler = FixedBucketSampler(lengths=[len(item[1]) for item in dev_dataset],\n                                 batch_size=args.batch_size,\n                                 shuffle=False)\n\ntrain_dataloader = DataLoader(train_dataset,\n                              batchify_fn=batchify_fn,\n                              batch_sampler=train_sampler,\n                              num_workers=10)\n\ndev_dataloader = DataLoader(dev_dataset,\n                            batchify_fn=batchify_fn,\n                            batch_sampler=dev_sampler,\n                            num_workers=10)\n\ntrainer = Trainer(model.collect_params(), 'adam', {'learning_rate': args.learning_rate})\nbest_model, best_val_threshold = run_training(model, trainer, train_dataloader,\n                                              dev_dataloader, args.epochs, context)\n\n\ntest_dataset = QuoraDataset('test')\nprocessed_test_dataset = load_and_process_dataset(test_dataset, word_vocab)\ntest_sampler = FixedBucketSampler(lengths=[len(item[1]) for item in processed_test_dataset],\n                                  batch_size=args.batch_size)\n\nbatchify_test_fn = Tuple(Stack(),\n                         Pad(axis=0, pad_val=word_vocab[word_vocab.padding_token],\n                             ret_length=True))\n\ntest_dataloader = DataLoader(processed_test_dataset,\n                             batchify_fn=batchify_test_fn,\n                             batch_sampler=test_sampler,\n                             num_workers=10,\n                             shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5f13611b2736a9a803f19008d608cb1610109519"},"cell_type":"markdown","source":"# Run evaluation"},{"metadata":{"trusted":true,"_uuid":"73d316f4f503e2b7ac3d3581a092272c09a5f69a"},"cell_type":"code","source":"predictions = run_evaluation(model, test_dataloader, best_val_threshold, context)\n\nsubmission = pd.DataFrame()\nmapping = {item[0]: item[1] for item in test_dataset}\nsubmission['qid'] = [mapping[p[0]] for p in predictions]\nsubmission['prediction'] = [p[1] for p in predictions]\n\n# some magic to restore the order, in case Kaggle is sensetive to order of items\nsample = pd.read_csv('../input/sample_submission.csv')\njoined = sample.join(submission.set_index('qid'), on='qid', lsuffix='_sample', rsuffix='_real')\n\njoined = joined.drop(['prediction_sample'], axis=1)\njoined = joined.rename(index=str, columns={'prediction_real': 'prediction'})\njoined.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}