{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport nltk\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nimport gensim.models\nimport torch\nfrom torch import nn\nimport torch.nn.functional as F\nimport torch.utils.data\nimport copy\nprint(os.listdir(\"../input\"))\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6fd243fc59e4a741ebab6661a90abc49f0b92d7c"},"cell_type":"code","source":"np.random.seed(481945)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fc3691d1aa5bd23f79089a6016d870cbada8f78c"},"cell_type":"code","source":"from sklearn.metrics import pairwise","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"62ae520429acfe5a097782221d5b10263074b26f"},"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.model_selection import train_test_split\nimport matplotlib.pyplot as plt\nfrom sklearn import metrics","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c0de99ab4008edb3f64776af93d5677d9f11b669"},"cell_type":"code","source":"!ls","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5b35704d7513538bbdb2a355df1bf763ce9ac7f4","_kg_hide-input":true},"cell_type":"code","source":"from torch.autograd import Variable\n\nfrom collections import OrderedDict\nimport numpy as np\n\n\ndef summary(model, input_size, batch_size=-1, device=\"cuda\", input_type=torch.float32):\n\n    def register_hook(module):\n\n        def hook(module, input, output):\n            class_name = str(module.__class__).split(\".\")[-1].split(\"'\")[0]\n            module_idx = len(summary)\n\n            m_key = \"%s-%i\" % (class_name, module_idx + 1)\n            summary[m_key] = OrderedDict()\n            summary[m_key][\"input_shape\"] = list(input[0].size())\n            summary[m_key][\"input_shape\"][0] = batch_size\n            if isinstance(output, (list, tuple)):\n                summary[m_key][\"output_shape\"] = [\n                    [-1] + list(o.size())[1:] for o in output\n                ]\n            else:\n                summary[m_key][\"output_shape\"] = list(output.size())\n                summary[m_key][\"output_shape\"][0] = batch_size\n\n            params = 0\n            if hasattr(module, \"weight\") and hasattr(module.weight, \"size\"):\n                params += torch.prod(torch.LongTensor(list(module.weight.size())))\n                summary[m_key][\"trainable\"] = module.weight.requires_grad\n            if hasattr(module, \"bias\") and hasattr(module.bias, \"size\"):\n                params += torch.prod(torch.LongTensor(list(module.bias.size())))\n            summary[m_key][\"nb_params\"] = params\n\n        if (\n            not isinstance(module, nn.Sequential)\n            and not isinstance(module, nn.ModuleList)\n            and not (module == model)\n        ):\n            hooks.append(module.register_forward_hook(hook))\n\n    device = device.lower()\n    assert device in [\n        \"cuda\",\n        \"cpu\",\n    ], \"Input device is not valid, please specify 'cuda' or 'cpu'\"\n\n    if device == \"cuda\" and torch.cuda.is_available():\n        dtype = torch.cuda.FloatTensor\n    else:\n        dtype = torch.FloatTensor\n\n    # # multiple inputs to the network\n    # if isinstance(input_size, tuple):\n    #     input_size = [input_size]\n\n    # batch_size of 2 for batchnorm\n  #  x = [torch.rand(2, *in_size).type(dtype) for in_size in input_size]\n    net_device = next(model.parameters()).device\n    x = torch.zeros((1,) + input_size ,dtype=input_type).to(net_device)\n    # print(type(x[0]))\n\n    # create properties\n    summary = OrderedDict()\n    hooks = []\n\n    # register hook\n    model.apply(register_hook)\n\n    # make a forward pass\n    # print(x.shape)\n    model(x)\n\n    # remove these hooks\n    for h in hooks:\n        h.remove()\n\n    print(\"----------------------------------------------------------------\")\n    line_new = \"{:>20}  {:>25} {:>15}\".format(\"Layer (type)\", \"Output Shape\", \"Param #\")\n    print(line_new)\n    print(\"================================================================\")\n    total_params = 0\n    total_output = 0\n    trainable_params = 0\n    for layer in summary:\n        # input_shape, output_shape, trainable, nb_params\n        line_new = \"{:>20}  {:>25} {:>15}\".format(\n            layer,\n            str(summary[layer][\"output_shape\"]),\n            \"{0:,}\".format(summary[layer][\"nb_params\"]),\n        )\n        total_params += summary[layer][\"nb_params\"]\n        total_output += np.prod(summary[layer][\"output_shape\"])\n        if \"trainable\" in summary[layer]:\n            if summary[layer][\"trainable\"] == True:\n                trainable_params += summary[layer][\"nb_params\"]\n        print(line_new)\n\n    # assume 4 bytes/number (float on cuda).\n    total_input_size = abs(np.prod(input_size) * batch_size * 4. / (1024 ** 2.))\n    total_output_size = abs(2. * total_output * 4. / (1024 ** 2.))  # x2 for gradients\n    total_params_size = abs(total_params.numpy() * 4. / (1024 ** 2.))\n    total_size = total_params_size + total_output_size + total_input_size\n\n    print(\"================================================================\")\n    print(\"Total params: {0:,}\".format(total_params))\n    print(\"Trainable params: {0:,}\".format(trainable_params))\n    print(\"Non-trainable params: {0:,}\".format(total_params - trainable_params))\n    print(\"----------------------------------------------------------------\")\n    print(\"Input size (MB): %0.2f\" % total_input_size)\n    print(\"Forward/backward pass size (MB): %0.2f\" % total_output_size)\n    print(\"Params size (MB): %0.2f\" % total_params_size)\n    print(\"Estimated Total Size (MB): %0.2f\" % total_size)\n    print(\"----------------------\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bfea60109ea1d42efe6f71bc6bea5bb4462a3e3d"},"cell_type":"code","source":"from torch.optim.lr_scheduler import _LRScheduler\nfrom torch.optim import Optimizer\nimport math\n\nclass CyclicLR(_LRScheduler):\n    \"\"\"Sets the learning rate of each parameter group according to\n    cyclical learning rate policy (CLR). The policy cycles the learning\n    rate between two boundaries with a constant frequency, as detailed in\n    the paper `Cyclical Learning Rates for Training Neural Networks`_.\n    The distance between the two boundaries can be scaled on a per-iteration\n    or per-cycle basis.\n    Cyclical learning rate policy changes the learning rate after every batch.\n    `step` should be called after a batch has been used for training.\n    To resume training, save `last_batch_iteration` and use it to instantiate `CycleLR`.\n    This class has three built-in policies, as put forth in the paper:\n    \"triangular\":\n        A basic triangular cycle w/ no amplitude scaling.\n    \"triangular2\":\n        A basic triangular cycle that scales initial amplitude by half each cycle.\n    \"exp_range\":\n        A cycle that scales initial amplitude by gamma**(cycle iterations) at each\n        cycle iteration.\n    This implementation was adapted from the github repo: `bckenstler/CLR`_\n    Args:\n        optimizer (Optimizer): Wrapped optimizer.\n        base_lr (float or list): Initial learning rate which is the\n            lower boundary in the cycle for eachparam groups.\n            Default: 0.001\n        max_lr (float or list): Upper boundaries in the cycle for\n            each parameter group. Functionally,\n            it defines the cycle amplitude (max_lr - base_lr).\n            The lr at any cycle is the sum of base_lr\n            and some scaling of the amplitude; therefore\n            max_lr may not actually be reached depending on\n            scaling function. Default: 0.006\n        step_size_up (int): Number of training iterations in the\n            increasing half of a cycle. Default: 2000\n        step_size_down (int): Number of training iterations in the\n            decreasing half of a cycle. If step_size_down is None,\n            it is set to step_size_up. Default: None\n        mode (str): One of {triangular, triangular2, exp_range}.\n            Values correspond to policies detailed above.\n            If scale_fn is not None, this argument is ignored.\n            Default: 'triangular'\n        gamma (float): Constant in 'exp_range' scaling function:\n            gamma**(cycle iterations)\n            Default: 1.0\n        scale_fn (function): Custom scaling policy defined by a single\n            argument lambda function, where\n            0 <= scale_fn(x) <= 1 for all x >= 0.\n            mode parameter is ignored\n            Default: None\n        scale_mode (str): {'cycle', 'iterations'}.\n            Defines whether scale_fn is evaluated on\n            cycle number or cycle iterations (training\n            iterations since start of cycle).\n            Default: 'cycle'\n        last_batch_idx (int): The index of the last batch. Default: -1\n    Example:\n        >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9)\n        >>> scheduler = torch.optim.CyclicLR(optimizer)\n        >>> data_loader = torch.utils.data.DataLoader(...)\n        >>> for epoch in range(10):\n        >>>     for batch in data_loader:\n        >>>         scheduler.step()\n        >>>         train_batch(...)\n    .. _Cyclical Learning Rates for Training Neural Networks: https://arxiv.org/abs/1506.01186\n    .. _bckenstler/CLR: https://github.com/bckenstler/CLR\n    \"\"\"\n\n    def __init__(self,\n                 optimizer,\n                 base_lr=1e-3,\n                 max_lr=6e-3,\n                 step_size_up=2000,\n                 step_size_down=None,\n                 mode='triangular',\n                 gamma=1.,\n                 scale_fn=None,\n                 scale_mode='cycle',\n                 last_batch_idx=-1):\n\n        if not isinstance(optimizer, Optimizer):\n            raise TypeError('{} is not an Optimizer'.format(\n                type(optimizer).__name__))\n        self.optimizer = optimizer\n\n        base_lrs = self._format_lr('base_lr', optimizer, base_lr)\n        if last_batch_idx == -1:\n            for base_lr, group in zip(base_lrs, optimizer.param_groups):\n                group['lr'] = base_lr\n\n        self.max_lrs = self._format_lr('max_lr', optimizer, max_lr)\n\n        step_size_down = step_size_down or step_size_up\n        self.total_size = float(step_size_up + step_size_down)\n        self.step_ratio = float(step_size_up) / self.total_size\n\n        if mode not in ['triangular', 'triangular2', 'exp_range'] \\\n                and scale_fn is None:\n            raise ValueError('mode is invalid and scale_fn is None')\n\n        self.mode = mode\n        self.gamma = gamma\n\n        if scale_fn is None:\n            if self.mode == 'triangular':\n                self.scale_fn = self._triangular_scale_fn\n                self.scale_mode = 'cycle'\n            elif self.mode == 'triangular2':\n                self.scale_fn = self._triangular2_scale_fn\n                self.scale_mode = 'cycle'\n            elif self.mode == 'exp_range':\n                self.scale_fn = self._exp_range_scale_fn\n                self.scale_mode = 'iterations'\n        else:\n            self.scale_fn = scale_fn\n            self.scale_mode = scale_mode\n        super(CyclicLR, self).__init__(optimizer, last_batch_idx)\n\n    def _format_lr(self, name, optimizer, lr):\n        \"\"\"Return correctly formatted lr for each param group.\"\"\"\n        if isinstance(lr, (list, tuple)):\n            if len(lr) != len(optimizer.param_groups):\n                raise ValueError(\"expected {} values for {}, got {}\".format(\n                    len(optimizer.param_groups), name, len(lr)))\n            return torch.tensor(lr)\n        else:\n            return lr * torch.ones(len(optimizer.param_groups))\n\n    def _triangular_scale_fn(self, x):\n        return 1.\n\n    def _triangular2_scale_fn(self, x):\n        return 1 / (2. ** (x - 1))\n\n    def _exp_range_scale_fn(self, x):\n        return self.gamma**(x)\n\n    def get_lr(self):\n        \"\"\"Calculates the learning rate at batch index. This function treats\n        `self.last_epoch` as the last batch index.\n        \"\"\"\n        cycle = math.floor(1 + self.last_epoch / self.total_size)\n        x = 1 + self.last_epoch / self.total_size - cycle\n        if x <= self.step_ratio:\n            scale_factor = x / self.step_ratio\n        else:\n            scale_factor = (x - 1) / (self.step_ratio - 1)\n\n        lrs = []\n        for base_lr, max_lr in zip(self.base_lrs, self.max_lrs):\n            base_height = (max_lr - base_lr) * scale_factor\n            if self.scale_mode == 'cycle':\n                lr = base_lr + base_height * self.scale_fn(cycle)\n            else:\n                lr = base_lr + base_height * self.scale_fn(self.last_epoch)\n            lrs.append(lr)\n        return lrs","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"quora_data = pd.read_csv('../input/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0002252a5ccee855de05335a4c83e29a78c0f014"},"cell_type":"code","source":"quora_test_data = pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"94a382f812dfef93a196f15734e1af2357ba7eab"},"cell_type":"code","source":"sample = quora_data.sample(100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c005116ad28e1d6e4840610640f910fe7a7ca7ac"},"cell_type":"code","source":"for s in sample[sample.target == 1].question_text:\n    print(s)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"11e9280cdd4d87f99569a8f453fed3c2628f6716"},"cell_type":"code","source":"from nltk.tokenize import TweetTokenizer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b51915bdfd3f83d560c7dd34e9eb82eeca22daeb"},"cell_type":"code","source":"print(nltk.tokenize.word_tokenize(\"Don't spoil the movie or I'll kill you\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c87402eba7d40d3545aab6ce58c40eebc508a82c"},"cell_type":"code","source":"print(TweetTokenizer().tokenize(\"Don't spoil the movie or I'll kill you\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"89845ec75b5b087aef0731fecb340fffe03fbbdb"},"cell_type":"code","source":"def tokenize(questions):\n    tokenized_questions = []\n    for iteration, text in enumerate(questions):\n        if iteration % 50000 == 0:\n            print(iteration, \"texts tokenized\")\n        tokenized_questions.append([t.lower() for t in TweetTokenizer().tokenize(text)])\n    return tokenized_questions","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":false,"trusted":true,"_uuid":"ea2c111bf6cfd309cbf8a71275de663908a29090"},"cell_type":"code","source":"dev_tokens = tokenize(quora_data.question_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0de6b5cce0c2f8d231e2648cab407fb11fd86b17"},"cell_type":"code","source":"test_tokens = tokenize(quora_test_data.question_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2da01e2b48f726c7355c0f08011b7d5eb5957530"},"cell_type":"code","source":"import torchtext","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"70cc82a7b9a78253321c45496d014c51d754d9bf"},"cell_type":"code","source":"from collections import Counter\nimport itertools","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4e624f75b91f6c924b1e75a3b1b0bca060284ca1"},"cell_type":"code","source":"print(dev_tokens[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cab3e9d38bda7519681adadfa06b482bf62168e1"},"cell_type":"code","source":"train_tokens, val_tokens, train_labels, val_labels = train_test_split(dev_tokens, quora_data.target, test_size=0.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fa6dd04547056e70c26c1461e2008c69dfcae2b2"},"cell_type":"code","source":"word_counts = Counter(itertools.chain(*train_tokens))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d5872d25efef09170c7aa6fe28c8a1d3bdaf44f2"},"cell_type":"code","source":"print(len(word_counts))\nprint(word_counts.most_common(10))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"28f84abea03b05a7bc32f3cd6143874ab032c998"},"cell_type":"code","source":"from gensim.models.keyedvectors import KeyedVectors","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3614cc788e9217f0cf435b789e8193d6943f0afb"},"cell_type":"code","source":"gensim_vectors = KeyedVectors.load_word2vec_format('../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin', binary=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"41522eb74621ecb8034b3ac4f3a7a4e3432a57af"},"cell_type":"code","source":"def filter_vectors(gensim_vectors, words):\n    result = {}\n    for w in words:\n        if w in gensim_vectors.vocab:\n            result[w] = gensim_vectors[w].copy()\n    return result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8b1cad8d88aa6da9045db8fe0b4c60fd8487c4d8"},"cell_type":"code","source":"filtered_vectors = filter_vectors(gensim_vectors, word_counts.keys())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"861875a53775544a33ce3de53a603d9e2d08be58"},"cell_type":"code","source":"print(len(filtered_vectors))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"23c96874306231cfdf56cf97939687f5a200fab7"},"cell_type":"code","source":"del gensim_vectors","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1a59221cbea30779f47b21dc5d40dc9fe006f92e"},"cell_type":"code","source":"min_occurences = 14\nfiltered_counts = {w:c for w,c in word_counts.items() if c >= min_occurences}\nprint(len(filtered_counts))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ec79d75835835dab85b04b461ad8580a29c08d46"},"cell_type":"code","source":"class VocabLike:\n    def __init__(self, itos, stoi):\n        self.itos = itos\n        self.stoi = stoi","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"932683ce96d18735cb51e657107e89127db21ff3"},"cell_type":"code","source":"from collections import defaultdict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e98e9ea2ff00be0b0f1371b61f8c91104df4ed39"},"cell_type":"code","source":"specials = ['<unk>', '<pad>', '<eos>']\nfiltered_counts.update({w:0 for w in specials})\n# vocab = torchtext.vocab.Vocab(filtered_counts,specials=specials, max_size=30000)\nstoi = defaultdict(lambda:0) #map to unk\nitos = [0] * len(filtered_counts)\n\ntrainable_words = {w for w in filtered_counts.keys() if w not in specials and w not in filtered_vectors}\npretrained_words = {w for w in filtered_counts.keys() if w not in specials and w in filtered_vectors}\n\nfor i,w in enumerate(itertools.chain(specials, trainable_words, pretrained_words)):\n    stoi[w] = i\n    itos[i] = w\n    \nvocab = VocabLike(itos, stoi)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fc0c1ee0d92781a6d2030927d68eda07afeb0e47"},"cell_type":"code","source":"print(len(pretrained_words))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bf73bb1f59a1fed21ee8e8ba10814bea018bf88d"},"cell_type":"code","source":"def extract_vectors(vocab, vec_dict, offset, total, vec_size):\n    vectors = np.zeros((total, vec_size), dtype=np.float32)\n    for i in range(total):\n        word = vocab.itos[i + offset]\n        assert word in vec_dict\n        vectors[i] = vec_dict[word]\n    return vectors","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a77252ecce4a39c421107ba3ec542881f64b663b"},"cell_type":"code","source":"np_vectors = extract_vectors(vocab, filtered_vectors,\n                             len(specials) + len(trainable_words), \n                             len(pretrained_words),\n                             300)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ab86134adbdfef76afa6e4b4004dbcf63c7d3f95"},"cell_type":"code","source":"from sklearn.metrics import pairwise\ndef nearest_neighbors(vocab, embeddings, word, topn, use_offset=False):\n    offset = len(specials) + len(trainable_words) if use_offset else 0\n    assert word in vocab.stoi\n    word_index = vocab.stoi[word] - offset\n    sims = pairwise.cosine_similarity(embeddings[word_index].reshape(1,-1),embeddings).ravel()\n    indices = np.argsort(sims)[::-1]\n    print(word)\n    for i in range(topn):\n        sim_word = vocab.itos[indices[i] + offset]\n        print(sim_word, sims[indices[i]])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2d2b2ae0f997bc7a84bf0114fe59d6aa55c26fd0"},"cell_type":"code","source":"nearest_neighbors(vocab, np_vectors, 'we\\'ll', 10, True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fcbde52ad7b511a9d9a4ac43ce8f135b3ee47ff8"},"cell_type":"code","source":"print(np_vectors.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"57992c2366f6d3b5a62b812613556a3450cf5473"},"cell_type":"code","source":"import gc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d33f0539fe631b6dc235f606391c59f737128b68"},"cell_type":"code","source":"def to_word_indices(tokens, vocab, start_index, end_index):\n    return [[start_index] + [vocab.stoi[w] for w in sent] + [end_index] for sent in tokens]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5876433b83c342253e6de4a6d1c572b4a6832a20"},"cell_type":"code","source":"class TokenToIdDataset(torch.utils.data.Dataset):\n    def __init__(self, tokens, labels, vocab, max_size=-1, min_size=10, precompute=False, precomputed=False): #TODO should I precompute word indices?\n        self._start_index = vocab.stoi['<sos>']\n        self._end_index = vocab.stoi['<eos>']\n        self._pad_index = vocab.stoi['<pad>']\n        \n        if precompute and not precomputed:\n            tokens = to_word_indices(tokens, vocab, self._start_index, self._end_index)\n            \n        self._precomputed = precompute or precomputed\n        self._tokens = tokens\n        self._labels = labels\n        print(len(self._labels))\n        self._vocab = vocab\n        self._len = len(tokens)\n        self._max_size = max_size\n        self._min_size = min_size\n        self._cache = []\n        self._batch_size = -1\n        \n        \n    def __len__(self):\n        return self._len\n    \n    def compute_cache(self, batch_size):\n        start = 0\n        batch_num = 0\n        cache = []\n        while start < len(self):\n            batch_end = min(start + batch_size, len(self))\n            cache.append(self.collate([self[j] for j in range(start, batch_end)]))\n            start = batch_end\n        self._cache = cache\n        self._batch_size = batch_size\n            \n\n    def get_cache(self, batch_size):\n        if not self._cache or self._batch_size != batch_size:\n            self.compute_cache(batch_size)\n        return self._cache\n    \n    def collate(self, samples):\n        if self._max_size == -1:\n            size = max(self._min_size, max(len(s[0]) for s in samples))\n        else:\n            size = self._max_size\n            \n        labels_tensor = torch.tensor([s[1] for s in samples], dtype=torch.float32)\n        texts_tensor = torch.zeros(len(samples), size, dtype=torch.long)\n        texts_tensor += self._pad_index\n        for i, pair in enumerate(samples):\n            text, _ = pair\n            text = text if len(text) <= size else text[:size]\n            texts_tensor[i, size - len(text):] = torch.tensor(text)\n        \n        return texts_tensor, labels_tensor\n        \n        \n    \n    def __getitem__(self, index):\n        text = self._tokens[index]\n        if self._precomputed:\n            indices = text\n        else:\n            indices = [self._start_index]\n            indices.extend(self._vocab.stoi[tok] for tok in text)\n            indices.append(self._end_index)\n        return indices, self._labels[index]\n            ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cff8607203ba786a5891833f284c26afc19e9705"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c18fc59505230686ac32633b5fd9ea2eb30ec3a5"},"cell_type":"code","source":"min_length=10\ntrain_dataset = TokenToIdDataset(train_tokens,train_labels.values, vocab,min_size=min_length, precompute=True)\nval_dataset = TokenToIdDataset(val_tokens, val_labels.values, vocab,min_size=min_length, precompute=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5d3003db5ac84930b3b62577f580a71f67ab782f"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3483386941b65e764cd39c401dfb768b282a4045"},"cell_type":"code","source":"import gc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"953b099f515ccca039d7a70bd671771330ff21fb"},"cell_type":"code","source":"import tqdm\nfrom tqdm import tqdm_notebook","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0a11823aa5b03a8b2d73130df6e7ac8e90784049"},"cell_type":"code","source":"class BestModel:\n    def __init__(self, model_path, optimizer_path, best_loss=10000):\n        self.best_loss = best_loss\n        self.model_path = model_path\n        self.optimizer_path = optimizer_path\n        \n    def update(self, loss, model, optimizer=None):\n        self.best_loss = loss\n        torch.save(model.state_dict(), self.model_path)\n        if optimizer:\n            torch.save(optimizer.state_dict(), self.optimizer_path)\n        \n    def load(self, model, optimizer=None):\n        model.load_state_dict(torch.load(self.model_path))\n        if optimizer:\n            optimizer.load_state_dict(torch.load(self.optimizer_path))\n        model.eval()\n        \n    def copy(self, new_model_path, new_optimizer_path):\n        from shutil import copyfile\n        copyfile(self.model_path, new_model_path)\n        copyfile(self.optimizer_path, new_optimizer_path)\n        return BestModel(new_model_path, new_optimizer_path, self.best_loss)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bat = torch.tensor([\n     [[1,2,3],[4,5,6]],\n     [[11,12,13],[14,15,16]],\n     [[5,9,2],[11,3,9]],\n     [[-1, 2,10],[10,4,5]]\n])\nprint(bat.shape) # 4 2 3 \nprint(torch.matmul(bat,torch.transpose(bat,1,2)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"90e4cdb953d79705b88db58ffce27d8df3fdd4d8"},"cell_type":"code","source":"def train_network(network, optimizer,\n                  criterion,\n                  train_loader, val_loader,\n                  n_epochs, patience,\n                  best_model, after_gradient=None, lr_scheduler=None, schedule_per_batch=False):\n    attempts_left = patience\n    for epoch in range(n_epochs):\n         # Training mode\n        network.train()\n        if lr_scheduler and not schedule_per_batch:\n            lr_scheduler.step()\n        print_every = len(train_loader) // 10\n        running_train_loss = 0.0\n        batches_since_last_print = 0\n        for i, batch in tqdm_notebook(enumerate(train_loader), total=len(train_loader)):\n            if lr_scheduler and schedule_per_batch:\n                lr_scheduler.step()\n            X, y = batch\n            X, y = X.cuda(), y.cuda()\n        \n            optimizer.zero_grad()\n            \n            output = network(X)\n            loss = criterion(output, y.view(-1,1))\n            \n            running_train_loss += loss.item()\n            batches_since_last_print += 1\n            \n            # for g in optimizer.param_groups:\n                #     g['lr'] = 0.0001\n            if i % print_every == print_every - 1:\n                learning_rate = next(iter(optimizer.param_groups))['lr']\n                print(\"Epoch {}, batch {}, loss {}, lr {}\".format(epoch + 1,\n                                                                  i + 1, \n                                                                  running_train_loss / batches_since_last_print,\n                                                                 learning_rate))\n                running_train_loss = 0.0\n                batches_since_last_print = 0\n            \n            loss.backward()\n            \n            if after_gradient:\n                after_gradient(epoch, network)\n#             network.zero_embedding_grad()\n            \n            optimizer.step()\n            \n        \n        network.eval()\n        with torch.no_grad():\n            running_val_loss = 0.0\n            for i, batch in tqdm_notebook(enumerate(val_loader),total=len(val_loader)):\n                X, y = batch\n                X, y = X.cuda(), y.cuda()\n                \n                output = network(X)\n                loss = criterion(output, y.view(-1,1))\n                \n                running_val_loss += loss.item()\n                \n            running_val_loss /= len(val_loader)\n            print(\"Epoch {}, validation loss {}\".format(epoch + 1, running_val_loss))\n            \n            if running_val_loss < best_model.best_loss:\n                print('Improved from {} to {}'.format(best_model.best_loss, running_val_loss))\n                best_model.update(running_val_loss, network, optimizer)\n                attempts_left = patience\n            elif attempts_left > 0:\n                print('No improvement, attempts left = {}'.format(attempts_left))\n                attempts_left -= 1\n            else:\n                print('Early stopping, best_weight are saved')\n#                 best_model.load(network, optimizer)\n                break\n        \n#         if attempts_left != patience:\n#             best_model.load(network, optimizer)\n            ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class GenericAttention(nn.Module):\n    def __init__(self, scoring_function, query_transform, key_transform, value_transform):\n        super().__init__()\n        self.scoring_function = scoring_function\n        self.query_transform = query_transform\n        self.key_transform = key_transform\n        self.value_transform = value_transform\n        \n    def forward(self, query, keys, values):\n        query, keys, values = self.query_transform(query), self.key_transform(keys), self.value_transform(values)\n        batch_size, output_length, query_dim = query.size()\n        _, keys_length, keys_dim = keys.size()\n        \n        # B Nq Dq ^ B Nk Dk = B, Nq, Nk\n        attention_scores = self.scoring_function(query, keys) \n        \n        at_b, at_q, at_k = attention_scores.size()\n        \n        assert at_b == batch_size\n        assert at_q == output_length\n        assert at_q == keys_length\n        \n        attention_weights = F.softmax(attention_scores, dim=2)\n        \n        # B Nq, Nk X B Nk Dv = B Nq Dv \n        alignments = torch.bmm(attention_weights, values) \n        \n        return alignments, attention_weights\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class DotProductAttentionScoring(nn.Module):\n    def __init__(self, scale):\n        super().__init__()\n        self.scale = np.sqrt(scale)\n        \n    def forward(self, query, keys):\n        # B Nq Dq ^ B Nk Dk = B, Nq, Nk\n        b_q, n_q, d_q = query.size()\n        b_k, n_k, d_k = keys.size()\n        \n        assert b_q == b_k\n        \n        dot_products = torch.bmm(query, torch.transpose(keys, 1, 2)) / self.scale\n        \n        return dot_products","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TimeInvariant(nn.Module):\n    def __init__(self, inner):\n        super().__init__()\n        self.inner = inner\n        \n    def forward(self, data):\n        batch,time,dim = data.size()\n        result = self.inner(data.view(batch * time, dim))\n        result = result.view(batch, time, -1)\n        return result\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class PositionalEncoding(nn.Module):\n    \"Implement the PE function.\"\n    def __init__(self, d_model, max_len=5000):\n        super(PositionalEncoding, self).__init__()\n        \n        # Compute the positional encodings once in log space.\n        pe = torch.zeros(max_len, d_model,dtype=torch.float32)\n        position = torch.arange(0., max_len).unsqueeze(1)\n        div_term = torch.exp(torch.arange(0., d_model, 2) *\n                             -(math.log(10000.0) / d_model))\n        pe[:, 0::2] = torch.sin(position * div_term) # pos, dim\n        pe[:, 1::2] = torch.cos(position * div_term)\n        pe = pe.unsqueeze(0) # 1, pos, dim\n        self.register_buffer('pe', pe)\n        \n    def forward(self, x):\n        x = x + Variable(self.pe[:, :x.size(1)],  #x B,N,D\n                         requires_grad=False)\n        return x\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class SelfAttentionNet(nn.Module):\n    \n    def __init__(self, vocab, embeddings, num_trainable):\n        super().__init__()\n        self.pos_encoding = PositionalEncoding(50,120)\n        self.num_trainable = num_trainable\n        self.embedding = nn.Embedding(num_embeddings=len(vocab.itos), embedding_dim=300,padding_idx=vocab.stoi['<pad>'])\n        self.embedding.weight.data[num_trainable:].copy_(torch.from_numpy(embeddings))\n#         self.drop_emb = nn.AlphaDropout(p=0.5)\n        self.drop_emb = nn.Dropout2d(p=0.5)\n        self.pretrained_projection = nn.Sequential(\n            nn.Linear(300,300))\n#             nn.ReLU(),\n#             nn.Linear(300,300))\n\n\n#         mlp_transform = TimeInvariant(\n#             nn.Sequential(\n#                 nn.Linear(300, 500),\n# #                 nn.BatchNorm1d(500),\n#                 nn.ReLU(),\n#                 nn.Linear(500, 500)\n#             )\n#         )\n        \n        key_transform = TimeInvariant(\n            nn.Sequential(\n                nn.Linear(350,300),\n#                 nn.BatchNorm1d(300),\n                nn.ReLU(),\n                nn.Linear(300,300)\n            ))\n        query_transform = TimeInvariant(\n            nn.Sequential(\n                nn.Linear(350,300),\n#                 nn.BatchNorm1d(300),\n                nn.ReLU(),\n                nn.Linear(300,300)\n            ))\n        value_transform = TimeInvariant(\n            nn.Sequential(\n                nn.Linear(350,300),\n#                 nn.BatchNorm1d(300),\n                nn.ReLU(),\n                nn.Linear(300,300)\n            ))\n        # Наложить маску на padding?\n        attention_scoring = DotProductAttentionScoring(300.) # Проверить \n        self.attention1 = GenericAttention(attention_scoring, query_transform, key_transform,value_transform)\n        \n                \n        self.fc = nn.Sequential(\n            nn.Dropout(p=0.2),\n#             nn.BatchNorm1d(300),\n            nn.Linear(300,500),\n            nn.BatchNorm1d(500),\n            nn.ReLU(),\n            nn.Linear(500, 1)\n        )\n        \n    \n    def zero_embedding_grad(self):\n        self.embedding.weight.grad[self.num_trainable:] = 0\n        \n    def set_emb_dropout(self, p):\n        self.emb_dropout = p\n        \n    def apply_spatial_dropout(self,dropout_layer, x):\n        x = torch.transpose(x,1,2) # B C N\n        x = x.view(x.shape[0], x.shape[1], x.shape[2], 1) # B C N 1\n        x = dropout_layer(x)\n        x = torch.squeeze(x,dim=3)\n        \n        x = torch.transpose(x,1,2) # B N C\n        return x\n    \n    def get_attention_weights(self, x):\n        x = self.embed(x)\n        x,weights = self.attention1(x, x, x) # B N C\n        return weights\n        \n    def mixed_embedding(self, x):\n        # x = B N\n        e = self.embedding(x)\n        # B N\n        pretrained_mask = x < self.num_trainable\n        pretrained = e.view(-1,300)[pretrained_mask.view(-1)]\n        e.view(-1,300)[pretrained_mask.view(-1)] = self.pretrained_projection(pretrained)\n        return e\n        \n    def embed(self, x):\n        x = self.mixed_embedding(x) # B, N, C\n        x = self.apply_spatial_dropout(self.drop_emb, x)\n        \n        pe = torch.zeros(x.size(0), x.size(1), 50,device=x.device)\n        pe = self.pos_encoding(pe)\n        x = torch.cat((x,pe), dim=2)\n        return x\n\n\n#         x = torch.cat((x,pe), dim=2)\n#         x = self.pos_encoding(x)\n#         x = F.dropout(x, p=self.emb_dropout,training=self.training)\n\n    def forward(self, x):\n#         print('x', x.size())\n        x = self.embed(x) # B, N, C\n\n        x,_ = self.attention1(x, x, x) # B N C\n#         x = self.drop_attention1(x)\n#         x = self.attention_result_transform(x)        \n#         x = self.attention2(x, x, x)\n\n        reduced,_ = torch.max(x, dim=1)\n        return self.fc(reduced)\n        \n        \n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"15dd1a56e9232c15805e48afd083d3d1eaec1694","_kg_hide-input":false},"cell_type":"code","source":"class Net(nn.Module):\n    def __init__(self, vocab, embeddings, num_trainable, normalize=False):\n        super().__init__()\n#         self.pretrained_embedding = nn.Embedding(num_embeddings=len(vocab.itos) - num_trainable, embedding_dim=300)\n#         # Рандомная инициализвция?\n#         self.pretrained_embedding.weight.requires_grad = False\n#         self.pretrained_embedding.weight.data.copy_(torch.from_numpy(embeddings[num_trainable:]))\n        self.num_trainable = num_trainable\n\n        self.embedding = nn.Embedding(num_embeddings=len(vocab.itos), embedding_dim=300,padding_idx=vocab.stoi['<pad>'])\n        if normalize:\n            embeddings = embeddings / np.linalg.norm(embeddings,axis=1).reshape(-1,1)\n        self.embedding.weight.data[num_trainable:].copy_(torch.from_numpy(embeddings))\n        self.drop_emb = nn.Dropout2d(p=0.5)\n\n        \n        self.rnn1 = nn.GRU(input_size=300, hidden_size=150,num_layers=1,bidirectional=True)\n        self.drop_rnn = nn.Dropout(p=0.2)\n        self.rnn2 = nn.GRU(input_size=300, hidden_size=150,num_layers=1,bidirectional=True)\n        self.fc = nn.Sequential(\n            nn.Dropout(p=0.3),\n            nn.Linear(300, 1)\n        )\n        \n    def embed(self, x):\n        return self.embedding(x)\n    \n    def zero_embedding_grad(self):\n        self.embedding.weight.grad[self.num_trainable:] = 0\n    \n    def get_hidden(self, x):\n        x = self.embedding(x) # B, N, C\n        x = torch.transpose(x,1,2) # B C N\n        x = x.view(x.shape[0], x.shape[1], x.shape[2], 1) # B C N 1\n        x = self.drop_emb(x)\n        x = torch.squeeze(x,dim=3)\n        \n        \n        x = torch.transpose(x,1,2) # N B C\n        x = torch.transpose(x,0,1)\n\n        x, hidden = self.rnn1(x)\n        return x\n    \n    \n    def forward(self, x):\n        x = self.embedding(x) # B, N, C\n        x = torch.transpose(x,1,2) # B C N\n        x = x.view(x.shape[0], x.shape[1], x.shape[2], 1) # B C N 1\n        x = self.drop_emb(x)\n        x = torch.squeeze(x,dim=3)\n        \n\n        x = torch.transpose(x,1,2) # N B C\n        x = torch.transpose(x,0,1)\n#         x = self.conv_block(x)\n        \n#         x = F.max_pool1d(x,kernel_size=x.shape[-1])\n#         x = x.view(x.shape[0],-1)\n        output, hidden = self.rnn1(x) #output T, B, dim * d\n        output = self.drop_rnn(output)\n        output = output + self.rnn2(output)[0]\n        output = output.transpose(0,1) # B N C\n        x,_ = torch.max(output, dim=1)\n    \n#         x = torch.cat((output[-1, :, :self.rnn1.hidden_size],output[0, :, self.rnn1.hidden_size:]), dim=1)\n        \n#         hidden = hidden.view(self.rnn1.num_layers,2,-1,self.rnn1.hidden_size)\n        \n#         fc_input = hidden[-1] #2 B H\n#         fc_input = torch.transpose(fc_input,0,1) #B 2 H\n#         fc_input = fc_input.reshape(-1, 2 * self.rnn1.hidden_size)\n        \n        x = self.fc(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"12972a1033c369be3584898071367c34e7393f58"},"cell_type":"code","source":"num_scratch = len(specials) + len(trainable_words)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1cea16fc65e4ce50459748f949ee7217cf88aadc"},"cell_type":"code","source":"def test_shape_1():\n    net = Net(vocab, np_vectors, num_scratch).cuda()\n    net.eval()\n    summary(net,(min_length,),input_type=torch.long)\n    \ndef test_shape():\n    net = SelfAttentionNet(vocab, np_vectors, num_scratch).cuda()\n    net.eval()\n    summary(net,(min_length,),batch_size=32, input_type=torch.long)\n    \ndef test_shape2():\n    net = SelfAttentionNet(vocab, np_vectors, num_scratch).cuda()\n    \n    net.mixed_embedding()\n    summary(net,(min_length,),batch_size=32, input_type=torch.long)\n\ntest_shape_1()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"49538e42779df52eecdec76de274130c16ec7a5b"},"cell_type":"code","source":"net = Net(vocab, np_vectors, num_scratch).cuda()\nbest_model = BestModel('best_model', 'best_optimizer')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b5fb6578ccece37a5d0514a221f77352742bb14f"},"cell_type":"code","source":"criterion = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([1.])).cuda()\noptimizer = torch.optim.Adam(net.parameters(), lr=0.0003)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"efb60deb80e0cb2747d2e8ebe04415cdc8721b48"},"cell_type":"code","source":"# scheduler = CyclicLR(optimizer,base_lr=0.0001,max_lr=0.006,step_size_up=9184)\n# scheduler = torch.optim.lr_scheduler.StepLR(optimizer,step_size=6,gamma=0.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def truncate_batch(batch):\n#     print(len(batch))\n    x,y = batch\n    if x.shape[1] > 100:\n        x = x[:,:100]\n    return x,y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dc903cfce4b94123156562f7bf149229b26e0a8b"},"cell_type":"code","source":"train_loader = torch.utils.data.DataLoader(train_dataset,\n                                           batch_size=256,\n                                           collate_fn=lambda samples: truncate_batch(train_dataset.collate(samples)), shuffle=True)\nval_loader = torch.utils.data.DataLoader(val_dataset, batch_size=256,collate_fn=lambda samples: truncate_batch(val_dataset.collate(samples)))\n# train_loader = train_dataset.get_cache(256)\n# val_loader = val_dataset.get_cache(256)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(128 * 112 * 112 * 300 * 4 / 1024 / 1024)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c05ccddcaa5535b5cab451d42e3d87bb406ccc10"},"cell_type":"code","source":"train_network(net, optimizer,criterion,train_loader,val_loader,16, 3,best_model, # lr_scheduler=scheduler,\n              after_gradient=lambda epoch, network: network.zero_embedding_grad())\nbest_model.load(net, optimizer)\n\n# train_network(net, optimizer,criterion,train_loader,val_loader,16, 4,best_model,lr_scheduler=scheduler)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_network(net, optimizer,criterion,train_loader,val_loader,10, 3,best_model, # lr_scheduler=scheduler,\n              after_gradient=lambda epoch, network: network.zero_embedding_grad())\nbest_model.load(net, optimizer)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_model.load(net, optimizer)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_model_with_fixed_embeddings = best_model.copy('best_model_fixed', 'best_optimizer_fixed')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_model_with_fixed_embeddings.load(net, optimizer)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_model_with_fixed_embeddings.load(net, optimizer)\nfor g in optimizer.param_groups:\n    g['lr'] = 0.00001\ntrain_network(net, optimizer,criterion,train_loader,val_loader,10, 3,best_model_with_fixed_embeddings, # lr_scheduler=scheduler,\n              after_gradient=lambda epoch, network: network.zero_embedding_grad())\nbest_model_with_fixed_embeddings.load(net, optimizer)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_model2 = best_model_with_fixed_embeddings.copy('best_model_low', 'best_optimizer_low')\nbest_model2.load(net, optimizer)\nfor g in optimizer.param_groups:\n    g['lr'] = 0.00003\ntrain_network(net, optimizer,criterion,train_loader,val_loader,10, 3,best_model2)\nbest_model2.load(net, optimizer)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_model2.load(net, optimizer)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # net.set_emb_dropout(0.5)\ntrain_network(net, optimizer,criterion,train_loader,val_loader,10, 3,best_model)\nbest_model.load(net, optimizer)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5bd7b8f1b67872f0938b5ad077a305a03dd94bee"},"cell_type":"code","source":"# for g in optimizer.param_groups:\n#     g['lr'] = 0.0001","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ddbc43185f8dcdd1d77e07f7af7dee401f9150d2"},"cell_type":"code","source":"best_model.load(net, optimizer)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"543a3dbe2f1a4a9553ca7e385460ec5e5d730c67"},"cell_type":"code","source":"# for g in optimizer.param_groups:\n#     g['lr'] = 0.00003","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"675a5c1ae35585e62e0e06b66c5ed0e3e1f6f53f"},"cell_type":"code","source":"# train_network(net, optimizer,criterion,train_loader,val_loader,16, 3,best_model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5681e95875c333e046ece2f98e15abee03a9896b"},"cell_type":"code","source":"def predict_loader(network, loader):\n    network.eval()\n    all_predictions = []\n    with torch.no_grad():\n        for X,_ in tqdm_notebook(loader, total=len(loader)):\n            X = X.cuda()\n            out = network(X).view(-1)\n            out = torch.sigmoid(out)\n            all_predictions.extend(out.tolist())\n\n    return np.array(all_predictions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0e558c4ad39f9016045d277e303fb82dfd80e350"},"cell_type":"code","source":"network_pred = predict_loader(net, val_loader)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ade155f41a68decaf3da0dce70738e03ae101917"},"cell_type":"code","source":"print(net.embedding.weight.requires_grad)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a42a94009632559cc91cbb057138876afbd85b54"},"cell_type":"code","source":"prc = metrics.precision_recall_curve(val_labels.values, network_pred, pos_label=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"097db858df284dd97c8ebbe9631aac1189a4ead0"},"cell_type":"code","source":"import matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"73c8652bc3207f6626a66e99498887168c586d9e"},"cell_type":"code","source":"plt.xlabel('precision')\nplt.ylabel('recall')\nplt.plot(prc[0], prc[1])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7ca59635ddd87ccf6ceb9777d192c17afb8e7d90"},"cell_type":"code","source":"exact_predictions = np.round(network_pred)\nprint(metrics.classification_report(val_labels.values, exact_predictions))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"634f295305bef43c08d89ea240ffe4e27fa348b8"},"cell_type":"code","source":"def try_different_thresholds(y_true, network_pred):\n    thresholds = np.arange(0,1,0.01)\n    scores = []\n    for t in thresholds:\n        exact_pred = (network_pred > t).astype(np.int8)\n        f1_score = metrics.f1_score(y_true, exact_pred)\n#         print(t, f1_score)\n        scores.append(f1_score)\n    return thresholds, np.array(scores)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"33d6e3e6e10ef72e4f632c1c71e4bc4e7e5e8b01"},"cell_type":"code","source":"thresholds, f1_scores = try_different_thresholds(val_labels.values, network_pred)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Результат со spatial dropout"},{"metadata":{"trusted":true,"_uuid":"207eceb2368cc034de2fb38e1b037f91512752b9"},"cell_type":"code","source":"max_f1_index = np.argmax(f1_scores)\nbest_threshold = thresholds[max_f1_index]\nprint(best_threshold)\nprint(f1_scores[max_f1_index])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3dda85f2c8194f7e80d496777b22c3e3c5b954f4"},"cell_type":"code","source":"def thresholded_predictions(probs, threshold):\n    return (probs > threshold).astype(np.int8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2c92dcb839f85c7c39723f4e188159f477b62011"},"cell_type":"code","source":"exact_predictions = thresholded_predictions(network_pred, best_threshold)\nprint(metrics.classification_report(val_labels.values, exact_predictions))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1cf8eae79878f8e1636449f7beec0864aca314e8"},"cell_type":"code","source":"print(metrics.confusion_matrix(val_labels.values, exact_predictions))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def test_weights(network):\n    network.eval()\n    with torch.no_grad():\n        xx, yy = next(iter(val_loader))\n        xx, yy = xx.cuda(), yy.cuda()\n        print(xx[10], [vocab.itos[w] for w in xx[10]])\n        weights = network.get_attention_weights(xx)\n        print(weights.size())\n        return weights.cpu()\n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_attention(sent_index, attn, sentences):\n    attn = attn.numpy()\n    attn_matrix = attn[sent_index]\n    sentence = sentences[sent_index]\n    sentence = [w if w in filtered_counts else w + '(unk)' for w in sentence]\n    sentence = sentence + ['</S>']\n    limit = len(sentence)\n    ticks = np.arange(0, len(sentence))\n    fig = plt.figure(figsize=(18,18))\n    ax = fig.add_subplot(111)\n    limm = attn_matrix[-limit:,-limit:]\n    print(limm[0])\n    cax = ax.matshow(limm,\n                     vmin=limm.min(),\n                     vmax=limm.max(), \n                     cmap=plt.cm.Greys)\n    fig.colorbar(cax)\n    ax.set_xticks(ticks)\n    ax.set_yticks(ticks)\n    ax.set_xticklabels(sentence)\n    ax.set_yticklabels(sentence)\n    plt.show()\n\n    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(val_tokens[9])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"attn = test_weights(net)\nshow_attention(15,attn,val_tokens[:256])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4603e62ccdd2122313ab809b91fb94174520666f"},"cell_type":"code","source":"def print_examples(questions, labels, y_pred):\n    assert len(questions) == len(labels) == len(y_pred)\n    sample = np.random.choice(len(questions),size=1000, replace=False)\n    questions, labels, y_pred = questions[sample], labels[sample], y_pred[sample]\n    \n    positive_pred = y_pred == 1\n    negative_pred = ~positive_pred\n    positives = labels == 1\n    negatives = ~positives\n    \n    true_pos = questions[positive_pred & positives][:10]\n    false_pos = questions[positive_pred & negatives][:10]\n    true_neg = questions[negative_pred & negatives][:10]\n    false_neg = questions[negative_pred & positives][:10]\n    for name, examples in zip(('TP', 'FP', 'TN', 'FN'),(true_pos, false_pos, true_neg, false_neg)):\n        print(name,\":\")\n        for q in examples:\n            print(\"\\t\", q)\n        print()\n    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cdce2e5c8bd67adcfef54493fcd8a73ece290755"},"cell_type":"code","source":"val_questions = quora_data.iloc[list(val_labels.index)].question_text.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7be6ce5fcb2bbf5c5e93bcfca3c2ee386b3a0f0c"},"cell_type":"code","source":"print_examples(val_questions, val_labels.values, exact_predictions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"070d34e34e92fcc8453a51f662f0b00c8188be26"},"cell_type":"code","source":"print(net.embedding.weight.data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e3fe3c866bbf459e0efff8505dc0e36951cfb045"},"cell_type":"code","source":"tuned_embeddings = net.embedding.weight.data.cpu().numpy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c88473281c3846ff9b9663e69dbacf6225b6c874"},"cell_type":"code","source":"print(tuned_embeddings.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn import decomposition","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e8478e54ff8fa06bd0367d3679beee2dd2219f4c"},"cell_type":"code","source":"def get_trajectory(batch):\n    X = batch.cuda()\n    with torch.no_grad():\n        states = net.get_hidden(X)\n        probs = F.sigmoid(net(X)).view(-1).cpu().numpy()\n    print(states.shape)\n    pca = decomposition.PCA(2)\n    timesteps = states.shape[0]\n    batch_size = states.shape[1]\n    states = torch.transpose(states,0,1) #example, t, h\n    pca_points = pca.fit_transform(states.cpu().reshape(-1,192).numpy())\n    print(pca.explained_variance_)\n    return pca_points.reshape(batch_size, timesteps, 2), probs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def display(pca_points, probs, tokens, grid: tuple, offset=0):\n    total = np.prod(grid)\n    fig = plt.figure(figsize=tuple(s * 8 for s in grid))\n    xmin, ymin = pca_points.reshape(-1,2).min(axis=0)\n    xmax, ymax = pca_points.reshape(-1,2).max(axis=0)\n    pca_points = pca_points[offset:offset + total]\n    tokens = tokens[offset:offset + total]\n    probs = probs[offset:offset + total]\n    \n    for i in range(total):\n        ax = fig.add_subplot(*(grid + (i+1,)))\n#         ax.scatter(pca_points[i][:,0], pca_points[i][:,1])\n        print(pca_points[i,0,:])\n        x = pca_points[i,:,0]\n        y = pca_points[i,:,1]\n#         ax.plot(, pca_points[i][:,1],marker='o')\n#         ax.scatter(x,y)\n        classified = probs[i] > best_threshold\n        ax.quiver(x[:-1], y[:-1], x[1:]-x[:-1], y[1:]-y[:-1], scale_units='xy', angles='xy', scale=1, color=('indianred' if classified else 'black'))\n        ax.set_xlim(xmin-0.5,xmax+1.5)\n        ax.set_ylim(ymin-0.5,ymax+0.5)\n        xtext = xmax + 2\n        ytext = ymax - 0.5\n        color_seq = ['red', 'green', 'blue', 'gray', 'darkviolet', 'olive' ]\n        color_cycle = iter(itertools.cycle(color_seq))\n        sx, sy = x[-len(tokens[i])-1:], y[-len(tokens[i])-1:]\n        for j, token in enumerate(tokens[i]):\n            xx, yy = sx[j],sy[j]\n            c = next(color_cycle)\n#             ax.annotate(token, (xx,yy), (xtext,ytext), color=c, arrowprops={'arrowstyle': '-', 'color': c})\n            ax.annotate(str(j + 1) + '.' + token, (xx,yy), (xtext,ytext), color=c)\n            ax.annotate(str(j + 1), (xx,yy), color=c)\n            ytext -= 0.5\n            \n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch, _ = next(iter(val_loader))\npca, probs = get_trajectory(batch)\ntoks = val_tokens[:len(batch)]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.colors","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cdict = {'red':  ((0.0, 0.0, 0.0),\n                 (1/6., 0.0, 0.0),\n                 (1/2., 0.8, 1.0),\n                 (5/6., 1.0, 1.0),\n                 (1.0, 0.4, 1.0)),\n\n             'green':  ((0.0, 0.0, 0.4),\n                 (1/6., 1.0, 1.0),\n                 (1/2., 1.0, 0.8),\n                 (5/6., 0.0, 0.0),\n                 (1.0, 0.0, 0.0)),\n\n             'blue': ((0.0, 0.0, 0.0),\n                 (1/6., 0.0, 0.0),\n                 (1/2., 0.9, 0.9),\n                 (5/6., 0.0, 0.0),\n                 (1.0, 0.0, 0.0))\n\n    }\n\ncmap=matplotlib.colors.LinearSegmentedColormap('rg',cdict, N=256)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.scatter(pca[:,-1,0],pca[:,-1,1],c=probs, cmap=cmap)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display(pca, probs, toks, (2,2), offset=132)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8f40b37cd5d0d1f833bcc2a416780568e1cc06e6"},"cell_type":"code","source":"sel_word = 'iq'\nnearest_neighbors(vocab, np_vectors, sel_word, 10, True)\nprint('\\nafter_training\\n')\nnearest_neighbors(vocab, tuned_embeddings, sel_word, 10, False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c321f7bb66da003c666a0fd718da4de97c45ad38"},"cell_type":"code","source":"def ngram_neighbors(conv_layer, vocab, vectors, idx):\n    ngram = conv_layer.weight.data[idx].cpu().numpy()\n    ngram = ngram.T\n    sims = pairwise.cosine_similarity(ngram, vectors)\n    ranking = np.argsort(sims,axis=1)[:,::-1]\n    for i, row in enumerate(ranking):\n        print(i)\n        for j in range(10):\n            word_index = ranking[i,j]\n            sim = sims[i, word_index]\n            print(j+1, vocab.itos[word_index], sim)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"357ff690ecec9ef1cbff1da05c1028c50745073b"},"cell_type":"code","source":"# ngram_neighbors(net.conv_block_width3[0], vocab,tuned_embeddings,0)\n# ngram_neighbors(net.conv_block_width3[0], vocab,tuned_embeddings,200)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"54e33996bb7d0bb98870386aaa9171df6dc7fe50"},"cell_type":"code","source":"test_dataset = TokenToIdDataset(test_tokens,np.broadcast_to(np.zeros(1),shape=(len(test_tokens,))),vocab,min_size=25, precompute=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ed32156900d9f8293cd925f3554df593f4a08739"},"cell_type":"code","source":"test_loader = torch.utils.data.DataLoader(test_dataset,batch_size=256,collate_fn=test_dataset.collate)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3cbee2ce0e97e31e1997f8143796544840447f69"},"cell_type":"code","source":"network_test_pred = predict_loader(net, test_loader)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"29075ebfa62d0af2c687692cc4d6e050cee22252"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5ecc9088e89c3be02a6876792499663cd473e50c"},"cell_type":"code","source":"exact_test_predictions = thresholded_predictions(network_test_pred, best_threshold) #Use different thresholds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"591a01076db4e77cf177152bf18273ee94c89ba6"},"cell_type":"code","source":"submission = pd.DataFrame({'qid': quora_test_data.qid, 'prediction': exact_test_predictions})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bc10beabe405d6f5d22e2477a9031d2f1c104575"},"cell_type":"code","source":"submission.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0abdb19817a7025ee3ff87ed0a5e9ea68692e304"},"cell_type":"code","source":"print(quora_test_data.head(5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cd5d6a090e60849bec6ac72fe224f3ecce20843f"},"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7981866a157478fc204d13d70dc29d9a8e52fdf6"},"cell_type":"code","source":"with open('submission.csv') as f:\n    for line in itertools.islice(f,0,5):\n        print(line)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"739d857a1732c16e34a496da9d03d2b7be89364e"},"cell_type":"code","source":"exact_test_predictions = thresholded_predictions(network_test_pred, 0.5)\nsubmission = pd.DataFrame({'qid': quora_test_data.qid, 'prediction': exact_test_predictions})\nsubmission.to_csv('submission_05.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}