{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# import standard numerical packages\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import f1_score\n\n# import pytorch modules\nimport torch\nimport torchtext\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim.optimizer import Optimizer","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# seed so results are reproducible\nseed = 0\ntorch.manual_seed(seed)\nif torch.cuda.is_available():\n    torch.cuda.manual_seed_all(seed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c3a17735daea764a20a60dded66ad8386489bb4e"},"cell_type":"code","source":"# read training data \ndata = pd.read_csv(\"../input/train.csv\")\n# split into validation and train sets\ndata_train, data_val = train_test_split(data, test_size=0.12, random_state=6)\n# read testing data\ndata_test = pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a2a2b9c417f67d175e447951535297557842dbf0"},"cell_type":"code","source":"# check proportion of positive examples in train and val set\nprint(np.sum(data_train.target) / len(data_train))\nprint(np.sum(data_val.target) / len(data_val))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f0374a379943cf588d71751fdb0cb6a32a29cde3"},"cell_type":"code","source":"# text preprocessing inspired from following kernel: https://www.kaggle.com/theoviel/improve-your-score-with-some-text-preprocessing/notebook\ncontraction_mapping = {\"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\", \"didn't\": \"did not\",  \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\", \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\",  \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\", \"i'd've\": \"i would have\", \"i'll\": \"i will\",  \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\", \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\", \"must've\": \"must have\", \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\", \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\"so's\": \"so as\", \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\", \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \"they'd've\": \"they would have\", \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\", \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\",  \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\", \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\", \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\", \"you're\": \"you are\", \"you've\": \"you have\" }\nmispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', \"mastrubating\": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', \"whst\": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}\nchar_mapping = {\"‘\": \"'\", \"₹\": \"e\", \"´\": \"'\", \"°\": \"\", \"€\": \"e\", \"™\": \"tm\", \"√\": \" sqrt \", \"×\": \"x\", \"²\": \"2\", \"—\": \"-\", \"–\": \"-\", \"’\": \"'\", \"_\": \"-\", \"`\": \"'\", '“': '\"', '”': '\"', '“': '\"', \"£\": \"e\", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', }    \n\ndef clean_text(x):\n    for dic in [contraction_mapping, mispell_dict, char_mapping]:\n        for word in dic.keys():\n            x = x.replace(word, dic[word])\n    return x\n\n# apply text cleaning to training, validation and test set\npd.set_option('mode.chained_assignment', None) # ignore copy on slice of DataFrame warning\ndata_train['question_text'] = data_train['question_text'].fillna(\"\").apply(lambda x: clean_text(x))\ndata_val['question_text'] = data_val['question_text'].fillna(\"\").apply(lambda x: clean_text(x))\ndata_test['question_text'] = data_test['question_text'].fillna(\"\").apply(lambda x: clean_text(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"369360a0118938e69507536988c0a69bba5b451e"},"cell_type":"code","source":"# save files to disk and remove unncessary memory afterwards\n!rm -rf split_data\n!mkdir split_data\ndata_train.to_csv('split_data/train.csv', index=False)\ndata_val.to_csv('split_data/val.csv', index=False)\ndata_test.to_csv('split_data/test.csv', index=False)\ndel data, data_train, data_val, data_test ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c92a3e5d51853b96e6f6ede8794a0be7e0511e6c"},"cell_type":"code","source":"# initialize torchtext Field objects \ntext = torchtext.data.Field(lower=True, batch_first=True, tokenize='spacy', include_lengths=True)\ntarget = torchtext.data.Field(sequential=False, use_vocab=False, is_target=True)\nqid = torchtext.data.Field()\n# use field objects to read training, validation and test sets\ntrain = torchtext.data.TabularDataset(path='split_data/train.csv', format='csv',\n                                      fields={'question_text': ('text',text),\n                                              'target': ('target',target)})\nval = torchtext.data.TabularDataset(path='split_data/val.csv', format='csv',\n                                    fields={'question_text': ('text',text),\n                                              'target': ('target',target)})\ntest = torchtext.data.TabularDataset(path='split_data/test.csv', format='csv',\n                                     fields={'qid': ('qid', qid),\n                                             'question_text': ('text',text)})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"82b366788ec39854c02bce123203ee20483f854c"},"cell_type":"code","source":"# build vocabulary object from datasets\ntext.build_vocab(train, val, test, min_freq=3)\nqid.build_vocab(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7ea5eb627b2d304211a9f5e5b86fbd78c7463ac6"},"cell_type":"code","source":"# load glove embedding into vocab object\ntext.vocab.load_vectors(torchtext.vocab.Vectors(\"../input/embeddings/glove.840B.300d/glove.840B.300d.txt\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"43f9f7d2a47951f7c52f010fca3639474fcac8bb"},"cell_type":"code","source":"print(text.vocab.vectors.shape)\nprint(f\"Unique tokens in text vocabulary: {len(text.vocab)}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1137b8c23b7a72114dab3eff7bb2331125f0479b"},"cell_type":"code","source":"# helper functions to be used later\ndef sigmoid(x):\n    return 1 / (1 + np.exp(-x))\n\ndef get_metrics(outs, y):\n    outs = sigmoid(outs.cpu().data.numpy())\n    y = y.cpu().data.numpy()\n    y_pred = (outs >= 0.5).astype(int)\n    acc = np.sum(y_pred == y) / len(y)\n    tp = np.sum((y_pred == y) & (y_pred == 1))\n    fp = np.sum((y_pred != y) & (y_pred == 1))\n    fn = np.sum((y_pred != y) & (y_pred == 0))\n    return acc, tp, fp, fn","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"78edb315250cd2cc65a606fad5970a1e5ed9e3e6"},"cell_type":"code","source":"# initialize iterators over datasets. we will use these to train our model\nbatch_size = 512\ntrain_iter = torchtext.data.BucketIterator(dataset=train,\n                                           batch_size=batch_size,\n                                           sort_key=lambda x: x.text.__len__(),\n                                           shuffle=True,\n                                           sort_within_batch=True) \nval_iter = torchtext.data.BucketIterator(dataset=val,\n                                         batch_size=batch_size,\n                                         sort_key=lambda x: x.text.__len__(),\n                                         train=False,\n                                         sort_within_batch=True)\ntest_iter = torchtext.data.BucketIterator(dataset=test,\n                                          batch_size=batch_size,\n                                          sort_key=lambda x: x.text.__len__(),\n                                          sort_within_batch=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fe72ab78407c9fb27657838a02bb5039e21b25db"},"cell_type":"code","source":"# attention layer code inspired from: https://discuss.pytorch.org/t/self-attention-on-words-and-masking/5671/4\nclass Attention(nn.Module):\n    def __init__(self, hidden_size, batch_first=False):\n        super(Attention, self).__init__()\n\n        self.hidden_size = hidden_size\n        self.batch_first = batch_first\n\n        self.att_weights = nn.Parameter(torch.Tensor(1, hidden_size), requires_grad=True)\n\n        stdv = 1.0 / np.sqrt(self.hidden_size)\n        for weight in self.att_weights:\n            nn.init.uniform_(weight, -stdv, stdv)\n\n    def get_mask(self):\n        pass\n\n    def forward(self, inputs, lengths):\n        if self.batch_first:\n            batch_size, max_len = inputs.size()[:2]\n        else:\n            max_len, batch_size = inputs.size()[:2]\n            \n        # apply attention layer\n        weights = torch.bmm(inputs,\n                            self.att_weights  # (1, hidden_size)\n                            .permute(1, 0)  # (hidden_size, 1)\n                            .unsqueeze(0)  # (1, hidden_size, 1)\n                            .repeat(batch_size, 1, 1) # (batch_size, hidden_size, 1)\n                            )\n    \n        attentions = torch.softmax(F.relu(weights.squeeze()), dim=-1)\n\n        # create mask based on the sentence lengths\n        mask = torch.ones(attentions.size(), requires_grad=True).cuda()\n        for i, l in enumerate(lengths):  # skip the first sentence\n            if l < max_len:\n                mask[i, l:] = 0\n\n        # apply mask and renormalize attention scores (weights)\n        masked = attentions * mask\n        _sums = masked.sum(-1).unsqueeze(-1)  # sums per row\n        \n        attentions = masked.div(_sums)\n\n        # apply attention weights\n        weighted = torch.mul(inputs, attentions.unsqueeze(-1).expand_as(inputs))\n\n        # get the final fixed vector representations of the sentences\n        representations = weighted.sum(1).squeeze()\n\n        return representations, attentions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"17da16991cda9e79544cd252ff23432f7f346824"},"cell_type":"code","source":"# define our own model which is an lstm followed by two dense layers\nclass MyLSTM(nn.Module):\n    def __init__(self, pretrained_lm, hidden_dim=128, lstm_layer=2, dropout=0.2):\n        super(MyLSTM, self).__init__()\n        self.dropout = nn.Dropout(p=dropout)\n        self.embedding = nn.Embedding.from_pretrained(pretrained_lm)\n        self.embedding.weight.requires_grad = False\n        self.lstm1 = nn.LSTM(input_size=self.embedding.embedding_dim,\n                            hidden_size=hidden_dim,\n                            num_layers=1, \n                            bidirectional=True)\n        self.atten1 = Attention(hidden_dim*2, batch_first=True) # 2 is bidrectional\n        self.lstm2 = nn.LSTM(input_size=hidden_dim*2,\n                            hidden_size=hidden_dim,\n                            num_layers=1, \n                            bidirectional=True)\n        self.atten2 = Attention(hidden_dim*2, batch_first=True)\n        self.fc1 = nn.Sequential(nn.Linear(hidden_dim*lstm_layer*2, hidden_dim*lstm_layer*2),\n                                 nn.BatchNorm1d(hidden_dim*lstm_layer*2),\n                                 nn.ReLU()) \n        self.fc2 = nn.Linear(hidden_dim*lstm_layer*2, 1)\n\n    \n    def forward(self, x, x_len):\n        x = self.embedding(x)\n        x = self.dropout(x)\n        \n        x = nn.utils.rnn.pack_padded_sequence(x, x_len, batch_first=True)\n        out1, (h_n, c_n) = self.lstm1(x)\n        x, lengths = nn.utils.rnn.pad_packed_sequence(out1, batch_first=True)\n        x, _ = self.atten1(x, lengths) # skip connect\n\n        out2, (h_n, c_n) = self.lstm2(out1)\n        y, lengths = nn.utils.rnn.pad_packed_sequence(out2, batch_first=True)\n        y, _ = self.atten2(y, lengths)\n        \n        z = torch.cat([x, y], dim=1)\n        z = self.fc1(self.dropout(z))\n        z = self.fc2(self.dropout(z))\n        return z","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fec0e6d201b7f51224841baa9de43bcd5c228646"},"cell_type":"code","source":"# code inspired from: https://github.com/anandsaha/pytorch.cyclic.learning.rate/blob/master/cls.py\nclass CyclicLR(object):\n    def __init__(self, optimizer, base_lr=1e-3, max_lr=6e-3,\n                 step_size=2000, mode='triangular', gamma=1.,\n                 scale_fn=None, scale_mode='cycle', last_batch_iteration=-1):\n\n        if not isinstance(optimizer, Optimizer):\n            raise TypeError('{} is not an Optimizer'.format(\n                type(optimizer).__name__))\n        self.optimizer = optimizer\n\n        if isinstance(base_lr, list) or isinstance(base_lr, tuple):\n            if len(base_lr) != len(optimizer.param_groups):\n                raise ValueError(\"expected {} base_lr, got {}\".format(\n                    len(optimizer.param_groups), len(base_lr)))\n            self.base_lrs = list(base_lr)\n        else:\n            self.base_lrs = [base_lr] * len(optimizer.param_groups)\n\n        if isinstance(max_lr, list) or isinstance(max_lr, tuple):\n            if len(max_lr) != len(optimizer.param_groups):\n                raise ValueError(\"expected {} max_lr, got {}\".format(\n                    len(optimizer.param_groups), len(max_lr)))\n            self.max_lrs = list(max_lr)\n        else:\n            self.max_lrs = [max_lr] * len(optimizer.param_groups)\n\n        self.step_size = step_size\n\n        if mode not in ['triangular', 'triangular2', 'exp_range'] \\\n                and scale_fn is None:\n            raise ValueError('mode is invalid and scale_fn is None')\n\n        self.mode = mode\n        self.gamma = gamma\n\n        if scale_fn is None:\n            if self.mode == 'triangular':\n                self.scale_fn = self._triangular_scale_fn\n                self.scale_mode = 'cycle'\n            elif self.mode == 'triangular2':\n                self.scale_fn = self._triangular2_scale_fn\n                self.scale_mode = 'cycle'\n            elif self.mode == 'exp_range':\n                self.scale_fn = self._exp_range_scale_fn\n                self.scale_mode = 'iterations'\n        else:\n            self.scale_fn = scale_fn\n            self.scale_mode = scale_mode\n\n        self.batch_step(last_batch_iteration + 1)\n        self.last_batch_iteration = last_batch_iteration\n\n    def batch_step(self, batch_iteration=None):\n        if batch_iteration is None:\n            batch_iteration = self.last_batch_iteration + 1\n        self.last_batch_iteration = batch_iteration\n        for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()):\n            param_group['lr'] = lr\n\n    def _triangular_scale_fn(self, x):\n        return 1.\n\n    def _triangular2_scale_fn(self, x):\n        return 1 / (2. ** (x - 1))\n\n    def _exp_range_scale_fn(self, x):\n        return self.gamma**(x)\n\n    def get_lr(self):\n        step_size = float(self.step_size)\n        cycle = np.floor(1 + self.last_batch_iteration / (2 * step_size))\n        x = np.abs(self.last_batch_iteration / step_size - 2 * cycle + 1)\n\n        lrs = []\n        param_lrs = zip(self.optimizer.param_groups, self.base_lrs, self.max_lrs)\n        for param_group, base_lr, max_lr in param_lrs:\n            base_height = (max_lr - base_lr) * np.maximum(0, (1 - x))\n            if self.scale_mode == 'cycle':\n                lr = base_lr + base_height * self.scale_fn(cycle)\n            else:\n                lr = base_lr + base_height * self.scale_fn(self.last_batch_iteration)\n            lrs.append(lr)\n        return lrs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"19e7ee1e0929fba3b87394e82f80af166f39845a"},"cell_type":"code","source":"# weight for positive observation is 15x negative observation\npos_weight = 15\n\n# computes validation score\ndef get_val_score(model, val_iter, loss_func):\n    epoch_loss, epoch_acc = 0, 0\n    epoch_tp, epoch_fp, epoch_fn = 0, 0, 0\n    model.eval()\n    \n    with torch.no_grad():\n        for batch in val_iter:\n            question, x_len = batch.text\n            x = question.cuda()\n            y = batch.target.type(torch.Tensor).cuda()\n            outs = model.forward(x, x_len).view(-1)\n            weight = torch.FloatTensor(np.ones(len(y))).cuda()\n            weight[(weight==1).nonzero()] = pos_weight\n            loss_func.weight = weight\n            loss = loss_func(outs, y)\n            acc, tp, fp, fn = get_metrics(outs, y)\n            \n            epoch_loss += loss.item() / len(val_iter)\n            epoch_acc += acc / len(val_iter)\n            epoch_tp += tp\n            epoch_fp += fp\n            epoch_fn += fn\n    \n    epoch_precision = epoch_tp / (epoch_tp + epoch_fp)\n    epoch_recall = epoch_tp / (epoch_tp + epoch_fn)\n    epoch_f1 = 2 * epoch_precision * epoch_recall / (epoch_precision + epoch_recall)\n    \n    return epoch_loss, epoch_acc, epoch_f1\n\n# does one epoch of training\ndef train_one_epoch(model, train_iter, val_iter, optimizer, loss_func, min_loss, scheduler, eval_every=1000):\n    epoch_loss, epoch_acc = 0, 0\n    epoch_tp, epoch_fp, epoch_fn = 0, 0, 0\n    \n    step = 0\n    # iterate over batches in training set\n    for batch in train_iter:\n        model.train()\n        # update learning rate\n        if scheduler:\n            scheduler.batch_step()\n        step += 1\n        model.zero_grad()\n        # get question and label from batch\n        question, x_len = batch.text\n        x = question.cuda()\n        y = batch.target.type(torch.Tensor).cuda()\n        # compute forward pass\n        outs = model.forward(x, x_len).view(-1)\n        # put weights on positive examples\n        weight = torch.FloatTensor(np.ones(len(y))).cuda()\n        weight[(weight==1).nonzero()] = pos_weight\n        loss_func.weight = weight\n        # compute loss function and metrics\n        loss = loss_func(outs, y)\n        acc, tp, fp, fn = get_metrics(outs, y)\n        # compute gradients wrt loss and do a step update\n        loss.backward()\n        optimizer.step()\n        # consolidate metrics for batch\n        epoch_loss += loss.item() / len(train_iter)\n        epoch_acc += acc / len(train_iter)\n        epoch_tp += tp\n        epoch_fp += fp\n        epoch_fn += fn\n        # save model if val_f1 > max_f1\n        if step % eval_every == 0:\n            val_loss, val_acc, val_f1 = get_val_score(model, val_iter, loss_func)\n            print('epoch', epoch, 'step', step, 'val_loss', val_loss, 'val_f1', val_f1, 'lr', scheduler.get_lr())\n            if val_loss < min_loss:\n                save(m=model, info={'step': step, 'epoch': epoch, 'val_loss': val_loss, 'val_f1': val_f1})\n                min_loss = val_loss\n                \n    # compute f1 score over training set\n    epoch_precision = epoch_tp / (epoch_tp + epoch_fp)\n    epoch_recall = epoch_tp / (epoch_tp + epoch_fn)\n    epoch_f1 = 2 * epoch_precision * epoch_recall / (epoch_precision + epoch_recall)\n    \n    return epoch_loss, epoch_acc, epoch_f1, min_loss\n\n# save and load model functions\ndef save(m, info):\n    torch.save(info, 'best_model.info')\n    torch.save(m, 'best_model.m')\n    \ndef load():\n    m = torch.load('best_model.m')\n    info = torch.load('best_model.info')\n    return m, info","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"73684d9221ef6b613045bce80df8046bf926c863"},"cell_type":"code","source":"# initialize model, loss function, optimizer and scheduler\nmodel = MyLSTM(text.vocab.vectors, hidden_dim=64, lstm_layer=2, dropout=0.3).cuda()\nloss_func = nn.BCEWithLogitsLoss()\nbase_lr, max_lr = 0.001, 0.003\noptimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), \n                             lr=max_lr)\nstep_size = 300\nscheduler = CyclicLR(optimizer, base_lr=base_lr, max_lr=max_lr,\n               step_size=step_size, mode='exp_range',\n               gamma=0.99994)\n# note that we always save our model at the minimum learning rate\neval_every = 2 * step_size","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e8d55dbb5d485015e4dfce3c3658e63887d229f0"},"cell_type":"code","source":"# training!\nmin_loss = float('inf')\nNUM_EPOCHS = 10\nfor epoch in range(NUM_EPOCHS):\n    train_loss, train_acc, train_f1, min_loss = train_one_epoch(model, train_iter, val_iter, optimizer, loss_func, min_loss, scheduler, eval_every)\n    print('train_loss', train_loss, 'train_acc', train_acc, 'train_f1', train_f1)\n    val_loss, val_acc, val_f1 = get_val_score(model, val_iter, loss_func)\n    print('val_loss', val_loss, 'val_acc', val_acc, 'val_f1', val_f1)\n    if val_loss < min_loss:\n        save(m=model, info={'step': 'none', 'epoch': epoch, 'val_loss': val_loss, 'val_f1': val_f1})\n        val_loss = min_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"88de53db0c28097b6ce72dcd84821e5b857bf53f"},"cell_type":"code","source":"# load best model\nmodel, m_info = load()\nm_info","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0d0057a6be0383e80a3e796302af5d280bfd2605"},"cell_type":"code","source":"# flatten parameters for evaluation\nmodel.lstm1.flatten_parameters()\nmodel.lstm2.flatten_parameters()\nmodel.eval()\nmodel.zero_grad()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fac1d8ea76aa63b636c85f2da046c411e10d7e48"},"cell_type":"code","source":"# save validation predictions \nval_preds = []\nval_labels = []\nfor batch in val_iter:\n    question, x_len = batch.text\n    x = question.cuda()\n    y = batch.target.type(torch.Tensor).cuda()\n    outs = model.forward(x, x_len).view(-1)\n    outs = sigmoid(outs.cpu().data.numpy()).tolist()\n    y = y.cpu().data.numpy().tolist()\n    val_preds += outs\n    val_labels += y\n\nval_preds = np.array(val_preds)\nval_labels = np.array(val_labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bc13ba27b3f0388b226f5825456f8533fbc20ce5"},"cell_type":"code","source":"# tune threshold of sigmoid to optimizer for f1 score\nval_scores = []\nthresholds = np.arange(0.1, 1.0, 0.001)\nfor threshold in thresholds:\n    threshold = np.round(threshold, 3)\n    f1 = f1_score(y_true=val_labels, y_pred=(val_preds > threshold).astype(int))\n    val_scores.append(f1)\n\nbest_threshold = np.argmax(val_scores)\nbest_val_f1 = np.max(val_scores)\nbest_threshold = np.round(thresholds[np.argmax(val_scores)], 3)\n\nplt.plot(thresholds, val_scores)\nprint('best_threshold', best_threshold, 'best_val_f1', best_val_f1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b544a0e47aa4ba9b521a4a6709e6371ec373fcfd"},"cell_type":"code","source":"# get test predictions\ntest_pred, test_id = [], []\nfor batch in test_iter:\n    question, x_len = batch.text\n    x = question.cuda()\n    outs = model.forward(x, x_len).view(-1) \n    outs = sigmoid(outs.cpu().data.numpy()).tolist()\n    test_pred += outs\n    test_id += batch.qid.view(-1).data.numpy().tolist()\n    \nsub_df = pd.DataFrame()\nsub_df['qid'] = [qid.vocab.itos[i] for i in test_id]\nsub_df['prediction'] = (np.array(test_pred) >= best_threshold).astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e3dc283cb6d98fefc1d3b4e8c1e16a190b4976b2"},"cell_type":"code","source":"sub_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}