{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import logging\nfrom nltk import word_tokenize\nimport pandas as pd\nimport numpy as np\nimport gc\nimport os\nimport torch\nfrom torch import nn\nfrom torch.autograd import Variable\nimport torch.nn.functional as F\nfrom torch.nn.utils.rnn import pad_sequence\nfrom sklearn.metrics import f1_score\nfrom torch import optim\nimport torchtext\nimport random","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"724bac94e97471bcc8bb5e6a9cfa45b1703ffa83"},"cell_type":"markdown","source":"## Build Vocabulary"},{"metadata":{"trusted":true,"_uuid":"6a6c212befce20c740297f5344facc5cdcd458bd","scrolled":false},"cell_type":"code","source":"text = torchtext.data.Field(lower=True, batch_first=True, tokenize=word_tokenize)\nqid = torchtext.data.Field()\ntarget = torchtext.data.Field(sequential=False, use_vocab=False, is_target=True)\ntrain = torchtext.data.TabularDataset(path='../input/train.csv', format='csv',\n                                      fields={'question_text': ('text',text),\n                                              'target': ('target',target)})\ntest = torchtext.data.TabularDataset(path='../input/test.csv', format='csv',\n                                     fields={'qid': ('qid', qid),\n                                             'question_text': ('text', text)})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"d852a81de412efdc17b589c6e8f2aa92140a06fe"},"cell_type":"code","source":"text.build_vocab(train, test, min_freq=3)\nqid.build_vocab(test)\ntext.vocab.load_vectors(torchtext.vocab.Vectors('../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'))\nprint(text.vocab.vectors.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2f387578566d4bbdb13e4b59221ec38b74f571e4"},"cell_type":"markdown","source":"## train/validate split"},{"metadata":{"trusted":true,"_uuid":"bd14753163254f9a00da6dec354476616c7f323e"},"cell_type":"code","source":"random.seed(2018)\ntrain, val = train.split(split_ratio=0.9, random_state=random.getstate())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4d2a26070e3b05aa477584c1f123f81fefa714d2"},"cell_type":"markdown","source":"## Model Define"},{"metadata":{"trusted":true,"_uuid":"288389faf86fa843c3503b058e6bb3bd03a5cc17"},"cell_type":"code","source":"class BiLSTM(nn.Module):\n    def __init__(self, pretrained_lm, padding_idx, static=True, hidden_dim=128, lstm_layer=2, dropout=0.2):\n        super(BiLSTM, self).__init__()\n        self.hidden_dim = hidden_dim\n        self.dropout = nn.Dropout(p=dropout)\n        self.embedding = nn.Embedding.from_pretrained(pretrained_lm)\n        self.embedding.padding_idx = padding_idx\n        if static:\n            self.embedding.weight.requires_grad = False\n        self.lstm = nn.LSTM(input_size=self.embedding.embedding_dim,\n                            hidden_size=hidden_dim,\n                            num_layers=lstm_layer, \n                            dropout = dropout,\n                            bidirectional=True)\n        self.hidden2label = nn.Linear(hidden_dim*lstm_layer*2, 1)\n    \n    def forward(self, sents):\n        x = self.embedding(sents)\n        x = torch.transpose(x, dim0=1, dim1=0)\n        lstm_out, (h_n, c_n) = self.lstm(x)\n        y = self.hidden2label(self.dropout(torch.cat([c_n[i,:, :] for i in range(c_n.shape[0])], dim=1)))\n        return y","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"da319a154a240c2c12bf33e3e359975bf40870df"},"cell_type":"markdown","source":"## Training"},{"metadata":{"trusted":true,"_uuid":"e04ce6e17a5da95b29ed5684429f9246fdf9b7dc","scrolled":true},"cell_type":"code","source":"def training(epoch, model, eval_every, loss_func, optimizer, train_iter, val_iter, early_stop=1, warmup_epoch=2):\n    \n    step = 0\n    max_loss = 1e5\n    no_improve_epoch = 0\n    no_improve_in_previous_epoch = False\n    fine_tuning = False\n    train_record = []\n    val_record = []\n    losses = []\n    \n    for e in range(epoch):\n        if e >= warmup_epoch:\n            if no_improve_in_previous_epoch:\n                no_improve_epoch += 1\n                if no_improve_epoch >= early_stop:\n                    break\n            else:\n                no_improve_epoch = 0\n            no_improve_in_previous_epoch = True\n        if not fine_tuning and e >= warmup_epoch:\n            model.embedding.weight.requires_grad = True\n            fine_tuning = True\n        train_iter.init_epoch()\n        for train_batch in iter(train_iter):\n            step += 1\n            model.train()\n            x = train_batch.text.cuda()\n            y = train_batch.target.type(torch.Tensor).cuda()\n            model.zero_grad()\n            pred = model.forward(x).view(-1)\n            loss = loss_function(pred, y)\n            losses.append(loss.cpu().data.numpy())\n            train_record.append(loss.cpu().data.numpy())\n            loss.backward()\n            optimizer.step()\n            if step % eval_every == 0:\n                model.eval()\n                model.zero_grad()\n                val_loss = []\n                for val_batch in iter(val_iter):\n                    val_x = val_batch.text.cuda()\n                    val_y = val_batch.target.type(torch.Tensor).cuda()\n                    val_pred = model.forward(val_x).view(-1)\n                    val_loss.append(loss_function(val_pred, val_y).cpu().data.numpy())\n                val_record.append({'step': step, 'loss': np.mean(val_loss)})\n                print('epcoh {:02} - step {:06} - train_loss {:.4f} - val_loss {:.4f} '.format(\n                            e, step, np.mean(losses), val_record[-1]['loss']))\n                if e >= warmup_epoch:\n                    if val_record[-1]['loss'] <= max_loss:\n                        save(m=model, info={'step': step, 'epoch': e, 'train_loss': np.mean(losses),\n                                            'val_loss': val_record[-1]['loss']})\n                        max_loss = val_record[-1]['loss']\n                        no_improve_in_previous_epoch = False\n    \n\ndef save(m, info):\n    torch.save(info, 'best_model.info')\n    torch.save(m, 'best_model.m')\n    \ndef load():\n    m = torch.load('best_model.m')\n    info = torch.load('best_model.info')\n    return m, info\n\n                ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"606f03eb93e4fdf886124a14b9dd7b58872bcfc0"},"cell_type":"markdown","source":"## model train"},{"metadata":{"trusted":true,"_uuid":"0c43318d20d81ddb49b0c33815559183dfcf6767"},"cell_type":"code","source":"batch_size = 128\ntrain_iter = torchtext.data.BucketIterator(dataset=train,\n                                               batch_size=batch_size,\n                                               sort_key=lambda x: x.text.__len__(),\n                                               shuffle=True,\n                                               sort=False)\nval_iter = torchtext.data.BucketIterator(dataset=val,\n                                             batch_size=batch_size,\n                                             sort_key=lambda x: x.text.__len__(),\n                                             train=False,\n                                             sort=False)\nmodel = BiLSTM(text.vocab.vectors, lstm_layer=2, padding_idx=text.vocab.stoi[text.pad_token], hidden_dim=128).cuda()\n# loss_function = nn.BCEWithLogitsLoss(pos_weight=torch.Tensor([pos_w]).cuda())\nloss_function = nn.BCEWithLogitsLoss()\noptimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()),\n                    lr=1e-3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b6b4abd4c2d7ad47f9d42e09bab7bfe11892822b"},"cell_type":"code","source":"training(model=model, epoch=20, eval_every=500,\n         loss_func=loss_function, optimizer=optimizer, train_iter=train_iter,\n        val_iter=val_iter, warmup_epoch=3, early_stop=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"11befd2f07f62c247addfe9ea9c9f2f68a400913"},"cell_type":"code","source":"model, m_info = load()\nm_info","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"66c6e71ae5f244bf57bfc66e678fec8e361d640a"},"cell_type":"code","source":"model.lstm.flatten_parameters()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"461ed33c5143c5f70c21ff8607393968b67a3487"},"cell_type":"markdown","source":"##  Prediction"},{"metadata":{"trusted":true,"_uuid":"cef10bf9f0c90bb2ea65c57bb7d912b1e97f4557"},"cell_type":"code","source":"model.eval()\nval_pred = []\nval_true = []\nval_iter.init_epoch()\nfor val_batch in iter(val_iter):\n    val_x = val_batch.text.cuda()\n    val_true += val_batch.target.data.numpy().tolist()\n    val_pred += torch.sigmoid(model.forward(val_x).view(-1)).cpu().data.numpy().tolist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8a87d01c5a560c8704c7462e0b110ecb628af123","scrolled":true},"cell_type":"code","source":"tmp = [0,0,0] # idx, cur, max\ndelta = 0\nfor tmp[0] in np.arange(0.1, 0.501, 0.01):\n    tmp[1] = f1_score(val_true, np.array(val_pred)>tmp[0])\n    if tmp[1] > tmp[2]:\n        delta = tmp[0]\n        tmp[2] = tmp[1]\nprint('best threshold is {:.4f} with F1 score: {:.4f}'.format(delta, tmp[2]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d8e24de0991105929980df7662d201b967cc104","_kg_hide-output":true},"cell_type":"code","source":"model.eval()\nmodel.zero_grad()\ntest_iter = torchtext.data.BucketIterator(dataset=test,\n                                    batch_size=batch_size,\n                                    sort_key=lambda x: x.text.__len__(),\n                                    sort=True)\ntest_pred = []\ntest_id = []\n\nfor test_batch in iter(test_iter):\n    test_x = test_batch.text.cuda()\n    test_pred += torch.sigmoid(model.forward(test_x).view(-1)).cpu().data.numpy().tolist()\n    test_id += test_batch.qid.view(-1).data.numpy().tolist()\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4bbb86d4637d1fa8b0d4bfc78f4ea43b15541591"},"cell_type":"code","source":"sub_df =pd.DataFrame()\nsub_df['qid'] = [qid.vocab.itos[i] for i in test_id]\nsub_df['prediction'] = (np.array(test_pred) >= delta).astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7bd9dbb2df8ba1e5ff2151c68a8a1b71497d96a1"},"cell_type":"code","source":"sub_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}