{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport torch\nfrom torch import nn\nfrom sklearn.metrics import f1_score\nimport torchtext\nfrom tqdm import tqdm, tqdm_notebook\nfrom nltk import word_tokenize\nimport random\nfrom torch import optim","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"markdown","source":"### Load Corpus"},{"metadata":{"trusted":true,"_uuid":"ce0d591319978336f5dedd08a0fa17d4e94f4107"},"cell_type":"code","source":"text = torchtext.data.Field(lower=True, batch_first=True, tokenize=word_tokenize, fix_length=70)\nqid = torchtext.data.Field()\ntarget = torchtext.data.Field(sequential=False, use_vocab=False, is_target=True)\ntrain = torchtext.data.TabularDataset(path='../input/train.csv', format='csv',\n                                      fields={'question_text': ('text',text),\n                                              'target': ('target',target)})\ntest = torchtext.data.TabularDataset(path='../input/test.csv', format='csv',\n                                     fields={'qid': ('qid', qid),\n                                             'question_text': ('text', text)})","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4aeea5fe5a4fd1b345dd8750278defa380193d55"},"cell_type":"markdown","source":"### Build Vocabulary"},{"metadata":{"trusted":true,"_uuid":"cc3c4a2a8378c618dd25e0581d84a4d22ffebe3d"},"cell_type":"code","source":"text.build_vocab(train, test, min_freq=3)\nqid.build_vocab(test)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"78b2077741cc28f0125a734a12cec7df7b70e67c"},"cell_type":"markdown","source":"### Load Pretrained Language Model"},{"metadata":{"trusted":true,"_uuid":"45ba9822c8f007c72d1142383a31cab3e291e276"},"cell_type":"code","source":"glove = torchtext.vocab.Vectors('../input/embeddings/glove.840B.300d/glove.840B.300d.txt')\ntqdm_notebook().pandas() ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0fe48c681699c1d26d237af2bd1d81a470e723da"},"cell_type":"code","source":"text.vocab.set_vectors(glove.stoi, glove.vectors, dim=300)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1f53eae53fbfb26eacebbbba921e474c4fa95a79"},"cell_type":"markdown","source":"### Network"},{"metadata":{"trusted":true,"_uuid":"74c6aa5c5b8cdce5d8702bbda04f2eaa2d238060"},"cell_type":"code","source":"class TextCNN(nn.Module):\n    \n    def __init__(self, lm, padding_idx, static=True, kernel_num=128, fixed_length=50, kernel_size=[2, 5, 10], dropout=0.2):\n        super(TextCNN, self).__init__()\n        self.dropout = nn.Dropout(p=dropout)\n        self.embedding = nn.Embedding.from_pretrained(lm)\n        if static:\n            self.embedding.weight.requires_grad = False\n        self.embedding.padding_idx = padding_idx\n        self.conv = nn.ModuleList([nn.Conv2d(1, kernel_num, (i, self.embedding.embedding_dim)) for i in kernel_size])\n        self.maxpools = [nn.MaxPool2d((fixed_length+1-i,1)) for i in kernel_size]\n        self.fc = nn.Linear(len(kernel_size)*kernel_num, 1)\n        \n    def forward(self, input):\n        x = self.embedding(input).unsqueeze(1)  # B X Ci X H X W\n        x = [self.maxpools[i](torch.tanh(cov(x))).squeeze(3).squeeze(2) for i, cov in enumerate(self.conv)]  # B X Kn\n        x = torch.cat(x, dim=1)  # B X Kn * len(Kz)\n        y = self.fc(self.dropout(x))\n        return y","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"764fcf732e8b550f37e4b6f5dfff4db43f7f4900"},"cell_type":"markdown","source":"### Training Strategy"},{"metadata":{"trusted":true,"_uuid":"005321b7b00a9b35c39bfdc97c4a7c6f156c6916"},"cell_type":"code","source":"def search_best_f1(true, pred):\n    tmp = [0,0,0] # idx, cur, max\n    delta = 0\n    for tmp[0] in np.arange(0.1, 0.501, 0.01):\n        tmp[1] = f1_score(true, np.array(pred)>tmp[0])\n        if tmp[1] > tmp[2]:\n            delta = tmp[0]\n            tmp[2] = tmp[1]\n    return tmp[2], delta\n\ndef training(epoch, model, loss_func, optimizer, train_iter):\n    e = 0\n    \n    while e < epoch:\n        train_iter.init_epoch()\n        losses, preds, true = [], [], []\n        for train_batch in tqdm(list(iter(train_iter)), 'epcoh {} training'.format(e)):\n            model.train()\n            x = train_batch.text.cuda()\n            y = train_batch.target.type(torch.Tensor).cuda()\n            true.append(train_batch.target.numpy())\n            model.zero_grad()\n            pred = model.forward(x).view(-1)\n            loss = loss_function(pred, y)\n            preds.append(torch.sigmoid(pred).cpu().data.numpy())\n            losses.append(loss.cpu().data.numpy())\n            loss.backward()\n#             clip_grad_norm_(model.parameters(), 2)\n            optimizer.step()\n        train_f1, alpha_train = search_best_f1([j for i in true for j in i], [j for i in preds for j in i])\n        print('epcoh {:02} - train_loss {:.4f} - train f1 {:.4f} - delta {:.4f}'.format(\n                            e, np.mean(losses), train_f1, alpha_train))\n                \n        e += 1\n    return alpha_train\n                ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c68fc3b0d580fd83200ccf50438bc9dacc825712"},"cell_type":"markdown","source":"### Batch Set and Train/Validation Split"},{"metadata":{"trusted":true,"_uuid":"5d351b6931ee750696297017d139313cffd089a9"},"cell_type":"code","source":"random.seed(1234)\nbatch_size = 512\ntrain_iter = torchtext.data.BucketIterator(dataset=train,\n                                               batch_size=batch_size,\n                                               shuffle=True,\n                                               sort=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"579ee79800e1a2fb7ab6ad269725a21a64450d07"},"cell_type":"markdown","source":"### Network Init"},{"metadata":{"trusted":true,"_uuid":"004a427cd48fe4b013a459879966aa8b1258c75f"},"cell_type":"code","source":"def init_network(model, method='xavier', exclude='embedding', seed=123):\n    torch.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed_all(seed)\n    for name, w in model.named_parameters():\n        if not exclude in name:\n            if 'weight' in name:\n                if method is 'xavier':\n                    nn.init.xavier_normal_(w)\n                elif method is 'kaiming':\n                    nn.init.kaiming_normal_(w)\n                else:\n                    nn.init.normal_(w)\n            elif 'bias' in name:\n                nn.init.constant_(w, 0.0)\n            else: \n                pass\n\ndef print_model(model, ignore='embedding'):\n    total = 0\n    for name, w in model.named_parameters():\n        if not ignore or ignore not in name:\n            total += w.nelement()\n            print('{} : {}  {} parameters'.format(name, w.shape, w.nelement()))\n    print('-------'*4)\n    print('Total {} parameters'.format(total))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0aa75a0a051c3b9f2d186805b0349e74edd1dd43"},"cell_type":"code","source":"text.fix_length = 70\nmodel = TextCNN(text.vocab.vectors, padding_idx=text.vocab.stoi[text.pad_token], kernel_size=[1, 2, 3, 5], kernel_num=128, static=False, fixed_length=text.fix_length, dropout=0.1).cuda()\ninit_network(model)\noptimizer = optim.Adam(params=model.parameters(), lr=1e-3)\nloss_function = nn.BCEWithLogitsLoss()\nprint_model(model, ignore=None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"14a22b81393395272cc1105f5515df40f72913e0"},"cell_type":"code","source":"alpha = training(3, model, loss_function, optimizer, train_iter)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"30858cb0c5d7e2998c8a019c53e241202950e527"},"cell_type":"markdown","source":"### Predict"},{"metadata":{"trusted":true,"_uuid":"b7ad66e0c53276e061ab1f725ebfc782e9d6511d"},"cell_type":"code","source":"def predict(model, test_list):\n    pred = []\n    with torch.no_grad():\n        for test_batch in test_list:\n            model.eval()\n            x = test_batch.text.cuda()\n            pred += torch.sigmoid(model.forward(x).view(-1)).cpu().data.numpy().tolist()\n    return pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d05b4f51d49ea612709ef93a233b6baf9e29722"},"cell_type":"code","source":"test_list = list(torchtext.data.BucketIterator(dataset=test,\n                                    batch_size=batch_size,\n                                    sort=False,\n                                    train=False))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cfb6cbdbb0d2eb641895aef0855648dd7faa0035"},"cell_type":"code","source":"preds = predict(model, test_list)\nsub = pd.DataFrame()\nsub['qid'] = [qid.vocab.itos[j] for i in test_list for j in i.qid.view(-1).numpy()]\nsub['prediction'] = (preds > alpha).astype(int)\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"159791be3e54fbe3fbb9e9e6e9a2e41fd2c66d52"},"cell_type":"code","source":"sub.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}