{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"markdown","source":"### General information\n\nIn this kernel I'll work with data from Quora Insincere Questions Classification Competition.\n\nThis dataset is interesting for NLP researching. We will try to find insincere questions which aren't usefull or are even harmful. I'll do a simple EDA and try an LSTM-CNN model using Pytorch.\n\n![](https://pbs.twimg.com/profile_images/1013607595616038912/pRq_huGc_400x400.jpg)\n\nA lot of ideas for working with Pytorch are taken from this kernel: https://www.kaggle.com/leighplt/gru-model"},{"metadata":{"trusted":true,"_uuid":"9db8eb93bb22bef6425a441c5e8f16924d42eca7","_kg_hide-input":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.autograd import Variable\nimport torch.nn.functional as F\nfrom torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence\n\nimport torch\nimport torchtext\nfrom torchtext import data, vocab\nfrom tqdm import tnrange, tqdm_notebook\nfrom tqdm import tqdm as tbar\nimport spacy # lib with tokenizer\n# from nltk.tokenize.stanford import StanfordTokenizer\nfrom nltk.tokenize import TweetTokenizer\nfrom sklearn.metrics import f1_score\n\nimport numpy as np\nimport pandas as pd\n\nimport os, re, random\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\npath = \"../input\"\nemb_path = \"../input/embeddings\"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1ae63cdb314af03d8f04ddc4a74d072e654d1bce"},"cell_type":"markdown","source":"## Data overview\n\nThis is a kernel competition, where we can't use external data. As a result we can use only train and test datasets as well as embeddings which were provided by organizers."},{"metadata":{"trusted":true,"_uuid":"6382fde5d4712c78ffc939fd1956281d771abe7c"},"cell_type":"code","source":"# Load only for checking fields name and its format\ntrain = pd.read_csv('../input/train.csv')\ntest = pd.read_csv(\"../input/test.csv\")\nsub = pd.read_csv('../input/sample_submission.csv')\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3d2d8cc1cac7767b95b9493d602688579bafa751"},"cell_type":"code","source":"import os\nprint('Available embeddings:', os.listdir(\"../input/embeddings/\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0c7299c8895405ef00049595ead1ef89649ba71b"},"cell_type":"code","source":"train[\"target\"].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cdbe2595e31608b72cfbdc8d4bfc75840bfe3a0d"},"cell_type":"markdown","source":"We have a serious disbalance - only ~6% of data are positive. No wonder the metric for the competition is f1-score."},{"metadata":{"trusted":true,"_uuid":"afaa845d44d72b9997ce037ab547ab4010701311"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0817c86624cc43ea1df0eba310ba41f4f799f8da"},"cell_type":"markdown","source":"In the dataset we have only texts of questions."},{"metadata":{"trusted":true,"_uuid":"54a553b7e92a2a0a3d491ccf92b437011b813c85"},"cell_type":"code","source":"print('Average word length of questions in train is {0:.0f}.'.format(np.mean(train['question_text'].apply(lambda x: len(x.split())))))\nprint('Average word length of questions in test is {0:.0f}.'.format(np.mean(test['question_text'].apply(lambda x: len(x.split())))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7861669f72f36145c25911926a51bc688f51d473"},"cell_type":"code","source":"print('Max word length of questions in train is {0:.0f}.'.format(np.max(train['question_text'].apply(lambda x: len(x.split())))))\nprint('Max word length of questions in test is {0:.0f}.'.format(np.max(test['question_text'].apply(lambda x: len(x.split())))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1b1303137eb44cc0de3329921751c48209037562"},"cell_type":"code","source":"print('Average character length of questions in train is {0:.0f}.'.format(np.mean(train['question_text'].apply(lambda x: len(x)))))\nprint('Average character length of questions in test is {0:.0f}.'.format(np.mean(test['question_text'].apply(lambda x: len(x)))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b82f366ef8b46b0115e9940c7966849023545733"},"cell_type":"markdown","source":"As we can see on average questions in train and test datasets are similar, but there are quite long questions in train dataset."},{"metadata":{"_uuid":"2d4059747168545e407d96927eeecc532c23c9eb"},"cell_type":"markdown","source":"## Create tokenizer"},{"metadata":{"trusted":true,"_uuid":"4eedf781083b6f6f052904ddf9a9d267b87739ab"},"cell_type":"code","source":"tknzr = TweetTokenizer(strip_handles=True, reduce_len=True)\ndef tokenizer(s): \n    return tknzr.tokenize(s)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"febcbe0e9db24e65e87718e798bc2e874063a833"},"cell_type":"markdown","source":"## Load data and split to train and validation set"},{"metadata":{"trusted":true,"_uuid":"373797c35a1143bded7769c7cf08537ec666d85f"},"cell_type":"code","source":"# define the columns that we want to process and how to process\ntxt_field = data.Field(sequential=True, tokenize=tokenizer, include_lengths=True,  use_vocab=True)\nlabel_field = data.Field(sequential=False, use_vocab=False, is_target=True)\n\ntrain_fields = [\n    ('qid', None),\n    ('question_text', txt_field), # process it as text\n    ('target', label_field) # process it as label\n]\ndf_file = data.TabularDataset(path=os.path.join(path, 'train.csv'), \n                              format='csv',\n                              fields=train_fields, \n                              skip_header=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4922b2c4286da5b704e505f96acd98c3acf5e15b"},"cell_type":"code","source":"random.seed = 42\ntrain, valid = df_file.split(split_ratio=0.9, stratified=True, strata_field='target', random_state=random.getstate())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"85007feb2d7e21629ef6475c59e1233c627aac10"},"cell_type":"code","source":"# check it\nprint('Train size: {}, validation size: {}'.format(len(train), len(valid)))\nprint('Our fields in data: \\n', train.fields.items())\nexample = train[0]\nprint('First question: \\n', example.question_text)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c8abd39395c3a6adac6fc67df454a958fb6f1ca4"},"cell_type":"markdown","source":"Create dir for cache"},{"metadata":{"trusted":true,"_uuid":"302d8308f26514ad06d73963264a6fe1371b77c2"},"cell_type":"code","source":"!mkdir cache","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"38777ead0506f01df44f96a460bb9f357ef86773"},"cell_type":"code","source":"# specify the path to the localy saved vectors\nvec = vocab.Vectors(os.path.join(emb_path, 'glove.840B.300d/glove.840B.300d.txt'), cache='cache/')\n# build the vocabulary using train and validation dataset and assign the vectors\ntxt_field.build_vocab(train, valid, max_size=300000, vectors=vec)\n# build vocab for labels\nlabel_field.build_vocab(train)\n\nprint(txt_field.vocab.vectors.shape)\n# torch.Size([274124, 300])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6af311c448999bd8e8ace452425ec928ec43f55c"},"cell_type":"code","source":"# Iterator\nclass BatchGenerator:\n    def __init__(self, dl, x_field, y_field):\n        self.dl, self.x_field, self.y_field = dl, x_field, y_field\n        \n    def __len__(self):\n        return len(self.dl)\n    \n    def __iter__(self):\n        for batch in self.dl:\n            X = getattr(batch, self.x_field)\n            y = getattr(batch, self.y_field)\n            yield (X,y.float())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5adb329ea6d6a62b9d2cf125f65def51cfef3f9d"},"cell_type":"code","source":"vocab_size = len(txt_field.vocab)\nembedding_dim = 300\nn_hidden = 256\nn_out = 1\ndevice = 'cuda'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"75c2487c63f8eec22cffd48ee103891632cc5c32"},"cell_type":"code","source":"class ConcatPoolingGRUAdaptive(nn.Module):\n    def __init__(self, vocab_size, embedding_dim, n_hidden, n_out, pretrained_vec, bidirectional=True):\n        super().__init__()\n        self.vocab_size = vocab_size\n        self.embedding_dim = embedding_dim\n        self.n_hidden = n_hidden\n        self.n_out = n_out\n        self.bidirectional = bidirectional\n        \n        self.emb = nn.Embedding(self.vocab_size, self.embedding_dim)\n        self.emb.weight.data.copy_(pretrained_vec) # load pretrained vectors\n        self.emb.weight.requires_grad = False # make embedding non trainable\n        self.gru = nn.GRU(self.embedding_dim, self.n_hidden, bidirectional=bidirectional, dropout=0.5)\n        if bidirectional:\n            self.out = nn.Linear(self.n_hidden*2*2, self.n_out)\n        else:\n            self.out = nn.Linear(self.n_hidden*2, self.n_out)\n        \n    def forward(self, seq, lengths):\n        bs = seq.size(1)\n        self.h = self.init_hidden(bs)\n        seq = seq.transpose(0,1)\n        embs = self.emb(seq)\n        embs = embs.transpose(0,1)\n        embs = pack_padded_sequence(embs, lengths)\n        gru_out, self.h = self.gru(embs, self.h)\n        gru_out, lengths = pad_packed_sequence(gru_out)        \n        \n        avg_pool = F.adaptive_avg_pool1d(gru_out.permute(1,2,0),1).view(bs,-1)\n        max_pool = F.adaptive_max_pool1d(gru_out.permute(1,2,0),1).view(bs,-1)        \n        outp = self.out(torch.cat([avg_pool,max_pool],dim=1))\n        return F.sigmoid(outp)\n    \n    def init_hidden(self, batch_size): \n        if self.bidirectional:\n            return torch.zeros((2,batch_size,self.n_hidden)).to(device)\n        else:\n            return torch.zeros((1,batch_size,self.n_hidden)).to(device)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"792ac60dea4b2e2ac532da6d5ef4a7b365645bd1"},"cell_type":"code","source":"traindl, valdl = data.BucketIterator.splits(datasets=(train, valid), # specify train and validation Tabulardataset\n                                            batch_sizes=(1024,1024),  # batch size of train and validation\n                                            # on what attribute the text should be sorted\n                                            sort_key=lambda x: len(x.question_text), \n                                            device='cuda', # -1 mean cpu and 0 or None mean gpu\n                                            sort_within_batch=True, \n                                            repeat=False)\n        \ntrain_dl = BatchGenerator(traindl, 'question_text', 'target') # use the wrapper to convert Batch to data\nval_dl = BatchGenerator(valdl, 'question_text', 'target')\n\nmodel = ConcatPoolingGRUAdaptive(vocab_size, embedding_dim, n_hidden, n_out, \n                             train.fields['question_text'].vocab.vectors).to(device)\n\nopt = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), 1e-3)\nloss_fn=F.binary_cross_entropy","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4e7cf4ab14a8358b9033cf7bb8c27610710dd611"},"cell_type":"code","source":"from time import sleep\nnum_batch = len(train_dl)\nepochs=4\nfor epoch in range(epochs):      \n    y_true_train = np.empty(0)\n    y_pred_train = np.empty(0)\n    total_loss_train = 0          \n    model.train()\n    for (X,lengths),y in train_dl:\n        lengths = lengths.cpu().numpy()\n        opt.zero_grad()\n        pred = model(X, lengths)\n        loss = loss_fn(pred, y)\n        loss.backward()\n        opt.step()\n\n        y_true_train = np.concatenate([y_true_train, y.cpu().data.numpy()], axis = 0)\n        y_pred_train = np.concatenate([y_pred_train, pred.cpu().squeeze().data.numpy()], axis = 0)\n        total_loss_train += loss.item()\n\n    tacc = f1_score(y_true_train, y_pred_train>0.5)\n    tloss = total_loss_train/len(train_dl)\n\n    if val_dl:\n        model.eval()\n        y_true_val = np.empty(0)\n        y_pred_val = np.empty(0)\n        total_loss_val = 0\n        for (X,lengths),y in val_dl:\n            pred = model(X, lengths.cpu().numpy())\n            loss = loss_fn(pred, y)\n            y_true_val = np.concatenate([y_true_val, y.cpu().data.numpy()], axis = 0)\n            y_pred_val = np.concatenate([y_pred_val, pred.cpu().squeeze().data.numpy()], axis = 0)\n            total_loss_val += loss.item()\n        vacc = f1_score(y_true_val, y_pred_val>0.5)\n        vloss = total_loss_val/len(valdl)\n        print(f'Epoch {epoch}: Train loss: {tloss:.4f} F1: {tacc:.4f} | Validation loss: {vloss:.4f} F1: {vacc:.4f}')\n    else:\n        print(f'Epoch {epoch}: Train loss: {tloss:.4f} F1: {train_acc:.4f}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d7cc32ebea46b5c2bc27523a6ec71684f83906f3"},"cell_type":"code","source":"# define the columns that we want to process and how to process\nqid_field = data.RawField()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"df3827fed36ad99ddc08b19ac1a753aae3c49e2e"},"cell_type":"code","source":"test_fields = [\n    ('qid', qid_field), \n    ('question_text', txt_field), \n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"03fed58d7c4ba3b165bc77d99a0650d2bec04437"},"cell_type":"code","source":"test_df = data.TabularDataset(path=os.path.join(path, 'test.csv'), \n                           format='csv',\n                           fields=test_fields, \n                           skip_header=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2767b5bae3ecc8a43150ccda4944e858c57f37f8"},"cell_type":"code","source":"test_df.fields['qid'].is_target = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"68ff2404d4f712796c81827d48ed4979bc3bb039"},"cell_type":"code","source":"test_ld = data.BucketIterator(test_df, batch_size=512, device='cuda',\n                              sort_key=lambda x: len(x.question_text), \n                              sort_within_batch=True, \n                              repeat=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3d7c566323438a8c82199a046145c3c7f841b917"},"cell_type":"code","source":"# Iterator\nclass BatchGeneratorTest:\n    def __init__(self, dl, x_field, qid_field):\n        self.dl, self.x_field, self.qid_field = dl, x_field, qid_field\n        \n    def __len__(self):\n        return len(self.dl)\n    \n    def __iter__(self):\n        for batch in self.dl:\n            X = getattr(batch, self.x_field)\n            qid = getattr(batch, self.qid_field)\n            yield (X, qid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6a88d5a5369696b379e8e3a5940687631c42e813"},"cell_type":"code","source":"test_loader = BatchGeneratorTest(test_ld, 'question_text', 'qid')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ccc95dec581d75fe7b63bea600e5c22a4c7ccc61"},"cell_type":"code","source":"model.eval()\npreds = np.empty((0,1))\nqids = []\nfor ((z, x), y) in test_loader:\n    pred = model(z, x.cpu().numpy())\n    qids.append(y)\n    preds = np.concatenate([preds, pred.cpu().data.numpy()], axis = 0)\npreds = preds.flatten()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b73d258753703a4e9acb2099fcce2937eff14d7a"},"cell_type":"code","source":"qids = [item for sublist in qids for item in sublist]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5a4ecf0cbea9fef4b3835dfdcb470b3f844ea3cc"},"cell_type":"code","source":"!rm -r cache","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9eb70678e00f6e8b099b911f1a791a16c741286a"},"cell_type":"code","source":"submission = pd.DataFrame(data=(preds > 0.5), index=qids,dtype=np.int8, columns=['prediction'])\nsubmission.index.name = 'qid'\nsubmission.to_csv('submission.csv')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}