{"cells":[{"metadata":{"_uuid":"38e7d605b52588dfa82fb54def70d25e511df5bd"},"cell_type":"markdown","source":"### Inspired by:\n* https://www.kaggle.com/sudalairajkumar/a-look-at-different-embeddings\n* https://www.kaggle.com/shujian/single-rnn-with-4-folds-v1-9\n* http://mlexplained.com/2018/01/13/weight-normalization-and-layer-normalization-explained-normalization-in-deep-learning-part-2/\n* https://arxiv.org/abs/1607.06450\n* https://github.com/keras-team/keras/issues/3878\n* https://www.kaggle.com/lystdo/lstm-with-word2vec-embeddings\n* https://www.kaggle.com/jhoward/improved-lstm-baseline-glove-dropout\n* https://www.kaggle.com/aquatic/entity-embedding-neural-net\n* https://www.kaggle.com/hireme/fun-api-keras-f1-metric-cyclical-learning-rate\n* https://ai.google/research/pubs/pub46697\n* https://blog.openai.com/quantifying-generalization-in-reinforcement-learning/\n* https://www.kaggle.com/rasvob/let-s-try-clr-v3\n* https://github.com/bentrevett/pytorch-sentiment-analysis/blob/master/3%20-%20Faster%20Sentiment%20Analysis.ipynb\n* https://www.kaggle.com/ziliwang/pytorch-text-cnn\n* https://github.com/yunjey/pytorch-tutorial/blob/master/tutorials/02-intermediate/bidirectional_recurrent_neural_network/main.py\n* https://github.com/clairett/pytorch-sentiment-classification/blob/master/bilstm.py\n* https://pytorch.org/tutorials/beginner/nlp/sequence_models_tutorial.html\n\n\ntrying torch...\n\nmuch harder then keras, but feels more rewarding when done"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport sys\n\nnp.set_printoptions(threshold=sys.maxsize)\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\nprint(os.listdir(\"../input/embeddings\"))\nprint(os.listdir(\"../input/embeddings/GoogleNews-vectors-negative300\"))\n\n# Any results you write to the current directory are saved as output.\n\nimport gensim\nfrom gensim.utils import simple_preprocess\nfrom keras.preprocessing.sequence import pad_sequences\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report,f1_score,precision_recall_fscore_support,recall_score,precision_score\nfrom sklearn.utils import class_weight\nimport matplotlib.pyplot as plt\n\n\n#https://www.kaggle.com/shujian/single-rnn-with-4-folds-v1-9\ndef threshold_search(y_true, y_proba):\n    best_threshold = 0\n    best_score = 0\n    for threshold in [i * 0.01 for i in range(100)]:\n        score = f1_score(y_true=y_true, y_pred=y_proba > threshold)\n#         print('\\rthreshold = %f | score = %f'%(threshold,score),end='')\n        if score > best_score:\n            best_threshold = threshold\n            best_score = score\n#     print('best threshold is % f with score %f'%(best_threshold,best_score))\n    search_result = {'threshold': best_threshold, 'f1': best_score}\n    return search_result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a4bc8f068282e3f99f87f354f1b16ae799fd42bc"},"cell_type":"code","source":"import random\nimport torch\n\ndef seed_everything(seed=1234):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    \nseed_everything(6017)\nprint('Seeding done...')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bd86c03fdf44b58e9a817f8781bc567f279e1f3e"},"cell_type":"code","source":"import torchtext\nimport random\nfrom nltk import word_tokenize\n\ntext = torchtext.data.Field(lower=True, batch_first=True, tokenize=word_tokenize, fix_length=100)\nqid = torchtext.data.Field()\ntarget = torchtext.data.Field(sequential=False, use_vocab=False, is_target=True)\ntrain_dataset = torchtext.data.TabularDataset(path='../input/train.csv', format='csv',\n                                      fields={'question_text': ('text',text),\n                                              'target': ('target',target)})\n\ntrain, test = train_dataset.split(split_ratio=[0.8,0.2],stratified=True,strata_field='target',random_state=random.getstate())\n\nsubmission_x = torchtext.data.TabularDataset(path='../input/test.csv', format='csv',\n                                     fields={'qid': ('qid', qid),\n                                             'question_text': ('text', text)})\n\ntext.build_vocab(train_dataset, submission_x, min_freq=3)\nqid.build_vocab(submission_x)\nprint('train dataset len:',len(train_dataset))\nprint('train len:',len(train))\nprint('test len:',len(test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1a087a225715b0f6100d68d363c88ffd26cb275d"},"cell_type":"code","source":"glove = torchtext.vocab.Vectors('../input/embeddings/glove.840B.300d/glove.840B.300d.txt')\ntext.vocab.set_vectors(glove.stoi, glove.vectors, dim=300)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3b2ce383510333a6090ebda5b7d4843194e419dd"},"cell_type":"code","source":"import torchtext.data\n\nbatch_size = 512\nprint('batch_size:',batch_size)\nprint('---')\n\ntrain_loader = torchtext.data.BucketIterator(dataset=train,\n                                                   batch_size=batch_size,\n                                                   shuffle=True,\n                                                   sort=False)\ntest_loader = torchtext.data.BucketIterator(dataset=test,\n                                               batch_size=batch_size,\n                                               shuffle=False,\n                                               sort=False)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"90c151f3f6e8d359ff3fc710b2f25e9b66309559","scrolled":false},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport torch.utils.data\nimport warnings\nfrom sklearn.metrics import accuracy_score\nfrom torch.autograd import Variable\n\ntorch.cuda.init()\ntorch.cuda.empty_cache()\nprint('CUDA MEM:',torch.cuda.memory_allocated())\nprint('cuda:', torch.cuda.is_available())\nprint('cude index:',torch.cuda.current_device())\n\n\nclass SentimentLSTM(nn.Module):\n    \n    def __init__(self,vocab_vectors,padding_idx,batch_size):\n        super(SentimentLSTM,self).__init__()\n        print('Vocab vectors size:',vocab_vectors.shape)\n        self.batch_size = batch_size\n        self.hidden_dim = 128\n        self.n_layers = 2 #bidirectional has 2 layers - forward and backward seq\n        \n        self.embedding = nn.Embedding.from_pretrained(vocab_vectors)\n        self.embedding.weight.requires_grad = False\n        self.embedding.padding_idx = padding_idx\n        \n        self.lstm = nn.LSTM(input_size=vocab_vectors.shape[1], hidden_size=self.hidden_dim, bidirectional=True,batch_first=True)        \n        self.linear1 = nn.Linear(self.n_layers*self.hidden_dim,self.hidden_dim)        \n        self.linear2 = nn.Linear(self.hidden_dim,1)\n        self.dropout = nn.Dropout(0.2)\n\n        \n    def forward(self,x):\n        #init h0,c0\n        hidden = (torch.zeros(self.n_layers, x.shape[0], self.hidden_dim).cuda(),\n                torch.zeros(self.n_layers, x.shape[0], self.hidden_dim).cuda())\n        e = self.embedding(x)\n        _, hidden = self.lstm(e, hidden)\n        out = torch.cat((hidden[0][-2,:,:], hidden[0][-1,:,:]), dim=1).cuda()\n        out = self.linear1(out)\n        return self.linear2(self.dropout(F.relu(out)))\n    \nclass SentimentBase(nn.Module):\n    \n    def __init__(self):\n        super(SentimentBase,self).__init__()\n        \n        self.embedding = nn.Embedding(75966,300)        \n        self.linear1 = nn.Linear(300*100,128)\n        self.linear2 = nn.Linear(128,1)\n    \n    def forward(self,x):\n        emb = self.embedding(x)\n        pooled = emb.reshape((emb.shape[0],emb.shape[1]*emb.shape[2]))\n        out = self.linear1(pooled)\n        out = self.linear2(F.relu(out))\n        return out\n\n    \nclass SentimentCNN(nn.Module):\n    \n    def __init__(self,vocab_vectors,padding_idx,batch_size):\n        super(SentimentCNN,self).__init__()\n        print('Vocab vectors size:',vocab_vectors.shape)\n        self.batch_size = batch_size\n        self.hidden_dim = 128\n        \n        self.embedding = nn.Embedding.from_pretrained(vocab_vectors)\n        self.embedding.weight.requires_grad = False\n        self.embedding.padding_idx = padding_idx\n        \n        self.cnns =  nn.ModuleList([nn.Conv1d(in_channels=vocab_vectors.shape[1], out_channels=self.hidden_dim, kernel_size=k) for k in [3,4,5]])\n        \n        self.linear1 = nn.Linear(3*self.hidden_dim,self.hidden_dim)        \n        self.linear2 = nn.Linear(self.hidden_dim,1)\n        self.dropout = nn.Dropout(0.2)\n\n    @staticmethod\n    def conv_and_max_pool(x, conv):\n        \"\"\"Convolution and global max pooling layer\"\"\"\n        return F.relu(conv(x).permute(0, 2, 1).max(1)[0])\n        \n    # https://github.com/gaussic/text-classification/blob/master/cnn_pytorch.py\n    def forward(self,x):\n        e = self.embedding(x)\n         # Conv1d takes in (batch, channels, seq_len), but raw embedded is (batch, seq_len, channels)\n        e = e.permute(0,2,1)\n        cnn_outs = []\n        for conv in self.cnns:\n            f =self.conv_and_max_pool(e,conv)\n            cnn_outs.append(f)\n        out = torch.cat(cnn_outs, dim=1).cuda()\n        out = self.linear1(out)\n        return self.linear2( self.dropout(F.relu(out)))\n\n\nclass SentimentGRU(nn.Module):\n    \n    def __init__(self,vocab_vectors,padding_idx,batch_size):\n        super(SentimentGRU,self).__init__()\n        print('Vocab vectors size:',vocab_vectors.shape)\n        self.batch_size = batch_size\n        self.hidden_dim = 128\n        self.n_layers = 2 #bidirectional has 2 layers - forward and backward seq\n        \n        self.embedding = nn.Embedding.from_pretrained(vocab_vectors)\n        self.embedding.weight.requires_grad = False\n        self.embedding.padding_idx = padding_idx\n        \n        self.gru = nn.GRU(input_size=vocab_vectors.shape[1], hidden_size=self.hidden_dim, bidirectional=True,batch_first=True)        \n        self.linear1 = nn.Linear(self.n_layers*self.hidden_dim,self.hidden_dim)        \n        self.linear2 = nn.Linear(self.hidden_dim,1)\n        self.dropout = nn.Dropout(0.2)\n\n        \n    def forward(self,x):\n        #init h0,c0\n        hidden = torch.zeros(self.n_layers, x.shape[0], self.hidden_dim).cuda()\n        e = self.embedding(x)\n        _, hidden = self.gru(e, hidden)\n        out = torch.cat((hidden[-2,:,:], hidden[-1,:,:]), dim=1).cuda()\n        out = self.linear1(out)\n        return self.linear2( self.dropout(F.relu(out)))\n    \ndef train(model,filename,epochs=3):    \n    loss_function = nn.BCEWithLogitsLoss().cuda()        \n    optimizer = optim.Adam(model.parameters())\n\n    for epoch in range(epochs):\n    #     print('-----%d-----'%epoch)\n        model.train()\n        avg_loss = 0\n        for batch,train_batch in enumerate(list(iter(train_loader)),1):\n            optimizer.zero_grad()\n\n            y_pred = model(train_batch.text.cuda()).squeeze(1)\n            y_true = train_batch.target.float().cuda()\n            loss = loss_function(y_pred,y_true)\n            avg_loss += loss.item()\n            loss.backward()\n            optimizer.step()\n\n        print('EPOCH: ',epoch,': ',avg_loss/batch)\n        print('-'*80)\n\n    torch.save(model.state_dict(), filename )\n    print('Training finished....')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2521d09195ef681afc5069e32384148563d0f9a8"},"cell_type":"code","source":"model = SentimentLSTM(text.vocab.vectors, padding_idx=text.vocab.stoi[text.pad_token], batch_size=batch_size).cuda()\nprint(model)\nprint('-'*80)\ntrain(model,'lstm.pt',3)\nprint('-'*80)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"212a0ca16fb217175b37832e55825ce9a2d41e75"},"cell_type":"code","source":"model = SentimentBase().cuda()\nprint(model)\nprint('-'*80)\ntrain(model,'base.pt',5)\nprint('-'*80)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"65b1dd5def3e180baaa6fff1ccb9f64bbe06e241"},"cell_type":"code","source":"model = SentimentCNN(text.vocab.vectors, padding_idx=text.vocab.stoi[text.pad_token], batch_size=batch_size).cuda()\nprint(model)\nprint('-'*80)\ntrain(model,'cnn.pt',3)\nprint('-'*80)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1c08b4bbd3aee01b61c9940e75f02a01a2e2752b"},"cell_type":"code","source":"model = SentimentGRU(text.vocab.vectors, padding_idx=text.vocab.stoi[text.pad_token], batch_size=batch_size).cuda()\nprint(model)\nprint('-'*80)\ntrain(model,'gru.pt',3)\nprint('-'*80)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"82265cceb7529df3aeb7b7b02e208d0153e79cf5"},"cell_type":"code","source":"def disable_grad(layer):\n    for p in layer.parameters():\n        p.requires_grad=False\n\n        \nclass Ensemble(nn.Module):\n    \n    def __init__(self,vocab_vectors,padding_idx,batch_size):\n        super(Ensemble,self).__init__()\n        self.lstm = SentimentLSTM(text.vocab.vectors, padding_idx=text.vocab.stoi[text.pad_token], batch_size=batch_size).cuda()\n        self.lstm.load_state_dict(torch.load('lstm.pt'))\n        disable_grad(self.lstm)\n        \n        self.gru = SentimentGRU(text.vocab.vectors, padding_idx=text.vocab.stoi[text.pad_token], batch_size=batch_size).cuda()\n        self.gru.load_state_dict(torch.load('gru.pt'))\n        disable_grad(self.gru)\n        \n        self.cnn = SentimentCNN(text.vocab.vectors, padding_idx=text.vocab.stoi[text.pad_token], batch_size=batch_size).cuda()\n        self.cnn.load_state_dict(torch.load('cnn.pt'))\n        disable_grad(self.cnn)\n\n        self.base = SentimentBase().cuda()\n        self.base.load_state_dict(torch.load('base.pt'))\n        disable_grad(self.base)\n        \n        self.l_in = nn.Linear(4,1024,bias=False)        \n        self.l_out = nn.Linear(1024,1,bias=False)\n          \n    def forward(self,x):        \n        o1 = self.lstm(x)\n        o2 = self.gru(x)\n        o3 = self.cnn(x)\n        o4 = self.base(x)\n        out = torch.cat([o1,o2,o3,o4],1)\n        return self.l_out(F.relu(self.l_in(out)))\n\nmodel = Ensemble(text.vocab.vectors, padding_idx=text.vocab.stoi[text.pad_token], batch_size=batch_size).cuda()\nprint(model)\nprint('-'*80)\ntrain(model,'ensemble.pt', 3)\nprint('-'*80)    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"85e6897336a9bf6a430223022f5f80f210a4ccc8"},"cell_type":"code","source":"print(os.listdir())\n\nmodel = Ensemble(text.vocab.vectors, padding_idx=text.vocab.stoi[text.pad_token], batch_size=batch_size).cuda()\nmodel.load_state_dict(torch.load('ensemble.pt'))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6d383bd5e7ea144d51bf539579482b11a6a2057a"},"cell_type":"code","source":"#test f1: 0.6742109752630083 - L(4,4), Softmax, L(4,1)\n#test f1: 0.6729460720913231 - L(4,4), Softmax, matmul\n# test f1: 0.669876044969732 - L(4,1) with bias \n# test f1: 0.6795008114611898- L(4,1) w/o bias\n# test f1: 0.6735841400498521- L(4,4),Soft,L(4,1) w/o bias\n# test f1: 0.6797504254112309 -L(4,4),relu,L(4,1) w/o bias\n# test f1: 0.6649719184156075 - relu,L(4,1) w/o bias\npred = []\ntargets = []\nwith torch.no_grad():\n    for test_batch in list(test_loader):\n        model.eval()\n        x = test_batch.text.cuda()\n        pred += torch.sigmoid(model(x).squeeze(1)).cpu().data.numpy().tolist()\n        targets += test_batch.target.cpu().data.numpy().tolist()\n\npred = np.array(pred)\ntargets =  np.array(targets)\nsearch_result = threshold_search(targets, pred)\npred = (pred > search_result['threshold']).astype(int)\nprint('test acc:',accuracy_score(pred,targets))\nprint('test f1:',search_result['f1'])\n\nprint('RESULTS ON TEST SET:\\n',classification_report(targets,pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0ad34df4ae095e1de08c64079fa6b0ecbc944423"},"cell_type":"code","source":"print('Threshold:',search_result['threshold'])\n\nsubmission_list = list(torchtext.data.BucketIterator(dataset=submission_x,\n                                    batch_size=batch_size,\n                                    sort=False,\n                                    train=False))\npred = []\nwith torch.no_grad():\n    for submission_batch in submission_list:\n        model.eval()\n        x = submission_batch.text.cuda()\n        pred += torch.sigmoid(model(x).squeeze(1)).cpu().data.numpy().tolist()\n\npred = np.array(pred)\n\ndf_subm = pd.DataFrame()\ndf_subm['qid'] = [qid.vocab.itos[j] for i in submission_list for j in i.qid.view(-1).numpy()]\n# df_subm['prediction'] = test_meta > search_result['threshold']\ndf_subm['prediction'] = (pred > search_result['threshold']).astype(int)\nprint(df_subm.head())\ndf_subm.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}