{"cells":[{"metadata":{"_uuid":"38e7d605b52588dfa82fb54def70d25e511df5bd"},"cell_type":"markdown","source":"### Inspired by:\n* https://www.kaggle.com/sudalairajkumar/a-look-at-different-embeddings\n* https://www.kaggle.com/shujian/single-rnn-with-4-folds-v1-9\n* http://mlexplained.com/2018/01/13/weight-normalization-and-layer-normalization-explained-normalization-in-deep-learning-part-2/\n* https://arxiv.org/abs/1607.06450\n* https://github.com/keras-team/keras/issues/3878\n* https://www.kaggle.com/lystdo/lstm-with-word2vec-embeddings\n* https://www.kaggle.com/jhoward/improved-lstm-baseline-glove-dropout\n* https://www.kaggle.com/aquatic/entity-embedding-neural-net\n* https://www.kaggle.com/hireme/fun-api-keras-f1-metric-cyclical-learning-rate\n* https://ai.google/research/pubs/pub46697\n* https://blog.openai.com/quantifying-generalization-in-reinforcement-learning/\n* https://www.kaggle.com/rasvob/let-s-try-clr-v3\n* https://github.com/bentrevett/pytorch-sentiment-analysis/blob/master/3%20-%20Faster%20Sentiment%20Analysis.ipynb\n* https://www.kaggle.com/ziliwang/pytorch-text-cnn\n* https://github.com/yunjey/pytorch-tutorial/blob/master/tutorials/02-intermediate/bidirectional_recurrent_neural_network/main.py\n* https://github.com/clairett/pytorch-sentiment-classification/blob/master/bilstm.py\n* https://pytorch.org/tutorials/beginner/nlp/sequence_models_tutorial.html\n\n\ntrying torch...\n\nmuch harder then keras, but feels more rewarding when done"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nnp.set_printoptions(threshold=np.nan)\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\nprint(os.listdir(\"../input/embeddings\"))\nprint(os.listdir(\"../input/embeddings/GoogleNews-vectors-negative300\"))\n\n# Any results you write to the current directory are saved as output.\n\nimport gensim\nfrom gensim.utils import simple_preprocess\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.callbacks import Callback\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report,f1_score,precision_recall_fscore_support,recall_score,precision_score\nfrom keras import backend as K\nfrom sklearn.utils import class_weight\nimport matplotlib.pyplot as plt\n\nimport tensorflow as tf\n\nSEED = 2019\n\nnp.random.seed(SEED)\n\n#https://www.kaggle.com/shujian/single-rnn-with-4-folds-v1-9\ndef threshold_search(y_true, y_proba):\n    best_threshold = 0\n    best_score = 0\n    for threshold in [i * 0.01 for i in range(100)]:\n        score = f1_score(y_true=y_true, y_pred=y_proba > threshold)\n#         print('\\rthreshold = %f | score = %f'%(threshold,score),end='')\n        if score > best_score:\n            best_threshold = threshold\n            best_score = score\n#     print('best threshold is % f with score %f'%(best_threshold,best_score))\n    search_result = {'threshold': best_threshold, 'f1': best_score}\n    return search_result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bd86c03fdf44b58e9a817f8781bc567f279e1f3e"},"cell_type":"code","source":"import torchtext\nimport random\nfrom nltk import word_tokenize\n\ntext = torchtext.data.Field(lower=True, batch_first=True, tokenize=word_tokenize, fix_length=100)\nqid = torchtext.data.Field()\ntarget = torchtext.data.Field(sequential=False, use_vocab=False, is_target=True)\ntrain_dataset = torchtext.data.TabularDataset(path='../input/train.csv', format='csv',\n                                      fields={'question_text': ('text',text),\n                                              'target': ('target',target)})\n\ntrain, val,test = train_dataset.split(split_ratio=[0.8,0.1,0.1],stratified=True,strata_field='target',random_state=random.getstate())\n\nsubmission_x = torchtext.data.TabularDataset(path='../input/test.csv', format='csv',\n                                     fields={'qid': ('qid', qid),\n                                             'question_text': ('text', text)})\n\ntext.build_vocab(train_dataset, submission_x, min_freq=3)\nqid.build_vocab(submission_x)\nprint('train dataset len:',len(train_dataset))\nprint('train len:',len(train))\nprint('val len:',len(val))\nprint('test len:',len(test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1a087a225715b0f6100d68d363c88ffd26cb275d"},"cell_type":"code","source":"file_path = '../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'\nw2v = gensim.models.KeyedVectors.load_word2vec_format(file_path,binary=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"460f9178a497fdcf5e75ad53df0a67e3accc7fee"},"cell_type":"code","source":"#src: https://github.com/Bjarten/early-stopping-pytorch/blob/master/pytorchtools.py\nimport numpy as np\nimport torch\n\nclass EarlyStopping:\n    \"\"\"Early stops the training if validation loss dosen't improve after a given patience.\"\"\"\n    def __init__(self, patience=7, verbose=False):\n        \"\"\"\n        Args:\n            patience (int): How long to wait after last time validation loss improved.\n                            Default: 7\n            verbose (bool): If True, prints a message for each validation loss improvement. \n                            Default: False\n        \"\"\"\n        self.patience = patience\n        self.verbose = verbose\n        self.counter = 0\n        self.best_score = None\n        self.early_stop = False\n        self.val_loss_min = np.Inf\n\n    def __call__(self, val_loss, model):\n\n        score = -val_loss\n\n        if self.best_score is None:\n            self.best_score = score\n            if self.verbose:\n                print(f'Validation loss decreased ({self.val_loss_min:.6f} --> {val_loss:.6f}).  Saving model ...')\n            self.save_checkpoint(val_loss, model)\n        elif score < self.best_score:\n            self.counter += 1\n            if self.verbose:\n                print(f'EarlyStopping counter: {self.counter} out of {self.patience}')\n            if self.counter >= self.patience:\n                self.early_stop = True\n        else:\n            self.best_score = score\n            self.save_checkpoint(val_loss, model)\n            self.counter = 0\n\n    def save_checkpoint(self, val_loss, model):\n        '''Saves model when validation loss decrease.'''\n        if self.verbose:\n            print(f'Validation loss decreased ({self.val_loss_min:.6f} --> {val_loss:.6f}).  Saving model ...')\n        torch.save(model.state_dict(), 'checkpoint.pt')\n        self.val_loss_min = val_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"90c151f3f6e8d359ff3fc710b2f25e9b66309559","scrolled":false},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport torch.utils.data\nimport torchtext.data\nimport warnings\nfrom sklearn.metrics import accuracy_score\nfrom torch.autograd import Variable\n\ntorch.cuda.init()\ntorch.cuda.empty_cache()\nprint('CUDA MEM:',torch.cuda.memory_allocated())\n\nprint('cuda:', torch.cuda.is_available())\nprint('cude index:',torch.cuda.current_device())\n\n\n# lr = 1e-3\n# batch_size = int(len(train_dataset)/100)\n# batch_size = int(lr*len(train))\nbatch_size = 512\nprint('batch_size:',batch_size)\nprint('---')\n\ntrain_loader = torchtext.data.BucketIterator(dataset=train,\n                                               batch_size=batch_size,\n                                               shuffle=True,\n                                               sort=False)\nval_loader = torchtext.data.BucketIterator(dataset=val,\n                                               batch_size=batch_size,\n                                               shuffle=False,\n                                               sort=False)\ntest_loader = torchtext.data.BucketIterator(dataset=test,\n                                               batch_size=batch_size,\n                                               shuffle=False,\n                                               sort=False)\n\nclass Sentiment(nn.Module):\n    \n    def __init__(self,vocab_vectors,batch_size):\n        super(Sentiment,self).__init__()\n        print('Vocab vectors size:',vocab_vectors.shape)\n        self.batch_size = batch_size\n        self.hidden_dim = 128\n        self.n_layers = 2 #bidirectional has 2 layers - forward and backward seq\n        \n        self.embedding = nn.Embedding.from_pretrained(vocab_vectors)\n        self.embedding.weight.requires_grad = False\n        \n        self.lstm = nn.LSTM(input_size=vocab_vectors.shape[1], hidden_size=self.hidden_dim, bidirectional=True,batch_first=True)        \n        self.linear1 = nn.Linear(self.n_layers*self.hidden_dim,self.hidden_dim)        \n        self.linear2 = nn.Linear(self.hidden_dim,1)\n        self.dropout = nn.Dropout(0.2)\n\n        \n    def forward(self,x):\n        #init h0,c0\n        hidden = (torch.zeros(self.n_layers, x.shape[0], self.hidden_dim).cuda(),\n                torch.zeros(self.n_layers, x.shape[0], self.hidden_dim).cuda())\n        e = self.embedding(x)\n        _, hidden = self.lstm(e, hidden)\n        out = torch.cat((hidden[0][-2,:,:], hidden[0][-1,:,:]), dim=1).cuda()\n        out = self.linear1(F.relu(out))\n        return self.linear2( self.dropout(out))\n        \nmodel = Sentiment(torch.FloatTensor(w2v.vectors), batch_size=batch_size).cuda()\nprint(model)\nprint('-'*80)\n\nearly_stopping = EarlyStopping(patience=2,verbose=True)\nloss_function = nn.BCEWithLogitsLoss().cuda()        \noptimizer = optim.Adam(model.parameters(),lr=1e-3)\n\ndef get_lr(optimizer):\n    for param_group in optimizer.param_groups:\n        return param_group['lr']\n\n    \nlosses = []\nval_losses=[]\nepoch_acc=[]\nepoch_val_acc=[]\nlrs = []\n\nfor epoch in range(100):\n#     print('-----%d-----'%epoch)\n    epoch_losses=[]\n    epoch_val_losses = []\n    preds = []\n    val_preds=[]\n    targets = []\n    acc = []\n    model.train()\n    for batch,train_batch in enumerate(list(iter(train_loader)),1):\n        optimizer.zero_grad()\n        \n        y_pred = model(train_batch.text.cuda()).squeeze(1)\n        y_numpy_pred =torch.sigmoid(y_pred).cpu().detach().numpy()\n        preds += y_numpy_pred.tolist()\n        \n        y_true = train_batch.target.float().cuda()\n        y_numpy_true = train_batch.target.cpu().detach().numpy()\n        targets += y_numpy_true.tolist()\n        loss = loss_function(y_pred,y_true)\n        epoch_losses.append(loss.item())\n\n        loss.backward()\n        optimizer.step()\n        lrs.append(get_lr(optimizer))\n        acc.append(accuracy_score(y_numpy_true,np.round(y_numpy_pred)))\n        if batch % 100 == 0:\n            print('\\rtraining (batch,loss,acc) | ',batch,' ===>',loss.item(),' acc ',np.mean(acc),end='')\n    \n    losses.append(np.mean(epoch_losses))\n    targets =  np.array(targets)\n    preds = np.array(preds)\n    search_result = threshold_search(targets, preds)\n    train_f1 = search_result['f1']\n    epoch_acc.append(np.mean(acc))\n    \n    targets = []\n    val_acc=[]\n    model.eval()\n    with torch.no_grad():\n        for batch,val_batch in enumerate(list(val_loader),1):\n            y_pred = model(val_batch.text.cuda()).squeeze(1)\n            y_numpy_pred = torch.sigmoid(y_pred).cpu().detach().numpy()\n            val_preds += y_numpy_pred.tolist()        \n            y_true = val_batch.target.float().cuda()\n            y_numpy_true = val_batch.target.cpu().detach().numpy()\n            targets += y_numpy_true.tolist()\n            val_loss = loss_function(y_pred,y_true)\n            epoch_val_losses.append(val_loss.item())\n            val_acc.append(accuracy_score(y_numpy_true,np.round(y_numpy_pred)))\n            if batch % 100 == 0:\n                print('\\rvalidation (batch,acc) | ',batch,' ===>', np.mean(val_acc),end='')\n    \n    val_losses.append(np.mean(epoch_val_losses))\n    epoch_val_acc.append(np.mean(val_acc))\n    \n    targets =  np.array(targets)\n    val_preds =  np.array(val_preds)\n    search_result = threshold_search(targets, val_preds)\n    val_f1 = search_result['f1']\n    \n    print('\\nEPOCH: ',epoch,'\\n has acc of ',epoch_acc[-1],' ,has loss of ',losses[-1], ' ,f1 of ',train_f1,'\\nval acc of ',epoch_val_acc[-1],' ,val loss of ',val_losses[-1],' ,val f1 of ',val_f1)\n    print('-'*80)\n            \n    if early_stopping.early_stop:        \n        print(\"Early stopping at \",epoch,\" epoch\")\n        break\n    else:\n        early_stopping(1.-val_f1, model)\n\n    \nprint('Training finished....')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"147165d678e7c531202dfc749273618949443bf5"},"cell_type":"code","source":"# # this was for fast text net\n# # to find out clr min and max...looks like its 0.0001 and 0.5153366336628805 - best results with big batch len(train)/100\n# # for batches with 914 (lr*len(train)), max is 0.9231538461538571 - not very good results...\n# # for lstm:\n# # 32 LSTM max is 0.5332183406113592\n# # 64 LSTM max is 0.3587205240174625\n# plt.figure(figsize=(20,10))\n# print(len(lrs))\n# print(len(acc))\n# acc = np.array(acc)\n# lrs = np.array(lrs)\n# indexes = np.argsort(lrs)\n# # print(indexes)\n# plt.plot(lrs[indexes],acc[indexes])\n# plt.xticks(np.arange(min(lrs), max(lrs), 1.0/len(lrs)*4))\n# plt.show()\n\n# max_acc_id =  np.argmax(acc)\n# print(lrs[max_acc_id])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ae403d8c4ccc8be12768d260a65d00a912c8b119"},"cell_type":"code","source":"print(os.listdir())\n\nmodel = Sentiment(torch.FloatTensor(w2v.vectors), batch_size=batch_size).cuda()\nmodel.load_state_dict(torch.load('checkpoint.pt'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d9213db94c556e3c42cdbeb741efef5c1423ae1a"},"cell_type":"code","source":"_,ax = plt.subplots(2,1,figsize=(20,10))\nax[0].plot(losses,label='loss')\nax[0].plot(val_losses,label='val_loss')\n\nax[1].plot(epoch_acc,label='acc')\nax[1].plot(epoch_val_acc,label='val_acc')\n\nplt.legend()\nplt.show()\n\npred = []\ntargets = []\nwith torch.no_grad():\n    for test_batch in list(test_loader):\n        model.eval()\n        x = test_batch.text.cuda()\n        pred += torch.sigmoid(model(x).squeeze(1)).cpu().data.numpy().tolist()\n        targets += test_batch.target.cpu().data.numpy().tolist()\n\npred = np.array(pred)\ntargets =  np.array(targets)\nsearch_result = threshold_search(targets, pred)\npred = (pred > search_result['threshold']).astype(int)\nprint('test acc:',accuracy_score(pred,targets))\nprint('test f1:',search_result['f1'])\n\nprint('RESULTS ON TEST SET:\\n',classification_report(targets,pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0ad34df4ae095e1de08c64079fa6b0ecbc944423"},"cell_type":"code","source":"print('Threshold:',search_result['threshold'])\n\nsubmission_list = list(torchtext.data.BucketIterator(dataset=submission_x,\n                                    batch_size=batch_size,\n                                    sort=False,\n                                    train=False))\npred = []\nwith torch.no_grad():\n    for submission_batch in submission_list:\n        model.eval()\n        x = submission_batch.text.cuda()\n        pred += torch.sigmoid(model(x).squeeze(1)).cpu().data.numpy().tolist()\n\npred = np.array(pred)\n\ndf_subm = pd.DataFrame()\ndf_subm['qid'] = [qid.vocab.itos[j] for i in submission_list for j in i.qid.view(-1).numpy()]\n# df_subm['prediction'] = test_meta > search_result['threshold']\ndf_subm['prediction'] = (pred > search_result['threshold']).astype(int)\nprint(df_subm.head())\ndf_subm.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}