{"cells":[{"metadata":{"_uuid":"38e7d605b52588dfa82fb54def70d25e511df5bd"},"cell_type":"markdown","source":"### Inspired by:\n* https://www.kaggle.com/sudalairajkumar/a-look-at-different-embeddings\n* https://www.kaggle.com/shujian/single-rnn-with-4-folds-v1-9\n* http://mlexplained.com/2018/01/13/weight-normalization-and-layer-normalization-explained-normalization-in-deep-learning-part-2/\n* https://arxiv.org/abs/1607.06450\n* https://github.com/keras-team/keras/issues/3878\n* https://www.kaggle.com/lystdo/lstm-with-word2vec-embeddings\n* https://www.kaggle.com/jhoward/improved-lstm-baseline-glove-dropout\n* https://www.kaggle.com/aquatic/entity-embedding-neural-net\n* https://www.kaggle.com/hireme/fun-api-keras-f1-metric-cyclical-learning-rate\n* https://ai.google/research/pubs/pub46697\n* https://blog.openai.com/quantifying-generalization-in-reinforcement-learning/\n* https://www.kaggle.com/rasvob/let-s-try-clr-v3\n* https://github.com/bentrevett/pytorch-sentiment-analysis/blob/master/3%20-%20Faster%20Sentiment%20Analysis.ipynb\n* https://www.kaggle.com/ziliwang/pytorch-text-cnn\n\n\ntrying torch...\n\nclr + adam is actually pretty awesome"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nnp.set_printoptions(threshold=np.nan)\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\nprint(os.listdir(\"../input/embeddings\"))\nprint(os.listdir(\"../input/embeddings/GoogleNews-vectors-negative300\"))\n\n# Any results you write to the current directory are saved as output.\n\nimport gensim\nfrom gensim.utils import simple_preprocess\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.callbacks import Callback\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report,f1_score,precision_recall_fscore_support,recall_score,precision_score\nfrom keras import backend as K\nfrom sklearn.utils import class_weight\nimport matplotlib.pyplot as plt\n\nimport tensorflow as tf\n\nSEED = 2019\n\nnp.random.seed(SEED)\n\n#https://www.kaggle.com/shujian/single-rnn-with-4-folds-v1-9\ndef threshold_search(y_true, y_proba):\n    best_threshold = 0\n    best_score = 0\n    for threshold in [i * 0.01 for i in range(100)]:\n        score = f1_score(y_true=y_true, y_pred=y_proba > threshold)\n#         print('\\rthreshold = %f | score = %f'%(threshold,score),end='')\n        if score > best_score:\n            best_threshold = threshold\n            best_score = score\n#     print('best threshold is % f with score %f'%(best_threshold,best_score))\n    search_result = {'threshold': best_threshold, 'f1': best_score}\n    return search_result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bd86c03fdf44b58e9a817f8781bc567f279e1f3e"},"cell_type":"code","source":"import torchtext\nimport random\nfrom nltk import word_tokenize\n\ntext = torchtext.data.Field(lower=True, batch_first=True, tokenize=word_tokenize, fix_length=100)\nqid = torchtext.data.Field()\ntarget = torchtext.data.Field(sequential=False, use_vocab=False, is_target=True)\ntrain_dataset = torchtext.data.TabularDataset(path='../input/train.csv', format='csv',\n                                      fields={'question_text': ('text',text),\n                                              'target': ('target',target)})\n\ntrain, val,test = train_dataset.split(split_ratio=[0.7,0.3,0.3],stratified=True,strata_field='target',random_state=random.getstate())\n\nsubmission_x = torchtext.data.TabularDataset(path='../input/test.csv', format='csv',\n                                     fields={'qid': ('qid', qid),\n                                             'question_text': ('text', text)})\n\ntext.build_vocab(train_dataset, submission_x, min_freq=3)\nqid.build_vocab(submission_x)\nprint('train dataset len:',len(train_dataset))\nprint('train len:',len(train))\nprint('val len:',len(val))\nprint('test len:',len(test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1a087a225715b0f6100d68d363c88ffd26cb275d"},"cell_type":"code","source":"glove = torchtext.vocab.Vectors('../input/embeddings/glove.840B.300d/glove.840B.300d.txt')\ntext.vocab.set_vectors(glove.stoi, glove.vectors, dim=300)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"460f9178a497fdcf5e75ad53df0a67e3accc7fee"},"cell_type":"code","source":"#src: https://github.com/Bjarten/early-stopping-pytorch/blob/master/pytorchtools.py\nimport numpy as np\nimport torch\n\nclass EarlyStopping:\n    \"\"\"Early stops the training if validation loss dosen't improve after a given patience.\"\"\"\n    def __init__(self, patience=7, verbose=False):\n        \"\"\"\n        Args:\n            patience (int): How long to wait after last time validation loss improved.\n                            Default: 7\n            verbose (bool): If True, prints a message for each validation loss improvement. \n                            Default: False\n        \"\"\"\n        self.patience = patience\n        self.verbose = verbose\n        self.counter = 0\n        self.best_score = None\n        self.early_stop = False\n        self.val_loss_min = np.Inf\n\n    def __call__(self, val_loss, model):\n\n        score = -val_loss\n\n        if self.best_score is None:\n            self.best_score = score\n            if self.verbose:\n                print(f'Validation loss decreased ({self.val_loss_min:.6f} --> {val_loss:.6f}).  Saving model ...')\n            self.save_checkpoint(val_loss, model)\n        elif score < self.best_score:\n            self.counter += 1\n            if self.verbose:\n                print(f'EarlyStopping counter: {self.counter} out of {self.patience}')\n            if self.counter >= self.patience:\n                self.early_stop = True\n        else:\n            self.best_score = score\n            self.save_checkpoint(val_loss, model)\n            self.counter = 0\n\n    def save_checkpoint(self, val_loss, model):\n        '''Saves model when validation loss decrease.'''\n        if self.verbose:\n            print(f'Validation loss decreased ({self.val_loss_min:.6f} --> {val_loss:.6f}).  Saving model ...')\n        torch.save(model.state_dict(), 'checkpoint.pt')\n        self.val_loss_min = val_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9799fb98c686b267cf62ac4c7b52586b0f4b3f87"},"cell_type":"code","source":"#src:https://github.com/pytorch/pytorch/pull/2016\ndef cyclical_lr(step_sz, min_lr=0.001, max_lr=1, mode='triangular', scale_func=None, scale_md='cycles', gamma=1.):\n    \"\"\"implements a cyclical learning rate policy (CLR).\n    Notes: the learning rate of optimizer should be 1\n\n    Parameters:\n    ----------\n    mode : str, optional\n        one of {triangular, triangular2, exp_range}. \n    scale_md : str, optional\n        {'cycles', 'iterations'}.\n    gamma : float, optional\n        constant in 'exp_range' scaling function: gamma**(cycle iterations)\n    \n    Examples:\n    --------\n    >>> # the learning rate of optimizer should be 1\n    >>> optimizer = optim.SGD(model.parameters(), lr=1.)\n    >>> step_size = 2*len(train_loader)\n    >>> clr = cyclical_lr(step_size, min_lr=0.001, max_lr=0.005)\n    >>> scheduler = lr_scheduler.LambdaLR(optimizer, [clr])\n    >>> # some other operations\n    >>> scheduler.step()\n    >>> optimizer.step()\n    \"\"\"\n    if scale_func == None:\n        if mode == 'triangular':\n            scale_fn = lambda x: 1.\n            scale_mode = 'cycles'\n        elif mode == 'triangular2':\n            scale_fn = lambda x: 1 / (2.**(x - 1))\n            scale_mode = 'cycles'\n        elif mode == 'exp_range':\n            scale_fn = lambda x: gamma**(x)\n            scale_mode = 'iterations'\n        else:\n            raise ValueError(f'The {mode} is not valid value!')\n    else:\n        scale_fn = scale_func\n        scale_mode = scale_md\n\n    lr_lambda = lambda iters: min_lr + (max_lr - min_lr) * rel_val(iters, step_sz, scale_mode)\n\n    def rel_val(iteration, stepsize, mode):\n        cycle = np.floor(1 + iteration / (2 * stepsize))\n        x = np.abs(iteration / stepsize - 2 * cycle + 1)\n        if mode == 'cycles':\n            return max(0, (1 - x)) * scale_fn(cycle)\n        elif mode == 'iterations':\n            return max(0, (1 - x)) * scale_fn(iteration)\n        else:\n            raise ValueError(f'The {scale_mode} is not valid value!')\n\n    return lr_lambda","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"90c151f3f6e8d359ff3fc710b2f25e9b66309559","scrolled":false},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport torch.utils.data\nimport torchtext.data\nimport warnings\nfrom sklearn.metrics import accuracy_score\n\ntorch.cuda.init()\ntorch.cuda.empty_cache()\nprint('CUDA MEM:',torch.cuda.memory_allocated())\n\nprint('cuda:', torch.cuda.is_available())\nprint('cude index:',torch.cuda.current_device())\n\nbatch_size = int(len(train_dataset)/100)\n\n# lr = 1e-3\n# batch_size = int(lr*len(train))\n# batch_size = 64\nprint('batch_size:',batch_size)\nprint('---')\n\ntrain_loader = torchtext.data.BucketIterator(dataset=train,\n                                               batch_size=batch_size,\n                                               shuffle=True,\n                                               sort=False)\nval_loader = torchtext.data.BucketIterator(dataset=val,\n                                               batch_size=batch_size,\n                                               shuffle=False,\n                                               sort=False)\ntest_loader = torchtext.data.BucketIterator(dataset=test,\n                                               batch_size=batch_size,\n                                               shuffle=False,\n                                               sort=False)\n\nclass Sentiment(nn.Module):\n    \n    def __init__(self,vocab_vectors,padding_idx):\n        super(Sentiment,self).__init__()\n        print('Vocab vectors size:',vocab_vectors.shape)\n        \n        self.embedding = nn.Embedding.from_pretrained(vocab_vectors)\n        self.embedding.weight.requires_grad = True\n        self.embedding.padding_idx = padding_idx\n        \n        self.linear2 = nn.Linear(300,1)\n    \n    def forward(self,x):\n        x = x.permute(1,0)\n#         print('input#[sent len, batch size]:',x.shape)\n        emb = self.embedding(x)\n#         print('emb out#[sent len, batch size, emb dim]:',emb.shape)\n        emb = emb.permute(1, 0, 2)\n#         print('emb out#[batch size, sent len, emb dim]:',emb.shape)\n        pooled = F.avg_pool2d(emb, (emb.shape[1], 1)).squeeze(1) \n#         print('pooled#[batch size, embedding_dim]:',pooled.shape)\n        out = self.linear2(pooled)\n#         print('linear2 out:',out.shape)        \n        return out\n        \nmodel = Sentiment(text.vocab.vectors, padding_idx=text.vocab.stoi[text.pad_token]).cuda()\nprint(model)\nprint('-'*80)\n\nearly_stopping = EarlyStopping(patience=2,verbose=True)\nloss_function = nn.BCEWithLogitsLoss().cuda()        \noptimizer = optim.Adam(model.parameters(),lr=1.)\n\nstep_size = len(train_loader)/batch_size\nclr = cyclical_lr(step_size, min_lr=0.001, max_lr=0.5153366336628805)\nscheduler = optim.lr_scheduler.LambdaLR(optimizer, [clr])\n\ndef get_lr(optimizer):\n    for param_group in optimizer.param_groups:\n        return param_group['lr']\n\n    \nlosses = []\nval_losses=[]\nepoch_acc=[]\nepoch_val_acc=[]\nlrs = []\n\nfor epoch in range(100):\n#     print('-----%d-----'%epoch)\n    epoch_losses=[]\n    epoch_val_losses = []\n    preds = []\n    val_preds=[]\n    targets = []\n    acc = []\n    model.train()\n    for batch,train_batch in enumerate(list(iter(train_loader)),1):\n        optimizer.zero_grad()\n        y_pred = model(train_batch.text.cuda()).squeeze(1)\n        y_numpy_pred =torch.sigmoid(y_pred).cpu().data.numpy()\n        preds += y_numpy_pred.tolist()\n        \n        y_true = train_batch.target.float().cuda()\n        y_numpy_true = train_batch.target.cpu().data.numpy()\n        targets += y_numpy_true.tolist()\n        loss = loss_function(y_pred,y_true)\n        epoch_losses.append(loss.item())\n\n        loss.backward()\n        optimizer.step()\n        scheduler.step()\n        lrs.append(get_lr(optimizer))\n        acc.append(accuracy_score(y_numpy_true,np.round(y_numpy_pred)))\n\n#         print(batch,' ===>',loss.item(),' target ',targets[-1])\n    \n    losses.append(np.mean(epoch_losses))\n    targets =  np.array(targets)\n    preds = np.array(preds)\n    search_result = threshold_search(targets, preds)\n    train_f1 = search_result['f1']\n    epoch_acc.append(np.mean(acc))\n    \n    targets = []\n    val_acc=[]\n    \n    model.eval()\n    for val_batch in list(val_loader):\n        y_pred = model(val_batch.text.cuda()).squeeze(1)\n        y_numpy_pred = torch.sigmoid(y_pred).cpu().data.numpy()\n        val_preds += y_numpy_pred.tolist()        \n        y_true = val_batch.target.float().cuda()\n        y_numpy_true = val_batch.target.cpu().data.numpy()\n        targets += y_numpy_true.tolist()\n        val_loss = loss_function(y_pred,y_true)\n        epoch_val_losses.append(val_loss.item())\n        val_acc.append(accuracy_score(y_numpy_true,np.round(y_numpy_pred)))\n    \n    val_losses.append(np.mean(epoch_val_losses))\n    epoch_val_acc.append(np.mean(val_acc))\n    \n    targets =  np.array(targets)\n    val_preds =  np.array(val_preds)\n    search_result = threshold_search(targets, val_preds)\n    val_f1 = search_result['f1']\n    \n    print('EPOCH: ',epoch,'\\n has acc of ',epoch_acc[-1],' ,has loss of ',losses[-1], ' ,f1 of ',train_f1,'\\nval acc of ',epoch_val_acc[-1],' ,val loss of ',val_losses[-1],' ,val f1 of ',val_f1)\n    print('-'*80)\n    \n#     early_stopping(val_losses[-1], model)\n    if epoch > 0:\n        early_stopping(1.-val_f1, model)\n        \n    if early_stopping.early_stop:\n        print(\"Early stopping at \",epoch,\" epoch\")\n        break\n    \nprint('Training finished....')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"147165d678e7c531202dfc749273618949443bf5"},"cell_type":"code","source":"# to find out clr min and max...looks like its 0.0001 and 0.5153366336628805 - best results with big batch len(train)/100\n# for batches with 914 (lr*len(train)), max is 0.9231538461538571 - not very good results...\n# plt.figure(figsize=(20,10))\n# print(len(lrs))\n# print(len(acc))\n# acc = np.array(acc)\n# lrs = np.array(lrs)\n# indexes = np.argsort(lrs)\n# # print(indexes)\n# plt.plot(lrs[indexes],acc[indexes])\n# plt.xticks(np.arange(min(lrs), max(lrs), 1.0/len(lrs)*4))\n# plt.show()\n\n# max_acc_id =  np.argmax(acc)\n# print(lrs[max_acc_id])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ae403d8c4ccc8be12768d260a65d00a912c8b119"},"cell_type":"code","source":"print(os.listdir())\n\nmodel = Sentiment(text.vocab.vectors, padding_idx=text.vocab.stoi[text.pad_token]).cuda()\nmodel.load_state_dict(torch.load('checkpoint.pt'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d9213db94c556e3c42cdbeb741efef5c1423ae1a"},"cell_type":"code","source":"_,ax = plt.subplots(2,1,figsize=(20,10))\nax[0].plot(losses,label='loss')\nax[0].plot(val_losses,label='val_loss')\n\nax[1].plot(epoch_acc,label='acc')\nax[1].plot(epoch_val_acc,label='val_acc')\n\nplt.legend()\nplt.show()\n\npred = []\ntargets = []\nfor test_batch in list(test_loader):\n    model.eval()\n    x = test_batch.text.cuda()\n    pred += torch.sigmoid(model(x).squeeze(1)).cpu().data.numpy().tolist()\n    targets += test_batch.target.cpu().data.numpy().tolist()\n\npred = np.array(pred)\ntargets =  np.array(targets)\nsearch_result = threshold_search(targets, pred)\npred = (pred > search_result['threshold']).astype(int)\nprint('test acc:',accuracy_score(pred,targets))\nprint('test f1:',search_result['f1'])\n\nprint('RESULTS ON TEST SET:\\n',classification_report(targets,pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0ad34df4ae095e1de08c64079fa6b0ecbc944423"},"cell_type":"code","source":"print('Threshold:',search_result['threshold'])\n\nsubmission_list = list(torchtext.data.BucketIterator(dataset=submission_x,\n                                    batch_size=batch_size,\n                                    sort=False,\n                                    train=False))\npred = []\nwith torch.no_grad():\n    for submission_batch in submission_list:\n        model.eval()\n        x = submission_batch.text.cuda()\n        pred += torch.sigmoid(model(x).squeeze(1)).cpu().data.numpy().tolist()\n\npred = np.array(pred)\n\ndf_subm = pd.DataFrame()\ndf_subm['qid'] = [qid.vocab.itos[j] for i in submission_list for j in i.qid.view(-1).numpy()]\n# df_subm['prediction'] = test_meta > search_result['threshold']\ndf_subm['prediction'] = (pred > search_result['threshold']).astype(int)\nprint(df_subm.head())\ndf_subm.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}