{"cells":[{"metadata":{"_uuid":"38e7d605b52588dfa82fb54def70d25e511df5bd"},"cell_type":"markdown","source":"### Inspired by:\n* https://www.kaggle.com/sudalairajkumar/a-look-at-different-embeddings\n* https://www.kaggle.com/shujian/single-rnn-with-4-folds-v1-9\n* http://mlexplained.com/2018/01/13/weight-normalization-and-layer-normalization-explained-normalization-in-deep-learning-part-2/\n* https://arxiv.org/abs/1607.06450\n* https://github.com/keras-team/keras/issues/3878\n* https://www.kaggle.com/lystdo/lstm-with-word2vec-embeddings\n* https://www.kaggle.com/jhoward/improved-lstm-baseline-glove-dropout\n* https://www.kaggle.com/aquatic/entity-embedding-neural-net\n* https://www.kaggle.com/hireme/fun-api-keras-f1-metric-cyclical-learning-rate\n* https://ai.google/research/pubs/pub46697\n* https://blog.openai.com/quantifying-generalization-in-reinforcement-learning/\n* https://www.kaggle.com/rasvob/let-s-try-clr-v3\n* https://github.com/bentrevett/pytorch-sentiment-analysis/blob/master/3%20-%20Faster%20Sentiment%20Analysis.ipynb\n* https://www.kaggle.com/ziliwang/pytorch-text-cnn\n* https://github.com/yunjey/pytorch-tutorial/blob/master/tutorials/02-intermediate/bidirectional_recurrent_neural_network/main.py\n* https://github.com/clairett/pytorch-sentiment-classification/blob/master/bilstm.py\n* https://pytorch.org/tutorials/beginner/nlp/sequence_models_tutorial.html\n\n\ntrying torch...\n\nmuch harder then keras, but feels more rewarding when done"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nnp.set_printoptions(threshold=np.nan)\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\nprint(os.listdir(\"../input/embeddings\"))\nprint(os.listdir(\"../input/embeddings/GoogleNews-vectors-negative300\"))\n\n# Any results you write to the current directory are saved as output.\n\nimport gensim\nfrom gensim.utils import simple_preprocess\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.callbacks import Callback\nfrom sklearn.metrics import classification_report,f1_score,precision_recall_fscore_support,recall_score,precision_score\nfrom keras import backend as K\nfrom sklearn.utils import class_weight\nimport matplotlib.pyplot as plt\n\nimport tensorflow as tf\n\nSEED = 2019\n\nnp.random.seed(SEED)\n\n#https://www.kaggle.com/shujian/single-rnn-with-4-folds-v1-9\ndef threshold_search(y_true, y_proba):\n    best_threshold = 0\n    best_score = 0\n    for threshold in [i * 0.01 for i in range(100)]:\n        score = f1_score(y_true=y_true, y_pred=y_proba > threshold)\n#         print('\\rthreshold = %f | score = %f'%(threshold,score),end='')\n        if score > best_score:\n            best_threshold = threshold\n            best_score = score\n#     print('best threshold is % f with score %f'%(best_threshold,best_score))\n    search_result = {'threshold': best_threshold, 'f1': best_score}\n    return search_result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fc45ae0bde97562a903da741fbc0b6801779c970"},"cell_type":"code","source":"df = pd.read_csv('../input/train.csv')\ndf[\"question_text\"].fillna(\"_##_\",inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bd86c03fdf44b58e9a817f8781bc567f279e1f3e"},"cell_type":"code","source":"import re\n\npuncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', \n '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]\n\ndef clean_text(x):\n    x = str(x)\n    for punct in puncts:\n        x = x.replace(punct, f' {punct} ')\n    return x\n\ndef clean_numbers(x):\n    x = re.sub('[0-9]{5,}', '#####', x)\n    x = re.sub('[0-9]{4}', '####', x)\n    x = re.sub('[0-9]{3}', '###', x)\n    x = re.sub('[0-9]{2}', '##', x)\n    return x\n\nmispell_dict = {\"aren't\" : \"are not\",\n\"can't\" : \"cannot\",\n\"couldn't\" : \"could not\",\n\"didn't\" : \"did not\",\n\"doesn't\" : \"does not\",\n\"don't\" : \"do not\",\n\"hadn't\" : \"had not\",\n\"hasn't\" : \"has not\",\n\"haven't\" : \"have not\",\n\"he'd\" : \"he would\",\n\"he'll\" : \"he will\",\n\"he's\" : \"he is\",\n\"i'd\" : \"I would\",\n\"i'd\" : \"I had\",\n\"i'll\" : \"I will\",\n\"i'm\" : \"I am\",\n\"isn't\" : \"is not\",\n\"it's\" : \"it is\",\n\"it'll\":\"it will\",\n\"i've\" : \"I have\",\n\"let's\" : \"let us\",\n\"mightn't\" : \"might not\",\n\"mustn't\" : \"must not\",\n\"shan't\" : \"shall not\",\n\"she'd\" : \"she would\",\n\"she'll\" : \"she will\",\n\"she's\" : \"she is\",\n\"shouldn't\" : \"should not\",\n\"that's\" : \"that is\",\n\"there's\" : \"there is\",\n\"they'd\" : \"they would\",\n\"they'll\" : \"they will\",\n\"they're\" : \"they are\",\n\"they've\" : \"they have\",\n\"we'd\" : \"we would\",\n\"we're\" : \"we are\",\n\"weren't\" : \"were not\",\n\"we've\" : \"we have\",\n\"what'll\" : \"what will\",\n\"what're\" : \"what are\",\n\"what's\" : \"what is\",\n\"what've\" : \"what have\",\n\"where's\" : \"where is\",\n\"who'd\" : \"who would\",\n\"who'll\" : \"who will\",\n\"who're\" : \"who are\",\n\"who's\" : \"who is\",\n\"who've\" : \"who have\",\n\"won't\" : \"will not\",\n\"wouldn't\" : \"would not\",\n\"you'd\" : \"you would\",\n\"you'll\" : \"you will\",\n\"you're\" : \"you are\",\n\"you've\" : \"you have\",\n\"'re\": \" are\",\n\"wasn't\": \"was not\",\n\"we'll\":\" will\",\n\"didn't\": \"did not\",\n\"tryin'\":\"trying\"}\n\ndef _get_mispell(mispell_dict):\n    mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n    return mispell_dict, mispell_re\n\nmispellings, mispellings_re = _get_mispell(mispell_dict)\ndef replace_typical_misspell(text):\n    def replace(match):\n        return mispellings[match.group(0)]\n    return mispellings_re.sub(replace, text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1a087a225715b0f6100d68d363c88ffd26cb275d"},"cell_type":"code","source":"from keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\n\nmaxlen = 100\nnum_words = 75966\ndim = 300\n\n#base w/o cleanup is 0.662\n\ndef load_and_prec():\n    train_df = pd.read_csv(\"../input/train.csv\")\n    test_df = pd.read_csv(\"../input/test.csv\")\n    print(\"Train shape : \",train_df.shape)\n    print(\"Test shape : \",test_df.shape)\n        \n    # lower - ++ 0.003%\n    print('lower...')\n    train_df[\"question_text\"] = train_df[\"question_text\"].apply(lambda x: x.lower())\n    test_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x: x.lower())\n\n    print('clean misspellings...')\n    train_df[\"question_text\"] = train_df[\"question_text\"].apply(lambda x: replace_typical_misspell(x))\n    test_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x: replace_typical_misspell(x))\n\n    print('lower...')\n    train_df[\"question_text\"] = train_df[\"question_text\"].apply(lambda x: x.lower())\n    test_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x: x.lower())\n    \n    # Clean the text - ++0.01%\n    print('clean text...')\n    train_df[\"question_text\"] = train_df[\"question_text\"].apply(lambda x: clean_text(x))\n    test_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x: clean_text(x))\n            \n    ## fill up the missing values\n    print('fill na...')\n    train_X = train_df[\"question_text\"].fillna(\"_##_\").values\n    test_X = test_df[\"question_text\"].fillna(\"_##_\").values\n\n    ## Tokenize the sentences\n    print('tokenize...')\n    tokenizer = Tokenizer(num_words=num_words)\n    tokenizer.fit_on_texts(list(train_X)+list(test_X))\n    train_X = tokenizer.texts_to_sequences(train_X)\n    test_X = tokenizer.texts_to_sequences(test_X)\n\n    ## Pad the sentences - this prob isn't necessary with torch...\n    train_X = pad_sequences(train_X, maxlen=maxlen)\n    test_X = pad_sequences(test_X, maxlen=maxlen)\n\n    ## Get the target values\n    train_y = train_df['target'].values\n        \n    return train_X, test_X, train_y, tokenizer.word_index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2a05d28cc139e786918e442d64e90b00c7910a13"},"cell_type":"code","source":"train_X, test_X, train_y, word_index = load_and_prec()\nprint(train_X.shape)\nprint(train_y.shape)\nprint(test_X.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cb8dad7f991c2cfa129117f0663e8c1ebce76ed4"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n# train_y =train_y.reshape((train_y.shape[0],1))\nx_train,x_val,y_train,y_val = train_test_split(train_X,train_y,random_state=SEED,stratify=train_y)\nx_train,x_test,y_train,y_test = train_test_split(x_train,y_train,test_size=0.1,random_state=SEED,stratify=y_train)\nprint(x_train.shape)\nprint(x_test.shape)\nprint(x_val.shape)\nprint(y_train.shape)\nprint(y_test.shape)\nprint(y_val.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"87c0b0bfb2aa2ded4bcfa51fae949932ce9759e4"},"cell_type":"code","source":"print('Glove ... ')\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open('../input/embeddings/glove.840B.300d/glove.840B.300d.txt'))\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nprint(len(all_embs))\n\n# embedding_matrix_glov = np.random.normal(emb_mean, emb_std, (num_words, dim))\nembedding_matrix_glov = np.zeros((num_words, dim))\n\ncount=0\nfor word, i in word_index.items():\n    if i >= num_words: \n        break\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: \n        embedding_matrix_glov[i] = embedding_vector\n    else:\n        count += 1\nprint('embedding matrix size:',embedding_matrix_glov.shape)\nprint('Number of words not in vocab:',count)\n\ndel embeddings_index,all_embs\nimport gc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"63328c2c14e9aede2a215809b435ef8aef379236"},"cell_type":"code","source":"print('Para...')\nEMBEDDING_FILE = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore') if len(o)>100)\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nprint(len(all_embs))\n\n# embedding_matrix_para = np.random.normal(emb_mean, emb_std, (num_words, dim))\nembedding_matrix_para = np.zeros((num_words, dim))\ncount=0\nfor word, i in word_index.items():\n    if i >= num_words: \n        break\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: \n        embedding_matrix_para[i] = embedding_vector\n    else:\n        count += 1\nprint('embedding matrix size:',embedding_matrix_glov.shape)\nprint('Number of words not in vocab:',count)\n\ndel embeddings_index,all_embs\nimport gc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ce4e38b868da9f6af3762032a0ca5ff2b0f07888"},"cell_type":"code","source":"print('Concatting matrixes...')\nmatrixes = [embedding_matrix_glov,embedding_matrix_para]\n\nmatrix = np.mean(matrixes,axis=0)\n\nprint(matrix.shape)\n\ndel embedding_matrix_glov,embedding_matrix_para\nimport gc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"460f9178a497fdcf5e75ad53df0a67e3accc7fee"},"cell_type":"code","source":"#src: https://github.com/Bjarten/early-stopping-pytorch/blob/master/pytorchtools.py\nimport numpy as np\nimport torch\n\nclass EarlyStopping:\n    \"\"\"Early stops the training if validation loss dosen't improve after a given patience.\"\"\"\n    def __init__(self, patience=7, verbose=False):\n        \"\"\"\n        Args:\n            patience (int): How long to wait after last time validation loss improved.\n                            Default: 7\n            verbose (bool): If True, prints a message for each validation loss improvement. \n                            Default: False\n        \"\"\"\n        self.patience = patience\n        self.verbose = verbose\n        self.counter = 0\n        self.best_score = None\n        self.early_stop = False\n        self.val_loss_min = np.Inf\n\n    def __call__(self, val_loss, model):\n\n        score = -val_loss\n\n        if self.best_score is None:\n            self.best_score = score\n            if self.verbose:\n                print(f'Validation loss decreased ({self.val_loss_min:.6f} --> {val_loss:.6f}).  Saving model ...')\n            self.save_checkpoint(val_loss, model)\n        elif score < self.best_score:\n            self.counter += 1\n            if self.verbose:\n                print(f'EarlyStopping counter: {self.counter} out of {self.patience}')\n            if self.counter >= self.patience:\n                self.early_stop = True\n        else:\n            self.best_score = score\n            self.save_checkpoint(val_loss, model)\n            self.counter = 0\n\n    def save_checkpoint(self, val_loss, model):\n        '''Saves model when validation loss decrease.'''\n        if self.verbose:\n            print(f'Validation loss decreased ({self.val_loss_min:.6f} --> {val_loss:.6f}).  Saving model ...')\n        torch.save(model.state_dict(), 'checkpoint.pt')\n        self.val_loss_min = val_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"90c151f3f6e8d359ff3fc710b2f25e9b66309559","scrolled":false},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport torch.utils.data\nimport torchtext.data\nimport warnings\nfrom sklearn.metrics import accuracy_score\nfrom torch.autograd import Variable\n\ntorch.cuda.init()\ntorch.cuda.empty_cache()\nprint('CUDA MEM:',torch.cuda.memory_allocated())\n\nprint('cuda:', torch.cuda.is_available())\nprint('cude index:',torch.cuda.current_device())\n\n\n# lr = 1e-3\n# batch_size = int(len(train_dataset)/100)\n# batch_size = int(lr*len(train))\nbatch_size = 512\nprint('batch_size:',batch_size)\nprint('---')\n\ntrain_loader = torch.utils.data.DataLoader(dataset=torch.utils.data.TensorDataset(torch.tensor(x_train, dtype=torch.long).cuda(),\n                                                                                    torch.tensor(y_train, dtype=torch.float32).cuda()),\n                                               batch_size=batch_size,\n                                               shuffle=True)\nval_loader = torch.utils.data.DataLoader(dataset=torch.utils.data.TensorDataset(torch.tensor(x_val, dtype=torch.long).cuda(),\n                                                                                  torch.tensor(y_val, dtype=torch.float32).cuda()),\n                                               batch_size=batch_size,\n                                               shuffle=False)\ntest_loader = torch.utils.data.DataLoader(dataset=torch.utils.data.TensorDataset(torch.tensor(x_test, dtype=torch.long).cuda(),\n                                                                                   torch.tensor(y_test, dtype=torch.float32).cuda()),\n                                               batch_size=batch_size,\n                                               shuffle=False)\n\nclass Sentiment(nn.Module):\n    \n    def __init__(self,matrix,batch_size):\n        super(Sentiment,self).__init__()\n        print('Vocab vectors size:',matrix.shape)\n        self.batch_size = batch_size\n        self.hidden_dim = 128\n        self.n_layers = 2 #bidirectional has 2 layers - forward and backward seq\n        \n#         self.embedding = nn.Embedding.from_pretrained(matrix)\n        self.embedding = nn.Embedding(matrix.shape[0],matrix.shape[1])\n        self.embedding.weight = nn.Parameter(torch.tensor(matrix, dtype=torch.float32))\n        self.embedding.weight.requires_grad = False\n        \n        self.lstm = nn.LSTM(input_size=matrix.shape[1], hidden_size=self.hidden_dim, bidirectional=True,batch_first=True)        \n        self.linear1 = nn.Linear(self.n_layers*self.hidden_dim,self.hidden_dim)        \n        self.linear2 = nn.Linear(self.hidden_dim,1)\n        self.dropout = nn.Dropout(0.2)\n\n        \n    def forward(self,x):\n        #init h0,c0\n        hidden = (torch.zeros(self.n_layers, x.shape[0], self.hidden_dim).cuda(),\n                torch.zeros(self.n_layers, x.shape[0], self.hidden_dim).cuda())\n        e = self.embedding(x)\n        _, hidden = self.lstm(e, hidden)\n        out = torch.cat((hidden[0][-2,:,:], hidden[0][-1,:,:]), dim=1).cuda()\n        out = self.linear1(F.relu(out))\n        return self.linear2( self.dropout(out))\n        \nmodel = Sentiment(matrix, batch_size=batch_size).cuda()\nprint(model)\nprint('-'*80)\n\nearly_stopping = EarlyStopping(patience=2,verbose=True)\nloss_function = nn.BCEWithLogitsLoss().cuda()        \noptimizer = optim.Adam(model.parameters(),lr=1e-3)\n\ndef get_lr(optimizer):\n    for param_group in optimizer.param_groups:\n        return param_group['lr']\n\n    \nlosses = []\nval_losses=[]\nepoch_acc=[]\nepoch_val_acc=[]\nlrs = []\n\nfor epoch in range(100):\n#     print('-----%d-----'%epoch)\n    epoch_losses=[]\n    epoch_val_losses = []\n    preds = []\n    val_preds=[]\n    targets = []\n    acc = []\n    model.train()\n    for batch,(x_batch,y_true) in enumerate(list(iter(train_loader)),1):\n        optimizer.zero_grad()\n        \n        y_pred = model(x_batch).squeeze(1)\n        y_numpy_pred =torch.sigmoid(y_pred).cpu().detach().numpy()\n        preds += y_numpy_pred.tolist()\n        \n        y_numpy_true = y_true.cpu().detach().numpy()\n        targets += y_numpy_true.tolist()\n        loss = loss_function(y_pred,y_true)\n        epoch_losses.append(loss.item())\n\n        loss.backward()\n        optimizer.step()\n        lrs.append(get_lr(optimizer))\n        acc.append(accuracy_score(y_numpy_true,np.round(y_numpy_pred)))\n        if batch % 100 == 0:\n            print('\\rtraining (batch,loss,acc) | ',batch,' ===>',loss.item(),' acc ',np.mean(acc),end='')\n    \n    losses.append(np.mean(epoch_losses))\n    targets =  np.array(targets)\n    preds = np.array(preds)\n    search_result = threshold_search(targets, preds)\n    train_f1 = search_result['f1']\n    epoch_acc.append(np.mean(acc))\n    \n    targets = []\n    val_acc=[]\n    model.eval()\n    with torch.no_grad():\n        for batch,(x_val_batch,y_true) in enumerate(list(val_loader),1):\n            y_pred = model(x_val_batch).squeeze(1)\n            y_numpy_pred = torch.sigmoid(y_pred).cpu().detach().numpy()\n            val_preds += y_numpy_pred.tolist()        \n            y_numpy_true = y_true.cpu().detach().numpy()\n            targets += y_numpy_true.tolist()\n            val_loss = loss_function(y_pred,y_true)\n            epoch_val_losses.append(val_loss.item())\n            val_acc.append(accuracy_score(y_numpy_true,np.round(y_numpy_pred)))\n            if batch % 100 == 0:\n                print('\\rvalidation (batch,acc) | ',batch,' ===>', np.mean(val_acc),end='')\n    \n    val_losses.append(np.mean(epoch_val_losses))\n    epoch_val_acc.append(np.mean(val_acc))\n    \n    targets =  np.array(targets)\n    val_preds =  np.array(val_preds)\n    search_result = threshold_search(targets, val_preds)\n    val_f1 = search_result['f1']\n    \n    print('\\nEPOCH: ',epoch,'\\n has acc of ',epoch_acc[-1],' ,has loss of ',losses[-1], ' ,f1 of ',train_f1,'\\nval acc of ',epoch_val_acc[-1],' ,val loss of ',val_losses[-1],' ,val f1 of ',val_f1)\n    print('-'*80)\n            \n    if early_stopping.early_stop:        \n        print(\"Early stopping at \",epoch,\" epoch\")\n        break\n    else:\n        early_stopping(1.-val_f1, model)\n\n    \nprint('Training finished....')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ae403d8c4ccc8be12768d260a65d00a912c8b119"},"cell_type":"code","source":"print(os.listdir())\n\nmodel = Sentiment(matrix, batch_size=batch_size).cuda()\nmodel.load_state_dict(torch.load('checkpoint.pt'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d9213db94c556e3c42cdbeb741efef5c1423ae1a"},"cell_type":"code","source":"_,ax = plt.subplots(2,1,figsize=(20,10))\nax[0].plot(losses,label='loss')\nax[0].plot(val_losses,label='val_loss')\n\nax[1].plot(epoch_acc,label='acc')\nax[1].plot(epoch_val_acc,label='val_acc')\n\nplt.legend()\nplt.show()\n\npred = []\ntargets = []\nwith torch.no_grad():\n    for x_test_batch,y_test_batch in list(test_loader):\n        model.eval()\n        pred += torch.sigmoid(model(x_test_batch).squeeze(1)).cpu().detach().numpy().tolist()\n        targets += y_test_batch.cpu().detach().numpy().tolist()\n\npred = np.array(pred)\ntargets =  np.array(targets)\nsearch_result = threshold_search(targets, pred)\npred = (pred > search_result['threshold']).astype(int)\nprint('test acc:',accuracy_score(pred,targets))\nprint('test f1:',search_result['f1'])\n\nprint('RESULTS ON TEST SET:\\n',classification_report(targets,pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0ad34df4ae095e1de08c64079fa6b0ecbc944423"},"cell_type":"code","source":"print('Threshold:',search_result['threshold'])\n\nprint(test_X.shape)\n\nsubmission_list = list(torch.utils.data.DataLoader(dataset=torch.utils.data.TensorDataset(torch.tensor(test_X, dtype=torch.long).cuda()),\n                                              batch_size=batch_size, \n                                                   shuffle=False))\n\npred = []\nwith torch.no_grad():\n    model.eval()\n    for (x,) in submission_list:       \n        y_pred = torch.sigmoid(model(x).squeeze(1)).detach()\n        pred += y_pred.cpu().numpy().tolist()\n\npred = np.array(pred)\n\ndf_subm = pd.read_csv('../input/sample_submission.csv')\ndf_subm.prediction = (pred > search_result['threshold']).astype(int)\nprint(df_subm.head())\ndf_subm.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}