{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import time\nimport numpy as np\nimport pandas as pd \nfrom tqdm import tqdm_notebook as tqdm\nfrom sklearn.model_selection import train_test_split\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nimport spacy\nimport time\nimport gc","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Some params"},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 512\nmaxlen = 100\nmax_features = int(120e3)\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Read data"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\ntest_df = pd.read_csv(\"../input/test.csv\")\n\ntrain_y = train_df['target'].values\nlen_train = len(train_df)\nprint(\"Train shape : \",train_df.shape)\nprint(\"Test shape : \",test_df.shape)\n\ntrain_text = train_df['question_text']\ntest_text = test_df['question_text']\ntext_list = pd.concat([train_text, test_text])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Tokenize/pad sequences with Keras"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## Tokenize the sentences\ntokenizer = Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(text_list))\nword_sequences = tokenizer.texts_to_sequences(text_list)\nword_dict = tokenizer.word_index\n\ndel train_text, test_text, text_list, train_df, test_df\ngc.collect()"},{"metadata":{},"cell_type":"markdown","source":"# Tokenize with spacy"},{"metadata":{"trusted":true},"cell_type":"code","source":"start_time = time.time()\nprint(\"Spacy NLP ...\")\nnlp = spacy.load('en_core_web_lg', disable=['parser','ner','tagger'])\nnlp.vocab.add_flag(lambda s: s.lower() in spacy.lang.en.stop_words.STOP_WORDS, spacy.attrs.IS_STOP)\nword_dict = {}\nword_index = 1\nlemma_dict = {}\ndocs = nlp.pipe(text_list, n_threads = 2)\nword_sequences = []\nfor doc in tqdm(docs):\n    word_seq = []\n    for token in doc:\n        if (token.text not in word_dict) and (token.pos_ is not \"PUNCT\"):\n            word_dict[token.text] = word_index\n            word_index += 1\n            lemma_dict[token.text] = token.lemma_\n        if token.pos_ is not \"PUNCT\":\n            word_seq.append(word_dict[token.text])\n    word_sequences.append(word_seq)\n\ndel docs, train_text, test_text, text_list, train_df, test_df\n\nimport gc\ngc.collect()\n\nprint(\"--- %s seconds ---\" % (time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_word_sequences = word_sequences[:len_train]\ntest_word_sequences = word_sequences[len_train:]\n\n## Pad the sentences \ntrain_word_sequences = pad_sequences(train_word_sequences, maxlen=maxlen)\ntest_word_sequences = pad_sequences(test_word_sequences, maxlen=maxlen)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# borrowed from https://www.kaggle.com/wowfattie/3rd-place\nfrom nltk.stem import PorterStemmer\nps = PorterStemmer()\nfrom nltk.stem.lancaster import LancasterStemmer\nlc = LancasterStemmer()\nfrom nltk.stem import SnowballStemmer\nsb = SnowballStemmer(\"english\")\n\ndef load_glove(word_dict, lemma_dict = None):\n    EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n    embed_size = 300\n    nb_words = len(word_dict)+1\n    embedding_matrix = np.zeros((nb_words, embed_size), dtype=np.float32)\n    unknown_vector = np.zeros((embed_size,), dtype=np.float32) - 1.\n    print(unknown_vector[:5])\n    for key in tqdm(word_dict):\n        word = key\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = key.lower()\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = key.upper()\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = key.capitalize()\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = ps.stem(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = lc.stem(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = sb.stem(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        if(lemma_dict is not None):    \n            word = lemma_dict[key]\n            embedding_vector = embeddings_index.get(word)\n            if embedding_vector is not None:\n                embedding_matrix[word_dict[key]] = embedding_vector\n                continue\n        embedding_matrix[word_dict[key]] = unknown_vector                    \n    return embedding_matrix, nb_words \n\ndef load_fasttext(word_dict, lemma_dict = None):\n    EMBEDDING_FILE = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE) if len(o)>100)\n    embed_size = 300\n    nb_words = len(word_dict)+1\n    embedding_matrix = np.zeros((nb_words, embed_size), dtype=np.float32)\n    unknown_vector = np.zeros((embed_size,), dtype=np.float32) - 1.\n    print(unknown_vector[:5])\n    for key in tqdm(word_dict):\n        word = key\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = key.lower()\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = key.upper()\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = key.capitalize()\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = ps.stem(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = lc.stem(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = sb.stem(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        if(lemma_dict is not None):    \n            word = lemma_dict[key]\n            embedding_vector = embeddings_index.get(word)\n            if embedding_vector is not None:\n                embedding_matrix[word_dict[key]] = embedding_vector\n                continue\n        embedding_matrix[word_dict[key]] = unknown_vector                    \n    return embedding_matrix, nb_words \n\ndef load_para(word_dict, lemma_dict = None):\n    EMBEDDING_FILE = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore') if len(o)>100)\n    embed_size = 300\n    nb_words = len(word_dict)+1\n    embedding_matrix = np.zeros((nb_words, embed_size), dtype=np.float32)\n    unknown_vector = np.zeros((embed_size,), dtype=np.float32) - 1.\n    print(unknown_vector[:5])\n    for key in tqdm(word_dict):\n        word = key\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = key.lower()\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = key.upper()\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = key.capitalize()\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = ps.stem(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = lc.stem(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = sb.stem(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        if(lemma_dict is not None):    \n            word = lemma_dict[key]\n            embedding_vector = embeddings_index.get(word)\n            if embedding_vector is not None:\n                embedding_matrix[word_dict[key]] = embedding_vector\n                continue\n        embedding_matrix[word_dict[key]] = unknown_vector                    \n    return embedding_matrix, nb_words ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Pytorch dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CustomDataset(Dataset):\n\n    def __init__(self, x, y):\n        super(CustomDataset, self).__init__()\n        self.x = torch.tensor(x, dtype = torch.long)\n        self.y = torch.tensor(y, dtype = torch.float32)\n    \n    def __len__(self):\n        return self.x.shape[0]\n\n    def __getitem__(self, idx):\n        x = self.x[idx, :]\n        y = self.y[idx, :]\n        return x, y\n\n#train_ds = CustomDataset(train_word_sequences, train_y.reshape((len(train_df), 1)))\n\n#train_loader = torch.utils.data.DataLoader(train_ds, batch_size=batch_size, shuffle=True)\nx_test = torch.tensor(test_word_sequences, dtype=torch.long)\ntest = torch.utils.data.TensorDataset(x_test)\ntest_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load embeddings"},{"metadata":{"trusted":true},"cell_type":"code","source":"start_time = time.time()\nprint(\"Loading embedding matrix ...\")\n\n# For spacy tokenizer \n#embedding_matrix_glove, _ = load_glove(word_dict, lemma_dict)\n#embedding_matrix_fasttext, _ = load_fasttext(word_dict, lemma_dict)\n#embedding_matrix_para, _ = load_para(word_dict, lemma_dict)\n\n# For keras tokenizer \nembedding_matrix_glove, _ = load_glove(word_dict)\nembedding_matrix_fasttext, _ = load_fasttext(word_dict)\nembedding_matrix_para, _ = load_para(word_dict)\n\nembedding_matrix = np.concatenate((embedding_matrix_glove, embedding_matrix_fasttext, embedding_matrix_para), axis=1)\n\ndel embedding_matrix_glove, embedding_matrix_fasttext, embedding_matrix_para\ngc.collect()\n\nprint(\"--- %s seconds ---\" % (time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Test stuff"},{"metadata":{"trusted":true},"cell_type":"code","source":"x_batch = next(iter(test_loader))[0]\nembed_size = 300\nembed = nn.Embedding(max_features, 300)\nx_batch.shape, embed(x_batch).shape\nhidden_size = 10\nlstm = nn.LSTM(embed_size, hidden_size, bidirectional=True, batch_first=True)\no, (h, c) = lstm(embed(x_batch))\no.shape, h.shape, c.shape\n\navg_pool = torch.mean(o, 1,)\nmax_pool, _ = torch.max(o, 1)\nconc = torch.cat((avg_pool, max_pool), 1)\nconc.shape, avg_pool.shape, max_pool.shape, c.shape, h.view(512, 2*hidden_size).shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## End Test stuff"},{"metadata":{},"cell_type":"markdown","source":"## Simple model"},{"metadata":{"trusted":true},"cell_type":"code","source":"class SimpleNet(nn.Module):\n    def __init__(self, embedding_matrix = None):\n        super(SimpleNet, self).__init__()\n        \n        self.hidden_size = 128\n        \n        self.embed_size = 300 if(embedding_matrix is None) else embedding_matrix.shape[1]\n        self.embedding = nn.Embedding(max_features, self.embed_size)\n        if(embedding_matrix is not None):\n            self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))\n            self.embedding.weight.requires_grad = False\n        \n        self.embedding_dropout = nn.Dropout2d(0.1)\n        self.lstm = nn.LSTM(self.embed_size, self.hidden_size, bidirectional=True, batch_first=True)\n        \n        self.linear = nn.Linear(self.hidden_size * 3*2, 32)\n        self.relu = nn.ReLU()\n        self.dropout = nn.Dropout(0.1)\n        self.out = nn.Linear(32, 1)\n        \n    def forward(self, x):\n        h_embedding = self.embedding(x)\n        h_embedding = torch.squeeze(self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))\n        \n        h_lstm, (h, c) = self.lstm(h_embedding)\n        \n        avg_pool = torch.mean(h_lstm, 1)\n        max_pool, _ = torch.max(h_lstm, 1)\n        #print(avg_pool.shape, max_pool.shape, c.shape, h.view(batch_size, 2*self.hidden_size).shape)\n        conc = torch.cat((avg_pool, max_pool, h.view(-1, 2*self.hidden_size)), 1)\n        conc = self.relu(self.linear(conc))\n        #conc = torch.cat((self.relu(self.linear(conc)), conc), 1)\n        conc = self.dropout(conc)\n        out = torch.sigmoid(self.out(conc))\n        \n        return out\n    \ndef unfreeze_embedding(model):\n    model.embedding.weight.requires_grad = True","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Attention model"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Attention(nn.Module):\n    def __init__(self, feature_dim, step_dim, bias=True, **kwargs):\n        super(Attention, self).__init__(**kwargs)\n        \n        self.supports_masking = True\n\n        self.bias = bias\n        self.feature_dim = feature_dim\n        self.step_dim = step_dim\n        self.features_dim = 0\n        \n        weight = torch.zeros(feature_dim, 1)\n        nn.init.xavier_uniform_(weight)\n        self.weight = nn.Parameter(weight)\n        \n        if bias:\n            self.b = nn.Parameter(torch.zeros(step_dim))\n        \n    def forward(self, x, mask=None):\n        feature_dim = self.feature_dim\n        step_dim = self.step_dim\n\n        eij = torch.mm(\n            x.contiguous().view(-1, feature_dim), \n            self.weight\n        ).view(-1, step_dim)\n        \n        if self.bias:\n            eij = eij + self.b\n            \n        eij = torch.tanh(eij)\n        a = torch.exp(eij)\n        \n        if mask is not None:\n            a = a * mask\n\n        a = a / torch.sum(a, 1, keepdim=True) + 1e-10\n\n        weighted_input = x * torch.unsqueeze(a, -1)\n        return torch.sum(weighted_input, 1)\n    \nclass NeuralNet(nn.Module):\n    def __init__(self, embedding_matrix):\n        super(NeuralNet, self).__init__()\n        \n        hidden_size = 128\n        \n        self.embed_size = 300 if(embedding_matrix is None) else embedding_matrix.shape[1]\n        self.embedding = nn.Embedding(max_features, self.embed_size)\n        if(embedding_matrix is not None):\n            self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))\n            self.embedding.weight.requires_grad = False\n        \n        self.embedding_dropout = nn.Dropout2d(0.1)\n        self.lstm = nn.LSTM(self.embed_size, hidden_size, bidirectional=True, batch_first=True)\n        self.gru = nn.GRU(hidden_size*2, hidden_size, bidirectional=True, batch_first=True)\n        \n        self.lstm_attention = Attention(hidden_size*2, maxlen)\n        self.gru_attention = Attention(hidden_size*2, maxlen)\n        \n        self.linear = nn.Linear(1024, 16)\n        self.relu = nn.ReLU()\n        self.dropout = nn.Dropout(0.1)\n        self.out = nn.Linear(16, 1)\n        \n    def forward(self, x):\n        h_embedding = self.embedding(x)\n        h_embedding = torch.squeeze(self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))\n        \n        h_lstm, _ = self.lstm(h_embedding)\n        h_gru, _ = self.gru(h_lstm)\n        \n        h_lstm_atten = self.lstm_attention(h_lstm)\n        h_gru_atten = self.gru_attention(h_gru)\n        \n        avg_pool = torch.mean(h_gru, 1)\n        max_pool, _ = torch.max(h_gru, 1)\n        \n        conc = torch.cat((h_lstm_atten, h_gru_atten, avg_pool, max_pool), 1)\n        conc = self.relu(self.linear(conc))\n        conc = self.dropout(conc)\n        out  = torch.sigmoid(self.out(conc))\n        \n        return out","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train function"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model(model, x_train, y_train, x_val, y_val, n_epochs, optimizer = None, validate=True):\n    \n    if(optimizer is None):\n        optimizer = torch.optim.Adam(model.parameters())\n    \n    train_ds = CustomDataset(x_train, y_train)\n    valid_ds  = CustomDataset(x_val, y_val)\n\n    train_loader = torch.utils.data.DataLoader(train_ds, batch_size=batch_size, shuffle=True)\n    valid_loader = torch.utils.data.DataLoader(valid_ds, batch_size=batch_size, shuffle=False)\n    \n    loss_fn = torch.nn.BCELoss(reduction='mean').cuda()\n    best_score = -np.inf\n    \n    for epoch in range(n_epochs):\n        start_time = time.time()\n        model.train()\n        avg_loss = 0.\n        \n        for x_batch, y_batch in tqdm(train_loader):\n            \n            x_batch = x_batch.to(device)\n            y_batch = y_batch.to(device)\n            \n            y_pred = model(x_batch)\n            loss = loss_fn(y_pred, y_batch)\n\n            optimizer.zero_grad()\n\n            loss.backward()\n\n            optimizer.step()\n            avg_loss += loss.item() / len(train_loader)\n            \n        model.eval()\n        \n        valid_preds = np.zeros((x_val_fold.shape[0]))\n        \n        if validate:\n            avg_val_loss = 0.\n            for i, (x_batch, y_batch) in tqdm(enumerate(valid_loader)):\n                x_batch = x_batch.to(device)\n                y_batch = y_batch.to(device)\n                y_pred = model(x_batch).detach()\n\n                avg_val_loss += loss_fn(y_pred, y_batch).item() / len(valid_loader)\n                valid_preds[i * batch_size:(i+1) * batch_size] = y_pred.cpu().numpy()[:, 0]\n            search_result = threshold_search(y_val, valid_preds)\n\n            val_f1, val_threshold = search_result['f1'], search_result['threshold']\n            elapsed_time = time.time() - start_time\n            print('Epoch {}/{} \\t loss={:.4f} \\t val_loss={:.4f} \\t val_f1={:.4f} best_t={:.2f} \\t time={:.2f}s'.format(\n                epoch + 1, n_epochs, avg_loss, avg_val_loss, val_f1, val_threshold, elapsed_time))\n        else:\n            elapsed_time = time.time() - start_time\n            print('Epoch {}/{} \\t loss={:.4f} \\t time={:.2f}s'.format(\n                epoch + 1, n_epochs, avg_loss, elapsed_time))\n    \n    valid_preds = np.zeros((x_val_fold.shape[0]))\n    \n    avg_val_loss = 0.\n    for i, (x_batch, y_batch) in tqdm(enumerate(valid_loader)):\n        x_batch = x_batch.to(device)\n        y_batch = y_batch.to(device)\n        y_pred = model(x_batch).detach()\n\n        avg_val_loss += loss_fn(y_pred, y_batch).item() / len(valid_loader)\n        valid_preds[i * batch_size:(i+1) * batch_size] = y_pred.cpu().numpy()[:, 0]\n\n    print('Validation loss: ', avg_val_loss)\n\n    test_preds = np.zeros((len(test_loader.dataset)))\n    \n    for i, (x_batch,) in tqdm(enumerate(test_loader)):\n        \n        x_batch = x_batch.to(device)\n        y_pred = model(x_batch).detach()\n\n        test_preds[i * batch_size:(i+1) * batch_size] = y_pred.cpu().numpy()[:, 0]\n    \n    return valid_preds, test_preds, optimizer\n\ndef threshold_search(y_true, y_proba):\n    best_threshold = 0\n    best_score = 0\n    for threshold in tqdm([i * 0.01 for i in range(100)], disable=True):\n        score = f1_score(y_true=y_true, y_pred=y_proba > threshold)\n        if score > best_score:\n            best_threshold = threshold\n            best_score = score\n    search_result = {'threshold': best_threshold, 'f1': best_score}\n    return search_result\n\ndef seed_everything(seed=1234):\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\nseed_everything()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\n\nsplits = list(StratifiedKFold(n_splits=3, shuffle=True, random_state=10).split(train_word_sequences, train_y))\n\ntest_preds = np.zeros((len(test_word_sequences), len(splits)))\ntrain_preds = np.zeros(len(train_word_sequences))\n\nn_epochs = 4\nfrom sklearn.metrics import f1_score\n\nseed = 10\n\nfor i, (train_idx, valid_idx) in enumerate(splits):    \n    #i = 0\n    train_idx, valid_idx =  splits[i]\n    x_train_fold = train_word_sequences[train_idx]\n    y_train_fold = train_y[train_idx, np.newaxis]\n    x_val_fold = train_word_sequences[valid_idx]\n    y_val_fold = train_y[valid_idx, np.newaxis]\n    print('Fold {}'.format(i+1))\n\n    seed_everything(seed + i)\n    model = SimpleNet(embedding_matrix)\n    #model = NeuralNet(embedding_matrix)\n    model.to(device)\n\n    valid_preds_fold, test_preds_fold, opt = train_model(model,\n                                                    x_train_fold, \n                                                    y_train_fold, \n                                                    x_val_fold, \n                                                    y_val_fold, \n                                                    n_epochs,\n                                                    validate=True)\n\n    test_preds[:, i] = test_preds_fold\n    train_preds[valid_idx] = valid_preds_fold","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unfreeze_embedding(model)\n\nfor param_group in opt.param_groups:\n        param_group['lr'] = 1e-4\n\nvalid_preds_fold, test_preds_fold, opt = train_model(model,\n                                                x_train_fold, \n                                                y_train_fold, \n                                                x_val_fold, \n                                                y_val_fold, \n                                                1,\n                                                opt,\n                                                validate=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv('../input/sample_submission.csv')\n\nsearch_result = threshold_search(train_y, train_preds)\nsub['prediction'] = test_preds.mean(1) > search_result['threshold']\nsub.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}