{"cells":[{"metadata":{},"cell_type":"markdown","source":"### Import Libraries "},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\n\nfrom nltk.tokenize import TweetTokenizer\nimport datetime\nimport lightgbm as lgb\nfrom scipy import stats\nfrom scipy.sparse import hstack, csr_matrix\nfrom sklearn.model_selection import train_test_split, cross_val_score\nfrom wordcloud import WordCloud\nfrom collections import Counter\nfrom nltk.corpus import stopwords\nfrom nltk.util import ngrams\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.svm import LinearSVC\nfrom sklearn.multiclass import OneVsRestClassifier\nimport time\npd.set_option('max_colwidth',400)\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom sklearn.preprocessing import OneHotEncoder\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence\nfrom torch.autograd import Variable\nimport torch.utils.data\nimport random\nimport warnings\nwarnings.filterwarnings(\"ignore\", message=\"F-score is ill-defined and being set to 0.0 due to no predicted samples.\")\nimport re\nfrom torch.optim.lr_scheduler import StepLR, ReduceLROnPlateau, CosineAnnealingLR","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bb40886b64534d7a8c0e424d3f2033e984ca9194"},"cell_type":"code","source":"def seed_torch(seed=1029):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Load data and overview"},{"metadata":{"trusted":true,"_uuid":"2b77f6a1c831c98851143feb25c9903cb1154bf2","_kg_hide-input":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/train.csv\")\ntest = pd.read_csv(\"../input/test.csv\")\nsub = pd.read_csv('../input/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1558909b0a5c120c1d5ddc5be4f5a952fcb4971e"},"cell_type":"code","source":"print('Available embeddings:', os.listdir(\"../input/embeddings/\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"afaa845d44d72b9997ce037ab547ab4010701311"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0c7299c8895405ef00049595ead1ef89649ba71b"},"cell_type":"code","source":"sin = len(train[train[\"target\"] == 0])\ninsin = len(train[train[\"target\"] == 1])\ntot = sin + insin \nprint(\"Train sincere text total {}\".format(sin))\nprint(\"Train insincere text total {}\".format(insin))\nprint(\"Sincere percentage = {0:.2f}\".format(sin*100/tot))\nprint(\"Sincere percentage = {0:.2f}\".format(insin*100/tot))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cdbe2595e31608b72cfbdc8d4bfc75840bfe3a0d"},"cell_type":"markdown","source":"We have a seriuos disbalance - only ~6% of data are positive. No wonder the metric for the competition is f1-score."},{"metadata":{"trusted":true,"_uuid":"54a553b7e92a2a0a3d491ccf92b437011b813c85"},"cell_type":"code","source":"print('Average word length of questions in train is {0:.0f}.'.format(np.mean(train['question_text'].apply(lambda x: len(x.split())))))\nprint('Average word length of questions in test is {0:.0f}.'.format(np.mean(test['question_text'].apply(lambda x: len(x.split())))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7861669f72f36145c25911926a51bc688f51d473"},"cell_type":"code","source":"print('Max word length of questions in train is {0:.0f}.'.format(np.max(train['question_text'].apply(lambda x: len(x.split())))))\nprint('Max word length of questions in test is {0:.0f}.'.format(np.max(test['question_text'].apply(lambda x: len(x.split())))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1b1303137eb44cc0de3329921751c48209037562"},"cell_type":"code","source":"print('Average character length of questions in train is {0:.0f}.'.format(np.mean(train['question_text'].apply(lambda x: len(x)))))\nprint('Average character length of questions in test is {0:.0f}.'.format(np.mean(test['question_text'].apply(lambda x: len(x)))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b82f366ef8b46b0115e9940c7966849023545733"},"cell_type":"markdown","source":"As we can see on average questions in train and test datasets are similar, but there are quite long questions in train dataset."},{"metadata":{},"cell_type":"markdown","source":"### Preprocessing"},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"4f1838e686d67670bb6429b8b43619a69803fde8"},"cell_type":"code","source":"puncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', \n '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]\n\ndef clean_text(x):\n    x = str(x)\n    for punct in puncts:\n        x = x.replace(punct, f' {punct} ')\n    return x\n\ndef clean_numbers(x):\n    x = re.sub('[0-9]{5,}', '#####', x)\n    x = re.sub('[0-9]{4}', '####', x)\n    x = re.sub('[0-9]{3}', '###', x)\n    x = re.sub('[0-9]{2}', '##', x)\n    return x\n\nmispell_dict = {\"aren't\" : \"are not\",\n\"can't\" : \"cannot\",\n\"couldn't\" : \"could not\",\n\"didn't\" : \"did not\",\n\"doesn't\" : \"does not\",\n\"don't\" : \"do not\",\n\"hadn't\" : \"had not\",\n\"hasn't\" : \"has not\",\n\"haven't\" : \"have not\",\n\"he'd\" : \"he would\",\n\"he'll\" : \"he will\",\n\"he's\" : \"he is\",\n\"i'd\" : \"I would\",\n\"i'd\" : \"I had\",\n\"i'll\" : \"I will\",\n\"i'm\" : \"I am\",\n\"isn't\" : \"is not\",\n\"it's\" : \"it is\",\n\"it'll\":\"it will\",\n\"i've\" : \"I have\",\n\"let's\" : \"let us\",\n\"mightn't\" : \"might not\",\n\"mustn't\" : \"must not\",\n\"shan't\" : \"shall not\",\n\"she'd\" : \"she would\",\n\"she'll\" : \"she will\",\n\"she's\" : \"she is\",\n\"shouldn't\" : \"should not\",\n\"that's\" : \"that is\",\n\"there's\" : \"there is\",\n\"they'd\" : \"they would\",\n\"they'll\" : \"they will\",\n\"they're\" : \"they are\",\n\"they've\" : \"they have\",\n\"we'd\" : \"we would\",\n\"we're\" : \"we are\",\n\"weren't\" : \"were not\",\n\"we've\" : \"we have\",\n\"what'll\" : \"what will\",\n\"what're\" : \"what are\",\n\"what's\" : \"what is\",\n\"what've\" : \"what have\",\n\"where's\" : \"where is\",\n\"who'd\" : \"who would\",\n\"who'll\" : \"who will\",\n\"who're\" : \"who are\",\n\"who's\" : \"who is\",\n\"who've\" : \"who have\",\n\"won't\" : \"will not\",\n\"wouldn't\" : \"would not\",\n\"you'd\" : \"you would\",\n\"you'll\" : \"you will\",\n\"you're\" : \"you are\",\n\"you've\" : \"you have\",\n\"'re\": \" are\",\n\"wasn't\": \"was not\",\n\"we'll\":\" will\",\n\"didn't\": \"did not\",\n\"tryin'\":\"trying\"}\n\ndef _get_mispell(mispell_dict):\n    mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n    return mispell_dict, mispell_re\n\nmispellings, mispellings_re = _get_mispell(mispell_dict)\ndef replace_typical_misspell(text):\n    def replace(match):\n        return mispellings[match.group(0)]\n    return mispellings_re.sub(replace, text)\n\n# Clean the text\ntrain[\"question_text\"] = train[\"question_text\"].apply(lambda x: clean_text(x.lower()))\ntest[\"question_text\"] = test[\"question_text\"].apply(lambda x: clean_text(x.lower()))\n\n# Clean numbers\ntrain[\"question_text\"] = train[\"question_text\"].apply(lambda x: clean_numbers(x))\ntest[\"question_text\"] = test[\"question_text\"].apply(lambda x: clean_numbers(x))\n\n# Clean speelings\ntrain[\"question_text\"] = train[\"question_text\"].apply(lambda x: replace_typical_misspell(x))\ntest[\"question_text\"] = test[\"question_text\"].apply(lambda x: replace_typical_misspell(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9d081b2f0d46faf01a943c309568c27f92462f94"},"cell_type":"code","source":"max_features = 120000\ntk = Tokenizer(lower = True, filters='', num_words=max_features)\nfull_text = list(train['question_text'].values) + list(test['question_text'].values)\ntk.fit_on_texts(full_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"33929a60e1872b73e40424daa1178a3d8fbf8f5a"},"cell_type":"code","source":"train_tokenized = tk.texts_to_sequences(train['question_text'].fillna('_##_'))\ntest_tokenized = tk.texts_to_sequences(test['question_text'].fillna('_##_'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4330f6c01064b5fda4ca9661dc4f1cefb439cf75"},"cell_type":"code","source":"train['question_text'].apply(lambda x: len(x.split())).plot(kind='hist');\nplt.yscale('log');\nplt.title('Distribution of train question text length in characters');","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"05c7e2c63afb343b64835432721a42f81dd53626"},"cell_type":"markdown","source":"We can see that most of the questions are 40 words long or shorter. Let's try having sequence length equal to 72 for now."},{"metadata":{"trusted":true,"_uuid":"15d7e3a2fb5fe5fb38da45ca3d0bd13c82b7eda5"},"cell_type":"code","source":"max_len = 72\nmaxlen = 72\nX_train = pad_sequences(train_tokenized, maxlen = max_len)\nX_test = pad_sequences(test_tokenized, maxlen = max_len)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"50ade6e2c4dab8ee4add2a9c080f132e012cedc7"},"cell_type":"markdown","source":"### Preparing data for Pytorch"},{"metadata":{"trusted":true,"_uuid":"31839c56ed1d7945decf176c97b49f0205cc40af"},"cell_type":"code","source":"y_train = train['target'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"69d325ea77439697e53143adef3a0da58e7f31f2"},"cell_type":"code","source":"def sigmoid(x):\n    return 1 / (1 + np.exp(-x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cca23aa2dc1207a6f08212ad8d87f8182e567e0e"},"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\nsplits = list(StratifiedKFold(n_splits=4, shuffle=True, random_state=10).split(X_train, y_train))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Load Embeddings : Glove, Para"},{"metadata":{"trusted":true,"_uuid":"744ee4a1fbc66cb47aae6a18f829dea284b860c9"},"cell_type":"code","source":"#Glove\nembed_size = 300\nembedding_path = \"../input/embeddings/glove.840B.300d/glove.840B.300d.txt\"\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembedding_index = dict(get_coefs(*o.split(\" \")) for o in open(embedding_path, encoding='utf-8', errors='ignore'))\n# all_embs = np.stack(embedding_index.values())\n# emb_mean,emb_std = all_embs.mean(), all_embs.std()\nemb_mean,emb_std = -0.005838499, 0.48782197\nword_index = tk.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words + 1, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embedding_index.get(word)\n    if embedding_vector is not None: embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cc553dd7ddbd3a81ed1a39084fdb4982b5887971"},"cell_type":"code","source":"#para\nembedding_path = \"../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt\"\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembedding_index = dict(get_coefs(*o.split(\" \")) for o in open(embedding_path, encoding='utf-8', errors='ignore') if len(o)>100)\n# all_embs = np.stack(embedding_index.values())\n# emb_mean,emb_std = all_embs.mean(), all_embs.std()\nemb_mean,emb_std = -0.0053247833, 0.49346462\nembedding_matrix1 = np.random.normal(emb_mean, emb_std, (nb_words + 1, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embedding_index.get(word)\n    if embedding_vector is not None: embedding_matrix1[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"333429eaa0c7c057769518c8f1b0fefbb68f1d1d"},"cell_type":"code","source":"embedding_matrix = np.mean([embedding_matrix, embedding_matrix1], axis=0)\ndel embedding_matrix1","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ed3cd4b2f73c7ed5d1896f6b43029c8efdd89d5e"},"cell_type":"markdown","source":"### Model\nEmbedding-LSTM-Attention + Embedding-GRU-Attention "},{"metadata":{"trusted":true,"_uuid":"23b0dcfa9bff61ef4c5aebd85f9521edd8f6009d","_kg_hide-input":true},"cell_type":"code","source":"class Attention(nn.Module):\n    def __init__(self, feature_dim, step_dim, bias=True, **kwargs):\n        super(Attention, self).__init__(**kwargs)\n        \n        self.supports_masking = True\n\n        self.bias = bias\n        self.feature_dim = feature_dim\n        self.step_dim = step_dim\n        self.features_dim = 0\n        \n        weight = torch.zeros(feature_dim, 1)\n        nn.init.xavier_uniform_(weight)\n        self.weight = nn.Parameter(weight)\n        \n        if bias:\n            self.b = nn.Parameter(torch.zeros(step_dim))\n        \n    def forward(self, x, mask=None):\n        feature_dim = self.feature_dim\n        step_dim = self.step_dim\n\n        eij = torch.mm(\n            x.contiguous().view(-1, feature_dim), \n            self.weight\n        ).view(-1, step_dim)\n        \n        if self.bias:\n            eij = eij + self.b\n            \n        eij = torch.tanh(eij)\n        a = torch.exp(eij)\n        \n        if mask is not None:\n            a = a * mask\n\n        a = a / torch.sum(a, 1, keepdim=True) + 1e-10\n\n        weighted_input = x * torch.unsqueeze(a, -1)\n        return torch.sum(weighted_input, 1)\n    \nclass NeuralNet(nn.Module):\n    def __init__(self):\n        super(NeuralNet, self).__init__()\n        \n        hidden_size = 128\n        \n        self.embedding = nn.Embedding(max_features, embed_size)\n        self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))\n        self.embedding.weight.requires_grad = False\n        \n        self.embedding_dropout = nn.Dropout2d(0.1)\n        self.lstm = nn.LSTM(embed_size, hidden_size, bidirectional=True, batch_first=True)\n        self.gru = nn.GRU(hidden_size*2, hidden_size, bidirectional=True, batch_first=True)\n        \n        self.lstm_attention = Attention(hidden_size*2, maxlen)\n        self.gru_attention = Attention(hidden_size*2, maxlen)\n        \n        self.linear = nn.Linear(512, 16)\n        self.relu = nn.ReLU()\n        self.dropout = nn.Dropout(0.1)\n        self.out = nn.Linear(16, 1)\n        \n    def forward(self, x):\n        h_embedding = self.embedding(x)\n        h_embedding = torch.squeeze(self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))\n        \n        h_lstm, _ = self.lstm(h_embedding)\n        h_gru, _ = self.gru(h_lstm)\n        \n        h_lstm_atten = self.lstm_attention(h_lstm)\n        h_gru_atten = self.gru_attention(h_gru)\n        \n        #avg_pool = torch.mean(h_gru, 1)\n        #max_pool, _ = torch.max(h_gru, 1)\n        \n        conc = torch.cat((h_lstm_atten, h_gru_atten), 1)\n        conc = self.relu(self.linear(conc))\n        conc = self.dropout(conc)\n        out = self.out(conc)\n        \n        return out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"efa73786bf93892a5c3f46329122c0c0275e4858"},"cell_type":"code","source":"m = NeuralNet()\nprint(m)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a29c7d10faaf54c58cda8a406cb31fb367816f6c"},"cell_type":"code","source":"def train_model(model, x_train, y_train, x_val, y_val, validate=True):\n    optimizer = torch.optim.Adam(model.parameters())\n\n    # scheduler = CosineAnnealingLR(optimizer, T_max=5)\n    # scheduler = StepLR(optimizer, step_size=3, gamma=0.1)\n    \n    train = torch.utils.data.TensorDataset(x_train, y_train)\n    valid = torch.utils.data.TensorDataset(x_val, y_val)\n    \n    train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)\n    valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)\n    \n    loss_fn = torch.nn.BCEWithLogitsLoss(reduction='mean').cuda()\n    best_score = -np.inf\n    \n    for epoch in range(n_epochs):\n        start_time = time.time()\n        model.train()\n        avg_loss = 0.\n        \n        for x_batch, y_batch in tqdm(train_loader, disable=True):\n            y_pred = model(x_batch)\n            \n            \n            loss = loss_fn(y_pred, y_batch)\n\n            optimizer.zero_grad()\n\n            loss.backward()\n\n            optimizer.step()\n            avg_loss += loss.item() / len(train_loader)\n            \n        model.eval()\n        \n        valid_preds = np.zeros((x_val_fold.size(0)))\n        \n        if validate:\n            avg_val_loss = 0.\n            for i, (x_batch, y_batch) in enumerate(valid_loader):\n                y_pred = model(x_batch).detach()\n\n                avg_val_loss += loss_fn(y_pred, y_batch).item() / len(valid_loader)\n                valid_preds[i * batch_size:(i+1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]\n            search_result = threshold_search(y_val.cpu().numpy(), valid_preds)\n\n            val_f1, val_threshold = search_result['f1'], search_result['threshold']\n            elapsed_time = time.time() - start_time\n            print('Epoch {}/{} \\t loss={:.4f} \\t val_loss={:.4f} \\t val_f1={:.4f} best_t={:.2f} \\t time={:.2f}s'.format(\n                epoch + 1, n_epochs, avg_loss, avg_val_loss, val_f1, val_threshold, elapsed_time))\n        else:\n            elapsed_time = time.time() - start_time\n            print('Epoch {}/{} \\t loss={:.4f} \\t time={:.2f}s'.format(\n                epoch + 1, n_epochs, avg_loss, elapsed_time))\n    \n    valid_preds = np.zeros((x_val_fold.size(0)))\n    \n    avg_val_loss = 0.\n    for i, (x_batch, y_batch) in enumerate(valid_loader):\n        y_pred = model(x_batch).detach()\n\n        avg_val_loss += loss_fn(y_pred, y_batch).item() / len(valid_loader)\n        valid_preds[i * batch_size:(i+1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]\n\n    print('Validation loss: ', avg_val_loss)\n\n    test_preds = np.zeros((len(test_loader.dataset)))\n    \n    for i, (x_batch,) in enumerate(test_loader):\n        y_pred = model(x_batch).detach()\n\n        test_preds[i * batch_size:(i+1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]\n    # scheduler.step()\n    \n    return valid_preds, test_preds#, test_preds_local","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2252ff85cae8b5096e7f10ccf64ce10299a18782"},"cell_type":"code","source":"x_test_cuda = torch.tensor(X_test, dtype=torch.long).cuda()\ntest = torch.utils.data.TensorDataset(x_test_cuda)\nbatch_size = 512\ntest_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9ec7632d3f2ef30b2e985a4662bc8178b9a49a94"},"cell_type":"code","source":"seed=1029\n\ndef threshold_search(y_true, y_proba):\n    best_threshold = 0\n    best_score = 0\n    for threshold in tqdm([i * 0.01 for i in range(100)], disable=True):\n        score = f1_score(y_true=y_true, y_pred=y_proba > threshold)\n        if score > best_score:\n            best_threshold = threshold\n            best_score = score\n    search_result = {'threshold': best_threshold, 'f1': best_score}\n    return search_result\n\ndef seed_everything(seed=1234):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\nseed_everything()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"68aa1d6303e656ca76184d53c44001836c431939","scrolled":true},"cell_type":"code","source":"train_preds = np.zeros(len(train))\ntest_preds = np.zeros((len(test), len(splits)))\nn_epochs = 5\nfrom tqdm import tqdm\nfrom sklearn.metrics import f1_score\nfor i, (train_idx, valid_idx) in enumerate(splits):    \n    x_train_fold = torch.tensor(X_train[train_idx], dtype=torch.long).cuda()\n    y_train_fold = torch.tensor(y_train[train_idx, np.newaxis], dtype=torch.float32).cuda()\n    x_val_fold = torch.tensor(X_train[valid_idx], dtype=torch.long).cuda()\n    y_val_fold = torch.tensor(y_train[valid_idx, np.newaxis], dtype=torch.float32).cuda()\n    \n    train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)\n    valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)\n    \n    train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)\n    valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)\n    \n    print(f'Fold {i + 1}')\n    \n    seed_everything(seed + i)\n    model = NeuralNet()\n    model.cuda()\n    valid_preds_fold, test_preds_fold = train_model(model,\n                                                                           x_train_fold, \n                                                                           y_train_fold, \n                                                                           x_val_fold, \n                                                                           y_val_fold, validate=False)\n\n    train_preds[valid_idx] = valid_preds_fold\n    test_preds[:, i] = test_preds_fold\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2c04e298f5c96a9561450874a0238f408984c39e"},"cell_type":"code","source":"search_result = threshold_search(y_train, train_preds)\nprint(search_result)\nsub['prediction'] = test_preds.mean(1) > search_result['threshold']\nsub.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}