{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\n\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\n\nfrom nltk.tokenize import TweetTokenizer\nimport datetime\nimport lightgbm as lgb\nfrom scipy import stats\nfrom scipy.sparse import hstack, csr_matrix\nfrom sklearn.model_selection import train_test_split, cross_val_score\nfrom wordcloud import WordCloud\nfrom collections import Counter\nfrom nltk.corpus import stopwords\nfrom nltk.util import ngrams\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.svm import LinearSVC\nfrom sklearn.multiclass import OneVsRestClassifier\nimport time\npd.set_option('max_colwidth',400)\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom sklearn.preprocessing import OneHotEncoder\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence\nfrom torch.autograd import Variable\nimport torch.utils.data\nimport random\nimport warnings\nwarnings.filterwarnings(\"ignore\", message=\"F-score is ill-defined and being set to 0.0 due to no predicted samples.\")\nimport re\nfrom torch.optim.lr_scheduler import StepLR, ReduceLROnPlateau, CosineAnnealingLR","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"99f9cf7db33a22ab614aa4f9bacc8523bc52e49a"},"cell_type":"code","source":"def seed_torch(seed=1029):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"78e1e805eb6c8da2797cdd37c96f3547e3104fb4"},"cell_type":"code","source":"import os\nprint('Available embeddings:', os.listdir(\"../input/embeddings/\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"17969602fa53f71c9fdd511988839f55ecce0b1f"},"cell_type":"code","source":"train = pd.read_csv(\"../input/train.csv\")\ntest = pd.read_csv(\"../input/test.csv\")\nsub = pd.read_csv('../input/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"99768fcf2788d1034b7ef3f41f8e0df472b3e326"},"cell_type":"code","source":"train[\"target\"].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7f5aea138ce1a8cda4a6efdde3861bc8728ac145"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ee022fd0c64c0a1b0f7728754adcffb63ff5338a"},"cell_type":"code","source":"print('Average word length of questions in train is {0:.0f}.'.format(np.mean(train['question_text'].apply(lambda x: len(x.split())))))\nprint('Average word length of questions in test is {0:.0f}.'.format(np.mean(test['question_text'].apply(lambda x: len(x.split())))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"34081a86adf734ae28349cb0cd7d4a9d65b5b761"},"cell_type":"code","source":"print('Max word length of questions in train is {0:.0f}.'.format(np.max(train['question_text'].apply(lambda x: len(x.split())))))\nprint('Max word length of questions in test is {0:.0f}.'.format(np.max(test['question_text'].apply(lambda x: len(x.split())))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a2326e4821e75ec3a9a27fae79fc83a21a33ab95"},"cell_type":"code","source":"print('Average character length of questions in train is {0:.0f}.'.format(np.mean(train['question_text'].apply(lambda x: len(x)))))\nprint('Average character length of questions in test is {0:.0f}.'.format(np.mean(test['question_text'].apply(lambda x: len(x)))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ebae821cf384d2bf6a2291d2eff86df181639841"},"cell_type":"code","source":"max_features = 120000\ntk = Tokenizer(lower = True, filters='', num_words=max_features)\nfull_text = list(train['question_text'].values) + list(test['question_text'].values)\ntk.fit_on_texts(full_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1cbf4c223dc232d75f3f34607181244583c5624b"},"cell_type":"code","source":"train_tokenized = tk.texts_to_sequences(train['question_text'].fillna('missing'))\ntest_tokenized = tk.texts_to_sequences(test['question_text'].fillna('missing'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f7365690e086107acc9d96d0f7a6acbe47f635c0"},"cell_type":"code","source":"train['question_text'].apply(lambda x: len(x.split())).plot(kind='hist');\nplt.yscale('log');\nplt.title('Distribution of question text length in characters');","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"90b57fbdb0800accd8caece24942161a94b46da8"},"cell_type":"code","source":"max_len = 72\nmaxlen = 72\nX_train = pad_sequences(train_tokenized, maxlen = max_len)\nX_test = pad_sequences(test_tokenized, maxlen = max_len)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5f1d52d8c3186c1594cac274ee1570c4f46a4e0c"},"cell_type":"code","source":"y_train = train['target'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fe31db845ffedcbdb5cde8ecca5737fc64a2cb03"},"cell_type":"code","source":"def sigmoid(x):\n    return 1 / (1 + np.exp(-x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fc751d620ccb6b21b1621364e6b34001519eef41"},"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\nsplits = list(StratifiedKFold(n_splits=4, shuffle=True, random_state=10).split(X_train, y_train))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"499f7aab5d76f11a2d614a7d6acc543d6b613c24"},"cell_type":"code","source":"embed_size = 300\nembedding_path = \"../input/embeddings/glove.840B.300d/glove.840B.300d.txt\"\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembedding_index = dict(get_coefs(*o.split(\" \")) for o in open(embedding_path, encoding='utf-8', errors='ignore'))\n# all_embs = np.stack(embedding_index.values())\n# emb_mean,emb_std = all_embs.mean(), all_embs.std()\nemb_mean,emb_std = -0.005838499, 0.48782197\nword_index = tk.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words + 1, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embedding_index.get(word)\n    if embedding_vector is not None: embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"04e2c15ad2171a1c10de9f49d75f5f7c500d0a4e"},"cell_type":"code","source":"embedding_path = \"../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt\"\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembedding_index = dict(get_coefs(*o.split(\" \")) for o in open(embedding_path, encoding='utf-8', errors='ignore') if len(o)>100)\n# all_embs = np.stack(embedding_index.values())\n# emb_mean,emb_std = all_embs.mean(), all_embs.std()\nemb_mean,emb_std = -0.0053247833, 0.49346462\nembedding_matrix1 = np.random.normal(emb_mean, emb_std, (nb_words + 1, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embedding_index.get(word)\n    if embedding_vector is not None: embedding_matrix1[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"35fb4b0c83f6b0e94324c24afcfaf7fb5248724f"},"cell_type":"code","source":"embedding_matrix = np.mean([embedding_matrix, embedding_matrix1], axis=0)\ndel embedding_matrix1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8f0ae9d50956fdb9eadfcb1e9effeb4c4351862e"},"cell_type":"code","source":"class Attention(nn.Module):\n    def __init__(self, feature_dim, step_dim, bias=True, **kwargs):\n        super(Attention, self).__init__(**kwargs)\n        \n        self.supports_masking = True\n\n        self.bias = bias\n        self.feature_dim = feature_dim\n        self.step_dim = step_dim\n        self.features_dim = 0\n        \n        weight = torch.zeros(feature_dim, 1)\n        nn.init.xavier_uniform_(weight)\n        self.weight = nn.Parameter(weight)\n        \n        if bias:\n            self.b = nn.Parameter(torch.zeros(step_dim))\n        \n    def forward(self, x, mask=None):\n        feature_dim = self.feature_dim\n        step_dim = self.step_dim\n\n        eij = torch.mm(\n            x.contiguous().view(-1, feature_dim), \n            self.weight\n        ).view(-1, step_dim)\n        \n        if self.bias:\n            eij = eij + self.b\n            \n        eij = torch.tanh(eij)\n        a = torch.exp(eij)\n        \n        if mask is not None:\n            a = a * mask\n\n        a = a / torch.sum(a, 1, keepdim=True) + 1e-10\n\n        weighted_input = x * torch.unsqueeze(a, -1)\n        return torch.sum(weighted_input, 1)\n    \nclass NeuralNet(nn.Module):\n    def __init__(self):\n        super(NeuralNet, self).__init__()\n        \n        hidden_size = 128\n        \n        self.embedding = nn.Embedding(max_features, embed_size)\n        self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))\n        self.embedding.weight.requires_grad = False\n        \n        self.embedding_dropout = nn.Dropout2d(0.1)\n        self.lstm = nn.LSTM(embed_size, hidden_size, bidirectional=True, batch_first=True)\n        self.gru = nn.GRU(hidden_size*2, hidden_size, bidirectional=True, batch_first=True)\n        \n        self.lstm_attention = Attention(hidden_size*2, maxlen)\n        self.gru_attention = Attention(hidden_size*2, maxlen)\n        \n        self.linear = nn.Linear(1024, 16)\n        self.relu = nn.ReLU()\n        self.dropout = nn.Dropout(0.1)\n        self.out = nn.Linear(16, 1)\n        \n    def forward(self, x):\n        h_embedding = self.embedding(x)\n        h_embedding = torch.squeeze(self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))\n        \n        h_lstm, _ = self.lstm(h_embedding)\n        h_gru, _ = self.gru(h_lstm)\n        \n        h_lstm_atten = self.lstm_attention(h_lstm)\n        h_gru_atten = self.gru_attention(h_gru)\n        \n        avg_pool = torch.mean(h_gru, 1)\n        max_pool, _ = torch.max(h_gru, 1)\n        \n        conc = torch.cat((h_lstm_atten, h_gru_atten, avg_pool, max_pool), 1)\n        conc = self.relu(self.linear(conc))\n        conc = self.dropout(conc)\n        out = self.out(conc)\n        \n        return out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f4d8d8d90c4ca4a623ddbee81750c4b6b0cbe025"},"cell_type":"code","source":"m = NeuralNet()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2fcf4e9d4b857ad38a64c13643d0ad4000651a5a"},"cell_type":"code","source":"def train_model(model, x_train, y_train, x_val, y_val, validate=True):\n    optimizer = torch.optim.Adam(model.parameters())\n\n    # scheduler = CosineAnnealingLR(optimizer, T_max=5)\n    # scheduler = StepLR(optimizer, step_size=3, gamma=0.1)\n    \n    train = torch.utils.data.TensorDataset(x_train, y_train)\n    valid = torch.utils.data.TensorDataset(x_val, y_val)\n    \n    train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)\n    valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)\n    \n    loss_fn = torch.nn.BCEWithLogitsLoss(reduction='mean').cuda()\n    best_score = -np.inf\n    \n    for epoch in range(n_epochs):\n        start_time = time.time()\n        model.train()\n        avg_loss = 0.\n        \n        for x_batch, y_batch in tqdm(train_loader, disable=True):\n            y_pred = model(x_batch)\n            \n            \n            loss = loss_fn(y_pred, y_batch)\n\n            optimizer.zero_grad()\n\n            loss.backward()\n\n            optimizer.step()\n            avg_loss += loss.item() / len(train_loader)\n            \n        model.eval()\n        \n        valid_preds = np.zeros((x_val_fold.size(0)))\n        \n        if validate:\n            avg_val_loss = 0.\n            for i, (x_batch, y_batch) in enumerate(valid_loader):\n                y_pred = model(x_batch).detach()\n\n                avg_val_loss += loss_fn(y_pred, y_batch).item() / len(valid_loader)\n                valid_preds[i * batch_size:(i+1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]\n            search_result = threshold_search(y_val.cpu().numpy(), valid_preds)\n\n            val_f1, val_threshold = search_result['f1'], search_result['threshold']\n            elapsed_time = time.time() - start_time\n            print('Epoch {}/{} \\t loss={:.4f} \\t val_loss={:.4f} \\t val_f1={:.4f} best_t={:.2f} \\t time={:.2f}s'.format(\n                epoch + 1, n_epochs, avg_loss, avg_val_loss, val_f1, val_threshold, elapsed_time))\n        else:\n            elapsed_time = time.time() - start_time\n            print('Epoch {}/{} \\t loss={:.4f} \\t time={:.2f}s'.format(\n                epoch + 1, n_epochs, avg_loss, elapsed_time))\n    \n    valid_preds = np.zeros((x_val_fold.size(0)))\n    \n    avg_val_loss = 0.\n    for i, (x_batch, y_batch) in enumerate(valid_loader):\n        y_pred = model(x_batch).detach()\n\n        avg_val_loss += loss_fn(y_pred, y_batch).item() / len(valid_loader)\n        valid_preds[i * batch_size:(i+1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]\n\n    print('Validation loss: ', avg_val_loss)\n\n    test_preds = np.zeros((len(test_loader.dataset)))\n    \n    for i, (x_batch,) in enumerate(test_loader):\n        y_pred = model(x_batch).detach()\n\n        test_preds[i * batch_size:(i+1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]\n    # scheduler.step()\n    \n    return valid_preds, test_preds#, test_preds_local","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8f1726fa675fd9c06b17f6960fe4a729b8cb4366"},"cell_type":"code","source":"x_test_cuda = torch.tensor(X_test, dtype=torch.long).cuda()\ntest = torch.utils.data.TensorDataset(x_test_cuda)\nbatch_size = 512\ntest_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3d44e7bf62e2e5b98ed81680240e401ccf49cfc7"},"cell_type":"code","source":"seed=1029\n\ndef threshold_search(y_true, y_proba):\n    best_threshold = 0\n    best_score = 0\n    for threshold in tqdm([i * 0.01 for i in range(100)], disable=True):\n        score = f1_score(y_true=y_true, y_pred=y_proba > threshold)\n        if score > best_score:\n            best_threshold = threshold\n            best_score = score\n    search_result = {'threshold': best_threshold, 'f1': best_score}\n    return search_result\n\ndef seed_everything(seed=1234):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\nseed_everything()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":""},{"metadata":{"trusted":true,"_uuid":"ff28ebb343da56bf13fef90a66b97c1826bfaef3"},"cell_type":"code","source":"train_preds = np.zeros(len(train))\ntest_preds = np.zeros((len(test), len(splits)))\nn_epochs = 5\nfrom tqdm import tqdm\nfrom sklearn.metrics import f1_score\nfor i, (train_idx, valid_idx) in enumerate(splits):    \n    x_train_fold = torch.tensor(X_train[train_idx], dtype=torch.long).cuda()\n    y_train_fold = torch.tensor(y_train[train_idx, np.newaxis], dtype=torch.float32).cuda()\n    x_val_fold = torch.tensor(X_train[valid_idx], dtype=torch.long).cuda()\n    y_val_fold = torch.tensor(y_train[valid_idx, np.newaxis], dtype=torch.float32).cuda()\n    \n    train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)\n    valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)\n    \n    train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)\n    valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)\n    \n    print(f'Fold {i + 1}')\n    \n    seed_everything(seed + i)\n    model = NeuralNet()\n    model.cuda()\n\n    valid_preds_fold, test_preds_fold = train_model(model,\n                                                                           x_train_fold, \n                                                                           y_train_fold, \n                                                                           x_val_fold, \n                                                                           y_val_fold, validate=False)\n\n    train_preds[valid_idx] = valid_preds_fold\n    test_preds[:, i] = test_preds_fold","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cfead10a68c0f72d55c40646307077c115fa8ba7"},"cell_type":"code","source":"search_result = threshold_search(y_train, train_preds)\nsub['prediction'] = test_preds.mean(1) > search_result['threshold']\nsub.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}