{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import re\nimport time\nimport gc\nimport random\nimport os\nimport nltk\nimport multiprocess as mp\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import f1_score, roc_curve, precision_recall_curve\nfrom sklearn.preprocessing import StandardScaler\n\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\n\nimport torch\nimport torch.nn as nn\nimport torch.utils.data\nfrom torch.nn import Parameter","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1860a9d0918c15f3102d8378d5d8790203b1e144"},"cell_type":"code","source":"embed_size = 300 # how big is each word vector\nmax_features = 120000 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 50 # max number of words in a question to use\n\nbatch_size = 512\ntrain_epochs = 6\n\nSEED = 1029","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"44873db795fa85a3ee2f32a38b4fe8747ffb9b74"},"cell_type":"code","source":"def seed_torch(seed=1029):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nseed_torch(SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d20cee89afb8267f4177e05be9236103fb638f3c"},"cell_type":"code","source":"puncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', \n '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]\n\ndef clean_text(x):\n    x = str(x)\n    for punct in puncts:\n        x = x.replace(punct, f' {punct} ')\n    return x\n\ndef clean_numbers(x):\n    x = re.sub('[0-9]{5,}', '#####', x)\n    x = re.sub('[0-9]{4}', '####', x)\n    x = re.sub('[0-9]{3}', '###', x)\n    x = re.sub('[0-9]{2}', '##', x)\n    return x\n\nmispell_dict = {\"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\", \"didn't\": \"did not\",  \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\", \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\",  \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\", \"i'd've\": \"i would have\", \"i'll\": \"i will\",  \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\", \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\", \"must've\": \"must have\", \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\", \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\"so's\": \"so as\", \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\", \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \"they'd've\": \"they would have\", \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\", \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\",  \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\", \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\", \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\", \"you're\": \"you are\", \"you've\": \"you have\", 'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', \"mastrubating\": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', \"whst\": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}\n\ndef _get_mispell(mispell_dict):\n    mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n    return mispell_dict, mispell_re\n\nmispellings, mispellings_re = _get_mispell(mispell_dict)\ndef replace_typical_misspell(text):\n    def replace(match):\n        return mispellings[match.group(0)]\n    return mispellings_re.sub(replace, text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"39bef4d4f07ee7b7a205a478e0649c125f0f6016"},"cell_type":"code","source":"def preprocess(df):\n    # cleaning\n    df.question_text = df[\"question_text\"].apply(lambda x: x.lower())\n    df.question_text = df[\"question_text\"].apply(lambda x: clean_text(x))\n    df.question_text = df[\"question_text\"].apply(lambda x: clean_numbers(x))\n    df.question_text = df[\"question_text\"].apply(lambda x: replace_typical_misspell(x))\n    \n    #  add features\n    df['total_length'] = df['question_text'].apply(len)\n    df['capitals'] = df['question_text'].apply(lambda comment: sum(1 for c in comment if c.isupper()))\n    df['caps_vs_length'] = df.apply(lambda row: float(row['capitals'])/float(row['total_length']), axis=1)\n    df['num_words'] = df.question_text.str.count('\\S+')\n    df['num_unique_words'] = df['question_text'].apply(lambda comment: len(set(w for w in comment.split())))\n    df['words_vs_unique'] = df['num_unique_words'] / df['num_words'] \n    \n    # add pos_tag\n    texts = df.question_text.tolist()\n    tagged_texts = nltk.pos_tag_sents(map(nltk.word_tokenize, texts))\n    df[\"tagged_texts\"] = tagged_texts\n    df.tagged_texts = df[\"tagged_texts\"].apply(lambda x: \" \".join([i[1] for i in x]))\n    \n    return df\n\nchunksize = 10000","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"df884a6277574e90dd7d5c3432bf2a664bb39bdb"},"cell_type":"code","source":"train_reader = pd.read_csv(\"../input/train.csv\", chunksize=chunksize)\nstart_time = time.time()\npool = mp.Pool(4)\nfunclist = []\nfor df in train_reader:\n    f = pool.apply_async(preprocess, [df])\n    funclist.append(f)\nresult = []\nfor f in funclist:\n    result.append(f.get(timeout=120))    \ntrain_df = pd.concat(result)\nelapsed_time = (time.time() - start_time) / 60\nprint(elapsed_time)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"14da6255a5564d3faaefe59fb27ffd5e8d8179a9"},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c7991c9d74cc5901b4850d3a0a763e11f1c97197"},"cell_type":"code","source":"test_reader = pd.read_csv(\"../input/test.csv\", chunksize=chunksize)\nstart_time = time.time()\npool = mp.Pool(4)\nfunclist = []\nfor df in test_reader:\n    f = pool.apply_async(preprocess, [df])\n    funclist.append(f)\nresult = []\nfor f in funclist:\n    result.append(f.get(timeout=120))    \ntest_df = pd.concat(result)\nelapsed_time = (time.time() - start_time) / 60\nprint(elapsed_time)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b40ce6dc8b632f0c6509cf5bae3eecbe3ea432c5"},"cell_type":"code","source":"test_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4ee6b3fa85bd696e4937919b6564232af0db9cda"},"cell_type":"code","source":"start_time = time.time()\n\ntrain_X = train_df[\"question_text\"].fillna(\"_##_\").values\ntest_X = test_df[\"question_text\"].fillna(\"_##_\").values\n\ntrain_tag_X = train_df[\"tagged_texts\"].fillna(\"_##_\").values\ntest_tag_X = test_df[\"tagged_texts\"].fillna(\"_##_\").values\n\ntokenizer = Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(train_X))\ntrain_X = tokenizer.texts_to_sequences(train_X)\ntest_X = tokenizer.texts_to_sequences(test_X)\n\ntag_tokenizer = Tokenizer(num_words=max_features)\ntag_tokenizer.fit_on_texts(list(train_tag_X))\ntrain_tag_X = tag_tokenizer.texts_to_sequences(train_tag_X)\ntest_tag_X = tag_tokenizer.texts_to_sequences(test_tag_X)\n\ntrain_X = pad_sequences(train_X, maxlen=maxlen)\ntest_X = pad_sequences(test_X, maxlen=maxlen)\n\n# train_tag_X = pad_sequences(train_tag_X, maxlen=maxlen, padding='post', truncating='post')\n# test_tag_X = pad_sequences(test_tag_X, maxlen=maxlen, padding='post', truncating='post')\n\ntrain_tag_X = pad_sequences(train_tag_X, maxlen=maxlen)\ntest_tag_X = pad_sequences(test_tag_X, maxlen=maxlen)\n\ntrain_features = train_df[['caps_vs_length', 'words_vs_unique']].fillna(0)\ntest_features = test_df[['caps_vs_length', 'words_vs_unique']].fillna(0)\nss = StandardScaler()\nss.fit(np.vstack((train_features, test_features)))\ntrain_features = ss.transform(train_features)\ntest_features = ss.transform(test_features)\n\nelapsed_time = (time.time() - start_time) / 60\nprint(elapsed_time)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5ac2781c5d82b4270a1694611e55464bc39f5677"},"cell_type":"code","source":"train_y = train_df['target'].values\n\nnp.random.seed(SEED)\ntrn_idx = np.random.permutation(len(train_X))\n\ntrain_X = train_X[trn_idx]\ntrain_tag_X = train_tag_X[trn_idx]\ntrain_features = train_features[trn_idx]\ntrain_y = train_y[trn_idx]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"14a490af2c3ad2ae3abe72a6f92110d142aa7ccf"},"cell_type":"code","source":"train_X.shape, train_tag_X.shape, train_features.shape, train_y.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"224a6ab1f67155b972ff8142ef135cf3a4c09f4c"},"cell_type":"code","source":"def load_glove(word_index):\n    EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n\n    # word_index = tokenizer.word_index\n    nb_words = min(max_features, len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n            \n    return embedding_matrix \n\ndef load_para(word_index):\n    EMBEDDING_FILE = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore') if len(o)>100)\n\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n\n    # word_index = tokenizer.word_index\n    nb_words = min(max_features, len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n    \n    return embedding_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"345d5aaef2d16f79852a1c595548c2150e74dc24"},"cell_type":"code","source":"start_time = time.time()\n\nword_index = tokenizer.word_index\nembedding_matrix_1 = load_glove(word_index)\nembedding_matrix_2 = load_para(word_index)\n\nembedding_matrix = np.mean([embedding_matrix_1, embedding_matrix_2], axis=0)\n# embedding_matrix = np.concatenate((embedding_matrix_1, embedding_matrix_2), axis=1)\nprint(np.shape(embedding_matrix))\n\ntotal_time = (time.time() - start_time) / 60\nprint(\"Took {:.2f} minutes\".format(total_time))\n\ndel embedding_matrix_1, embedding_matrix_2\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5f2473e4ad39dfb61e310cc348349e5e80cdb8b5"},"cell_type":"code","source":"class Attention(nn.Module):\n    def __init__(self, feature_dim, step_dim, bias=True, **kwargs):\n        super(Attention, self).__init__(**kwargs)\n        \n        self.supports_masking = True\n\n        self.bias = bias\n        self.feature_dim = feature_dim\n        self.step_dim = step_dim\n        self.features_dim = 0\n        \n        weight = torch.zeros(feature_dim, 1)\n        nn.init.xavier_uniform_(weight)\n        self.weight = nn.Parameter(weight)\n        \n        if bias:\n            self.b = nn.Parameter(torch.zeros(step_dim))\n        \n    def forward(self, x, mask=None):\n        feature_dim = self.feature_dim\n        step_dim = self.step_dim\n\n        eij = torch.mm(\n            x.contiguous().view(-1, feature_dim), \n            self.weight\n        ).view(-1, step_dim)\n        \n        if self.bias:\n            eij = eij + self.b\n            \n        eij = torch.tanh(eij)\n        a = torch.exp(eij)\n        \n        if mask is not None:\n            a = a * mask\n\n        a = a / torch.sum(a, 1, keepdim=True) + 1e-10\n\n        weighted_input = x * torch.unsqueeze(a, -1)\n        return torch.sum(weighted_input, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"df9c1427a5a0f79acb5581eb1a8ef1142bae525a"},"cell_type":"code","source":"# https://github.com/salesforce/awd-lstm-lm/issues/86\n\nclass BackHook(torch.nn.Module):\n    def __init__(self, hook):\n        super(BackHook, self).__init__()\n        self._hook = hook\n        self.register_backward_hook(self._backward)\n\n    def forward(self, *inp):\n        return inp\n\n    @staticmethod\n    def _backward(self, grad_in, grad_out):\n        self._hook()\n        return None\n\n\nclass WeightDrop(torch.nn.Module):\n    \"\"\"\n    Implements drop-connect, as per Merity et al https://arxiv.org/abs/1708.02182\n    \"\"\"\n    def __init__(self, module, weights, dropout=0, variational=False):\n        super(WeightDrop, self).__init__()\n        self.module = module\n        self.weights = weights\n        self.dropout = dropout\n        self.variational = variational\n        self._setup()\n        self.hooker = BackHook(lambda: self._backward())\n\n    def _setup(self):\n        for name_w in self.weights:\n            # print('Applying weight drop of {} to {}'.format(self.dropout, name_w))\n            w = getattr(self.module, name_w)\n            self.register_parameter(name_w + '_raw', Parameter(w.data))\n\n    def _setweights(self):\n        for name_w in self.weights:\n            raw_w = getattr(self, name_w + '_raw')\n            mask = raw_w.new_ones((raw_w.size(0), 1))\n            mask = torch.nn.functional.dropout(mask, p=self.dropout, training=True)\n            w = mask.expand_as(raw_w) * raw_w\n            rnn_w = getattr(self.module, name_w)\n            rnn_w.data.copy_(w)\n            setattr(self, name_w + \"_mask\", mask)\n\n    def _backward(self):\n        # transfer gradients from embeddedRNN to raw params\n        for name_w in self.weights:\n            raw_w = getattr(self, name_w + '_raw')\n            rnn_w = getattr(self.module, name_w)\n            raw_w.grad = rnn_w.grad * getattr(self, name_w + \"_mask\")\n\n    def forward(self, *args):\n        self._setweights()\n        return self.module(*self.hooker(*args))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a31ef63ba507c7ab4d701c932188528ac27915c0"},"cell_type":"code","source":"class NeuralNet(nn.Module):\n    def __init__(self):\n        super(NeuralNet, self).__init__()\n        \n        hidden_size = 60\n        \n        self.embedding = nn.Embedding(max_features, embed_size)\n        self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))\n        self.embedding.weight.requires_grad = False\n                \n        # self.lstm = nn.LSTM(embed_size, hidden_size, bidirectional=True, batch_first=True)\n        self.lstm = WeightDrop(nn.LSTM(embed_size, hidden_size, bidirectional=True, batch_first=True), ['weight_hh_l0'], dropout=0.1)\n        # self.gru = nn.GRU(hidden_size*2, hidden_size, bidirectional=True, batch_first=True) \n        self.gru = WeightDrop(nn.GRU(hidden_size*2, hidden_size, bidirectional=True, batch_first=True), ['weight_hh_l0'], dropout=0.1)\n        \n        self.lstm_attention = Attention(hidden_size*2, maxlen)\n        self.gru_attention = Attention(hidden_size*2, maxlen)\n        \n        self.tag_lstm = nn.LSTM(maxlen, hidden_size, bidirectional=True, batch_first=True)\n        self.tag_gru = nn.GRU(hidden_size*2, hidden_size, bidirectional=True, batch_first=True)\n        \n        # self.tag_lstm_attention = Attention(hidden_size*2, maxlen)\n        # self.tag_gru_attention = Attention(hidden_size*2, maxlen)\n        \n        # self.linear = nn.Linear(hidden_size*8, 16)\n        self.linear = WeightDrop(nn.Linear(hidden_size*8+2, 32), ['weight'], dropout=0.1)\n        self.relu = nn.ReLU()\n        self.dropout = nn.Dropout(0.1)\n        self.out = nn.Linear(32, 1)\n        \n    def forward(self, x, y, z):\n        x_embedding = self.embedding(x)     \n        \n        x_lstm, _ = self.lstm(x_embedding)\n        x_gru, _ = self.gru(x_lstm)\n        \n        x_lstm_atten = self.lstm_attention(x_lstm)\n        x_gru_atten = self.gru_attention(x_gru)\n        \n        x_max_pool, _ = torch.max(x_gru, 1)\n        \n        # y_embedding = self.tag_embedding(y)\n        y_lstm, _ = self.tag_lstm(y.view(len(y), -1, maxlen))\n        # y_lstm, _ = self.tag_lstm(y_embedding)\n        y_gru, _ = self.tag_gru(y_lstm)\n        \n        # y_lstm_atten = self.tag_lstm_attention(y_lstm)\n        # y_gru_atten = self.tag_gru_attention(y_gru)\n        y_max_pool_1, _ = torch.max(y_lstm, 1)\n        y_max_pool_2, _ = torch.max(y_gru, 1)\n        \n        y_max_pool = torch.add(y_max_pool_1, y_max_pool_2)\n        \n        conc = torch.cat((x_lstm_atten, x_gru_atten, x_max_pool, y_max_pool, z), 1)\n        conc = self.relu(self.linear(conc))\n        conc = self.dropout(conc)\n        out = self.out(conc)\n        \n        return out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1e559784e58d0ac533c6bd1a631e1243afcea2a8"},"cell_type":"code","source":"splits = list(StratifiedKFold(n_splits=4, shuffle=True, random_state=SEED).split(train_X, train_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c35fe85d3f7f018fc12256faea3354c523cd9989"},"cell_type":"code","source":"def sigmoid(x):\n    return 1 / (1 + np.exp(-x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fd451e4daa5625e38fd1fd2c6a7a9eaa6a9a0037"},"cell_type":"code","source":"def threshold_search(y_true, y_proba):\n    precision, recall, thresholds = precision_recall_curve(y_true, y_proba)\n    thresholds = np.append(thresholds, 1.001) \n    F = 2 / (1/precision + 1/recall)\n    best_score = np.max(F)\n    best_th = thresholds[np.argmax(F)]\n    search_result = {'threshold': best_th , 'f1': best_score}\n    return search_result ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6e3767070c46b7c910494aa4c3210071d618475f"},"cell_type":"code","source":"x_test_cuda = torch.tensor(test_X, dtype=torch.long).cuda()\nx_tag_test_cuda = torch.tensor(test_tag_X, dtype=torch.float32).cuda()\nfeatures_test_cuda = torch.tensor(test_features, dtype=torch.float32).cuda()\ntest = torch.utils.data.TensorDataset(x_test_cuda, x_tag_test_cuda, features_test_cuda)\ntest_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"03d95ccee548c93633ec17efb5714929076b032d"},"cell_type":"code","source":"train_preds = np.zeros((len(train_X)))\ntest_preds = np.zeros((len(test_X)))\n\nfor i, (train_idx, valid_idx) in enumerate(splits):\n    x_train_fold = torch.tensor(train_X[train_idx], dtype=torch.long).cuda()\n    x_tag_train_fold = torch.tensor(train_tag_X[train_idx], dtype=torch.float32).cuda()\n    feature_train_fold = torch.tensor(train_features[train_idx], dtype=torch.float32).cuda()\n    y_train_fold = torch.tensor(train_y[train_idx, np.newaxis], dtype=torch.float32).cuda()\n    x_val_fold = torch.tensor(train_X[valid_idx], dtype=torch.long).cuda()\n    x_tag_val_fold = torch.tensor(train_tag_X[valid_idx], dtype=torch.float32).cuda()\n    feature_val_fold = torch.tensor(train_features[valid_idx], dtype=torch.float32).cuda()\n    y_val_fold = torch.tensor(train_y[valid_idx, np.newaxis], dtype=torch.float32).cuda()\n    \n    model = NeuralNet()\n    model.cuda()\n    \n    loss_fn = torch.nn.BCEWithLogitsLoss(reduction=\"sum\")\n    # optimizer = torch.optim.Adam(model.parameters())\n    \n    train = torch.utils.data.TensorDataset(x_train_fold, x_tag_train_fold, feature_train_fold, y_train_fold)\n    valid = torch.utils.data.TensorDataset(x_val_fold, x_tag_val_fold, feature_val_fold, y_val_fold)\n    \n    train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)\n    valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)\n    \n    print(f'Fold {i + 1}')\n    \n    best_f1 = 0.\n    train_preds_tmp = np.zeros((len(train_X)))\n    test_preds_tmp = np.zeros((len(test_X)))\n    for epoch in range(train_epochs):\n        start_time = time.time()\n        if epoch < 4:\n            optimizer = torch.optim.Adam(model.parameters())\n        else:\n            optimizer = torch.optim.Adam(model.parameters(), lr=0.0001)\n        model.train()\n        avg_loss = 0.\n        for x_batch, x_tag_batch, feature_batch, y_batch in tqdm(train_loader, disable=True):\n            y_pred = model(x_batch, x_tag_batch, feature_batch)\n            loss = loss_fn(y_pred, y_batch)\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n            avg_loss += loss.item() / len(train_loader)\n        \n        model.eval()\n        valid_preds_fold = np.zeros((x_val_fold.size(0)))\n        valid_true_fold = np.zeros((x_val_fold.size(0)))\n        test_preds_fold = np.zeros(len(test_X))\n        avg_val_loss = 0.\n        for i, (x_batch, x_tag_batch, feature_batch, y_batch) in enumerate(valid_loader):\n            y_pred = model(x_batch, x_tag_batch, feature_batch).detach()\n            avg_val_loss += loss_fn(y_pred, y_batch).item() / len(valid_loader)\n            valid_preds_fold[i * batch_size:(i+1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]\n            valid_true_fold[i * batch_size:(i+1) * batch_size] = y_batch.cpu().numpy()[:, 0]\n        \n        elapsed_time = time.time() - start_time \n        search_result = threshold_search(valid_true_fold, valid_preds_fold)\n        \n        if search_result['f1'] > best_f1:\n            train_preds_tmp[valid_idx] = valid_preds_fold\n            for i, (x_batch, x_tag_batch, feature_batch) in enumerate(test_loader):\n                y_pred = model(x_batch, x_tag_batch, feature_batch).detach()\n                test_preds_fold[i * batch_size:(i+1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]\n            test_preds_tmp = test_preds_fold / len(splits)\n            print('Epoch {}/{}\\tloss={:.4f}\\tval_loss={:.4f}\\tthreshold={:.2f}\\tval_f1={:.8f}\\ttime={:.2f}s [impoved]'.format(\n                epoch + 1, train_epochs, avg_loss, avg_val_loss, search_result['threshold'], search_result['f1'], elapsed_time))\n            best_f1 = search_result['f1']\n        else:\n            print('Epoch {}/{}\\tloss={:.4f}\\tval_loss={:.4f}\\tthreshold={:.2f}\\tval_f1={:.8f}\\ttime={:.2f}s [not impoved]'.format(\n                epoch + 1, train_epochs, avg_loss, avg_val_loss, search_result['threshold'], search_result['f1'], elapsed_time))\n            \n    train_preds[valid_idx] += train_preds_tmp[valid_idx]\n    test_preds += test_preds_tmp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6b8211f3e398b72f0d5c2a9b25a38f303cdfc0fa"},"cell_type":"code","source":"search_result = threshold_search(train_y, train_preds)\nsearch_result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"caeec66fc09d9e98520789002e863c320e17cee3"},"cell_type":"code","source":"sub = pd.read_csv('../input/sample_submission.csv')\nsub.prediction = test_preds > search_result['threshold']\nsub.to_csv(\"submission.csv\", index=False)\n\ngrouped = sub.groupby(\"prediction\").size()\nprint(grouped)\n\nprint(grouped[1] / grouped[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"29d36d5bfea29c117086d03d120a7d80a8c1ab21"},"cell_type":"code","source":"import re\n\ndef find_word(word, x):\n    regex = r\"\\b{}\\b\".format(word)\n    return True if re.search(regex, x) else False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a9bd19cec5a4d72d656723e871131adfc7036170"},"cell_type":"code","source":"for i,q in enumerate(test_df.question_text.iloc[:]):\n    if find_word(\"anal\", q):\n        print(sub.prediction.iloc[i], \" : \", q)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fa0e455c18ac85fabad09f377e6f863845b0d97a"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}