{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# standard imports\nimport time\nimport random\nimport os\nfrom IPython.display import display\nimport numpy as np\nimport pandas as pd\n\n# pytorch imports\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.utils.data\nfrom torch.optim.lr_scheduler import _LRScheduler\n\n# imports for preprocessing the questions\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\n\n# cross validation and metrics\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import f1_score\nfrom torch.optim.optimizer import Optimizer\n\n# progress bars\nfrom tqdm import tqdm\ntqdm.pandas()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"scrolled":true},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\ntest_df = pd.read_csv(\"../input/test.csv\")\nprint('Train data dimension: ', train_df.shape)\ndisplay(train_df.head())\nprint('Test data dimension: ', test_df.shape)\ndisplay(test_df.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"31fbc1c50b1bba6b29a18094cdbca6b94481693d"},"cell_type":"code","source":"enable_local_test = False\nif enable_local_test:\n    n_test = len(test_df)*4\n    train_df,local_test_df = (train_df.iloc[:-n_test].reset_index(drop=True),\n                             train_df.iloc[-n_test:].reset_index(drop=True))\nelse:\n    local_test_df = pd.DataFrame([[None,None,0],[None,None,0]],columns=['qid','question_text','target'])\n    n_test = 2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2826717e96febf5911cd557d3913b6ee99aa2f51"},"cell_type":"code","source":"def seed_everything(seed=1234):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\nseed_everything(4396)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aa32f73d2a275d322e2c1f1f21cde70bd6960967"},"cell_type":"code","source":"def threshold_search(y_true, y_proba):\n    best_threshold = 0\n    best_score = 0\n    for threshold in tqdm([i * 0.01 for i in range(100)]):\n        score = f1_score(y_true=y_true, y_pred=y_proba > threshold)\n        if score > best_score:\n            best_threshold = threshold\n            best_score = score\n    search_result = {'threshold': best_threshold, 'f1': best_score}\n    return search_result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2fc64b0c53235cd0b56389f8dcb2724d11927866"},"cell_type":"code","source":"def sigmoid(x):\n    return 1 / (1 + np.exp(-x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"421647c70d16bcf47d53cc01dfb0d5855ae4ea48"},"cell_type":"code","source":"from contextlib import contextmanager\n@contextmanager\ndef timer(name):\n    \"\"\"\n    Taken from Konstantin Lopuhin https://www.kaggle.com/lopuhin\n    in script named : Mercari Golf: 0.3875 CV in 75 LOC, 1900 s\n    https://www.kaggle.com/lopuhin/mercari-golf-0-3875-cv-in-75-loc-1900-s\n    \"\"\"\n    t0 = time.time()\n    yield\n    print(f'[{name}] done in {time.time() - t0:.0f} s')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f65096178e16a177e8ecfe7ffbe81cfa12cd6fa5"},"cell_type":"code","source":"embed_size = 300 # how big is each word vector\nmax_features = 120000 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 70 # max number of words in a question to use","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c9ad126757b52802d415c4a9b10d82f59544c1cf"},"cell_type":"code","source":"#2.map contraction\ncontraction_mapping = {\"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\", \"didn't\": \"did not\",  \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\", \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\",  \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\", \"i'd've\": \"i would have\", \"i'll\": \"i will\",  \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\", \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\", \"must've\": \"must have\", \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\", \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\"so's\": \"so as\", \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\", \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \"they'd've\": \"they would have\", \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\", \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\",  \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\", \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\", \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\", \"you're\": \"you are\", \"you've\": \"you have\" }\ndef clean_contractions(text, mapping):\n    specials = [\"’\", \"‘\", \"´\", \"`\"]\n    for s in specials:\n        text = text.replace(s, \"'\")\n    text = ' '.join([mapping[t] if t in mapping else t for t in text.split(\" \")])\n    return text","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d11270ded13000b40690b1a038518583b353de1a"},"cell_type":"code","source":"puncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', \n '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]\ndef clean_text(x):\n    x = str(x)\n    for punct in puncts:\n        x = x.replace(punct, f' {punct} ')\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2fa8d581bb4556c44d40fe7a5dd4591aea221161"},"cell_type":"code","source":"def reverse_text(text):\n    t = text.split()\n    list.reverse(t)\n    return \" \".join(t)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f996b43093bae6296362bd638dc6687bca05dc0b"},"cell_type":"code","source":"with timer(\"processing\"):\n    train_df[\"clean_question\"] = train_df[\"question_text\"].str.lower()\n    test_df[\"clean_question\"] = test_df[\"question_text\"].str.lower()\n\n    #train_df['clean_question'] = train_df['clean_question'].apply(lambda x:clean_contractions(x,contraction_mapping))\n    #local_test_df['clean_question'] = local_test_df['clean_question'].apply(lambda x:clean_contractions(x,contraction_mapping))\n    \n    train_df[\"clean_question\"] = train_df[\"clean_question\"].apply(lambda x: clean_text(x))\n    test_df[\"clean_question\"] = test_df[\"clean_question\"].apply(lambda x:clean_text(x))\n\n    train_df['reverse_question'] = train_df['clean_question'].apply(lambda x:reverse_text(x))\n    test_df['reverse_question'] = test_df['clean_question'].apply(lambda x:reverse_text(x))\n    \n    x_train = train_df[\"reverse_question\"].fillna(\"_##_\").values\n    x_test = test_df[\"reverse_question\"].fillna(\"_##_\").values\n    \n    # Tokenize the sentences\n    tokenizer = Tokenizer(num_words=max_features)\n    tokenizer.fit_on_texts(list(x_train)+list(x_test))\n    \n    x_train = tokenizer.texts_to_sequences(x_train)\n    x_test = tokenizer.texts_to_sequences(x_test)\n    \n    # Pad the sentences \n    x_train = pad_sequences(x_train, maxlen=maxlen)\n    x_test = pad_sequences(x_test, maxlen=maxlen)\n    \n    # Get the target values\n    y_train = train_df['target'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6af5d1c995d30b59d6b4a2099496988b4fd314ee"},"cell_type":"code","source":"from gensim import utils\ndef fast_load_all(embedding_path,emb_mean,emb_std,ready_pos = None,\\\n                  encoding='latin1',unicode_errors='strict',datatype=np.float32,word_index=None):\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (max_features, 300))\n    in_words = 0\n    stop = 0\n    new_pos = []\n    count = 0\n    with utils.smart_open(embedding_path) as fin:\n        #reading word by word\n        #for _ in range(embedding_length):\n        end = False\n        while end==False:\n            count+=1\n            word = []\n            while True:\n                ch = fin.read(1)\n                if ch == b' ':\n                    break\n                elif ch == b'':\n                    end=True\n                    break\n                    #raise EOFError(\"unexpected end of input\")\n                else:\n                    word.append(ch)\n            word_length = len(word)\n            try:\n                word = utils.to_unicode(b''.join(word), encoding=encoding, errors=unicode_errors)\n            except:\n                word = utils.to_unicode(b''.join(word), encoding='latin1', errors=unicode_errors)\n            #replace digits by ##,###....\n            #if word.isdigit():\n            #    word = ''.join([\"c\"]*len(word))\n            if end==True:\n                print(count)\n                break\n            if word not in word_index:\n                fin.readline()\n                stop += 1\n            else:\n                in_words+=1\n                i = word_index[word]\n                if i>=max_features:\n                    continue\n                if ready_pos is not None:\n                    if ready_pos[i]==1:\n                        continue\n                    else:\n                        ready_pos[i]=1\n                        new_pos.append(i)\n                #in_words.append(word)\n                embedding_matrix[i] = np.asarray(fin.readline().split()[:300],dtype='float32')\n                stop = 0\n    print(\"total ooi words now:{0},total finding words:{2},stopping at:{1}\".format(in_words,stop,np.sum(ready_pos)))\n    return in_words,embedding_matrix,new_pos","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"539ff73a07ce584c55654e091b200446e3d4a3c9"},"cell_type":"code","source":"def load_word2vec(fname,emb_mean,emb_std,ready_pos = None,\\\n                  encoding='utf-8',unicode_errors='strict',datatype=np.float32,word_index=None):\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (max_features, 300))\n    in_words = 0\n    stop = 0\n    new_pos = []\n    count = 0\n    with utils.smart_open(fname) as fin:\n        header = utils.to_unicode(fin.readline(), encoding=encoding)\n        vocab_size,vector_size = (int(x) for x in header.split())\n        #print(vocab_size,vector_size)\n        binary_len = np.dtype(datatype).itemsize*vector_size\n        for _ in range(vocab_size-180000):\n            word = []\n            while True:\n                ch = fin.read(1)\n                if ch==b' ':\n                    break\n                if ch==b'':\n                    raise EOFError(\"unexpected end of input\")\n                if ch!=b'\\n':\n                    word.append(ch)\n            try:\n                word = utils.to_unicode(b''.join(word), encoding=encoding, errors=unicode_errors)\n            except:\n                word = utils.to_unicode(b''.join(word), encoding='latin1', errors=unicode_errors)\n            weights = np.fromstring(fin.read(binary_len),dtype=datatype).astype(datatype)\n            if word not in word_index:\n                stop += 1\n                continue\n            else:\n                in_words+=1\n                i = word_index[word]\n                if i>=max_features:\n                    continue\n                if ready_pos is not None:\n                    if ready_pos[i]==1:\n                        continue\n                    else:\n                        ready_pos[i]=1\n                        new_pos.append(i)\n                embedding_matrix[i] = weights\n                stop = 0\n    print(\"total ooi words now:{0},total finding words:{2},stopping at:{1}\".format(in_words,stop,np.sum(ready_pos)))\n    return in_words,embedding_matrix,new_pos","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"25e9c3e0905bdbcfb5b40538bcb6a07680cf52ca"},"cell_type":"code","source":"#remember that ....encoding may infrence the embedding ...\nseed_everything(4396)\nglove_pos = np.zeros(max_features)\npara_pos = np.zeros(max_features)\nfast_pos = np.zeros(max_features)\nwith timer(\"build embeddings\"):\n    in_glove,embedding_glove,glove_pos = fast_load_all('../input/embeddings/glove.840B.300d/glove.840B.300d.txt',-0.005838499,0.48782197,encoding='utf-8',word_index=tokenizer.word_index,ready_pos=glove_pos)\n    in_para,embedding_para,para_pos = fast_load_all('../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt',-0.0053247833,0.49346462,encoding='utf-8',word_index=tokenizer.word_index,ready_pos=para_pos)\n    in_fast,embedding_fasttext,fast_pos = fast_load_all('../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec',-0.0051106834, 0.18445626,encoding='utf-8',word_index=tokenizer.word_index,ready_pos=fast_pos)\n    \n    #in_google,embedding_google,google_pos = load_word2vec('../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin',-0.0051106834, 0.18445626,encoding='utf-8',word_index=tokenizer.word_index,ready_pos = None)\n    #embedding_matrix = np.mean([embedding_glove,embedding_para],axis=0)\n    print(len(glove_pos),len(para_pos),len(fast_pos))\n    #embedding_glove[para_pos] = embedding_para[para_pos]\n    #embedding_glove[fast_pos] = embedding_fasttext[fast_pos]\n    #embedding_glove[google_pos] = embedding_google[google_pos]\n    #embedding_matrix = embedding_glove\n\n    #print(np.shape(embedding_matrix))\nimport gc\n#del embedding_para,embedding_fasttext,embedding_google\n#gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0b40d76addc7ba4c7d1a188d6d83640f61b5d8be"},"cell_type":"code","source":"diff_glove_para = list(set(para_pos)-set(glove_pos))\ndiff_glove_fast = list(set(fast_pos)-set(glove_pos))\ndiff_glove_p_f = list(set(diff_glove_fast)-set(diff_glove_para))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"af61b1e447f5f892f4e42e1cfaf00c5cef471d76"},"cell_type":"code","source":"splits = list(StratifiedKFold(n_splits=9, shuffle=True, random_state=10).split(x_train, y_train))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"60a1b0d8b3efa124329a60fc1e9fd014554ff678"},"cell_type":"code","source":"# code inspired from: https://github.com/anandsaha/pytorch.cyclic.learning.rate/blob/master/cls.py\nclass CyclicLR(object):\n    def __init__(self, optimizer, base_lr=1e-3, max_lr=6e-3,\n                 step_size=2000, mode='triangular', gamma=1.,\n                 scale_fn=None, scale_mode='cycle', last_batch_iteration=-1):\n\n        if not isinstance(optimizer, Optimizer):\n            raise TypeError('{} is not an Optimizer'.format(\n                type(optimizer).__name__))\n        self.optimizer = optimizer\n        self.lr_history = []\n        if isinstance(base_lr, list) or isinstance(base_lr, tuple):\n            if len(base_lr) != len(optimizer.param_groups):\n                raise ValueError(\"expected {} base_lr, got {}\".format(\n                    len(optimizer.param_groups), len(base_lr)))\n            self.base_lrs = list(base_lr)\n        else:\n            self.base_lrs = [base_lr] * len(optimizer.param_groups)\n\n        if isinstance(max_lr, list) or isinstance(max_lr, tuple):\n            if len(max_lr) != len(optimizer.param_groups):\n                raise ValueError(\"expected {} max_lr, got {}\".format(\n                    len(optimizer.param_groups), len(max_lr)))\n            self.max_lrs = list(max_lr)\n        else:\n            self.max_lrs = [max_lr] * len(optimizer.param_groups)\n\n        self.step_size = step_size\n\n        if mode not in ['triangular', 'triangular2', 'exp_range'] \\\n                and scale_fn is None:\n            raise ValueError('mode is invalid and scale_fn is None')\n\n        self.mode = mode\n        self.gamma = gamma\n        \n        #scheduler althorighms\n        if scale_fn is None:\n            if self.mode == 'triangular':\n                self.scale_fn = self._triangular_scale_fn\n                self.scale_mode = 'cycle'\n            elif self.mode == 'triangular2':\n                self.scale_fn = self._triangular2_scale_fn\n                self.scale_mode = 'cycle'\n            elif self.mode == 'exp_range':\n                self.scale_fn = self._exp_range_scale_fn\n                self.scale_mode = 'iterations'\n        else:\n            self.scale_fn = scale_fn\n            self.scale_mode = scale_mode\n        \n        self.batch_step(last_batch_iteration + 1)\n        self.last_batch_iteration = last_batch_iteration\n\n    def batch_step(self, batch_iteration=None):\n        if batch_iteration is None:\n            batch_iteration = self.last_batch_iteration + 1\n        self.last_batch_iteration = batch_iteration\n        for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()):\n            param_group['lr'] = lr\n\n    def _triangular_scale_fn(self, x):\n        return 1.\n\n    def _triangular2_scale_fn(self, x):\n        return 1 / (2. ** (x - 1))\n\n    def _exp_range_scale_fn(self, x):\n        return self.gamma**(x)\n\n    def get_lr(self):\n        step_size = float(self.step_size)\n        #why 2* step_size?\n        cycle = np.floor(1 + self.last_batch_iteration / (2 * step_size))\n        x = np.abs(self.last_batch_iteration / step_size - 2 * cycle + 1)\n\n        lrs = []\n        param_lrs = zip(self.optimizer.param_groups, self.base_lrs, self.max_lrs)\n        for param_group, base_lr, max_lr in param_lrs:\n            base_height = (max_lr - base_lr) * np.maximum(0, (1 - x))\n            if self.scale_mode == 'cycle':\n                lr = base_lr + base_height * self.scale_fn(cycle)\n            else:\n                lr = base_lr + base_height * self.scale_fn(self.last_batch_iteration)\n            lrs.append(lr)\n        self.lr_history.append(lrs)\n        return lrs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"13af43888c8c81effed8c0ddaf6f1c870f7cf82d"},"cell_type":"code","source":"class Attention(nn.Module):\n    def __init__(self, feature_dim, step_dim, bias=True, **kwargs):\n        super(Attention, self).__init__(**kwargs)\n        \n        self.supports_masking = True\n\n        self.bias = bias\n        self.feature_dim = feature_dim\n        self.step_dim = step_dim\n        self.features_dim = 0\n        \n        weight = torch.zeros(feature_dim, 1)\n        #this is important...\n        nn.init.xavier_uniform_(weight)\n        self.weight = nn.Parameter(weight)\n        \n        if bias:\n            self.b = nn.Parameter(torch.zeros(step_dim))\n        \n    def forward(self, x, mask=None):\n        feature_dim = self.feature_dim\n        step_dim = self.step_dim\n\n        eij = torch.mm(\n            x.contiguous().view(-1, feature_dim), \n            self.weight\n        ).view(-1, step_dim)\n        \n        if self.bias:\n            eij = eij + self.b\n            \n        eij = torch.tanh(eij)\n        a = torch.exp(eij)\n        \n        if mask is not None:\n            a = a * mask\n\n        a = a / torch.sum(a, 1, keepdim=True) + 1e-10\n\n        weighted_input = x * torch.unsqueeze(a, -1)\n        return torch.sum(weighted_input, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ae1047845e81ddbb942919e03e0697ecf71b8856"},"cell_type":"code","source":"use_pretrained_embedding = True\nhidden_size = 60\ngru_len = hidden_size\nRoutings = 4\nNum_capsule = 5\nDim_capsule = 5\nT_epsilon = 1e-7\n# core caps_layer with squash func\nclass Caps_layer(nn.Module):\n    def __init__(self, input_dim_capsule=gru_len * 2, num_capsule=Num_capsule, dim_capsule=Dim_capsule, \\\n                 routings=Routings, kernel_size=(9, 1), share_weights=True,\n                 activation='default', **kwargs):\n        super(Caps_layer, self).__init__(**kwargs)\n\n        self.num_capsule = num_capsule\n        self.dim_capsule = dim_capsule\n        self.routings = routings\n        self.kernel_size = kernel_size  # 暂时没用到\n        self.share_weights = share_weights\n        if activation == 'default':\n            self.activation = self.squash\n        else:\n            self.activation = nn.ReLU(inplace=True)\n\n        if self.share_weights:\n            self.W = nn.Parameter(\n                nn.init.xavier_normal_(torch.empty(1, input_dim_capsule, self.num_capsule * self.dim_capsule)))\n        else:\n            self.W = nn.Parameter(\n                torch.randn(BATCH_SIZE, input_dim_capsule, self.num_capsule * self.dim_capsule))  # 64即batch_size\n\n    def forward(self, x):\n\n        if self.share_weights:\n            u_hat_vecs = torch.matmul(x, self.W)\n        else:\n            print('add later')\n\n        batch_size = x.size(0)\n        input_num_capsule = x.size(1)\n        u_hat_vecs = u_hat_vecs.view((batch_size, input_num_capsule,\n                                      self.num_capsule, self.dim_capsule))\n        u_hat_vecs = u_hat_vecs.permute(0, 2, 1, 3)  # 转成(batch_size,num_capsule,input_num_capsule,dim_capsule)\n        b = torch.zeros_like(u_hat_vecs[:, :, :, 0])  # (batch_size,num_capsule,input_num_capsule)\n\n        for i in range(self.routings):\n            b = b.permute(0, 2, 1)\n            c = F.softmax(b, dim=2)\n            c = c.permute(0, 2, 1)\n            b = b.permute(0, 2, 1)\n            outputs = self.activation(torch.einsum('bij,bijk->bik', (c, u_hat_vecs)))  # batch matrix multiplication\n            # outputs shape (batch_size, num_capsule, dim_capsule)\n            if i < self.routings - 1:\n                b = torch.einsum('bik,bijk->bij', (outputs, u_hat_vecs))  # batch matrix multiplication\n        return outputs  # (batch_size, num_capsule, dim_capsule)\n\n    # text version of squash, slight different from original one\n    def squash(self, x, axis=-1):\n        s_squared_norm = (x ** 2).sum(axis, keepdim=True)\n        scale = torch.sqrt(s_squared_norm + T_epsilon)\n        return x / scale\n\"\"\"\nclass Capsule_Main(nn.Module):\n    def __init__(self, embedding_matrix=None, vocab_size=None):\n        super(Capsule_Main, self).__init__()\n        self.embed_layer = Embed_Layer(embedding_matrix, vocab_size)\n        self.gru_layer = GRU_Layer()\n        # 【重要】初始化GRU权重操作，这一步非常关键，acc上升到0.98，如果用默认的uniform初始化则acc一直在0.5左右\n        self.gru_layer.init_weights()\n        self.caps_layer = Caps_Layer()\n        self.dense_layer = Dense_Layer()\n\n    def forward(self, content):\n        content1 = self.embed_layer(content)\n        content2, _ = self.gru_layer(\n            content1)  # 这个输出是个tuple，一个output(seq_len, batch_size, num_directions * hidden_size)，一个hn\n        content3 = self.caps_layer(content2)\n        output = self.dense_layer(content3)\n        return output\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4e6fc90a5846f7b90a213b69c6bf30028cf32cca"},"cell_type":"code","source":"class NeuralNet_2(nn.Module):\n    def __init__(self,embeddings=None):\n        super(NeuralNet_2,self).__init__()\n        hidden_size = 60\n        fc_layer = 16\n        fc_layer1 = 16\n        self.embedding = nn.Embedding(max_features,embed_size)\n        self.embedding.weight = nn.Parameter(torch.tensor(embeddings,dtype=torch.float32))\n        self.embedding.weight.requires_grad = False\n        \n        #self.embedding_dropoout = nn.Dropout2d(0.1)\n        self.lstm = nn.LSTM(embed_size,hidden_size,bidirectional=True,batch_first = True)\n        self.gru = nn.GRU(hidden_size*2,hidden_size,bidirectional=True,batch_first=True)\n        #self.lstm2 = nn.LSTM(hidden_size*2,hidden_size,bidirectional=True,batch_first=True)\n        \n        self.lstm_attention = Attention(hidden_size*2,maxlen)\n        self.gru_attention = Attention(hidden_size*2,maxlen)\n        self.bn = nn.BatchNorm1d(16,momentum=0.5)\n        self.linear = nn.Linear(hidden_size*8+1,fc_layer1)\n        self.relu = nn.ReLU()\n        self.dropout = nn.Dropout(0.1)\n        self.fc = nn.Linear(fc_layer**2,fc_layer)\n        self.out = nn.Linear(fc_layer,1)\n        \n        self.lincaps = nn.Linear(Num_capsule*Dim_capsule,1)\n        self.caps_layer = Caps_layer()\n    def forward(self,x):\n        h_embedding = self.embedding(x)\n        \n        h_lstm,_ = self.lstm(h_embedding)\n        h_gru,_ = self.gru(h_lstm)\n        #caps layer\n        contents3 = self.caps_layer(h_gru)\n        contents3 = self.dropout(contents3)\n        batch_size =contents3.size(0)\n        contents3 = contents3.view(batch_size,-1)\n        contents3 = self.relu(self.lincaps(contents3))\n        #attention layer\n        h_lstm_atten = self.lstm_attention(h_lstm)\n        h_gru_atten = self.gru_attention(h_gru)\n        \n        avg_pool = torch.mean(h_gru,1)\n        max_pool,_ = torch.max(h_gru,1)\n        #extra features\n        #f = torch.tensor(x[1],dtype=torch.float).cuda()\n        \n        conc = torch.cat((h_lstm_atten,h_gru_atten,contents3,avg_pool,max_pool),1)\n        #print(conc.size(),h_lstm_atten.size(),h_gru_atten.size(),contents3.size(),\n        #     avg_pool.size(),max_pool.size())\n        conc = self.relu(self.linear(conc))\n        #conc = self.bn(conc)\n        conc = self.dropout(conc)\n        out = self.out(conc)\n        return out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8a344b26283e13cadaf1089f70b72cebc71c38b9"},"cell_type":"code","source":"class NeuralNet(nn.Module):\n    def __init__(self,embeddings=None):\n        super(NeuralNet, self).__init__()\n        \n        hidden_size = 60\n        \n        self.embedding = nn.Embedding(max_features, embed_size)\n        self.embedding.weight = nn.Parameter(torch.tensor(embeddings, dtype=torch.float32))\n        self.embedding.weight.requires_grad = False\n        \n        self.embedding_dropout = nn.Dropout2d(0.1)\n        #seq_len,batch,input_size\n        self.lstm = nn.LSTM(embed_size, hidden_size, bidirectional=True, batch_first=True)\n        self.gru = nn.GRU(hidden_size * 2, hidden_size, bidirectional=True, batch_first=True)\n        \n        self.lstm_attention = Attention(hidden_size * 2, maxlen)\n        self.gru_attention = Attention(hidden_size * 2, maxlen)\n        \n        self.linear = nn.Linear(hidden_size*8, 16)\n        self.relu = nn.ReLU()\n        self.dropout = nn.Dropout(0.1)\n        self.out = nn.Linear(16, 1)\n    \n    def forward(self, x):\n        h_embedding = self.embedding(x)\n        #h_embedding = torch.squeeze(\n        #    self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))\n        \n        h_lstm, _ = self.lstm(h_embedding)\n        h_gru, _ = self.gru(h_lstm)\n        \n        h_lstm_atten = self.lstm_attention(h_lstm)\n        h_gru_atten = self.gru_attention(h_gru)\n        \n        # global average pooling\n        avg_pool = torch.mean(h_gru, 1)\n        # global max pooling\n        max_pool, _ = torch.max(h_gru, 1)\n        \n        conc = torch.cat((h_lstm_atten, h_gru_atten, avg_pool, max_pool), 1)\n        \n        conc = self.relu(self.linear(conc))\n        conc = self.dropout(conc)\n        out = self.out(conc)\n        \n        return out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fb80af14d889d50663a8a562eb00be24f4b4d4cb"},"cell_type":"code","source":"def create_pseudo_data(model,unlabeled_data_loader,batch_size,sample_rate = 0.2):\n    num_of_samples = int(len(unlabeled_data_loader.dataset)*sample_rate)\n    pseudo_preds = np.zeros(len(unlabeled_data_loader.dataset))\n    print(\"creating pseudo labels!!\")\n    model.eval()\n    for i,(x_batch,) in enumerate(unlabeled_data_loader):\n        y_pred = model(x_batch).detach()\n        pseudo_preds[i*batch_size:(i+1)*batch_size] = sigmoid(y_pred.cpu().numpy())[:,0]\n    model.train()\n    pseudo_index = np.random.choice(range(len(pseudo_preds)),int(len(pseudo_preds)*sample_rate),replace=False)\n    return pseudo_index,pseudo_preds[pseudo_index]\n\ndef create_augmented_train(model,train_data,train_y,unlabeled_data,batch_size,sample_rate=0.2):\n    unlabeled_data_cuda = torch.tensor(unlabeled_data,dtype=torch.long).cuda()\n    unlabeled_data_ = torch.utils.data.TensorDataset(unlabeled_data_cuda)\n    unlabeled_data_loader = torch.utils.data.DataLoader(unlabeled_data_,batch_size=batch_size,shuffle=False)\n    \n    pseudo_index,pseudo_labels = create_pseudo_data(model,unlabeled_data_loader,batch_size,sample_rate=sample_rate)\n    pseudo_data = unlabeled_data[pseudo_index].copy()\n    new_train_data = train_data.copy()\n    new_train_y = train_y.copy()\n    new_train_data = np.concatenate([new_train_data,pseudo_data],axis=0)\n    new_train_y = np.concatenate([new_train_y,pseudo_labels[:,np.newaxis]],axis=0)\n    \n    new_x_train_fold = torch.tensor(new_train_data,dtype=torch.long).cuda()\n    new_y_train_fold = torch.tensor(new_train_y,dtype=torch.float32).cuda()\n    \n    new_train = torch.utils.data.TensorDataset(new_x_train_fold,new_y_train_fold)\n    new_train_loader = torch.utils.data.DataLoader(new_train,batch_size=batch_size,shuffle=True)\n    return new_train_loader","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"42f9402a1eb5b5982299aeab7c8add50e1eb0f8f"},"cell_type":"code","source":"batch_size = 512 # how many samples to process at once\nn_epochs =4# how many times to iterate over all samples","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4efad305845638ade03cb6b4d35da5a0bd57c544"},"cell_type":"code","source":"def train_model(model,x_train,y_train,x_val=None,y_val=None,n_epochs=4,validate=True,using_pseudo=False):\n    x_train_fold = torch.tensor(x_train, dtype=torch.long).cuda()\n    y_train_fold = torch.tensor(y_train, dtype=torch.float32).cuda()\n    train = torch.utils.data.TensorDataset(x_train_fold,y_train_fold)\n    train_loader = torch.utils.data.DataLoader(train,batch_size=batch_size,shuffle=True)\n    if validate:\n        x_val_fold = torch.tensor(x_val, dtype=torch.long).cuda()\n        y_val_fold = torch.tensor(y_val, dtype=torch.float32).cuda()\n        valid = torch.utils.data.TensorDataset(x_val_fold,y_val_fold)\n        val_loader = torch.utils.data.DataLoader(valid,batch_size=batch_size,shuffle=False)\n\n    base_lr, max_lr = 0.001, 0.003 \n    optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), \n                             lr=max_lr)\n\n    scheduler = CyclicLR(optimizer,base_lr =base_lr ,max_lr=max_lr,\n                        step_size =len(train_loader) ,mode='triangular',gamma=0.994)\n    \n    loss_fn = torch.nn.BCEWithLogitsLoss(reduction='mean')\n    best_score = -np.inf\n    test_preds_local = np.zeros(len(test_local_loader.dataset))\n    #now the test is local test\n    for epoch in range(n_epochs):\n        #print(optimizer.param_groups[0]['lr'])\n        # set train mode of the model. This enables operations which are only applied during training like dropout\n        start_time = time.time()\n        model.train()\n        avg_loss = 0.\n        aug_num = 0\n        \n        if epoch==n_epochs-1:\n            if using_pseudo==True:\n                #model,train_data,train_y,unlabeled_data,batch_size,sample_rate=0.2\n                train_loader = create_augmented_train(model,x_train,y_train,x_test_local,batch_size=batch_size,sample_rate=0.2)\n        \n        for x_batch, y_batch in tqdm(train_loader, disable=True):\n            #sgdr.step()\n            \n            #x_batch,aug_ = augment(x_batch)\n            #aug_num += aug_\n            # Forward pass: compute predicted y by passing x to the model.\n            y_pred = model(x_batch)\n            #print(y_pred.shape,y_batch.shape)\n            # Compute and print loss.\n            if scheduler:\n                scheduler.batch_step()\n            \n            loss = loss_fn(y_pred, y_batch)\n\n            # Before the backward pass, use the optimizer object to zero all of the\n            # gradients for the Tensors it will update (which are the learnable weights\n            # of the model)\n            optimizer.zero_grad()\n            \n            # Backward pass: compute gradient of the loss with respect to model parameters\n            loss.backward()\n            #nn.utils.clip_grad_norm(self.G.parameters(),1.0)\n            # Calling the step function on an Optimizer makes an update to its parameters\n            optimizer.step()\n            avg_loss += loss.item() / len(train_loader)\n        #print(\"aug num:{0}\".format(aug_num))\n        model.eval()\n        if validate:\n            valid_preds = np.zeros((x_val_fold.size(0)))\n            avg_val_loss = 0.\n            for i,(x_batch,y_batch) in enumerate(val_loader):\n                y_pred = model(x_batch).detach()\n                avg_val_loss+= loss_fn(y_pred,y_batch).item()/len(val_loader)\n                valid_preds[i*batch_size:(i+1)*batch_size] = sigmoid(y_pred.cpu().numpy())[:,0]\n            search_result = threshold_search(y_val, valid_preds)\n            val_f1, val_threshold = search_result['f1'], search_result['threshold']\n            elapsed_time = time.time() - start_time\n            print('Epoch {}/{} \\t loss={:.4f} \\t val_loss={:.4f} \\t val_f1={:.4f} best_t={:.2f} \\t time={:.2f}s'.format(\n                epoch + 1, n_epochs, avg_loss, avg_val_loss, val_f1, val_threshold, elapsed_time))\n        else:\n            elapsed_time = time.time()-start_time\n            print('Epoch {}/{} \\t loss={:.4f} \\t time={:.2f}s'.format(\n                epoch + 1, n_epochs, avg_loss, elapsed_time))\n    if validate:\n        valid_preds = np.zeros((x_val_fold.size(0)))\n        avg_val_loss = 0.\n        for i, (x_batch, y_batch) in enumerate(val_loader):\n            y_pred = model(x_batch).detach()\n\n            avg_val_loss += loss_fn(y_pred, y_batch).item() / len(val_loader)\n            valid_preds[i * batch_size:(i+1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]\n\n        print('Validation loss: ', avg_val_loss)\n          \n    test_preds = np.zeros(len(test_df))\n    test_preds_local = np.zeros(len(test_local_loader.dataset))\n    #now the test is local test\n    for i,(x_batch,)in enumerate(test_local_loader):\n        y_pred = model(x_batch).detach()\n        test_preds_local[i*batch_size:(i+1)*batch_size]=sigmoid(y_pred.cpu().numpy())[:,0]\n    if validate:\n        return valid_preds,test_preds,test_preds_local\n    else:\n        return test_preds,test_preds_local","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cd8b7db3f1018aa4efef5615a128c00a9e513e17"},"cell_type":"code","source":"x_test_local = x_test\nx_train = x_train\ny_train = y_train\nsingle_or_multi = 2","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c1436b3a877add6bee5ca149d8bbf8a2a52e64b0"},"cell_type":"markdown","source":"**Training for single models with folds**"},{"metadata":{"trusted":true,"_uuid":"68ff5e0605c47428b62264935b9a173e600333a3","scrolled":true},"cell_type":"code","source":"if single_or_multi==1:\n    seed=6017\n    train_preds = np.zeros(len(train_df))\n    test_preds = np.zeros((len(test_df), len(splits)))\n    test_preds_local = np.zeros((n_test, len(splits)))\n\n    x_test_local_cuda = torch.tensor(x_test_local,dtype=torch.long).cuda()\n    test_local = torch.utils.data.TensorDataset(x_test_local_cuda)\n    test_local_loader = torch.utils.data.DataLoader(test_local, batch_size=batch_size, shuffle=False)\n\n    for i,(train_idx,valid_idx) in enumerate(splits):\n        x_train_fold = x_train[train_idx]\n        y_train_fold = y_train[train_idx, np.newaxis]\n        x_val_fold = x_train[valid_idx]\n        y_val_fold = y_train[valid_idx, np.newaxis]\n\n        print(f'Fold {i+1}')\n        #########################\n        #using diff seeds for adding randomness....for better ensemble\n        seed_everything(seed+i)\n        model = NeuralNet()\n        model.cuda()\n        valid_preds_fold, test_preds_fold, test_preds_local_fold = train_model(model,\n                                                                               x_train_fold, \n                                                                               y_train_fold, \n                                                                               x_val_fold, \n                                                                               y_val_fold, \n                                                                               n_epochs=n_epochs,\n                                                                               validate=True,\n                                                                              using_pseudo=False)\n        train_preds[valid_idx] = valid_preds_fold\n        test_preds[:,i] = test_preds_fold\n        test_preds_local[:,i] = test_preds_local_fold","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6467e38442ac415bc0da932f1f5c1373d4b7b5c2"},"cell_type":"markdown","source":"**Training many models and stacking them**"},{"metadata":{"trusted":true,"_uuid":"2017b4a10865c44f03fb1008533c9cba89cc9808"},"cell_type":"code","source":"embedding_matrix_1 = embedding_glove.copy()\nembedding_matrix_1[diff_glove_para] = embedding_para[diff_glove_para]\nembedding_matrix_1[diff_glove_p_f] = embedding_fasttext[diff_glove_p_f]\n\nembedding_matrix_2 = embedding_para.copy()\nembedding_matrix_2[diff_glove_p_f] = embedding_fasttext[diff_glove_p_f]\n\nembedding_matrix_3 = (embedding_glove+embedding_para)/2.0\n\nembedding_matrix_4 = (0.45*embedding_glove+0.35*embedding_para+0.20*embedding_fasttext)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7105c27c812ca82c1776305d8cc7b7055b71ab54"},"cell_type":"code","source":"del embedding_glove,embedding_para,embedding_fasttext\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"39e379ba9293a3f56a9e87dc1e515eba93e15ebd"},"cell_type":"markdown","source":"**defing models**"},{"metadata":{"trusted":true,"_uuid":"90bacafc3fc0541f48474083d78cde2381f7ac93"},"cell_type":"code","source":"models={\n    \"glove_main_model_1\":NeuralNet(embeddings=embedding_matrix_1),\n    #\"para_main_model_1\":NeuralNet(embeddings=embedding_matrix_2),\n    \"avg_2_model_1\":NeuralNet(embeddings=embedding_matrix_3),\n    \"avg_3_model_1\":NeuralNet(embeddings=embedding_matrix_4),\n    \"glove_main_model_2\":NeuralNet_2(embeddings=embedding_matrix_1),\n    \"para_main_model_2\":NeuralNet_2(embeddings=embedding_matrix_2),\n    \"avg_2_model_2\":NeuralNet_2(embeddings=embedding_matrix_3),\n    \"avg_3_model_2\":NeuralNet_2(embeddings=embedding_matrix_4)\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7d8de592809acf472c0ce3dcc627e488543c770c"},"cell_type":"code","source":"if single_or_multi==2:\n    seed = 6017\n    \n    epochs = [4]*len(models)\n\n    y_train = y_train[:, np.newaxis]\n    \n    test_preds = np.zeros((len(test_df),len(models)))\n    test_preds_local = np.zeros((n_test,len(models)))\n\n    x_test_local_cuda = torch.tensor(x_test_local,dtype=torch.long).cuda()\n    test_local = torch.utils.data.TensorDataset(x_test_local_cuda)\n    test_local_loader = torch.utils.data.DataLoader(test_local, batch_size=batch_size, shuffle=False)\n    \n    i=0\n    #for i in range(len(models)):\n    for key,model in models.items():\n        print('model {0}'.format(key))\n        #seed_everything(seed+i)\n        model.cuda()\n        test_preds_fold,test_preds_local_fold = train_model(\n            model,x_train,y_train,n_epochs=epochs[i],validate=False)\n\n        test_preds[:,i] = test_preds_local_fold\n        #test_preds_local[:,i] = test_preds_local_fold\n        i+=1\n    \"\"\"\n    for i in range(len(models)):\n        search_result = threshold_search(y_test,test_preds_local[:,i])\n        print(\"model {0}:\".format(i))\n        print(search_result)\n    \"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f3ddcd65a00ac51d51d5d3389529aed6bfede71e"},"cell_type":"code","source":"magic_numbers = [0.22161071,0.13261581,0.13514855,0.18424001,0.09617107,0.13212665,0.0977806]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9af4d7f9774a82f666242473e35c3a6e6af0307d"},"cell_type":"code","source":"submission = test_df[['qid']].copy()\nsubmission['prediction'] = np.sum(test_preds*np.array(magic_numbers),axis=1)>0.37\nsubmission.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}