{"cells":[{"metadata":{"trusted":true,"_uuid":"526ee93c6d5039ad3fb0f06f465f8e05e1f2a64e"},"cell_type":"code","source":"import gc\n#import spacy\nfrom tqdm import tqdm_notebook, tnrange\nfrom tqdm.auto import tqdm\ntqdm.pandas(desc='Progress')\nimport torch.nn.functional as F","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nimport re\nfrom torch.utils import data\nimport torch\nimport torch.nn as nn\nimport numpy as np\nimport random\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn import metrics\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nimport tqdm\nimport time\nfrom torch.utils import data\nfrom torch.autograd import Variable\nfrom torch.optim.optimizer import Optimizer\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6abdaf45543837fd3ecdeef340fdf57e5fdbd654"},"cell_type":"code","source":"embed_size = 300 # how big is each word vector\nmax_features = 200000 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 40 # max number of words in a question to use\nbatch_size=1024","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b07ca3dd47c32a9c136dcf6dd8a8af95cfd74df0"},"cell_type":"code","source":"def seed_everything(seed=1029):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\nseed_everything()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5c0958fdac2475ba8b6a831007b361b7d2ad16ef"},"cell_type":"code","source":"def sigmoid(x):\n    return 1 / (1 + np.exp(-x))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9f43d78c1501b69e1f5cbe5d58cb7f3ee7f70187"},"cell_type":"markdown","source":"**数据处理**"},{"metadata":{"trusted":true,"_uuid":"5136865814c735080c214c4251b916008ed9de97"},"cell_type":"code","source":"puncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', \n '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]\n\ndef clean_text(x):\n    x = str(x)\n    for punct in puncts:\n        if punct in x:\n            x = x.replace(punct, f' {punct} ')\n    return x\n\n\ndef clean_numbers(x):\n    if bool(re.search(r'\\d', x)):\n        x = re.sub('[0-9]{5,}', '#####', x)\n        x = re.sub('[0-9]{4}', '####', x)\n        x = re.sub('[0-9]{3}', '###', x)\n        x = re.sub('[0-9]{2}', '##', x)\n    return x\n\nmispell_dict = {\"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\", \"didn't\": \"did not\",  \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\", \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\",  \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\", \"i'd've\": \"i would have\", \"i'll\": \"i will\",  \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\", \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\", \"must've\": \"must have\", \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\", \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\"so's\": \"so as\", \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\", \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \"they'd've\": \"they would have\", \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\", \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\",  \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\", \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\", \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\", \"you're\": \"you are\", \"you've\": \"you have\", 'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', \"mastrubating\": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', \"whst\": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}\n\ndef _get_mispell(mispell_dict):\n    mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n    return mispell_dict, mispell_re\n\nmispellings, mispellings_re = _get_mispell(mispell_dict)\ndef replace_typical_misspell(text):\n    def replace(match):\n        return mispellings[match.group(0)]\n    return mispellings_re.sub(replace, text)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9fff018480098e8fe5c58628ea3e9c6489635181"},"cell_type":"code","source":"def load_and_prec():\n    train_df = pd.read_csv(\"../input/train.csv\")\n    test_df = pd.read_csv(\"../input/test.csv\")\n    print(\"Train shape : \",train_df.shape)\n    print(\"Test shape : \",test_df.shape)    \n    # lower\n    train_df[\"question_text\"] = train_df[\"question_text\"].apply(lambda x: x.lower())\n    test_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x: x.lower())\n\n    # Clean the text\n    train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: clean_text(x))\n    test_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x: clean_text(x))\n    \n    # Clean numbers\n    train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: clean_numbers(x))\n    test_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x: clean_numbers(x))\n    \n    # Clean speelings\n    train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: replace_typical_misspell(x))\n    test_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x: replace_typical_misspell(x))\n    \n    ## fill up the missing values\n    train_X = train_df[\"question_text\"].fillna(\"_##_\").values\n    test_X = test_df[\"question_text\"].fillna(\"_##_\").values\n    \n    ###########################################################################\n\n    ## Tokenize the sentences\n    tokenizer = Tokenizer(num_words=max_features)\n    tokenizer.fit_on_texts(list(train_X))\n    train_X = tokenizer.texts_to_sequences(train_X)\n    test_X = tokenizer.texts_to_sequences(test_X)\n\n    ## Pad the sentences \n    train_X = pad_sequences(train_X, maxlen=maxlen)\n    test_X = pad_sequences(test_X, maxlen=maxlen)\n\n    ## Get the target values\n    train_y = train_df['target'].values\n    return train_X, test_X, train_y,tokenizer.word_index","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3e64035007c96c6517d0469a2ebe2ded3481c3b9"},"cell_type":"markdown","source":"**预训练模型**"},{"metadata":{"trusted":true,"_uuid":"8c1fabc74fede0d7511613dd5cd8c10014dbec1c"},"cell_type":"code","source":"def load_glove(word_index):\n    EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')[:300]\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n    \n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = -0.005838499,0.48782197\n    embed_size = all_embs.shape[1]\n\n    # word_index = tokenizer.word_index\n    nb_words = min(max_features, len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words+1, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n            \n    return embedding_matrix \n    \ndef load_fasttext(word_index):    \n    EMBEDDING_FILE = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE) if len(o)>100)\n\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n\n    # word_index = tokenizer.word_index\n    nb_words = min(max_features, len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words+1, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n\n    return embedding_matrix\n\ndef load_para(word_index):\n    EMBEDDING_FILE = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore') if len(o)>100)\n\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = -0.0053247833,0.49346462\n    embed_size = all_embs.shape[1]\n\n    # word_index = tokenizer.word_index\n    nb_words = min(max_features, len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words+1, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n    \n    return embedding_matrix","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2234b009100153e65e861f64cc41015173c6ef8f"},"cell_type":"markdown","source":"**数据集**"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"class QuraData(data.Dataset):\n    def __init__(self,questions,labels,augument=False,training=True):\n        super(QuraData, self).__init__()\n        self.augument=augument\n        self.questions=questions\n        self.labels= labels\n        self.len_ = len(self.questions)\n        self.training=training\n    def shuffle(self,d):\n        return np.random.permutation(d.tolist())\n\n    def dropout(self,d,p=0.5):\n        len_ = len(d)\n        index = np.random.choice(len_,int(len_*p))\n        d[index]=0\n        return d     \n    def __getitem__(self,index):\n        question,label =  self.questions[index],self.labels[index,np.newaxis]\n    \n        if self.training and self.augument :\n            question= self.dropout(question,p=0.05)\n        question=torch.from_numpy(question).long()\n        label=torch.LongTensor(label).long()\n        return question,label\n\n    def __len__(self):\n        return self.len_\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f797532df240a5c1aca292b4a95e168d9a2f2f94"},"cell_type":"markdown","source":"**模型**"},{"metadata":{"trusted":true,"_uuid":"b976b62645dacb61bdae4e1fa8cbfe33fdd3e330"},"cell_type":"code","source":"class Attention(nn.Module):\n    def __init__(self, feature_dim, step_dim, bias=True, **kwargs):\n        super(Attention, self).__init__(**kwargs)\n        \n        self.supports_masking = True\n\n        self.bias = bias\n        self.feature_dim = feature_dim\n        self.step_dim = step_dim\n        self.features_dim = 0\n        \n        weight = torch.zeros(feature_dim, 1)\n        nn.init.xavier_uniform_(weight)\n        self.weight = nn.Parameter(weight)\n        \n        if bias:\n            self.b = nn.Parameter(torch.zeros(step_dim))\n        \n    def forward(self, x, mask=None):\n        feature_dim = self.feature_dim\n        step_dim = self.step_dim\n\n        eij = torch.mm(\n            x.contiguous().view(-1, feature_dim), \n            self.weight\n        ).view(-1, step_dim)\n        \n        if self.bias:\n            eij = eij + self.b\n            \n        eij = torch.tanh(eij)\n        a = torch.exp(eij)\n        \n        if mask is not None:\n            a = a * mask\n\n        a = a / torch.sum(a, 1, keepdim=True) + 1e-10\n\n        weighted_input = x * torch.unsqueeze(a, -1)\n        return torch.sum(weighted_input, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a4c61b46f84c534e73a41303877480612ca5bec2"},"cell_type":"code","source":"# code inspired from: https://github.com/anandsaha/pytorch.cyclic.learning.rate/blob/master/cls.py\nclass CyclicLR(object):\n    def __init__(self, optimizer, base_lr=1e-3, max_lr=6e-3,\n                 step_size=2000, mode='triangular', gamma=1.,\n                 scale_fn=None, scale_mode='cycle', last_batch_iteration=-1):\n\n        if not isinstance(optimizer, Optimizer):\n            raise TypeError('{} is not an Optimizer'.format(\n                type(optimizer).__name__))\n        self.optimizer = optimizer\n\n        if isinstance(base_lr, list) or isinstance(base_lr, tuple):\n            if len(base_lr) != len(optimizer.param_groups):\n                raise ValueError(\"expected {} base_lr, got {}\".format(\n                    len(optimizer.param_groups), len(base_lr)))\n            self.base_lrs = list(base_lr)\n        else:\n            self.base_lrs = [base_lr] * len(optimizer.param_groups)\n\n        if isinstance(max_lr, list) or isinstance(max_lr, tuple):\n            if len(max_lr) != len(optimizer.param_groups):\n                raise ValueError(\"expected {} max_lr, got {}\".format(\n                    len(optimizer.param_groups), len(max_lr)))\n            self.max_lrs = list(max_lr)\n        else:\n            self.max_lrs = [max_lr] * len(optimizer.param_groups)\n\n        self.step_size = step_size\n\n        if mode not in ['triangular', 'triangular2', 'exp_range'] \\\n                and scale_fn is None:\n            raise ValueError('mode is invalid and scale_fn is None')\n\n        self.mode = mode\n        self.gamma = gamma\n\n        if scale_fn is None:\n            if self.mode == 'triangular':\n                self.scale_fn = self._triangular_scale_fn\n                self.scale_mode = 'cycle'\n            elif self.mode == 'triangular2':\n                self.scale_fn = self._triangular2_scale_fn\n                self.scale_mode = 'cycle'\n            elif self.mode == 'exp_range':\n                self.scale_fn = self._exp_range_scale_fn\n                self.scale_mode = 'iterations'\n        else:\n            self.scale_fn = scale_fn\n            self.scale_mode = scale_mode\n\n        self.batch_step(last_batch_iteration + 1)\n        self.last_batch_iteration = last_batch_iteration\n\n    def batch_step(self, batch_iteration=None):\n        if batch_iteration is None:\n            batch_iteration = self.last_batch_iteration + 1\n        self.last_batch_iteration = batch_iteration\n        for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()):\n            param_group['lr'] = lr\n\n    def _triangular_scale_fn(self, x):\n        return 1.\n\n    def _triangular2_scale_fn(self, x):\n        return 1 / (2. ** (x - 1))\n\n    def _exp_range_scale_fn(self, x):\n        return self.gamma**(x)\n\n    def get_lr(self):\n        step_size = float(self.step_size)\n        cycle = np.floor(1 + self.last_batch_iteration / (2 * step_size))\n        x = np.abs(self.last_batch_iteration / step_size - 2 * cycle + 1)\n\n        lrs = []\n        param_lrs = zip(self.optimizer.param_groups, self.base_lrs, self.max_lrs)\n        for param_group, base_lr, max_lr in param_lrs:\n            base_height = (max_lr - base_lr) * np.maximum(0, (1 - x))\n            if self.scale_mode == 'cycle':\n                lr = base_lr + base_height * self.scale_fn(cycle)\n            else:\n                lr = base_lr + base_height * self.scale_fn(self.last_batch_iteration)\n            lrs.append(lr)\n        return lrs\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"697ec5500495ebda91deb4a25199ed93c1c94b34"},"cell_type":"code","source":"def kmax_pooling(x, dim, k):\n    index = x.topk(k, dim = dim)[1].sort(dim = dim)[0]\n    return x.gather(dim, index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f7840e00d341c6e1034679a89a7ac5f1a33257a0"},"cell_type":"code","source":"class Gate(nn.Module):\n    def __init__(self, input_size):\n        super(Gate, self).__init__()\n        self.linear = nn.Linear(input_size, input_size)\n\n    def forward(self, x):\n        x_proj = self.linear(x)\n        gate = torch.sigmoid(x_proj)\n        return x * gate","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b130d011c6610d28ab507445ef126d19d63ac6a1"},"cell_type":"code","source":"class SFU(nn.Module):\n    \"\"\"Semantic Fusion Unit\n    The ouput vector is expected to not only retrieve correlative information from fusion vectors,\n    but also retain partly unchange as the input vector\n    \"\"\"\n    def __init__(self, input_size, fusion_size):\n        super(SFU, self).__init__()\n        self.linear_r = nn.Linear(input_size + fusion_size, input_size)\n        self.linear_g = nn.Linear(input_size + fusion_size, input_size)\n\n    def forward(self, x, fusions):\n        r_f = torch.cat([x, fusions], 2)\n        r = F.tanh(self.linear_r(r_f))\n        g = F.sigmoid(self.linear_g(r_f))\n        o = g * r + (1-g) * x\n        return o","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bc2db89e212dbf8c0fe84ca20ce1aa2de0fb7af8"},"cell_type":"code","source":"kernel_sizes =  [2,3,5]\nclass mergeNN(nn.Module): \n    def __init__(self,embedding_matrix):\n        super(mergeNN, self).__init__()\n        self.model_name = 'LSTMText'\n        self.encoder = nn.Embedding.from_pretrained(torch.FloatTensor(embedding_matrix))\n        self.title_lstm = nn.LSTM(input_size = embed_size*2,\\\n                            hidden_size = 128,\n                            num_layers =1,\n                            bias = True,\n                            batch_first = True,\n                            #dropout = 0.2,\n                            bidirectional = True\n                            )\n        self.title_gru = nn.GRU(input_size = 256,\\\n                            hidden_size = 128,\n                            num_layers =1,\n                            bias = True,\n                            batch_first = True,\n                            #dropout = 0.2,\n                            bidirectional = True\n                            )\n        question_convs = [nn.Sequential(nn.Conv1d(in_channels = embed_size*2,out_channels = 48,kernel_size = kernel_size),\n                          nn.BatchNorm1d(48),\n                          nn.ReLU(inplace=True),\n                          nn.MaxPool1d(kernel_size = (maxlen - kernel_size + 1))\n                          ) for kernel_size in kernel_sizes]\n        self.question_convs = nn.ModuleList(question_convs)\n        self.lstm_attention = Attention(256, maxlen)\n        self.fc = nn.Sequential(\n            nn.Linear(912,256),\n            nn.BatchNorm1d(256),\n            nn.ReLU(inplace=True),\n            nn.Dropout(0.3),\n            nn.Linear(256,1)\n        )\n        self.dropout=nn.Dropout2d(0.1)\n        self.linear1=nn.Linear(600,256)\n        for name, param in self.title_lstm.named_parameters():\n            if 'bias' in name:\n                 nn.init.constant_(param, 0.0)\n            elif 'weight_ih' in name:\n                 nn.init.kaiming_normal_(param)\n            elif 'weight_hh' in name:\n                 nn.init.orthogonal_(param)\n        for name, param in self.title_gru.named_parameters():\n            if 'bias' in name:\n                 nn.init.constant_(param, 0.0)\n            elif 'weight_ih' in name:\n                 nn.init.kaiming_normal_(param)\n            elif 'weight_hh' in name:\n                 nn.init.orthogonal_(param)\n    def forward(self, question):\n        question = self.encoder(question)\n        question = torch.squeeze(self.dropout(torch.unsqueeze(question, 0)))\n        \n        question_lstm_out = self.title_lstm(question)[0]\n        question_res_out=self.linear1(question)+question_lstm_out\n        question_gru_out = self.title_gru(question_res_out)[0]\n        question_res_out2=question_res_out+question_gru_out\n        \n        question_conv_out = kmax_pooling((question_res_out2.permute(0,2,1)),2,2)\n        question_conv_out = question_conv_out.view(question_conv_out.size(0), -1)#b,512\n        \n        question_attention_out=self.lstm_attention(question_res_out2)\n        \n        cnn_question_out = [question_conv(question.permute(0, 2, 1)) for question_conv in self.question_convs]\n        conv_out = torch.cat(cnn_question_out,dim=2)\n        conv_out = conv_out.view(conv_out.size(0), -1)#b,144\n        \n        reshaped=torch.cat((question_conv_out,question_attention_out,conv_out),1)\n        \n        logits = self.fc((reshaped))\n        return logits","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"132549e69116f9a03db5133e9d3a55c45908098b"},"cell_type":"code","source":"#train_X,val_X,test_X,train_y,val_y,word_index=load_and_prec()\ntrain_X,test_X,train_y,word_index=load_and_prec()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0119ed12058882d5c1779080af0d9eb0efba39f8"},"cell_type":"code","source":"splits = list(StratifiedKFold(n_splits=5, shuffle=True, random_state=2018).split(train_X, train_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"620aec41b7a6a49b4981e6a53763f43069d38489"},"cell_type":"code","source":"glove_embeddings = load_glove(word_index)\nparagram_embeddings = load_para(word_index)\nembedding_matrix = np.concatenate((glove_embeddings, paragram_embeddings), axis=1)\nnp.shape(embedding_matrix)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"afd80434ef5e55b1bee5b79310924c19c3bba4ab"},"cell_type":"code","source":"x_test_cuda = torch.tensor(test_X, dtype=torch.long).cuda()\ntest = torch.utils.data.TensorDataset(x_test_cuda)\ntest_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)\n\"\"\"\nx_valid_dataset= QuraData(val_X,val_y,augument=False,training=False)\nx_valid_loader = torch.utils.data.DataLoader(x_valid_dataset, batch_size=batch_size, shuffle=False)\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"48a1786800107894197a43d446d670c88d291187"},"cell_type":"code","source":"def train(n_epochs):\n    train_preds = np.zeros((len(train_X)))\n    # matrix for the predictions on the test set\n    test_preds = np.zeros((len(test_X)))\n    #oof_fold= np.zeros((len(val_X)))\n    #outputs=[]\n    for i, (train_idx, valid_idx) in enumerate(splits): \n        train_dataset= QuraData(train_X[train_idx.astype(int)],train_y[train_idx.astype(int)],augument=False,training=True)\n        valid_dataset= QuraData(train_X[valid_idx.astype(int)],train_y[valid_idx.astype(int)],augument=False,training=False)\n        train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n        valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=batch_size, shuffle=False)\n         # make sure everything in the model is running on the GPU\n        model=mergeNN(embedding_matrix=embedding_matrix).cuda()\n        \n        loss_fn = torch.nn.BCEWithLogitsLoss(reduction='mean') \n        step_size = 500\n        base_lr, max_lr = 0.001, 0.003  \n        optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), \n                             lr=max_lr)\n        scheduler = CyclicLR(optimizer, base_lr=base_lr, max_lr=max_lr,\n               step_size=step_size, mode='exp_range',\n               gamma=0.99994)\n        valid_preds_fold= np.zeros((len(valid_idx)))\n        test_preds_fold= np.zeros((len(test_X)))\n        print(f'Fold {i + 1}')\n        for epoch in range(n_epochs):\n            start_time = time.time()\n            model.train(True)\n            avg_loss = 0 \n            for ii,(x_batch,y_batch) in enumerate(train_loader):\n                x_batch=Variable(x_batch).cuda()\n                y_batch=Variable(y_batch).cuda()\n                y_pred = model(x_batch)\n                #print(y_pred.shape)\n                scheduler.batch_step()\n                loss = loss_fn(y_pred, y_batch.float())\n                optimizer.zero_grad()\n                loss.backward()\n                optimizer.step()\n                avg_loss += loss.item() / len(train_loader)\n            model.train(False)\n            model.eval()   \n            avg_val_loss = 0.\n            with torch.no_grad():\n                for i, (x_batch, y_batch) in enumerate(valid_loader):\n                    x_batch=Variable(x_batch).cuda()\n                    y_batch=Variable(y_batch).cuda()\n                    y_pred = model(x_batch)\n                    avg_val_loss += loss_fn(y_pred, y_batch.float()).item() / len(valid_loader)\n                    valid_preds_fold[i * batch_size:(i+1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]\n                train_preds[valid_idx] = valid_preds_fold\n                elapsed_time = time.time() - start_time\n                print('Epoch {}/{} \\t loss={:.4f} \\t val_loss={:.4f} \\t time={:.2f}s'.format(epoch + 1, n_epochs, avg_loss, avg_val_loss,elapsed_time))\n        model.eval()\n        with torch.no_grad():\n            \"\"\"\n            for i, (x_batch, y_batch) in enumerate(x_valid_loader):\n                x_batch=Variable(x_batch).cuda()\n                y_pred = model(x_batch)\n                oof_fold[i * batch_size:(i+1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]   \n            \"\"\"\n            for i, (x_batch,) in enumerate(test_loader):\n                x_batch=Variable(x_batch).cuda()\n                y_pred = model(x_batch)\n                test_preds_fold[i * batch_size:(i+1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]\n            test_preds+=test_preds_fold/len(splits)\n        #outputs.append([oof_fold,test_preds])\n    #print('All \\t loss={:.4f} \\t val_loss={:.4f} \\t '.format(np.average(avg_losses_f),np.average(avg_val_losses_f)))\n    #return train_preds,outputs\n    return train_preds,test_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b562e9128809341007424b6e89bc7e52e5419fb5"},"cell_type":"code","source":"train_preds,test_preds=train(n_epochs=3)\n#train_preds,outputs=train(model_lstm,n_epochs=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a462fb9bf3204bbdcdbf2ba74ac9f5001f4e28ba"},"cell_type":"code","source":"thresholds = []\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    res = metrics.f1_score(train_y, (train_preds > thresh).astype(int))\n    thresholds.append([thresh, res])\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, res))  \nthresholds.sort(key=lambda x: x[1], reverse=True)\nbest_thresh = thresholds[0][0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7181e17af1e403c4fe1630d27aa6858c7e5c52f9"},"cell_type":"code","source":"\"\"\"\nfrom sklearn.linear_model import LinearRegression\nX = np.asarray([outputs[i][0] for i in range(len(outputs))])\nX = X[...]\nreg = LinearRegression().fit(X.T, val_y)\nprint(reg.score(X.T, val_y),reg.coef_)\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"081e847ef3002dfb6744b7d285e46d35b1dc2ea7"},"cell_type":"code","source":"#pred_test_y = np.sum([outputs[i][1]*reg.coef_[i] for i in range(len(outputs))], axis = 0)\npred_test_y = (test_preds > best_thresh).astype(int)\n#pred_test_y = (outputs[0][1]> outputs[0][2]).astype(int)\ntest_df = pd.read_csv(\"../input/test.csv\", usecols=[\"qid\"])\nout_df = pd.DataFrame({\"qid\":test_df[\"qid\"].values})\nout_df['prediction'] = pred_test_y\nout_df.to_csv(\"submission.csv\", index=False)\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}