{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import re\nimport time\nimport gc\nimport random\nimport os\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\nfrom sklearn.model_selection import GridSearchCV, StratifiedKFold\nfrom sklearn.metrics import f1_score, roc_auc_score\n\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\n\nimport torch\nimport torch.nn as nn\nimport torch.utils.data\n\ntqdm.pandas()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"def seed_torch(seed=43):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"85ee6b26944da2cd972c5038fa63c78b763184d3"},"cell_type":"code","source":"embed_size = 300      # how big is each word vector\nmax_features = 100000 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 70           # max number of words in a question to use\n\nbatch_size = 512\ntrain_epochs = 4\n\nSEED = 43","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5013e3945f2454854337ff3ee7eee98efa79fd25"},"cell_type":"code","source":"puncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', \n '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]\n\ndef clean_text(x):\n    x = str(x)\n    for punct in puncts:\n        x = x.replace(punct, f' {punct} ')\n    return x\n\ndef clean_numbers(x):\n    x = re.sub('[0-9]{5,}', '#####', x)\n    x = re.sub('[0-9]{4}', '####', x)\n    x = re.sub('[0-9]{3}', '###', x)\n    x = re.sub('[0-9]{2}', '##', x)\n    return x\n\nmispell_dict = {\"aren't\" : \"are not\",\n\"can't\" : \"cannot\",\n\"couldn't\" : \"could not\",\n\"didn't\" : \"did not\",\n\"doesn't\" : \"does not\",\n\"don't\" : \"do not\",\n\"hadn't\" : \"had not\",\n\"hasn't\" : \"has not\",\n\"haven't\" : \"have not\",\n\"he'd\" : \"he would\",\n\"he'll\" : \"he will\",\n\"he's\" : \"he is\",\n\"i'd\" : \"I would\",\n\"i'd\" : \"I had\",\n\"i'll\" : \"I will\",\n\"i'm\" : \"I am\",\n\"isn't\" : \"is not\",\n\"it's\" : \"it is\",\n\"it'll\":\"it will\",\n\"i've\" : \"I have\",\n\"let's\" : \"let us\",\n\"mightn't\" : \"might not\",\n\"mustn't\" : \"must not\",\n\"shan't\" : \"shall not\",\n\"she'd\" : \"she would\",\n\"she'll\" : \"she will\",\n\"she's\" : \"she is\",\n\"shouldn't\" : \"should not\",\n\"that's\" : \"that is\",\n\"there's\" : \"there is\",\n\"they'd\" : \"they would\",\n\"they'll\" : \"they will\",\n\"they're\" : \"they are\",\n\"they've\" : \"they have\",\n\"we'd\" : \"we would\",\n\"we're\" : \"we are\",\n\"weren't\" : \"were not\",\n\"we've\" : \"we have\",\n\"what'll\" : \"what will\",\n\"what're\" : \"what are\",\n\"what's\" : \"what is\",\n\"what've\" : \"what have\",\n\"where's\" : \"where is\",\n\"who'd\" : \"who would\",\n\"who'll\" : \"who will\",\n\"who're\" : \"who are\",\n\"who's\" : \"who is\",\n\"who've\" : \"who have\",\n\"won't\" : \"will not\",\n\"wouldn't\" : \"would not\",\n\"you'd\" : \"you would\",\n\"you'll\" : \"you will\",\n\"you're\" : \"you are\",\n\"you've\" : \"you have\",\n\"'re\": \" are\",\n\"wasn't\": \"was not\",\n\"we'll\":\" will\",\n\"didn't\": \"did not\",\n\"tryin'\":\"trying\"}\n\ndef _get_mispell(mispell_dict):\n    mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n    return mispell_dict, mispell_re\n\nmispellings, mispellings_re = _get_mispell(mispell_dict)\ndef replace_typical_misspell(text):\n    def replace(match):\n        return mispellings[match.group(0)]\n    return mispellings_re.sub(replace, text)\n\n\nfirst_word_mispell_dict = {\n                'whta': 'what', 'howdo': 'how do', 'Whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', \n                'howmany': 'how many', 'whydo': 'why do', 'doi': 'do i', 'howdoes': 'how does', \"whst\": 'what', \n                'shoupd': 'should', 'whats': 'what is', \"im\": \"i am\", \"whatis\": \"what is\", \"iam\": \"i am\", \"wat\": \"what\",\n                \"wht\": \"what\",\"whts\": \"what is\", \"whtwh\": \"what\", \"whtat\": \"what\", \"whtlat\": \"what\", \"dueto to\": \"due to\",\n                \"dose\": \"does\", \"wha\": \"what\", 'hw': \"how\", \"its\": \"it is\", \"whay\": \"what\", \"ho\": \"how\", \"whart\": \"what\", \n                \"woe\": \"wow\", \"wt\": \"what\", \"ive\": \"i have\",\"wha\": \"what\", \"wich\": \"which\", \"whic\": \"which\", \"whys\": \"why\", \n                \"doe\": \"does\", \"wjy\": \"why\", \"wgat\": \"what\", \"hiw\": \"how\",\"howto\": \"how to\", \"lets\": \"let us\", \"haw\": \"how\", \n                \"witch\": \"which\", \"wy\": \"why\", \"girlfriend\": \"girl friend\", \"hows\": \"how is\",\"whyis\": \"why is\", \"whois\": \"who is\",\n                \"dont\": \"do not\", \"hat\": \"what\", \"whos\": \"who is\", \"whydoes\": \"why does\", \"whic\": \"which\",\"hy\": \"why\", \"w? hy\": \"why\",\n                \"ehat\": \"what\", \"whate\": \"what\", \"whai\": \"what\", \"whichis\": \"which is\", \"whi\": \"which\", \"isit\": \"is it\",\"ca\": \"can\", \n                \"wwhat\": \"what\", \"wil\": \"will\", \"wath\": \"what\", \"plz\": \"please\", \"ww\": \"how\", \"hou\": \"how\", \"whch\": \"which\",\n                \"ihave\": \"i have\", \"cn\": \"can\", \"doesnt\": \"does not\", \"shoul\": \"should\", \"whatdo\": \"what do\", \"isnt\": \"is not\", \n                \"whare\": \"what are\",\"whick\": \"which\", \"whatdoes\": \"what does\", \"hwo\": \"how\", \"howdid\": \"how did\", \"why dose\": \"why does\"\n}\ndef correct_first_word(x):\n    for key in first_word_mispell_dict.keys():\n        if x.startswith(key + \" \"):\n            x = x.replace(key + \" \", first_word_mispell_dict[key] + \" \")\n            break\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"87a5fb67cdd061388568a1cd27250cecf5fb24a7"},"cell_type":"code","source":"def load_and_prec():\n    train_df = pd.read_csv(\"../input/train.csv\")\n    test_df = pd.read_csv(\"../input/test.csv\")\n    print(\"Train shape : \",train_df.shape)\n    print(\"Test shape : \",test_df.shape)\n    \n    # lower\n    train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: x.lower())\n    test_df[\"question_text\"] = test_df[\"question_text\"].progress_apply(lambda x: x.lower())\n    \n    # clean first word\n    test_df[\"question_text\"] = test_df[\"question_text\"].progress_apply(lambda x: correct_first_word(x))\n    \n    # Clean the text\n    train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: clean_text(x))\n    test_df[\"question_text\"] = test_df[\"question_text\"].progress_apply(lambda x: clean_text(x))\n    \n    # Clean numbers\n    train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: clean_numbers(x))\n    test_df[\"question_text\"] = test_df[\"question_text\"].progress_apply(lambda x: clean_numbers(x))\n    \n    # Clean speelings\n    train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: replace_typical_misspell(x))\n    test_df[\"question_text\"] = test_df[\"question_text\"].progress_apply(lambda x: replace_typical_misspell(x))\n    \n    ## fill up the missing values\n    train_X = train_df[\"question_text\"].fillna(\"_##_\").values\n    test_X = test_df[\"question_text\"].fillna(\"_##_\").values\n\n    ## Tokenize the sentences\n    tokenizer = Tokenizer(num_words=max_features, oov_token='OOV', filters='')\n    tokenizer.fit_on_texts(list(train_X))\n    train_X = tokenizer.texts_to_sequences(train_X)\n    test_X = tokenizer.texts_to_sequences(test_X)\n    print(\"tokenization finished, word count: %d\" % len(tokenizer.word_index))\n\n    ## Pad the sentences \n    train_X = pad_sequences(train_X, maxlen=maxlen)\n    test_X = pad_sequences(test_X, maxlen=maxlen)\n\n    ## Get the target values\n    train_y = train_df['target'].values\n    \n    #shuffling the data\n    np.random.seed(SEED)\n    trn_idx = np.random.permutation(len(train_X))\n\n    train_X = train_X[trn_idx]\n    train_y = train_y[trn_idx]\n    \n    return train_X, test_X, train_y, tokenizer.word_index\ntrain_X, test_X, train_y, word_index = load_and_prec()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9fa4b3016d2edf2c14340de1f23988b086850851"},"cell_type":"code","source":"def load_glove(word_index):\n    EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'    \n    emb_mean, emb_std = -0.005838499, 0.48782197\n    nb_words = min(max_features, len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n\n    with open(EMBEDDING_FILE, 'r', encoding=\"utf8\") as f:\n        for line in tqdm(f):\n            word, vec = line.split(' ', 1)\n            if word not in word_index:\n                continue\n            i = word_index[word]\n            if i >= nb_words:\n                continue\n            embedding_vector = np.asarray(vec.split(' '), dtype='float32')[:300]\n            if len(embedding_vector) == 300:\n                embedding_matrix[i] = embedding_vector                \n            \n    return embedding_matrix\n    \ndef load_fasttext(word_index):    \n    EMBEDDING_FILE = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\n    emb_mean, emb_std = -0.0033469985, 0.109855495\n    nb_words = min(max_features, len(word_index) + 1)\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    with open(EMBEDDING_FILE, 'r') as f:\n        for line in tqdm(f):\n            if len(line) <= 100:\n                continue\n            word, vec = line.split(' ', 1)\n            if word not in word_index:\n                continue\n            i = word_index[word]\n            if i >= nb_words:\n                continue\n            embedding_vector = np.asarray(vec.split(' '), dtype='float32')[:300]\n            if len(embedding_vector) == 300:\n                embedding_matrix[i] = embedding_vector   \n\n    return embedding_matrix\n\ndef load_para(word_index):\n    EMBEDDING_FILE = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\n    emb_mean, emb_std = -0.0053247833, 0.49346462\n    nb_words = min(max_features, len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    with open(EMBEDDING_FILE, 'r', encoding=\"utf8\", errors='ignore') as f:\n        for line in tqdm(f):\n            if len(line) <= 100:\n                continue\n            word, vec = line.split(' ', 1)\n            if word not in word_index:\n                continue\n            i = word_index[word]\n            if i >= nb_words:\n                continue\n            embedding_vector = np.asarray(vec.split(' '), dtype='float32')[:300]\n            if len(embedding_vector) == 300:\n                embedding_matrix[i] = embedding_vector \n    \n    return embedding_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"06944041d5cad5617c641b73f513c4840eb502fc"},"cell_type":"code","source":"embedding_matrix_1 = load_glove(word_index)\nembedding_matrix_2 = load_fasttext(word_index)\nembedding_matrix_3 = load_para(word_index)\nembedding_matrix123 = np.concatenate([embedding_matrix_1, embedding_matrix_2, embedding_matrix_3], axis = 1)\nembedding_matrix12 = np.concatenate([embedding_matrix_1, embedding_matrix_2], axis = 1)\nembedding_matrix13 = np.concatenate([embedding_matrix_1, embedding_matrix_3], axis = 1)\nembedding_matrix23 = np.concatenate([embedding_matrix_2, embedding_matrix_3], axis = 1)\nembedding_matrixs = [embedding_matrix12, embedding_matrix13, embedding_matrix23, embedding_matrix13]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e78df8d5ca5898ada4e6bf99132c00640493d351"},"cell_type":"code","source":"class Attention(nn.Module):\n    def __init__(self, feature_dim, step_dim, bias=True, **kwargs):\n        super(Attention, self).__init__(**kwargs)\n        \n        self.supports_masking = True\n\n        self.bias = bias\n        self.feature_dim = feature_dim\n        self.step_dim = step_dim\n        self.features_dim = 0\n        \n        weight = torch.zeros(feature_dim, 1)\n        nn.init.xavier_uniform_(weight)\n        self.weight = nn.Parameter(weight)\n        \n        if bias:\n            self.b = nn.Parameter(torch.zeros(step_dim))\n        \n    def forward(self, x, mask=None):\n        feature_dim = self.feature_dim\n        step_dim = self.step_dim\n\n        eij = torch.mm(\n            x.contiguous().view(-1, feature_dim), \n            self.weight\n        ).view(-1, step_dim)\n        \n        if self.bias:\n            eij = eij + self.b\n            \n        eij = torch.tanh(eij)\n        a = torch.exp(eij)\n        \n        if mask is not None:\n            a = a * mask\n\n        a = a / torch.sum(a, 1, keepdim=True) + 1e-10\n\n        weighted_input = x * torch.unsqueeze(a, -1)\n        return torch.sum(weighted_input, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"71eeb24155b28b82e08b89becf8ba2cf3e207d67"},"cell_type":"code","source":"class NeuralNet(nn.Module):\n    def __init__(self, embedding_matrix):\n        super(NeuralNet, self).__init__()\n        \n        hidden_size = 128\n        \n        self.embedding = nn.Embedding(max_features, embed_size*2)\n        self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))\n        self.embedding.weight.requires_grad = False\n        \n        self.embedding_dropout = nn.Dropout2d(0.15)\n        self.lstm = nn.LSTM(embed_size*2, hidden_size, bidirectional=True, batch_first=True)\n        self.gru = nn.GRU(hidden_size*2, hidden_size, bidirectional=True, batch_first=True)\n        \n        self.lstm_attention = Attention(hidden_size*2, maxlen)\n        self.gru_attention = Attention(hidden_size*2, maxlen)\n        \n        self.linear = nn.Linear(1024, 64)\n        self.relu = nn.ReLU()\n        self.dropout = nn.Dropout(0.15)\n        self.out = nn.Linear(64, 1)\n        \n    def forward(self, x):\n        h_embedding = self.embedding(x)\n        h_embedding = torch.squeeze(self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))\n        \n        h_lstm, _ = self.lstm(h_embedding)\n        h_gru, _ = self.gru(h_lstm)\n        \n        h_lstm_atten = self.lstm_attention(h_lstm)\n        h_gru_atten = self.gru_attention(h_gru)\n        \n        avg_pool = torch.mean(h_gru, 1)\n        max_pool, _ = torch.max(h_gru, 1)\n        \n        conc = torch.cat((h_lstm_atten, h_gru_atten, avg_pool, max_pool), 1)\n        conc = self.relu(self.linear(conc))\n        conc = self.dropout(conc)\n        out = self.out(conc)\n        \n        return out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fd7935d72ecdb040469fed2c01d5cf9e2f45566c"},"cell_type":"code","source":"class CyclicLR(object):\n    def __init__(self, optimizer, base_lr=1e-3, max_lr=6e-3,\n                 step_size=2000, factor=0.6, min_lr=1e-4, mode='triangular', gamma=1.,\n                 scale_fn=None, scale_mode='cycle', last_batch_iteration=-1):\n\n        if not isinstance(optimizer, torch.optim.Optimizer):\n            raise TypeError('{} is not an Optimizer'.format(\n                type(optimizer).__name__))\n        self.optimizer = optimizer\n\n        if isinstance(base_lr, list) or isinstance(base_lr, tuple):\n            if len(base_lr) != len(optimizer.param_groups):\n                raise ValueError(\"expected {} base_lr, got {}\".format(\n                    len(optimizer.param_groups), len(base_lr)))\n            self.base_lrs = list(base_lr)\n        else:\n            self.base_lrs = [base_lr] * len(optimizer.param_groups)\n\n        if isinstance(max_lr, list) or isinstance(max_lr, tuple):\n            if len(max_lr) != len(optimizer.param_groups):\n                raise ValueError(\"expected {} max_lr, got {}\".format(\n                    len(optimizer.param_groups), len(max_lr)))\n            self.max_lrs = list(max_lr)\n        else:\n            self.max_lrs = [max_lr] * len(optimizer.param_groups)\n\n        self.step_size = step_size\n\n        if mode not in ['triangular', 'triangular2', 'exp_range'] \\\n                and scale_fn is None:\n            raise ValueError('mode is invalid and scale_fn is None')\n\n        self.mode = mode\n        self.gamma = gamma\n\n        if scale_fn is None:\n            if self.mode == 'triangular':\n                self.scale_fn = self._triangular_scale_fn\n                self.scale_mode = 'cycle'\n            elif self.mode == 'triangular2':\n                self.scale_fn = self._triangular2_scale_fn\n                self.scale_mode = 'cycle'\n            elif self.mode == 'exp_range':\n                self.scale_fn = self._exp_range_scale_fn\n                self.scale_mode = 'iterations'\n        else:\n            self.scale_fn = scale_fn\n            self.scale_mode = scale_mode\n\n        self.batch_step(last_batch_iteration + 1)\n        self.last_batch_iteration = last_batch_iteration\n        \n        self.last_loss = np.inf\n        self.min_lr = min_lr\n        self.factor = factor\n        \n    def batch_step(self, batch_iteration=None):\n        if batch_iteration is None:\n            batch_iteration = self.last_batch_iteration + 1\n        self.last_batch_iteration = batch_iteration\n        for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()):\n            param_group['lr'] = lr\n\n    def step(self, loss):\n        if loss > self.last_loss:\n            self.base_lrs = [max(lr * self.factor, self.min_lr) for lr in self.base_lrs]\n            self.max_lrs = [max(lr * self.factor, self.min_lr) for lr in self.max_lrs]\n            \n    def _triangular_scale_fn(self, x):\n        return 1.\n\n    def _triangular2_scale_fn(self, x):\n        return 1 / (2. ** (x - 1))\n\n    def _exp_range_scale_fn(self, x):\n        return self.gamma**(x)\n\n    def get_lr(self):\n        step_size = float(self.step_size)\n        cycle = np.floor(1 + self.last_batch_iteration / (2 * step_size))\n        x = np.abs(self.last_batch_iteration / step_size - 2 * cycle + 1)\n\n        lrs = []\n        param_lrs = zip(self.optimizer.param_groups, self.base_lrs, self.max_lrs)\n        for param_group, base_lr, max_lr in param_lrs:\n            base_height = (max_lr - base_lr) * np.maximum(0, (1 - x))\n            if self.scale_mode == 'cycle':\n                lr = base_lr + base_height * self.scale_fn(cycle)\n            else:\n                lr = base_lr + base_height * self.scale_fn(self.last_batch_iteration)\n            lrs.append(lr)\n        return lrs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b6c179e5478f1ac92b69020fb875bf3d8154bff3"},"cell_type":"code","source":"def sigmoid(x):\n    return 1 / (1 + np.exp(-x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2ad4bf11f982e199ff5d64004eeedd9f3d3d454c"},"cell_type":"code","source":"def threshold_search(y_true, y_proba):\n    best_threshold = 0\n    best_score = 0\n    for threshold in tqdm([i * 0.01 for i in range(100)]):\n        score = f1_score(y_true=y_true, y_pred=y_proba > threshold)\n        if score > best_score:\n            best_threshold = threshold\n            best_score = score\n    search_result = {'threshold': best_threshold, 'f1': best_score}\n    return search_result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1f80aa8c099cb76eede9c0c5db859a835fc9d1f9","scrolled":true},"cell_type":"code","source":"def train_and_predict1():\n\n    seed_torch(SEED)\n\n    x_test_cuda = torch.tensor(test_X, dtype=torch.long).cuda()\n    test = torch.utils.data.TensorDataset(x_test_cuda)\n    test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)\n\n    train_x0, valid_x0, train_y0, valid_y0 = train_test_split(train_X, train_y, test_size=0.05, shuffle=True, random_state=43)\n    valid_preds = np.zeros((len(valid_y0)))\n    test_preds = np.zeros((len(test_X)))\n\n    epochs=3\n    for i in range(epochs):\n        x_train_fold = torch.tensor(train_x0, dtype=torch.long).cuda()\n        y_train_fold = torch.tensor(train_y0.reshape(len(train_y0), 1), dtype=torch.float32).cuda()\n        x_val_fold = torch.tensor(valid_x0, dtype=torch.long).cuda()\n        y_val_fold = torch.tensor(valid_y0.reshape(len(valid_y0), 1), dtype=torch.float32).cuda()\n\n        model = NeuralNet(embedding_matrixs[i])\n        model.cuda()\n\n        loss_fn = torch.nn.BCEWithLogitsLoss(reduction=\"mean\")\n        optimizer = torch.optim.Adam(model.parameters())\n        step_size = 300\n        scheduler = CyclicLR(optimizer, base_lr=0.001, max_lr=0.002,\n                         step_size=step_size, mode='exp_range',\n                         gamma=0.99994)\n\n        train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)\n        valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)\n\n        train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)\n        valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)\n\n        print(f'Fold {i + 1}')\n    \n        train_epochs = 4\n        for epoch in range(train_epochs):\n            start_time = time.time()\n\n            model.train()\n            avg_loss = 0.\n            for x_batch, y_batch in tqdm(train_loader, disable=True):\n                y_pred = model(x_batch)\n                scheduler.batch_step()\n                loss = loss_fn(y_pred, y_batch)\n                optimizer.zero_grad()\n                loss.backward()\n                optimizer.step()\n                avg_loss += loss.item() / len(train_loader)\n\n            model.eval()\n            valid_preds_fold = np.zeros((x_val_fold.size(0)))\n            test_preds_fold = np.zeros(len(test_X))\n            avg_val_loss = 0.\n            for i, (x_batch, y_batch) in enumerate(valid_loader):\n                y_pred = model(x_batch).detach()\n                avg_val_loss += loss_fn(y_pred, y_batch).item() / len(valid_loader)\n                valid_preds_fold[i * batch_size:(i+1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]\n\n            elapsed_time = time.time() - start_time \n            print('Epoch {}/{} \\t loss={:.4f} \\t val_loss={:.4f} \\t time={:.2f}s'.format(\n                epoch + 1, train_epochs, avg_loss, avg_val_loss, elapsed_time))\n            search_result = threshold_search(valid_y0, valid_preds_fold)\n            print(search_result)\n\n        for i, (x_batch,) in enumerate(test_loader):\n            y_pred = model(x_batch).detach()\n\n            test_preds_fold[i * batch_size:(i+1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]\n\n        valid_preds += valid_preds_fold / epochs\n        test_preds += test_preds_fold / epochs  \n        \n    return valid_y0, valid_preds, test_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"640586b2651445a603c01569e11ce4d542b61ee1"},"cell_type":"code","source":"def train_and_predict2():\n\n    seed_torch(SEED)\n\n    x_test_cuda = torch.tensor(test_X, dtype=torch.long).cuda()\n    test = torch.utils.data.TensorDataset(x_test_cuda)\n    test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)\n    test_preds = np.zeros((len(test_X)))\n\n    epochs=3\n    avgThreshold = 0\n    for i in range(epochs):\n        train_x0, valid_x0, train_y0, valid_y0 = train_test_split(train_X, train_y, test_size=0.05, shuffle=True, random_state=43 * i)\n        valid_preds = np.zeros((len(valid_y0)))\n        \n        x_train_fold = torch.tensor(train_x0, dtype=torch.long).cuda()\n        y_train_fold = torch.tensor(train_y0.reshape(len(train_y0), 1), dtype=torch.float32).cuda()\n        x_val_fold = torch.tensor(valid_x0, dtype=torch.long).cuda()\n        y_val_fold = torch.tensor(valid_y0.reshape(len(valid_y0), 1), dtype=torch.float32).cuda()\n\n        model = NeuralNet(embedding_matrixs[i])\n        model.cuda()\n\n        loss_fn = torch.nn.BCEWithLogitsLoss(reduction=\"mean\")\n        optimizer = torch.optim.Adam(model.parameters())\n\n        train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)\n        valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)\n\n        train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)\n        valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)\n\n        print(f'Fold {i + 1}')\n    \n        train_epochs = 4\n        for epoch in range(train_epochs):\n            start_time = time.time()\n\n            model.train()\n            avg_loss = 0.\n            for x_batch, y_batch in tqdm(train_loader, disable=True):\n                y_pred = model(x_batch)\n                loss = loss_fn(y_pred, y_batch)\n                optimizer.zero_grad()\n                loss.backward()\n                optimizer.step()\n                avg_loss += loss.item() / len(train_loader)\n\n            model.eval()\n            valid_preds_fold = np.zeros((x_val_fold.size(0)))\n            test_preds_fold = np.zeros(len(test_X))\n            avg_val_loss = 0.\n            for i, (x_batch, y_batch) in enumerate(valid_loader):\n                y_pred = model(x_batch).detach()\n                avg_val_loss += loss_fn(y_pred, y_batch).item() / len(valid_loader)\n                valid_preds_fold[i * batch_size:(i+1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]\n\n            elapsed_time = time.time() - start_time \n            print('Epoch {}/{} \\t loss={:.4f} \\t val_loss={:.4f} \\t time={:.2f}s'.format(\n                epoch + 1, train_epochs, avg_loss, avg_val_loss, elapsed_time))\n            search_result = threshold_search(valid_y0, valid_preds_fold)\n            if epoch == train_epochs - 1:\n                avgThreshold += search_result['threshold'] / epochs\n            print(search_result)\n\n        for i, (x_batch,) in enumerate(test_loader):\n            y_pred = model(x_batch).detach()\n\n            test_preds_fold[i * batch_size:(i+1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]\n\n        test_preds += test_preds_fold / epochs  \n        \n    return avgThreshold, test_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b5828120ca9fd0d6890e667f004dad9571b95a37"},"cell_type":"code","source":"def train_and_predict3():\n\n    seed_torch(SEED)\n\n    x_test_cuda = torch.tensor(test_X, dtype=torch.long).cuda()\n    test = torch.utils.data.TensorDataset(x_test_cuda)\n    test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)\n    test_preds = np.zeros((len(test_X)))\n    \n    splits = list(StratifiedKFold(n_splits=4, shuffle=True, random_state=SEED).split(train_X, train_y))\n    train_preds = np.zeros((len(train_X)))\n\n    for i, (train_idx, valid_idx) in enumerate(splits):\n        x_train_fold = torch.tensor(train_X[train_idx], dtype=torch.long).cuda()\n        y_train_fold = torch.tensor(train_y[train_idx, np.newaxis], dtype=torch.float32).cuda()\n        x_val_fold = torch.tensor(train_X[valid_idx], dtype=torch.long).cuda()\n        y_val_fold = torch.tensor(train_y[valid_idx, np.newaxis], dtype=torch.float32).cuda()\n\n        model = NeuralNet(embedding_matrixs[i])\n        model.cuda()\n\n        loss_fn = torch.nn.BCEWithLogitsLoss(reduction=\"mean\")\n        optimizer = torch.optim.Adam(model.parameters())\n        #step_size = 300\n        #scheduler = CyclicLR(optimizer, base_lr=0.001, max_lr=0.003, step_size=step_size, mode='exp_range', gamma=0.99994)\n\n        train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)\n        valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)\n\n        train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)\n        valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)\n\n        print(f'Fold {i + 1}')\n    \n        train_epochs = 4\n        for epoch in range(train_epochs):\n            start_time = time.time()\n\n            model.train()\n            avg_loss = 0.\n            for x_batch, y_batch in tqdm(train_loader, disable=False):\n                y_pred = model(x_batch)\n                #scheduler.batch_step()\n                loss = loss_fn(y_pred, y_batch)\n                optimizer.zero_grad()\n                loss.backward()\n                optimizer.step()\n                avg_loss += loss.item() / len(train_loader)\n\n            model.eval()\n            valid_preds_fold = np.zeros((x_val_fold.size(0)))\n            test_preds_fold = np.zeros(len(test_X))\n            avg_val_loss = 0.\n            for i, (x_batch, y_batch) in enumerate(valid_loader):\n                y_pred = model(x_batch).detach()\n                avg_val_loss += loss_fn(y_pred, y_batch).item() / len(valid_loader)\n                valid_preds_fold[i * batch_size:(i+1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]\n\n            elapsed_time = time.time() - start_time \n            print('Epoch {}/{} \\t loss={:.4f} \\t val_loss={:.4f} \\t time={:.2f}s'.format(\n                epoch + 1, train_epochs, avg_loss, avg_val_loss, elapsed_time))\n            search_result = threshold_search(train_y[valid_idx], valid_preds_fold)\n            print(search_result)\n\n        for i, (x_batch,) in enumerate(test_loader):\n            y_pred = model(x_batch).detach()\n\n            test_preds_fold[i * batch_size:(i+1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]\n\n        train_preds[valid_idx] = valid_preds_fold\n        test_preds += test_preds_fold / len(splits)   \n        \n    return train_preds, test_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f0c7655686bd19130f7f2c8074b2ec44b1451848"},"cell_type":"code","source":"#valid_y0, valid_preds, test_preds = train_and_predict1()\n#search_result = threshold_search(valid_y0, valid_preds)\n#print(search_result)\n#sub = pd.read_csv('../input/sample_submission.csv')\n#sub.prediction = (test_preds > search_result['threshold']).astype(int)\n#sub.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cf8a1aeeb47e71c5ad96c78514af58035798f592","scrolled":false},"cell_type":"code","source":"#avgThreshold, test_preds = train_and_predict2()\n#print(\"avg threshold: %.6f\" % avgThreshold)\n#sub = pd.read_csv('../input/sample_submission.csv')\n#sub.prediction = (test_preds > avgThreshold).astype(int)\n#sub.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d85de8f83b8c7e314b9123771061e1dae72481e4"},"cell_type":"code","source":"train_preds, test_preds = train_and_predict3()\nsearch_result = threshold_search(train_y, train_preds)\nsub = pd.read_csv('../input/sample_submission.csv')\nsub.prediction = (test_preds > search_result['threshold']).astype(int)\nsub.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"84fe0e08ff4bc4c34a0b875975122d6d6171c3e6"},"cell_type":"code","source":"print(search_result)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}