{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import time\nimport os\nimport pandas as pd\nimport numpy as np\nimport random\nimport copy\n\nimport re\nfrom gensim.models import KeyedVectors\nimport gc\n\nfrom keras.preprocessing.text import Tokenizer\n\nimport torch\nimport torch.nn as nn\nfrom torch.optim import Adam\nfrom torch.utils.data import Dataset\nfrom torch.nn.utils.rnn import pad_sequence\nt0 = time.time()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_path = '../input/'\ntrain_df = pd.read_csv(data_path + 'train.csv')\ntest_df = pd.read_csv(data_path + 'test.csv')\nprint('Train : ', train_df.shape)\nprint('Test : ', test_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"seed = 2077\n\n# Preprocessing\nmax_len = 50\nlower = False\ntrunc = 'pre'\nmax_features = 120000\nn_vocab = max_features\nclean_num = 0\n\n# Training\nn_models = 6\nepochs = 8\nbatch_size = 512\ndrop_last = True\n\nhidden_dim = 128\n\n# Embedding\nfix_embedding = True\nunk_uni = True  # Initializer for unknown words\nn_embed = 2\nembed_dim = n_embed * 300\nproj_dim = hidden_dim\n\n# GRU\nbidirectional = True\nn_layers = 1\nrnn_dim = hidden_dim\n\n# The second last Linear layer\ndense_dim = 2 * rnn_dim if bidirectional else rnn_dim\n\n# EMA\nmu = 0.9\nupdates_per_epoch = 10\n\n# Test set\nthreshold = 0.37\ntest_batch_size = 8 * batch_size","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_torch(seed=1):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\n\nseed_torch(seed)\ndevice = torch.device('cuda') if torch.cuda.is_available() else 'cpu'\n\n\ndef get_param_size(model, trainable=True):\n    if trainable:\n        psize = np.sum([np.prod(p.size()) for p in model.parameters() if p.requires_grad])\n    else:\n        psize = np.sum([np.prod(p.size()) for p in model.parameters()])\n    return psize\n\n\n# https://discuss.pytorch.org/t/how-to-apply-exponential-moving-average-decay-for-variables/10856\nclass EMA():\n    def __init__(self, model, mu, level='batch', n=1):\n        \"\"\"\n        level: 'batch' or 'epoch'\n          'batch': Update params every n batches.\n          'epoch': Update params every epoch.\n        \"\"\"\n        # self.ema_model = copy.deepcopy(model)\n        self.mu = mu\n        self.level = level\n        self.n = n\n        self.cnt = self.n\n        self.shadow = {}\n        for name, param in model.named_parameters():\n            if param.requires_grad:\n                self.shadow[name] = param.data\n\n    def _update(self, model):\n        for name, param in model.named_parameters():\n            if param.requires_grad:\n                new_average = (1 - self.mu) * param.data + self.mu * self.shadow[name]\n                self.shadow[name] = new_average.clone()\n\n    def set_weights(self, ema_model):\n        for name, param in ema_model.named_parameters():\n            if param.requires_grad:\n                param.data = self.shadow[name]\n\n    def on_batch_end(self, model):\n        if self.level is 'batch':\n            self.cnt -= 1\n            if self.cnt == 0:\n                self._update(model)\n                self.cnt = self.n\n\n    def on_epoch_end(self, model):\n        if self.level is 'epoch':\n            self._update(model)\n\n\nclass GlobalMaxPooling1D(nn.Module):\n    def __init__(self):\n        super(GlobalMaxPooling1D, self).__init__()\n\n    def forward(self, inputs):\n        z, _ = torch.max(inputs, 1)\n        return z\n\n    def __repr__(self):\n        return self.__class__.__name__ + '()'\n\n\nclass GRUModel(nn.Module):\n    def __init__(self, n_vocab, embed_dim, proj_dim, rnn_dim, n_layers, bidirectional, dense_dim,\n                 padding_idx=0, pretrained_embedding=None, fix_embedding=True,\n                 n_out=1):\n        super(GRUModel, self).__init__()\n        self.n_vocab = n_vocab\n        self.embed_dim = embed_dim\n        self.n_layers = n_layers\n        self.dense_dim = dense_dim\n        self.n_out = n_out\n        self.bidirectional = bidirectional\n        self.fix_embedding = fix_embedding\n        self.padding_idx = padding_idx\n        if pretrained_embedding is not None:\n            self.embed = nn.Embedding.from_pretrained(pretrained_embedding, freeze=fix_embedding)\n            self.embed.padding_idx = self.padding_idx\n        else:\n            self.embed = nn.Embedding(self.n_vocab, self.embed_dim, padding_idx=self.padding_idx)\n        self.proj = nn.Linear(embed_dim, proj_dim)\n        self.proj_act = nn.ReLU()\n        self.gru = nn.GRU(proj_dim, rnn_dim, self.n_layers,\n                          batch_first=True, bidirectional=bidirectional)\n        self.pooling = GlobalMaxPooling1D()\n        in_dim = 2 * rnn_dim if self.bidirectional else rnn_dim\n        self.dense = nn.Linear(in_dim, dense_dim)\n        self.dense_act = nn.ReLU()\n        self.out_linear = nn.Linear(dense_dim, n_out)\n        self.init_weights()\n\n    def init_weights(self):\n        for name, param in self.named_parameters():\n            if name.find('embed') > -1:\n                continue\n            elif name.find('weight') > -1 and len(param.size()) > 1:\n                nn.init.xavier_uniform_(param)\n\n    def forward(self, inputs):\n        # inputs: (bs, max_len)\n        x = self.embed(inputs)\n        x = self.proj_act(self.proj(x))\n        x, hidden = self.gru(x)\n        x = self.pooling(x)\n        x = self.dense_act(self.dense(x))\n        x = self.out_linear(x)\n        return x\n\n    def fit(self, dataloader, epochs, optimizer, callbacks=None):\n        for i in range(epochs):\n            run_epoch(model, dataloader, optimizer, callbacks=callbacks)\n\n    def predict(self, dataloader):\n        preds = []\n        with torch.no_grad():\n            for batch in dataloader:\n                batch = tuple(t.to(device) for t in batch)\n                X_batch = batch[0]\n                preds.append(self.forward(X_batch).data.cpu())\n        return torch.cat(preds)\n\n    def predict_proba(self, dataloader):\n        return torch.sigmoid(self.predict(dataloader)).data.numpy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TextClassificationDataset(Dataset):\n    \"\"\"\n    token_ids_s : List of token ids\n    labels   : Target labels\n    training: Sort token_ids_s by length if training is True.\n    \"\"\"\n    def __init__(self, token_ids_s, labels=None, max_len=1000, training=True, sort=True):\n        self.training = training\n\n        if labels is None:\n            self.labels = torch.ones(len(token_ids_s), dtype=torch.long)  # dummy\n        else:\n            self.labels = torch.LongTensor(labels)\n\n        seq_lens = []\n        self.inputs = []\n        for e, token_ids in enumerate(token_ids_s):\n            seq_lens.append(len(token_ids))\n            input_ids = torch.LongTensor(token_ids[:max_len])\n            self.inputs.append(input_ids)\n\n        if self.training and sort:\n            self.indices = np.argsort(seq_lens)\n            self.inputs = [self.inputs[i] for i in self.indices]\n            self.labels = self.labels[self.indices]\n\n    def __len__(self):\n        return len(self.inputs)\n\n    def __getitem__(self, idx):\n        return self.inputs[idx], self.labels[idx]\n\n    def set_labels(self, labels):\n        self.labels = torch.tensor(labels, dtype=torch.long)\n\n\n# Always drop last\nclass BatchIterator(object):\n\n    def __init__(self, dataset, collate_fn, batch_size,\n                 shuffle=True, drop_last=True):\n        self.dataset = dataset\n        self.collate_fn = collate_fn\n        self.batch_size = batch_size\n        self.size = len(dataset)\n        self.shuffle = shuffle\n        if drop_last:\n            self.num_batches = self.size // batch_size\n        else:\n            self.num_batches = (self.size + self.batch_size - 1) // self.batch_size\n\n    def __iter__(self):\n        if self.shuffle:\n            indices = np.random.choice(self.num_batches, self.num_batches, replace=False)\n        else:\n            indices = range(self.num_batches)\n        for idx in indices:\n            left = self.batch_size * idx\n            yield(self.collate_fn(self.dataset[left: left + self.batch_size]))\n\n    def __len__(self):\n        return self.num_batches\n\n\ndef collate_fn(batch):\n    xy_batch = [pad_sequence(batch[0], batch_first=True), batch[1]]\n    return xy_batch\n\n\ndef run_epoch(model, dataloader, optimizer, callbacks=None,\n              criterion=nn.BCEWithLogitsLoss(), verbose_step=10000):\n    t1 = time.time()\n    tr_loss = 0\n    for step, batch in enumerate(dataloader):\n        batch = tuple(t.to(device) for t in batch)\n        x_batch, y_batch = batch\n        model.zero_grad()\n        outputs = model(x_batch)\n        loss = criterion(outputs[:, 0], y_batch.float())\n        loss.backward()\n        optimizer.step()\n        tr_loss += loss.item()\n        if callbacks is not None:\n            for func in callbacks:\n                func.on_batch_end(model)\n        if (step + 1) % verbose_step == 0:\n            loss_now = tr_loss / (step + 1)\n            print(f'step:{step+1} loss:{loss_now:.7f} time:{time.time() - t1:.1f}s')\n    if callbacks is not None:\n        for func in callbacks:\n            func.on_epoch_end(model)\n    return tr_loss / (step + 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_glove(word_index, max_features, unk_uni):\n    def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32')\n    EMBEDDING_FILE = data_path + 'embeddings/glove.840B.300d/glove.840B.300d.txt'\n    embeddings_index = dict(get_coefs(*o.split(' ')) for o in open(EMBEDDING_FILE))\n\n    all_embs = np.stack(list(embeddings_index.values()))\n    emb_mean, emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n\n    unknown_words = []\n    nb_words = min(max_features, len(word_index))\n\n    if unk_uni:\n        embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    else:\n        embedding_matrix = np.zeros((nb_words, embed_size))\n    for word, i in word_index.items():\n        if i >= nb_words:\n            continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is None:\n            embedding_vector = embeddings_index.get(word.lower())\n            if embedding_vector is None:\n                unknown_words.append((word, i))\n            else:\n                embedding_matrix[i] = embedding_vector\n        else:\n            embedding_matrix[i] = embedding_vector\n    print('\\nTotal unknowns glove', len(unknown_words))\n    print(unknown_words[:10])\n\n    del embeddings_index\n    gc.collect()\n    return embedding_matrix, unknown_words\n\n\ndef load_wiki(word_index, max_features, unk_uni):\n    def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32')\n    EMBEDDING_FILE = data_path + 'embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\n    embeddings_index = dict(get_coefs(*o.split(' ')) for o in open(EMBEDDING_FILE) if len(o) > 100)\n\n    all_embs = np.stack(list(embeddings_index.values()))\n    emb_mean, emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n\n    unknown_words = []\n    nb_words = min(max_features, len(word_index))\n\n    if unk_uni:\n        embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    else:\n        embedding_matrix = np.zeros((nb_words, embed_size))\n    for word, i in word_index.items():\n        if i >= nb_words:\n            continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is None:\n            embedding_vector = embeddings_index.get(word.lower())\n            if embedding_vector is None:\n                unknown_words.append((word, i))\n            else:\n                embedding_matrix[i] = embedding_vector\n        else:\n            embedding_matrix[i] = embedding_vector\n    print('\\nTotal unknowns wiki', len(unknown_words))\n    print(unknown_words[:10])\n\n    del embeddings_index\n    gc.collect()\n    return embedding_matrix, unknown_words\n\n\ndef load_parag(word_index, max_features, unk_uni):\n    def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32')\n    EMBEDDING_FILE = data_path + 'embeddings/paragram_300_sl999/paragram_300_sl999.txt'\n    embeddings_index = dict(get_coefs(*o.split(' '))\n                            for o in open(EMBEDDING_FILE, encoding='utf8', errors='ignore')\n                            if len(o) > 100)\n\n    all_embs = np.stack(list(embeddings_index.values()))\n    emb_mean, emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n\n    unknown_words = []\n    nb_words = min(max_features, len(word_index))\n    if unk_uni:\n        embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    else:\n        embedding_matrix = np.zeros((nb_words, embed_size))\n    for word, i in word_index.items():\n        if i >= nb_words:\n            continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is None:\n            embedding_vector = embeddings_index.get(word.lower())\n            if embedding_vector is None:\n                unknown_words.append((word, i))\n            else:\n                embedding_matrix[i] = embedding_vector\n        else:\n            embedding_matrix[i] = embedding_vector\n    print('\\nTotal unknowns parag', len(unknown_words))\n    print(unknown_words[:10])\n\n    del embeddings_index\n    gc.collect()\n    return embedding_matrix, unknown_words\n\n\n# https://www.kaggle.com/strideradu/word2vec-and-gensim-go-go-go\ndef load_ggle(word_index, max_features, unk_uni):\n    EMBEDDING_FILE = data_path + 'embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'\n    embeddings_index = KeyedVectors.load_word2vec_format(EMBEDDING_FILE, binary=True)\n    embed_size = embeddings_index.get_vector('known').size\n\n    unknown_words = []\n    nb_words = min(max_features, len(word_index))\n    if unk_uni:\n        embedding_matrix = (np.random.rand(nb_words, embed_size) - 0.5) / 5.0\n    else:\n        embedding_matrix = np.zeros((nb_words, embed_size))\n    for word, i in word_index.items():\n        if i >= nb_words:\n            continue\n        if word in embeddings_index:\n            embedding_vector = embeddings_index.get_vector(word)\n            embedding_matrix[i] = embedding_vector\n        else:\n            word_lower = word.lower()\n            if word_lower in embeddings_index:\n                embedding_matrix[i] = embeddings_index.get_vector(word_lower)\n            else:\n                unknown_words.append((word, i))\n\n    print('\\nTotal unknowns ggle', len(unknown_words))\n    print(unknown_words[:10])\n\n    del embeddings_index\n    gc.collect()\n    return embedding_matrix, unknown_words\n\n\ndef load_all_embeddings(tokenizer, max_features, clean_num=False, unk_uni=True):\n    word_index = tokenizer.word_index\n    if clean_num == 2:\n        ggle_word_index = {}\n        for word, i in word_index.items():\n            ggle_word_index[clean_numbers(word)] = i\n    else:\n        ggle_word_index = word_index\n\n    embedding_matrix_1, u1 = load_glove(word_index, max_features, unk_uni)\n    embedding_matrix_2, u2 = load_wiki(word_index, max_features, unk_uni)\n    embedding_matrix_3, u3 = load_parag(word_index, max_features, unk_uni)\n    embedding_matrix_4, u4 = load_ggle(ggle_word_index, max_features, unk_uni)\n    embedding_matrix = np.concatenate((embedding_matrix_1,\n                                       embedding_matrix_2,\n                                       embedding_matrix_3,\n                                       embedding_matrix_4), axis=1)\n    del embedding_matrix_1, embedding_matrix_2, embedding_matrix_3, embedding_matrix_4\n    gc.collect()\n    # with open('unknowns.pkl', 'wb') as f:\n    #     pickle.dump({'glove': u1, 'wiki': u2, 'parag': u3, 'ggle': u4}, f)\n    # print('Embedding:', embedding_matrix.shape)\n    return embedding_matrix\n\n\ndef setup_emb(tr_X, max_features=50000, clean_num=2, unk_uni=True):\n    tokenizer = Tokenizer(num_words=max_features, lower=False, filters='')\n    tokenizer.fit_on_texts(tr_X)\n    print('len(vocab)', len(tokenizer.word_index))\n    embedding_matrix = load_all_embeddings(tokenizer, max_features=max_features,\n                                           clean_num=clean_num, unk_uni=unk_uni)\n    # np.save(embed_path, embedding_matrix)\n    return tokenizer, embedding_matrix","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0554d849-8dc9-400c-b907-09fc5fe92428","_cell_guid":"da21a73e-9eec-45e2-a3c9-7cfcf37a8b2b","trusted":true},"cell_type":"code","source":"puncts = ',.\":)(-!?|;\\'$&/[]>%=#*+\\\\•~@£·_{}©^®`<→°€™›♥←ק″′Â█½à…“★”–●â►−¢²¬░¶↑±¿▾═¦║―¥▓—‹─▒：¼⊕▼▪†■’▀¨▄♫☆é¯♦¤▲踾Ã⋅‘∞∙）↓、│（»，♪╩╚³・╦╣╔╗▬❤ïع≤‡√'\n\n\ndef clean_text(x, puncts=puncts):\n    x = str(x)\n    for punct in puncts:\n        x = x.replace(punct, f' {punct} ')\n    return x\n\n\ndef clean_numbers(x):\n    x = re.sub('[0-9]{5,}', '#####', x)\n    x = re.sub('[0-9]{4}', '####', x)\n    x = re.sub('[0-9]{3}', '###', x)\n    x = re.sub('[0-9]{2}', '##', x)\n    return x\n\n\ndef prepare_data(train_df, test_df, max_len, max_features, trunc='pre',\n                 lower=False, clean_num=2, unk_uni=True):\n    train_df = train_df.copy()\n    train_y = train_df['target'].values\n\n    # lower\n    if lower:\n        train_df['question_text'] = train_df['question_text'].apply(lambda x: x.lower())\n        test_df['question_text'] = test_df['question_text'].apply(lambda x: x.lower())\n\n    # Clean the text\n    train_df['question_text'] = train_df['question_text'].apply(\n        lambda x: clean_text(x))\n    test_df['question_text'] = test_df['question_text'].apply(\n        lambda x: clean_text(x))\n\n    # Clean numbers\n    if clean_num == 1:\n        train_df['question_text'] = train_df['question_text'].apply(\n            lambda x: clean_numbers(x))\n        test_df['question_text'] = test_df['question_text'].apply(\n            lambda x: clean_numbers(x))\n\n    # fill up the missing values\n    train_df['question_text'] = train_df['question_text'].fillna('_##_')\n    test_df['question_text'] = test_df['question_text'].fillna('_##_')\n\n    train_X = train_df['question_text'].values\n    test_X = test_df['question_text'].values\n\n    tokenizer, embedding_matrix = setup_emb(train_X,\n                                            max_features=max_features,\n                                            clean_num=clean_num, unk_uni=unk_uni)\n\n    train_X_ids = tokenizer.texts_to_sequences(train_X)\n    train_dataset = TextClassificationDataset(train_X_ids, train_y, sort=False)\n    train_loader = BatchIterator(train_dataset, collate_fn, batch_size)\n\n    test_X_ids = tokenizer.texts_to_sequences(test_X)\n    test_dataset = TextClassificationDataset(test_X_ids, training=False)\n    test_loader = BatchIterator(test_dataset, collate_fn, batch_size, shuffle=False, drop_last=False)\n\n    embedding_matrix = torch.Tensor(embedding_matrix)\n    return train_loader, test_loader, embedding_matrix\n\n\nids_s = [list(range(300)), list(range(300, 600)),\n         list(range(600, 900)), list(range(900, 1200))]\n\ncols_s = [ids_s[0] + ids_s[1],\n          ids_s[0] + ids_s[2],\n          ids_s[1] + ids_s[2],\n          ids_s[0] + ids_s[3],\n          ids_s[1] + ids_s[3],\n          ids_s[2] + ids_s[3]]\n\n\n\nobj = prepare_data(train_df, test_df, max_len, max_features,\n                   trunc=trunc, lower=lower, clean_num=clean_num, unk_uni=unk_uni)\n\ntrain_loader, test_loader, embedding_matrix = obj\n\n\nema_n = int(train_df.shape[0] / (updates_per_epoch * batch_size))\ntest_pr = np.zeros((len(test_df), 1))\nfor i in range(n_models):\n    cols_in_use = cols_s[i % len(cols_s)]\n    model = GRUModel(n_vocab, embed_dim, proj_dim, rnn_dim, n_layers, bidirectional, dense_dim,\n                     pretrained_embedding=embedding_matrix[:, cols_in_use],\n                     fix_embedding=fix_embedding, padding_idx=0)\n    if i == 0:\n        print(model)\n        print('#Trainable params', get_param_size(model))\n    model.cuda()\n    ema_model = copy.deepcopy(model)\n    ema_model.eval()\n    optimizer = Adam([p for n, p in model.named_parameters() if p.requires_grad is True])\n    ema = EMA(model, mu, n=ema_n)\n\n    t2 = time.time()\n    model.train()\n    model.fit(train_loader, epochs, optimizer, callbacks=[ema])\n    print(f'n_model:{i + 1} {(time.time() - t2) / epochs:.1f}s/epoch')\n\n    ema.set_weights(ema_model)\n    # To avoid RuntimeWarning:\n    #   RNN module weights are not part of\n    #   single contiguous chunk of memory. This means they need to\n    #   be compacted at every call, possibly greatly increasing memory usage.\n    #   To compact weights again call flatten_parameters().\n    ema_model.gru.flatten_parameters()\n    t3 = time.time()\n    test_pr += ema_model.predict_proba(test_loader)\n    print(f'{time.time() - t3:.1f}s')\n\n\ntest_pr /= n_models\ntest_pr = (test_pr > threshold).astype(int)\nout_df = pd.DataFrame({\"qid\": test_df[\"qid\"].values})\nout_df['prediction'] = test_pr\nout_df.to_csv(\"submission.csv\", index=False)\nprint(f'Done:{time.time() - t0:.1f}s')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}