{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ef71975a1672bd1cccba823d31d078da2d900581"},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.\n\n%matplotlib inline\nimport os, sys\nimport re\nimport string\nimport pathlib\nimport random\nfrom collections import Counter, OrderedDict\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport nltk\nimport spacy\nfrom tqdm import tqdm, tqdm_notebook, tnrange\ntqdm.pandas(desc='Progress')\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader, WeightedRandomSampler\nfrom torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence\nfrom torch.autograd import Variable\nfrom sklearn.model_selection import StratifiedShuffleSplit, train_test_split\nfrom sklearn.metrics import accuracy_score, f1_score\nimport warnings\nwarnings.filterwarnings('ignore')\nfrom IPython.core.interactiveshell import InteractiveShell\nInteractiveShell.ast_node_interactivity='all'\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.utils import class_weight\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"385710bd42383457248aec0e30a8de27d8352d13"},"cell_type":"code","source":"\n# Build Pytorch DataLoader Class (will change it to torchtext)\nclass QuestionDataLoader(Dataset):\n    def __init__(self, df, word2idx, nlp, is_test=False, maxlen=70):\n        self.maxlen = maxlen\n        self.word2idx = word2idx\n        self.nlp = nlp\n        self.is_test = is_test\n        self.df = df #pd.read_csv(df_path, error_bad_lines=False)\n        self.df['question_text'] = self.df.question_text.apply(lambda x: x.strip())\n        print('Indexing...')\n        self.df['question_ids'] = self.df.question_text.progress_apply(self.indexer)\n        print('Calculating lengths')\n        self.df['lengths'] = self.df.question_ids.progress_apply(lambda x: self.maxlen if len(x) > self.maxlen else len(x))\n        print('Padding')\n        self.df['question_padded'] = self.df.question_ids.progress_apply(self.pad_data)\n\n    @classmethod\n    def fromfilename(cls, name, word2idx, nlp, is_test=False):\n        return cls(pd.read_csv(name, error_bad_lines=False), word2idx, nlp,  is_test)\n    \n    def __len__(self):\n        return self.df.shape[0]\n    \n    \n    def __getitem__(self, idx):\n        X = self.df.question_padded[idx]\n        lens = self.df.lengths[idx]\n        qids = self.df.qid[idx]\n        if not self.is_test:\n            y = self.df.target[idx]\n        else:\n            return X, lens, qids\n        return X,y,lens\n    \n    def pad_data(self, s):\n        padded = np.zeros((self.maxlen,), dtype=np.int64)\n        if len(s) > self.maxlen: \n            padded[:] = s[:self.maxlen]\n        else:\n            padded[:len(s)] = s\n        return padded\n    \n    def indexer(self, s):\n        return [self.word2idx[w.text.lower()] for w in self.nlp(s)]\n        \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b6991f0f71f40c81a46a2a03d49c4fd2594108e0"},"cell_type":"code","source":"# Build Model RNN (GRU)\n\nclass AttentionBiLSTM(nn.Module):\n    def __init__(self, vocab_size, embedding_dim, embedding_matrix, n_hidden, n_out):\n        super().__init__()\n        self.vocab_size, self.embedding_dim, self.embedding_matrix, self.n_hidden, self.n_out = vocab_size, embedding_dim, embedding_matrix, n_hidden, n_out\n        self.emb = nn.Embedding(self.vocab_size, self.embedding_dim)\n        self.emb.weight = nn.Parameter(torch.tensor(self.embedding_matrix, dtype=torch.float32))\n        self.emb.weight.requires_grad = False\n        self.dropout = 0.8\n        self.relu = nn.ReLU()\n        self.bilstm = nn.LSTM(self.embedding_dim, self.n_hidden, dropout=self.dropout, bidirectional=True)\n        self.W_s1 = nn.Linear(2*self.n_hidden, 350)\n        self.W_s2 = nn.Linear(350, 30)\n        self.fc_layer = nn.Linear(30*2*self.n_hidden, 500)\n        self.label = nn.Linear(500, self.n_out)\n    \n    def attn_bahdanau(self, lstm_output):\n        attn_weight_matrix = self.W_s2(F.tanh(self.W_s1(lstm_output)))\n        # batch, num_seq, 30\n        attn_weight_matrix = attn_weight_matrix.permute(0, 2, 1)\n        # batch, 30, num_seq\n        attn_weight_matrix = F.softmax(attn_weight_matrix, dim=2)\n        return attn_weight_matrix\n    \n    def forward(self, seq, lengths, gpu=True):\n        bs = seq.size(1) # batch size\n        h0, c0 = self.init_hidden(bs, gpu) # initialize hidden state of GRU\n        embs = self.emb(seq)\n        embs = pack_padded_sequence(embs, lengths) # unpad\n        output, (hn, cn) = self.bilstm(embs, (h0, c0)) # \n        output, lengths = pad_packed_sequence(output) # pad the sequence to the max length in the batch\n        \n        output = output.permute(1, 0, 2)\n        # batch, 30, num_seq # batch, num_seq, 2*hidden\n        attn_matrix = self.attn_bahdanau(output)\n        # batch, num_seq, 2*hidden\n        hidden_matrix = torch.bmm(attn_matrix, output)\n        # hidden_matrix.size() = (batch_size, 30, 2*hidden_size)\n        # Let's now concatenate the hidden_matrix and connect it to the fully connected layer.\n        fc_out = self.fc_layer(hidden_matrix.view(-1, hidden_matrix.size()[1]*hidden_matrix.size()[2]))\n        fc_out = self.relu(fc_out)\n        outp = self.label(fc_out) # \n        return F.log_softmax(outp, dim=-1)\n    \n    def init_hidden(self, batch_size, gpu):\n        if gpu: return (Variable(torch.zeros((2,batch_size,self.n_hidden)).cuda()), Variable(torch.zeros((2,batch_size,self.n_hidden)).cuda()))\n        else: return Variable(torch.zeros((2,batch_size,self.n_hidden)))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# Utils\n\n\ndef eval_model(model, data_iter, criterion, device):\n    model.eval()\n    with torch.no_grad():\n        y_true_eval = list()\n        y_pred_eval = list()\n        ind_ids =list()\n        for X, y, lengths in data_iter:\n            X,y,lengths, indx = sort_batch(X,y,lengths)\n            X = Variable(X.cuda())\n            pred = model(X, lengths, gpu=True)\n            pred_idx = torch.max(pred, dim=1)[1]\n            y_pred_eval += list(pred_idx.cpu().data.numpy())\n            y_true_eval += list(y.cpu().data.numpy())\n            ind_ids += list(indx.cpu().numpy())\n        eval_acc = f1_score(y_true_eval, y_pred_eval)\n        print(f'Accuracy on eval set: {eval_acc}')\n        return y_pred_eval, y_true_eval, ind_ids\n\n\ndef predict(model, test_iter, criterion, device):\n    model.eval()\n    with torch.no_grad():\n        y_pred_test = list()\n        ind_ids =list()\n        X_temp =list()\n        for X, lengths, qids in test_iter:\n            X,lengths, qids = sort_batch_test(X,lengths, np.array(qids))\n            X = Variable(X.cuda())\n            pred = model(X, lengths, gpu=True)\n            pred_idx = torch.max(pred, dim=1)[1]\n            y_pred_test += list(pred_idx.cpu().data.numpy())\n            ind_ids += list(np.array(qids))\n            X = X.transpose(1,0)\n            X_temp += list(X.cpu().numpy())\n        return np.array(y_pred_test), np.array(ind_ids), X_temp\n\ndef sort_batch(X, y, lengths):\n    lengths, indx = lengths.sort(dim=0, descending=True)\n    X = X[indx]\n    y = y[indx]\n    return X.transpose(0,1), y, lengths, indx\n\ndef sort_batch_test(X, lengths, qids):\n    lengths, indx = lengths.sort(dim=0, descending=True)\n    X = X[indx]\n    return X.transpose(0,1), lengths, qids[indx]\n\ndef fit(model, train_dl, val_dl, loss_fn, opt, epochs=3):\n    num_batch = len(train_dl)\n    best_loss = None\n    for epoch in tnrange(epochs):      \n        y_true_train = list()\n        y_pred_train = list()\n        total_loss_train = 0\n        \n        if val_dl:\n            y_true_val = list()\n            y_pred_val = list()\n            total_loss_val = 0\n        \n        t = tqdm_notebook(iter(train_dl), leave=False, total=num_batch)\n        count = 0\n        for X,y, lengths in t:\n            t.set_description(f'Epoch {epoch}')\n            X,y,lengths, indx = sort_batch(X,y,lengths)\n            X = Variable(X.cuda())\n            y = Variable(y.cuda())\n            lengths = lengths.numpy()\n            count += 1\n            opt.zero_grad()\n            pred = model(X, lengths, gpu=True)\n            loss = loss_fn(pred, y)\n            #if count % 100 == 0:\n            #    import pdb;pdb.set_trace()\n            loss.backward()\n            opt.step()\n            #import pdb;pdb.set_trace()\n            t.set_postfix(loss=loss.data.item())\n            pred_idx = torch.max(pred, dim=1)[1]#torch.max(pred, dim=1)[1]\n            \n            y_true_train += list(y.cpu().data.numpy())\n            y_pred_train += list(pred_idx.cpu().data.numpy())\n            total_loss_train += loss.data.item()\n            \n        train_acc = f1_score(y_true_train, y_pred_train)\n        train_loss = total_loss_train/len(train_dl)\n        print(f' Epoch {epoch}: Train loss: {train_loss} acc: {train_acc}')\n        \n        if val_dl:\n            for X,y,lengths in tqdm_notebook(val_dl, leave=False):\n                X, y,lengths, idx = sort_batch(X,y,lengths)\n                X = Variable(X.cuda())\n                y = Variable(y.cuda())\n                pred = model(X, lengths.numpy())\n                loss = loss_fn(pred, y)\n                pred_idx = torch.max(pred, 1)[1]#torch.max(pred, 1)[1]\n                y_true_val += list(y.cpu().data.numpy())\n                y_pred_val += list(pred_idx.cpu().data.numpy())\n                total_loss_val += loss.data.item()\n            valacc = f1_score(y_true_val, y_pred_val)\n            valloss = total_loss_val/len(val_dl)\n            print(best_loss, valloss)\n            if (epoch > 2) and valloss > best_loss:\n                print(f'Val loss: {valloss} acc: {valacc}')\n                break\n            best_loss = valloss\n            print(f'Val loss: {valloss} acc: {valacc}')\n            \n\ndef main(sample = False):\n    data_root = pathlib.Path('../input')\n    df_train = pd.read_csv(data_root/'train.csv', error_bad_lines=False)\n    df_test = pd.read_csv(data_root/'test.csv', error_bad_lines=False)\n    \n    if sample:\n        df_train = df_train[:50000]\n    \n    df_train.question_text.progress_apply(lambda x: x.strip())\n    df_test.question_text.progress_apply(lambda x: x.strip())\n    \n    # Construct vaobaulary\n    nlp = spacy.load('en',disable=['parser', 'tagger', 'ner'])\n    words = Counter()\n    for sent in tqdm_notebook(df_train.question_text.values):\n        words.update(w.text.lower() for w in nlp(sent))\n    for sent in tqdm_notebook(df_test.question_text.values):\n        words.update(w.text.lower() for w in nlp(sent))\n    words = sorted(words, key=words.get, reverse=True)\n    words = ['_PAD','_UNK'] + words\n    word2idx = {o:i for i,o in enumerate(words)}\n    idx2word = {i:o for i,o in enumerate(words)}\n    def indexer(s): return [word2idx[w.text.lower()] for w in nlp(s)]\n    \n    train_df, other_df = train_test_split(df_train, test_size=0.2)\n    val_df, eval_df = train_test_split(other_df, test_size=0.5)\n    \n    \n    # Build train, val, eval and test Datasets\n    dtrain = QuestionDataLoader(train_df.reset_index(drop=True), word2idx, nlp)\n    dval = QuestionDataLoader(val_df.reset_index(drop=True), word2idx, nlp)\n    deval = QuestionDataLoader(eval_df.reset_index(drop=True), word2idx, nlp)\n    dtest = QuestionDataLoader.fromfilename(data_root/'test.csv', word2idx, nlp, True)\n    dl_train = DataLoader(\n        dtrain,\n        batch_size=256,\n        drop_last=True)\n    dl_val = DataLoader(\n        dval,\n        batch_size=256,\n        drop_last=True)\n    dl_eval = DataLoader(\n        deval,\n        batch_size=256,\n        drop_last=True)\n    dl_test = DataLoader(dtest, batch_size= 256)#len(df_test))\n    \n    \n    # Load Embeddings\n    print(\"Loading embeddings\")\n    maxlen = 120000\n    embed_size = 300\n    embedding_path = \"../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec\"\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    \n    nb_words = max(maxlen, len(word2idx))\n    emb_mean,emb_std = -0.0033469985, 0.109855495\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    \n    if not sample:\n        embedding_index = dict(get_coefs(*o.split(\" \")) for o in open(embedding_path, encoding='utf-8', errors='ignore') if len(o)>100)\n        for word, i in word2idx.items():\n            if i >= nb_words:\n                continue\n            embedding_vector = embedding_index.get(word)\n            if embedding_vector is not None:\n                embedding_matrix[i] = embedding_vector\n    print(\"Loaded embeddings\")\n    # Model Params\n    vocab_size = nb_words#max(nb_words, len(words))\n    embedding_dim = 300\n    n_hidden = 200\n    n_out = 2\n    num_epochs = 10\n    lr_rate = 3e-4\n    # Start Train\n    model_bilstm = AttentionBiLSTM(vocab_size, embedding_dim, embedding_matrix, n_hidden, n_out).cuda()\n    opt = optim.Adam(model_bilstm.parameters(),  lr_rate)\n    #fit(model=model_bilstm, train_dl=dl_train, val_dl=dl_val, loss_fn=F.nll_loss, opt=opt, epochs=num_epochs)\n    fit(model=model_bilstm, train_dl=dl_train, val_dl=dl_val, loss_fn=F.nll_loss, opt=opt, epochs=num_epochs)\n    # Accuracy on Eval set\n    _, _, _ = eval_model(model_bilstm, dl_eval, F.nll_loss, torch.device(\"cuda\") )\n    \n    # Predict on test-set\n    dl_test = DataLoader(dtest, batch_size= 256)\n    test_pred, test_ids, X_temp = predict(model_bilstm, dl_test, F.nll_loss, torch.device(\"cuda\") )\n    print(\"Predict Done\")\n    # Prepare submission\n    sub = pd.read_csv('../input/sample_submission.csv')\n    new_sub = pd.DataFrame()\n    #new_sub = sub.reindex(test_ids)\n    new_sub[\"qid\"] = test_ids\n    new_sub[\"prediction\"] = test_pred\n    #new_sub.sort_index(inplace=True)\n    new_sub.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e231a7ef8d3a45525abdcc291bbb9a24789e671f","scrolled":false},"cell_type":"code","source":"main()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fd28cd6be3b03b4507814f58a6753f9a7a559f2a"},"cell_type":"code","source":"# weights = [0.4, 1]\n# class_weights=torch.FloatTensor(weights).cuda()\n# learn.crit = nn.CrossEntropyLoss(weight=class_weights)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c7283ae79b0a3fbd09e61bc12b7b96df9fb24d31"},"cell_type":"code","source":"#dtest = QuestionDataLoader.fromfilename(data_root/'test.csv', word2idx, nlp, True)\n#X,lengths, indx = sort_batch_test(X,lengths)\n#dl_test = DataLoader(dtest, batch_size= 256)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e0f7b69c89a4b40edf392f0a2ffa6d7e9b81cb8d"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"789d4bf8c08bf3f78e66e5f5d2a1d793b2fc542b"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9cedda9fde2224f8a082931383fb7bfa613778ea"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b792dfd3f6b06a1b90aba907da09663dd791670d"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"87f4d831b2973f632b856dae92ba3d4ff38b0efd"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}