{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport nltk\nfrom sklearn.metrics import f1_score\nimport re\nimport string\nimport xgboost as xgb\nimport lightgbm as lgb\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.ensemble import GradientBoostingClassifier\nfrom sklearn.model_selection import StratifiedKFold, StratifiedShuffleSplit, KFold, GridSearchCV, RandomizedSearchCV\nimport catboost as cat\nimport pandas as pd\nimport numpy as np\nimport xgboost as xgb\nfrom tqdm import tqdm\nfrom sklearn.svm import SVC\nfrom keras.models import Sequential\nfrom keras.layers.recurrent import LSTM, GRU\nfrom keras.layers.core import Dense, Activation, Dropout\nfrom keras.layers.embeddings import Embedding\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.utils import np_utils\nfrom sklearn import preprocessing, decomposition, model_selection, metrics, pipeline\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\nfrom sklearn.decomposition import TruncatedSVD\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.naive_bayes import MultinomialNB\nfrom keras.layers import GlobalMaxPooling1D, Conv1D, MaxPooling1D, Flatten, Bidirectional, SpatialDropout1D\nfrom keras.preprocessing import sequence, text\nfrom keras.callbacks import EarlyStopping\nfrom nltk import word_tokenize\nfrom nltk.corpus import stopwords\n# stop_words = stopwords.words('english')\n\nimport warnings\nfrom sklearn.multiclass import OneVsRestClassifier\nwarnings.filterwarnings(\"ignore\")\neng_stopwords = set(stopwords.words(\"english\"))\nimport os\n\n####################\nimport os\nimport time\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom tqdm import tqdm\nimport math\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D\nfrom keras.layers import Bidirectional, GlobalMaxPool1D\nfrom keras.models import Model\nfrom keras import initializers, regularizers, constraints, optimizers, layers","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Final data preparation for modelling","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"os.listdir('../input')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Building vocabulary","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Process whcih will be followed below\n- Define fields for Question text, Label\n- Define a TabularDataset object for both train and test data\n- Split train data into train, validation split\n- Now build the vocabulary for the complete data using the build_vocab with WIKI news as vectors\n- Create a BucketIterator for the train, validation and test data sets","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.read_csv('../input/quora-insincere-questions-classification/train.csv', nrows = 100).head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# We need to define fields for how the every column will be processed in the tabular dataset\nfrom torchtext.data import Field, TabularDataset, BucketIterator\nimport torchtext\nimport torch\nimport torch.nn as nn","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- `Fields in the dataset`\n- `BucketIterator creation`\n- Removing the header from the fields","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# Ensure there is a field expression for every column in the data\nfrom nltk.corpus import stopwords \nfrom nltk.tokenize import word_tokenize \n\nexample_sent = \"This is a sample sentence, showing off the stop words filtration.\"\nstop_words = set(stopwords.words('english'))\n\n# Fields are ready for being used in the tabular dataset \ntokenize = lambda x: x.split()\nquestion = Field(sequential=True, use_vocab=True,tokenize=word_tokenize, stop_words = stop_words)\ntarget = Field(sequential=False, use_vocab=False, is_target = True, dtype=torch.float64)\nqid = torchtext.data.Field(use_vocab=False, sequential=False)\n\n\ntrain_set = TabularDataset(path = '../input/quora-insincere-questions-classification/train.csv',\n                      format='csv',\n                      fields = [('qid', None), ('question_text', question), ('target', target)],\n                          skip_header=True)\ntest_set = TabularDataset(path = '../input/quora-insincere-questions-classification/test.csv',\n                      format='csv',\n                      fields = [('qid', qid), ('question_text', question)],\n                          skip_header=True)\n\nprint(train_set[0].__dict__.keys())\ntrain_set[1].question_text, train_set[1].target\n\n# Building vocabulary using glove pretrained vectors\nquestion.build_vocab(train_set, min_freq = 3, max_size = 2000000)\nquestion.vocab.load_vectors(torchtext.vocab.Vectors('../input/glove6b/glove.6B.300d.txt'))\nprint(question.vocab.vectors.shape)\n\n\nimport random\ntrain_data, valid_data = train_set.split(split_ratio = 0.8, random_state = random.seed(123))\ntrain_iter, valid_iter = BucketIterator.splits((train_data, valid_data),\n                                        batch_size = 128,\n                                       sort_key = lambda x: len(x.question_text),\n                                       sort_within_batch = True,\n                                        device='cuda')\n\ntest_iter = BucketIterator(test_set,\n                          batch_size = 128,\n                          sort = False, sort_within_batch=False,\n                          device='cuda')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Testing the iterator for train and valid data\nfor batch in train_iter:\n    print(batch.question_text.shape)\n    break","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Building the Bi-LSTM model with Glove embeddings ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class params():\n    input_dim = question.vocab.vectors\n    batch_size = 128\n    embedding_dim = 300\n    hidden_dim = 128\n    output_dim = 1\n    learning_rate = 1e-3\n    num_layers = 3\n    bidirectional =True\n    dropout_prob = 0.2\n    padding_idx = question.vocab.stoi[question.pad_token]\n    static=False\n    device='cuda'\n    \nargs = params()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch.nn as nn\nclass BiLSTMS(nn.Module):\n    def __init__(self,vocab_size,  embedding_dim, static, hidden_dim, output_dim, padding_idx, num_layers,\n        bidirectional, dropout_prob):\n        super(BiLSTMS, self).__init__()\n        \n        # Initializing the embedding layer for the network\n        self.embedding = nn.Embedding.from_pretrained(vocab_size, embedding_dim, \n                                                      padding_idx=padding_idx)\n        self.static = static\n        # Making embeddings trainable \n        if self.static:\n            self.embedding.weight.requires_grad = False\n            \n        self.LSTM = nn.LSTM(embedding_dim, hidden_dim, num_layers,\n                    bidirectional= bidirectional, dropout=dropout_prob)\n        self.fc = nn.Linear(hidden_dim * 2, output_dim)\n        self.dropout = nn.Dropout(dropout_prob)\n\n    def forward(self, text):\n        # Text input  dimensions = [sentence_len, batch_size]\n        embedding = self.embedding(text)\n        embedding = self.dropout(embedding)\n        \n        # Removed the neurons who have a probability < dropout_prob\n\n        # Padding the sequences beyond the max sequence length for prediction of sent.\n#         print(text.shape[0])\n#         packed_embedding = nn.utils.rnn.pack_padded_sequence(embedding, text.shape[0])\n\n        packed_output, (hidden_state, cell_state) = self.LSTM(embedding)\n#         print(hidden_state.shape, cell_state.shape)\n        \n        # Unpack the sequences using\n#         output, output_lengths = nn.utils.rnn.pad_packed_sequence(packed_output)\n        # Concating the final forward and backward hidden state to make the predictions \n        # for the sentiment \n\n        hidden_final = self.dropout(torch.cat((hidden_state[-2, : , : ], hidden_state[-1, :, : ]), dim = 1))\n\n        return self.fc(hidden_final)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training the LSTM loop","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# args.static","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch.optim as optim\nfrom sklearn.metrics import f1_score, accuracy_score\nfrom tqdm import tqdm_notebook, tqdm\n\n# Model Instantiation\nmodel = BiLSTMS(args.input_dim, args.embedding_dim, args.static, args.hidden_dim, args.output_dim, \n                args.padding_idx, args.num_layers, args.bidirectional, args.dropout_prob)\nmodel = model.to(args.device)\n\n\noptimizer = optim.Adam(model.parameters())\nscheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode ='min', factor = 0.2,\n                                                patience=4, verbose=True)\ncriterion = nn.BCEWithLogitsLoss()\ncriterion = criterion.to(args.device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def evaluate(model, iterator, criterion):\n    targets= []\n    preds = []\n    with torch.no_grad():\n        loop = tqdm(enumerate(iterator), position = 0, leave =True)\n        for i, batch in loop:\n            text = batch.question_text\n            predictions = model(text).squeeze(1).double()\n            actual_labels = batch.target\n            targets += actual_labels.to('cpu').numpy().tolist()\n            preds += predictions.to('cpu').numpy().tolist()\n            \n            loop.set_description(f\"Evaluating model performance on validation dataset\")\n    print(len(preds), len(targets))\n    \n    label = [1 if pred >= 0.80 else 0 for pred in preds]\n    return f1_score(targets, label), accuracy_score(targets, label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_params(model):\n    return sum(p.numel() for p in model.parameters() if p.requires_grad)\nprint(f\"Number of trainable parameters in the Bi-LSTM model are {count_params(model)}\")\n\n# pretrained_embedding_vec = question.vocab.vectors\n# pretrained_embedding_vec.shape\n\n# # Changing the default Embeddings in the model to pretrained embeddings\n# model.embedding.weight.data.copy_(pretrained_embedding_vec)\n\n# # Getting UNK index from the vocab for question text\n# UNK_idx = question.vocab.stoi[question.unk_token]\n# model.embedding.weight.data[UNK_idx] = torch.zeros(args.embedding_dim)\n# model.embedding.weight.data[args.padding_idx] = torch.zeros(args.embedding_dim)\n# print(model.embedding.weight.data)\n# ## Training Loop for the LSTM model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f1_score_list = [0]\naccuracy_list = [0]\nnum_epochs = 15\n\n\nfor epoch in range(num_epochs):\n    epoch_loss = 0\n    no_improve_count = 0\n        \n    loop = tqdm(enumerate(train_iter),  position = 0, leave=True)\n    for i, batch in loop:\n        optimizer.zero_grad()\n        text = batch.question_text\n        model.train()\n        predictions = model(text).squeeze(1).double()\n        loss = criterion(batch.target, predictions)\n        loss.backward()\n        optimizer.step()\n        epoch_loss += loss.item()\n        loop.set_description(f\"Epoch {epoch +1}/{num_epochs}\")\n        loop.set_postfix(loss = loss.item(), acc = accuracy_list[epoch], f1_score = f1_score_list[epoch])\n\n    # Metrics on the evaluation set\n    model.eval()\n    f1, acc = evaluate(model, valid_iter, criterion)\n    mean_loss = epoch_loss / len(train_iter)\n    scheduler.step(mean_loss)\n    \n    f1_score_list.append(f1)\n    accuracy_list.append(acc)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# f1, acc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# targets= []\n# preds = []\n# with torch.no_grad():\n#     loop = tqdm(enumerate(valid_iter), position = 0, leave =True)\n#     for i, batch in loop:\n#         text = batch.question_text\n#         predictions = model(text).squeeze(1).double()\n#         actual_labels = batch.target\n#         targets += actual_labels.to('cpu').numpy().tolist()\n#         preds += predictions.to('cpu').numpy().tolist()\n\n#         loop.set_description(f\"Evaluating model performance on validation dataset\")\n# print(len(preds), len(targets))\n\n# label = [1 if pred >= 0.80 else 0 for pred in preds]\n# f1_score(targets, label), accuracy_score(targets, label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.sum(label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# Testing tutorial code","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# os.listdir('../input')\n\n# train = pd.read_csv('../input/quora-insincere-questions-classification/train.csv',)\n# test = pd.read_csv('../input/quora-insincere-questions-classification/test.csv')\n\n# train.shape, test.shape\n\n# train['target'].value_counts()\n\n# # Preprocessing the text\n\n# train['len_of_sentence'] = train['question_text'].apply(lambda x: len(x.split()))\n# import seaborn as sns\n# sns.kdeplot(train['len_of_sentence'])\n# plt.show()\n\n# import matplotlib.pyplot as plt\n\n# plt.figure(figsize = (15, 8))\n# plt.title(\"Disbution of lengths of sentences\")\n# train['len_of_sentence'].value_counts().plot(kind='bar')\n# plt.xticks(rotation = 45)\n# plt.xlabel(\"Length of sentence\")\n# plt.ylabel(\"#Sentences\")\n# plt.show()\n\n# puncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', \n#  '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n#  '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n#  '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n#  '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]\n\n# def clean_text(x):\n#     x = str(x)\n#     for punct in puncts:\n#         x = x.replace(punct, f' {punct} ')\n#     return x\n\n# def clean_numbers(x):\n#     x = re.sub('[0-9]{5,}', '#####', x)\n#     x = re.sub('[0-9]{4}', '####', x)\n#     x = re.sub('[0-9]{3}', '###', x)\n#     x = re.sub('[0-9]{2}', '##', x)\n#     return x\n\n# mispell_dict = {\"aren't\" : \"are not\",\n# \"can't\" : \"cannot\",\n# \"couldn't\" : \"could not\",\n# \"didn't\" : \"did not\",\n# \"doesn't\" : \"does not\",\n# \"don't\" : \"do not\",\n# \"hadn't\" : \"had not\",\n# \"hasn't\" : \"has not\",\n# \"haven't\" : \"have not\",\n# \"he'd\" : \"he would\",\n# \"he'll\" : \"he will\",\n# \"he's\" : \"he is\",\n# \"i'd\" : \"I would\",\n# \"i'd\" : \"I had\",\n# \"i'll\" : \"I will\",\n# \"i'm\" : \"I am\",\n# \"isn't\" : \"is not\",\n# \"it's\" : \"it is\",\n# \"it'll\":\"it will\",\n# \"i've\" : \"I have\",\n# \"let's\" : \"let us\",\n# \"mightn't\" : \"might not\",\n# \"mustn't\" : \"must not\",\n# \"shan't\" : \"shall not\",\n# \"she'd\" : \"she would\",\n# \"she'll\" : \"she will\",\n# \"she's\" : \"she is\",\n# \"shouldn't\" : \"should not\",\n# \"that's\" : \"that is\",\n# \"there's\" : \"there is\",\n# \"they'd\" : \"they would\",\n# \"they'll\" : \"they will\",\n# \"they're\" : \"they are\",\n# \"they've\" : \"they have\",\n# \"we'd\" : \"we would\",\n# \"we're\" : \"we are\",\n# \"weren't\" : \"were not\",\n# \"we've\" : \"we have\",\n# \"what'll\" : \"what will\",\n# \"what're\" : \"what are\",\n# \"what's\" : \"what is\",\n# \"what've\" : \"what have\",\n# \"where's\" : \"where is\",\n# \"who'd\" : \"who would\",\n# \"who'll\" : \"who will\",\n# \"who're\" : \"who are\",\n# \"who's\" : \"who is\",\n# \"who've\" : \"who have\",\n# \"won't\" : \"will not\",\n# \"wouldn't\" : \"would not\",\n# \"you'd\" : \"you would\",\n# \"you'll\" : \"you will\",\n# \"you're\" : \"you are\",\n# \"you've\" : \"you have\",\n# \"'re\": \" are\",\n# \"wasn't\": \"was not\",\n# \"we'll\":\" will\",\n# \"didn't\": \"did not\",\n# \"tryin'\":\"trying\"}\n\n# def _get_mispell(mispell_dict):\n#     mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n#     return mispell_dict, mispell_re\n\n# mispellings, mispellings_re = _get_mispell(mispell_dict)\n# def replace_typical_misspell(text):\n#     def replace(match):\n#         return mispellings[match.group(0)]\n#     return mispellings_re.sub(replace, text)\n\n# # Clean the text\n# train[\"question_text\"] = train[\"question_text\"].apply(lambda x: clean_text(x.lower()))\n# test[\"question_text\"] = test[\"question_text\"].apply(lambda x: clean_text(x.lower()))\n\n# # Clean numbers\n# train[\"question_text\"] = train[\"question_text\"].apply(lambda x: clean_numbers(x))\n# test[\"question_text\"] = test[\"question_text\"].apply(lambda x: clean_numbers(x))\n\n# # Clean speelings\n# train[\"question_text\"] = train[\"question_text\"].apply(lambda x: replace_typical_misspell(x))\n# test[\"question_text\"] = test[\"question_text\"].apply(lambda x: replace_typical_misspell(x))\n\n# # os.listdir('../input/glove840b300dtxt/glove.840B.300d.txt')\n\n# train['length'] = train['question_text'].apply(lambda x: len(x.split()))\n# test['length'] = test['question_text'].apply(lambda x: len(x.split()))\n\n# np.mean(train['length']), np.mean(test['length']), np.max(train['length']), np.max(test['length'])\n\n# from keras.preprocessing.text import Tokenizer\n# from keras.preprocessing.sequence import pad_sequences\n\n# num_words = 120000\n# tx = Tokenizer(num_words = num_words, lower = True, filters = '')\n# full_text = list(train['question_text'].values) + list(test['question_text'].values)\n# tx.fit_on_texts(full_text)\n\n# train_tokenized = tx.texts_to_sequences(train['question_text'].fillna('missing'))\n# test_tokenized = tx.texts_to_sequences(test['question_text'].fillna('missing'))\n\n# max_len = 100\n# X_train = pad_sequences(train_tokenized, maxlen = max_len)\n# X_test = pad_sequences(test_tokenized, maxlen = max_len)\n\n# X_train.shape","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}