{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"from zipfile import *\nfrom tqdm import tqdm\nimport os\nimport random\n\nimport seaborn as sns\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nimport torch\nimport torch.nn as nn\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import f1_score\n\nimport matplotlib.pyplot as plt\nfrom wordcloud import WordCloud\nfrom collections import Counter","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Set the random seeds for deterministic results.**"},{"metadata":{"trusted":true},"cell_type":"code","source":"SEED = 1234\n\nrandom.seed(SEED)\nnp.random.seed(SEED)\ntorch.manual_seed(SEED)\ntorch.cuda.manual_seed(SEED)\ntorch.backends.cudnn.deterministic = True","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# kaggle starter code, just to know full paths of files\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data analysis"},{"metadata":{"trusted":true},"cell_type":"code","source":"# load data from train.csv file and split it to train and validation\ntrain_df = pd.read_csv(\"/kaggle/input/quora-insincere-questions-classification/train.csv\")\ntrain_df, val_df = train_test_split(train_df, test_size=0.1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Draw graph which shows number of words in sentences of train and validation sets."},{"metadata":{"trusted":true},"cell_type":"code","source":"train_sentences_lens = train_df['question_text'].apply(lambda x: len(x.split(' '))).tolist()\nval_sentences_lens = val_df['question_text'].apply(lambda x: len(x.split(' '))).tolist()\nsns.distplot(train_sentences_lens)\nsns.distplot(val_sentences_lens)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"As we see from the graph, the number of cases where words counts greater than 40 is too small."},{"metadata":{"trusted":true},"cell_type":"code","source":"SENTENCE_MAX_LEN = 40","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Draw pie chart which shows distribution of positive and negative examples"},{"metadata":{"trusted":true},"cell_type":"code","source":"def pie_chart(positive_samples_num, negative_samples_num, set_type):\n    labels = 'Insincere', 'Sincere'\n    sizes = [positive_samples_num, negative_samples_num]\n\n    fig1, ax1 = plt.subplots()\n    ax1.set_title(set_type)\n    ax1.pie(sizes, labels=labels, autopct='%1.1f%%',shadow=True, startangle=90)\n\n    plt.show()\n\ntrain_positive_samples_num = sum(train_df['target'].values)\nval_positive_samples_num = sum(val_df['target'].values)\npie_chart(train_positive_samples_num, len(train_df['target']) - train_positive_samples_num, 'train set')\npie_chart(val_positive_samples_num, len(train_df['target']) - val_positive_samples_num, 'validation set')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"From the graphs we can see that data is unbalanced and later we need to use some techniques to avoid overfitting.\nIn this case accuracy won't be the best metric to evaluate your model."},{"metadata":{},"cell_type":"markdown","source":"## Word clouds"},{"metadata":{"trusted":true},"cell_type":"code","source":"sincere_text = ' '.join(train_df[train_df['target'] == 0]['question_text'].tolist())\ninsincere_text = ' '.join(train_df[train_df['target'] == 1]['question_text'].tolist())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def draw_word_cloud(text):\n    wordcloud = WordCloud(background_color=\"white\", max_words=100, min_word_length=5).generate(text)\n\n    plt.figure(figsize=(15,10))\n    plt.imshow(wordcloud, interpolation='bilinear')\n    plt.axis(\"off\")\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Sincere words cloud"},{"metadata":{"trusted":true},"cell_type":"code","source":"draw_word_cloud(sincere_text)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"As sincere words clouds shows, there are words like use, work, help, make and so on. Such kind of words used very frequently in the normal questions and They are mostly neutral and indicate someone actually looking for advice."},{"metadata":{},"cell_type":"markdown","source":"### Insincere words cloud"},{"metadata":{"trusted":true},"cell_type":"code","source":"draw_word_cloud(insincere_text)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We can see difference from the previous graph, because in this case the most frequently used words are all very political(for example Donald Trump) Instead of the generic advice topics. \n\nAlso can be seen that the words in general are mostly not negative. for example word \"people\" is neither positive or negative. But coupled with other words it should be possible to perform useful topic analysis, thats why simple sentiment analysis of words is not sufficient and we need something more powerful, for example lstm."},{"metadata":{},"cell_type":"markdown","source":"# Word2vec functions"},{"metadata":{},"cell_type":"markdown","source":"**store embeddings in dictionary for every word.\nembeddings_index (where key is word and value is embedding array)**\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"embeddings_index = {}\n\nwith ZipFile('/kaggle/input/quora-insincere-questions-classification/embeddings.zip') as myzip:\n    with myzip.open('glove.840B.300d/glove.840B.300d.txt') as myfile:\n        lines = myfile.readlines()\n        for line in tqdm(lines):\n            values = line.decode().split(\" \")\n            word = values[0] \n            embeddings_index[word] = np.asarray(values[1:], dtype='float32')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# this function takes text as argument and returns array of embeddings.\ndef to_emb(text, embedding_dim): \n    unk_emb = np.zeros(embedding_dim)\n    text = text[:-1].split()[:SENTENCE_MAX_LEN]\n    embeds = [embeddings_index.get(x, unk_emb) for x in text]\n    \n    padding = np.zeros(embedding_dim)\n    embeds += [padding] * (SENTENCE_MAX_LEN - len(embeds))\n    return np.array(embeds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# this function takes array of texts and returs array of embedding arrays\ndef to_embeddings(texts, embedding_dim):\n    return torch.tensor(np.array([to_emb(text, embedding_dim) for text in texts])).float().cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# this function works like iterator, returns next batch_size arrays of texts and targets on every call\ndef batch_iterator(frame, batch_size):\n    frame = frame.sample(frac=1).reset_index(drop=True)\n    frame_len = len(frame)\n    \n    for ind in range(0, frame_len, batch_size):\n      x = frame['question_text'][ind : ind + batch_size].values\n      y = torch.tensor(frame['target'][ind : ind + batch_size].values).cuda()\n      yield x, y","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"class LSTMModel(nn.Module):\n  def __init__(self, embedding_dim, hidden_dim, output_dim, dropout):\n\n    super().__init__()\n    \n    self.embedding_dim = embedding_dim\n    self.hidden_dim = hidden_dim\n    self.output_dim = output_dim\n    self.dropout = nn.Dropout(dropout)\n    \n    self.lstm = nn.LSTM(input_size=embedding_dim, \n                   hidden_size=hidden_dim, \n                   bias=True,\n                   batch_first=True \n                  )\n    \n    self.classifier = nn.Linear(hidden_dim, output_dim)\n\n  def forward(self, inp):\n    # translate texts to embeddings\n    # after this operation inp will be (batch_size, max_seq_len, embedding_dim)\n    inp = to_embeddings(inp, self.embedding_dim)\n    \n    # add dropout to prevent neural network from overfitting.\n    inp = self.dropout(inp)\n    \n    # Initialize hidden state with zeros\n    h0 = torch.zeros(1, inp.size(0), self.hidden_dim, device=inp.device) \n    \n    # Initialize cell state\n    c0 = torch.zeros(1, inp.size(0), self.hidden_dim, device=inp.device)\n\n    out, (hn,cn) = self.lstm(inp, (h0, c0))\n\n    hn = hn.reshape(-1, self.hidden_dim)\n    \n    return self.classifier(hn).view(inp.size(0), self.output_dim)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training"},{"metadata":{},"cell_type":"markdown","source":"This part is similar to the one discussed in the lecture, but difference is f1 score metric."},{"metadata":{"trusted":true},"cell_type":"code","source":"# No need many epochs, because after 2-3 epochs model does not learn new things. \nEPOCHS = 3\nBS = 128\nEMBEDDING_DIM = 300","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# this function returns arg max of predictions.\ndef get_predictions_from_prob(y_pred):\n    return [1 if predictions[1] > predictions[0] else 0 for predictions in y_pred]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def compute_f1_and_perplexity(model, val_df):\n  model.eval()\n\n  val_BS = 4 * BS\n  loss = 0\n  score = 0\n  with torch.no_grad(): # tells Pytorch not to store values of intermediate computations for backward pass because we not gonna need gradients.\n    iterator = batch_iterator(val_df, val_BS)\n    for x, y in iterator:\n      y_pred = model(x)\n      loss += torch.nn.functional.cross_entropy(y_pred, y).item()\n      score += f1_score(y.cpu(), get_predictions_from_prob(y_pred), zero_division=0)\n  \n  model.train()\n\n\n  batchs_num = int((len(val_df) + val_BS - 1) / val_BS)\n  return score / batchs_num,  np.exp(loss / batchs_num)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_lr(optimizer):\n  for param_group in optimizer.param_groups:\n    return param_group['lr']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_loop(model, train_df, val_df):\n\n  model.train() \n\n  # we add weight decay (L2 regularization) to avoid overfitting.\n  optimizer = torch.optim.Adam(model.parameters(), lr=1e-4, weight_decay=1e-4)\n\n  # we will reduce initial learning rate by 'lr=lr*factor' every time validation perplexity doesn't improve within certain range.\n  lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.5, min_lr=1e-6, patience=10) \n\n  crit = nn.CrossEntropyLoss(reduction='mean')\n\n  it = 1\n  total_loss = 0\n  curr_perplexity = None\n  perplexity = None\n  f1_score = None\n\n  for epoch in range(EPOCHS):\n    iterator = batch_iterator(train_df, BS)\n    for x, y in iterator:\n\n      optimizer.zero_grad()\n      \n      # do forward pass, will save intermediate computations of the graph for later backprop use.\n      y_pred = model(x)\n         \n      loss = crit(y_pred, y)\n      \n      total_loss += loss.item()\n      \n      # running backprop.\n      loss.backward()\n\n      # doing gradient descent step.\n      optimizer.step()\n\n      # we are logging current loss/perplexity in every 1000 iteration\n      if it % 1000 == 0:\n      \n        # computing validation set perplexity in every 2000 iteration.\n        if it % 2000 == 0:\n          f1_score, curr_perplexity = compute_f1_and_perplexity(model, val_df)\n\n          lr_scheduler.step(curr_perplexity)\n\n          # making checkpoint of best model weights.\n          if not perplexity or curr_perplexity < perplexity:\n            torch.save(model.state_dict(), 'model')\n            perplexity = curr_perplexity\n\n        print('Epoch', epoch + 1, '| Iter', it, '| Avg Train Loss', total_loss / 1000, '| F1 score', f1_score, '| Dev Perplexity', curr_perplexity, '| LR ', get_lr(optimizer))\n        total_loss = 0\n\n      it += 1\n    \nmodel = LSTMModel(EMBEDDING_DIM, 100, 2, 0.1).cuda()\ntrain_loop(model, train_df, val_df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Testing"},{"metadata":{"trusted":true},"cell_type":"code","source":"# load test data from test.csv file\ntest_df = pd.read_csv(\"/kaggle/input/quora-insincere-questions-classification/test.csv\"); test_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# this function predicts targets for every test question.\ndef get_predictions(model, question_texts, batch_size):\n    model.eval()\n    res = []\n    question_texts_len = len(question_texts)\n    with torch.no_grad():\n        for ind in range(0, question_texts_len, batch_size):\n          x = question_texts[ind : ind + batch_size]\n          y_pred = model(x)\n          res += get_predictions_from_prob(y_pred)\n    return res\n\npredicitons = get_predictions(model, test_df['question_text'].values, 1000)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# write down answers in prediciton column.\ntest_df['prediction'] = predicitons","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df[['qid', 'prediction']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# save results to submission.csv. this is file, which should be commited\ntest_df[['qid', 'prediction']].to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Evaluation"},{"metadata":{},"cell_type":"markdown","source":"evaluate our model on some examples"},{"metadata":{"trusted":true},"cell_type":"code","source":"def evaluate_some_examples(model, texts, realTarget):\n    print('{} questions evaluation'.format('Sincere' if realTarget == 0 else 'Incinecere'))\n    predictions = get_predictions(model, texts, len(texts))\n    for text, prediction in zip(texts, predictions):\n        print('For text \"{}\" Model predicts {} and real target is {}'.format(text, prediction, realTarget))\n    print('\\n')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# sincere text are real quora questions.\nsincere_text_examples = ['Why are there crushed stones alongside rail tracks?',\n                        'What are some of the most inspirational photos ever taken?',\n                        'What is the greatest single image in movie history?',\n                        'What are the best proggraming blogs?',\n                        'What is the most horrific picture you have ever seen?']\n\ninsincere_text_examples = ['Why Jews Did not Leave Europe?',\n                          'What are some of the false things the U.S. government claimed about the conflict in the Ukraine?',\n                          'Why are women shameless?']\n\n\n\nevaluate_some_examples(model, sincere_text_examples, 0)\nevaluate_some_examples(model, insincere_text_examples, 1)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}