{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom sklearn.model_selection import train_test_split\nfrom zipfile import ZipFile \nimport seaborn as sns\nfrom sklearn.metrics import confusion_matrix\nfrom wordcloud import WordCloud\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import f1_score\n\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n    \n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### set random seed"},{"metadata":{"trusted":true},"cell_type":"code","source":"import random\nrandom.seed(1234)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\n\ntorch.__version__","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Unzip word embeddings**"},{"metadata":{"trusted":true},"cell_type":"code","source":"with ZipFile('/kaggle/input/quora-insincere-questions-classification/embeddings.zip') as z: \n    z.extract('glove.840B.300d/glove.840B.300d.txt')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Read embeddings and store in dictionary.**"},{"metadata":{"trusted":true},"cell_type":"code","source":"word2vec = {}\nf = open('glove.840B.300d/glove.840B.300d.txt')\nfor line in f:\n    values = line.split(\" \")\n    word = values[0]\n    coefs = np.asarray(values[1:], dtype='float32')\n    word2vec[word] = coefs\nf.close()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Read and split data into train and val sets.**"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/train.csv')\ntrain_df, val_df = train_test_split(train_df, test_size=0.1)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Choose only sentences that have lenght less than 40, for performance purpouces and see all sentences length distribution**"},{"metadata":{},"cell_type":"markdown","source":"# Data analyze"},{"metadata":{"trusted":true},"cell_type":"code","source":"max_length = 40\n\n# distribution of sentence lengths\nlengths = train_df['question_text'].apply(lambda x: len(x.split(' '))).to_list()\nsns.distplot(lengths)\n\ntrain_df = train_df[train_df.apply(lambda x : len(x['question_text'].split(\" \")) <= max_length,axis=1)]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Distribution of positive and negative sentences"},{"metadata":{"trusted":true},"cell_type":"code","source":"import plotly.express as px\nfig = px.pie(train_df, names='target', title='Distribution of sentiment',width=600, height=400)\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"As we can see data is pretty unbalanced, thats why we use f1 score and not regular accuracy."},{"metadata":{},"cell_type":"markdown","source":"## WordCloud for negative sentences"},{"metadata":{"trusted":true},"cell_type":"code","source":"wordcloud = WordCloud().generate(' '.join(train_df[train_df['target'] == 0]['question_text'].tolist()))\n\n# Display the generated image:\nplt.figure( figsize=(20,10) )\nplt.imshow(wordcloud, interpolation='bilinear')\nplt.axis(\"off\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## WordCloud for positive sentences"},{"metadata":{"trusted":true},"cell_type":"code","source":"wordcloud = WordCloud().generate(' '.join(train_df[train_df['target'] == 1]['question_text'].tolist()))\n\n# Display the generated image:\nplt.figure( figsize=(20,10) )\nplt.imshow(wordcloud, interpolation='bilinear')\nplt.axis(\"off\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Batch generator, returns tensor [bath_size, sentence_length, embedding_size], where sentence_length=40 and  embedding_size= 300**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def put_embeddings(text):\n    words = text.split(' ')\n    embs = [word2vec.get(x.lower(), np.zeros(300)) for x in words] + [np.zeros(300)] * (max_length - len(words))\n    return np.asarray(embs[0:max_length])\n\ndef batch_gen(df, batch_size):\n    n_batches = (len(df) + batch_size - 1) // batch_size\n    df = df.sample(frac=1.)  # Shuffle the data.\n    for i in range(n_batches):\n        texts = df[i*batch_size:(i+1)*batch_size]['question_text'].to_list()\n        embs = np.array([put_embeddings(txt) for txt in texts], dtype=np.dtype('float64'))\n        targets = (df[\"target\"][i*batch_size:(i+1)*batch_size]).to_list()\n        yield torch.tensor(embs, device=torch.device('cuda:0')), torch.tensor(targets, device=torch.device('cuda:0'))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training"},{"metadata":{},"cell_type":"markdown","source":"### LSTM model which uses pytorchs lstm module, adds dropout layer and also adds linear layer for output, so output could be 2 dimensional for our problem."},{"metadata":{"trusted":true},"cell_type":"code","source":"class LSTMModel(nn.Module):\n    def __init__(self, embedding_dim, hidden_dim, dropout):\n\n        super().__init__()\n\n        self.hidden_dim = hidden_dim\n        \n        self.lstm = torch.nn.LSTM(input_size=embedding_dim, \n                                  hidden_size=hidden_dim, \n                                  bias=True,\n                                  batch_first=True\n                                 )\n        \n        self.dropout = nn.Dropout(dropout)\n        self.classifier = nn.Linear(hidden_dim, 2)\n\n    def forward(self, inp):   \n        inp = self.dropout(inp)\n        _, (hn, _) = self.lstm(inp)\n\n        return self.classifier(hn)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Classical Train loop, uses learning rate scheduler, for adapting learning rate. I only use 30 epoch, but increasing amount of epoch gives us better results."},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_score(y_pred, y):\n    y_pred = [0 if p[0] > p[1] else 1 for p in y_pred]\n    return f1_score(y.cpu(), y_pred, zero_division=0)\n\ndef compute_perplexity(model, dl):\n    model.eval()\n    score = 0\n    length = 0\n    with torch.no_grad(): # tells Pytorch not to store values of intermediate computations for backward pass because we not gonna need gradients.\n        loss = 0\n        for x, y in dl:\n            x = x.float()\n            y_pred = model(x).squeeze(0)\n            if y_pred.shape[0] == 128:\n                score += get_score(y_pred, y)\n                length += 1\n            loss += torch.nn.functional.cross_entropy(y_pred, y).item()\n        model.train()\n\n    return np.exp(loss / length), score/(length)\n \n    \ndef get_lr(optimizer):\n    for param_group in optimizer.param_groups:\n        return param_group['lr']\n    \ndef train_loop(model):\n    model.train()\n    optimizer = torch.optim.Adam(model.parameters(), lr=1e-4, weight_decay=1e-4)\n    lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.5, min_lr=1e-6, patience=10)\n    crit = nn.CrossEntropyLoss(reduction='mean')\n    bs = 128\n    \n    \n    curr_perplexity = None\n    perplexity = None\n    \n    for epoch in range(30):\n        train_dl = batch_gen(train_df, bs)\n        total_loss = 0 \n        batch_num = 0\n\n        for x, y in train_dl:\n            optimizer.zero_grad()\n            x = x.float()            \n            y_pred = model(x).squeeze(0)\n            loss = crit(y_pred, y)\n            total_loss += loss.item()\n            batch_num += 1\n            loss.backward()\n\n            # doing gradient descent step.\n            optimizer.step()\n            \n        \n        eval_dl = batch_gen(val_df, bs)\n        curr_perplexity, score = compute_perplexity(model, eval_dl)\n        lr_scheduler.step(curr_perplexity)\n\n        print('Epoch', epoch + 1, '| Avg Train Loss', total_loss / batch_num, '| Dev Perplexity', curr_perplexity, '| LR ', get_lr(optimizer), '| F1 score', score)\n\nhidden_dim = 30 \nembedding_dim = 300\ndropout = 0.2\n            \nmodel = LSTMModel(embedding_dim, hidden_dim, dropout).cuda()\ntrain_loop(model)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Testing"},{"metadata":{},"cell_type":"markdown","source":"### Read test data frame"},{"metadata":{"trusted":true},"cell_type":"code","source":"import math\n\ntest_df = pd.read_csv(\"/kaggle/input/quora-insincere-questions-classification/test.csv\")\n\nprint(len(test_df))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Compute model results for every test and put output in submission.csv"},{"metadata":{"trusted":true},"cell_type":"code","source":"def getBatchedTest(batch_size):\n    n_batches = math.ceil(len(test_df) / batch_size)\n    for i in range(n_batches):\n        texts = test_df.iloc[i*batch_size:(i+1)*batch_size, 1]\n        text_arr = np.array([put_embeddings(txt[:max_length]) for txt in texts])\n        yield torch.Tensor(text_arr).cuda()\n        \n        \nall_preds = []\nmodel.eval()\nwith torch.no_grad():\n    for X_batch in getBatchedTest(256):\n        y_test_pred = model(X_batch).squeeze(0)\n        all_preds += [0 if y[0] > y[1] else 1 for y in y_test_pred]\n\nsubmit_df = pd.DataFrame({\"qid\": test_df[\"qid\"], \"prediction\": all_preds})\nsubmit_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Test for custom sentence"},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef get_prediction(sentence):\n    inp = put_embeddings(sentence)\n    inp = torch.tensor(inp, device=torch.device('cuda:0')).unsqueeze(0).float()\n    y_pred = model(inp).squeeze()    \n    return 0 if y_pred[0] > y_pred[1] else 1 \n\nneg = ['Why do so many women become so rude and arrogant when they get just a little bit of wealth and power?',\n             'Why do Bengali and Kolkata people dominate Hindi language speaking people from Delhi, Mumbai and other parts of India?']\n             \npos = ['When should I apply for RV college of engineering and BMS college of engineering? Should I wait for the COMEDK result or am I supposed to apply before the result?',\n       'What kind of knife can I own in California?'\n      ]\n\nprint('negatives')\nprint(neg[0], ' prediction --', get_prediction(neg[0]), '\\n')\nprint(neg[1], ' prediction --', get_prediction(neg[1]), '\\n')\nprint('positives')\nprint(pos[0], ' prediction --', get_prediction(pos[0]), '\\n')\nprint(pos[1], ' prediction --', get_prediction(pos[1]), '\\n')\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}