{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from zipfile import ZipFile \n  \nwith ZipFile('/kaggle/input/quora-insincere-questions-classification/embeddings.zip', 'r') as embd_zip: \n    print(embd_zip.namelist())\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**START WORKING ON DATA**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def DEBUG_DICTIONARY(dct, limit=10):\n    for i, key in enumerate(dct.keys()):\n        if i > limit: break\n        print(key, dct[key])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\n# configure train and validation data\ntrain_data, val_data = train_test_split(pd.read_csv('/kaggle/input/quora-insincere-questions-classification/train.csv'), test_size=0.2, random_state=42)\nsentences, targets = train_data['question_text'], train_data['target']\nval_sentences, val_targets = val_data['question_text'], val_data['target']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"targets.value_counts(), val_targets.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for each word - counts how many times it occurs totally in sentences \ndef configure_sentences(sentences, lower = True):\n    words = {}\n    for sentence in sentences:\n        for word in sentence.split():\n            if lower: word = word.lower()\n            words[word] = words.get(word, 0) + 1\n    return words\n\nwords = configure_sentences(sentences)\n# look to frequencies of words in sentences\nDEBUG_DICTIONARY(words)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# find out what kind of words are frequently used\nDEBUG_DICTIONARY({word: cnt for word, cnt in sorted(words.items(), key=lambda item: item[1], reverse=True)})","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# indexing words - in case needed\ndef configure_words(words):\n    vocabulary = {}\n    for i, word in enumerate(words.keys()):\n        vocabulary[word] = i # vocabulary[i] = word\n    return vocabulary\n\nvocabulary = configure_words(words)\nDEBUG_DICTIONARY(vocabulary)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# returns min, avrg and max sentence length - and also displays plot(histogram) for length distribution \ndef configure_sentence_statistic(sentences):\n    def sentence_len(s):\n        return len(s.split())\n    \n    sentences.apply(sentence_len).plot(title='Sentence Length Distribution',y='Length Frequency',kind='hist', colormap='autumn', logy=True);\n    return np.min(sentences.apply(sentence_len)), np.round(np.mean(sentences.apply(sentence_len))), np.max(sentences.apply(sentence_len))\n\nmin, avrg, max = configure_sentence_statistic(sentences)\n\nprint('minimum sentence length {} - average sentence length {} - maximum sentence length {}'.format(min, avrg, max))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"HIDDEN_SIZE = 30","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Average length is 13, but if we take length as hiden dimension about 30-35 it should be better - as we see these lengths(30-35) are in the middle of data.\nif we got 13 - most of sentences would be cut and their 70-80% would be lost...  also the cut parts should be important for the final target, but i think its no need to filter them - as start of 30 word must have enough content to predict final target - if dont, some almost same sentences should have different targets and i think this will help to learn still the right way."},{"metadata":{"trusted":true},"cell_type":"code","source":"import plotly.offline as py\nimport plotly.graph_objs as go\npy.init_notebook_mode(connected=True)\n\n# display how the targets are distributed\ndef configure_target_statistic(targets):\n    trg_cnt = targets.value_counts()\n    labels, sizes = (np.array(trg_cnt.index)), (np.array(100*(trg_cnt/trg_cnt.sum())))\n    py.iplot(go.Figure(data=[go.Pie(labels=labels, values=sizes)], layout=go.Layout(title='Target Distribution',font=dict(size=15),width=500, height=500)))\n    return trg_cnt\n\nconfigure_target_statistic(targets)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"it seems that only 6.2% of targets are 1. so we need some careful model to dont go overboard - overfit ^^"},{"metadata":{"trusted":true},"cell_type":"code","source":"# filters data according to given parameters\ndef filter_and_display_data(sentences, targets, target=0, min_len=5, max_len=30, limit=3):\n    result = []\n    for i, sentence in enumerate(sentences):\n        sent_len = len(sentence.split(' '))\n        if min_len <= sent_len and sent_len <= max_len:\n            if targets[i] == target:\n                result.append(sentence)\n                if len(result) >= limit: break\n    \n    if(len(result) ==- 0):\n        print('no such sequencies found.')\n        return\n    \n    print('{} {} sentences with length between {}-{}:\\n'.format(limit, 'GOOD' if target == 0 else 'BAD', min_len, max_len))\n    for i, s in enumerate(result):\n        print(str(i+1)+\")\",s)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"lets see some examples of our data"},{"metadata":{"trusted":true},"cell_type":"code","source":"filter_and_display_data(sentences, np.asarray(targets, dtype='int'), target=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"filter_and_display_data(sentences, np.asarray(targets, dtype='int'), target=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"for length <= 30 - its seems our data is normally."},{"metadata":{"trusted":true},"cell_type":"code","source":"filter_and_display_data(sentences, np.asarray(targets, dtype='int'), target=0, min_len=120, max_len=140)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"filter_and_display_data(sentences, np.asarray(targets, dtype='int'), target=1, min_len=120, max_len=140)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"for length 120-140 its seems our data is some kind of hard - to determine its target even by human and for target = 1 we dont have any examples - so its bad for train data to dont have all kind of basic examples, but nvm."},{"metadata":{"trusted":true},"cell_type":"code","source":"# reads and returns dictionary - key: word; value: word's embedding vector (vec. length=300)\ndef confnigure_embeddings(embd_path):\n    word2vecs = {}\n    with ZipFile('/kaggle/input/quora-insincere-questions-classification/embeddings.zip') as embd_zip:\n        for embd in embd_zip.open(embd_path, 'r'):\n            word2vec = embd.decode().split(' ')\n            word2vecs[word2vec[0]] = np.asarray(word2vec[1:], dtype='float32')\n    return word2vecs\n            \nword2vecs = confnigure_embeddings('glove.840B.300d/glove.840B.300d.txt')\nDEBUG_DICTIONARY(word2vecs, limit=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# in each sentence replaces words with its own embedding vectors \ndef configure_word2vecs(sentences, word2vecs):\n    def configure_sentence(sentence, len=HIDDEN_SIZE):\n        return ([word2vecs.get(word.lower(), np.zeros(300)) for word in sentence.split()] + [np.zeros(300)]*len)[:len] \n    \n    return [configure_sentence(sentence) for sentence in sentences]\n\n# embedding_sentences = configure_word2vecs(sentences, word2vecs)\n# print(embedding_sentences[:10])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**START WORKING ON MODEL**"},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\n\nimport seaborn as sns\nimport numpy as np\nimport pandas as pd","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 256\nBATCHES = (len(sentences)+BATCH_SIZE-1)//BATCH_SIZE\n\nEPOCHS = 2 # gpu :(\nEMBD_SIZE = 300","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gpu = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\ngpu, torch.cuda.is_available()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# long short-term memory is best suit for this case i think - as we got 1M+ train data and embedding vectors with length 300, \n# if we just convert evrything once in tensors we need more than 16gb ram and much more resources to train this data.\n# also use linear layer should be good enough as there is no hard dependences - as if sentence contains 'bad' word its target is most likly 1.\n# dropout layer would be good also - but as we are using only one lstm layer bc of cpu - we dont...\n\nclass LSTM(nn.Module):\n    def __init__(self, input_dim=1, emb_dim=EMBD_SIZE, hid_dim=HIDDEN_SIZE, n_layers=1, output_dim=1, dropout=0.3):\n        super().__init__()\n        self.hid_dim, self.n_layers = hid_dim, n_layers\n        \n        # nn's\n        self.lstm = nn.LSTM(emb_dim, hid_dim, n_layers, batch_first=True)\n        self.linear = nn.Linear(hid_dim, output_dim)\n        \n        ### NOTE ### for dropout lstm layers has to be more than 1 - but bc of my code works only cpu i got one layer :( so it doesn't works...\n#         self.dropout = nn.Dropout(dropout)\n        \n        \n    def forward(self, src):\n        outputs, (hidden, cell) = self.lstm(src)\n        return self.linear(hidden.reshape(-1, self.hid_dim))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# craete model - with lstm and linear layers\nmodel = LSTM().to(gpu)\n\n# init loss function\nloss_function = nn.BCEWithLogitsLoss().to(gpu) #nn.MSELoss()\n\n# init optimizer with learning rate 0.001\noptimizer = torch.optim.Adam(model.parameters(), lr=0.001)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# evaluates and returns accuracy for predicted Y by model \ndef acc_function(y_pred, y_test):\n    y_pred = torch.round(torch.sigmoid(y_pred).to(gpu)).to(gpu)\n    correct = (y_pred == y_test).sum().float()\n    return torch.round(100*(correct/y_pred.shape[0]))\n\n# generates and returns idx-th batch as torch tensor according to given data(sentences and targets)\ndef get_batch(sentences, targets, idx):\n    src = configure_word2vecs(sentences[BATCH_SIZE*idx:BATCH_SIZE*(idx+1)], word2vecs)\n    trg = np.asarray(targets[BATCH_SIZE*idx:BATCH_SIZE*(idx+1)], dtype='bool')\n    return torch.FloatTensor(src).to(gpu), torch.FloatTensor(trg).to(gpu)\n\n# evaluates and returns f1 score for predicted Y by model \ndef f1_score(y_pred, y_test):\n    tp = (y_test * y_pred).sum().to(torch.float32)\n    tn = ((1 - y_test) * (1 - y_pred)).sum().to(torch.float32)\n    fp = ((1 - y_test) * y_pred).sum().to(torch.float32)\n    fn = (y_test * (1 - y_pred)).sum().to(torch.float32)\n    \n    epsilon = 1e-7 # for avoid crash\n    precision, recall = tp / (tp + fp + epsilon), tp / (tp + fn + epsilon)\n    \n    return 2*(precision*recall)/(precision + recall + epsilon)\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**START TRAINING OF MODEL**"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ready for training\nmodel.train()\n\nVALIDATION_BATCHES = 10\n# init validation data for accuracy while training - but taking only VALIDATION_BATCHES while whole data is too big.\nval_sents = configure_word2vecs(val_sentences[:VALIDATION_BATCHES*BATCH_SIZE], word2vecs)\nval_targs = np.asarray(val_targets[:VALIDATION_BATCHES*BATCH_SIZE], dtype='bool')\n\nval_batch = torch.FloatTensor(val_sents).to(gpu)\nval_target = torch.FloatTensor(val_targs).to(gpu)\nprint(type(val_batch), val_batch.shape, type(val_targets), val_targets.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCHES, BATCH_SIZE, get_batch(sentences, targets, 0)[0].shape, get_batch(sentences, targets, 0)[1].shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# training\nfor e in range(EPOCHS):\n    # save epoch loss and accuracy\n    epoch_loss, epoch_acc = 0, 0\n    for b in range(BATCHES):\n        # get current batch from data\n        X_batch, y_batch = get_batch(sentences, targets, b)\n        \n        # set the gradients to zero, before starting to do backpropragation - avoiding gradient miss direction for minimum. \n        optimizer.zero_grad()\n\n        # predict targets for current batch and learn by comparing it to real targets with loss func.\n        y_pred = model(X_batch)\n        loss = loss_function(y_pred, y_batch.unsqueeze(1))\n        \n        # predict targets for validation data and eval. accuracy\n        val_pred = model(val_batch)\n        acc = acc_function(val_pred, val_target.unsqueeze(1))\n\n        # gradients are \"stored\" by the tensors themselves - once call backward on the loss.\n        loss.backward()\n        \n        # updates the model parameters\n        optimizer.step()\n        \n        # add batch loss and acc to evaluate epoch loss/acc\n        epoch_loss += loss.item()\n        epoch_acc += acc.item()\n        \n        if b == 0 or (b+1) % 100 == 0:\n            print(f'Epoch {(e+1)+0:03} | Batch {(b+1)+0:04}: | Loss: {epoch_loss/(b+1):.5f} | Acc: {epoch_acc/(b+1):.3f} | F1: {f1_score(val_pred, val_target.unsqueeze(1)):.3f}')\n            # print(next(model.parameters()).is_cuda, X_batch.get_device(), y_batch.get_device(), y_pred.get_device(), val_pred.get_device())\n\n    print(f'Epoch {(e+1)+0:03}: | Epoch Loss: {epoch_loss/BATCHES:.5f} | Epoch Acc: {epoch_acc/BATCHES:.3f}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**START WORKING ON TEST DATA**"},{"metadata":{"trusted":true},"cell_type":"code","source":"# init test data\ntest_data = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/test.csv')\nsentences, targets = test_data['question_text'], []\nTEST_BATCHES = (len(sentences)+BATCH_SIZE-1)//BATCH_SIZE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"min, avrg, max = configure_sentence_statistic(sentences)\n\nprint('minimum sentence length {} - average sentence length {} - maximum sentence length {}'.format(min, avrg, max))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(sentences), len(targets), TEST_BATCHES","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**PREDICT TEST DATA ACCORDING TO OUR MODEL**"},{"metadata":{"trusted":true},"cell_type":"code","source":"model.eval()\nwith torch.no_grad():\n    for b in range(TEST_BATCHES):\n        # get current batch\n        X_batch = torch.FloatTensor(configure_word2vecs(sentences[BATCH_SIZE*b:BATCH_SIZE*(b+1)], word2vecs)).to(gpu)\n        \n        # predict batch according to our trained model\n        trg = torch.round(torch.sigmoid(model(X_batch))).cpu().numpy().squeeze()\n        targets.extend(trg)\n        \n        if b == 0 or (b+1) % 100 == 0: print(f'Batch {(b+1)+0:04} predicted')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# save data to submit\ntest_targets = (np.array(targets) >= 0.5).astype(np.int)\n\nsubmit = pd.DataFrame({\"qid\": test_data['qid'], \"prediction\": test_targets})\nsubmit.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# display results\nsubmit.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"As it seems % of our prediction is almost like train data. that seems good ^^ "},{"metadata":{},"cell_type":"markdown","source":"lets see some examples of our prediction"},{"metadata":{"trusted":true},"cell_type":"code","source":"configure_target_statistic(submit['prediction'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"filter_and_display_data(sentences, np.asarray(targets, dtype='int'), target=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"filter_and_display_data(sentences, np.asarray(targets, dtype='int'), target=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"in one look - it seems our model is working well."},{"metadata":{"trusted":true},"cell_type":"code","source":"filter_and_display_data(sentences, np.asarray(targets, dtype='int'), target=1, min_len=100, max_len=150)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"filter_and_display_data(sentences, np.asarray(targets, dtype='int'), target=1, min_len=100, max_len=150)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"if there are no long sequencies in test set - it's just fine."}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}