{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.utils as utils\nimport torch.nn.init as init\nfrom torch.autograd import Variable\nimport torchvision.utils as v_utils\nimport torchvision.datasets as dset\nimport torchvision.transforms as transforms\nfrom torch.nn.utils.rnn import pad_sequence\nfrom tqdm import tqdm_notebook\nfrom sklearn.metrics import roc_curve, auc, precision_recall_curve, average_precision_score\nfrom IPython.display import clear_output\nfrom scipy.special import expit\n\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom collections import OrderedDict\nimport torch\nimport torch.nn as nn\nimport torch.utils as utils\nimport torch.nn.init as init\nfrom torch.autograd import Variable\nimport torchvision.utils as v_utils\nimport torchvision.datasets as dset\nimport torchvision.transforms as transforms\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom collections import OrderedDict\nimport pandas as pd\nimport numpy as np\nimport nltk\nfrom nltk.tokenize import RegexpTokenizer\nfrom nltk.corpus import stopwords\nfrom nltk import word_tokenize\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import f1_score\nfrom sklearn.utils import shuffle\nfrom gensim import models\n\nimport re\nfrom collections import Counter\nimport gensim\nimport heapq\nfrom operator import itemgetter\nfrom multiprocessing import Pool","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#data = pd.read_csv('../input/quora-insincere-questions-classification/train.csv')\n#data.head()\ntrain = pd.read_csv('../input/kernel7d1c9fd560/processed_train.csv')\ntest = pd.read_csv('../input/kernel7d1c9fd560/processed_test.csv')\ntrain = train.fillna('nan')\ntest = test.fillna('nan')\ndisplay(train.head())\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.rename(columns={'no_misspels' : 'no_misspells'}, inplace=True)\ntest.rename(columns={'no_misspels' : 'no_misspells'}, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MAX_THRESHOLD = 0.21\ntrain['len'] = train['basic'].str.split().apply(len)\nX, y = train['basic'].str.split().to_numpy(), train['target'].to_numpy()\n\nX_train, X_test, y_train, y_test = train_test_split(\n    X, y, test_size=0.1, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"word2vec_path = '../input/quora-insincere-questions-classification/embeddings/\\\nGoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'\nword2vec = models.KeyedVectors.load_word2vec_format(word2vec_path, binary=True)\nwords = word2vec.index2word","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(train['len'], bins=100);\nlens = np.array(train['len'])\nnp.quantile(lens, 0.99)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#https://discuss.pytorch.org/t/vanishing-gradients/46824/5\ndef plot_grad_flow(named_parameters, title):\n    '''Plots the gradients flowing through different layers in the net during training.\n    Can be used for checking for possible gradient vanishing / exploding problems.\n    \n    Usage: Plug this function in Trainer class after loss.backwards() as \n    \"plot_grad_flow(self.model.named_parameters())\" to visualize the gradient flow'''\n    ave_grads = []\n    max_grads= []\n    layers = []\n    for n, p in named_parameters:\n        if(p.requires_grad) and (\"bias\" not in n):\n            try:\n                layers.append(n)\n                ave_grads.append(p.grad.abs().mean())\n                max_grads.append(p.grad.abs().max())\n                \n            except:\n                print(n)\n    plt.bar(np.arange(len(max_grads)), max_grads, alpha=0.1, lw=1, color=\"c\")\n    plt.bar(np.arange(len(max_grads)), ave_grads, alpha=0.1, lw=1, color=\"b\")\n    plt.hlines(0, 0, len(ave_grads)+1, lw=2, color=\"k\" )\n    plt.xticks(range(0,len(ave_grads), 1), layers, rotation=\"vertical\")\n    plt.xlim(left=0, right=len(ave_grads))\n    plt.ylim(bottom = -0.001, top=0.02) # zoom in on the lower gradient regions\n    plt.xlabel(\"Layers\")\n    plt.ylabel(\"average gradient\")\n    plt.title(title)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_batch(data):\n    dim=300\n    vectorized = [torch.Tensor([word2vec[word] if word in word2vec else np.random.rand(dim) for word in sen])\n                  for sen in data]\n    batch = torch.Tensor(pad_sequence(vectorized, batch_first=True))\n    return batch","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Discriminator receives 1x28x28 image and returns a float number\n# we can name each layer using OrderedDict\n\nclass CNN(nn.Module):\n    def __init__(self, dim=300):\n        super(CNN,self).__init__()\n        self.global_pool_1 = nn.AdaptiveMaxPool2d((1, 1))\n        self.global_pool_2 = nn.AdaptiveMaxPool2d((2, 1))\n        self.conv1 = nn.Sequential(\n                     nn.Conv2d(1, 128, kernel_size=(1, 300)), \n                     nn.BatchNorm2d(128),\n                     nn.LeakyReLU()\n        )\n        self.conv2 = nn.Sequential(\n                     nn.Conv2d(1, 128, kernel_size=(2, 300), padding=(1, 0)), \n                     nn.BatchNorm2d(128),\n                     nn.LeakyReLU()\n        )\n        self.conv2_dilation = nn.Sequential(\n                     nn.Conv2d(1, 128, kernel_size=(2, 300), padding=(1, 0), dilation=(2, 1)), \n                     nn.BatchNorm2d(128),\n                     nn.LeakyReLU()\n        )\n        self.conv3 = nn.Sequential(\n                     nn.Conv2d(1, 128, kernel_size=(3, 300), padding=(1, 0)), \n                     nn.BatchNorm2d(128),\n                     nn.LeakyReLU()\n        )\n        self.conv4 = nn.Sequential(\n                     nn.Conv2d(1, 128, kernel_size=(4, 300), padding=(2, 0)), \n                     nn.BatchNorm2d(128),\n                     nn.LeakyReLU()\n        )\n        self.conv5 = nn.Sequential(\n                        nn.Conv2d(1, 128, kernel_size=(5, 300), padding=(2, 0)),\n                        nn.BatchNorm2d(128),\n                        nn.LeakyReLU()\n        )\n        \n        self.bottleneck = nn.Sequential(\n                        nn.Conv2d(128, 64, kernel_size=1),\n                        nn.BatchNorm2d(64),\n                        nn.LeakyReLU()\n        )\n        self.fc1 = nn.Sequential(\n                        nn.Linear(512, 128),\n                        nn.BatchNorm1d(128),\n                        nn.Dropout(0.5),\n                        nn.LeakyReLU()\n        )\n        \n        self.fc2 = nn.Sequential(\n                        nn.Linear(128, 32),\n                        nn.BatchNorm1d(32),\n                        nn.Dropout(0.5),\n                        nn.LeakyReLU()\n        )\n        self.fc3 = nn.Sequential(\n                        nn.Linear(32, 1)\n        )\n        \n\n\n    def forward(self,x):\n        conv1 = self.global_pool_2(self.conv1(x)) #batch x 64 x 2\n        conv2 = self.global_pool_2(self.conv2(x)) #batch x 64  x 2\n        conv2_dilation = self.global_pool_1(self.conv2_dilation(x)) #batch x 64 x 1\n        conv3 = self.global_pool_1(self.conv3(x)) #batch x 64 x 1\n        conv4 = self.global_pool_1(self.conv4(x)) #batch x 64 x 1\n        conv5 = self.global_pool_1(self.conv5(x)) #batch x 64 x 1\n        concatenated = torch.cat((conv1, conv2, conv2_dilation, conv3, conv4, conv5), 2) #batch x 64 x 8\n        res = self.bottleneck(concatenated) #batch x 64 x 8\n        res = res.view(res.shape[0], -1) #batch x 256\n        res = self.fc1(res)\n        res = self.fc2(res)\n        res = self.fc3(res)\n        return res","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cnn = CNN().to(device)\ncnn.load_state_dict(torch.load('/kaggle/input/quora-models/model2.ckpt'))\nopt = torch.optim.Adam(cnn.parameters(), lr=0.1)\ncriterion = nn.BCEWithLogitsLoss()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_n_test(epochs, data_train, target_train, data_test, target_test, batch_size=128):\n    batch_num = len(data_train) // batch_size\n    for epoch in tqdm_notebook(range(epochs)):\n        cnn.train()\n        losses = []\n        data_train, target_train = shuffle(data_train, target_train)\n        for i in tqdm_notebook(range(batch_num)):\n            opt.zero_grad()\n            batch = make_batch(data_train[i * batch_size : (i + 1) * batch_size]).to(device)\n            s = batch.shape\n            batch = batch.view(s[0], 1, s[1], s[2])\n            batch_y = torch.Tensor(target_train[i * batch_size : (i + 1) * batch_size]).to(device)\n            predict = cnn(batch).view(-1)\n            loss = criterion(predict, batch_y)\n            losses.append(loss)\n            loss.backward()\n            opt.step()\n            #plot_grad_flow(cnn.named_parameters(), 'cnn')\n            if i % 1000 == 0:\n                print(\"Epoch: {}, i: {}, loss: {}\".format(epoch, i, loss))\n        cnn.eval()\n        data_test, target_test = shuffle(data_test, target_test)\n        batch_size_test = 1024\n        batch_test_num = len(data_test) // batch_size_test\n        predictions = np.array([])\n        for i in tqdm_notebook(range(batch_test_num)):\n            batch_test = make_batch(data_test[i * batch_size_test : (i + 1) * batch_size_test]).to(device)\n            s = batch_test.shape\n            batch_test = batch_test.view(s[0], 1, s[1], s[2])\n            predict = cnn(batch_test).view(-1)\n            if predictions.size == 0:\n                predictions = predict.cpu().detach().numpy()\n            else:\n                predictions = np.hstack((predictions, predict.view(predict.shape[0]).cpu().detach().numpy()))\n        average_precision = average_precision_score(target_test[: len(predictions)], predictions)\n        plt.hist(expit(predictions[target_test[: len(predictions)] == 0]), alpha=0.2, color='c')\n        plt.hist(expit(predictions[target_test[: len(predictions)] != 0]), alpha=0.2, color='b')\n        plt.show()\n        f1 = f1_score(target_test[: len(predictions)], (expit(predictions) >= MAX_THRESHOLD).astype(int))\n        print(\"Epoch num: {}, average_precision: {}, f1: {}\".format(epoch, average_precision, f1), flush=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size_test = 64\nbatch_test_num = len(X_test) // batch_size_test\npredictions = np.array([])\nfor i in tqdm_notebook(range(batch_test_num)):\n    batch_test = make_batch(X_test[i * batch_size_test : (i + 1) * batch_size_test]).to(device)\n    s = batch_test.shape\n    batch_test = batch_test.view(s[0], 1, s[1], s[2])\n    predict = cnn(batch_test).view(-1)\n    if predictions.size == 0:\n        predictions = predict.cpu().detach().numpy()\n    else:\n        predictions = np.hstack((predictions, predict.view(predict.shape[0]).cpu().detach().numpy()))\naverage_precision = average_precision_score(y_test[: len(predictions)], predictions)\nthresholds = np.linspace(0, 1, 100)\nmax_f1, max_threshold = 0, 0 \nfor threshold in thresholds:\n    f1 = f1_score(y_test[: len(predictions)], (expit(predictions) >= threshold).astype(int))\n    if f1 > max_f1:\n        max_f1, max_threshold = f1, threshold\nprint(max_f1, max_threshold)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_real_test = test['basic'].str.split().to_numpy()\ntest_size, batch_size = len(X_real_test), 256\nbatch_size_num = test_size // batch_size","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = np.array([])\nfor i in tqdm_notebook(range(batch_size_num)):\n    batch_test = make_batch(X_real_test[i * batch_size : (i + 1) * batch_size]).to(device)\n    s = batch_test.shape\n    batch_test = batch_test.view(s[0], 1, s[1], s[2])\n    predict = cnn(batch_test).view(-1)\n    if predictions.size == 0:\n        predictions = predict.cpu().detach().numpy()\n    else:\n        predictions = np.hstack((predictions, predict.view(predict.shape[0]).cpu().detach().numpy()))\nbatch_test = make_batch(X_real_test[batch_size_num * batch_size : ]).to(device)\ns = batch_test.shape\nbatch_test = batch_test.view(s[0], 1, s[1], s[2])\npredict = cnn(batch_test).view(-1)\npredictions = np.hstack((predictions, predict.view(predict.shape[0]).cpu().detach().numpy()))\nres = expit(predictions) >= threshold","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame({'qid' : test['qid'], 'prediction' : res})\nprint(len(submission.index))\nsubmission.to_csv('submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\ntrain_size = len(X_train)\ntrain_n_test(1, X_train[: train_size // 2], y_train[: train_size // 2], X_test, y_test)\ntorch.cuda.empty_cache()\ntrain_n_test(1, X_train[train_size // 2 :], y_train[train_size // 2 :], X_test, y_test)\ntorch.save(cnn.state_dict(), 'model3.ckpt')\n''';","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}