{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport torch\nimport torch.nn as nn\nimport time\nfrom torch.utils.data import TensorDataset, DataLoader\nfrom tqdm import tqdm\n#import keras\nfrom keras.preprocessing import sequence, text\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score\nimport os\nfrom numba import cuda\nimport sys\nfrom numpy import save, load\nfrom random import sample","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\")\nvalid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')\n#test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')\nembedding_matrix = load('/kaggle/input/jigsaw-gru/embedding_matrix.npy')\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Split data into training and validation","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train.drop(['severe_toxic','obscene','threat','insult','identity_hate'],axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xtrain, xvalid, ytrain, yvalid = train_test_split(train.comment_text.values, train.toxic.values, \n                                                  stratify = train.toxic.values, \n                                                  random_state = 42, \n                                                  test_size = 0.1, shuffle = True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Tokenize input data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# using keras tokenizer here\ntoken = text.Tokenizer(num_words = None)\nmax_len = 1500\n\ntoken.fit_on_texts(list(xtrain) + list(xvalid))\nxtrain_seq = token.texts_to_sequences(xtrain)\nxvalid_seq = token.texts_to_sequences(xvalid)\n\n#zero pad the sequences\nxtrain_pad = sequence.pad_sequences(xtrain_seq, maxlen = max_len)\nxvalid_pad = sequence.pad_sequences(xvalid_seq, maxlen = max_len)\n\nword_index = token.word_index\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#save('/kaggle/working/xtrain_pad.npy', xtrain_pad)\n#save('/kaggle/working/xvalid_pad.npy', xvalid_pad)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 32\ntrain_data = TensorDataset(torch.LongTensor(xtrain_pad), torch.tensor(ytrain, dtype=torch.int8))\ntrain_loader = DataLoader(train_data, shuffle = True, batch_size = batch_size, drop_last = True)\n#valid_data = TensorDataset(torch.LongTensor(xvalid_pad), torch.from_numpy(yvalid))\n#valid_loader = DataLoader(valid_data, shuffle = False, batch_size = batch_size, drop_last = True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Pre-trained Glove embedding","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# %%time\n# embeddings_index = {}\n# f = open('/kaggle/input/glove840b300dtxt/glove.840B.300d.txt','r',encoding='utf-8')\n# for line in tqdm(f):\n#     values = line.split(' ')\n#     word = values[0]\n#     coefs = np.asarray([float(val) for val in values[1:]])\n#     embeddings_index[word] = coefs\n# f.close()\n\n# print('Found %s word vectors.' % len(embeddings_index))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create an embedding matrix for the words we have in the dataset\n# embedding_matrix = np.zeros((len(word_index) + 1, 300))\n# for word, i in tqdm(word_index.items()):\n#     embedding_vector = embeddings_index.get(word)\n#     if embedding_vector is not None:\n#         embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#save('/kaggle/working/embedding_matrix.npy', embedding_matrix)\n#embedding_matrix,'jigsaw/embedding_matrix.csv')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embedding_matrix = embedding_matrix.astype('float32')\n#del embeddings_index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# local_vars = list(locals().items())\n# var_mem = []\n# for var, obj in local_vars:\n#     tmp = round(sys.getsizeof(obj)/1024**2,2)\n#     if tmp > 0:\n#         var_mem.append((var, tmp))\n    \n# var_mem = sorted(var_mem, key = lambda x: x[1], reverse= True)\n# var_mem","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Checking if GPU is available","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# torch.cuda.is_available() checks and returns a Boolean True if a GPU is available, else it'll return False\nis_cuda = torch.cuda.is_available()\n\n# If we have a GPU available, we'll set our device to GPU. We'll use this device variable later in our code.\nif is_cuda:\n    device = torch.device(\"cuda\")\nelse:\n    device = torch.device(\"cpu\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#device = torch.device(\"cpu\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# RNN model with GRU layer","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class GRUNet(nn.Module):\n    def __init__(self, embd_mat, hidden_size = 300, trainable = False):\n        super(GRUNet, self).__init__()\n        embd_num, embd_dim = embd_mat.shape\n        self.embd = nn.Embedding(embd_num, embd_dim)\n        self.embd.load_state_dict({'weight': torch.tensor(embd_mat)})        \n        self.embd.weight.requires_grad = trainable\n        self.embd_dropout = nn.Dropout2d(0.3)\n        self.hidden_size = hidden_size\n        self.gru = nn.GRU(embd_dim, hidden_size, batch_first = True)\n        self.fc = nn.Linear(hidden_size, 1)\n        self.sigmoid = nn.Sigmoid()\n        \n    def forward(self, inputs, h0):\n        embeddings = self.embd_dropout(self.embd(inputs))\n        out, h = self.gru(embeddings, h0)\n        out = self.sigmoid(self.fc(out[:, -1]))\n        return out, h\n    \n    def init_hidden(self, batch_size):\n        weight = next(self.parameters()).data\n        hidden = weight.new(1, batch_size, self.hidden_size).zero_().to(device)\n        return hidden\n    \n    def predict(self, inputs):\n        out, h = self.forward(inputs, h0 = self.init_hidden(len(inputs)))\n        return out   \n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model(train_loader, learn_rate = 0.01, EPOCHS = 5):\n    \n    # Setting common hyperparameters\n    #input_dim = next(iter(train_loader))[0].shape[1]\n    #output_dim = 1\n    \n    # Instantiating the models\n    model = GRUNet(embedding_matrix)\n    model.to(device)\n        \n    # Defining loss function and optimizer\n    criterion = nn.BCELoss()\n    #criterion = nn.BCEWithLogitsLoss()\n    optimizer = torch.optim.Adam(model.parameters(), lr = learn_rate)\n    \n    model.train()\n    print(\"Starting Training of {} model\".format('GRU'))\n    epoch_times = []\n    # Start training loop\n    for epoch in range(1, EPOCHS + 1):\n        start_time = time.time()\n        h = model.init_hidden(batch_size)\n        avg_loss = 0.\n        counter = 0\n        avg_accuracy = 0.\n        for x, label in train_loader:\n            counter += 1\n            h = h.data\n            model.zero_grad()\n            \n            out, h = model(x.to(device), h)\n            #out, h = model(x.to(device))\n            loss = criterion(out[:,-1], label.to(device).float())\n            loss.backward()\n            optimizer.step()\n            avg_loss += loss.item()                    \n            if counter % 200 == 0:\n                print(\"Epoch {}......Step: {}/{}....... Average Loss for Epoch: {}\"\\\n                      .format(epoch, counter, len(train_loader), \\\n                              round(avg_loss/counter, 2)))\n        current_time = time.time()\n        print(\"Epoch {}/{} Done, Total Loss: {}\"\\\n              .format(epoch, EPOCHS, round(avg_loss/len(train_loader),2)))\n        print(\"Total Time Elapsed: {} seconds\".format(str(round(current_time - start_time,2))))\n        epoch_times.append(round(current_time - start_time,2))\n    print(\"Total Training Time: {} seconds\".format(str(round(sum(epoch_times),2))))\n    return model, optimizer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def score(model, test_x, test_y):\n    pred = model.predict(torch.LongTensor(test_x).to(device).long())\n    return round(roc_auc_score(torch.tensor(test_y).cpu().detach().numpy(),pred.cpu().detach().numpy()), 2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_gru, optimizer = train_model(train_loader)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"index = sample(range(len(xvalid_pad)),500)\nscore(model_gru, xvalid_pad[index], yvalid[index])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"checkpoint = {'model': model_gru,\n              'state_dict': model_gru.state_dict(),\n              'optimizer' : optimizer.state_dict()}\n\ntorch.save(checkpoint, '/kaggle/working/checkpoint.pth')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def load_checkpoint(filepath):\n#     checkpoint = torch.load(filepath)\n#     model = checkpoint['model']\n#     model.load_state_dict(checkpoint['state_dict'])\n#     for parameter in model.parameters():\n#         parameter.requires_grad = False\n\n#     model.eval()\n#     return model\n\n# model = load_checkpoint('/kaggle/working/checkpoint.pth')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#del model_gru\n#import gc\n#torch.cuda.empty_cache()\n#del model\n#score(model, xvalid_pad[index], yvalid[index])\n#%time model_gru.save('kaggle/working/model_gru.h5')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}