{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nfrom sklearn.model_selection import train_test_split\nfrom keras.preprocessing import sequence, text\nfrom tensorflow.keras.utils import pad_sequences\nfrom tqdm import tqdm\nimport numpy as np\n","metadata":{"execution":{"iopub.status.busy":"2023-06-15T18:34:11.438736Z","iopub.execute_input":"2023-06-15T18:34:11.439292Z","iopub.status.idle":"2023-06-15T18:34:21.333398Z","shell.execute_reply.started":"2023-06-15T18:34:11.439264Z","shell.execute_reply":"2023-06-15T18:34:21.331466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv')\nvalidation = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')\ntest = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')","metadata":{"execution":{"iopub.status.busy":"2023-06-15T18:34:21.335881Z","iopub.execute_input":"2023-06-15T18:34:21.336646Z","iopub.status.idle":"2023-06-15T18:34:24.675169Z","shell.execute_reply.started":"2023-06-15T18:34:21.336612Z","shell.execute_reply":"2023-06-15T18:34:24.673811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.drop(['severe_toxic','obscene','threat','insult','identity_hate'],axis=1,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-15T18:34:24.676461Z","iopub.execute_input":"2023-06-15T18:34:24.67674Z","iopub.status.idle":"2023-06-15T18:34:24.704676Z","shell.execute_reply.started":"2023-06-15T18:34:24.676717Z","shell.execute_reply":"2023-06-15T18:34:24.7035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train.columns)\ntrain = train.loc[:12000,:]\ntrain.shape","metadata":{"execution":{"iopub.status.busy":"2023-06-15T18:34:24.708321Z","iopub.execute_input":"2023-06-15T18:34:24.708776Z","iopub.status.idle":"2023-06-15T18:34:24.721245Z","shell.execute_reply.started":"2023-06-15T18:34:24.708745Z","shell.execute_reply":"2023-06-15T18:34:24.720127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['comment_text'].apply(lambda x:len(x.split())).max()","metadata":{"execution":{"iopub.status.busy":"2023-06-15T18:34:24.722548Z","iopub.execute_input":"2023-06-15T18:34:24.722871Z","iopub.status.idle":"2023-06-15T18:34:24.780109Z","shell.execute_reply.started":"2023-06-15T18:34:24.722847Z","shell.execute_reply":"2023-06-15T18:34:24.778047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xtrain, xvalid, ytrain, yvalid = train_test_split(train.comment_text.values, train.toxic.values, \n                                                  stratify=train.toxic.values, \n                                                  random_state=42, \n                                                  test_size=0.2, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-15T18:34:24.781579Z","iopub.execute_input":"2023-06-15T18:34:24.781904Z","iopub.status.idle":"2023-06-15T18:34:24.795147Z","shell.execute_reply.started":"2023-06-15T18:34:24.781882Z","shell.execute_reply":"2023-06-15T18:34:24.793499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"token = text.Tokenizer(num_words=None)\ntoken.fit_on_texts(list(xtrain) + list(xvalid))\nxtrain_seq = token.texts_to_sequences(xtrain)\nxtest_seq = token.texts_to_sequences(xvalid)\n\nmax_len = 1500\nxtrain_pad = pad_sequences(xtrain_seq, maxlen=max_len)\nxtest_pad =  pad_sequences(xtest_seq, maxlen=max_len)\nword_index = token.word_index\nprint(len(word_index))","metadata":{"execution":{"iopub.status.busy":"2023-06-15T18:34:24.796307Z","iopub.execute_input":"2023-06-15T18:34:24.797395Z","iopub.status.idle":"2023-06-15T18:34:26.246552Z","shell.execute_reply.started":"2023-06-15T18:34:24.797335Z","shell.execute_reply":"2023-06-15T18:34:26.245498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Xtrain = pd.DataFrame(xtrain_pad)\nXtest = pd.DataFrame(xtest_pad)\n\nYtrain = pd.DataFrame(ytrain)\nYtest = pd.DataFrame(yvalid)\nprint(Xtrain.shape)\nprint(Xtest.shape)\nprint(Ytrain.shape)\nprint(Ytest.shape)","metadata":{"execution":{"iopub.status.busy":"2023-06-15T18:34:26.249327Z","iopub.execute_input":"2023-06-15T18:34:26.24982Z","iopub.status.idle":"2023-06-15T18:34:26.25727Z","shell.execute_reply.started":"2023-06-15T18:34:26.24978Z","shell.execute_reply":"2023-06-15T18:34:26.25578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim","metadata":{"execution":{"iopub.status.busy":"2023-06-15T18:34:26.258822Z","iopub.execute_input":"2023-06-15T18:34:26.259221Z","iopub.status.idle":"2023-06-15T18:34:26.272703Z","shell.execute_reply.started":"2023-06-15T18:34:26.259188Z","shell.execute_reply":"2023-06-15T18:34:26.270656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xtrainT = torch.tensor(Xtrain.values)\nxtestT = torch.tensor(Xtest.values)\nytrainT = torch.tensor(Ytrain.values)\nytestT = torch.tensor(Ytest.values)       ","metadata":{"execution":{"iopub.status.busy":"2023-06-15T18:34:26.277478Z","iopub.execute_input":"2023-06-15T18:34:26.277809Z","iopub.status.idle":"2023-06-15T18:34:26.323682Z","shell.execute_reply.started":"2023-06-15T18:34:26.277783Z","shell.execute_reply":"2023-06-15T18:34:26.322217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embeddings_index = {}\nf = open('/kaggle/input/glove840b300dtxt/glove.840B.300d.txt','r',encoding='utf-8')\nfor line in tqdm(f):\n    values = line.split(' ')\n    word = values[0]\n    coefs = np.asarray([float(val) for val in values[1:]])\n    embeddings_index[word] = coefs\nf.close()\n\nprint('Found %s word vectors.' % len(embeddings_index))","metadata":{"execution":{"iopub.status.busy":"2023-06-15T18:34:26.32565Z","iopub.execute_input":"2023-06-15T18:34:26.326151Z","iopub.status.idle":"2023-06-15T18:37:34.647178Z","shell.execute_reply.started":"2023-06-15T18:34:26.326111Z","shell.execute_reply":"2023-06-15T18:37:34.645895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(embeddings_index['hello']))\nprint(xtrainT.size())","metadata":{"execution":{"iopub.status.busy":"2023-06-15T18:37:34.648678Z","iopub.execute_input":"2023-06-15T18:37:34.649084Z","iopub.status.idle":"2023-06-15T18:37:34.655273Z","shell.execute_reply.started":"2023-06-15T18:37:34.649049Z","shell.execute_reply":"2023-06-15T18:37:34.654105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embedding_matrix = np.zeros((len(word_index) + 1, 300))\nprint(embedding_matrix.shape)\nfor word, i in tqdm(word_index.items()):\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None:\n        embedding_matrix[i] = embedding_vector","metadata":{"execution":{"iopub.status.busy":"2023-06-15T18:37:34.65684Z","iopub.execute_input":"2023-06-15T18:37:34.657341Z","iopub.status.idle":"2023-06-15T18:37:34.803736Z","shell.execute_reply.started":"2023-06-15T18:37:34.657308Z","shell.execute_reply":"2023-06-15T18:37:34.802398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embedding_tensor = torch.tensor(embedding_matrix)\nembedding_tensor.shape[0]","metadata":{"execution":{"iopub.status.busy":"2023-06-15T18:37:34.805316Z","iopub.execute_input":"2023-06-15T18:37:34.805638Z","iopub.status.idle":"2023-06-15T18:37:34.838867Z","shell.execute_reply.started":"2023-06-15T18:37:34.805611Z","shell.execute_reply":"2023-06-15T18:37:34.837834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nclass LSTMmodel(nn.Module):\n    def __init__(self, inp_matrix, max_len):\n        super(LSTMmodel, self).__init__()\n\n        self.embedding = nn.Embedding(inp_matrix.shape[0], inp_matrix.shape[1])\n        self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix))\n        self.embedding.weight.requires_grad = False\n\n        self.lstm = nn.LSTM(embedding_matrix.shape[1], 100, dropout=0.3, bidirectional=False)\n        self.dense = nn.Linear(100, 1)\n        self.sigmoid = nn.Sigmoid()\n\n    def forward(self, x):\n        embedded = self.embedding(x)\n        lstm_output, _ = self.lstm(embedded)\n        lstm_output = lstm_output[:, -1, :]  # Extract the last output of LSTM for classification\n        dense_output = self.dense(lstm_output)\n        output = self.sigmoid(dense_output)\n        return output\n\n\n","metadata":{"execution":{"iopub.status.busy":"2023-06-15T18:37:34.840247Z","iopub.execute_input":"2023-06-15T18:37:34.840771Z","iopub.status.idle":"2023-06-15T18:37:34.849155Z","shell.execute_reply.started":"2023-06-15T18:37:34.840743Z","shell.execute_reply":"2023-06-15T18:37:34.847354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nlearning_rate = 0.001\nnum_epochs = 10\nbatch_size = 32\nmodel = LSTMmodel(embedding_matrix, 0)\nprint(model)","metadata":{"execution":{"iopub.status.busy":"2023-06-15T18:37:34.85038Z","iopub.execute_input":"2023-06-15T18:37:34.850757Z","iopub.status.idle":"2023-06-15T18:37:34.989263Z","shell.execute_reply.started":"2023-06-15T18:37:34.85073Z","shell.execute_reply":"2023-06-15T18:37:34.988092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xtrainT = torch.tensor(Xtrain.values)\nxtestT = torch.tensor(Xtest.values)\nytrainT = torch.tensor(Ytrain.values)\nytestT = torch.tensor(Ytest.values)                                                            ","metadata":{"execution":{"iopub.status.busy":"2023-06-15T18:39:59.202075Z","iopub.execute_input":"2023-06-15T18:39:59.203053Z","iopub.status.idle":"2023-06-15T18:39:59.736342Z","shell.execute_reply.started":"2023-06-15T18:39:59.203015Z","shell.execute_reply":"2023-06-15T18:39:59.734995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(model(xtrainT))","metadata":{"execution":{"iopub.status.busy":"2023-06-15T18:37:35.017894Z","iopub.execute_input":"2023-06-15T18:37:35.018141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## define loss function and optimiser\ncriterion = nn.BCELoss()\noptimizer = optim.Adam(model.parameters(), lr=learning_rate)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Training loop\n# for epoch in range(num_epochs):\n#     model.train()\n#     running_loss = 0.0\n#     for i in range(0, len(xtrainT), batch_size):\n#         # Get the batch of training data\n#         inputs = xtrainT[i:i+batch_size]\n#         labels = ytrainT[i:i+batch_size]\n\n#         # Zero the gradients\n#         optimizer.zero_grad()\n\n#         # Forward pass\n#         outputs = model(inputs)\n# #         loss = criterion(outputs, labels.float())\n        \n# #         loss.backward()\n# #         optimizer.step()\n\n# #         running_loss += loss.item()\n        \n\n# #     # Compute average training loss for the epoch\n# #     avg_loss = running_loss / (len(xtrainT) / batch_size)\n# #     print(avg_loss)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\n\nclass MyModel(nn.Module):\n    def __init__(self, vocab_size, embedding_dim):\n        super(MyModel, self).__init__()\n        self.embedding = nn.Embedding(vocab_size, embedding_dim)\n    \n    def forward(self, x):\n        embedded = self.embedding(x)\n        return embedded\n\n# Create an instance of the model\nvocab_size = 1000\nembedding_dim = 100\nmodel = MyModel(vocab_size, embedding_dim)\n\n# Generate sample input tensor\nbatch_size = 4\nsequence_length = 10\ninput_tensor = torch.randint(low=0, high=vocab_size, size=(batch_size, sequence_length))\n\n# Pass input through the model\noutput_tensor = model(input_tensor)\n\nprint(\"Input tensor shape:\", input_tensor.shape)\nprint(\"Output tensor shape:\", output_tensor.shape)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}