{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from torch.utils.data import Dataset,DataLoader\nfrom sklearn.model_selection import train_test_split\nfrom gensim.models import KeyedVectors\nfrom keras.preprocessing.sequence import pad_sequences\nfrom tqdm import tqdm\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport pandas as pd\nimport numpy as np\nimport re","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-30T06:21:07.099944Z","iopub.execute_input":"2022-01-30T06:21:07.100429Z","iopub.status.idle":"2022-01-30T06:21:15.176354Z","shell.execute_reply.started":"2022-01-30T06:21:07.100278Z","shell.execute_reply":"2022-01-30T06:21:15.175359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define LSTM-based model","metadata":{}},{"cell_type":"code","source":"class bi_lstm(nn.Module):\n    def __init__(self, input_size, hidden_size,weight_mattrix):\n        super(bi_lstm, self).__init__()\n        self.input_size = input_size\n        self.hidden_size = hidden_size\n        self.embed = nn.Embedding.from_pretrained(weight_mattrix)\n        self.bilstm = nn.LSTM(input_size, hidden_size, batch_first=True, bidirectional=True)\n        self.conv1d = nn.Conv1d(256, 64, 3)\n        self.max_pool_1d = nn.AdaptiveMaxPool1d(1)\n        self.fc1 = nn.Linear(64, 128)\n        self.drop = nn.Dropout(0.2)\n        self.fc2 = nn.Linear(128, 64)\n        self.fc3 = nn.Linear(64,2)\n    def forward(self, x):\n        batch, seq_len = x.size()\n        out = self.embed(x)\n        h0 = torch.zeros(2, batch, self.hidden_size,dtype=torch.double).to(device)\n        c0 = torch.zeros(2, batch, self.hidden_size,dtype=torch.double).to(device)\n        out1, _ = self.bilstm(out.float(), (h0.float(),c0.float()))\n        batch,lin,cin = out1.size()\n        out1 = out1.reshape(batch,cin,lin)\n        out2 = self.conv1d(out1)\n        out3 = self.max_pool_1d(out2)\n        out3 = out3.reshape(batch,-1)\n        out4 = self.fc1(out3)\n        out4 = self.drop(out4)\n        out5 = self.fc2(out4)\n        out6 = self.fc3(out5)\n        return out6","metadata":{"execution":{"iopub.status.busy":"2022-01-30T06:21:15.178327Z","iopub.execute_input":"2022-01-30T06:21:15.178678Z","iopub.status.idle":"2022-01-30T06:21:15.192085Z","shell.execute_reply.started":"2022-01-30T06:21:15.178629Z","shell.execute_reply":"2022-01-30T06:21:15.19111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define Question Dataset Class","metadata":{}},{"cell_type":"code","source":"class QuestionDataset(Dataset):\n    def __init__(self,df,encode_mattrix,test=False):\n        super(QuestionDataset, self).__init__()\n        self.df = df\n        self.encode = encode_mattrix\n        self.is_test = test\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, index):\n        if self.is_test == False:\n            question_text = self.encode[index]\n            target = self.df['target'].iloc[index]\n            return {'question_text': question_text, 'target': target}\n        else:\n            qid = self.df['qid'].iloc[index]\n            question_text = self.encode[index]\n            return {'question_text': question_text,'qid':qid}","metadata":{"execution":{"iopub.status.busy":"2022-01-30T06:21:15.193437Z","iopub.execute_input":"2022-01-30T06:21:15.194141Z","iopub.status.idle":"2022-01-30T06:21:15.205786Z","shell.execute_reply.started":"2022-01-30T06:21:15.194092Z","shell.execute_reply":"2022-01-30T06:21:15.204729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define EarlyStoping module","metadata":{}},{"cell_type":"code","source":"class EarlyStopping():\n    def __init__(self, patience, min_delta=0):\n        self.patience = patience\n        self.min_delta = min_delta\n        self.counter = 0\n        self.best_loss = None\n        self.early_stop = False\n    def __call__(self, val_loss):\n        if self.best_loss == None:\n            self.best_loss = val_loss\n        elif self.best_loss - val_loss > self.min_delta:\n            self.best_loss = val_loss\n            # reset counter if validation loss improves\n            self.counter = 0\n        elif self.best_loss - val_loss < self.min_delta:\n            self.counter += 1\n            print(f\"INFO: Early stopping counter {self.counter} of {self.patience}\")\n            if self.counter >= self.patience:\n                print('INFO: Early stopping')\n                self.early_stop = True","metadata":{"execution":{"iopub.status.busy":"2022-01-30T06:21:15.208819Z","iopub.execute_input":"2022-01-30T06:21:15.209572Z","iopub.status.idle":"2022-01-30T06:21:15.221816Z","shell.execute_reply.started":"2022-01-30T06:21:15.209525Z","shell.execute_reply":"2022-01-30T06:21:15.220741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train and valid function","metadata":{}},{"cell_type":"code","source":"#Train function\ndef train_model(dataloader,dataset,optimizer,device,model,loss_function):\n    print('Training')\n    model.train()\n    train_loss=0.0\n    train_acc = 0.0\n    counter = 0\n    total = 0\n    for batch,data in tqdm(enumerate(dataloader),total=int(len(dataset)/dataloader.batch_size)):\n        counter += 1\n        question = data['question_text']\n        question = question.to(device)\n        target = data['target'].to(device)\n        total += target.size(0)\n        result = model(question)\n        loss = loss_function(result,target)\n        train_loss += loss.item()\n        _, preds = result.max(1)\n        train_acc += (preds == target).sum().item()\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n    return train_loss/counter,100. * train_acc/total","metadata":{"execution":{"iopub.status.busy":"2022-01-30T06:21:15.223573Z","iopub.execute_input":"2022-01-30T06:21:15.2243Z","iopub.status.idle":"2022-01-30T06:21:15.237246Z","shell.execute_reply.started":"2022-01-30T06:21:15.224253Z","shell.execute_reply":"2022-01-30T06:21:15.236176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#valid function\ndef valid_model(dataloader,dataset,model,device,loss_function):\n    print('Validating')\n    model.eval()\n    val_loss = 0.0\n    val_acc = 0.0\n    counter=0\n    total=0\n    with torch.no_grad():\n        for batch,data in tqdm(enumerate(dataloader),total=int(len(dataset)/dataloader.batch_size)):\n            counter += 1\n            question = data['question_text'].to(device)\n            target = data['target'].to(device)\n            total += target.size(0)\n            result = model(question)\n            _,preds = result.max(1)\n            val_acc += (preds == target).sum().item()\n            loss = loss_function(result,target)\n            val_loss += loss.item()\n        return val_loss/counter, 100. * val_acc/total","metadata":{"execution":{"iopub.status.busy":"2022-01-30T06:21:15.239065Z","iopub.execute_input":"2022-01-30T06:21:15.239618Z","iopub.status.idle":"2022-01-30T06:21:15.254349Z","shell.execute_reply.started":"2022-01-30T06:21:15.239557Z","shell.execute_reply":"2022-01-30T06:21:15.253031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define contractions and their normal words","metadata":{}},{"cell_type":"code","source":"\ncontractions={\"I'm\": 'I am',\n \"I'm'a\": 'I am about to',\n \"I'm'o\": 'I am going to',\n \"I've\": 'I have',\n \"I'll\": 'I will',\n \"I'll've\": 'I will have',\n \"I'd\": 'I would',\n \"I'd've\": 'I would have',\n 'Whatcha': 'What are you',\n \"amn't\": 'am not',\n \"ain't\": 'are not',\n \"aren't\": 'are not',\n \"'cause\": 'because',\n \"can't\": 'can not',\n \"can't've\": 'can not have',\n \"could've\": 'could have',\n \"couldn't\": 'could not',\n \"couldn't've\": 'could not have',\n \"daren't\": 'dare not',\n \"daresn't\": 'dare not',\n \"dasn't\": 'dare not',\n \"didn't\": 'did not',\n 'didn’t': 'did not',\n \"don't\": 'do not',\n 'don’t': 'do not',\n \"doesn't\": 'does not',\n \"e'er\": 'ever',\n \"everyone's\": 'everyone is',\n 'finna': 'fixing to',\n 'gimme': 'give me',\n \"gon't\": 'go not',\n 'gonna': 'going to',\n 'gotta': 'got to',\n \"hadn't\": 'had not',\n \"hadn't've\": 'had not have',\n \"hasn't\": 'has not',\n \"haven't\": 'have not',\n \"he've\": 'he have',\n \"he's\": 'he is',\n \"he'll\": 'he will',\n \"he'll've\": 'he will have',\n \"he'd\": 'he would',\n \"he'd've\": 'he would have',\n \"here's\": 'here is',\n \"how're\": 'how are',\n \"how'd\": 'how did',\n \"how'd'y\": 'how do you',\n \"how's\": 'how is',\n \"how'll\": 'how will',\n \"isn't\": 'is not',\n \"it's\": 'it is',\n \"'tis\": 'it is',\n \"'twas\": 'it was',\n \"it'll\": 'it will',\n \"it'll've\": 'it will have',\n \"it'd\": 'it would',\n \"it'd've\": 'it would have',\n 'kinda': 'kind of',\n \"let's\": 'let us',\n 'luv': 'love',\n \"ma'am\": 'madam',\n \"may've\": 'may have',\n \"mayn't\": 'may not',\n \"might've\": 'might have',\n \"mightn't\": 'might not',\n \"mightn't've\": 'might not have',\n \"must've\": 'must have',\n \"mustn't\": 'must not',\n \"mustn't've\": 'must not have',\n \"needn't\": 'need not',\n \"needn't've\": 'need not have',\n \"ne'er\": 'never',\n \"o'\": 'of',\n \"o'clock\": 'of the clock',\n \"ol'\": 'old',\n \"oughtn't\": 'ought not',\n \"oughtn't've\": 'ought not have',\n \"o'er\": 'over',\n \"shan't\": 'shall not',\n \"sha'n't\": 'shall not',\n \"shalln't\": 'shall not',\n \"shan't've\": 'shall not have',\n \"she's\": 'she is',\n \"she'll\": 'she will',\n \"she'd\": 'she would',\n \"she'd've\": 'she would have',\n \"should've\": 'should have',\n \"shouldn't\": 'should not',\n \"shouldn't've\": 'should not have',\n \"so've\": 'so have',\n \"so's\": 'so is',\n \"somebody's\": 'somebody is',\n \"someone's\": 'someone is',\n \"something's\": 'something is',\n 'sux': 'sucks',\n \"that're\": 'that are',\n \"that's\": 'that is',\n \"that'll\": 'that will',\n \"that'd\": 'that would',\n \"that'd've\": 'that would have',\n 'em': 'them',\n \"there're\": 'there are',\n \"there's\": 'there is',\n \"there'll\": 'there will',\n \"there'd\": 'there would',\n \"there'd've\": 'there would have',\n \"these're\": 'these are',\n \"they're\": 'they are',\n \"they've\": 'they have',\n \"they'll\": 'they will',\n \"they'll've\": 'they will have',\n \"they'd\": 'they would',\n \"they'd've\": 'they would have',\n \"this's\": 'this is',\n \"those're\": 'those are',\n \"to've\": 'to have',\n 'wanna': 'want to',\n \"wasn't\": 'was not',\n \"we're\": 'we are',\n \"we've\": 'we have',\n \"we'll\": 'we will',\n \"we'll've\": 'we will have',\n \"we'd\": 'we would',\n \"we'd've\": 'we would have',\n \"weren't\": 'were not',\n \"what're\": 'what are',\n \"what'd\": 'what did',\n \"what've\": 'what have',\n \"what's\": 'what is',\n \"what'll\": 'what will',\n \"what'll've\": 'what will have',\n \"when've\": 'when have',\n \"when's\": 'when is',\n \"where're\": 'where are',\n \"where'd\": 'where did',\n \"where've\": 'where have',\n \"where's\": 'where is',\n \"which's\": 'which is',\n \"who're\": 'who are',\n \"who've\": 'who have',\n \"who's\": 'who is',\n \"who'll\": 'who will',\n \"who'll've\": 'who will have',\n \"who'd\": 'who would',\n \"who'd've\": 'who would have',\n \"why're\": 'why are',\n \"why'd\": 'why did',\n \"why've\": 'why have',\n \"why's\": 'why is',\n \"will've\": 'will have',\n \"won't\": 'will not',\n \"won't've\": 'will not have',\n \"would've\": 'would have',\n \"wouldn't\": 'would not',\n \"wouldn't've\": 'would not have',\n \"y'all\": 'you all',\n \"y'all're\": 'you all are',\n \"y'all've\": 'you all have',\n \"y'all'd\": 'you all would',\n \"y'all'd've\": 'you all would have',\n \"you're\": 'you are',\n \"you've\": 'you have',\n \"you'll've\": 'you shall have',\n \"you'll\": 'you will',\n \"you'd\": 'you would',\n \"you'd've\": 'you would have',\n 'jan.': 'january',\n 'feb.': 'february',\n 'mar.': 'march',\n 'apr.': 'april',\n 'jun.': 'june',\n 'jul.': 'july',\n 'aug.': 'august',\n 'sep.': 'september',\n 'oct.': 'october',\n 'nov.': 'november',\n 'dec.': 'december',\n 'I’m': 'I am',\n 'I’m’a': 'I am about to',\n 'I’m’o': 'I am going to',\n 'I’ve': 'I have',\n 'I’ll': 'I will',\n 'I’ll’ve': 'I will have',\n 'I’d': 'I would',\n 'I’d’ve': 'I would have',\n 'amn’t': 'am not',\n 'ain’t': 'are not',\n 'aren’t': 'are not',\n '’cause': 'because',\n 'can’t': 'can not',\n 'can’t’ve': 'can not have',\n 'could’ve': 'could have',\n 'couldn’t': 'could not',\n 'couldn’t’ve': 'could not have',\n 'daren’t': 'dare not',\n 'daresn’t': 'dare not',\n 'dasn’t': 'dare not',\n 'doesn’t': 'does not',\n 'e’er': 'ever',\n 'everyone’s': 'everyone is',\n 'gon’t': 'go not',\n 'hadn’t': 'had not',\n 'hadn’t’ve': 'had not have',\n 'hasn’t': 'has not',\n 'haven’t': 'have not',\n 'he’ve': 'he have',\n 'he’s': 'he is',\n 'he’ll': 'he will',\n 'he’ll’ve': 'he will have',\n 'he’d': 'he would',\n 'he’d’ve': 'he would have',\n 'here’s': 'here is',\n 'how’re': 'how are',\n 'how’d': 'how did',\n 'how’d’y': 'how do you',\n 'how’s': 'how is',\n 'how’ll': 'how will',\n 'isn’t': 'is not',\n 'it’s': 'it is',\n '’tis': 'it is',\n '’twas': 'it was',\n 'it’ll': 'it will',\n 'it’ll’ve': 'it will have',\n 'it’d': 'it would',\n 'it’d’ve': 'it would have',\n 'let’s': 'let us',\n 'ma’am': 'madam',\n 'may’ve': 'may have',\n 'mayn’t': 'may not',\n 'might’ve': 'might have',\n 'mightn’t': 'might not',\n 'mightn’t’ve': 'might not have',\n 'must’ve': 'must have',\n 'mustn’t': 'must not',\n 'mustn’t’ve': 'must not have',\n 'needn’t': 'need not',\n 'needn’t’ve': 'need not have',\n 'ne’er': 'never',\n 'o’': 'of',\n 'o’clock': 'of the clock',\n 'ol’': 'old',\n 'oughtn’t': 'ought not',\n 'oughtn’t’ve': 'ought not have',\n 'o’er': 'over',\n 'shan’t': 'shall not',\n 'sha’n’t': 'shall not',\n 'shalln’t': 'shall not',\n 'shan’t’ve': 'shall not have',\n 'she’s': 'she is',\n 'she’ll': 'she will',\n 'she’d': 'she would',\n 'she’d’ve': 'she would have',\n 'should’ve': 'should have',\n 'shouldn’t': 'should not',\n 'shouldn’t’ve': 'should not have',\n 'so’ve': 'so have',\n 'so’s': 'so is',\n 'somebody’s': 'somebody is',\n 'someone’s': 'someone is',\n 'something’s': 'something is',\n 'that’re': 'that are',\n 'that’s': 'that is',\n 'that’ll': 'that will',\n 'that’d': 'that would',\n 'that’d’ve': 'that would have',\n 'there’re': 'there are',\n 'there’s': 'there is',\n 'there’ll': 'there will',\n 'there’d': 'there would',\n 'there’d’ve': 'there would have',\n 'these’re': 'these are',\n 'they’re': 'they are',\n 'they’ve': 'they have',\n 'they’ll': 'they will',\n 'they’ll’ve': 'they will have',\n 'they’d': 'they would',\n 'they’d’ve': 'they would have',\n 'this’s': 'this is',\n 'those’re': 'those are',\n 'to’ve': 'to have',\n 'wasn’t': 'was not',\n 'we’re': 'we are',\n 'we’ve': 'we have',\n 'we’ll': 'we will',\n 'we’ll’ve': 'we will have',\n 'we’d': 'we would',\n 'we’d’ve': 'we would have',\n 'weren’t': 'were not',\n 'what’re': 'what are',\n 'what’d': 'what did',\n 'what’ve': 'what have',\n 'what’s': 'what is',\n 'what’ll': 'what will',\n 'what’ll’ve': 'what will have',\n 'when’ve': 'when have',\n 'when’s': 'when is',\n 'where’re': 'where are',\n 'where’d': 'where did',\n 'where’ve': 'where have',\n 'where’s': 'where is',\n 'which’s': 'which is',\n 'who’re': 'who are',\n 'who’ve': 'who have',\n 'who’s': 'who is',\n 'who’ll': 'who will',\n 'who’ll’ve': 'who will have',\n 'who’d': 'who would',\n 'who’d’ve': 'who would have',\n 'why’re': 'why are',\n 'why’d': 'why did',\n 'why’ve': 'why have',\n 'why’s': 'why is',\n 'will’ve': 'will have',\n 'won’t': 'will not',\n 'won’t’ve': 'will not have',\n 'would’ve': 'would have',\n 'wouldn’t': 'would not',\n 'wouldn’t’ve': 'would not have',\n 'y’all': 'you all',\n 'y’all’re': 'you all are',\n 'y’all’ve': 'you all have',\n 'y’all’d': 'you all would',\n 'y’all’d’ve': 'you all would have',\n 'you’re': 'you are',\n 'you’ve': 'you have',\n 'you’ll’ve': 'you shall have',\n 'you’ll': 'you will',\n 'you’d': 'you would',\n 'you’d’ve': 'you would have'}\n\n#Đưa các từ viết tắt về dạng chuẩn\ndef contraction_fix(word):\n    try:\n        a=contractions[word]\n    except KeyError:\n        a=word\n    return a","metadata":{"execution":{"iopub.status.busy":"2022-01-30T06:21:15.256415Z","iopub.execute_input":"2022-01-30T06:21:15.257004Z","iopub.status.idle":"2022-01-30T06:21:15.306482Z","shell.execute_reply.started":"2022-01-30T06:21:15.256951Z","shell.execute_reply":"2022-01-30T06:21:15.305332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def Preprocess(doc):\n    corpus=[] #Tập văn bản câu hỏi  được xử lý\n    for text in tqdm(doc):\n        text=\" \".join([contraction_fix(w) for w in text.split()])\n        text=re.sub(r'[^a-z0-9A-Z]',\" \",text) #bỏ dấu gạch nối giữa các kí tự\n        text=re.sub(r'[0-9]{1}',\"#\",text) \n        text=re.sub(r'[0-9]{2}','##',text)   #thay các kí tự số bằng #\n        text=re.sub(r'[0-9]{3}','###',text)\n        text=re.sub(r'[0-9]{4}','####',text)\n        text=re.sub(r'[0-9]{5,}','#####',text)\n        corpus.append(text)\n    return corpus","metadata":{"execution":{"iopub.status.busy":"2022-01-30T06:21:15.308309Z","iopub.execute_input":"2022-01-30T06:21:15.308779Z","iopub.status.idle":"2022-01-30T06:21:15.321529Z","shell.execute_reply.started":"2022-01-30T06:21:15.308729Z","shell.execute_reply":"2022-01-30T06:21:15.320425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### tạo bộ từ vựng từ tập văn phạm\n### cấu trúc {từ vựng : số lần xuất hiện từ vựng đó trong tập văn phạm}\ndef vocab_build(corpus):\n    vocab={}\n    for text in tqdm(corpus):\n        for word in text.split():\n            try:\n                vocab[word]+=1\n            except KeyError:\n                vocab[word]=1\n    return vocab","metadata":{"execution":{"iopub.status.busy":"2022-01-30T06:21:15.323083Z","iopub.execute_input":"2022-01-30T06:21:15.323703Z","iopub.status.idle":"2022-01-30T06:21:15.334211Z","shell.execute_reply.started":"2022-01-30T06:21:15.323632Z","shell.execute_reply":"2022-01-30T06:21:15.333209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Lấy vị trí các từ trong bộ từ vựng\ndef get_word_index(vocab):\n    word_index=dict((w,i+1) for i,w in enumerate(vocab.keys()))\n    return word_index\n###Encoding các văn bản trong corpus\ndef fit_one_hot(word_index,corpus):\n    all_questions=[]\n    for text in tqdm(corpus):\n        #Một câu được mã hoá bằng một vector chứa các thứ tự của các từ trong câu\n        question=[]\n        for word in text.split():\n            try:\n                #Mỗi từ trong câu sẽ được mã hoá bằng thứ tự từ đó trong bộ từ vựng\n                question.append(word_index[word])\n            except KeyError:\n                #Nếu từ đó không có trong bộ từ vựng sẽ được mã hoá là 0\n                question.append(0)\n        all_questions.append(question)\n    return all_questions","metadata":{"execution":{"iopub.status.busy":"2022-01-30T06:21:15.339237Z","iopub.execute_input":"2022-01-30T06:21:15.339907Z","iopub.status.idle":"2022-01-30T06:21:15.349702Z","shell.execute_reply.started":"2022-01-30T06:21:15.33986Z","shell.execute_reply":"2022-01-30T06:21:15.348727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!unzip ../input/quora-insincere-questions-classification/embeddings.zip","metadata":{"execution":{"iopub.status.busy":"2022-01-30T06:21:15.35162Z","iopub.execute_input":"2022-01-30T06:21:15.35252Z","iopub.status.idle":"2022-01-30T06:24:56.627849Z","shell.execute_reply.started":"2022-01-30T06:21:15.35247Z","shell.execute_reply":"2022-01-30T06:24:56.626741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Download Embedding packages","metadata":{}},{"cell_type":"code","source":"### Load Google News pretrain embedding (model phụ trách việc embedding từ)\nfile_name=\"./GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin\"\nmodel_embed=KeyedVectors.load_word2vec_format(file_name,binary=True)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T06:24:56.629711Z","iopub.execute_input":"2022-01-30T06:24:56.630321Z","iopub.status.idle":"2022-01-30T06:26:06.043317Z","shell.execute_reply.started":"2022-01-30T06:24:56.630239Z","shell.execute_reply":"2022-01-30T06:26:06.042233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data=pd.read_csv(\"../input/quora-insincere-questions-classification/test.csv\")\ntrain_data=pd.read_csv(\"../input/quora-insincere-questions-classification/train.csv\")\ntrain,val = train_test_split(train_data,test_size=0.2,stratify=train_data.target,random_state=123)\n#keep,ignore=train_test_split(train_data,test_size=0.7,stratify=train_data.target,random_state=123)\n#train,val = train_test_split(keep,test_size=0.2,stratify=keep.target,random_state=123)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T06:26:06.045295Z","iopub.execute_input":"2022-01-30T06:26:06.045608Z","iopub.status.idle":"2022-01-30T06:26:13.178515Z","shell.execute_reply.started":"2022-01-30T06:26:06.045565Z","shell.execute_reply":"2022-01-30T06:26:13.1774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Lấy toàn bộ câu hỏi trong train và test dataset\ntotal_text=pd.concat([train_data.question_text,test_data.question_text])\n#Tiền xử lý \npre_text=Preprocess(total_text)\n#Tạo bộ từ vựng\nvocabulary=vocab_build(pre_text)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T06:26:13.179998Z","iopub.execute_input":"2022-01-30T06:26:13.180316Z","iopub.status.idle":"2022-01-30T06:27:26.812387Z","shell.execute_reply.started":"2022-01-30T06:26:13.180276Z","shell.execute_reply":"2022-01-30T06:27:26.811098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vocab_size=len(vocabulary)+1\n#Khởi tạo độ dài mỗi câu đã được encoding cố định là 40\n\nword_index=get_word_index(vocabulary)\n\ntrain_text=Preprocess(train.question_text)\nval_text=Preprocess(val.question_text)\ntest_text=Preprocess(test_data.question_text)\n\ntrain_encode = fit_one_hot(word_index,train_text)\nval_encode = fit_one_hot(word_index,val_text)\ntest_encode = fit_one_hot(word_index,test_text)\n\n\ntrain_padded=pad_sequences(train_encode,maxlen=40,padding=\"post\")\nval_padded=pad_sequences(val_encode,maxlen=40,padding=\"post\")\ntest_padded=pad_sequences(test_encode,maxlen=40,padding=\"post\")","metadata":{"execution":{"iopub.status.busy":"2022-01-30T06:27:26.814746Z","iopub.execute_input":"2022-01-30T06:27:26.81514Z","iopub.status.idle":"2022-01-30T06:29:00.40532Z","shell.execute_reply.started":"2022-01-30T06:27:26.815092Z","shell.execute_reply":"2022-01-30T06:29:00.404178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-30T06:29:00.406987Z","iopub.execute_input":"2022-01-30T06:29:00.407317Z","iopub.status.idle":"2022-01-30T06:29:00.431337Z","shell.execute_reply.started":"2022-01-30T06:29:00.407273Z","shell.execute_reply":"2022-01-30T06:29:00.430292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create dataset","metadata":{}},{"cell_type":"code","source":"train_dataset = QuestionDataset(train,train_padded)\nvalid_dataset = QuestionDataset(val,val_padded)\ntest_dataset = QuestionDataset(test_data,test_padded,True)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T06:29:00.433292Z","iopub.execute_input":"2022-01-30T06:29:00.434052Z","iopub.status.idle":"2022-01-30T06:29:00.440317Z","shell.execute_reply.started":"2022-01-30T06:29:00.434003Z","shell.execute_reply":"2022-01-30T06:29:00.439001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Embedding questions","metadata":{}},{"cell_type":"code","source":"#Khởi tạo ma trận embedding cho các từ trong bộ từ vựng\nembedding_mat=np.zeros((vocab_size,300))\nfor word,i in tqdm(word_index.items()):\n    try:\n        vec=model_embed[word] \n        embedding_mat[i]=vec\n    except KeyError:\n        continue\n        \nweight_mattrix = torch.from_numpy(embedding_mat)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T06:29:00.442477Z","iopub.execute_input":"2022-01-30T06:29:00.443229Z","iopub.status.idle":"2022-01-30T06:29:02.10838Z","shell.execute_reply.started":"2022-01-30T06:29:00.443182Z","shell.execute_reply":"2022-01-30T06:29:02.107194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create dataloader","metadata":{}},{"cell_type":"code","source":"train_dataloader = DataLoader(train_dataset,batch_size=512,shuffle=True)\nval_dataloader = DataLoader(valid_dataset,batch_size=512,shuffle=False)\ntest_dataloader = DataLoader(test_dataset,batch_size=512,shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T06:29:02.110408Z","iopub.execute_input":"2022-01-30T06:29:02.110776Z","iopub.status.idle":"2022-01-30T06:29:02.1197Z","shell.execute_reply.started":"2022-01-30T06:29:02.110731Z","shell.execute_reply":"2022-01-30T06:29:02.118289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device_name = 'cuda' if torch.cuda.is_available() else 'cpu'\ndevice = torch.device(device_name)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T06:29:02.121452Z","iopub.execute_input":"2022-01-30T06:29:02.122177Z","iopub.status.idle":"2022-01-30T06:29:02.267825Z","shell.execute_reply.started":"2022-01-30T06:29:02.122128Z","shell.execute_reply":"2022-01-30T06:29:02.266478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create model,optimizer, loss function","metadata":{}},{"cell_type":"code","source":"model = bi_lstm(300,128,weight_mattrix).to(device)\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(model.parameters(),lr=0.001)\nscheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min',patience=1)\nearlystopping = EarlyStopping(patience=1)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T06:29:02.26997Z","iopub.execute_input":"2022-01-30T06:29:02.27089Z","iopub.status.idle":"2022-01-30T06:29:09.263029Z","shell.execute_reply.started":"2022-01-30T06:29:02.270637Z","shell.execute_reply":"2022-01-30T06:29:09.262037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training and Validating","metadata":{}},{"cell_type":"code","source":"for epoch in range(10):\n    print('Epoch{epoch} of 10:'.format(epoch=epoch))\n    train_loss , train_acc = train_model(train_dataloader,train_dataset,optimizer,device,model,criterion)\n    print('Trainloss:{loss:.3f},Train-accuracy:{acc:.3f}'.format(loss=train_loss,acc=train_acc))\n    val_loss, val_acc  = valid_model(val_dataloader,valid_dataset,model,device,criterion)\n    print('Valloss:{loss:.3f},Val-accuracy:{acc:.3f}'.format(loss=val_loss,acc=val_acc))\n    scheduler.step(val_loss)\n    earlystopping(val_loss)\n    if earlystopping.early_stop:\n        break\n","metadata":{"execution":{"iopub.status.busy":"2022-01-30T06:29:09.264412Z","iopub.execute_input":"2022-01-30T06:29:09.264737Z","iopub.status.idle":"2022-01-30T06:34:03.45081Z","shell.execute_reply.started":"2022-01-30T06:29:09.264692Z","shell.execute_reply":"2022-01-30T06:34:03.449731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_qid = []\nall_preds = []\nfor batch,data in tqdm(enumerate(test_dataloader),total = len(test_dataset)/test_dataloader.batch_size):\n    question_text = data['question_text'].to(device)\n    qid = data['qid']\n    out = model(question_text)\n    _,preds = out.max(1)\n    all_qid += qid\n    all_preds += preds.tolist()\n","metadata":{"execution":{"iopub.status.busy":"2022-01-30T06:34:03.452659Z","iopub.execute_input":"2022-01-30T06:34:03.453274Z","iopub.status.idle":"2022-01-30T06:34:16.228166Z","shell.execute_reply.started":"2022-01-30T06:34:03.453211Z","shell.execute_reply":"2022-01-30T06:34:16.219976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# To submission file","metadata":{}},{"cell_type":"code","source":"submit = pd.DataFrame()\nsubmit['qid'] = all_qid\nsubmit['prediction'] = all_preds\n\nsubmit.to_csv(\"submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T06:34:16.229615Z","iopub.execute_input":"2022-01-30T06:34:16.230086Z","iopub.status.idle":"2022-01-30T06:34:17.639184Z","shell.execute_reply.started":"2022-01-30T06:34:16.230025Z","shell.execute_reply":"2022-01-30T06:34:17.638093Z"},"trusted":true},"execution_count":null,"outputs":[]}]}