{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport re\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-10T13:49:01.027971Z","iopub.execute_input":"2021-06-10T13:49:01.028477Z","iopub.status.idle":"2021-06-10T13:49:01.050309Z","shell.execute_reply.started":"2021-06-10T13:49:01.028356Z","shell.execute_reply":"2021-06-10T13:49:01.049401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn import model_selection, preprocessing, linear_model, naive_bayes, metrics, svm\nfrom sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\nfrom sklearn import decomposition, ensemble\nfrom sklearn.decomposition import TruncatedSVD\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.datasets import load_digits\nfrom sklearn.model_selection import learning_curve\nfrom sklearn.model_selection import ShuffleSplit\nfrom keras.preprocessing import text, sequence\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.layers import Input, Conv1D, GRU, merge, Dense, Flatten, LSTM,Activation, Dropout,Bidirectional,Reshape,Conv2D,MaxPool2D,Concatenate\nfrom keras.layers.experimental.preprocessing import TextVectorization\nfrom keras.models import Model, Sequential\nfrom keras.regularizers import l2\nfrom keras import backend as K\nfrom keras.optimizers import Adam,SGD\nfrom keras import initializers,regularizers\nfrom keras.optimizers import Adam,SGD\nimport keras\nfrom keras.preprocessing.text import one_hot,Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.models import Sequential\nfrom keras.layers import Dense , Flatten ,Embedding,Input\nfrom keras.models import Model","metadata":{"execution":{"iopub.status.busy":"2021-06-10T13:49:01.051573Z","iopub.execute_input":"2021-06-10T13:49:01.051978Z","iopub.status.idle":"2021-06-10T13:49:03.805811Z","shell.execute_reply.started":"2021-06-10T13:49:01.051945Z","shell.execute_reply":"2021-06-10T13:49:03.802214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data","metadata":{}},{"cell_type":"code","source":"dir = \"../input/quora-insincere-questions-classification/\"","metadata":{"execution":{"iopub.status.busy":"2021-06-10T13:49:03.806887Z","iopub.status.idle":"2021-06-10T13:49:03.807304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Đọc tập train và test từ dataset.","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(\"../input/quora-insincere-questions-classification/train.csv\")\ntrain = train.sample(frac = 1)\ntrain.head(20)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T13:49:03.808467Z","iopub.status.idle":"2021-06-10T13:49:03.808868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Xem qua nhưng câu hỏi có nhãn bằng 1","metadata":{}},{"cell_type":"code","source":"train[train.target == 1]","metadata":{"execution":{"iopub.status.busy":"2021-06-10T13:49:03.809838Z","iopub.status.idle":"2021-06-10T13:49:03.810212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv(\"../input/quora-insincere-questions-classification/test.csv\")\ntest.head(20)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T13:49:03.811023Z","iopub.status.idle":"2021-06-10T13:49:03.811405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub = pd.read_csv(\"../input/quora-insincere-questions-classification/sample_submission.csv\")\nprint(sample_sub[sample_sub.prediction <1 ])","metadata":{"execution":{"iopub.status.busy":"2021-06-10T13:49:03.812593Z","iopub.status.idle":"2021-06-10T13:49:03.813171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub.shape","metadata":{"execution":{"iopub.status.busy":"2021-06-10T13:49:03.814453Z","iopub.status.idle":"2021-06-10T13:49:03.815078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Pre-processing","metadata":{}},{"cell_type":"markdown","source":"# Remove stopword, lowercase, remove punctual, lematize","metadata":{}},{"cell_type":"code","source":"from gensim.utils import simple_preprocess \nfrom nltk.corpus import stopwords\nfrom nltk.stem import WordNetLemmatizer\n\nstop_words = set(stopwords.words('english')) \nwordnet_lemmatizer = WordNetLemmatizer()\ndef preprocessing(corpus):\n    res = []\n    for doc in corpus:\n        words = []\n        for word in simple_preprocess(doc):\n            if word not in stop_words:\n                word1 = wordnet_lemmatizer.lemmatize(word, pos = \"n\")\n                word2 = wordnet_lemmatizer.lemmatize(word1,pos = \"v\")\n                word3 = wordnet_lemmatizer.lemmatize(word2, pos = (\"a\"))\n                words.append(word3)\n                pass\n            pass\n        res.append(' '.join(words))        \n        pass\n    return res","metadata":{"execution":{"iopub.status.busy":"2021-06-10T13:49:03.816167Z","iopub.status.idle":"2021-06-10T13:49:03.816766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain['question_text'] = preprocessing(train['question_text'])\ntest['question_text'] = preprocessing(test['question_text'])","metadata":{"execution":{"iopub.status.busy":"2021-06-10T13:49:03.817908Z","iopub.status.idle":"2021-06-10T13:49:03.818567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Tính độ dài lớn nhất của câu hỏi trong tập train","metadata":{}},{"cell_type":"code","source":"m = 0\nfor i in train['question_text']:\n    m  = max(m, len(i.split()))\n    if (len(i.split()) == m):\n        print(i, m)\nprint(m)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T13:49:03.819754Z","iopub.status.idle":"2021-06-10T13:49:03.820347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Biến đổi từ câu sang vector bằng one hot, mỗi câu tương ứng với một list các số tương ứng với từ","metadata":{}},{"cell_type":"code","source":"%%time\nvocab_size=150000\nmax_lenth = 33\ntrain_X = []\nfor i in range(0, len(train)):\n    train_X.append(one_hot(train.iloc[i]['question_text'], vocab_size))\ntest_X = []\nfor i in range(0, len(test)):\n    test_X.append(one_hot(test.iloc[i]['question_text'], vocab_size))","metadata":{"execution":{"iopub.status.busy":"2021-06-10T13:49:03.821737Z","iopub.status.idle":"2021-06-10T13:49:03.822287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Load các vector word embedding từ GloVe","metadata":{}},{"cell_type":"code","source":"embeddings_index = dict()\nf = open('/kaggle/input/glove-global-vectors-for-word-representation/glove.6B.50d.txt')\nfor line in f:\n    values = line.split()\n    word = values[0]\n    coefs = np.asarray(values[1:], dtype='float32')\n    embeddings_index[word] = coefs\nf.close()\nprint('Loaded %s word vectors.' % len(embeddings_index))","metadata":{"execution":{"iopub.status.busy":"2021-06-10T13:49:03.823541Z","iopub.status.idle":"2021-06-10T13:49:03.824091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer = Tokenizer(num_words=150000)\ntokenizer.fit_on_texts(train['question_text'])","metadata":{"execution":{"iopub.status.busy":"2021-06-10T13:49:03.825306Z","iopub.status.idle":"2021-06-10T13:49:03.825877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Tạo embedding matrix","metadata":{}},{"cell_type":"code","source":"vocab_size = len(tokenizer.word_index) + 1\n\ncnt = 0\nembedding_matrix = np.zeros((vocab_size, 50))\nfor word, i in tokenizer.word_index.items():\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None:\n        embedding_matrix[i] = embedding_vector\n        cnt = cnt + 1\nprint(vocab_size, ' ', cnt)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T13:49:03.827534Z","iopub.status.idle":"2021-06-10T13:49:03.828101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_X[0]","metadata":{"execution":{"iopub.status.busy":"2021-06-10T13:49:03.829216Z","iopub.status.idle":"2021-06-10T13:49:03.829782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Padding để tạo thành các chuỗi có độ dài bằng nhau, thuận tiện cho mô hình","metadata":{}},{"cell_type":"code","source":"train_X = sequence.pad_sequences(train_X, maxlen=max_lenth, padding='post')\ntest_X = sequence.pad_sequences(test_X, maxlen=max_lenth, padding='post')\n\ntrain_Y = train['target']","metadata":{"execution":{"iopub.status.busy":"2021-06-10T13:49:03.830861Z","iopub.status.idle":"2021-06-10T13:49:03.831517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Metrics","metadata":{}},{"cell_type":"markdown","source":"2 Metric chủ yếu : accuracy, F1","metadata":{}},{"cell_type":"code","source":"from keras import backend as K\n\ndef recall_m(y_true, y_pred):\n    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n    recall = true_positives / (possible_positives + K.epsilon())\n    return recall\n\ndef precision_m(y_true, y_pred):\n    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n    precision = true_positives / (predicted_positives + K.epsilon())\n    return precision\n\ndef f1_m(y_true, y_pred):\n    precision = precision_m(y_true, y_pred)\n    recall = recall_m(y_true, y_pred)\n    return 2*((precision*recall)/(precision+recall+K.epsilon()))\n","metadata":{"execution":{"iopub.status.busy":"2021-06-10T13:49:03.832780Z","iopub.status.idle":"2021-06-10T13:49:03.833355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Bi - GRU","metadata":{}},{"cell_type":"markdown","source":"Mô hình với Bidirectional GRU với embeddings \n","metadata":{}},{"cell_type":"code","source":"_input = Input(max_lenth)\nembed = Embedding(input_dim=vocab_size, output_dim=50, weights=[embedding_matrix], trainable=True, input_length=max_lenth)(_input)\nlayer = Bidirectional(GRU(128, return_sequences=True))(embed)\nlayer = Dense(128, activation = 'relu')(layer)\nlayer = Dropout(0.5)(layer)\nlayer = Dense(64, activation = 'relu')(layer)\nlayer = Dropout(0.5)(layer)\nlayer = Flatten()(layer)\nprediction = Dense(1, activation = 'sigmoid')(layer)\n\nmodel = Model(inputs=_input,outputs=prediction)\nmodel.compile(loss=keras.losses.BinaryCrossentropy(),optimizer=Adam(lr=0.0003),metrics = ['accuracy',f1_m])\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-06-10T13:49:03.834339Z","iopub.status.idle":"2021-06-10T13:49:03.834752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(train_X, train['target'], epochs = 10, validation_split = 0.2, verbose = 2, batch_size = 128)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T13:49:03.835495Z","iopub.status.idle":"2021-06-10T13:49:03.835831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict = model.predict(test_X)\npredict[:20]","metadata":{"execution":{"iopub.status.busy":"2021-06-10T13:49:03.836562Z","iopub.status.idle":"2021-06-10T13:49:03.836887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['prediction'] = (predict > 0.35).astype(int)\nresults = test[['qid', 'prediction']]\nresults.to_csv('submission.csv', index=False)\nresults.shape","metadata":{"execution":{"iopub.status.busy":"2021-06-10T13:49:03.837638Z","iopub.status.idle":"2021-06-10T13:49:03.838114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head(20)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T13:49:03.838754Z","iopub.status.idle":"2021-06-10T13:49:03.839076Z"},"trusted":true},"execution_count":null,"outputs":[]}]}