{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# # This Python 3 environment comes with many helpful analytics libraries installed\n# # It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# # For example, here's several helpful packages to load\n\n# import numpy as np # linear algebra\n# import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# # Input data files are available in the read-only \"../input/\" directory\n# # For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# # You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# # You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Note:** Vì yêu cầu submit của challenge đã kết thúc là không được sử dụng '+Add data' dữ liệu từ bên ngoài nên notebook này em sẽ chỉ thực hiện lại các bước xử lý từ đầu đến cuối để có thể nộp bài, còn phần EDA và phần Feature Extraction em đã thực hiện ở 2 notebook trước đó ạ.","metadata":{}},{"cell_type":"markdown","source":"---------------------------------------------------------------------------------------------------------------------------------------------------------","metadata":{}},{"cell_type":"markdown","source":"> # Xử lý dữ liệu và chạy model cuối","metadata":{}},{"cell_type":"markdown","source":"(Các block code này em đã thực hiện ở 2 notebook trước rồi ạ)","metadata":{}},{"cell_type":"code","source":"import re\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt \nimport seaborn as sns\nimport nltk\nimport pickle\nimport zipfile\nimport tensorflow as tf\n\nfrom nltk.corpus import stopwords\nfrom nltk.stem.wordnet import WordNetLemmatizer\nfrom nltk import word_tokenize\nfrom nltk.tokenize import word_tokenize\n\nfrom collections import Counter\n\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.model_selection import KFold\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import f1_score\nfrom sklearn.metrics import classification_report\n\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\n\nfrom tensorflow import keras\nfrom keras.models import Model\nfrom keras.layers import Dense, Input, LSTM, GRU, Embedding, Dropout, Activation, Conv1D\nfrom keras.layers import Bidirectional, GlobalMaxPool1D, GlobalMaxPooling1D, GlobalAveragePooling1D\nfrom keras.layers import Input, Embedding, Dense, Conv2D, MaxPool2D, concatenate\nfrom keras.layers import Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D\nfrom keras.optimizers import Adam\nfrom keras.callbacks import EarlyStopping, ReduceLROnPlateau","metadata":{"execution":{"iopub.status.busy":"2021-06-10T20:05:35.528508Z","iopub.execute_input":"2021-06-10T20:05:35.528876Z","iopub.status.idle":"2021-06-10T20:05:41.600855Z","shell.execute_reply.started":"2021-06-10T20:05:35.528795Z","shell.execute_reply":"2021-06-10T20:05:41.600000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/quora-insincere-questions-classification/train.csv')\ntest = pd.read_csv('../input/quora-insincere-questions-classification/test.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-10T20:05:41.603296Z","iopub.execute_input":"2021-06-10T20:05:41.603654Z","iopub.status.idle":"2021-06-10T20:05:45.810261Z","shell.execute_reply.started":"2021-06-10T20:05:41.603618Z","shell.execute_reply":"2021-06-10T20:05:45.809377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stop_words = stopwords.words('english')\nwordnet = WordNetLemmatizer() ","metadata":{"execution":{"iopub.status.busy":"2021-06-10T20:05:45.814121Z","iopub.execute_input":"2021-06-10T20:05:45.814395Z","iopub.status.idle":"2021-06-10T20:05:45.833621Z","shell.execute_reply.started":"2021-06-10T20:05:45.814369Z","shell.execute_reply":"2021-06-10T20:05:45.832850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Xóa bỏ các từ là stopword\ndef delStopwords(text):\n    return ' '.join([w for w in word_tokenize(text) if not w in stop_words])\n\n# Đưa các từ về dạng từ gốc của nó\ndef lemmatizeText(text):\n    return ' '.join([wordnet.lemmatize(w, pos='v') for w in word_tokenize(text)])","metadata":{"execution":{"iopub.status.busy":"2021-06-10T20:05:45.834912Z","iopub.execute_input":"2021-06-10T20:05:45.835188Z","iopub.status.idle":"2021-06-10T20:05:45.842759Z","shell.execute_reply.started":"2021-06-10T20:05:45.835162Z","shell.execute_reply":"2021-06-10T20:05:45.841790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(text):\n    cleaned_text = re.sub('[\\n]',' ',text)    #xóa các ký tự xuống dòng\n    cleaned_text = re.sub('[A-Z]+', lambda m: m.group(0).lower(), cleaned_text)    #lower các ký tự \n    cleaned_text = re.sub('[^a-zA-Z]',' ',cleaned_text).strip()    #xóa các ký tự không phải là chữ cái\n    cleaned_text = delStopwords(cleaned_text)    #xóa stopwords\n    cleaned_text = lemmatizeText(cleaned_text)    #lemmatize các từ về danh noun\n    return cleaned_text","metadata":{"execution":{"iopub.status.busy":"2021-06-10T20:06:00.658301Z","iopub.execute_input":"2021-06-10T20:06:00.658621Z","iopub.status.idle":"2021-06-10T20:06:00.666309Z","shell.execute_reply.started":"2021-06-10T20:06:00.658591Z","shell.execute_reply":"2021-06-10T20:06:00.665450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_text = []\ntarget = []\nfor i in range(len(train)):\n    train_text.append(clean_text(train['question_text'][i]))\n    target.append((train['target'][i]).astype('int32'))\n    \ntest_text = []\nfor i in range(len(test)):\n    test_text.append(clean_text(test['question_text'][i]))","metadata":{"execution":{"iopub.status.busy":"2021-06-10T20:06:03.078277Z","iopub.execute_input":"2021-06-10T20:06:03.078598Z","iopub.status.idle":"2021-06-10T20:16:19.085635Z","shell.execute_reply.started":"2021-06-10T20:06:03.078568Z","shell.execute_reply":"2021-06-10T20:16:19.084822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_text), len(target), len(test_text)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T20:23:12.014395Z","iopub.execute_input":"2021-06-10T20:23:12.014716Z","iopub.status.idle":"2021-06-10T20:23:12.022210Z","shell.execute_reply.started":"2021-06-10T20:23:12.014687Z","shell.execute_reply":"2021-06-10T20:23:12.021282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with zipfile.ZipFile('../input/quora-insincere-questions-classification/embeddings.zip', 'r') as zip_ref:\n    zip_ref.extractall('./embeddings')","metadata":{"execution":{"iopub.status.busy":"2021-06-10T20:23:15.960669Z","iopub.execute_input":"2021-06-10T20:23:15.961021Z","iopub.status.idle":"2021-06-10T20:26:49.385901Z","shell.execute_reply.started":"2021-06-10T20:23:15.960984Z","shell.execute_reply":"2021-06-10T20:26:49.385050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_text = np.array(train_text)\ntest_text = np.array(test_text)\n\ntype(train_text), type(test_text), type(target)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T20:28:17.043637Z","iopub.execute_input":"2021-06-10T20:28:17.043978Z","iopub.status.idle":"2021-06-10T20:28:19.488473Z","shell.execute_reply.started":"2021-06-10T20:28:17.043947Z","shell.execute_reply":"2021-06-10T20:28:19.487690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MAX_LEN = 30    # Độ dài của 1 câu \nEMBEDDING_SIZE = 300    # Số chiều embedding khi sử dụng fastText mặc định là (300,) ","metadata":{"execution":{"iopub.status.busy":"2021-06-10T20:28:19.489798Z","iopub.execute_input":"2021-06-10T20:28:19.490097Z","iopub.status.idle":"2021-06-10T20:28:19.496266Z","shell.execute_reply.started":"2021-06-10T20:28:19.490069Z","shell.execute_reply":"2021-06-10T20:28:19.492830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from gensim.models import FastText \n\ndef load_fasttext():\n    print('loading word embeddings...')\n    embeddings_index = {}\n    f = open('./embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec',encoding='utf-8')\n    for line in f:\n        values = line.strip().rsplit(' ')\n        word = values[0]\n        coefs = np.asarray(values[1:], dtype='float32')\n        embeddings_index[word] = coefs\n    f.close()\n    print('found %s word vectors' % len(embeddings_index))\n\n    return embeddings_index","metadata":{"execution":{"iopub.status.busy":"2021-06-10T20:28:38.730827Z","iopub.execute_input":"2021-06-10T20:28:38.731176Z","iopub.status.idle":"2021-06-10T20:28:38.874394Z","shell.execute_reply.started":"2021-06-10T20:28:38.731143Z","shell.execute_reply":"2021-06-10T20:28:38.873498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embeddings_index = load_fasttext()","metadata":{"execution":{"iopub.status.busy":"2021-06-10T20:28:40.501471Z","iopub.execute_input":"2021-06-10T20:28:40.501843Z","iopub.status.idle":"2021-06-10T20:30:17.339145Z","shell.execute_reply.started":"2021-06-10T20:28:40.501812Z","shell.execute_reply":"2021-06-10T20:30:17.337904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_text = list(train_text)+list(test_text)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T20:31:33.708758Z","iopub.execute_input":"2021-06-10T20:31:33.709099Z","iopub.status.idle":"2021-06-10T20:31:36.779659Z","shell.execute_reply.started":"2021-06-10T20:31:33.709069Z","shell.execute_reply":"2021-06-10T20:31:36.778817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer = Tokenizer()\ntokenizer.fit_on_texts(all_text)\n\ntrain_sequences = tokenizer.texts_to_sequences(train_text)\ntest_sequences = tokenizer.texts_to_sequences(test_text)\n\ntrain_padded_sequences = pad_sequences(train_sequences, maxlen=MAX_LEN, padding='post', truncating='post')\ntest_padded_sequences = pad_sequences(test_sequences, maxlen=MAX_LEN, padding='post', truncating='post')","metadata":{"execution":{"iopub.status.busy":"2021-06-10T20:31:38.250280Z","iopub.execute_input":"2021-06-10T20:31:38.250593Z","iopub.status.idle":"2021-06-10T20:32:31.777291Z","shell.execute_reply.started":"2021-06-10T20:31:38.250566Z","shell.execute_reply":"2021-06-10T20:32:31.776326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_index = tokenizer.word_index\nnum_words = len(word_index)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T20:32:31.778785Z","iopub.execute_input":"2021-06-10T20:32:31.779139Z","iopub.status.idle":"2021-06-10T20:32:31.783826Z","shell.execute_reply.started":"2021-06-10T20:32:31.779101Z","shell.execute_reply":"2021-06-10T20:32:31.782953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prepare_matrix(embedding_index, emb_size = EMBEDDING_SIZE):\n    embedding_matrix = np.zeros((num_words+1, emb_size))\n\n    for word, i in word_index.items():\n        if i > num_words:\n            continue\n\n        emb_vec = embedding_index.get(word)\n        if emb_vec is not None:\n            embedding_matrix[i] = emb_vec\n\n    return embedding_matrix","metadata":{"execution":{"iopub.status.busy":"2021-06-10T20:32:31.785802Z","iopub.execute_input":"2021-06-10T20:32:31.786177Z","iopub.status.idle":"2021-06-10T20:32:31.795926Z","shell.execute_reply.started":"2021-06-10T20:32:31.786141Z","shell.execute_reply":"2021-06-10T20:32:31.795074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embedding_matrix = prepare_matrix(embeddings_index)\nembedding_matrix.shape","metadata":{"execution":{"iopub.status.busy":"2021-06-10T20:32:31.797227Z","iopub.execute_input":"2021-06-10T20:32:31.797605Z","iopub.status.idle":"2021-06-10T20:32:32.215538Z","shell.execute_reply.started":"2021-06-10T20:32:31.797570Z","shell.execute_reply":"2021-06-10T20:32:32.214541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LEARNING_RATE = 0.001\nBATCH_SIZE = 512\nEPOCHS = 10","metadata":{"execution":{"iopub.status.busy":"2021-06-10T20:32:32.216989Z","iopub.execute_input":"2021-06-10T20:32:32.217328Z","iopub.status.idle":"2021-06-10T20:32:32.221812Z","shell.execute_reply.started":"2021-06-10T20:32:32.217293Z","shell.execute_reply":"2021-06-10T20:32:32.220666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras import backend as K\n\ndef f1(y_true, y_pred):\n    def recall(y_true, y_pred):\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n        recall = true_positives / (possible_positives + K.epsilon())\n        return recall\n\n    def precision(y_true, y_pred):\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n        precision = true_positives / (predicted_positives + K.epsilon())\n        return precision\n    \n    precision = precision(y_true, y_pred)\n    recall = recall(y_true, y_pred)\n    return 2*((precision*recall)/(precision+recall+K.epsilon()))","metadata":{"execution":{"iopub.status.busy":"2021-06-10T20:32:32.223624Z","iopub.execute_input":"2021-06-10T20:32:32.224141Z","iopub.status.idle":"2021-06-10T20:32:32.233394Z","shell.execute_reply.started":"2021-06-10T20:32:32.224106Z","shell.execute_reply":"2021-06-10T20:32:32.232455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_train_val_f1score(hist):\n    plt.figure(figsize=(12,8))\n    epoch_list = list(range(1,len(hist.history['f1'])+1))\n    plt.plot(epoch_list, hist.history['f1'],label='f1')\n    plt.plot(epoch_list, hist.history['val_f1'],label='val_f1')\n    plt.xlabel('epoches')\n    plt.ylabel('score')\n    plt.legend()\n    plt.show()\n\ndef tweak_threshold(pred_prob, truth):\n    thresholds = []\n    scores = []\n    for thresh in np.arange(0.1, 0.501, 0.01):\n        thresh = np.round(thresh, 2)\n        thresholds.append(thresh)\n        score = f1_score(truth, (pred_prob>thresh).astype(int))\n        scores.append(score)\n    return np.max(scores), thresholds[np.argmax(scores)]","metadata":{"execution":{"iopub.status.busy":"2021-06-10T20:46:17.258917Z","iopub.execute_input":"2021-06-10T20:46:17.259261Z","iopub.status.idle":"2021-06-10T20:46:17.266557Z","shell.execute_reply.started":"2021-06-10T20:46:17.259215Z","shell.execute_reply":"2021-06-10T20:46:17.265653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Sau một vài lần chạy thử với việc thay đổi các tham số cùng với việc chạy StratifiedKFold ở notebook trước,  em đã chọn ra mô hình tốt nhất mình đạt được","metadata":{}},{"cell_type":"code","source":"def build_model(embedding_matrix, num_words, EMBEDDING_SIZE):\n    inp = Input(shape=(MAX_LEN,))\n    x = Embedding(num_words+1, EMBEDDING_SIZE, weights=[embedding_matrix], trainable=False)(inp)\n    x = Dropout(0.3)(x)\n    x1 = Bidirectional(LSTM(256, return_sequences=True))(x)\n    x2 = Bidirectional(GRU(256, return_sequences=True))(x1)\n    max_pool1 = GlobalMaxPooling1D()(x1)\n    max_pool2 = GlobalMaxPooling1D()(x2)\n    conc = Concatenate()([max_pool1, max_pool2])\n    predictions = Dense(1, activation='sigmoid')(conc)\n    model = Model(inputs=inp, outputs=predictions)\n    adam = Adam(lr=LEARNING_RATE)\n    model.compile(optimizer = adam, loss = 'binary_crossentropy', metrics = ['accuracy',f1])\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-06-10T21:10:25.559079Z","iopub.execute_input":"2021-06-10T21:10:25.559415Z","iopub.status.idle":"2021-06-10T21:10:25.568639Z","shell.execute_reply.started":"2021-06-10T21:10:25.559385Z","shell.execute_reply":"2021-06-10T21:10:25.567867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"es = EarlyStopping(monitor=f1, mode='max', patience=3, restore_best_weights=True)\nreduce_lr = ReduceLROnPlateau(monitor='val_f1', factor=0.1, patience=2, min_=0.0005, verbose=1)\nclass_weights = {0: 1, 1: 5}","metadata":{"execution":{"iopub.status.busy":"2021-06-10T20:38:26.245374Z","iopub.execute_input":"2021-06-10T20:38:26.245689Z","iopub.status.idle":"2021-06-10T20:38:26.250250Z","shell.execute_reply.started":"2021-06-10T20:38:26.245658Z","shell.execute_reply":"2021-06-10T20:38:26.249178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = build_model(embedding_matrix, num_words, EMBEDDING_SIZE)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T21:10:30.711206Z","iopub.execute_input":"2021-06-10T21:10:30.711536Z","iopub.status.idle":"2021-06-10T21:10:32.670906Z","shell.execute_reply.started":"2021-06-10T21:10:30.711507Z","shell.execute_reply":"2021-06-10T21:10:32.670083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_tr, X_val, y_tr, y_val = train_test_split(train_padded_sequences, np.array(target), test_size=0.2, shuffle=True, stratify=target)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T21:10:36.052644Z","iopub.execute_input":"2021-06-10T21:10:36.052979Z","iopub.status.idle":"2021-06-10T21:10:37.243215Z","shell.execute_reply.started":"2021-06-10T21:10:36.052947Z","shell.execute_reply":"2021-06-10T21:10:37.242342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(X_tr, y_tr,\n                   batch_size=BATCH_SIZE, \n                   epochs=EPOCHS, \n                   validation_data=(X_val,y_val), \n                   callbacks=[es, reduce_lr],\n                   class_weight=class_weights,\n                   verbose=1\n                   )","metadata":{"execution":{"iopub.status.busy":"2021-06-10T21:11:11.434647Z","iopub.execute_input":"2021-06-10T21:11:11.435037Z","iopub.status.idle":"2021-06-10T21:32:46.137190Z","shell.execute_reply.started":"2021-06-10T21:11:11.435003Z","shell.execute_reply":"2021-06-10T21:32:46.136304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_train_val_f1score(history)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T21:32:46.139157Z","iopub.execute_input":"2021-06-10T21:32:46.139526Z","iopub.status.idle":"2021-06-10T21:32:46.292727Z","shell.execute_reply.started":"2021-06-10T21:32:46.139485Z","shell.execute_reply":"2021-06-10T21:32:46.291822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Kết quả thu được là mô hình không còn bị overfit hay underfit","metadata":{}},{"cell_type":"code","source":"val_pred = np.squeeze(model.predict([X_val], batch_size=BATCH_SIZE))\nf1score, threshold = tweak_threshold(val_pred, np.squeeze(y_val))\nprint(f\"F1score: {round(f1score, 4)} for threshold: {threshold} on validation data\")\nval_prediction = (val_pred > threshold).astype(int)\nprint('Classification Report on validation data: \\n', classification_report(y_val, val_prediction))","metadata":{"execution":{"iopub.status.busy":"2021-06-10T21:34:16.388658Z","iopub.execute_input":"2021-06-10T21:34:16.389009Z","iopub.status.idle":"2021-06-10T21:34:30.574551Z","shell.execute_reply.started":"2021-06-10T21:34:16.388978Z","shell.execute_reply":"2021-06-10T21:34:30.573734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Tuy nhiên, khi chạy với model mới kết quả thu được cho precision của nhãn '1' vẫn là khá thấp, chỉ 51%. Điều này có thể là do mô hình em sử dụng vẫn còn đơn giản nên chưa trích chọn được các feature tốt nhất. Em xin phép tiếp tục cải thiện thêm mô hình của mình sau ạ.","metadata":{}},{"cell_type":"markdown","source":"----------------------------------------------------------------------------------------------------------------------------------------------------------","metadata":{}},{"cell_type":"markdown","source":"> # Dự đoán trên tập test.csv và nộp bài","metadata":{}},{"cell_type":"code","source":"y_test = np.squeeze(model.predict([test_padded_sequences], batch_size=BATCH_SIZE))","metadata":{"execution":{"iopub.status.busy":"2021-06-10T21:34:38.084368Z","iopub.execute_input":"2021-06-10T21:34:38.084679Z","iopub.status.idle":"2021-06-10T21:34:54.599989Z","shell.execute_reply.started":"2021-06-10T21:34:38.084648Z","shell.execute_reply":"2021-06-10T21:34:54.599123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_test = (y_test > threshold).astype(int)\npd.DataFrame(label_test).value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-06-10T21:35:06.895090Z","iopub.execute_input":"2021-06-10T21:35:06.895420Z","iopub.status.idle":"2021-06-10T21:35:06.917981Z","shell.execute_reply.started":"2021-06-10T21:35:06.895391Z","shell.execute_reply":"2021-06-10T21:35:06.917198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output = pd.DataFrame({'qid': test['qid'].values})\noutput['prediction'] = label_test\noutput.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T21:35:12.147665Z","iopub.execute_input":"2021-06-10T21:35:12.148091Z","iopub.status.idle":"2021-06-10T21:35:13.149925Z","shell.execute_reply.started":"2021-06-10T21:35:12.148050Z","shell.execute_reply":"2021-06-10T21:35:13.149030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save('model_Bidirectional_LSTM_GRU.h5')","metadata":{"execution":{"iopub.status.busy":"2021-06-10T21:35:20.416593Z","iopub.execute_input":"2021-06-10T21:35:20.416948Z","iopub.status.idle":"2021-06-10T21:35:21.270705Z","shell.execute_reply.started":"2021-06-10T21:35:20.416903Z","shell.execute_reply":"2021-06-10T21:35:21.269892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---------------------------------------------------------------------------------------------------------------------------------------------","metadata":{}},{"cell_type":"markdown","source":"> # Summary","metadata":{}},{"cell_type":"markdown","source":"Kết quả submit kaggle: 0.62463  ","metadata":{}},{"cell_type":"markdown","source":"Những gì đã học được:  \n- Quá trình EDA cũng như là việc xây dựng các bước tiền xử lý từ EDA.\n- Xử lý bài toán với dữ liệu imbalance.\n- Học máy cơ bản với các kiến thức nền bắt đầu từ khái niệm, các hàm loss, compile model, fit model, ...\n- Các kiến thức liên quan đến mạng học sâu như CNN, RNN, LSTM, GRU cũng như là set up các tham số cho mạng.\n- Các cách nhận biết, đánh giá model học sâu và cách tinh chỉnh model sao cho đạt kết quả tốt nhất.","metadata":{}},{"cell_type":"markdown","source":"Những gì cần cải thiện trong tương lai:\n- Xây dựng mô hình phức tạp hơn để có thể học cũng như dự đoán tốt hơn cho nhãn thiểu số.\n- Áp dụng cơ chế Attention cho bài toán dạng input sequences như challenge này để tập trung vào các đặc trưng tốt nhất.\n- Thay đổi các bước tiền xử lý vì như đã tham khảo thì có vẻ rằng em đã tiền xử lý quá kĩ khi áp dụng các pre-trained word embedding.","metadata":{}},{"cell_type":"markdown","source":"------------------------------------------------------------------------------------------------------------------------------------------------------------","metadata":{}}]}