{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"```\nStudent name: Tran Xuan Hai\nStudent ID: 19021263\nClass name: Machine Learning\nClass ID: INT3405E 20\n```","metadata":{}},{"cell_type":"markdown","source":"# Preparing","metadata":{"id":"hz3T0B_emlfS"}},{"cell_type":"code","source":"import pandas as pd\nimport re, string\nimport numpy as np\nimport os\nimport nltk\nfrom nltk.corpus import stopwords\nfrom nltk.tokenize import word_tokenize\nfrom matplotlib import pyplot as plt\n!pip install tensorflow-addons\n\n\nKAGGLE = True  # True if run on Kaggle Kernel, False if run on Google Colab\nSHOW_PLOT = True  # True to show word plot (more execution time)\n\n\nnltk.download('stopwords')\nnltk.download('punkt')\npd.set_option('display.max_colwidth', 2)\n\nif KAGGLE:\n    DATA_DIR = \"/kaggle/working\"\nelse:\n    DATA_DIR = \"/content/drive/Shareddrives/AML/embeddings\"\n\nif KAGGLE:\n    import zipfile\n    with zipfile.ZipFile(\"/kaggle/input/quora-insincere-questions-classification/embeddings.zip\", 'r') as f:\n        f.extractall(\".\")\n    !cp \"/kaggle/input/quora-insincere-questions-classification/train.csv\" .\n    !cp \"/kaggle/input/quora-insincere-questions-classification/test.csv\" .","metadata":{"id":"znjPc91f3PQ9","outputId":"4e675969-c9f7-42b1-cd76-0c7d8e1dac42"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data analyzing","metadata":{"id":"2FD-25FThO8O"}},{"cell_type":"code","source":"train_data_path = os.path.join(DATA_DIR, \"train.csv\")\n\ndf = pd.read_csv(train_data_path)\ndf.head()","metadata":{"id":"A8f4isdS4fTu","outputId":"74e8f27b-916d-457a-b924-d773fd6e4373"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.groupby(\"target\").size()","metadata":{"id":"ovD2ZTAohwju","outputId":"3786a90e-6235-47b6-8c51-c2ddee2217c7"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Word distribution by length\nimport seaborn as sns\n\nlengths = df['question_text'].apply(lambda x: len(x.split()))\nsns.distplot(lengths)","metadata":{"id":"JOmbEWoBE9aA","outputId":"854606c8-4ecc-42e2-bbe0-19d25782e62b"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to show word distribution by frequency\nfrom nltk.probability import FreqDist\n\ndef show_words_freq_plot(frame):\n    def get_token(fr):\n        for st in fr:\n            for token in word_tokenize(st):\n                yield token\n    fd = FreqDist(get_token(frame))\n    fd.plot(30)","metadata":{"id":"F3mNV20Y_PES"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Word distribution by frequency for sincere questions\nif SHOW_PLOT:\n    show_words_freq_plot(df[df['target'] == 0]['question_text'])","metadata":{"id":"DzD0W8hvGkwG","outputId":"23c509cf-a0bc-4f1b-e4a3-7534afd1178e"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Word distribution by frequency for insincere questions\nif SHOW_PLOT:\n    show_words_freq_plot(df[df['target'] == 1]['question_text'])","metadata":{"id":"lwMoMRjqEq0C","outputId":"50ff1c84-caae-4111-b5d5-abe0a99ee359"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data preprocessing","metadata":{"id":"qIa1DbjJixLh"}},{"cell_type":"code","source":"stop_words = list(stopwords.words('english'))\n\n\ndef preprocess_text(text):\n    # Remove icon\n    icon_re = re.compile(\"[\"\n        u\"\\U0001F600-\\U0001F64F\"  # emoticons\n        u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n        u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n        u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n        u\"\\U00002500-\\U00002BEF\"  # chinese char\n        u\"\\U00002702-\\U000027B0\"\n        u\"\\U00002702-\\U000027B0\"\n        u\"\\U000024C2-\\U0001F251\"\n        u\"\\U0001f926-\\U0001f937\"\n        u\"\\U00010000-\\U0010ffff\"\n        u\"\\u2640-\\u2642\" \n        u\"\\u2600-\\u2B55\"\n        u\"\\u200d\"\n        u\"\\u23cf\"\n        u\"\\u23e9\"\n        u\"\\u231a\"\n        u\"\\ufe0f\"  # dingbats\n        u\"\\u3030\"\n                      \"]+\", re.UNICODE)\n    text = icon_re.sub(r'', text)\n\n    # Remove punctuation\n    punctuation = string.punctuation.replace(\"-\", \"\").replace(\"'\", \"\")\n    text = \"\".join([i if i not in punctuation else \" \" for i in text])\n\n    # Remove number\n    text = re.sub(r'\\d+', '', text)\n\n    # Lowercase text\n    text = text.lower()\n\n    # Remove duplicated characters\n    text = re.sub(r'(.)\\1{2,}', r'\\1\\1', text)\n\n    # Remove \\n and strip\n    text = text.replace(\"\\n\", \" \").strip()\n\n    # Remove double space\n    text = re.sub(\" +\", \" \", text)\n\n    # Remove stopwords\n    word_token = word_tokenize(text)\n    filtered = [w for w in word_token if not w in stop_words]\n    text = \" \".join(filtered)\n\n    # Return\n    return text","metadata":{"id":"X7pdS48giz3N"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['preprocessed'] = df['question_text'].apply(lambda x: preprocess_text(x))\ndf.head()","metadata":{"id":"l_FyQJBajqJs","outputId":"538f96c9-9e20-410f-fa2f-5150c0f0055c"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Word distribution by frequency for sincere questions\nif SHOW_PLOT:\n    show_words_freq_plot(df[df['target'] == 0].astype('str')['preprocessed'])","metadata":{"id":"ZnzpqxWPEzmb","outputId":"25637666-4ff4-403c-f060-c14986f972cb"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Word distribution by frequency for insincere questions\nif SHOW_PLOT:\n    show_words_freq_plot(df[df['target'] == 1].astype('str')['preprocessed'])","metadata":{"id":"adHiBUSXH21J","outputId":"3e41ab7f-02af-477a-93f9-a8381f9e88a9"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Word distribution by length\nimport seaborn as sns\n\nlengths = df['preprocessed'].astype('str').apply(lambda x: len(x.split()))\nsns.distplot(lengths)","metadata":{"id":"5g5AdHg1LHe9","outputId":"81d1c1f7-a49b-4b2d-b88e-dd0c636e54e6"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not KAGGLE:\n    df.to_csv(os.path.join(DATA_DIR, 'preprocessed.csv'), columns=['qid', 'preprocessed', 'target'], index=False)","metadata":{"id":"gDyJJzJGl1aL"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{"id":"IA9_HNEYmOlK"}},{"cell_type":"code","source":"if not KAGGLE:\n    df = pd.read_csv(os.path.join(DATA_DIR, 'preprocessed.csv'))\ndf.head()","metadata":{"id":"S877evrOmVqy","outputId":"b1f4f758-1599-407d-9a54-50850ecfee48"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_data = df['preprocessed'].astype('str')\ny_data = df['target']\nprint(X_data.shape)\nprint(y_data.shape)","metadata":{"id":"DCDIf2QCPDBn","outputId":"1faf4def-fcda-4745-cbc0-4ceb0af7114a"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Undersampling (ratio: 1:1)\n\ndata_one = df[df['target'] == 1]\ndata_zero = df[df['target'] == 0]\ndata_zero_new = data_zero.sample(data_one.shape[0])\ndata = pd.concat([data_one, data_zero_new])\n\nX_data = data['preprocessed'].astype('str')\ny_data = data['target']\nprint(X_data.shape)\nprint(y_data.shape)","metadata":{"id":"AjYeOsKHQZjk","outputId":"2ce4a54e-0db3-404f-c1b5-19aa41df20bf"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Tokenize all dataset\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\n\nmaxlen = 100  # Max word count in sentences\n\ntokenizer = Tokenizer(num_words=100000)\ntokenizer.fit_on_texts(X_data)\nsequences = tokenizer.texts_to_sequences(X_data)\nX_data_padded = pad_sequences(sequences, maxlen=maxlen)\n\nword_index = tokenizer.word_index","metadata":{"id":"mRRPfKW2m7Wc"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Split train-validation dataset\nfrom sklearn.model_selection import train_test_split\n\nX_train, X_test, y_train, y_test = train_test_split(X_data_padded, y_data, test_size=0.2, stratify=y_data, random_state=2)","metadata":{"id":"kTNN1IouX21r"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prepare embedding layer from word embedding\n# (develop from this tutorial: https://blog.keras.io/using-pre-trained-word-embeddings-in-a-keras-model.html)\nword_path = os.path.join(DATA_DIR, \"glove.840B.300d/glove.840B.300d.txt\")\n\nembeddings_index = {}\nwith open(word_path, 'r') as f:\n    for line in f:\n        values = line.split()\n        word = \" \".join(values[:-300])\n        coefs = np.array(values[-300:], dtype='float32')\n        embeddings_index[word] = coefs\n\nembedding_matrix = np.zeros((len(word_index) + 1, 300))\nfor word, i in word_index.items():\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None:\n        embedding_matrix[i] = embedding_vector\n\nfrom keras.layers import Embedding\nembedding_layer = Embedding(len(embedding_matrix), 300, weights=[embedding_matrix], trainable=False, input_shape=(maxlen,))","metadata":{"id":"n2G53f5QQ_tG"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Build model\nfrom keras.layers import Dense, Input, LSTM, Bidirectional\nfrom keras.models import Model\nimport tensorflow_addons as tfa\n\nipt = Input(shape=(maxlen,), dtype='int32')\nx = embedding_layer(ipt)\nx = Bidirectional(LSTM(128, return_sequences=True))(x)\nx = Bidirectional(LSTM(64))(x)\nx = Dense(64, activation='relu')(x)\nopt = Dense(1, activation='sigmoid')(x)\n\n\nf1 = tfa.metrics.F1Score(num_classes=1, threshold=0.5)  # Use f1-score for training with imbalanced dataset\n\nmodel = Model(ipt, opt)\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=[f1, 'accuracy'])\nmodel.summary()","metadata":{"id":"yR5DzNp0WNw2","outputId":"3244eb47-3aa0-4a83-e2d3-5bfccc816111"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Start training with some earlystopping and reduceLR\nfrom keras.callbacks import EarlyStopping, ReduceLROnPlateau\n\nes = EarlyStopping(monitor='val_loss', patience=3, mode='min', restore_best_weights=True)\nrl = ReduceLROnPlateau(monitor='val_loss', patience=2, factor=0.5, mode='auto')\n\nhistory = model.fit(X_train, y_train, batch_size=128, epochs=10, validation_data=(X_test, y_test), callbacks=[es, rl])","metadata":{"id":"RN7VzZWcXiC9","outputId":"6bf2001d-58e7-4fec-d711-3190a2e49293"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Graph of model loss\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'val'])\nplt.show()","metadata":{"id":"g3NKoW-dNp9P","outputId":"5c1a1cf0-9ec2-4eb9-e861-5239f4497f91"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Graph of training accuracy\nplt.plot(history.history['accuracy'])\nplt.plot(history.history['val_accuracy'])\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'val'])\nplt.show()","metadata":{"id":"3JMIZkVRTENx","outputId":"35325039-1044-4772-8944-1246041a74c8"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Graph of f1-score\nplt.plot(history.history['f1_score'])\nplt.plot(history.history['val_f1_score'])\nplt.title('model accuracy')\nplt.ylabel('f1_score')\nplt.xlabel('epoch')\nplt.legend(['train', 'val'])\nplt.show()","metadata":{"id":"gkDh0_pkk_yV","outputId":"5a670bd7-b557-4d27-a022-c6cce1cf59fb"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission\n\n","metadata":{"id":"VNJVCUmRhviK"}},{"cell_type":"code","source":"test_data_path = os.path.join(DATA_DIR, \"test.csv\")\n\ndf_test = pd.read_csv(test_data_path)\ndf_test.head()","metadata":{"id":"6HaT0Pm2hxe9","outputId":"4c0c84aa-d893-4cd9-9f82-7b1fa79a28a1"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"t_preprocessed = df_test['question_text'].apply(lambda x: preprocess_text(x))\nt_sequences = tokenizer.texts_to_sequences(t_preprocessed)\nt_data = pad_sequences(t_sequences, maxlen=maxlen)\n\nt_pred = model.predict(t_data).round().astype('int32').flatten()","metadata":{"id":"0yRiLQIUif8p"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit_path = \"submission.csv\" if KAGGLE else os.path.join(DATA_DIR, \"submission.csv\")\nsm_df = pd.DataFrame({\"qid\": df_test['qid'], \"prediction\": t_pred})\nsm_df.to_csv(submit_path, index=False)\nsm_df.head()","metadata":{"id":"c1V_ZAN5xJll","outputId":"da15f8c6-af50-43a4-d414-6c45f4703dbf"},"execution_count":null,"outputs":[]}]}