{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Extract a zip file\nimport zipfile\nzip_ref = zipfile.ZipFile('/kaggle/input/quora-insincere-questions-classification/embeddings.zip', 'r')\nprint(zip_ref.namelist())\nembeddings = zip_ref.open('glove.840B.300d/glove.840B.300d.txt', 'r')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.decode().split(\" \")) for o in embeddings)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Stopwords list from https://github.com/Yoast/YoastSEO.js/blob/develop/src/config/stopwords.js","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"stopwords = [ \"a\", \"about\", \"above\", \"after\", \"again\", \"against\", \"all\", \"am\", \"an\", \"and\", \"any\", \"are\", \"as\", \"at\", \"be\", \"because\", \"been\", \"before\", \"being\", \"below\", \"between\", \"both\", \"but\", \"by\", \"could\", \"did\", \"do\", \"does\", \"doing\", \"down\", \"during\", \"each\", \"few\", \"for\", \"from\", \"further\", \"had\", \"has\", \"have\", \"having\", \"he\", \"he'd\", \"he'll\", \"he's\", \"her\", \"here\", \"here's\", \"hers\", \"herself\", \"him\", \"himself\", \"his\", \"how\", \"how's\", \"i\", \"i'd\", \"i'll\", \"i'm\", \"i've\", \"if\", \"in\", \"into\", \"is\", \"it\", \"it's\", \"its\", \"itself\", \"let's\", \"me\", \"more\", \"most\", \"my\", \"myself\", \"nor\", \"of\", \"on\", \"once\", \"only\", \"or\", \"other\", \"ought\", \"our\", \"ours\", \"ourselves\", \"out\", \"over\", \"own\", \"same\", \"she\", \"she'd\", \"she'll\", \"she's\", \"should\", \"so\", \"some\", \"such\", \"than\", \"that\", \"that's\", \"the\", \"their\", \"theirs\", \"them\", \"themselves\", \"then\", \"there\", \"there's\", \"these\", \"they\", \"they'd\", \"they'll\", \"they're\", \"they've\", \"this\", \"those\", \"through\", \"to\", \"too\", \"under\", \"until\", \"up\", \"very\", \"was\", \"we\", \"we'd\", \"we'll\", \"we're\", \"we've\", \"were\", \"what\", \"what's\", \"when\", \"when's\", \"where\", \"where's\", \"which\", \"while\", \"who\", \"who's\", \"whom\", \"why\", \"why's\", \"with\", \"would\", \"you\", \"you'd\", \"you'll\", \"you're\", \"you've\", \"your\", \"yours\", \"yourself\", \"yourselves\" ]\n\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/train.csv')\ntest_data = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_data.columns)\ntrain_input = list(train_data['question_text'])\ntrain_label = list(train_data['target'])\n\ntest_input = list(test_data['question_text'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Remove stop words from the data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def remove_stop_words(x):\n    for word in stopwords:\n        token = \" \" + word + \" \"\n        if (x.find(token) != -1):\n            x = x.replace(token, \" \")\n    return x\n\ntrain_input_rsw = list(map(remove_stop_words, train_input))\ntest_input_rsw = list(map(remove_stop_words, test_input))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Create a embedding matrix using embedding imported and words in train data.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenizer = Tokenizer()\ntokenizer.fit_on_texts(train_input_rsw)\nword_index = tokenizer.word_index\n\nembedding_matrix = np.zeros((len(word_index)+1, 300))\n\nfor word, index in word_index.items():\n    embedding_vector = embeddings_index.get(word)\n    if (embedding_vector is not None):\n        embedding_matrix[index] = embedding_vector\n        \nprint(embedding_matrix.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del embeddings_index\nimport gc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"max_length = 30","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Convert text to numbers and pad them for processing","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sequences = tokenizer.texts_to_sequences(train_input_rsw)\ntrain_input_padded = pad_sequences(sequences, maxlen=max_length, padding='post', truncating='post')\nprint(train_input_padded.shape)\n\nsequences = tokenizer.texts_to_sequences(test_input_rsw)\ntest_input_padded = pad_sequences(sequences, maxlen=max_length, padding='post', truncating='post')\nprint(test_input_padded.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"has_cv to control the split of train data, if false model will train on whole data else splits into 9:1 ratio and train's on 90 of the data. ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"has_cv = False\ntest_split_size = 0.1 if has_cv else 0\n\n# Train split\nif (has_cv) :\n    train_text, cv_text, train_target, cv_target = train_test_split(train_input_padded, train_label, test_size = test_split_size, random_state=2)\nelse:\n    train_text = train_input_padded\n    train_target = train_label\n    \n\nprint(f'Train Input Shape : {len(train_text)}')\nprint(f'Train Label Shape : {len(train_target)}')\nif (has_cv) :\n    print(f'CV Input Shape : {len(cv_text)}')\n    print(f'CV label Shape : {len(cv_target)}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tf.random.set_seed(123)\n\nmodel = tf.keras.models.Sequential([\n    tf.keras.layers.Embedding(len(word_index)+1, 300, input_length=max_length, weights=[embedding_matrix], trainable=False),\n    tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(128,return_sequences=True)),\n    tf.keras.layers.Dropout(0.2),\n    tf.keras.layers.Conv1D(64, 5, activation='relu'),\n    tf.keras.layers.MaxPooling1D(pool_size=4),\n    tf.keras.layers.LSTM(64),\n    tf.keras.layers.Dropout(0.2),\n    tf.keras.layers.Dense(1, activation=\"sigmoid\")\n])\n\nmodel.summary()\n\nmodel.compile(loss=\"binary_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs = 5\nif (has_cv):\n    history = model.fit(np.array(train_text), np.array(train_target), epochs = epochs, validation_data=(np.array(cv_text),np.array(cv_target)), batch_size=1024)\nelse:\n    history = model.fit(np.array(train_text), np.array(train_target), epochs = epochs, batch_size=1024)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"When trained model on 90% of data, best threshold achieved is 0.29. So used it for the full train data model.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"if (has_cv):\n    # calculate F1 Score\n    from sklearn.metrics import f1_score\n    cv_predictions = model.predict(cv_text, batch_size=1024)\n\n    thresholds = []\n    for thresh in np.arange(0.1, 0.501, 0.01):\n        thresh = np.round(thresh, 2)\n        result = f1_score(cv_target, (cv_predictions>thresh).astype(int))\n        thresholds.append([thresh, result])\n        print(\"F1 score at threshold {0} is {1}\".format(thresh, result))\n\n    thresholds.sort(key=lambda x: x[1], reverse=True)\n    print(\"Best value {0}\".format(thresholds[0]))\n    best_thresh = thresholds[0]\nelse:\n    best_thresh = 0.29","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"To Analyse training","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":" if (has_cv):\n    import matplotlib.image  as mpimg\n    import matplotlib.pyplot as plt\n\n    #-----------------------------------------------------------\n    # Retrieve a list of list results on training and test data\n    # sets for each training epoch\n    #-----------------------------------------------------------\n    acc=history.history['accuracy']\n    val_acc=history.history['val_accuracy']\n    loss=history.history['loss']\n    val_loss=history.history['val_loss']\n\n    epochs=range(len(acc)) # Get number of epochs\n\n    #------------------------------------------------\n    # Plot training and validation accuracy per epoch\n    #------------------------------------------------\n    plt.plot(epochs, acc, 'r')\n    plt.plot(epochs, val_acc, 'b')\n    plt.title('Training and validation accuracy')\n    plt.xlabel(\"Epochs\")\n    plt.ylabel(\"Accuracy\")\n    plt.legend([\"Accuracy\", \"Validation Accuracy\"])\n\n    plt.figure()\n\n    #------------------------------------------------\n    # Plot training and validation loss per epoch\n    #------------------------------------------------\n    plt.plot(epochs, loss, 'r')\n    plt.plot(epochs, val_loss, 'b')\n    plt.title('Training and validation loss')\n    plt.xlabel(\"Epochs\")\n    plt.ylabel(\"Loss\")\n    plt.legend([\"Loss\", \"Validation Loss\"])\n\n    plt.figure()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"To print confusion matrix for the cv","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"if (has_cv):\n    predictions = model.predict(cv_text)\n    predictions = np.around(predictions).astype(int)\n    df = pd.DataFrame({'pred': predictions.flatten(), 'actual': cv_target})\n    df.head()\n    pd.crosstab(df['pred'], df['actual'], margins=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"predictions = model.predict(test_input_padded)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = (predictions>best_thresh).astype(int)\n\noutput = pd.DataFrame({'qid': test_data.qid, 'prediction': predictions.flatten()})\n\noutput.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}