{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport tensorflow as tf\nprint(tf.__version__)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-01T20:27:55.986047Z","iopub.execute_input":"2021-11-01T20:27:55.986400Z","iopub.status.idle":"2021-11-01T20:27:58.750378Z","shell.execute_reply.started":"2021-11-01T20:27:55.986310Z","shell.execute_reply":"2021-11-01T20:27:58.748156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset Exploration","metadata":{}},{"cell_type":"code","source":"%matplotlib inline\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\ndf = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/train.csv')\ndf['target'].value_counts().plot.bar(title='Target')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-01T20:27:58.752249Z","iopub.execute_input":"2021-11-01T20:27:58.752510Z","iopub.status.idle":"2021-11-01T20:28:03.426501Z","shell.execute_reply.started":"2021-11-01T20:27:58.752458Z","shell.execute_reply":"2021-11-01T20:28:03.425774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%matplotlib inline\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\ndef summarize_data(corpus):\n    \"\"\"\n    print statements and visualizations to summarize the corpus\n    \"\"\"\n    \n    # get the documents size\n    df_doc_size = pd.Series([len(str(doc).split(\" \")) for doc in corpus])\n    \n    # get the tokens in the corpus\n    df_tokens = pd.Series([token for doc in corpus for token in str(doc).split(\" \")])\n    \n    print(\"---------------------------\")\n    print(\"num docs\", len(corpus))\n    print(\"median tokens\", df_doc_size.median())\n    print(\"num tokens\", len(df_tokens))\n    print(\"unique tokens\", len(df_tokens.value_counts()))\n    print(\"---------------------------\")\n    \n    # make plots\n    fig = plt.figure(figsize=(14,6))\n    ax1 = fig.add_subplot(121)\n    ax2 = fig.add_subplot(122)\n    \n    df_doc_size.plot.hist(ax=ax1, title='Document Sizes')\n    df_tokens.value_counts().plot.hist(ax=ax2, title='Tokens Counts')\n    \nsummarize_data(df.question_text.values.tolist())","metadata":{"execution":{"iopub.status.busy":"2021-11-01T20:28:03.428202Z","iopub.execute_input":"2021-11-01T20:28:03.428779Z","iopub.status.idle":"2021-11-01T20:28:16.773432Z","shell.execute_reply.started":"2021-11-01T20:28:03.428737Z","shell.execute_reply":"2021-11-01T20:28:16.772719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import shutil\nfrom sklearn.model_selection import train_test_split\n\ntrain_set, valid_set = train_test_split(df, test_size=0.2, stratify=df.target)\n\nprint(train_set.shape)\nprint(valid_set.shape)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T20:28:16.775244Z","iopub.execute_input":"2021-11-01T20:28:16.775776Z","iopub.status.idle":"2021-11-01T20:28:18.162131Z","shell.execute_reply.started":"2021-11-01T20:28:16.775735Z","shell.execute_reply":"2021-11-01T20:28:18.161314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# delete temp dir\nif os.path.exists('/kaggle/temp/'):\n    shutil.rmtree('/kaggle/temp/')\n\nos.mkdir(\"/kaggle/temp/\")\n\ntrain_path = \"/kaggle/temp/train.csv\"\nvalid_path = \"/kaggle/temp/valid.csv\" \n\ntrain_set.to_csv(train_path, index=False)\nvalid_set.to_csv(valid_path, index=False)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T20:28:18.164910Z","iopub.execute_input":"2021-11-01T20:28:18.165492Z","iopub.status.idle":"2021-11-01T20:28:25.468671Z","shell.execute_reply.started":"2021-11-01T20:28:18.165448Z","shell.execute_reply":"2021-11-01T20:28:25.467929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# keras Text preprocessing with Tokenizer","metadata":{}},{"cell_type":"code","source":"train_sentences = train_set.question_text.values.tolist()\ntrain_labels = train_set.target\n\nvalid_sentences = valid_set.question_text.values.tolist()\nvalid_labels = valid_set.target\n\ntrain_sentences[:5]","metadata":{"execution":{"iopub.status.busy":"2021-11-01T20:28:25.469904Z","iopub.execute_input":"2021-11-01T20:28:25.470203Z","iopub.status.idle":"2021-11-01T20:28:25.531601Z","shell.execute_reply.started":"2021-11-01T20:28:25.470165Z","shell.execute_reply":"2021-11-01T20:28:25.530789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\n\nsequence_length = 50\ntrunc_type='post'\npadding_type='post'\noov_tok = \"<OOV>\"\nvocab_size = 100000\n\ntokenizer = Tokenizer(num_words=vocab_size, oov_token=oov_tok)\ntokenizer.fit_on_texts(train_sentences)\nword_index = tokenizer.word_index\n\ntrain_sequences = tokenizer.texts_to_sequences(train_sentences)\ntrain_padded = pad_sequences(train_sequences, maxlen=sequence_length, padding=padding_type, truncating=trunc_type)\n\nvalid_sequences = tokenizer.texts_to_sequences(valid_sentences)\nvalid_padded = pad_sequences(valid_sequences, maxlen=sequence_length, padding=padding_type, truncating=trunc_type)\n\nprint(train_sentences[:4])\nprint(train_padded[:4])","metadata":{"execution":{"iopub.status.busy":"2021-11-01T20:28:25.532937Z","iopub.execute_input":"2021-11-01T20:28:25.533265Z","iopub.status.idle":"2021-11-01T20:29:15.380441Z","shell.execute_reply.started":"2021-11-01T20:28:25.533226Z","shell.execute_reply":"2021-11-01T20:29:15.379608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from collections import Counter\ntoken_sentences = tokenizer.sequences_to_texts(train_sequences)\nvocabulary = Counter()\n\nfor sentence in token_sentences:\n    vocabulary.update(sentence.split())","metadata":{"execution":{"iopub.status.busy":"2021-11-01T20:29:15.381691Z","iopub.execute_input":"2021-11-01T20:29:15.382194Z","iopub.status.idle":"2021-11-01T20:29:24.655915Z","shell.execute_reply.started":"2021-11-01T20:29:15.382150Z","shell.execute_reply":"2021-11-01T20:29:24.655156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vocab = [word for word, count in vocabulary.most_common()]\nlen(vocab)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T20:29:24.657779Z","iopub.execute_input":"2021-11-01T20:29:24.658366Z","iopub.status.idle":"2021-11-01T20:29:24.733423Z","shell.execute_reply.started":"2021-11-01T20:29:24.658325Z","shell.execute_reply":"2021-11-01T20:29:24.732715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# keras Data API and TextVectorization Layer","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\n\ntrain_ds = tf.data.experimental.CsvDataset(train_path, record_defaults=[\"\"] + [tf.constant([], dtype=tf.int32)], select_cols=[1, 2], header=True)\ntrain_ds = train_ds.shuffle(10000).batch(512).prefetch(1)\n\nvalid_ds = tf.data.experimental.CsvDataset(valid_path, record_defaults=[\"\"] + [tf.constant([], dtype=tf.int32)], select_cols=[1, 2], header=True)\nvalid_ds = valid_ds.batch(512).prefetch(1)\n\nfor X, y in train_ds.take(5):\n    print(X[0], y[0])","metadata":{"execution":{"iopub.status.busy":"2021-11-01T20:29:24.734663Z","iopub.execute_input":"2021-11-01T20:29:24.735360Z","iopub.status.idle":"2021-11-01T20:29:26.013408Z","shell.execute_reply.started":"2021-11-01T20:29:24.735315Z","shell.execute_reply":"2021-11-01T20:29:26.012629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vectorize_layer = tf.keras.layers.TextVectorization(max_tokens=vocab_size, output_mode='int', output_sequence_length=sequence_length)\ntext_ds = train_ds.map(lambda x, y: x)\nvectorize_layer.adapt(text_ds)\n\nmodel = tf.keras.models.Sequential()\nmodel.add(tf.keras.Input(shape=(1,), dtype=tf.string))\nmodel.add(vectorize_layer)\n\nfor X in text_ds.take(1):\n    print(X[0].numpy())\n    print(model.predict(X)[0])","metadata":{"execution":{"iopub.status.busy":"2021-11-01T20:29:26.014907Z","iopub.execute_input":"2021-11-01T20:29:26.015207Z","iopub.status.idle":"2021-11-01T20:29:36.614344Z","shell.execute_reply.started":"2021-11-01T20:29:26.015168Z","shell.execute_reply":"2021-11-01T20:29:36.613562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(vectorize_layer.get_vocabulary()))\nprint(vectorize_layer.get_vocabulary()[:10])","metadata":{"execution":{"iopub.status.busy":"2021-11-01T20:29:36.616042Z","iopub.execute_input":"2021-11-01T20:29:36.616563Z","iopub.status.idle":"2021-11-01T20:29:37.305370Z","shell.execute_reply.started":"2021-11-01T20:29:36.616521Z","shell.execute_reply":"2021-11-01T20:29:37.303444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create a classification model with pretrained Embeddings","metadata":{}},{"cell_type":"code","source":"import zipfile\nlocal_zip = \"/kaggle/input/quora-insincere-questions-classification/embeddings.zip\"\nzip_ref = zipfile.ZipFile(local_zip, 'r')\nzip_ref.extractall('/kaggle/temp/')\nzip_ref.close()","metadata":{"execution":{"iopub.status.busy":"2021-11-01T20:29:37.306937Z","iopub.execute_input":"2021-11-01T20:29:37.307235Z","iopub.status.idle":"2021-11-01T20:33:16.802267Z","shell.execute_reply.started":"2021-11-01T20:29:37.307195Z","shell.execute_reply":"2021-11-01T20:33:16.801424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load word embeddings\nembeddings_index = {}\nwith open('/kaggle/temp/glove.840B.300d/glove.840B.300d.txt') as f:\n    for line in f:\n        values = line.split()\n        word = values[0]\n        coefs = np.asarray(values[-300:], dtype='float32')\n        embeddings_index[word] = coefs","metadata":{"execution":{"iopub.status.busy":"2021-11-01T20:33:16.805304Z","iopub.execute_input":"2021-11-01T20:33:16.805744Z","iopub.status.idle":"2021-11-01T20:37:43.531353Z","shell.execute_reply.started":"2021-11-01T20:33:16.805704Z","shell.execute_reply":"2021-11-01T20:37:43.530601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embed_matrix = np.zeros((vocab_size, 300))\nfor idx, word in enumerate(vectorize_layer.get_vocabulary()):\n    embed_vector = embeddings_index.get(word)\n    if embed_vector is not None:\n        embed_matrix[idx] = embed_vector\n        \nembed_matrix.shape","metadata":{"execution":{"iopub.status.busy":"2021-11-01T20:37:43.532768Z","iopub.execute_input":"2021-11-01T20:37:43.533058Z","iopub.status.idle":"2021-11-01T20:37:44.256537Z","shell.execute_reply.started":"2021-11-01T20:37:43.533021Z","shell.execute_reply":"2021-11-01T20:37:44.255775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = tf.keras.Sequential([\n    tf.keras.layers.Input(shape=(1,), dtype=tf.string),\n    vectorize_layer,\n    tf.keras.layers.Embedding(vocab_size, 300, input_length=sequence_length, weights=[embed_matrix], trainable=False, mask_zero=True),\n    tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32, return_sequences=True)),\n    tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32)),\n    tf.keras.layers.Dense(128, activation='relu'),\n    tf.keras.layers.Dropout(0.5),\n    tf.keras.layers.Dense(1, activation='sigmoid')\n])\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-11-01T20:37:44.258258Z","iopub.execute_input":"2021-11-01T20:37:44.258862Z","iopub.status.idle":"2021-11-01T20:37:47.559867Z","shell.execute_reply.started":"2021-11-01T20:37:44.258815Z","shell.execute_reply":"2021-11-01T20:37:47.559077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Calculate class weights","metadata":{}},{"cell_type":"code","source":"# Scaling by total/2 helps keep the loss to a similar magnitude.\n# The sum of the weights of all examples stays the same.\n\nneg = train_set.target.value_counts().loc[0]\npos = train_set.target.value_counts().loc[1]\ntotal = train_set.shape[0]\n\nweight_for_0 = (1 / neg) * (total / 2.0)\nweight_for_1 = (1 / pos) * (total / 2.0)\n\nclass_weight = {0: weight_for_0, 1: weight_for_1}\nclass_weight","metadata":{"execution":{"iopub.status.busy":"2021-11-01T20:37:47.561107Z","iopub.execute_input":"2021-11-01T20:37:47.561458Z","iopub.status.idle":"2021-11-01T20:37:47.588392Z","shell.execute_reply.started":"2021-11-01T20:37:47.561418Z","shell.execute_reply":"2021-11-01T20:37:47.587765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"K = tf.keras.backend\nK.clear_session()\ntf.random.set_seed(42)\nnp.random.seed(42)\n\nMETRICS = [\n      tf.keras.metrics.BinaryAccuracy(name='accuracy'),\n      tf.keras.metrics.Precision(name='precision'),\n      tf.keras.metrics.Recall(name='recall'),\n      tf.keras.metrics.AUC(name='auc')\n]\n\nmodel.compile(loss='binary_crossentropy',optimizer='adam',metrics=METRICS, )\nearly_stopping_cb = tf.keras.callbacks.EarlyStopping(patience=5)\n\nhistory = model.fit(train_ds, epochs=30, validation_data=valid_ds, callbacks=[early_stopping_cb], class_weight=class_weight)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T20:37:47.589749Z","iopub.execute_input":"2021-11-01T20:37:47.590015Z","iopub.status.idle":"2021-11-01T20:56:00.703817Z","shell.execute_reply.started":"2021-11-01T20:37:47.589979Z","shell.execute_reply":"2021-11-01T20:56:00.702969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\n\ndef plot_cm(labels, predictions, p=0.5):\n    cm = confusion_matrix(labels, predictions > p)\n    plt.figure(figsize=(5,5))\n    sns.heatmap(cm, annot=True, fmt=\"d\")\n    plt.title('Confusion matrix @{:.2f}'.format(p))\n    plt.ylabel('Actual label')\n    plt.xlabel('Predicted label')\n    \nlabels = valid_set.target.values\npredictions = model.predict(valid_ds.map(lambda x, y: x))\nplot_cm(labels, predictions)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T20:56:00.705265Z","iopub.execute_input":"2021-11-01T20:56:00.705642Z","iopub.status.idle":"2021-11-01T20:57:10.464460Z","shell.execute_reply.started":"2021-11-01T20:56:00.705603Z","shell.execute_reply":"2021-11-01T20:57:10.463752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_auc=history.history['auc']\nvalid_auc=history.history['val_auc']\ntrain_loss=history.history['loss']\nvalid_loss=history.history['val_loss']\n\nepochs=range(len(train_auc)) # Get number of epochs\n\nplt.plot(epochs, train_auc, 'r')\nplt.plot(epochs, valid_auc, 'b')\nplt.title('Training and validation AUC')\nplt.xlabel(\"Epochs\")\nplt.ylabel(\"AUC\")\nplt.legend([\"Training AUC\", \"Validation AUC\"])\n\nplt.figure()\n\n#------------------------------------------------\n# Plot training and validation loss per epoch\n#------------------------------------------------\nplt.plot(epochs, train_loss, 'r')\nplt.plot(epochs, valid_loss, 'b')\nplt.title('Training and validation loss')\nplt.xlabel(\"Epochs\")\nplt.ylabel(\"Loss\")\nplt.legend([\"Loss\", \"Validation Loss\"])\nplt.figure()\n\n\n# Expected Output\n# A chart where the validation loss does not increase sharply!","metadata":{"execution":{"iopub.status.busy":"2021-11-01T20:57:10.466032Z","iopub.execute_input":"2021-11-01T20:57:10.466535Z","iopub.status.idle":"2021-11-01T20:57:10.957760Z","shell.execute_reply.started":"2021-11-01T20:57:10.466494Z","shell.execute_reply":"2021-11-01T20:57:10.957043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Make Predictions","metadata":{}},{"cell_type":"code","source":"test_path = '/kaggle/input/quora-insincere-questions-classification/test.csv'\ntest_ds = tf.data.experimental.CsvDataset(test_path, record_defaults=[\"\"], select_cols=[1], header=True).batch(512).prefetch(1)\ny_pred = model.predict(test_ds)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T20:57:10.959282Z","iopub.execute_input":"2021-11-01T20:57:10.959765Z","iopub.status.idle":"2021-11-01T20:57:25.579645Z","shell.execute_reply.started":"2021-11-01T20:57:10.959725Z","shell.execute_reply":"2021-11-01T20:57:25.578835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_set = pd.read_csv(test_path)\ntest_set['prediction'] = np.where(y_pred >= 0.5, 1, 0)\ntest_set[['qid', 'prediction']].to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T20:57:25.581254Z","iopub.execute_input":"2021-11-01T20:57:25.581519Z","iopub.status.idle":"2021-11-01T20:57:27.050050Z","shell.execute_reply.started":"2021-11-01T20:57:25.581473Z","shell.execute_reply":"2021-11-01T20:57:27.049169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.read_csv('submission.csv').head()","metadata":{"execution":{"iopub.status.busy":"2021-11-01T20:57:27.051946Z","iopub.execute_input":"2021-11-01T20:57:27.052415Z","iopub.status.idle":"2021-11-01T20:57:27.277584Z","shell.execute_reply.started":"2021-11-01T20:57:27.052374Z","shell.execute_reply":"2021-11-01T20:57:27.276837Z"},"trusted":true},"execution_count":null,"outputs":[]}]}