{"cells":[{"metadata":{},"cell_type":"markdown","source":"This notebook makes use of the translated/cleaned dataset: https://www.kaggle.com/kerneler/starter-jigsaw-toxic-comment-classific-e0420f1a-7. Let's see how far we can get just using Tensorflow 2 and Keras Preprocessing Layers, without bringing in more heavy-duty stuff like BERT.","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train= pd.read_csv(\"../input/jigsaw-toxic-comment-classification-cleaned-data/train_data.csv\")\nval = pd.read_csv(\"../input/jigsaw-toxic-comment-classification-cleaned-data/val_data.csv\")\nprint(len(train), len(val))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.columns.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val.columns.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"translated_test = pd.read_csv(\"../input/jigsaw-toxic-comment-classification-cleaned-data/test_data.csv\")\ntest = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/test.csv\")\nprint(len(translated_test), len(test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"translated_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"There are some values in the translated test set that are null, so I'm going to set those to a \"dummy\" value so that the model doesn't mess up. And, I'll remove null values from the training set.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"val.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dummy = train.cleaned_text.values[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"translated_test[pd.isnull(translated_test.cleaned_text)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"translated_test.cleaned_text[pd.isnull(translated_test.cleaned_text)] = dummy\ntranslated_test[pd.isnull(translated_test.cleaned_text)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train[pd.notnull(train.cleaned_text)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train[train.toxic == 0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train[train.toxic == 1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_train = pd.concat((train[train.toxic == 1], train[train.toxic == 0].sample(100000)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(new_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\n\nvocab_size = 50000\nmax_length = 192","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_vals = train[['toxic']]\nval_vals = val[['toxic']]\ntrain_vals","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(val_vals)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Instead of tokenizing the text manually, I'm going use the keras text vectorization layer. This way, we can feed the text directly into the model. For more information, see https://www.tensorflow.org/api_docs/python/tf/keras/layers/experimental/preprocessing/TextVectorization","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.layers.experimental.preprocessing import TextVectorization\n\nvectorize_layer = TextVectorization(\n max_tokens=vocab_size,\n output_mode='int',\n output_sequence_length=max_length)\n\nvectorize_layer.adapt(np.concatenate((train.cleaned_text.values, val.cleaned_text.values, translated_test.cleaned_text.values)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from tensorflow.keras.models import Sequential\n# from tensorflow.keras.layers import Embedding, LSTM, Dense, Bidirectional, Dropout, GRU, GlobalMaxPooling1D\n# from tensorflow.keras.optimizers import Adam\n# from tensorflow.keras import Input\n\n# loss = tf.keras.losses.BinaryCrossentropy(from_logits=True)\n\n# def create_model():\n#     model = Sequential()\n#     model.add(tf.keras.Input(shape=(1,), dtype=tf.string))\n#     model.add(vectorize_layer)\n#     model.add(Embedding(vocab_size + 1, 64, input_length = max_length))\n#     model.add(Bidirectional(LSTM(20, return_sequences = True)))\n#     model.add(Bidirectional(LSTM(20, return_sequences = True)))\n#     model.add(GlobalMaxPooling1D())\n#     model.add(Dense(1, activation='sigmoid'))\n#     model.compile(loss='binary_crossentropy', optimizer = tf.keras.optimizers.Adam(0.00005), metrics=['accuracy'])\n#     return model\n# #model.summary()\n# model = create_model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# history = model.fit(new_train.cleaned_text.values[::10],new_train.toxic.values[::10], epochs = 10, verbose = 1, \n#                     validation_data = (val.cleaned_text.values[::10], val_vals.values[::10]),\n#                    callbacks = [tf.keras.callbacks.EarlyStopping(patience = 3)])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"I'm going to use KerasTuner to find the optimal model for this data. For more information, visit: https://keras-team.github.io/keras-tuner/","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -U keras-tuner","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Embedding, LSTM, Dense, Bidirectional, Dropout, GRU, GlobalMaxPooling1D\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras import Input\nimport kerastuner as kt\n\nloss = tf.keras.losses.BinaryCrossentropy(from_logits=True)\n\ndef create_model(hp):\n    model = Sequential()\n    model.add(tf.keras.Input(shape=(1,), dtype=tf.string))\n    model.add(vectorize_layer)\n    model.add(Embedding(vocab_size + 1, hp.Int('units', min_value = 5, max_value = 200, step = 25), input_length = max_length))\n#     model.add(tf.keras.layers.Conv1D(hp.Int('units', min_value = 5, max_value = 200, step = 25), 5, activation='relu'))\n#     model.add(tf.keras.layers.GlobalMaxPooling1D())\n    model.add(Bidirectional(LSTM(hp.Int('units', min_value = 5, max_value = 200, step = 25), return_sequences = True)))\n    model.add(Bidirectional(LSTM(hp.Int('units', min_value = 5, max_value = 200, step = 25), return_sequences = True)))\n    model.add(GlobalMaxPooling1D())\n    model.add(Dense(1, activation='sigmoid'))\n    hp_learning_rate = hp.Choice('learning_rate', values = [1e-4, 5e-5, 1e-5]) \n    model.compile(loss='binary_crossentropy', optimizer = tf.keras.optimizers.Adam(hp_learning_rate), metrics=['accuracy'])\n    return model\n#model.summary()\n\ntuner = kt.Hyperband(create_model,\n                     objective = 'val_accuracy', \n                     max_epochs = 15,\n                     factor = 3)     \n\ntuner.search(new_train.cleaned_text.values[::100],new_train.toxic.values[::100], epochs = 10,verbose = 2,\n             validation_data = (val.cleaned_text.values[::100], val_vals.values[::100]), callbacks=[tf.keras.callbacks.EarlyStopping('val_loss', patience=3)])\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now, we can get the best model from KerasTuner and train it on our data.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"best_hps = tuner.get_best_hyperparameters(num_trials = 1)[0]\nmodel = tuner.hypermodel.build(best_hps)\n\nhistory = model.fit(new_train.cleaned_text.values,new_train.toxic.values, epochs = 20, verbose = 2, validation_data = (val.cleaned_text.values, val_vals.values),\n                   callbacks = [tf.keras.callbacks.EarlyStopping(patience = 3)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(val.cleaned_text.values, val_vals.values,epochs = 7, verbose = 2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\ndef plot_graphs(history, string):\n  plt.plot(history.history[string])\n  plt.xlabel(\"Epochs\")\n  plt.ylabel(string)\n  plt.show()\n\nplot_graphs(history, 'val_accuracy')\nplot_graphs(history, 'loss')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now, we can predict the values for the test set.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"translated_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_toxic = model.predict(translated_test.cleaned_text.values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"evaluation = translated_test.id.copy().to_frame()\nevaluation['toxic'] = np.round(test_toxic)\nevaluation","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"evaluation.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}