{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# import library\nimport tensorflow as tf \n\nfrom sklearn.preprocessing import MultiLabelBinarizer\nfrom sklearn.utils import shuffle\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # Detect hardware, return appropriate distribution strategy\n# try:\n#     # TPU detection. No parameters necessary if TPU_NAME environment variable is\n#     # set: this is always the case on Kaggle.\n#     tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n#     print('Running on TPU ', tpu.master())\n# except ValueError:\n#     tpu = None\n\n# if tpu:\n#     tf.config.experimental_connect_to_cluster(tpu)\n#     tf.tpu.experimental.initialize_tpu_system(tpu)\n#     strategy = tf.distribute.experimental.TPUStrategy(tpu)\n# else:\n#     # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n#     strategy = tf.distribute.get_strategy()\n\n# print(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#BATCH_SIZE = 18 * strategy.num_replicas_in_sync","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df1 = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\", usecols=[\"comment_text\", \"toxic\"])\ndf2 = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv\",  usecols=[\"comment_text\", \"toxic\"])\n#df3 = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/validation.csv\", usecols=[\"comment_text\", \"toxic\"])\ndf3 = pd.read_csv(\"../input/jigsaw-multilingual-toxic-test-translated/jigsaw_miltilingual_valid_translated.csv\", usecols=[\"translated\", \"toxic\"])\ndf2.toxic = df2.toxic.round().astype(int)\ndf3.toxic = df3.toxic.round().astype(int)\ndf3.rename(columns={\"translated\":\"comment_text\"}, inplace = True)\ndf_train = pd.concat([\n                      df1[[\"comment_text\", \"toxic\"]],\n                      df3[[\"comment_text\", \"toxic\"]],\n                      df2[[\"comment_text\", \"toxic\"]].query('toxic==1'),\n                      df2[[\"comment_text\", \"toxic\"]].query('toxic==0').sample(n=200000, random_state=0)],\n                      axis=0).reset_index(drop=True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = df_train.dropna()\n\ndf_train = shuffle(df_train, random_state=22)\ndf_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Training Shape:-\", df_train.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Split our data into train and test sets\ntrain_size = int(len(df_train) * .8)\nprint (\"Train size: %d\" % train_size)\nprint (\"Test size: %d\" % (len(df_train) - train_size))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Split our labels into train and test sets\nnum_labels =1\ntrain_toxic = df_train['toxic'].values[:train_size]\ntest_toxic = df_train['toxic'].values[train_size:]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%writefile preprocess.py\n# Pre-processing data: create our tokenizer class\nfrom tensorflow.keras.preprocessing import text\n\nclass TextPreprocessor(object):\n  def __init__(self, vocab_size):\n    self._vocab_size = vocab_size\n    self._tokenizer = None\n  \n  def create_tokenizer(self, text_list):\n    tokenizer = text.Tokenizer(num_words=self._vocab_size)\n    tokenizer.fit_on_texts(text_list)\n    self._tokenizer = tokenizer\n\n  def transform_text(self, text_list):\n    text_matrix = self._tokenizer.texts_to_matrix(text_list)\n    return text_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create vocab from training corpus\nfrom preprocess import TextPreprocessor\n\nVOCAB_SIZE=400 # This is a hyperparameter, try out different values for your dataset\n\ntrain_text = df_train['comment_text'].values[:train_size]\ntest_text = df_train['comment_text'].values[train_size:]\n\nprocessor = TextPreprocessor(VOCAB_SIZE)\nprocessor.create_tokenizer(train_text)\n\nbody_train = processor.transform_text(train_text) \nbody_test = processor.transform_text(test_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Save the processor state of the tokenizer\nimport pickle\nwith open('processor_state.pkl', 'wb') as f:\n  pickle.dump(processor, f)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_model(vocab_size, num_labels):\n  \n  model = tf.keras.models.Sequential()\n  model.add(tf.keras.layers.Dense(50, input_shape=(VOCAB_SIZE,), activation='relu'))\n  model.add(tf.keras.layers.Dense(25, activation='relu'))\n  model.add(tf.keras.layers.Dense(num_labels, activation='sigmoid'))\n\n  #model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n  model.compile(\n  optimizer=tf.keras.optimizers.Adam(1e-5),\n  loss=tf.keras.losses.BinaryCrossentropy(),#losses.SparseCategoricalCrossentropy(from_logits=True),\n  metrics=[tf.keras.metrics.AUC()]#metrics.SparseCategoricalAccuracy(name=\"acc\")]\n  )\n  return model\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# with strategy.scope():\n#     model = create_model(VOCAB_SIZE, num_labels)\n    \n# model.summary()\n\nmodel = create_model(VOCAB_SIZE, num_labels)\n    \n#model.summary()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#TPU_WORKER = 'grpc://10.0.0.2:8470'\n#tf.logging.set_verbosity(tf.logging.INFO)\n\n\nmirrored_strategy = tf.distribute.MirroredStrategy()\n\nwith mirrored_strategy.scope():\n    tpu_model = model\n\n# tpu_model = tf.distribute.cluster_resolver.contrib.tpu.keras_to_tpu_model(\n#     model,\n#     strategy=tf.contrib.tpu.TPUDistributionStrategy(\n#         tf.contrib.cluster_resolver.TPUClusterResolver(TPU_WORKER)))\n\n\ntpu_model.summary()\n\n\n\n# tf.config.experimental_connect_to_host('grpc://' + os.environ['COLAB_TPU_ADDR'])\n# resolver = tf.distribute.cluster_resolver.TPUClusterResolver('grpc://' + os.environ['COLAB_TPU_ADDR'])\n# tf.tpu.experimental.initialize_tpu_system(resolver)\n# strategy = tf.distribute.experimental.TPUStrategy(resolver) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nfrom tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau\n#n_steps = train_text.shape[0] // BATCH_SIZE\n\nearlyStopping = EarlyStopping(monitor='val_loss', patience=10, verbose=0, mode='min')\nmcp_save = ModelCheckpoint('.mdl_wts.hdf5', save_best_only=True, monitor='val_loss', mode='min')\nreduce_lr_loss = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=7, verbose=1, min_delta=1e-5, mode='min')\n\n\n# # Train and evaluate the model\n# history =model.fit(body_train, train_toxic, \n#           epochs=500,\n#           batch_size=21751,\n#           validation_split=0.1,\n#           callbacks=[earlyStopping, mcp_save, reduce_lr_loss]\n#          )\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# Train and evaluate the model\nhistory =tpu_model.fit(body_train, train_toxic, \n          epochs=1500,\n          batch_size=21751,\n          validation_split=0.1,\n          callbacks=[earlyStopping, mcp_save, reduce_lr_loss]\n         )\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nprint('Eval loss/accuracy:{}'.format(\n  model.evaluate(body_test, test_toxic, batch_size=128)))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(history.history['loss'], label='train loss')\nplt.plot(history.history['val_loss'], label='val loss')\nplt.xlabel(\"epoch\")\nplt.ylabel(\"Cross-entropy loss\")\nplt.legend();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df=pd.read_csv(\"../input/jigsaw-multilingual-toxic-test-translated/jigsaw_miltilingual_test_translated.csv\")\n\n\ntest_df.rename(columns={\"translated\":\"comment_text\"}, inplace = True)\ntest_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df_text=test_df['comment_text'].values\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv\")\nbody_submit = processor.transform_text(test_df_text) \npred = model.predict(body_submit)\nsample_submission['toxic'] =pred \nsample_submission.to_csv(\"submission.csv\", index=False)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}