{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# TF Imports\nfrom tensorflow.python.keras import backend as K\nfrom tensorflow.python.keras.models import Sequential\nfrom tensorflow.python.keras.layers import Dense, CuDNNLSTM, CuDNNGRU, Dropout, Bidirectional, Conv1D, Input\nfrom tensorflow.python.keras.models import Model\nfrom tensorflow.python.keras.layers import SpatialDropout1D, GlobalAveragePooling1D, GlobalMaxPooling1D, concatenate\nfrom tensorflow.python.keras.layers.embeddings import Embedding\nfrom tensorflow.python.keras.preprocessing import sequence\nfrom tensorflow.python.keras.wrappers.scikit_learn import KerasClassifier\nfrom tensorflow.python.keras.optimizers import Adam\nfrom tensorflow.python.keras.callbacks import EarlyStopping, LearningRateScheduler\n# Numpy\nimport numpy\nnumpy.random.seed(1331)\n# Pandas\nimport pandas as pd\n# Sklearn\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import f1_score\nfrom sklearn.model_selection import GridSearchCV, RandomizedSearchCV, StratifiedKFold, cross_val_score\n# Visualizations\nimport matplotlib.pyplot as plt\n%matplotlib inline\n# Garbage Collector\nimport gc\nimport sys\n# Hyperopt\nfrom hyperopt import fmin, tpe, hp, anneal, Trials, space_eval\n# Random\nimport random\n# codecs + collections + csv\nimport codecs\nimport collections\nimport csv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tf.__version__","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Tokenize\n\nFirst we need to read a good population of the data, in order to tokenize the data"},{"metadata":{"trusted":true},"cell_type":"code","source":"# load training data\nTRAIN_DATA_FILE = \"../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv\"\nj_df = pd.read_csv(TRAIN_DATA_FILE)\n\n# Parameter settings\nmaxlen = 220\nmax_features = 200000\n\n# create a tokenizer\ntoken = tf.keras.preprocessing.text.Tokenizer(num_words=max_features)\n# fit tokenizer on data\ntoken.fit_on_texts(j_df['comment_text'])\n# get word index from tokenizer\nword_index = token.word_index\n\n# Memory Clean-up\ndel j_df\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Validation"},{"metadata":{"trusted":true},"cell_type":"code","source":"# load validation data\nVAL_DATA_FILE = \"../input/jigsaw-multilingual-toxic-test-translated/jigsaw_miltilingual_valid_translated.csv\"\nj_df = pd.read_csv(VAL_DATA_FILE)\nj_df.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nX_val = sequence.pad_sequences(token.texts_to_sequences(j_df['translated']), maxlen=maxlen)\ny_val = j_df['toxic']\nprint(X_val.shape)\n\n# Memory Clean-up\ndel j_df\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Embedding"},{"metadata":{"trusted":true},"cell_type":"code","source":"embed_size = 100\n\nEMBEDDING_FILES = [\n    '../input/jigsaw-custom-word2vec-100d-5iter/custom_word2vec_100d_5iter.txt'\n]\n\ndef get_coefs(word, *arr):\n    return word, np.asarray(arr, dtype='float32')\n\ndef load_embeddings(path):\n    with open(path) as f:\n        return dict(get_coefs(*line.strip().split(' ')) for line in f)\n\ndef build_matrix(word_index, path):\n    embedding_index = load_embeddings(path)\n    embedding_matrix = np.zeros((len(word_index) + 1, 100))\n    for word, i in word_index.items():\n        try:\n            embedding_matrix[i] = embedding_index[word]\n        except KeyError:\n            pass\n    return embedding_matrix\n\nembedding_matrix = np.concatenate([build_matrix(token.word_index, f) for f in EMBEDDING_FILES], axis=-1)\nembedding_matrix = embedding_matrix[0:max_features,:]\nembedding_matrix.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Load & Padding"},{"metadata":{"trusted":true},"cell_type":"code","source":"def sent_generator(TRAIN_DATA_FILE, chunksize, threshold, maxlen):\n    reader = pd.read_csv(TRAIN_DATA_FILE, chunksize=chunksize, iterator=True)\n    for df in reader:\n        texts = df.iloc[:,1].astype(str)\n        target = np.where(df.iloc[:,2]>threshold,1,0)\n        sequences = token.texts_to_sequences(texts)\n        data_train = sequence.pad_sequences(sequences, maxlen=maxlen)\n        yield data_train, target","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def test_data_prep():\n    # load test data\n    j_df = pd.read_csv(\"../input/jigsaw-multilingual-toxic-test-translated/jigsaw_miltilingual_test_translated.csv\")\n    X_test = j_df['translated'].astype(str)\n    del j_df\n    gc.collect()\n    X_test = sequence.pad_sequences(token.texts_to_sequences(X_test), maxlen=maxlen)\n    return X_test\n\nX_test = test_data_prep()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Create & Compile Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Results from Hyperopt\ndrpt_amt = 0.30\nlstm2_nrns = 23\nepochs = 1\nbatches = 641","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model():\n    # create model\n    model = Sequential()\n    model.add(Embedding(max_features, embed_size, input_length=maxlen, weights=[embedding_matrix], trainable=False))\n    model.add(Dropout(drpt_amt))\n    model.add(Bidirectional(CuDNNLSTM(lstm2_nrns)))\n    model.add(Dropout(drpt_amt))\n    model.add(Dense(1, activation='sigmoid'))\n    # Compile model\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"checkpoint_predictions = []\nweights = []\nnb_epoch = epochs\nbatch_size = batches\nthreshold = 0.48\nfor model_idx in range(2):\n    model = build_model()\n    n_steps = (1209267) // batch_size\n    threshold += 0.02\n    for counter in range(nb_epoch):\n        print('-------epoch: ',counter,'--------')\n        scheduler = lambda _: 1e-3 * (0.55 ** counter)\n        callback = tf.keras.callbacks.LearningRateScheduler(scheduler)\n        model.fit_generator(sent_generator(TRAIN_DATA_FILE, batch_size, threshold, 220),\n                            steps_per_epoch=n_steps, \n                            epochs=3, \n                            validation_data=(X_val, y_val),\n                            callbacks=[callback])\n        prediction = model.predict_proba(X_test).flatten()\n        checkpoint_predictions.append(prediction)\n        weights.append(2 ** counter)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Score"},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = np.average(checkpoint_predictions, weights=weights, axis=0)\n\ntest_df = pd.read_csv(\"../input/jigsaw-multilingual-toxic-test-translated/jigsaw_miltilingual_test_translated.csv\")\nsubmission = pd.DataFrame.from_dict({\n    'id': test_df.id,\n    'toxic': predictions\n})\nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}