{"cells":[{"metadata":{},"cell_type":"markdown","source":"LSTM for solving this Problem:https://www.kaggle.com/c/jigsaw-multilingual-toxic-comment-classification \nby-Queen Saikia","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"from tensorflow.python.keras import backend as K\nfrom tensorflow.python.keras.models import Sequential\nfrom tensorflow.python.keras.layers import Dense, CuDNNLSTM, CuDNNGRU, Dropout, Bidirectional, Conv1D, Input\nfrom tensorflow.python.keras.models import Model\nfrom tensorflow.python.keras.layers import SpatialDropout1D, GlobalAveragePooling1D, GlobalMaxPooling1D, concatenate\nfrom tensorflow.python.keras.layers.embeddings import Embedding\nfrom tensorflow.python.keras.preprocessing import sequence\nfrom tensorflow.python.keras.wrappers.scikit_learn import KerasClassifier\nfrom tensorflow.python.keras.optimizers import Adam\nfrom tensorflow.python.keras.callbacks import EarlyStopping, LearningRateScheduler\n# Numpy\nimport numpy\nnumpy.random.seed(1331)\n# Pandas\nimport pandas as pd\n# Sklearn\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import f1_score\nfrom sklearn.model_selection import GridSearchCV, RandomizedSearchCV, StratifiedKFold, cross_val_score\n# Visualizations\nimport matplotlib.pyplot as plt\n%matplotlib inline\n# Garbage Collector\nimport gc\nimport sys\n# Hyperopt\nfrom hyperopt import fmin, tpe, hp, anneal, Trials, space_eval\n# Random\nimport random\n# codecs + collections + csv\nimport codecs\nimport collections\nimport csv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tf.__version__","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Tokenize\n\nFirst we need to read a good population of the data, in order to tokenize the data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# load training data\nTRAIN_DATA_FILE = \"../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv\"\nj_df = pd.read_csv(TRAIN_DATA_FILE)\n\n# Parameter settings\nmaxlen = 220\nmax_features = 200000\n\n# create a tokenizer\ntoken = tf.keras.preprocessing.text.Tokenizer(num_words=max_features)\n# fit tokenizer on data\ntoken.fit_on_texts(j_df['comment_text'])\n# get word index from tokenizer\nword_index = token.word_index\n\n# Memory Clean-up\ndel j_df\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Validation","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# load validation data\nVAL_DATA_FILE = \"../input/jigsaw-multilingual-toxic-test-translated/jigsaw_miltilingual_valid_translated.csv\"\nj_df = pd.read_csv(VAL_DATA_FILE)\nj_df.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_val = sequence.pad_sequences(token.texts_to_sequences(j_df['translated']), maxlen=maxlen)\ny_val = j_df['toxic']\nprint(X_val.shape)\n\n# Memory Clean-up\ndel j_df\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Embedding","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"embed_size = 100\n\nEMBEDDING_FILES = [\n    '../input/jigsaw-custom-word2vec-100d-5iter/custom_word2vec_100d_5iter.txt'\n]\n\ndef get_coefs(word, *arr):\n    return word, np.asarray(arr, dtype='float32')\n\ndef load_embeddings(path):\n    with open(path) as f:\n        return dict(get_coefs(*line.strip().split(' ')) for line in f)\n\ndef build_matrix(word_index, path):\n    embedding_index = load_embeddings(path)\n    embedding_matrix = np.zeros((len(word_index) + 1, 100))\n    for word, i in word_index.items():\n        try:\n            embedding_matrix[i] = embedding_index[word]\n        except KeyError:\n            pass\n    return embedding_matrix\n\nembedding_matrix = np.concatenate([build_matrix(token.word_index, f) for f in EMBEDDING_FILES], axis=-1)\nembedding_matrix = embedding_matrix[0:max_features,:]\nembedding_matrix.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Load & Padding","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def sent_generator(TRAIN_DATA_FILE, chunksize, threshold, maxlen):\n    reader = pd.read_csv(TRAIN_DATA_FILE, chunksize=chunksize, iterator=True)\n    for df in reader:\n        texts = df.iloc[:,1].astype(str)\n        target = np.where(df.iloc[:,2]>threshold,1,0)\n        sequences = token.texts_to_sequences(texts)\n        data_train = sequence.pad_sequences(sequences, maxlen=maxlen)\n        yield data_train, target","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def test_data_prep():\n    # load test data\n    j_df = pd.read_csv(\"../input/jigsaw-multilingual-toxic-test-translated/jigsaw_miltilingual_test_translated.csv\")\n    X_test = j_df['translated'].astype(str)\n    del j_df\n    gc.collect()\n    X_test = sequence.pad_sequences(token.texts_to_sequences(X_test), maxlen=maxlen)\n    return X_test\n\nX_test = test_data_prep()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Create & Compile Model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Results from Hyperopt\ndrpt_amt = 0.30\nlstm2_nrns = 23\nepochs = 1\nbatches = 641","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model():\n    # create model\n    model = Sequential()\n    model.add(Embedding(max_features, embed_size, input_length=maxlen, weights=[embedding_matrix], trainable=False))\n    model.add(Dropout(drpt_amt))\n    model.add(Bidirectional(CuDNNLSTM(lstm2_nrns)))\n    model.add(Dropout(drpt_amt))\n    model.add(Dense(1, activation='sigmoid'))\n    # Compile model\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"checkpoint_predictions = []\nweights = []\nnb_epoch = epochs\nbatch_size = batches\nthreshold = 0.48\nfor model_idx in range(2):\n    model = build_model()\n    n_steps = (1209267) // batch_size\n    threshold += 0.02\n    for counter in range(nb_epoch):\n        print('-------epoch: ',counter,'--------')\n        scheduler = lambda _: 1e-3 * (0.55 ** counter)\n        callback = tf.keras.callbacks.LearningRateScheduler(scheduler)\n        model.fit_generator(sent_generator(TRAIN_DATA_FILE, batch_size, threshold, 220),\n                            steps_per_epoch=n_steps, \n                            epochs=3, \n                            validation_data=(X_val, y_val),\n                            callbacks=[callback])\n        prediction = model.predict_proba(X_test).flatten()\n        checkpoint_predictions.append(prediction)\n        weights.append(2 ** counter)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Score","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = np.average(checkpoint_predictions, weights=weights, axis=0)\n\ntest_df = pd.read_csv(\"../input/jigsaw-multilingual-toxic-test-translated/jigsaw_miltilingual_test_translated.csv\")\nsubmission = pd.DataFrame.from_dict({\n    'id': test_df.id,\n    'toxic': predictions\n})\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}