{"cells":[{"metadata":{"_cell_guid":"9d2dbdb3-6c74-4f96-9865-2951dfd653ce","_uuid":"bb41ad86b25fecf332927b0c8f55dd710101e33f"},"cell_type":"markdown","source":"# Improved LSTM baseline\n\nThis kernel is a somewhat improved version of [Keras - Bidirectional LSTM baseline](https://www.kaggle.com/CVxTz/keras-bidirectional-lstm-baseline-lb-0-051) "},{"metadata":{"_cell_guid":"2f9b7a76-8625-443d-811f-8f49781aef81","_uuid":"598f965bc881cfe6605d92903b758778d400fa8b","trusted":true},"cell_type":"code","source":"import sys, os, re, csv, codecs, numpy as np, pandas as pd\n\nfrom keras.utils import plot_model\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.utils.np_utils import to_categorical\nfrom keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, GRU,SpatialDropout1D\nfrom keras.layers import Bidirectional, GlobalMaxPool1D\nfrom keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint\nfrom keras.models import Model, Sequential\nfrom keras import initializers, regularizers, constraints, optimizers, layers\n\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.pipeline import make_union","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"66a6b5fd-93f0-4f95-ad62-3253815059ba","_uuid":"729b0f0c2a02c678631b8c072d62ff46146a82ef","trusted":true},"cell_type":"code","source":"path = '../input/'\ncomp = 'jigsaw-toxic-comment-classification-challenge/'\nEMBEDDING_FILE=f'{path}glove6b50d/glove.6B.50d.txt'\nFAST_TEXT=f'{path}FastText crawl 300d 2M/crawl-300d-2M.vec'\nTRAIN_DATA_FILE=f'{path}{comp}train.csv'\nTEST_DATA_FILE=f'{path}{comp}test.csv'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aa509dbcd8d7bc9291cf70494f660a631ec1807c"},"cell_type":"code","source":"file = open(FAST_TEXT, 'r')\nprint(file.readline())","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"2807a0a5-2220-4af6-92d6-4a7100307de2","_uuid":"d365d5f8d9292bb9bf57d21d6186f8b619cbe8c3","trusted":true},"cell_type":"code","source":"embed_size = 50 # how big is each word vector\nmax_features = 100000 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 50 # max number of words in a comment to use","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"ac2e165b-1f6e-4e69-8acf-5ad7674fafc3","_uuid":"8ab6dad952c65e9afcf16e43c4043179ef288780","trusted":true},"cell_type":"code","source":"train = pd.read_csv(TRAIN_DATA_FILE)\ntest = pd.read_csv(TEST_DATA_FILE)\n\nlist_sentences_train = train[\"comment_text\"].fillna(\"_na_\").values\nlist_classes = [\"toxic\", \"severe_toxic\", \"obscene\", \"threat\", \"insult\", \"identity_hate\"]\ny = train[list_classes].values\nlist_sentences_test = test[\"comment_text\"].fillna(\"_na_\").values","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79afc0e9-b5f0-42a2-9257-a72458e91dbb","_uuid":"c292c2830522bfe59d281ecac19f3a9415c07155","trusted":true},"cell_type":"code","source":"tokenizer = Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(list_sentences_train))\n#list_tokenized_train = tokenizer.texts_to_sequences(list_sentences_train)\n#list_tokenized_test = tokenizer.texts_to_sequences(list_sentences_test)\n#X_t = pad_sequences(list_tokenized_train, maxlen=maxlen)\n#X_te = pad_sequences(list_tokenized_test, maxlen=maxlen)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"414a4d07b1c3786fa12d53f51038e542dabde67b"},"cell_type":"code","source":"def preprocess(list_sentences):\n    list_tokenized = tokenizer.texts_to_sequences(list_sentences)\n    return pad_sequences(list_tokenized, maxlen=maxlen)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"22c5a8f28a5d9e9e180d90faf2820b34f6c28f4f"},"cell_type":"code","source":"X_t = preprocess(list_sentences_train)\nX_te = preprocess(list_sentences_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e4a003fd09592929000c649368ede47a1555ae44"},"cell_type":"code","source":"X_t.shape","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"7d19392b-7750-4a1b-ac30-ed75b8a62d52","_uuid":"e9e3b4fa7c4658e0f22dd48cb1a289d9deb745fc","trusted":true},"cell_type":"code","source":"def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.strip().split()) for o in open(EMBEDDING_FILE))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"7370416a-094a-4dc7-84fa-bdbf469f6579","_uuid":"20cea54904ac1eece20874e9346905a59a604985"},"cell_type":"markdown","source":"Use these vectors to create our embedding matrix, with random initialization for words that aren't in GloVe. We'll use the same mean and stdev of embeddings the GloVe has when generating the random init."},{"metadata":{"_cell_guid":"4d29d827-377d-4d2f-8582-4a92f9569719","_uuid":"96fc33012e7f07a2169a150c61574858d49a561b","trusted":true},"cell_type":"code","source":"all_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nemb_mean,emb_std","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"62acac54-0495-4a26-ab63-2520d05b3e19","_uuid":"574c91e270add444a7bc8175440274bdd83b7173","trusted":true},"cell_type":"code","source":"word_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: \n        embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"0d4cb718-7f9a-4eab-acda-8f55b4712439","_uuid":"dc51af0bd046e1eccc29111a8e2d77bdf7c60d28","trusted":true,"_kg_hide-output":false},"cell_type":"code","source":"def get_model():\n    model = Sequential()\n    model.add(Embedding(max_features, embed_size, weights=[embedding_matrix],trainable=False,input_shape=(maxlen,)))\n    model.add(Bidirectional(LSTM(maxlen, return_sequences=True, dropout=0.1, recurrent_dropout=0.1)))\n    model.add(GlobalMaxPool1D())\n    model.add(Dense(50, activation=\"relu\"))\n    model.add(Dropout(0.1))\n    model.add(Dense(6, activation=\"sigmoid\"))\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"728a8f7abdd6c47b0585d4cc38470a47794564f6"},"cell_type":"code","source":"get_model().summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3e29c05c423c65bd48bacd00f3bc01be30274fc6"},"cell_type":"code","source":"def MyCallback(lr_patience=10):\n    es = EarlyStopping(monitor='loss', min_delta=0, patience=0, verbose=0, mode='auto', baseline=None, restore_best_weights=False)\n    rlr = ReduceLROnPlateau(monitor='loss', factor=0.1, patience=lr_patience, verbose=0, mode='auto', min_delta=0.0001, cooldown=0, min_lr=0)\n    \n    return [mc,es,rlr]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b22699e22c419bcbf78abe74d2681523e3d15baa"},"cell_type":"code","source":"'''\nskf = KFold(n_splits=5,random_state=1)\ni = 0\nscores=np.zeros((5,))\nfor tr_index,val_index in skf.split(X_t,y):\n    print('Fold ', i)\n    x_tr_cv = X_t[tr_index]\n    y_tr_cv = y[tr_index]\n    x_val_cv = X_t[val_index]\n    y_val_cv = y[val_index]\n    model = get_model()\n    model.fit(x=x_tr_cv,y=y_tr_cv,batch_size=128,epochs=2,callbacks=None,shuffle=True)\n    print('proba.')\n    y_pred = model.predict(x_val_cv)\n    print('roc_auc...')\n    score = roc_auc_score(y_true=y_val_cv,y_score=y_pred)\n    print('auc_roc: ', score)\n    scores[i]=score\n    i = i+1\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5ad06ce0044aaca5a49956cb8317aeb2bac8e133"},"cell_type":"code","source":"def tester():\n    m = get_model()\n    m.fit(X_t,y,batch_size=128)\n    return m","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"94cc87674c1962f1ac617fc71d707e02fa3dd7d6"},"cell_type":"code","source":"m = tester()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f852f9ae1c9901ba39d5282f6936c57991f6b9f1"},"cell_type":"code","source":"m.save('toxic-model.h5')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"066737b29465d09c337842a19004f53d4421fc43"},"cell_type":"code","source":"'''\nsample = preprocess([\"You should be added to the list I was creating, ergo you are a cunt.\"])\nprint(sample.shape)\nprint(model.predict(sample))\nprint(model.predict_proba(sample))\n'''","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"28ce30e3-0f21-48e5-af3c-7e5512c9fbdc","collapsed":true,"_uuid":"e59ad8a98ac5bb25a6bddd72718f3ed8a7fb52e0","trusted":false},"cell_type":"code","source":"'''\ny_test = model.predict([X_te], batch_size=1024, verbose=1)\nsample_submission = pd.read_csv(f'{path}{comp}sample_submission.csv')\nsample_submission[list_classes] = y_test\nsample_submission.to_csv('submission.csv', index=False)\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b38e079a8e26b153863b8e1ec358183e13e91ba5"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}