{"cells":[{"metadata":{"_uuid":"eeb550315f128f0f79a3a1613531c86428d45df9"},"cell_type":"markdown","source":"Model used to make predictions for the Antidote web extension.\n\nBasic model structure is forked from [here](https://www.kaggle.com/joviis/keras-2bilstm-ensemble).\n\nUses the dataset from the [Jigsaw Toxic Comment Competition](https://www.kaggle.com/c/jigsaw-toxic-comment-classification-challenge/overview) to train the model."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import json\nimport os\nimport time\nimport numpy as np \nimport pandas as pd \nfrom tqdm import tqdm\nimport math\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D, CuDNNLSTM, Bidirectional, GlobalMaxPool1D\nfrom keras.models import Model\nfrom keras import initializers, regularizers, constraints, optimizers, layers\n\ntrain_df = pd.read_csv(\"../input/google-data/train.csv\")\ntest_df = pd.read_csv(\"../input/google-data/val.csv\")\nprint(\"Train shape : \",train_df.shape)\nprint(\"Test shape : \",test_df.shape)\n\n\n\n\nembed_size = 300\nmax_features = 100000 \nmaxlen = 60\nnum_folds = 2\nnum_epochs = 20\n\ntrain_X = train_df[\"comment_text\"].fillna(\"_na_\").values\ntest_X = test_df[\"comment_text\"].fillna(\"_na_\").values\n\n\ntokenizer = Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(train_X))\ntrain_X = tokenizer.texts_to_sequences(train_X)\ntest_X = tokenizer.texts_to_sequences(test_X)\n\ntrain_X = pad_sequences(train_X, maxlen=maxlen)\ntest_X = pad_sequences(test_X, maxlen=maxlen)\n\ntrain_y = train_df['toxic'].values\n\n\nwith open('word_index.json', 'w') as outfile:  \n    json.dump(tokenizer.word_index, outfile)\n\n\nEMBEDDING_FILE = '../input/quora-insincere-questions-classification/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nf = open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in f)\nf.close()\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n        \n        \nclass NumpyEncoder(json.JSONEncoder):\n    def default(self, obj):\n        if isinstance(obj, np.ndarray):\n            return obj.tolist()\n        return json.JSONEncoder.default(self, obj)\n\n    \nwith open('embedding_matrix.json', 'w') as outfile: \n    json.dump(embedding_matrix, outfile, cls=NumpyEncoder)\n\n\ndel train_df, test_df, tokenizer, embeddings_index, all_embs,word_index\nimport gc; gc.collect()\ntime.sleep(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"935701d5a673640c22e709c201f014212efd35f0"},"cell_type":"code","source":"from keras import regularizers \nfrom keras.layers import BatchNormalization,Activation\nfrom keras.callbacks import ReduceLROnPlateau, EarlyStopping, ModelCheckpoint\nimport keras\n\nadam = keras.optimizers.Adam(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)\nlearning_rate_reduction = ReduceLROnPlateau(monitor='val_acc', \n                                            patience=1, \n                                            verbose=1, \n                                            factor=0.5, \n                                            min_lr=0.0001)\ncallbacks = [EarlyStopping(monitor='val_loss', patience=3),\n             ModelCheckpoint(filepath='best_model.h5', monitor='val_loss', save_best_only=True), learning_rate_reduction]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a0b2498ee4657677c0c087eb90d6705cd5e56709"},"cell_type":"code","source":"def get_model():\n    inp = Input(shape=(maxlen,))\n    x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False)(inp)\n    x = Dropout(0.3)(x)\n    x = Bidirectional(LSTM(128, return_sequences=True))(x)\n    x = Bidirectional(LSTM(128, return_sequences=True))(x)\n    x = GlobalMaxPool1D()(x)\n    x = Dropout(0.3)(x)\n    x = BatchNormalization()(x)\n    x = Dense(128, activation=\"elu\")(x)\n    x = Dropout(0.3)(x)\n    x = BatchNormalization()(x)\n    x = Dense(64, activation=\"elu\")(x)\n    x = Dropout(0.3)(x)\n    x = Dense(1, activation=\"sigmoid\")(x)\n    model = Model(inputs=inp, outputs=x)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c059b288015707348997d03f48f0993c60dabde7"},"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\n\nskf = StratifiedKFold(n_splits=num_folds, shuffle=True)\nskf.get_n_splits(train_X, train_y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eb618dc5838111317e8265407beeba084438983a"},"cell_type":"code","source":"from sklearn.metrics import f1_score\ndef thresh_search(y_true, y_proba):\n    best_thresh = 0\n    best_score = 0\n    for thresh in np.arange(0, 1, 0.01):\n        score = f1_score(y_true, y_proba > thresh)\n        if score > best_score:\n            best_thresh = thresh\n            best_score = score\n    return best_thresh, best_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a92351176ea11e6d04657085456c42f3dc354bef"},"cell_type":"code","source":"from sklearn.metrics import f1_score\n\nmodels = []\ny_pred = []\npred_val_y = []\ntresh_f1 = []\nfor i,(train_index, test_index) in enumerate(skf.split(train_X, train_y)):\n    X_train, X_val = train_X[train_index], train_X[test_index]\n    y_train, y_val = train_y[train_index], train_y[test_index]\n    models.append(get_model())\n    models[i].compile(loss='binary_crossentropy', optimizer=adam, metrics=['accuracy'])\n    history = models[i].fit(X_train, y_train, batch_size=512, callbacks=callbacks, epochs=num_epochs, validation_data=(X_val, y_val))\n    y_pred.append(models[i].predict(test_X, batch_size=1024, verbose=True))\n    pred_val_y.append(models[i].predict([X_val], batch_size=1024, verbose=1))\n    tresh_f1.append(thresh_search(y_val, pred_val_y[i]))\n    \nfor i, model in enumerate(models):\n    model.save(\"model_{}\".format(i))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d981eb170a9020d43e01d1465490fe85f9bc3c2"},"cell_type":"code","source":"for i,j in tresh_f1:\n    print('{}\\n'.format((i,j)))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}