{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import nltk\nimport random\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport re\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from nltk.corpus import stopwords\n\nfrom numpy import array\nfrom keras.preprocessing.text import one_hot\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.models import Sequential\nfrom keras.layers.core import Activation, Dropout, Dense\nfrom keras.layers import Flatten\nfrom keras.layers import GlobalMaxPooling1D\nfrom keras.layers.embeddings import Embedding","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv(\"../input/jigsaw-multilingual-toxic-test-translated/jigsaw_miltilingual_test_translated.csv\")\nsubmission = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Reading the training csv files\n\ndf1 = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\", usecols=[\"comment_text\", \"toxic\"])\n\ndf2 = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv\", usecols=[\"comment_text\", \"toxic\"])\n\n# Merging the training csv files\n\ntrain = pd.concat([df1, df2], axis = 0, sort = False).reset_index(drop=True)\n\nvalid = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/validation.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_multi = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_multi.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TAG_RE = re.compile(r'<[^>]+>')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def remove_tags(text):\n    return TAG_RE.sub('', text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocess_text(sen):\n    \n    sentence = sen.lower()\n    \n    # Removing html tags\n    sentence = remove_tags(sen)\n\n    # Remove punctuations and numbers\n    sentence = re.sub('[^a-zA-Z]', ' ', sentence)\n\n    # Single character removal\n    sentence = re.sub(r\"\\s+[a-zA-Z]\\s+\", ' ', sentence)\n\n    # Removing multiple spaces\n    sentence = re.sub(r'\\s+', ' ', sentence)\n\n    return sentence","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = []\nsentences = list(train[\"comment_text\"])\nfor sen in sentences:\n    X.append(preprocess_text(sen))\n\ny = train[\"toxic\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = y.astype('int')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.preprocessing.text import Tokenizer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenizer = Tokenizer(num_words=5000)\ntokenizer.fit_on_texts(X_train)\n\nX_train = tokenizer.texts_to_sequences(X_train)\nX_test = tokenizer.texts_to_sequences(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Adding 1 because of reserved 0 index\nvocab_size = len(tokenizer.word_index) + 1\n\nmaxlen = 100\n\nX_train = pad_sequences(X_train, padding='post', maxlen=maxlen)\nX_test = pad_sequences(X_test, padding='post', maxlen=maxlen)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from numpy import array\nfrom numpy import asarray\nfrom numpy import zeros\n\nembeddings_dictionary = dict()\nglove_file = open('../input/glove6b100dtxt/glove.6B.100d.txt', encoding=\"utf8\")\n\nfor line in glove_file:\n    records = line.split()\n    word = records[0]\n    vector_dimensions = asarray(records[1:], dtype='float32')\n    embeddings_dictionary [word] = vector_dimensions\nglove_file.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embedding_matrix = zeros((vocab_size, 100))\nfor word, index in tokenizer.word_index.items():\n    embedding_vector = embeddings_dictionary.get(word)\n    if embedding_vector is not None:\n        embedding_matrix[index] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# LSTM_RNN"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.layers import LSTM","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = Sequential()\nembedding_layer = Embedding(vocab_size, 100, weights=[embedding_matrix], input_length=maxlen , trainable=False)\nmodel.add(embedding_layer)\nmodel.add(LSTM(128))\n\nmodel.add(Dense(1, activation='sigmoid'))\nmodel.compile(optimizer='adam', loss='binary_crossentropy', metrics=['acc'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(model.summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit(X_train, y_train, batch_size=2000, epochs=6, verbose=1, validation_split=0.2)\n\n# score = model.evaluate(X_test, y_test, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pickle\n\nprint('Exporting the model to tokic_lstm.pkl')\nfile = open('tokic_lstm.pkl','wb')\npickle.dump(model, file)\nfile.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nplt.plot(history.history['acc'])\nplt.plot(history.history['val_acc'])\n\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train','test'], loc='upper left')\nplt.show()\n\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\n\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train','test'], loc='upper left')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Loading the LSTM model using toxic_lstm.pkl\n\n\"\"\"\nimport pickle\nwith open('outputs/tokic_lstm.pkl','rb') as pickled_file:\n    model_lstm = pickle.load(pickled_file)\n    pickled_file.close()\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loading Test Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"X_new = []\nsentences = list(test[\"translated\"])\nfor sen in sentences:\n    X_new.append(preprocess_text(sen))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_new = tokenizer.texts_to_sequences(X_new)\n\nvocab_size = len(tokenizer.word_index) + 1\n\nmaxlen = 100\nX_new = pad_sequences(X_new, padding='post', maxlen=maxlen)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import classification_report, confusion_matrix\nimport numpy as np","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"targets = [\"toxic\"]\n\nfor label in targets:\n    print(\"On the label:{} with X_test\".format(label))\n    predictions = model.predict(X_test)\n    print('Classification Report:')\n    print(classification_report(y_test, np.argmax(predictions, axis=1)))\n    print()\n    print('Confusion Matrix:')\n    print(confusion_matrix(y_test, np.argmax(predictions, axis=1)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"targets = [\"toxic\"]\ntrans_prob = submission\n\nfor label in targets:\n    print(\"On the label:{} with translated testing data\".format(label))\n    probability = model.predict_proba(X_new)[:,:]\n    trans_prob[label] = probability","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trans_prob.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"targets = [\"toxic\"]\nmulti_prob = submission\n\nfor label in targets:\n    print(\"On the label:{} with multilingual testing data\".format(label))\n    probability = model.predict_proba(X_new_multi)[:,:]\n    multi_prob[label] = probability","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"multi_prob.head(20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"multi_prob.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}