{"cells":[{"metadata":{"_cell_guid":"9d2dbdb3-6c74-4f96-9865-2951dfd653ce","_uuid":"bb41ad86b25fecf332927b0c8f55dd710101e33f"},"cell_type":"markdown","source":"# Improved LSTM baseline\n\nThis kernel is a somewhat improved version of [Keras - Bidirectional LSTM baseline](https://www.kaggle.com/CVxTz/keras-bidirectional-lstm-baseline-lb-0-051) along with some additional documentation of the steps. (NB: this notebook has been re-run on the new test set.)"},{"metadata":{"_cell_guid":"2f9b7a76-8625-443d-811f-8f49781aef81","_uuid":"598f965bc881cfe6605d92903b758778d400fa8b","trusted":true},"cell_type":"code","source":"import sys, os, re, csv, codecs, numpy as np, pandas as pd\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation\nfrom keras.layers import Bidirectional, GlobalMaxPool1D\nfrom keras.models import Model\nfrom keras import initializers, regularizers, constraints, optimizers, layers\nimport numpy as np \nimport pandas as pd \nfrom time import time\nimport re\nimport string\nimport os\n#import emoji\nfrom pprint import pprint\nimport collections\nfrom keras import Sequential\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nsns.set(style=\"darkgrid\")\nsns.set(font_scale=1.3)\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.pipeline import Pipeline, FeatureUnion\nfrom sklearn.metrics import classification_report\nfrom sklearn.naive_bayes import MultinomialNB\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.externals import joblib\n#import gensim\nfrom nltk.corpus import stopwords\nfrom nltk.stem import PorterStemmer\nfrom nltk.tokenize import word_tokenize\nimport warnings\nwarnings.filterwarnings('ignore')\nnp.random.seed(37)\nimport json\nimport string\nfrom pandas.io.json import json_normalize\nimport matplotlib.pyplot as plt\nimport seaborn as sns\ncolor = sns.color_palette()\n\n\nfrom sklearn import model_selection, preprocessing, metrics, ensemble, naive_bayes, linear_model\nfrom sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\nfrom sklearn.decomposition import TruncatedSVD\nimport lightgbm as lgb\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation,GRU, CuDNNGRU,CuDNNLSTM, Conv1D\nfrom keras.layers import Bidirectional, GlobalMaxPool1D, GlobalAveragePooling1D, GlobalMaxPooling1D\nfrom keras.models import Model\nfrom keras import initializers, regularizers, constraints, optimizers, layers\nfrom keras.utils.np_utils import to_categorical\nfrom sklearn.preprocessing import LabelEncoder\nfrom keras.layers import Input, Dense, Embedding, SpatialDropout1D, concatenate\nfrom sklearn.metrics import roc_auc_score\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"66a6b5fd-93f0-4f95-ad62-3253815059ba","_uuid":"729b0f0c2a02c678631b8c072d62ff46146a82ef","trusted":true},"cell_type":"code","source":"path = '../input/'\ncomp = 'jigsaw-toxic-comment-classification-challenge/'\nEMBEDDING_FILE=f'{path}glove6b50d/glove.6B.50d.txt'\nTRAIN_DATA_FILE=f'{path}{comp}train.csv'\nTEST_DATA_FILE=f'{path}{comp}test.csv'","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"2807a0a5-2220-4af6-92d6-4a7100307de2","_uuid":"d365d5f8d9292bb9bf57d21d6186f8b619cbe8c3","trusted":true},"cell_type":"code","source":"embed_size = 50 # how big is each word vector\nmax_features = 20000 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 100 # max number of words in a comment to use","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"ac2e165b-1f6e-4e69-8acf-5ad7674fafc3","_uuid":"8ab6dad952c65e9afcf16e43c4043179ef288780","trusted":true},"cell_type":"code","source":"train = pd.read_csv(TRAIN_DATA_FILE)\ntest = pd.read_csv(TEST_DATA_FILE)\n\nlist_sentences_train = train[\"comment_text\"].fillna(\"_na_\").values\nlist_classes = [\"toxic\", \"severe_toxic\", \"obscene\", \"threat\", \"insult\", \"identity_hate\"]\ny = train[list_classes].values\nlist_sentences_test = test[\"comment_text\"].fillna(\"_na_\").values","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79afc0e9-b5f0-42a2-9257-a72458e91dbb","_uuid":"c292c2830522bfe59d281ecac19f3a9415c07155","trusted":true},"cell_type":"code","source":"tokenizer = Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(list_sentences_train))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"711b11ac2d003ca0a05584e10cc4e3ab8efb51b6"},"cell_type":"code","source":"import pickle\nfile_Name = \"testfile\"\nfileObject = open(file_Name,'wb')\npickle.dump(tokenizer,fileObject)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"617e974a-57ee-436e-8484-0fb362306db2","collapsed":true,"_uuid":"2b969bab77ab952ecd5abf2abe2596a0e23df251","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}