{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":10737,"databundleVersionId":290346,"sourceType":"competition"}],"dockerImageVersionId":18199,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Use of the CNN and LSTM models with Keras and text Classifications\n","metadata":{"_uuid":"3edcf1d743dcdf3d1deebc50a67652cc10155058"}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nprint(os.listdir(\"../input\"))\n# Keras\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Flatten, LSTM, Conv1D, MaxPooling1D, Dropout, Activation\nfrom keras.layers.embeddings import Embedding\nimport nltk\nimport string\nfrom nltk.corpus import stopwords\nimport re\nfrom nltk.stem.snowball import SnowballStemmer","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df=pd.read_csv('../input/train.csv')","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape","metadata":{"_uuid":"efb7f37dca9e1decc4ca12836dbf63f4c8a0bb53","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Text Preprocessing","metadata":{"_uuid":"fccb45e8bc42fca2c78c8f4ce3937777fd7d0dc9"}},{"cell_type":"code","source":"def clean_text(text):\n    \n    ## Remove puncuation\n    text = text.translate(string.punctuation)\n    \n    ## Convert words to lower case and split them\n    text = text.lower().split()\n    \n    ## Remove stop words\n    stops = set(stopwords.words(\"english\"))\n    text = [w for w in text if not w in stops and len(w) >= 3]\n    \n    text = \" \".join(text)\n\n    ## Clean the text\n    text = re.sub(r\"[^A-Za-z0-9^,!.\\/'+-=]\", \" \", text)\n    text = re.sub(r\"what's\", \"what is \", text)\n    text = re.sub(r\"\\'s\", \" \", text)\n    text = re.sub(r\"\\'ve\", \" have \", text)\n    text = re.sub(r\"n't\", \" not \", text)\n    text = re.sub(r\"i'm\", \"i am \", text)\n    text = re.sub(r\"\\'re\", \" are \", text)\n    text = re.sub(r\"\\'d\", \" would \", text)\n    text = re.sub(r\"\\'ll\", \" will \", text)\n    text = re.sub(r\",\", \" \", text)\n    text = re.sub(r\"\\.\", \" \", text)\n    text = re.sub(r\"!\", \" ! \", text)\n    text = re.sub(r\"\\/\", \" \", text)\n    text = re.sub(r\"\\^\", \" ^ \", text)\n    text = re.sub(r\"\\+\", \" + \", text)\n    text = re.sub(r\"\\-\", \" - \", text)\n    text = re.sub(r\"\\=\", \" = \", text)\n    text = re.sub(r\"'\", \" \", text)\n    text = re.sub(r\"(\\d+)(k)\", r\"\\g<1>000\", text)\n    text = re.sub(r\":\", \" : \", text)\n    text = re.sub(r\" e g \", \" eg \", text)\n    text = re.sub(r\" b g \", \" bg \", text)\n    text = re.sub(r\" u s \", \" american \", text)\n    text = re.sub(r\"\\0s\", \"0\", text)\n    text = re.sub(r\" 9 11 \", \"911\", text)\n    text = re.sub(r\"e - mail\", \"email\", text)\n    text = re.sub(r\"j k\", \"jk\", text)\n    text = re.sub(r\"\\s{2,}\", \" \", text)\n\n    ## Stemming\n    text = text.split()\n    stemmer = SnowballStemmer('english')\n    stemmed_words = [stemmer.stem(word) for word in text]\n    text = \" \".join(stemmed_words)\n\n    return text","metadata":{"_uuid":"2e1bf1460e0a464a7066900c6ad6e7459145e1a2","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['question_text'] = train_df['question_text'].map(lambda x: clean_text(x))","metadata":{"_uuid":"4f46a3e8a18860982b0f969333c30bc0ac2902f3","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Maximum length of the question text is set to be 100. If the text is less 100 characters, zeroes will be padded else the text will be truncated","metadata":{"_uuid":"b07a07fa6f8fa22c44e0b14a8fcffca046a27b19"}},{"cell_type":"code","source":"tokenizer = Tokenizer()\ntokenizer.fit_on_texts(train_df['question_text'])\n\ntrain_sequences = tokenizer.texts_to_sequences(train_df['question_text'])\ntrain_data = pad_sequences(train_sequences, maxlen=100)","metadata":{"_uuid":"139ad2632ed86bb0559a977c06d25317af5669b4","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Then using Glove pre-trained model","metadata":{"_uuid":"59bf618ea6af9d621ed34fa328850a6779047d74"}},{"cell_type":"code","source":"\nEMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))","metadata":{"_uuid":"604eaf0297ec051906eae9e27d32f10196053a8d","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create a weight matrix for words in training docs\nembedding_matrix = np.zeros((50000, 300))\nfor word, index in tokenizer.word_index.items():\n    if index > 50000 - 1:\n        break\n    else:\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[index] = embedding_vector","metadata":{"_uuid":"9dc3194ef242a18a839e0f21d27d599f0b1cb541","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Embedding vector forms the first layer followed by Convolutional network and finally wrapped by LSTM.","metadata":{"_uuid":"448fe1d7fe31fbace181e6f69a75bbdce4fb51d9"}},{"cell_type":"code","source":"model_glove = Sequential()\nmodel_glove.add(Embedding(50000, 300, input_length=100, weights=[embedding_matrix], trainable=False))\nmodel_glove.add(Dropout(0.2))\nmodel_glove.add(Conv1D(64, 5, activation='relu'))\nmodel_glove.add(MaxPooling1D(pool_size=4))\nmodel_glove.add(LSTM(300))\nmodel_glove.add(Dense(1, activation='sigmoid'))\nmodel_glove.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])","metadata":{"_uuid":"42a9d56ffed8260ef851569d59e062ada6494a34","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_glove.fit(train_data, train_df['target'], validation_split=0.2, epochs = 2)","metadata":{"_uuid":"91d3ab6cbc6c99f0ff19f3fc2b24fd0035e120c2","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### It takes around 1 hour to fit the model on the data for 2 epochs. Please note that I haven't split the training data into train and valid sets for simplicity.","metadata":{"_uuid":"b43c32ebd797a25e44303cbd9c20b26d28df2208"}},{"cell_type":"code","source":"test_df=pd.read_csv('../input/test.csv')","metadata":{"_uuid":"1dd17430256d93e9cb254ae0b7ab4865ca61b0aa","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Pre-processing on the text data.","metadata":{"_uuid":"a87dc4a4e913c6b4c3ccb854a837e5df121c0a36"}},{"cell_type":"code","source":"test_df['question_text'] = test_df['question_text'].map(lambda x: clean_text(x))","metadata":{"_uuid":"e2624d2fdb46ba1213e1bd8f42f94887a46daf43","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_sequences = tokenizer.texts_to_sequences(test_df['question_text'])\ntest_data = pad_sequences(test_sequences, maxlen=100)","metadata":{"_uuid":"ab840dc7045ab249e00f7d13ba60c7b81da426bd","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions= model_glove.predict_classes(test_data)","metadata":{"_uuid":"59ca8b128be8039dca3ceeb4371b409f44df08c3","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df = pd.DataFrame({\"qid\":test_df[\"qid\"].values})\nsubmission_df['prediction'] = predictions\nsubmission_df.to_csv(\"submission.csv\", index=False)","metadata":{"_uuid":"f9f4ef29136cb848c52de3d8cdf08fdc7fe10894","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{"_uuid":"41e7c064804933ac311883345a804ebf778f3097"}},{"cell_type":"code","source":"","metadata":{"_uuid":"18e285c7666f556733ae7cebd757332e0530d24a","trusted":true},"execution_count":null,"outputs":[]}]}