{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport os\nimport pandas as pd\nimport sys\nimport matplotlib.pyplot as plt\n%matplotlib inline\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.feature_extraction.text import TfidfTransformer\nfrom sklearn.feature_extraction.text import TfidfVectorizer,HashingVectorizer\nfrom sklearn.naive_bayes import MultinomialNB\nfrom sklearn.svm import LinearSVC\nfrom sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier\nfrom sklearn.linear_model import LogisticRegression,SGDClassifier\nfrom nltk.corpus import wordnet as wn\nfrom nltk.corpus import stopwords\nfrom nltk.stem.snowball import SnowballStemmer\nfrom nltk.stem import PorterStemmer\nimport nltk\nfrom nltk import word_tokenize, ngrams\nfrom nltk.classify import SklearnClassifier\nfrom wordcloud import WordCloud,STOPWORDS\nimport xgboost as xgb\nimport seaborn as sns\nnp.random.seed(25)\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nfrom subprocess import check_output\nprint(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/train.csv\")\ntest = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bf868cda9c3aadf2a2a65904e7e37c50734ec734"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5a12501aa6a5a1da9693b4801c6a60ee02b10415"},"cell_type":"code","source":"test_id = test['qid']\ntarget = train['target']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fae089249104a77266990199599f7641e14b2a1e"},"cell_type":"code","source":"sns.countplot(train['target'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ba0b711dcb877c4ec681c860bc0f8cf0a6326859"},"cell_type":"code","source":"import keras\nimport keras.backend as K\nfrom keras.layers import Dense, GlobalAveragePooling1D, Embedding\nfrom keras.callbacks import EarlyStopping\nfrom keras.models import Sequential\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.utils import to_categorical","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dedf51dfa2318ca6f34db4e9bd2b138d9d8d9613"},"cell_type":"code","source":"y = to_categorical(target)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a93138ac2524b4cdfdbf5ace920bba505b2c4c91"},"cell_type":"code","source":"# function to clean data\nimport string\nimport itertools \nimport re\nfrom nltk.stem import WordNetLemmatizer\nfrom nltk.corpus import stopwords\nfrom string import punctuation\n\nstop_words = set(stopwords.words('english'))\n\ndef cleanData(text, lowercase = False, remove_stops = False, stemming = False, lemmatization = False):\n    txt = str(text)\n    \n    # Replace apostrophes with standard lexicons\n    txt = txt.replace(\"isn't\", \"is not\")\n    txt = txt.replace(\"aren't\", \"are not\")\n    txt = txt.replace(\"ain't\", \"am not\")\n    txt = txt.replace(\"won't\", \"will not\")\n    txt = txt.replace(\"didn't\", \"did not\")\n    txt = txt.replace(\"shan't\", \"shall not\")\n    txt = txt.replace(\"haven't\", \"have not\")\n    txt = txt.replace(\"hadn't\", \"had not\")\n    txt = txt.replace(\"hasn't\", \"has not\")\n    txt = txt.replace(\"don't\", \"do not\")\n    txt = txt.replace(\"wasn't\", \"was not\")\n    txt = txt.replace(\"weren't\", \"were not\")\n    txt = txt.replace(\"doesn't\", \"does not\")\n    txt = txt.replace(\"'s\", \" is\")\n    txt = txt.replace(\"'re\", \" are\")\n    txt = txt.replace(\"'m\", \" am\")\n    txt = txt.replace(\"'d\", \" would\")\n    txt = txt.replace(\"'ll\", \" will\")\n    txt = txt.replace(\"--th\", \" \")\n    \n    # More cleaning\n    txt = re.sub(r\"alot\", \"a lot\", txt)\n    txt = re.sub(r\"what's\", \"\", txt)\n    txt = re.sub(r\"What's\", \"\", txt)\n    \n    \n    # Remove urls and emails\n    txt = re.sub(r'^https?:\\/\\/.*[\\r\\n]*', ' ', txt, flags=re.MULTILINE)\n    txt = re.sub(r'[\\w\\.-]+@[\\w\\.-]+', ' ', txt, flags=re.MULTILINE)\n    \n    # Replace words like sooooooo with so\n    txt = ''.join(''.join(s)[:2] for _, s in itertools.groupby(txt))\n    \n    # Remove punctuation from text\n    txt = ''.join([c for c in text if c not in punctuation])\n    \n    # Remove all symbols\n    txt = re.sub(r'[^A-Za-z\\s]',r' ',txt)\n    txt = re.sub(r'\\n',r' ',txt)\n    \n    if lowercase:\n        txt = \" \".join([w.lower() for w in txt.split()])\n        \n    if remove_stops:\n        txt = \" \".join([w for w in txt.split() if w not in stop_words])\n        \n    if stemming:\n        st = PorterStemmer()\n        txt = \" \".join([st.stem(w) for w in txt.split()])\n    \n    if lemmatization:\n        wordnet_lemmatizer = WordNetLemmatizer()\n        txt = \" \".join([wordnet_lemmatizer.lemmatize(w, pos='v') for w in txt.split()])\n\n    return txt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6906cb329ad498e83411e3355dc21558b1a1ee86"},"cell_type":"code","source":"# clean comments\ntrain['question_text'] = train['question_text'].map(lambda x: cleanData(x, lowercase=True, remove_stops=True, stemming=False, lemmatization = False))\ntest['question_text'] = test['question_text'].map(lambda x: cleanData(x, lowercase=True, remove_stops=True, stemming=False, lemmatization = False))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"346f03355d0c423f43c9f944cc98ab3253925f5e"},"cell_type":"code","source":"MAX_SEQUENCE_LENGTH = 100\nMAX_NB_WORDS = 50000 #200000","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f5a1eaff40c0ccc463323ed6e90733c986c0633f"},"cell_type":"code","source":"tokenizer = Tokenizer(lower=False, filters='',num_words = MAX_NB_WORDS)\ntokenizer.fit_on_texts(train['question_text'])\n\nsequences = tokenizer.texts_to_sequences(train['question_text'])\ntest_sequences = tokenizer.texts_to_sequences(test['question_text'])\n\ntrain_data = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH)\n\nprint('Shape of train data tensor:', train_data.shape)\n\ntest_data = pad_sequences(test_sequences, maxlen=MAX_SEQUENCE_LENGTH)\n\nnb_words = (np.max(train_data) + 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"88e9c731d3d2205e7400de1f98cc877bed1caae2"},"cell_type":"code","source":"from keras.layers.recurrent import LSTM, GRU\nmodel = Sequential()\nmodel.add(Embedding(nb_words,50,input_length=MAX_SEQUENCE_LENGTH))\nmodel.add(GlobalAveragePooling1D())\nmodel.add(Dense(2, activation='softmax'))\nmodel.summary()\nmodel.compile(loss='categorical_crossentropy', optimizer='adam', metrics = ['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ad0fc19778ec84f38ef48c83bfe1540abb9b2c4b"},"cell_type":"code","source":"model.fit(train_data, y, validation_split=0.2, nb_epoch=2, batch_size=16)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"75b25baca81943566b38184494a8ae9b9a0acd8a"},"cell_type":"code","source":"pred = model.predict(test_data)\npred = pred.argmax(axis=-1)\npred[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"720f87fa627ccec823e3ef447d0edc93fb7bca84"},"cell_type":"code","source":"result = pd.DataFrame()\nresult['qid'] = test_id\nresult['prediction'] = pred\nresult.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b1b84420add9d9fdf557900ab4343e3c1157da3d"},"cell_type":"code","source":"result.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}