{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nfrom nltk import word_tokenize\nfrom imblearn.under_sampling import RandomUnderSampler\n\n# Keras\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Flatten, LSTM, Conv1D, MaxPooling1D, Dropout, Activation\nfrom keras.layers.embeddings import Embedding\nfrom keras.callbacks import EarlyStopping\nfrom keras.callbacks import ModelCheckpoint\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"markdown","source":"###### 1. Read data\nRead the data from CSV and apply some basic pre-processing (remove non-ascii characters, convert our target variable to an integer label)."},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(r'../input/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"docs = train_df[\"question_text\"].values\nlabels = train_df[\"target\"].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.unique(labels,return_counts=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 2. Preprocessing\nTokenize text, convert words / tokens to indexed integers. Take each document and convert to a sequence of max length 20 (pad with zeroes if shorter)."},{"metadata":{"trusted":true},"cell_type":"code","source":"# prepare tokenizer\nt = Tokenizer()\nt.fit_on_texts(docs)\nvocab_size = len(t.word_index) + 1\n# integer encode the documents\nencoded_docs = t.texts_to_sequences(docs)\nprint(vocab_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# pad documents to a max length of 20 words\nmax_length = 20\npadded_docs = pad_sequences(encoded_docs, maxlen=max_length, padding='post')\nprint(len(padded_docs))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"padded_docs.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rus = RandomUnderSampler(random_state=42)\npadded_docs_rus,labels_rus = rus.fit_resample(padded_docs,labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"padded_docs_rus.shape,labels_rus.shape,padded_docs.shape,labels.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.unique(labels_rus,return_counts=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 3. Import embeddings\nThe clever part: import a dictionary of word embeddings that translates each word into a 300 dimensional vector."},{"metadata":{"trusted":true},"cell_type":"code","source":"# load the whole embedding into memory\nEMBEDDING_FILE = r\"../input/embeddings/glove.840B.300d/glove.840B.300d.txt\"\nEMBEDDING_DIM = 100\n\ndef get_embedding():\n    embeddings_index = {}\n    f = open(EMBEDDING_FILE,'r', errors = 'ignore', encoding='utf-8')\n    for line in f:\n        values = line.split()\n        word = values[0]\n        try:\n            coefs = np.asarray(values[1:], dtype=\"float32\")\n            embeddings_index[word] = coefs\n        except:\n            pass\n            \n    f.close()\n    return embeddings_index\n\nembeddings_index = get_embedding()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create a weight matrix for words in training docs\n\ndef create_embedding_weights(vocab_size,t):\n    embedding_matrix = np.zeros((vocab_size, 300))\n    for word, i in t.word_index.items():\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[i] = embedding_vector\n    print(embedding_matrix.shape)\n    return embedding_matrix\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embedding_matrix = create_embedding_weights(vocab_size,t)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 4. Network architecture"},{"metadata":{"trusted":true},"cell_type":"code","source":"## create model\nmodel_glove = Sequential()\nmodel_glove.add(Embedding(vocab_size, 300, input_length=20, weights=[embedding_matrix], trainable=False))\nmodel_glove.add(Dropout(0.2))\nmodel_glove.add(Conv1D(64, 5, activation='relu'))\nmodel_glove.add(MaxPooling1D(pool_size=4))\nmodel_glove.add(LSTM(100))\nmodel_glove.add(Dense(1, activation='sigmoid'))\nmodel_glove.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_glove.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 5. Training and Evaluation\nIs it any good? Let's find out.\nDivide our dataset using a holdout strategy:"},{"metadata":{"trusted":true},"cell_type":"code","source":"# split dataset into train and test\nfrom sklearn.model_selection import train_test_split\nX_train, X_test, y_train, y_test = train_test_split(padded_docs_rus, labels_rus, test_size=0.1, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"filepath=\"/weights-{epoch:02d}-{val_acc:.4f}.h5\"\ncheckpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, \n                             save_best_only=False, save_weights_only=True, \n                             mode='auto', period=10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"earlystop = EarlyStopping(monitor='val_acc', min_delta=0.0001, patience=5,\n                          verbose=1, mode='auto')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# fit the model\nmodel_glove.fit(X_train, y_train, epochs=50,validation_data=(X_test, y_test),\n          verbose=1,callbacks=[checkpoint,earlystop])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# evaluate the model\nloss, accuracy = model_glove.evaluate(X_test, y_test, verbose=1)\nprint('Accuracy: %f' % (accuracy*100))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_glove.predict_classes(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ques = ['Has the United States become the largest dictatorship in the world?','Why do so many women become so rude and arrogant when they get just a little bit of wealth and power?','How should I prepare for IIT K/IIM C/ ISI K PGDBA course exam and interview?']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"t.fit_on_texts(ques)\n# integer encode the documents\nencoded_ques = t.texts_to_sequences(ques)\nmax_length = 20\npadded_ques = pad_sequences(encoded_ques, maxlen=max_length, padding='post')\nprint(len(padded_ques))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_glove.predict_classes(padded_ques)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_file = r\"../input/test.csv\"\ntest_df = pd.read_csv(test_file)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_docs = test_df['question_text'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"t.fit_on_texts(test_docs)\n# integer encode the documents\nencoded_ques_test = t.texts_to_sequences(test_docs)\nvocab_size_test = len(t.word_index) + 1\nprint(vocab_size_test)\nmax_length = 20\npadded_ques_test = pad_sequences(encoded_ques_test, maxlen=max_length, padding='post')\nprint(len(padded_ques_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predicted_output = model_glove.predict_classes(padded_ques_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df['predicted_labels'] = predicted_output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df[test_df['predicted_labels'] == 1].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.to_csv(r'submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}