{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"513207f4be121ee32a79ff06ea4dda2e39fa3b84"},"cell_type":"code","source":"from numpy import array\nfrom numpy import asarray\nfrom numpy import zeros\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.models import Sequential\nfrom keras.layers import LSTM\nfrom keras.layers import Dense\nfrom keras.layers import Flatten\nfrom keras.layers import Dropout\nfrom keras.layers import Bidirectional\nfrom keras.layers import CuDNNGRU\nfrom keras.layers.embeddings import Embedding","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"df_train = pd.read_csv(\"../input/train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5ffacfc8e40ba07d7493d6c4a269f5c754cf2b58"},"cell_type":"code","source":"df_test = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4344e693d11e8a9f19602aa4379164145b16107e"},"cell_type":"code","source":"# Using Keras Tokenizer to fit on training data\n\ntokenizer = Tokenizer()\ntokenizer.fit_on_texts(df_train['question_text'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"06ae475afc1962fe7071b2ff5a724c8776a35226"},"cell_type":"code","source":"vocab_size = len(tokenizer.word_index) + 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0ee83817254c59461b00d56b481691249259774e"},"cell_type":"code","source":"# Using Keras function to convert text token to number sequence\n\nts_train=tokenizer.texts_to_sequences(df_train['question_text'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cd9be8371c1e5ce1f927bcc1f51372530e9128bb"},"cell_type":"code","source":"ts_test=tokenizer.texts_to_sequences(df_test['question_text'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"00ee6a34a985e342859437eeddbecb482cac999b"},"cell_type":"code","source":"# Padding number sequence to a length that could be token size of a longest question\n# Small questions will have zero's at the end\n\nX_train_vectorized=pad_sequences(ts_train,maxlen=135,padding='post')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8c08dd1e8b002566ababb7da9dc6799fe40fefb0"},"cell_type":"code","source":"X_test_vectorized=pad_sequences(ts_test,maxlen=135,padding='post')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fa39886eb3964af0c2ff98dbea5d39eb93c1d391"},"cell_type":"code","source":"y_train = df_train['target']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"093b7866a4806b5e5ca788305861951c5415c6d0"},"cell_type":"code","source":"# Using only Glove word embedding out of other 3 provided for this competition(Google , Paragram , Wiki)\n# Glove considered to have slight better accuracy and same could be checked with resulting F1 accuracy/score\n# Word embedding has token and corresponding 300 weight features , we will load file entire in memory\n\nembeddings_index = {}\nf = open('../input/embeddings/glove.840B.300d/glove.840B.300d.txt', encoding='utf8')\nfor line in f:\n    values = line.split()\n    word = ''.join(values[:-300])\n    coefs = np.asarray(values[-300:], dtype='float32')\n    embeddings_index[word] = coefs\nf.close()\nprint('Loaded %s word vectors.' % len(embeddings_index))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"43972d402e67d3c027c9b16856e8e2f36d5da28e"},"cell_type":"code","source":"# Weight matrix is created for only those tokens present in our question corpus\n\nembedding_matrix = zeros((vocab_size, 300))\nfor word, i in tokenizer.word_index.items():\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None:\n        embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9c715f43c073b9b997c0858ed57c80dd5ed5f596"},"cell_type":"code","source":"# Sequential Neural netwok model with weights as embedding\n# Model parameters (output dimension ,dropout, activation function have been manually tuned for better output)\n\nmodel = Sequential()\ne = Embedding(vocab_size, 300, weights=[embedding_matrix], input_length=135, trainable=False)\nmodel.add(e)\nmodel.add(Bidirectional(LSTM(128, return_sequences=True)))\nmodel.add(Flatten())\nmodel.add(Dense(128, activation='sigmoid'))\nmodel.add(Dropout(0.2))\nmodel.add(Dense(1, activation='sigmoid'))\nmodel.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5e18b0e64e9d8f25e6d49ec9054f57e49b5f5f4f"},"cell_type":"code","source":"# Model is trained on the vectorized token data as input and label as output\n# Less epochs has been used , higher may overfit and may not generalize well and can bring less score\n# High batch size will help in faster training\n\nmodel.fit(X_train_vectorized, y_train, epochs=3, batch_size=1024, verbose=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a199eb05ae1679b1c48a97e0732cef6019c8fd74"},"cell_type":"code","source":"# Resulting prediction will come in float , but we have to transform it to integer for 0/1 labelling.\n\npredictions = model.predict(X_test_vectorized)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"46ad65690e3a8cf5afe43fce27679bb1d3bc4d17"},"cell_type":"code","source":"# Float value greater than 0.33 is converted to integer 1 and below to 0\n# 0.33 gives better result than 0.5 if used for 0 and 1 distinction \n\npreds_class = (predictions > 0.33).astype(np.int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7d353847b720fffa257f8ea7a742b63e9fc3a3f6"},"cell_type":"code","source":"# Reshaping size of prediction array so as to conver it to series\n\npreds=preds_class.reshape(len(df_test),)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f30ae0cc50a4111fd22a782d9e336b24f52a191e"},"cell_type":"code","source":"prediction = pd.Series(preds,name=\"prediction\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"71b2030b1778d5769e205dd843f205a6ee841f14","scrolled":true},"cell_type":"code","source":"qid = df_test['qid']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5a48c62ef81d2887353f7da7ddc4a37c6b101d43"},"cell_type":"code","source":"# Concatting qid and prediction for submission\n\nsubmission_df = pd.concat([qid, prediction], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f076264e43cf393d77f26e66866b5b88a226e1e0"},"cell_type":"code","source":"submission_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ad37193e7056a8c65042a6c3fdcd3664d312d67b"},"cell_type":"code","source":"submission_df.to_csv(\"submission.csv\", columns = submission_df.columns, index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}