{"cells":[{"metadata":{"_uuid":"2d83c1c78260505d929746bbaa6decd073ba8275"},"cell_type":"markdown","source":"Honorable mentions:\n\nAttention - https://www.kaggle.com/suicaokhoailang/beating-the-baseline-with-one-weird-trick-0-691"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\nprint(os.listdir(\"../input/embeddings/GoogleNews-vectors-negative300\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport pickle\n\n# preprocessing\nfrom sklearn import preprocessing\n\nfrom keras.preprocessing.text import Tokenizer, text_to_word_sequence\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.models import Model\nfrom keras.layers import Input, Embedding, GRU, TimeDistributed, Dense, CuDNNGRU, Bidirectional, Dropout\nfrom keras import initializers, regularizers, constraints, optimizers, layers\n\nimport gensim\nfrom gensim.models import Word2Vec\nfrom gensim.models.keyedvectors import KeyedVectors\nfrom gensim.scripts.glove2word2vec import glove2word2vec\nfrom gensim.utils import simple_preprocess","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"886961c28257cd5b70d03973a2add6d1629c52ec"},"cell_type":"code","source":"NUM_WORDS = 60000\nMAX_NUM_WORDS = NUM_WORDS\nEMBEDDING_DIM = 300\nVALIDATION_SPLIT = 0.005\nNUM_FILTERS = 25\nMAX_LEN = 64 #128 #256\nSLICE = 2\nBatch_size = 512","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4d40b49be8cd4f8c697792a95c1b3407ca316d49"},"cell_type":"code","source":"train_data = pd.read_csv('../input/train.csv', low_memory=False) # 450MB\ntest_data = pd.read_csv('../input/test.csv', low_memory=False) # 20MB","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"82964b888784b6adc754099ee6cc53c337e77441"},"cell_type":"code","source":"texts = pd.concat([train_data['question_text'], test_data['question_text']]) # 20MB","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"55fa3165ffbcd5387694025ed6c970d1768be974"},"cell_type":"code","source":"#texts = pd.concat([train_data['question_text'], test_data['question_text']]) \n\n# 100MB +-\ntokenizer = Tokenizer(num_words=NUM_WORDS)\ntokenizer.fit_on_texts(texts)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c2c4f4d13d62bdff5e86e14f30cda3eed7cdcd51"},"cell_type":"code","source":"# transfer sentences into sequences of word indexes\nsequences_train = tokenizer.texts_to_sequences(train_data['question_text']) # 300MB +-\nsequences_test = tokenizer.texts_to_sequences(test_data['question_text'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f7a2bceea83ac1e85c0e35a348e5aeb32b24d079"},"cell_type":"code","source":"word_index = tokenizer.word_index\n\nprint(sequences_train[0])\nprint('\\n')\nprint('Found %s unique tokens.' % len(word_index))\n\nprint(sequences_test[0])\nprint('\\n')\nprint('Found %s unique tokens.' % len(word_index))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0537b019a5e66109b8795d42296850ed368f3b8f"},"cell_type":"code","source":"# 1.4GB\nX_train = pad_sequences(sequences_train, maxlen=MAX_LEN)\n#X_test = pad_sequences(sequences_test, maxlen=X_train.shape[1])\nX_test = pad_sequences(sequences_test, maxlen=MAX_LEN)\n\ny_train = train_data['target']\n\nprint('Shape of X train: {0}'.format(X_train.shape))\nprint('Shape of label train: {0}'.format(y_train.shape) )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e4e0b4b3df6bdcbb4bf658fcea4817b09103a40f"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"85edcd63b0231caa671f3c9eadcfc2765838d777"},"cell_type":"code","source":"#load pre-trained GloVe word embeddings\nglove_path = \"../input/embeddings/glove.840B.300d/glove.840B.300d.txt\"\nembeddings_index = {}\n\nf = open(glove_path)\nfor line in f:\n values = line.split(\" \")\n word = values[0]\n coefs = np.asarray(values[1:], dtype='float32')\n embeddings_index[word] = coefs\nf.close()\nprint('Found %s word vectors.' % len(embeddings_index))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"46de5663a0860cfb9bba746fea7d6388f02f4637"},"cell_type":"code","source":"#use pre-trained GloVe word embeddings to initialize the embedding layer\nembedding_matrix = np.random.random((NUM_WORDS + 1, EMBEDDING_DIM))\n\nfor word, i in word_index.items():\n if i=NUM_WORDS:\n# continue\n# try:\n# # found.append(word)\n# # get vector for each word \n# embedding_vector = word_vectors[word]\n# # save vector into embedding matrix\n# embedding_matrix[i] = embedding_vector\n# except KeyError:\n# # generate random vector if the word was not found in pretrained vectors\n# # not_found.append(word)\n# embedding_matrix[i] = np.random.normal(0, np.sqrt(0.25), EMBEDDING_DIM)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7784ef54379f24533f921c82832ea4a63f553eff","scrolled":true},"cell_type":"code","source":"# to free up some memory\n#del(word_vectors)\ndel(sequences_train)\ndel(sequences_test)\ndel(tokenizer)\ndel(texts)\ndel(train_data)\ndel(embeddings_index)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8d396012c3fd9a5081c07e13b3c3e7e43e84ceb8"},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true,"_uuid":"9d7a344a8125f0d4709e0605c4ff3e3873777bbb"},"cell_type":"code","source":"# ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"77a09b5e831f47518f74bc5962e582578683d110"},"cell_type":"code","source":"#slice sequences into many subsequences\nx_test_padded_seqs_split = []\n\nfor i in range(X_test.shape[0]):\n split1=np.split(X_test[i], SLICE)\n a=[]\n for j in range(SLICE):\n s=np.split(split1[j], SLICE*4)\n a.append(s)\n x_test_padded_seqs_split.append(a)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9f1ce0f93547537b983c8c6d168d7a136c198e80"},"cell_type":"code","source":"x_train_padded_seqs_split = []\n\nfor i in range(X_train.shape[0]):\n split1=np.split(X_train[i], SLICE)\n a=[]\n for j in range(SLICE):\n s=np.split(split1[j], SLICE*4)\n a.append(s)\n x_train_padded_seqs_split.append(a)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7c6ff154330f8878b68d3fce5775464ed7fc6312"},"cell_type":"code","source":"x_test_padded_seqs_split[1231]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3d666d34c4d43859be434e6e545dda1956ffbb6f"},"cell_type":"code","source":"del(X_train)\ndel(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a457b6f9a84fe17df51d8d98b3a64bc4e79c2eeb","scrolled":true},"cell_type":"code","source":"embedding_layer = Embedding(MAX_NUM_WORDS + 1,\n EMBEDDING_DIM,\n weights=[embedding_matrix],\n input_length=4,\n trainable=True)\n\n#build model\ninput1 = Input(shape=(4, ), dtype='int32')\nembed = embedding_layer(input1)\ngru1 = Bidirectional(CuDNNGRU(NUM_FILTERS, return_sequences=False))(embed)\nEncoder1 = Model(input1, gru1)\n\ninput2 = Input(shape=(8, 4, ), dtype='int32')\nembed2 = TimeDistributed(Encoder1)(input2)\ngru2 = Bidirectional(CuDNNGRU(NUM_FILTERS, return_sequences=False))(embed2)\nEncoder2 = Model(input2,gru2)\n\n# expected input_16 to have shape (8, 4, 2) but got array with shape (2, 8, 4)\ninput3 = Input(shape=(2, 8, 4), dtype='int32')\nembed3 = TimeDistributed(Encoder2)(input3)\ngru3 = Bidirectional(CuDNNGRU(NUM_FILTERS, return_sequences=False))(embed3)\n\npreds = Dense(1, activation='sigmoid')(gru3)\nmodel = Model(input3, preds)\n\nprint(Encoder1.summary())\nprint(Encoder2.summary())\nprint(model.summary())\n\n#use adam optimizer\n# from keras.optimizers import Adam\nopt = optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08)\n\nmodel.compile(loss='binary_crossentropy', optimizer=opt, metrics=['acc'])\n\n\n#use the best model to evaluate on test set\n# from keras.models import load_model\n# best_model= load_model(savebestmodel) \n# print best_model.evaluate(np.array(x_test_padded_seqs_split),y_test,batch_size=Batch_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6fef4fe0fbb0a57e5cd0658d2133d13c9154c592","scrolled":true},"cell_type":"code","source":"model.fit(np.array(x_train_padded_seqs_split), y_train, validation_split=VALIDATION_SPLIT,\n epochs=2, batch_size=256)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1500a543b7312ae0b3769d881b0ed9baa2b9faa9"},"cell_type":"code","source":"preds = model.predict(np.array(x_test_padded_seqs_split))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d8fbcd3e9029d26297521fe703028e485a21fd7b"},"cell_type":"code","source":"#preds = np.round(preds)\npreds = (preds > 0.35).astype(np.int)\npreds = preds.reshape((preds.shape[0], ))\npreds[:3]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"44f375186cc34c717d26fde4f60fa92ea721335b"},"cell_type":"code","source":"df_test = pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2f98a5e81528bb3f07d3aa40bb284ec61558eba5"},"cell_type":"code","source":"submission = pd.DataFrame({\"qid\":df_test.qid, \"prediction\":preds})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"592f5bc44ecf09204e90ea37fc8643d51d6d62d8"},"cell_type":"code","source":"submission.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"38f6cccca78748754278c2a9768d36efd3fd1007"},"cell_type":"code","source":"submission.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"349e736ec143abd2882478d745618745a0e60c47"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}