{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"%matplotlib inline\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport pandas as pd\nimport numpy as np\nfrom scipy.spatial.distance import cdist\n# from tf.keras.models import Sequential  # This does not work!\nfrom tensorflow.python.keras.models import Sequential\nfrom tensorflow.python.keras.layers import Dense, GRU, Embedding\nfrom tensorflow.python.keras.optimizers import Adam\nfrom keras.preprocessing.text import Tokenizer\nfrom tensorflow.python.keras.preprocessing.sequence import pad_sequences\nimport gc","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"tf.__version__","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ad817cef495634ded49246c46c830b59b9793db6"},"cell_type":"code","source":"tf.keras.__version__","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a3103f8311c5aa44f832e4e35bef2c29d559fadc"},"cell_type":"code","source":"train_data = pd.read_csv(\"../input/train.csv\")\ntest_data = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0aa870f40a8b5721137a5b48323bdc99b1eb6f91"},"cell_type":"code","source":"test_data.isnull().any()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d73ec51702f4228e57b9155d051abfe429a8958b"},"cell_type":"code","source":"test_data.isnull().any()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a0140eb1152ccb49ec446c915a98c9c43e04bac5"},"cell_type":"code","source":"x_train_text = train_data.question_text\ny_train = train_data.target\nx_test_text = test_data.question_text","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"314359060d3b32f08afa040b3157e2438eb116ef"},"cell_type":"code","source":"print(\"Train-set size: \", len(x_train_text))\nprint(\"Test-set size:  \", len(x_test_text))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7ad0bb33ccc825facc817b903620e33ea6edee56"},"cell_type":"code","source":"data_text = list(train_data['question_text'].values) + list(test_data['question_text'].values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f58793d2acfc905ed8fd082d8c7fc48d25ddd096"},"cell_type":"code","source":"x_train_text[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d30760a4b03e6161f476196fde8dcb6887a9fe95"},"cell_type":"code","source":"y_train[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cfcd4f8b7eeb6dafd03131144336730d459cbda1"},"cell_type":"code","source":"num_words = 50000","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0c5fb621da105fe4e8da96987a3a7d28fba30151"},"cell_type":"code","source":"tokenizer = Tokenizer(num_words=num_words)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"de8d788fafe6dbb552586ca2b52d7fb22f725ea7"},"cell_type":"code","source":"%%time\ntokenizer.fit_on_texts(data_text)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"trusted":true,"_uuid":"65de33a35b28454a8a83ccc762c23c526a8f22f7"},"cell_type":"code","source":"if num_words is None:\n    num_words = len(tokenizer.word_index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"34cd8411d9c563f5d03eb501e767f57eddcae1d6"},"cell_type":"code","source":"tokenizer.word_index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8b2b1452670047377bf27434290d3288bcae8443"},"cell_type":"code","source":"x_train_tokens = tokenizer.texts_to_sequences(x_train_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2df897b676e24ff08ec7544c587b02c7ec3a63ca"},"cell_type":"code","source":"x_train_text[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"98dc3b432fdafc947863787a888bd4da55af0bb5"},"cell_type":"code","source":"np.array(x_train_tokens[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"74816010320bc612e14ffe8c65504da18c3a01f6"},"cell_type":"code","source":"x_test_tokens = tokenizer.texts_to_sequences(x_test_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a138eb6d33c98f9a6809ac5c51acbf62831a6a5a"},"cell_type":"code","source":"num_tokens = [len(tokens) for tokens in x_train_tokens + x_test_tokens]\nnum_tokens = np.array(num_tokens)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"28ecf4ca63129e1797031bc884ddc493bb189160"},"cell_type":"code","source":"np.mean(num_tokens)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9084757f82a072964785167cc5484c95c4260371"},"cell_type":"code","source":"np.max(num_tokens)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d655c9f608f299e4dc8ecf1c79a3afe3afd53154"},"cell_type":"code","source":"max_tokens = 100","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d3ae4b6e7cf5dbcf5859d9c32d90a99665279771"},"cell_type":"code","source":"pad = 'pre'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e654b517a8928c86a7cf41acf6eb0a381924423b"},"cell_type":"code","source":"x_train_pad = pad_sequences(x_train_tokens, maxlen=max_tokens,\n                            padding=pad, truncating=pad)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e4a7cb20a934f0bb437ccd3cad34b2130cd3f6a1"},"cell_type":"code","source":"x_test_pad = pad_sequences(x_test_tokens, maxlen=max_tokens,\n                           padding=pad, truncating=pad)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bc5161c167ec1d020db35e23492de55877d822b9"},"cell_type":"code","source":"x_train_pad.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"10a4cb253d978715273014b7e58c4fb7a5bd38c6"},"cell_type":"code","source":"x_test_pad.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"910e3763e663cf10e8dd741312f53a827d1e0031"},"cell_type":"code","source":"x_train_pad[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"da9feec4340149aef1e820c7cb8b9e805ca9e6dd"},"cell_type":"code","source":"idx = tokenizer.word_index\ninverse_map = dict(zip(idx.values(), idx.keys()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"916253389572168f2612eb25742e7ed5195c618f"},"cell_type":"code","source":"def tokens_to_string(tokens):\n    # Map from tokens back to words.\n    words = [inverse_map[token] for token in tokens if token != 0]\n    \n    # Concatenate all words.\n    text = \" \".join(words)\n\n    return text","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"93a8faee3f8d310b6bd43ccf8176661400091d06"},"cell_type":"code","source":"x_train_text[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d984607101f970ecb068079cfba4cb040073a2f6"},"cell_type":"code","source":"tokens_to_string(x_train_tokens[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c870f95b5c61467801456ba459ab56f1f657b003"},"cell_type":"code","source":"EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nword_index = tokenizer.word_index\nnb_words = min(num_words, len(word_index))\nembedding_matrix_1 = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\nfor word, i in word_index.items():\n    if i >= num_words: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix_1[i] = embedding_vector\n\ndel embeddings_index; gc.collect() ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0ce4458af0c383287a73b569d7fe46f1e838d392"},"cell_type":"code","source":"EMBEDDING_FILE = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE) if len(o)>100)\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nword_index = tokenizer.word_index\nnb_words = min(num_words, len(word_index))\nembedding_matrix_2 = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\nfor word, i in word_index.items():\n    if i >= num_words: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix_2[i] = embedding_vector\ndel embeddings_index; gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e4977b43287227be7ee1de1708fe84d6fa33d6b9"},"cell_type":"code","source":"EMBEDDING_FILE = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore') if len(o)>100)\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nword_index = tokenizer.word_index\nnb_words = min(num_words, len(word_index))\nembedding_matrix_3 = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\nfor word, i in word_index.items():\n    if i >= num_words: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix_3[i] = embedding_vector\n        \ndel embeddings_index; gc.collect()  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f7c813b76d7a8732839a073efcba62a13f36eff5"},"cell_type":"code","source":"embedding_matrix = np.concatenate((embedding_matrix_1, embedding_matrix_2, embedding_matrix_3), axis=1)  \ndel embedding_matrix_1, embedding_matrix_2, embedding_matrix_3\ngc.collect()\nnp.shape(embedding_matrix)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ce37822ed025fd988689e53fb88c6c0ed4442755"},"cell_type":"code","source":"model = Sequential()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fa084d9b1307a75cdbf5b7285a34c56a22ed1c03"},"cell_type":"code","source":"embedding_size = 300","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1157ff42b09a7d0efcc1aa4c487bbc539433fffc"},"cell_type":"code","source":"model.add(Embedding(num_words, embed_size * 3, weights=[embedding_matrix], trainable=False))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"71fa85ce25bcb6918fd16b98ffeb9b7c0e79d7c9"},"cell_type":"code","source":"model.add(GRU(units=16, return_sequences=True))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8044059403cc4e66b98ee27ea7061e72fe0253ca"},"cell_type":"code","source":"model.add(GRU(units=8, return_sequences=True))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"058d825aada55dd744be0d8aaa28b41cab9788da"},"cell_type":"code","source":"model.add(GRU(units=4))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"abd59a536f1a51d9954c9ed6601e23b432dd6e5b"},"cell_type":"code","source":"model.add(Dense(1, activation='sigmoid'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cbd81ac4d8d66890731f296a4db0db1c82ff8dcb"},"cell_type":"code","source":"optimizer = Adam(lr=1e-3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b651aa4cc90e53db07672ed7a06ad853b9cb6245"},"cell_type":"code","source":"model.compile(loss='binary_crossentropy',\n              optimizer=optimizer,\n              metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2dede70a8d446a66228b51cfc1ae69b877adef86"},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"554b90688012927b2d74d007ecf84cfe01d43c6e"},"cell_type":"code","source":"%%time\nmodel.fit(x_train_pad, y_train,\n          validation_split=0.05, epochs=3, batch_size=64)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a68a0396e4e30c8892f17291a9649d3a0052b8a9"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}