{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport math\nfrom sklearn import model_selection\nfrom sklearn.metrics import confusion_matrix, f1_score, precision_score, recall_score\nimport tensorflow as tf\nimport tensorflow_hub as hub\nimport tqdm\nfrom keras.optimizers import SGD, Adam\nfrom keras.models import Sequential, Model\nfrom keras.callbacks import Callback, ReduceLROnPlateau\nfrom keras.layers import (Input, Dense, \n                          LeakyReLU, BatchNormalization,\n                          Activation, Dropout)\nprint(os.listdir(\"../input\"))\ntrain = pd.read_csv('../input/train.csv').fillna(' ')\ntest = pd.read_csv('../input/test.csv').fillna(' ')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aa0af71961cd48e00379b46a95592385867c2863"},"cell_type":"code","source":"g = tf.Graph()\nwith g.as_default():\n    text_input = tf.placeholder(dtype=tf.string, shape=[None])\n    use = hub.Module(\"https://tfhub.dev/google/universal-sentence-encoder-large/3\", trainable=False)\n    text_use_embedded = use(text_input)\n    init_op = tf.group([tf.global_variables_initializer(), tf.tables_initializer()])\ng.finalize()\n\n# Create session and initialize.\nsession = tf.Session(graph=g)\nsession.run(init_op)\n\n_ = session.run(text_use_embedded, feed_dict={text_input: [\"Hello world\"]})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"65b18e0fbcc4c82308e59375565a9dd11a470afe"},"cell_type":"code","source":"print(len(train), len(test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fd7fe3b924dc604b698d2ae31dd7ffa76f6774b9"},"cell_type":"code","source":"def embed(texts):\n    use_batch_size = 1024\n    if len(texts) <= use_batch_size:\n        return session.run(text_use_embedded, feed_dict={text_input: texts})\n    # we query in small sizes to avoid blowing up\n    n_batches = math.ceil(len(texts) / use_batch_size)\n    np_arrs = []\n    for i in tqdm.tqdm_notebook(range(n_batches), total=n_batches):\n        start = i * use_batch_size\n        end = (i + 1) * use_batch_size\n        texts_batch = texts[start:end]\n        np_arrs.append(session.run(text_use_embedded, feed_dict={text_input: texts_batch}))\n    return np.concatenate(np_arrs, axis=0)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":false,"trusted":true,"_uuid":"f360b9efd282370b75eee8bb42702e2f0e5d3ff9"},"cell_type":"code","source":"# Speed test\n\ntrain_vecs_tmp = embed(train['question_text'].iloc[:2 ** 12].tolist())\nprint(train_vecs_tmp.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3ec8d1396445ca7479d0800606cd95d2da59cc38"},"cell_type":"code","source":"batch_size = 256\n\ndef batch_gen(_train_df):\n    _train_neg_df = _train_df[_train_df['target'] == 0]\n    _train_pos_df = _train_df[_train_df['target'] == 1]\n    _batch_size = batch_size // 2\n    # Since positive samples are way too less\n    n_batches = math.ceil(len(_train_pos_df) / _batch_size)\n    while True: \n        _train_neg_df = _train_neg_df.sample(frac=1.)  # Shuffle the data.\n        _train_pos_df = _train_pos_df.sample(frac=1.)  # Shuffle the data.\n        for i in range(n_batches):\n            start = i * _batch_size\n            end = (i + 1) * _batch_size\n            batch_df = pd.concat([_train_neg_df.iloc[start:end], _train_pos_df.iloc[start:end]]).sample(frac=1.)\n            # print('-- DEBUG --', len(batch_df))\n            texts_vectors_batch = embed(batch_df['question_text'].tolist())\n            texts_targets_batch = np.array(batch_df['target'])\n            yield texts_vectors_batch, texts_targets_batch","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bed54ae70ddc1c3130cfdc6097427ac272d61c71","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"# # From https://medium.com/@thongonary/how-to-compute-f1-score-for-each-epoch-in-keras-a1acd17715a2\n# class Metrics(Callback):\n#     def on_train_begin(self, logs={}):\n# #         self.val_f1s = []\n# #         self.val_recalls = []\n# #         self.val_precisions = []\n#         pass\n\n#     def on_epoch_end(self, epoch, logs={}):\n#         val_predict = np.asarray(self.model.predict(self.validation_data[0]))\n#         val_targ = self.validation_data[1]\n#         for thresh in np.arange(0.1, 1.0, 0.05):\n#             thresh = np.round(thresh, 2)\n#             print('F1 score at threshold {0} is {1}'.format(thresh, f1_score(val_targ, (val_predict > thresh).astype(int))))\n# #         _val_f1 = f1_score(val_targ, val_predict)\n# #         _val_recall = recall_score(val_targ, val_predict)\n# #         _val_precision = precision_score(val_targ, val_predict)\n# #         self.val_f1s.append(_val_f1)\n# #         self.val_recalls.append(_val_recall)\n# #         self.val_precisions.append(_val_precision)\n# #         print('— val_f1: %f — val_precision: %f — val_recall %f' % (_val_f1, _val_precision, _val_recall))\n\n# metrics = Metrics()\n# reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=5, min_lr=0.0001)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"40ac06424029cf51f3942f4b0896d7406b4a1c09","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"# def run_model(_train_df, _val_df, epochs=5, steps_per_epoch=5000):\n#     model = Sequential([\n#             Dropout(0.3, input_shape=(512,)),\n#             Dense(1),\n#             BatchNormalization(),\n#             Activation('sigmoid'),\n#         ])\n#     model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n#     val_vectors = embed(_val_df['question_text'].tolist())\n#     val_y = _val_df['target'].values\n#     train_batch_generator = batch_gen(_train_df)\n#     # val_batch_generator = batch_gen(_val_df)\n#     # steps_per_epoch = min(5000, len(_train_df) // batch_size)\n#     model.fit_generator(generator=train_batch_generator,\n#                         steps_per_epoch=steps_per_epoch,\n#                         validation_data=(val_vectors, val_y),\n#                         # validation_data=val_batch_generator,\n#                         # validation_steps=len(_val_df) // batch_size,\n#                         epochs=epochs,\n#                         callbacks=[reduce_lr, metrics],\n#                         verbose=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9e8a79ddf06446ea3ae7b6998f46fb480bc6c738","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"# # Test the setup\n\n# sss = model_selection.StratifiedShuffleSplit(n_splits=1, test_size=0.01, random_state=42)\n# for dev_index, val_index in sss.split(train['question_text'], train['target']):    \n#     _train = train.iloc[dev_index, :]\n#     _val = train.iloc[val_index, :]\n#     print('------')\n#     print(len(_train), len(_val))\n#     run_model(_train, _val, 10, 50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"84fc42ae103b3fedafbd933b2b6461ef8fa237d9"},"cell_type":"code","source":"# final_model = Sequential([\n#                 Dropout(0.1, input_shape=(512,)),\n#                 Dense(32),\n#                 BatchNormalization(),\n#                 Activation('relu'),\n#                 Dropout(0.3),\n#                 Dense(1, input_shape=(512,)),\n#                 BatchNormalization(),\n#                 Activation('sigmoid'),\n#               ])\n\n\nfinal_model = Sequential([\n                Dropout(0.3, input_shape=(512,)),\n                Dense(1),\n                BatchNormalization(),\n                Activation('sigmoid'),\n              ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"043e84c59706d0cff72ae9c2a67cafd2c845aeff"},"cell_type":"code","source":"sgd = SGD(lr=0.1, decay=1e-5, momentum=0.9, nesterov=True)\nfinal_model.compile(loss='binary_crossentropy', optimizer=sgd, metrics=['accuracy'])\nbatch_generator = batch_gen(train)\nfinal_model.fit_generator(batch_generator,\n                          epochs=20,\n                          steps_per_epoch=50,\n                          verbose=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"da27f91d2b08222f43093469db5952304b741d20"},"cell_type":"code","source":"submission = pd.DataFrame.from_dict({'qid': test['qid']})\npredictions = final_model.predict(embed(test['question_text'].tolist()))\npredictions = (predictions > 0.65).astype(int)\nsubmission['prediction'] = predictions\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"51a5ad1996cfd10a294f95cfcd346092996371a2"},"cell_type":"code","source":"pd.value_counts(submission['prediction'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e8295f9682418a8d16200af30361fce2163f5dd4"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}