{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport re\nimport json\nfrom sklearn.model_selection import train_test_split\nimport tensorflow as tf\n\nfrom tensorflow.python.keras.preprocessing.text import Tokenizer\nfrom tensorflow.python.keras.preprocessing.sequence import pad_sequences","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"79cc5ec7105a4b8fcfebfd875379bc80b857124f"},"cell_type":"code","source":"DATA_DIR = '../input/'\nTRAIN_DATA = 'train.npy'\nTEST_DATA = 'test.npy'\nTEST_ID_DATA = 'test_id.npy'\nTRAIN_LABEL_DATA = 'train_label.npy'\nDATA_CONFIGS = 'data_configs.jsonb'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"56eccca173266e1b733efe6f52e964d66eaa3e3d"},"cell_type":"code","source":"MAX_SEQUENCE_LENGTH = 31","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"02de6fa07cabcc12aef434642ab2f4941db5b0ab"},"cell_type":"code","source":"dataset = pd.read_csv(DATA_DIR + 'train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a4f2b11e6993ae55e84f515915a1bf0e95be1916"},"cell_type":"code","source":"question_text = list(dataset['question_text'])\nlabels = np.array(dataset['target'], dtype=np.int64)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a9be4505801ccf0b967d860bda629aab5ce182fc"},"cell_type":"code","source":"tokenizer = Tokenizer()\ntokenizer.fit_on_texts(question_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a94853ee41fd49f474b29b45a31057d9d3b2eadc"},"cell_type":"code","source":"sequence = tokenizer.texts_to_sequences(question_text)\ntrain_data = pad_sequences(sequence, maxlen=MAX_SEQUENCE_LENGTH, padding='post')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b85dfe5142774338fe8e8335af73bc03ef9b41d5"},"cell_type":"code","source":"test_dataset = pd.read_csv(DATA_DIR + 'test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ca6b7e15066c5466a8448d1213d3059c76c85108"},"cell_type":"code","source":"test_question_text = list(test_dataset['question_text'])\ntest_id = np.array(test_dataset['qid'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e299e8793e09a8ee050410173b0ce4395a2214bb"},"cell_type":"code","source":"test_sequence = tokenizer.texts_to_sequences(test_question_text)\ntest_data = pad_sequences(test_sequence, maxlen=MAX_SEQUENCE_LENGTH, padding='post')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"700dc7da0b81b6291adb15853156b8a2823d9cca"},"cell_type":"code","source":"TEST_SPLIT = 0.1\nRNG_SEED = 13371447\nVOCAB_SIZE = len(tokenizer.word_index) + 1\nEMB_SIZE = 128\nBATCH_SIZE = 64\nNUM_EPOCHS = 1\n\ninput_train, input_eval, label_train, label_eval = train_test_split(train_data, labels, test_size=TEST_SPLIT, random_state=RNG_SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"56f4e419f1b21b70b34b8bdbaf4af36a3c647ae1"},"cell_type":"code","source":"CONV_FEATURE_DIM = 128\nCONV_WINDOW_SIZE = 3\nFC_FEATURE_DIM = 128\n\nNUM_CONV_LAYERS = 5\nNUM_FC_LAYERS = 10","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"863182a679bddc7977a44d099343297aabb592eb"},"cell_type":"code","source":"def mapping_fn(X, Y):\n    input, label = {'x': X}, Y\n    return input, label\n\ndef train_input_fn():\n    dataset = tf.data.Dataset.from_tensor_slices((input_train, label_train))\n    dataset = dataset.shuffle(buffer_size=len(input_train))\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.map(mapping_fn)\n    dataset = dataset.repeat(count=NUM_EPOCHS)\n    iterator = dataset.make_one_shot_iterator()\n    \n    return iterator.get_next()\n\ndef eval_input_fn():\n    dataset = tf.data.Dataset.from_tensor_slices((input_eval, label_eval))\n    dataset = dataset.batch(64)\n    dataset = dataset.map(mapping_fn)\n    iterator = dataset.make_one_shot_iterator()\n    \n    return iterator.get_next()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"568cbeb2c09847b3c587315826d6d5337656edc7"},"cell_type":"code","source":"def model_fn(features, labels, mode):\n    TRAIN = mode == tf.estimator.ModeKeys.TRAIN\n    EVAL = mode == tf.estimator.ModeKeys.EVAL\n    PREDICT = mode == tf.estimator.ModeKeys.PREDICT\n    \n    def conv_block(inputs):\n        conv_layer = tf.keras.layers.Conv1D(CONV_FEATURE_DIM, \n                                            CONV_WINDOW_SIZE,  \n                                            padding='same')(inputs)\n\n        glu_layer = tf.keras.layers.Dense(CONV_FEATURE_DIM * 2, \n                                             activation=tf.nn.relu)(conv_layer)\n\n        scored_output, output_layer = tf.split(glu_layer, 2, axis=-1)\n\n        output_layer = output_layer * tf.nn.sigmoid(scored_output)\n\n        return output_layer\n\n    embedding_layer = tf.keras.layers.Embedding(VOCAB_SIZE,EMB_SIZE)(features['x'])\n    embedding_layer = tf.keras.layers.Dropout(0.2)(embedding_layer)\n\n    with tf.variable_scope('conv_layers'):\n        for i in range(NUM_CONV_LAYERS):\n            input_layer = conv_output_layer if i > 0 else embedding_layer\n            conv_output_layer = conv_block(input_layer)\n            conv_output_layer = tf.keras.layers.Dropout(0.2)(input_layer + conv_output_layer)\n    \n    flatten_layer = tf.keras.layers.Flatten()(conv_output_layer)\n    flatten_layer = tf.keras.layers.Dense(FC_FEATURE_DIM, activation=tf.nn.relu)(flatten_layer)\n    with tf.variable_scope('dense_layers'):\n        for i in range(NUM_FC_LAYERS):\n            input_layer = fc_output_layer if i > 0 else flatten_layer\n            fc_output_layer = tf.keras.layers.Dense(FC_FEATURE_DIM, activation=tf.nn.relu)(input_layer)\n            fc_output_layer = tf.keras.layers.Dropout(0.2)(input_layer + fc_output_layer)\n\n    logits = tf.keras.layers.Dense(1)(fc_output_layer)\n    \n    if PREDICT:\n        return tf.estimator.EstimatorSpec(\n            mode=mode,\n            predictions={\n                'prob': tf.round(tf.nn.sigmoid(logits))\n            })\n    \n    labels = tf.reshape(labels, [-1, 1])\n    \n    if TRAIN:\n        global_step = tf.train.get_global_step()\n        loss = tf.losses.sigmoid_cross_entropy(labels, logits)\n        train_op = tf.train.AdamOptimizer(0.001).minimize(loss, global_step)\n\n        return tf.estimator.EstimatorSpec(mode=mode, train_op=train_op, loss = loss)\n    \n    if EVAL:\n        loss = tf.losses.sigmoid_cross_entropy(labels, logits)\n        pred = tf.nn.sigmoid(logits)\n        accuracy = tf.metrics.accuracy(labels, tf.round(pred))\n        f1_score = tf.contrib.metrics.f1_score(labels, tf.round(pred))\n        return tf.estimator.EstimatorSpec(mode=mode, loss=loss, eval_metric_ops={'f1 score': f1_score, 'acc': accuracy})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"72376b4051da3fe3eed19f0279b9cece9c514b40"},"cell_type":"code","source":"est = tf.estimator.Estimator(model_fn, model_dir=\"model/checkpoint/cnn_model\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8ab328a93722186649c24f0349c146ec9801ea0d","_kg_hide-output":false},"cell_type":"code","source":"est.train(train_input_fn)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bacde86085219d06353a3db409798cd4dc40e6b3","_kg_hide-output":false,"_kg_hide-input":false},"cell_type":"code","source":"est.evaluate(eval_input_fn)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"887f1d9d2478ac60e8aa8041a1eab09bea4d6bfd"},"cell_type":"code","source":"predict_input_fn = tf.estimator.inputs.numpy_input_fn(x={\"x\":test_data}, shuffle=False)\npredictions = np.array([int(p['prob'][0]) for p in est.predict(input_fn=predict_input_fn)], dtype=np.int32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6681b88756720682ebb0ff96ae3baf6bed15726a"},"cell_type":"code","source":"output = pd.DataFrame( data={\"qid\": test_id, \"prediction\": predictions} )\noutput.to_csv(\"submission.csv\", index=False, quoting=3 )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e106c08beb35c59116d78e54b450821703aaec1a"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}