{"cells":[{"metadata":{"_uuid":"c3b55951f4396677c4e4251f4b8762408ed7dfa6"},"cell_type":"markdown","source":"**Pre-trained language model(LM) ** shows great improvement for NLP systems in 2018. Wonderful results such as Elmo，OpenAI GPT and BERT.  \n\nthere are some kernels about how to use BERT for QIQC：\n\n@sergeykalutsky [Introducing BERT with Tensorflow](https://www.kaggle.com/sergeykalutsky/introducing-bert-with-tensorflow)\n\n@ishitori  [What if we could finetune BERT from Apache MXNet?](https://www.kaggle.com/ishitori/what-if-we-could-finetune-bert-from-apache-mxnet)\n\n**You can run this off kernel to get a great score reach 0.710.**  But limitted by external data usage, it doesn't work for the second stage.\n\nBut we can make a light version without externel data by your favorate kreas."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"markdown","source":"![bert](http://www.uxforthemasses.com/wp-content/uploads/2010/06/BERT-min.jpg)"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"markdown","source":"## Let's make our light pretrained LM for text classification"},{"metadata":{"trusted":true,"_uuid":"2697c750f9833199009cc7666879d04afa6b0d94"},"cell_type":"code","source":"import os\nimport time\nimport re\nimport gc\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm\nimport math\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\nfrom sklearn.model_selection import GridSearchCV, StratifiedKFold\nfrom sklearn.metrics import f1_score, roc_auc_score\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import *\nfrom keras.layers import Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D\nfrom keras.layers import Bidirectional, GlobalMaxPool1D, GlobalMaxPooling1D, GlobalAveragePooling1D\nfrom keras.layers import Input, Embedding, Dense, Conv2D, MaxPool2D, concatenate, add, BatchNormalization\nfrom keras.layers import Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D\nfrom keras.optimizers import Adam\nfrom keras.models import Model\nfrom keras import backend as K\nfrom keras.engine.topology import Layer\nfrom keras import initializers, regularizers, constraints, optimizers, layers\nfrom keras.initializers import glorot_normal, orthogonal\nfrom keras.layers import concatenate\nfrom keras.callbacks import *\n\nimport tensorflow as tf","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4fb3a94cf24fcef2bc3a518cf0e8a5d82758c7d5"},"cell_type":"markdown","source":"## some config"},{"metadata":{"trusted":true,"_uuid":"af4efbab167d41d88233536ce75409d0cdb552a8"},"cell_type":"code","source":"embed_size = 300 \nmax_features = 95000\nmaxlen = 72 \ntrain_batch_size = 512\ntrain_epochs = 6\nSEED = 2018","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4b703b15c3a5b741feff3049483d2a4cec48781d"},"cell_type":"markdown","source":"## Load data without preprocess"},{"metadata":{"trusted":true,"_uuid":"74dfc5372d0af15d1064067c2731171921ab3a6b"},"cell_type":"code","source":"def load_and_prec():\n    train_df = pd.read_csv(\"../input/train.csv\")\n    test_df = pd.read_csv(\"../input/test.csv\")\n    tokenizer = Tokenizer(num_words=max_features)\n    train_X = train_df[\"question_text\"].fillna(\"_##_\").values\n    test_X = test_df[\"question_text\"].fillna(\"_##_\").values\n    tokenizer.fit_on_texts(list(train_X))\n    train_X = tokenizer.texts_to_sequences(train_X)\n    test_X = tokenizer.texts_to_sequences(test_X)\n    train_X = pad_sequences(train_X, maxlen=maxlen)\n    test_X = pad_sequences(test_X, maxlen=maxlen)\n    train_y = train_df['target'].values\n    trn_idx = np.random.permutation(len(train_X))\n    train_X = train_X[trn_idx]\n    train_y = train_y[trn_idx]\n    return train_X, test_X, train_y, tokenizer.word_index, tokenizer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cee3372ee4e035908efe426cd7f4484ca77f499e"},"cell_type":"code","source":"def load_glove(word_index):\n    EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n\n    # word_index = tokenizer.word_index\n    nb_words = min(max_features, len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n            \n    return embedding_matrix \n\ndef load_para(word_index):\n    EMBEDDING_FILE = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore') if len(o)>100)\n\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n\n    # word_index = tokenizer.word_index\n    nb_words = min(max_features, len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n    \n    return embedding_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"24a274424dbe630837b51e5b41b84a4e134eb131"},"cell_type":"code","source":"from tqdm import tqdm\nstart_time = time.time()\ntrain_X, test_X, train_y, word_index, tockennizer = load_and_prec()\nembedding_matrix_1 = load_glove(word_index)\nembedding_matrix_2 = load_para(word_index)\ntotal_time = (time.time() - start_time) / 60\nprint(\"Took {:.2f} minutes\".format(total_time))\nembedding_matrix = np.mean([embedding_matrix_1, embedding_matrix_2], axis=0)\ndel embedding_matrix_1, embedding_matrix_2\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"07b6a1d37a1c9c797c90b719cabef5a22586d07f"},"cell_type":"markdown","source":"## define our LM batch generator"},{"metadata":{"trusted":true,"_uuid":"a68caa52d1b40d8acd6755948edcef4399b8cc8e"},"cell_type":"code","source":"from keras.utils import to_categorical\n\nclass KerasBatchGenerator(object):\n\n    def __init__(self, data, num_steps, batch_size, vocabulary , skip_step=5):\n        self.data = np.concatenate(data,axis=0)\n        self.num_steps = num_steps\n        self.batch_size = batch_size\n        self.vocabulary = vocabulary\n        self.current_idx = 0\n        self.skip_step = skip_step\n\n    def generate(self):\n        x = np.zeros((self.batch_size, self.num_steps))\n        y = np.zeros((self.batch_size, self.num_steps, self.vocabulary))\n        while True:\n            for i in range(self.batch_size):\n                if self.current_idx + self.num_steps >= len(self.data):\n                    self.current_idx = 0\n                x[i, :] = self.data[self.current_idx:self.current_idx + self.num_steps]\n                temp_y = self.data[self.current_idx + 1:self.current_idx + self.num_steps + 1]\n                y[i, :, :] = to_categorical(temp_y, num_classes=self.vocabulary)\n                self.current_idx += self.skip_step\n            yield x, y","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c76401657874c2fda2f4bab4d33c020e11d5eb2a"},"cell_type":"markdown","source":"## trianed a single Forward LM and save the pretrained parameters"},{"metadata":{"trusted":true,"_uuid":"3045eb9613083085f4c989d032b97641001937f6"},"cell_type":"code","source":"from keras import Sequential\nvocabulary = min(max_features, len(tockennizer.word_index))\nnum_steps = 5\nlm_batch_size = 512\nnum_epochs=1\nlm_train_X = train_X[:len(train_X)-len(train_X)//3]\ntrain_data_generator = KerasBatchGenerator(lm_train_X, num_steps, lm_batch_size, vocabulary,skip_step=num_steps)\nvalid_data_generator = KerasBatchGenerator(test_X, num_steps, lm_batch_size, vocabulary,skip_step=num_steps)\n\nmodel = Sequential()\nmodel.add(Embedding(vocabulary, embed_size, weights=[embedding_matrix], trainable=False, input_length=num_steps))\nmodel.add(CuDNNGRU(80, return_sequences=True, name = 'gru1', kernel_initializer=glorot_normal(seed=1029), recurrent_initializer=orthogonal(gain=1.0, seed=1029)))\nmodel.add(Dropout(0.5))\nmodel.add(TimeDistributed(Dense(vocabulary)))\nmodel.add(Activation('softmax'))\nmodel.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['categorical_accuracy'])\nmodel.fit_generator(train_data_generator.generate(), len(lm_train_X) // (lm_batch_size * num_steps), num_epochs,\n                    validation_data=valid_data_generator.generate(),\n                    validation_steps=len(test_X) // (lm_batch_size * num_steps),verbose = 0)\nmodel.save_weights('lm.hdf5')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ec38e056af0e4fa7bfa6d2bb3339d892020942d2"},"cell_type":"markdown","source":"## Attention Layer"},{"metadata":{"trusted":true,"_uuid":"c025dbd2e4c53477bccaf07fcc7a8223313dee77"},"cell_type":"code","source":"class Attention(Layer):\n    def __init__(self, step_dim,\n                 W_regularizer=None, b_regularizer=None,\n                 W_constraint=None, b_constraint=None,\n                 bias=True, **kwargs):\n        self.supports_masking = True\n        self.init = initializers.get('glorot_uniform')\n        self.W_regularizer = regularizers.get(W_regularizer)\n        self.b_regularizer = regularizers.get(b_regularizer)\n        self.W_constraint = constraints.get(W_constraint)\n        self.b_constraint = constraints.get(b_constraint)\n        self.bias = bias\n        self.step_dim = step_dim\n        self.features_dim = 0\n        super(Attention, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        assert len(input_shape) == 3\n        self.W = self.add_weight((input_shape[-1],),\n                                 initializer=self.init,\n                                 name='{}_W'.format(self.name),\n                                 regularizer=self.W_regularizer,\n                                 constraint=self.W_constraint)\n        self.features_dim = input_shape[-1]\n        if self.bias:\n            self.b = self.add_weight((input_shape[1],),\n                                     initializer='zero',\n                                     name='{}_b'.format(self.name),\n                                     regularizer=self.b_regularizer,\n                                     constraint=self.b_constraint)\n        else:\n            self.b = None\n        self.built = True\n\n    def compute_mask(self, input, input_mask=None):\n        return None\n\n    def call(self, x, mask=None):\n        features_dim = self.features_dim\n        step_dim = self.step_dim\n        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)),\n                        K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n        if self.bias:\n            eij += self.b\n        eij = K.tanh(eij)\n        a = K.exp(eij)\n        if mask is not None:\n            a *= K.cast(mask, K.floatx())\n        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n        a = K.expand_dims(a)\n        weighted_input = x * a\n        return K.sum(weighted_input, axis=1)\n\n    def compute_output_shape(self, input_shape):\n        return input_shape[0],  self.features_dim\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1c2361601f8d05011718f630b0126a2163da026f"},"cell_type":"markdown","source":"## capsule Layer"},{"metadata":{"trusted":true,"_uuid":"cce5f494761e5571dad42213e5870139c43d6fc0"},"cell_type":"code","source":"def squash(x, axis=-1):\n    s_squared_norm = K.sum(K.square(x), axis, keepdims=True)\n    scale = K.sqrt(s_squared_norm + K.epsilon())\n    return x / scale\n\nclass Capsule(Layer):\n    def __init__(self, num_capsule, dim_capsule, routings=3, kernel_size=(9, 1), share_weights=True,\n                 activation='default', **kwargs):\n        super(Capsule, self).__init__(**kwargs)\n        self.num_capsule = num_capsule\n        self.dim_capsule = dim_capsule\n        self.routings = routings\n        self.kernel_size = kernel_size\n        self.share_weights = share_weights\n        if activation == 'default':\n            self.activation = squash\n        else:\n            self.activation = Activation(activation)\n\n    def build(self, input_shape):\n        super(Capsule, self).build(input_shape)\n        input_dim_capsule = input_shape[-1]\n        if self.share_weights:\n            self.W = self.add_weight(name='capsule_kernel',\n                                     shape=(1, input_dim_capsule,\n                                            self.num_capsule * self.dim_capsule),\n                                     # shape=self.kernel_size,\n                                     initializer='glorot_uniform',\n                                     trainable=True)\n        else:\n            input_num_capsule = input_shape[-2]\n            self.W = self.add_weight(name='capsule_kernel',\n                                     shape=(input_num_capsule,\n                                            input_dim_capsule,\n                                            self.num_capsule * self.dim_capsule),\n                                     initializer='glorot_uniform',\n                                     trainable=True)\n\n    def call(self, u_vecs):\n        if self.share_weights:\n            u_hat_vecs = K.conv1d(u_vecs, self.W)\n        else:\n            u_hat_vecs = K.local_conv1d(u_vecs, self.W, [1], [1])\n\n        batch_size = K.shape(u_vecs)[0]\n        input_num_capsule = K.shape(u_vecs)[1]\n        u_hat_vecs = K.reshape(u_hat_vecs, (batch_size, input_num_capsule,\n                                            self.num_capsule, self.dim_capsule))\n        u_hat_vecs = K.permute_dimensions(u_hat_vecs, (0, 2, 1, 3))\n        # final u_hat_vecs.shape = [None, num_capsule, input_num_capsule, dim_capsule]\n\n        b = K.zeros_like(u_hat_vecs[:, :, :, 0])  # shape = [None, num_capsule, input_num_capsule]\n        for i in range(self.routings):\n            b = K.permute_dimensions(b, (0, 2, 1))  # shape = [None, input_num_capsule, num_capsule]\n            c = K.softmax(b)\n            c = K.permute_dimensions(c, (0, 2, 1))\n            b = K.permute_dimensions(b, (0, 2, 1))\n            outputs = self.activation(tf.keras.backend.batch_dot(c, u_hat_vecs, [2, 2]))\n            if i < self.routings - 1:\n                b = tf.keras.backend.batch_dot(outputs, u_hat_vecs, [2, 3])\n\n        return outputs\n\n    def compute_output_shape(self, input_shape):\n        return (None, self.num_capsule, self.dim_capsule)\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0e127ba3cc289f658bbadbb10445f1f3d072e413"},"cell_type":"markdown","source":"## FC layer"},{"metadata":{"trusted":true,"_uuid":"640e2d1064096b8dc8d5e944429f93cec73d35b7"},"cell_type":"code","source":"from keras.layers import Wrapper\n# https://github.com/andry9454/KerasDropconnect\nclass DropConnect(Wrapper):\n    def __init__(self, layer, prob=1., **kwargs):\n        self.prob = prob\n        self.layer = layer\n        super(DropConnect, self).__init__(layer, **kwargs)\n        if 0. < self.prob < 1.:\n            self.uses_learning_phase = True\n\n    def build(self, input_shape):\n        if not self.layer.built:\n            self.layer.build(input_shape)\n            self.layer.built = True\n        super(DropConnect, self).build()\n\n    def compute_output_shape(self, input_shape):\n        return self.layer.compute_output_shape(input_shape)\n\n    def call(self, x):\n        if 0. < self.prob < 1.:\n            self.layer.kernel = K.in_train_phase(K.dropout(self.layer.kernel, self.prob), self.layer.kernel)\n            self.layer.bias = K.in_train_phase(K.dropout(self.layer.bias, self.prob), self.layer.bias)\n        return self.layer.call(x)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"09fcb30f5ec5e44a182b05ace9b213287b170d04"},"cell_type":"code","source":"def f1(y_true, y_pred):\n    '''\n    metric from here \n    https://stackoverflow.com/questions/43547402/how-to-calculate-f1-macro-in-keras\n    '''\n    def recall(y_true, y_pred):\n        \"\"\"Recall metric.\n\n        Only computes a batch-wise average of recall.\n\n        Computes the recall, a metric for multi-label classification of\n        how many relevant items are selected.\n        \"\"\"\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n        recall = true_positives / (possible_positives + K.epsilon())\n        return recall\n\n    def precision(y_true, y_pred):\n        \"\"\"Precision metric.\n\n        Only computes a batch-wise average of precision.\n\n        Computes the precision, a metric for multi-label classification of\n        how many selected items are relevant.\n        \"\"\"\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n        precision = true_positives / (predicted_positives + K.epsilon())\n        return precision\n    precision = precision(y_true, y_pred)\n    recall = recall(y_true, y_pred)\n    return 2*((precision*recall)/(precision+recall+K.epsilon()))\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"62ef3b445f85b8e06306edddba8f14c25e2dece5"},"cell_type":"markdown","source":"# load the pretrained parameters into GRU Layer and finture it"},{"metadata":{"trusted":true,"_uuid":"c2cdc86176448dda640fe6a56d7a64aedcaf5cd3"},"cell_type":"code","source":"def build_my_model(embedding_matrix):\n    inp = Input(shape=(maxlen,))\n    x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False)(inp)\n    x = SpatialDropout1D(rate=0.1)(x)\n    x = Bidirectional(CuDNNGRU(80, \n                                return_sequences=True, name = 'gru1'))(x)\n                                #kernel_initializer=glorot_normal(seed=1029), \n                                #recurrent_initializer=orthogonal(gain=1.0, seed=1029), name = 'gru1'))(x)\n\n    x_1 = Attention(maxlen)(x)\n    x_1 = DropConnect(Dense(32, activation=\"relu\"), prob=0.1)(x_1)\n    \n    x_2 = Capsule(num_capsule=10, dim_capsule=10, routings=4, share_weights=True)(x)\n    x_2 = Flatten()(x_2)\n    x_2 = DropConnect(Dense(32, activation=\"relu\"), prob=0.1)(x_2)\n\n    conc = concatenate([x_1, x_2])\n    #conc =  DropConnect(Dense(16, activation=\"relu\"), prob=0.1)(conc)\n    # conc = add([x_1, x_2])\n    outp = Dense(1, activation=\"sigmoid\")(conc)\n    model = Model(inputs=inp, outputs=outp)\n    model.compile(loss='binary_crossentropy', optimizer=Adam(lr=0.0006), metrics=[f1])\n    return model\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5ccb7a26e72d74c154d155e2cb48353ef0f10fc8"},"cell_type":"code","source":"def train_pred(model, train_X, train_y, val_X, val_y, epochs=2, callback=None):\n    for e in range(epochs):\n        start_time = time.time()\n        model.fit(train_X, train_y, batch_size=train_batch_size, epochs=1, callbacks=callback, verbose=0)\n        pred_val_y = model.predict([val_X], batch_size=2048, verbose=0)\n        pred_test_y = model.predict([test_X], batch_size=2048, verbose=0)      \n        total_time = (time.time() - start_time) / 60.0\n\n        best_thresh = 0.5\n        best_score = 0.0\n        for thresh in np.arange(0.1, 0.501, 0.01):\n            thresh = np.round(thresh, 2)\n            score = metrics.f1_score(val_y, (pred_val_y > thresh).astype(int))\n            if score > best_score:\n                best_thresh = thresh\n                best_score = score\n        \n        print(\"Epoch: \", e, \"-    Val F1 Score: {:.4f} Took {:.2f} minutes\".format(best_score, total_time))\n    \n    print('='*80)\n    return pred_val_y, pred_test_y, best_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ff63932e4f257d2f32b2a58af70e5499720c7a84"},"cell_type":"code","source":"count = 0\n\ntrain_meta = np.zeros(train_y.shape)\ntest_meta = np.zeros(test_X.shape[0])\nsplits = list(StratifiedKFold(n_splits=5, shuffle=True, random_state=SEED).split(train_X, train_y))\nfor idx, (train_idx, valid_idx) in enumerate(splits):\n        X_train = train_X[train_idx]\n        y_train = train_y[train_idx]\n        X_val = train_X[valid_idx]\n        y_val = train_y[valid_idx]\n        model = build_my_model(embedding_matrix)\n        model.load_weights(\"lm.hdf5\",by_name = True)\n        #if idx >= 2:\n        #    train_epochs = 5\n        pred_val_y, pred_test_y, best_score = train_pred(model, X_train, y_train, X_val, y_val, epochs=train_epochs)\n        #if best_score > 0.68:\n        train_meta[valid_idx] = pred_val_y.reshape(-1)\n        test_meta += pred_test_y.reshape(-1)\n        count += 1\nif count > 0:\n    test_meta = test_meta / count \nprint(\"count is {}\".format(count))\n\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0c832c9c1bd319343ea3b4dc35038dada869d2fd"},"cell_type":"code","source":"\n\ndef threshold_search(y_true, y_proba):\n    best_threshold = 0\n    best_score = 0\n    for threshold in [i * 0.01 for i in range(100)]:\n        score = f1_score(y_true=y_true, y_pred=y_proba > threshold)\n        if score > best_score:\n            best_threshold = threshold\n            best_score = score\n    search_result = {'threshold': best_threshold, 'f1': best_score}\n    return search_result\n\n\n\n\nsearch_result = threshold_search(train_y, train_meta)\nprint(search_result)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"06bc31fa2f64173f785d6b9fd9bf90c46f27994c"},"cell_type":"code","source":"sub = pd.read_csv('../input/sample_submission.csv')\nsub.prediction = test_meta > search_result['threshold']\nsub.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8e9d222a4901464a12aea357379af28e30aef2d3"},"cell_type":"markdown","source":""},{"metadata":{"_uuid":"382a091652563db6b622e723e94a5b74c9ebd48e"},"cell_type":"markdown","source":""},{"metadata":{"_uuid":"89fe0759e724a4f762c0b6c3b34e1a7532c38266"},"cell_type":"markdown","source":""}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}