{"cells":[{"metadata":{"_uuid":"6d5221cf58859cd6496cf4294414a3bc37d4c95f"},"cell_type":"markdown","source":"In this kernel, I have implemented the encoder part of the transformer architecture as mentioned in the famous paper: Attention is all you need.(https://arxiv.org/abs/1706.03762).\n\nMany of other codes are adopted from other kernels. For example, loading the embeddings,  load the training and test data and preprocessing, etc. I really appreciate their contributions.\n\np.s. When I run this locally, I get validation f1-score around 0.688.\n\nHappy transforming!"},{"metadata":{"_uuid":"9a947373c706a15ed71a686d92703b9677561894"},"cell_type":"markdown","source":"## Imports"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import os\nimport time\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom tqdm import tqdm\nimport math\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D\nfrom keras.layers import Bidirectional, GlobalMaxPool1D, GlobalMaxPooling1D, GlobalAveragePooling1D\nfrom keras.layers import Input, Embedding, Dense, Conv2D, MaxPool2D, concatenate\nfrom keras.layers import Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D\nfrom keras.layers import BatchNormalization, InputSpec, add\nfrom keras.optimizers import Adam\nfrom keras.models import Model, load_model\nfrom keras import backend as K\nfrom keras.engine.topology import Layer\nfrom keras import initializers, regularizers, constraints, optimizers, layers, activations\nfrom keras.callbacks import Callback, EarlyStopping, ModelCheckpoint\nfrom keras.utils import Sequence","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b37e0d09f42f5bc5ad73a366580f6b778c9aad5a"},"cell_type":"markdown","source":"## Some pre-configurations"},{"metadata":{"trusted":true,"_uuid":"c7498e1cc6e3dd7e7c58f24e10fd5ad1b06b4489"},"cell_type":"code","source":"embed_size = 300 # how big is each word vector\nmax_features = 95000 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 70 # max number of words in a question to use\nn_heads = 4 # Number of heads as in Multi-head attention","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"05385c91a5e5603c346bfe53010aa4cb0f3ddd4a"},"cell_type":"code","source":"def load_and_prec():\n    train_df = pd.read_csv(\"../input/train.csv\")\n    test_df = pd.read_csv(\"../input/test.csv\")\n    print(\"Train shape : \",train_df.shape)\n    print(\"Test shape : \",test_df.shape)\n    \n    ## split to train and val\n    train_df, val_df = train_test_split(train_df, test_size=0.001, random_state=2018) # hahaha\n\n\n    ## fill up the missing values\n    train_X = train_df[\"question_text\"].fillna(\"_##_\").values\n    val_X = val_df[\"question_text\"].fillna(\"_##_\").values\n    test_X = test_df[\"question_text\"].fillna(\"_##_\").values\n\n    ## Tokenize the sentences\n    tokenizer = Tokenizer(num_words=max_features)\n    tokenizer.fit_on_texts(list(train_X))\n    train_X = tokenizer.texts_to_sequences(train_X)\n    val_X = tokenizer.texts_to_sequences(val_X)\n    test_X = tokenizer.texts_to_sequences(test_X)\n\n    ## Pad the sentences \n    train_X = pad_sequences(train_X, maxlen=maxlen)\n    val_X = pad_sequences(val_X, maxlen=maxlen)\n    test_X = pad_sequences(test_X, maxlen=maxlen)\n\n    ## Get the target values\n    train_y = train_df['target'].values\n    val_y = val_df['target'].values  \n    \n    #shuffling the data\n    np.random.seed(2018)\n    trn_idx = np.random.permutation(len(train_X))\n    val_idx = np.random.permutation(len(val_X))\n\n    train_X = train_X[trn_idx]\n    val_X = val_X[val_idx]\n    train_y = train_y[trn_idx]\n    val_y = val_y[val_idx]    \n    \n    return train_X, val_X, test_X, train_y, val_y, tokenizer.word_index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"20682431e22eab3cbf634777cb0d2bc2730ab754"},"cell_type":"markdown","source":"## Load Embeddings"},{"metadata":{"trusted":true,"_uuid":"cf6dc22b3b2a9f2ec94f70e2aa5dfe36f6f142d3"},"cell_type":"code","source":"def load_glove(word_index):\n    EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = -0.005838499,0.48782197\n    embed_size = all_embs.shape[1]\n\n    # word_index = tokenizer.word_index\n    nb_words = min(max_features, len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    for word, i in tqdm(word_index.items()):\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n            \n    return embedding_matrix \n    \ndef load_fasttext(word_index):    \n    EMBEDDING_FILE = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE) if len(o)>100)\n\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n\n    # word_index = tokenizer.word_index\n    nb_words = min(max_features, len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    for word, i in tqdm(word_index.items()):\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n\n    return embedding_matrix\n\ndef load_para(word_index):\n    EMBEDDING_FILE = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore') if len(o)>100)\n\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = -0.0053247833,0.49346462\n    embed_size = all_embs.shape[1]\n    print(emb_mean,emb_std,\"para\")\n\n    # word_index = tokenizer.word_index\n    nb_words = min(max_features, len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    for word, i in tqdm(word_index.items()):\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n    \n    return embedding_matrix","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a35ee76c0f926bcdf817fcb91dbd20ee90007f06"},"cell_type":"markdown","source":"## Scaled Dot-product attention"},{"metadata":{"trusted":true,"_uuid":"92c050cb313508d5c88b288dd1561493bcfacbed"},"cell_type":"code","source":"class DotProdSelfAttention(Layer):\n    \"\"\"The self-attention layer as in 'Attention is all you need'.\n    paper reference: https://arxiv.org/abs/1706.03762\n    \n    \"\"\"\n    def __init__(self, units,\n                 activation=None,\n                 use_bias=False,\n                 kernel_initializer='glorot_uniform',\n                 bias_initializer='zeros',\n                 kernel_regularizer=None,\n                 bias_regularizer=None,\n                 activity_regularizer=None,\n                 kernel_constraint=None,\n                 bias_constraint=None,\n                 **kwargs):\n        if 'input_shape' not in kwargs and 'input_dim' in kwargs:\n            kwargs['input_shape'] = (kwargs.pop('input_dim'),)\n        super(DotProdSelfAttention, self).__init__(*kwargs)\n        self.units = units\n        self.activation = activations.get(activation)\n        self.use_bias = use_bias\n        self.kernel_initializer = initializers.get(kernel_initializer)\n        self.bias_initializer = initializers.get(bias_initializer)\n        self.kernel_regularizer = regularizers.get(kernel_regularizer)\n        self.bias_regularizer = regularizers.get(bias_regularizer)\n        self.activity_regularizer = regularizers.get(activity_regularizer)\n        self.kernel_constraint = constraints.get(kernel_constraint)\n        self.bias_constraint = constraints.get(bias_constraint)\n        self.input_spec = InputSpec(min_ndim=2)\n        self.supports_masking = True\n\n    def build(self, input_shape):\n        assert len(input_shape) == 3\n        input_dim = input_shape[-1]\n        # We assume the output-dim of Q, K, V are the same\n        self.kernels = dict.fromkeys(['Q', 'K', 'V'])\n        for key, _ in self.kernels.items():\n            self.kernels[key] = self.add_weight(shape=(input_dim, self.units),\n                                                initializer=self.kernel_initializer,\n                                                name='kernel_{}'.format(key),\n                                                regularizer=self.kernel_regularizer,\n                                                constraint=self.kernel_constraint)\n        if self.use_bias:\n            raise NotImplementedError\n        super(DotProdSelfAttention, self).build(input_shape)\n        \n    def call(self, x):\n        Q = K.dot(x, self.kernels['Q'])\n        K_mat = K.dot(x, self.kernels['K'])\n        V = K.dot(x, self.kernels['V'])\n        attention = K.batch_dot(Q, K.permute_dimensions(K_mat, [0, 2, 1]))\n        d_k = K.constant(self.units, dtype=K.floatx())\n        attention = attention / K.sqrt(d_k)\n        attention = K.batch_dot(K.softmax(attention, axis=-1), V)\n        return attention\n    \n    def compute_output_shape(self, input_shape):\n        assert input_shape and len(input_shape) >= 2\n        assert input_shape[-1]\n        output_shape = list(input_shape)\n        output_shape[-1] = self.units\n        return tuple(output_shape)\n      \n        \n    ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d931bd69e53756c2f8aa8cf63d07d4c7eb02a8d9"},"cell_type":"markdown","source":"## The Encoder Block"},{"metadata":{"trusted":true,"_uuid":"399ddb0b7367ac0f6bd4121396b24ac3a6d1edfe"},"cell_type":"code","source":"def encoder(input_tensor):\n    \"\"\"One encoder as in Attention Is All You Need\n    \"\"\"\n    # Sub-layer 1\n    # Multi-Head Attention\n    multiheads = []\n    d_v = embed_size // n_heads\n    for i in range(n_heads):\n        multiheads.append(DotProdSelfAttention(d_v)(input_tensor))\n    multiheads = concatenate(multiheads, axis=-1)\n    multiheads = Dense(embed_size)(multiheads)\n    multiheads = Dropout(0.1)(multiheads)\n    \n    # Residual Connection\n    res_con = add([input_tensor, multiheads])\n    # Didn't use layer normalization, use Batch Normalization instead here\n    res_con = BatchNormalization(axis=-1)(res_con)\n    \n    # Sub-layer 2\n    # 2 Feed forward layer\n    ff1 = Dense(64, activation='relu')(res_con)\n    ff2 = Dense(embed_size)(ff1)\n    output = add([res_con, ff2])\n    output = BatchNormalization(axis=-1)(output)\n    \n    return output","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8740be2622e8195ed3f0c7d9568a06ddb64cd98b"},"cell_type":"markdown","source":"## Positional Encoding"},{"metadata":{"trusted":true,"_uuid":"8bd0cd11b96080f965195da697ba34c865f5b7e2"},"cell_type":"code","source":"# https://github.com/kpot/keras-transformer/blob/master/keras_transformer/position.py\ndef positional_signal(hidden_size: int, length: int,\n                      min_timescale: float = 1.0, max_timescale: float = 1e4):\n    \"\"\"\n    Helper function, constructing basic positional encoding.\n    The code is partially based on implementation from Tensor2Tensor library\n    https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/layers/common_attention.py\n    \"\"\"\n\n    if hidden_size % 2 != 0:\n        raise ValueError(\n            f\"The hidden dimension of the model must be divisible by 2.\"\n            f\"Currently it is {hidden_size}\")\n    position = K.arange(0, length, dtype=K.floatx())\n    num_timescales = hidden_size // 2\n    log_timescale_increment = K.constant(\n        (np.log(float(max_timescale) / float(min_timescale)) /\n         (num_timescales - 1)),\n        dtype=K.floatx())\n    inv_timescales = (\n            min_timescale *\n            K.exp(K.arange(num_timescales, dtype=K.floatx()) *\n                  -log_timescale_increment))\n    scaled_time = K.expand_dims(position, 1) * K.expand_dims(inv_timescales, 0)\n    signal = K.concatenate([K.sin(scaled_time), K.cos(scaled_time)], axis=1)\n    return K.expand_dims(signal, axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"309135be6416acde8d5088af910eee3597e25d4e"},"cell_type":"code","source":"# https://github.com/kpot/keras-transformer/blob/master/keras_transformer/position.py\nclass AddPositionalEncoding(Layer):\n    \"\"\"\n    Injects positional encoding signal described in section 3.5 of the original\n    paper \"Attention is all you need\". Also a base class for more complex\n    coordinate encoding described in \"Universal Transformers\".\n    \"\"\"\n\n    def __init__(self, min_timescale: float = 1.0,\n                 max_timescale: float = 1.0e4, **kwargs):\n        self.min_timescale = min_timescale\n        self.max_timescale = max_timescale\n        self.signal = None\n        super().__init__(**kwargs)\n\n    def get_config(self):\n        config = super().get_config()\n        config['min_timescale'] = self.min_timescale\n        config['max_timescale'] = self.max_timescale\n        return config\n\n    def build(self, input_shape):\n        _, length, hidden_size = input_shape\n        self.signal = positional_signal(\n            hidden_size, length, self.min_timescale, self.max_timescale)\n        return super().build(input_shape)\n\n    def call(self, inputs, **kwargs):\n        return inputs + self.signal\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"312bd7b8472de749134273fead7f939a234c551c"},"cell_type":"markdown","source":"## Transformer Encoder model"},{"metadata":{"trusted":true,"_uuid":"235ae9ecb6873475900b1ad88e3a07a161195370"},"cell_type":"code","source":"def model_transformer(embedding_matrix, n_encoder=3):\n    inp = Input(shape=(maxlen,))\n    x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=True)(inp)\n    # Add positional encoding\n    x = AddPositionalEncoding()(x)\n    x = Dropout(0.1)(x)\n    for i in range(n_encoder):\n        x = encoder(x)\n    # These are my own experiments\n    avg_pool = GlobalAveragePooling1D()(x)\n    max_pool = GlobalMaxPooling1D()(x)\n    conc = concatenate([avg_pool, max_pool])\n    conc = Dense(64, activation=\"relu\")(conc)\n    conc = Dropout(0.1)(conc)\n    outp = Dense(1, activation=\"sigmoid\")(conc)\n    \n    model = Model(inputs=inp, outputs=outp)\n    model.compile(loss='binary_crossentropy', optimizer='adam',\n                  metrics=['accuracy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e2f42600ff37e6c286215562afc11a1be28f0981"},"cell_type":"code","source":"# https://stanford.edu/~shervine/blog/keras-how-to-generate-data-on-the-fly\nclass BaseDataGenerator(Sequence):\n    \"\"\"A data generator\"\"\"\n    def __init__(self, list_IDs, batch_size=128, shuffle=True):\n        self.list_IDs = list_IDs\n        self.batch_size = batch_size\n        self.shuffle = shuffle\n        self.on_epoch_end()\n\n    def __len__(self):\n        \"\"\"number of steps in one epoch\"\"\"\n        # Here is the trick\n        return len(self.list_IDs) // (self.batch_size * 2**2)\n\n    def __getitem__(self, index):\n        'Generate one batch of data'\n        indexes = self.indexes[index*self.batch_size: (index+1)*self.batch_size]\n        list_IDs_temp = [self.list_IDs[k] for k in indexes]\n\n        # Generate data\n        X, y = self.__data_generation(list_IDs_temp)\n        return X, y\n\n    def on_epoch_end(self):\n        'Updates indexes after each epoch'\n        self.indexes = np.arange(len(self.list_IDs))\n        if self.shuffle:\n            np.random.shuffle(self.indexes)\n\n    def __data_generation(self, list_IDs_temp):\n        'Generates data containing batch_size samples' \n        X = train_X[list_IDs_temp, :]\n        y = train_y[list_IDs_temp]\n        return X, y","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ee0611a48264188ec3063bc6c6ce221eb3724d8e"},"cell_type":"markdown","source":"### Train and Predict"},{"metadata":{"_uuid":"8c9d6121293c82701f3c07ae00f396564d8aa2c9"},"cell_type":"markdown","source":"Here I used early stopping and model checkpoint to load the best_val model"},{"metadata":{"trusted":true,"_uuid":"5f90446889d864f0a318d305eca2d3a04d1baa55"},"cell_type":"code","source":"# https://www.kaggle.com/strideradu/word2vec-and-gensim-go-go-go\ndef train_pred(model, epochs=2):\n    # learning schedule callback\n#     loss_history = LossHistory()\n#     lrate = BatchLRScheduler(step_decay)\n#     callbacks_list = [loss_history, lrate]\n#     es = EarlyStopping(monitor='val_loss', min_delta=0, patience=5)\n#     model_path = 'keras_models.h5'\n#     mc = ModelCheckpoint(filepath=model_path, monitor='val_loss', save_best_only=True)\n#     callbacks = [es, mc]\n#     train_generator = BaseDataGenerator(list(np.arange(train_X.shape[0])), batch_size=512)\n#     model.fit_generator(train_generator,\n#                         epochs=epochs,\n#                         validation_data=(val_X, val_y),)\n#                         callbacks=callbacks)\n#     model = load_model(model_path)\n    model.fit(train_X, train_y, batch_size=512,\n              epochs=epochs,\n              validation_data=(val_X, val_y),)\n\n    pred_val_y = model.predict([val_X], batch_size=1024, verbose=0)\n    pred_test_y = model.predict([test_X], batch_size=1024, verbose=0)\n    return pred_val_y, pred_test_y","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d9b632c9bea6c5e8c7e111fc97474fc9d8b099c4"},"cell_type":"markdown","source":"### Main part: load, train, pred and blend "},{"metadata":{"trusted":true,"_uuid":"6ddf2ae0c374759ef040db80346faaf609850e58"},"cell_type":"code","source":"train_X, val_X, test_X, train_y, val_y, word_index = load_and_prec()\nvocab = []\nfor w,k in word_index.items():\n    vocab.append(w)\n    if k >= max_features:\n        break\nembedding_matrix_1 = load_glove(word_index)\n# embedding_matrix_2 = load_fasttext(word_index)\nembedding_matrix_3 = load_para(word_index)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ffc83124a9f220b31e698b922cad56d59f8c83e5"},"cell_type":"markdown","source":"### Create New Embeddings"},{"metadata":{"trusted":true,"_uuid":"622e0a2f777d15a9f74b3d1d56a0e5ad60ee90bc"},"cell_type":"code","source":"## Simple average: http://aclweb.org/anthology/N18-2031\n\n# We have presented an argument for averaging as\n# a valid meta-embedding technique, and found experimental\n# performance to be close to, or in some cases \n# better than that of concatenation, with the\n# additional benefit of reduced dimensionality  \n\n\n## Unweighted DME in https://arxiv.org/pdf/1804.07983.pdf\n\n# “The downside of concatenating embeddings and \n#  giving that as input to an RNN encoder, however,\n#  is that the network then quickly becomes inefficient\n#  as we combine more and more embeddings.”\n  \n# embedding_matrix = np.mean([embedding_matrix_1, embedding_matrix_2, embedding_matrix_3], axis = 0)\nembedding_matrix = np.mean([embedding_matrix_1, embedding_matrix_3], axis = 0)\nnp.shape(embedding_matrix)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"52627f8490364ed29f81e574c49af644726701e1"},"cell_type":"markdown","source":"## Train and Predict"},{"metadata":{"_uuid":"c3270c045c747a7c5da32704e6db44c18f470646"},"cell_type":"markdown","source":"Here I am experimenting with 2 encoders, it's not guaranteed to be optimal, you can try out other numbers. Notice that I used epochs = 8"},{"metadata":{"trusted":true,"_uuid":"573d8e65c007aefd1fb4a0deef16e73894327486"},"cell_type":"code","source":"outputs = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"091d7915c8c00d088d60c09c306298950cd2afe6"},"cell_type":"code","source":"n_encoder = 1\npred_val_y, pred_test_y = train_pred(model_transformer(embedding_matrix, n_encoder=n_encoder), epochs = 3)\noutputs.append([pred_val_y, pred_test_y, 'transformer_enc{}'.format(n_encoder)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e3dc6e5362fa13e00b291986b94ea9d6e5acdebf"},"cell_type":"code","source":"for thresh in np.arange(0.1, 0.51, 0.01):\n    thresh = np.round(thresh, 2)\n    print(\"F1 score at threshold {0:.2f} is {1}\".format(thresh, metrics.f1_score(val_y, (pred_val_y>thresh).astype(int))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"43e0969fd3af4b65decfbba8cce0a8a36d552176"},"cell_type":"code","source":"pred_test_y = (pred_test_y > 0.42).astype(int)\ntest_df = pd.read_csv(\"../input/test.csv\", usecols=[\"qid\"])\nout_df = pd.DataFrame({\"qid\":test_df[\"qid\"].values})\nout_df['prediction'] = pred_test_y\nout_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c378fe1cfd53529831f3928fb9866329eb7a9184"},"cell_type":"code","source":"idx = (pred_test_y > 0.42).astype(int)\ntest_df = pd.read_csv(\"../input/test.csv\", usecols=[\"qid\"])\nout_df = pd.DataFrame({\"qid\":test_df[\"qid\"].values})\nout_df['prediction'] = idx","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c378fe1cfd53529831f3928fb9866329eb7a9184"},"cell_type":"code","source":"mylist = out_df[out_df.prediction == 1].index\nfor i in mylist:\n    print(i, end=',')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1a3b5a15d3745fa2ffb2e0bfd1c98ba890e27550"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}