{"cells":[{"metadata":{"_uuid":"e154a47bf09b8770980486e87786317a1b3038e1"},"cell_type":"markdown","source":"## TL;DR\n\n- Feature engineering: https://www.kaggle.com/braquino/5-fold-lstm-attention-fully-commented-0-694\n\n- Transformer kernel from Quora competition: https://www.kaggle.com/shujian/transformer-initial-attempt\n\n- And of course: https://arxiv.org/abs/1706.03762\n\n## Major changes to the original Transformer architecture:\n\n- No decoder, basically we cut it in half. We're doing classical classification not auto-regression like in machine translation so it's not needed.\n\n- No masked attention. Like above, we do not need to ensure causality in inference like machine transalation, so we can afford to make the information flow freely."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport pyarrow.parquet as pq # Used to read the data\nimport os \nimport numpy as np\nfrom keras.layers import * # Keras is the most friendly Neural Network library, this Kernel use a lot of layers classes\nfrom keras.models import Model\nfrom tqdm import tqdm # Processing time measurement\nfrom sklearn.model_selection import train_test_split \nfrom keras import backend as K # The backend give us access to tensorflow operations and allow us to create the Attention class\nfrom keras import optimizers # Allow us to access the Adam class to modify some parameters\nfrom sklearn.model_selection import GridSearchCV, StratifiedKFold # Used to use Kfold to train our model\nfrom keras.callbacks import * # This object helps the model to train in a smarter way, avoiding overfitting","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6e6379386e44afc69bee8895a52da22199e888fb"},"cell_type":"code","source":"# select how many folds will be created\nN_SPLITS = 5\n# it is just a constant with the measurements data size\nsample_size = 800000","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c3340ee96becb5ca8f075d9c44b7df383ddba5ee"},"cell_type":"code","source":"# It is the official metric used in this competition\n# below is the declaration of a function used inside the keras model, calculation with K (keras backend / thensorflow)\nepsilon = K.epsilon()\ndef matthews_correlation(y_true, y_pred, K=K):\n    '''Calculates the Matthews correlation coefficient measure for quality\n    of binary classification problems.\n    '''\n    y_pred_pos = K.round(K.clip(y_pred, 0, 1))\n    y_pred_neg = 1 - y_pred_pos\n\n    y_pos = K.round(K.clip(y_true, 0, 1))\n    y_neg = 1 - y_pos\n\n    tp = K.sum(y_pos * y_pred_pos)\n    tn = K.sum(y_neg * y_pred_neg)\n\n    fp = K.sum(y_neg * y_pred_pos)\n    fn = K.sum(y_pos * y_pred_neg)\n\n    numerator = (tp * tn - fp * fn)\n    denominator = K.sqrt((tp + fp) * (tp + fn) * (tn + fp) * (tn + fn))\n\n    return numerator / (denominator + epsilon)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# just load train data\ndf_train = pd.read_csv('../input/vsb-power-line-fault-detection/metadata_train.csv')\n# set index, it makes the data access much faster\ndf_train = df_train.set_index(['id_measurement', 'phase'])\ndf_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"26df6c7fbfecd537404866faec13d1238ae3ebc6"},"cell_type":"code","source":"# in other notebook I have extracted the min and max values from the train data, the measurements\nmax_num = 127\nmin_num = -128","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7b0717b14bcfcba1f48d33c8161ae51c778687af"},"cell_type":"code","source":"# This function standardize the data from (-128 to 127) to (-1 to 1)\n# Theoretically it helps in the NN Model training, but I didn't tested without it\ndef min_max_transf(ts, min_data, max_data, range_needed=(-1,1)):\n    if min_data < 0:\n        ts_std = (ts + abs(min_data)) / (max_data + abs(min_data))\n    else:\n        ts_std = (ts - min_data) / (max_data - min_data)\n    if range_needed[0] < 0:    \n        return ts_std * (range_needed[1] + abs(range_needed[0])) + range_needed[0]\n    else:\n        return ts_std * (range_needed[1] - range_needed[0]) + range_needed[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c6137bbbe75c3a1509a5f98e08805dbbd492aa37"},"cell_type":"code","source":"# This is one of the most important peace of code of this Kernel\n# Any power line contain 3 phases of 800000 measurements, or 2.4 millions data \n# It would be praticaly impossible to build a NN with an input of that size\n# The ideia here is to reduce it each phase to a matrix of <n_dim> bins by n features\n# Each bean is a set of 5000 measurements (800000 / 160), so the features are extracted from this 5000 chunk data.\ndef transform_ts(ts, n_dim=160, min_max=(-1,1)):\n    # convert data into -1 to 1\n    ts_std = min_max_transf(ts, min_data=min_num, max_data=max_num)\n    # bucket or chunk size, 5000 in this case (800000 / 160)\n    bucket_size = int(sample_size / n_dim)\n    # new_ts will be the container of the new data\n    new_ts = []\n    # this for iteract any chunk/bucket until reach the whole sample_size (800000)\n    for i in range(0, sample_size, bucket_size):\n        # cut each bucket to ts_range\n        ts_range = ts_std[i:i + bucket_size]\n        # calculate each feature\n        mean = ts_range.mean()\n        std = ts_range.std() # standard deviation\n        std_top = mean + std # I have to test it more, but is is like a band\n        std_bot = mean - std\n        # I think that the percentiles are very important, it is like a distribuiton analysis from eath chunk\n        percentil_calc = np.percentile(ts_range, [0, 1, 25, 50, 75, 99, 100]) \n        max_range = percentil_calc[-1] - percentil_calc[0] # this is the amplitude of the chunk\n        relative_percentile = percentil_calc - mean # maybe it could heap to understand the asymmetry\n        # now, we just add all the features to new_ts and convert it to np.array\n        new_ts.append(np.concatenate([np.asarray([mean, std, std_top, std_bot, max_range]),percentil_calc, relative_percentile]))\n    return np.asarray(new_ts)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7460e718a605803f1d9e4fbec61750a0deb02a47"},"cell_type":"code","source":"# this function take a piece of data and convert using transform_ts(), but it does to each of the 3 phases\n# if we would try to do in one time, could exceed the RAM Memmory\ndef prep_data(start, end):\n    # load a piece of data from file\n    praq_train = pq.read_pandas('../input/train.parquet', columns=[str(i) for i in range(start, end)]).to_pandas()\n    X = []\n    y = []\n    # using tdqm to evaluate processing time\n    # takes each index from df_train and iteract it from start to end\n    # it is divided by 3 because for each id_measurement there are 3 id_signal, and the start/end parameters are id_signal\n    for id_measurement in tqdm(df_train.index.levels[0].unique()[int(start/3):int(end/3)]):\n        X_signal = []\n        # for each phase of the signal\n        for phase in [0,1,2]:\n            # extract from df_train both signal_id and target to compose the new data sets\n            signal_id, target = df_train.loc[id_measurement].loc[phase]\n            # but just append the target one time, to not triplicate it\n            if phase == 0:\n                y.append(target)\n            # extract and transform data into sets of features\n            X_signal.append(transform_ts(praq_train[str(signal_id)]))\n        # concatenate all the 3 phases in one matrix\n        X_signal = np.concatenate(X_signal, axis=1)\n        # add the data to X\n        X.append(X_signal)\n    X = np.asarray(X)\n    y = np.asarray(y)\n    return X, y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"51ad0e25b00536de6170168499923d82ae1d735f"},"cell_type":"code","source":"X = np.load(\"../input/5-fold-lstm-attention-fully-commented-0-694/X.npy\")\ny = np.load(\"../input/5-fold-lstm-attention-fully-commented-0-694/y.npy\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"289bc7d1ab8048a60025801b457f8df1d848acbc"},"cell_type":"code","source":"import random, os, sys\nimport numpy as np\nfrom keras.models import *\nfrom keras.layers import *\nfrom keras.callbacks import *\nfrom keras.initializers import *\nimport tensorflow as tf\nfrom keras.engine.topology import Layer\n\ntry:\n    from dataloader import TokenList, pad_to_longest\n    # for transformer\nexcept: pass\n\nclass LayerNormalization(Layer):\n    def __init__(self, eps=1e-6, **kwargs):\n        self.eps = eps\n        super(LayerNormalization, self).__init__(**kwargs)\n    def build(self, input_shape):\n        self.gamma = self.add_weight(name='gamma', shape=input_shape[-1:],\n                                     initializer=Ones(), trainable=True)\n        self.beta = self.add_weight(name='beta', shape=input_shape[-1:],\n                                    initializer=Zeros(), trainable=True)\n        super(LayerNormalization, self).build(input_shape)\n    def call(self, x):\n        mean = K.mean(x, axis=-1, keepdims=True)\n        std = K.std(x, axis=-1, keepdims=True)\n        return self.gamma * (x - mean) / (std + self.eps) + self.beta\n    def compute_output_shape(self, input_shape):\n        return input_shape\n\nclass ScaledDotProductAttention():\n    def __init__(self, d_model, attn_dropout=0.1):\n        self.temper = np.sqrt(d_model)\n        self.dropout = Dropout(attn_dropout)\n    def __call__(self, q, k, v, mask):\n        attn = Lambda(lambda x:K.batch_dot(x[0],x[1],axes=[2,2])/self.temper)([q, k])\n        if mask is not None:\n            mmask = Lambda(lambda x:(-1e+10)*(1-x))(mask)\n            attn = Add()([attn, mmask])\n        attn = Activation('softmax')(attn)\n        attn = self.dropout(attn)\n        output = Lambda(lambda x:K.batch_dot(x[0], x[1]))([attn, v])\n        return output, attn\n\nclass MultiHeadAttention():\n    # mode 0 - big martixes, faster; mode 1 - more clear implementation\n    def __init__(self, n_head, d_model, d_k, d_v, dropout, mode=0, use_norm=True):\n        self.mode = mode\n        self.n_head = n_head\n        self.d_k = d_k\n        self.d_v = d_v\n        self.dropout = dropout\n        if mode == 0:\n            self.qs_layer = Dense(n_head*d_k, use_bias=False)\n            self.ks_layer = Dense(n_head*d_k, use_bias=False)\n            self.vs_layer = Dense(n_head*d_v, use_bias=False)\n        elif mode == 1:\n            self.qs_layers = []\n            self.ks_layers = []\n            self.vs_layers = []\n            for _ in range(n_head):\n                self.qs_layers.append(TimeDistributed(Dense(d_k, use_bias=False)))\n                self.ks_layers.append(TimeDistributed(Dense(d_k, use_bias=False)))\n                self.vs_layers.append(TimeDistributed(Dense(d_v, use_bias=False)))\n        self.attention = ScaledDotProductAttention(d_model)\n        self.layer_norm = LayerNormalization() if use_norm else None\n        self.w_o = TimeDistributed(Dense(d_model))\n\n    def __call__(self, q, k, v, mask=None):\n        d_k, d_v = self.d_k, self.d_v\n        n_head = self.n_head\n\n        if self.mode == 0:\n            qs = self.qs_layer(q)  # [batch_size, len_q, n_head*d_k]\n            ks = self.ks_layer(k)\n            vs = self.vs_layer(v)\n\n            def reshape1(x):\n                s = tf.shape(x)   # [batch_size, len_q, n_head * d_k]\n                x = tf.reshape(x, [s[0], s[1], n_head, d_k])\n                x = tf.transpose(x, [2, 0, 1, 3])  \n                x = tf.reshape(x, [-1, s[1], d_k])  # [n_head * batch_size, len_q, d_k]\n                return x\n            qs = Lambda(reshape1)(qs)\n            ks = Lambda(reshape1)(ks)\n            vs = Lambda(reshape1)(vs)\n\n            if mask is not None:\n                mask = Lambda(lambda x:K.repeat_elements(x, n_head, 0))(mask)\n            head, attn = self.attention(qs, ks, vs, mask=mask)  \n                \n            def reshape2(x):\n                s = tf.shape(x)   # [n_head * batch_size, len_v, d_v]\n                x = tf.reshape(x, [n_head, -1, s[1], s[2]]) \n                x = tf.transpose(x, [1, 2, 0, 3])\n                x = tf.reshape(x, [-1, s[1], n_head*d_v])  # [batch_size, len_v, n_head * d_v]\n                return x\n            head = Lambda(reshape2)(head)\n        elif self.mode == 1:\n            heads = []; attns = []\n            for i in range(n_head):\n                qs = self.qs_layers[i](q)   \n                ks = self.ks_layers[i](k) \n                vs = self.vs_layers[i](v) \n                head, attn = self.attention(qs, ks, vs, mask)\n                heads.append(head); attns.append(attn)\n            head = Concatenate()(heads) if n_head > 1 else heads[0]\n            attn = Concatenate()(attns) if n_head > 1 else attns[0]\n\n        outputs = self.w_o(head)\n        outputs = Dropout(self.dropout)(outputs)\n        if not self.layer_norm: return outputs, attn\n        outputs = Add()([outputs, q])\n        return self.layer_norm(outputs), attn\n\nclass PositionwiseFeedForward():\n    def __init__(self, d_hid, d_inner_hid, dropout=0.1):\n        self.w_1 = Conv1D(d_inner_hid, 1, activation='relu')\n        self.w_2 = Conv1D(d_hid, 1)\n        self.layer_norm = LayerNormalization()\n        self.dropout = Dropout(dropout)\n    def __call__(self, x):\n        output = self.w_1(x) \n        output = self.w_2(output)\n        output = self.dropout(output)\n        output = Add()([output, x])\n        return self.layer_norm(output)\n\nclass EncoderLayer():\n    def __init__(self, d_model, d_inner_hid, n_head, d_k, d_v, dropout=0.1):\n        self.self_att_layer = MultiHeadAttention(n_head, d_model, d_k, d_v, dropout=dropout)\n        self.pos_ffn_layer  = PositionwiseFeedForward(d_model, d_inner_hid, dropout=dropout)\n    def __call__(self, enc_input, mask=None):\n        output, slf_attn = self.self_att_layer(enc_input, enc_input, enc_input, mask=mask)\n        output = self.pos_ffn_layer(output)\n        return output, slf_attn\n\ndef GetPosEncodingMatrix(max_len, d_emb):\n    pos_enc = np.array([\n        [pos / np.power(10000, 2 * (j // 2) / d_emb) for j in range(d_emb)] \n        if pos != 0 else np.zeros(d_emb) \n            for pos in range(max_len)\n            ])\n    pos_enc[1:, 0::2] = np.sin(pos_enc[1:, 0::2]) # dim 2i\n    pos_enc[1:, 1::2] = np.cos(pos_enc[1:, 1::2]) # dim 2i+1\n    return pos_enc\n\ndef GetPadMask(q, k):\n    ones = K.expand_dims(K.ones_like(q, 'float32'), -1)\n    mask = K.cast(K.expand_dims(K.not_equal(k, 0), 1), 'float32')\n    mask = K.batch_dot(ones, mask, axes=[2,1])\n    return mask\n\ndef GetSubMask(s):\n    len_s = tf.shape(s)[1]\n    bs = tf.shape(s)[:1]\n    mask = K.cumsum(tf.eye(len_s, batch_shape=bs), 1)\n    return mask\n\nclass Encoder():\n    def __init__(self, d_model, d_inner_hid, n_head, d_k, d_v, \\\n                layers=6, dropout=0.1, pos_emb=None):\n        self.pos_layer = pos_emb\n        self.dropout = Dropout(dropout)\n        self.layers = [EncoderLayer(d_model, d_inner_hid, n_head, d_k, d_v, dropout) for _ in range(layers)]\n        \n    def __call__(self, src_seq, src_pos, return_att=False, active_layers=999):\n        x = src_seq\n        if src_pos is not None:\n            pos = self.pos_layer(src_pos)\n            x = Add()([x, pos])\n        x = self.dropout(x)\n        if return_att: atts = []\n        # mask = Lambda(lambda x:GetPadMask(x, x))(src_seq)\n        mask = None\n        for enc_layer in self.layers[:active_layers]:\n            x, att = enc_layer(x, mask)\n            if return_att: atts.append(att)\n        return (x, atts) if return_att else x\n\n\nclass Transformer():\n    def __init__(self, len_limit, d_model=X.shape[-1], \\\n              d_inner_hid=512, n_head=8, d_k=64, d_v=64, layers=2, dropout=0.1, \\\n              share_word_emb=False, **kwargs):\n        self.name = 'Transformer'\n        self.len_limit = len_limit\n        self.src_loc_info = True\n        self.d_model = d_model\n        d_emb = d_model\n\n        pos_emb = Embedding(len_limit, d_emb, trainable=False, \\\n                            weights=[GetPosEncodingMatrix(len_limit, d_emb)])\n\n        self.encoder = Encoder(d_model, d_inner_hid, n_head, d_k, d_v, layers, dropout, \\\n                            pos_emb=pos_emb)\n\n        \n    def get_pos_seq(self, x):\n        pos = K.cumsum(K.ones_like(x[...,0], 'int32'), 1)\n        return pos\n\n    def compile(self, active_layers=999):\n        src_seq = Input(shape=(X.shape[1],X.shape[2]))\n        src_pos = Lambda(self.get_pos_seq)(src_seq)\n        if not self.src_loc_info: src_pos = None\n\n        x = self.encoder(src_seq, src_pos, active_layers=active_layers)\n        x = GlobalAveragePooling1D()(x)\n#         x = concatenate([GlobalMaxPool1D()(x),GlobalAveragePooling1D()(x)])\n#         print(x.shape)\n#         x = Reshape((X.shape[1]*X.shape[2],))(x)\n        outp = Dense(1, activation=\"sigmoid\")(x)\n\n        self.model = Model(inputs=src_seq, outputs=outp)\n        self.model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[matthews_correlation])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d6f4ca319c383b1b4f671a37c5a324136e7a466"},"cell_type":"code","source":"# Here is where the training happens\n\n# First, create a set of indexes of the 5 folds\nsplits = list(StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=2019).split(X, y))\npreds_val = []\ny_val = []\n# Then, iteract with each fold\n# If you dont know, enumerate(['a', 'b', 'c']) returns [(0, 'a'), (1, 'b'), (2, 'c')]\nfor idx, (train_idx, val_idx) in enumerate(splits):\n    K.clear_session() # I dont know what it do, but I imagine that it \"clear session\" :)\n    print(\"Beginning fold {}\".format(idx+1))\n    # use the indexes to extract the folds in the train and validation data\n    train_X, train_y, val_X, val_y = X[train_idx], y[train_idx], X[val_idx], y[val_idx]\n    # instantiate the model for this fold\n    model = Transformer(train_X.shape[1], layers=1,d_inner_hid=256)\n    model.compile()\n    model = model.model\n    # This checkpoint helps to avoid overfitting. It just save the weights of the model if it delivered an\n    # validation matthews_correlation greater than the last one.\n    ckpt = ModelCheckpoint('weights_{}.h5'.format(idx), save_best_only=True, save_weights_only=True, verbose=1, monitor='val_matthews_correlation', mode='max')\n    # Train, train, train\n    model.fit(train_X, train_y, batch_size=128, epochs=100, validation_data=[val_X, val_y], callbacks=[ckpt])\n    # loads the best weights saved by the checkpoint\n    model.load_weights('weights_{}.h5'.format(idx))\n    # Add the predictions of the validation to the list preds_val\n    preds_val.append(model.predict(val_X, batch_size=512))\n    # and the val true y\n    y_val.append(val_y)\n\n# concatenates all and prints the shape    \npreds_val = np.concatenate(preds_val)[...,0]\ny_val = np.concatenate(y_val)\npreds_val.shape, y_val.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d28151fd0be9fd9762f3f55e307d82f89bfbd291"},"cell_type":"code","source":"# The output of this kernel must be binary (0 or 1), but the output of the NN Model is float (0 to 1).\n# So, find the best threshold to convert float to binary is crucial to the result\n# this piece of code is a function that evaluates all the possible thresholds from 0 to 1 by 0.01\ndef threshold_search(y_true, y_proba):\n    best_threshold = 0\n    best_score = 0\n    for threshold in tqdm([i * 0.01 for i in range(100)]):\n        score = matthews_correlation(y_true, (y_proba > threshold).astype(np.float64),K=np)\n        if score > best_score:\n            best_threshold = threshold\n            best_score = score\n    search_result = {'threshold': best_threshold, 'matthews_correlation': best_score}\n    print(search_result)\n    return search_result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6fee7f722ed08bc1453a822a4371ed2d48e08abc"},"cell_type":"code","source":"best_threshold = threshold_search(y_val, preds_val)['threshold']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ae9bd3fa9d8c0781c0708846bb7f2a9f9e6cbd3c"},"cell_type":"code","source":"%%time\n# Now load the test data\n# This first part is the meta data, not the main data, the measurements\nmeta_test = pd.read_csv('../input/vsb-power-line-fault-detection/metadata_test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3eb186d032f79c99ffba05dd1a7fabb77e13cec5"},"cell_type":"code","source":"meta_test = meta_test.set_index(['signal_id'])\nmeta_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"af9aa6b2b8f8a2beda1a02ff998e3072fcad8d06"},"cell_type":"code","source":"\nX_test_input = np.load(\"../input/5-fold-lstm-attention-fully-commented-0-694/X_test.npy\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cfd265d3e07c4cc1679d2c4d55fe7de631c813e7"},"cell_type":"code","source":"submission = pd.read_csv('../input/vsb-power-line-fault-detection/sample_submission.csv')\nprint(len(submission))\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2f7342296138f6bfd3e9cedd029e1035de3b98fc"},"cell_type":"code","source":"preds_test = []\nfor i in range(N_SPLITS):\n    model.load_weights('weights_{}.h5'.format(i))\n    pred = model.predict(X_test_input, batch_size=300, verbose=1)\n    pred_3 = []\n    for pred_scalar in pred:\n        for i in range(3):\n            pred_3.append(pred_scalar)\n    preds_test.append(pred_3)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9f76c471eaf983707d446c5081ab3d50c4e40ea5"},"cell_type":"code","source":"preds_test = (np.squeeze(np.mean(preds_test, axis=0)) > best_threshold).astype(np.int)\npreds_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b35723f85d494b4b6ec630dd7c79135a110a4062"},"cell_type":"code","source":"submission['target'] = preds_test\nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d7600d0093a9880003240ef9ce0a1f1303e4d982"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}