{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This code is basicly based on this tutorial [Solve GLUE tasks using BERT on TPU](https://www.tensorflow.org/text/tutorials/bert_glue)\n:)","metadata":{}},{"cell_type":"code","source":"!pip install -q -U \"tensorflow-text==2.8.*\"\n\n!pip install -q -U tf-models-official==2.7.0\n\n!pip install -U tfds-nightly","metadata":{"execution":{"iopub.status.busy":"2022-07-09T13:26:21.401032Z","iopub.execute_input":"2022-07-09T13:26:21.401294Z","iopub.status.idle":"2022-07-09T13:26:51.911758Z","shell.execute_reply.started":"2022-07-09T13:26:21.401264Z","shell.execute_reply":"2022-07-09T13:26:51.910756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Importing the libraries\nimport os\nimport re\nimport string\nimport tensorflow as tf\nimport pandas as pd \nimport numpy as np\n\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n\nfrom sklearn.preprocessing import LabelEncoder\n\nimport tensorflow_hub as hub\nimport tensorflow_text as text  # A dependency of the preprocessing model\nimport tensorflow_addons as tfa\nimport tensorflow_datasets as tfds\n\nfrom official.nlp import optimization\n\nfrom sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2022-07-09T13:26:51.914036Z","iopub.execute_input":"2022-07-09T13:26:51.914323Z","iopub.status.idle":"2022-07-09T13:26:51.976017Z","shell.execute_reply.started":"2022-07-09T13:26:51.914288Z","shell.execute_reply":"2022-07-09T13:26:51.973996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# reading the data\nsubmission = pd.read_csv('../input/contradictory-my-dear-watson/sample_submission.csv')\n\ntrainPath = '../input/trnwatson/translated_train.csv'\ntestPath = '../input/trnwatson/translated_test.csv'\n\n# train = pd.read_csv('train.csv')\n# test = pd.read_csv('test.csv')\n\ntrain = pd.read_csv(trainPath)\ntest = pd.read_csv(testPath)\n\ntrain['premise'] = train['premise_en']\ntrain['hypothesis'] = train['hypothesis_en']\n\ntest['premise'] = test['premise_en']\ntest['hypothesis'] = test['hypothesis_en']","metadata":{"execution":{"iopub.status.busy":"2022-07-09T13:26:51.977564Z","iopub.status.idle":"2022-07-09T13:26:51.978002Z","shell.execute_reply.started":"2022-07-09T13:26:51.977781Z","shell.execute_reply":"2022-07-09T13:26:51.977804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Strategy to use TPU or GPU.')\ndef Init_TPU():  \n    try:\n        resolver = tf.distribute.cluster_resolver.TPUClusterResolver()\n        tf.config.experimental_connect_to_cluster(resolver)\n        tf.tpu.experimental.initialize_tpu_system(resolver)\n        strategy = tf.distribute.experimental.TPUStrategy(resolver)\n        REPLICAS = strategy.num_replicas_in_sync\n        print(\"Connected to TPU Successfully:\\n TPUs Initialised with Replicas:\",REPLICAS)\n        return strategy\n    except ValueError:\n        print(\"Connection to TPU Falied\")\n        print(\"Using default strategy for CPU and single GPU\")\n        strategy = tf.distribute.get_strategy()\n        return strategy\n    \nstrategy=Init_TPU()","metadata":{"execution":{"iopub.status.busy":"2022-07-09T13:26:51.979789Z","iopub.status.idle":"2022-07-09T13:26:51.980286Z","shell.execute_reply.started":"2022-07-09T13:26:51.980029Z","shell.execute_reply":"2022-07-09T13:26:51.980054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"map_name_to_handle = {\n    'bert_en_uncased_L-12_H-768_A-12':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/3',\n    'bert_en_cased_L-12_H-768_A-12':\n        'https://tfhub.dev/tensorflow/bert_en_cased_L-12_H-768_A-12/3',\n    'bert_multi_cased_L-12_H-768_A-12':\n        'https://tfhub.dev/tensorflow/bert_multi_cased_L-12_H-768_A-12/3',\n    'small_bert/bert_en_uncased_L-2_H-128_A-2':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-128_A-2/1',\n    'small_bert/bert_en_uncased_L-2_H-256_A-4':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-256_A-4/1',\n    'small_bert/bert_en_uncased_L-2_H-512_A-8':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-512_A-8/1',\n    'small_bert/bert_en_uncased_L-2_H-768_A-12':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-768_A-12/1',\n    'small_bert/bert_en_uncased_L-4_H-128_A-2':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-128_A-2/1',\n    'small_bert/bert_en_uncased_L-4_H-256_A-4':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-256_A-4/1',\n    'small_bert/bert_en_uncased_L-4_H-512_A-8':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-512_A-8/1',\n    'small_bert/bert_en_uncased_L-4_H-768_A-12':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-768_A-12/1',\n    'small_bert/bert_en_uncased_L-6_H-128_A-2':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-128_A-2/1',\n    'small_bert/bert_en_uncased_L-6_H-256_A-4':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-256_A-4/1',\n    'small_bert/bert_en_uncased_L-6_H-512_A-8':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-512_A-8/1',\n    'small_bert/bert_en_uncased_L-6_H-768_A-12':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-768_A-12/1',\n    'small_bert/bert_en_uncased_L-8_H-128_A-2':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-128_A-2/1',\n    'small_bert/bert_en_uncased_L-8_H-256_A-4':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-256_A-4/1',\n    'small_bert/bert_en_uncased_L-8_H-512_A-8':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-512_A-8/1',\n    'small_bert/bert_en_uncased_L-8_H-768_A-12':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-768_A-12/1',\n    'small_bert/bert_en_uncased_L-10_H-128_A-2':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-128_A-2/1',\n    'small_bert/bert_en_uncased_L-10_H-256_A-4':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-256_A-4/1',\n    'small_bert/bert_en_uncased_L-10_H-512_A-8':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-512_A-8/1',\n    'small_bert/bert_en_uncased_L-10_H-768_A-12':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-768_A-12/1',\n    'small_bert/bert_en_uncased_L-12_H-128_A-2':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-128_A-2/1',\n    'small_bert/bert_en_uncased_L-12_H-256_A-4':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-256_A-4/1',\n    'small_bert/bert_en_uncased_L-12_H-512_A-8':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-512_A-8/1',\n    'small_bert/bert_en_uncased_L-12_H-768_A-12':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-768_A-12/1',\n    'albert_en_base':\n        'https://tfhub.dev/tensorflow/albert_en_base/2',\n    'electra_small':\n        'https://tfhub.dev/google/electra_small/2',\n    'electra_base':\n        'https://tfhub.dev/google/electra_base/2',\n    'experts_pubmed':\n        'https://tfhub.dev/google/experts/bert/pubmed/2',\n    'experts_wiki_books':\n        'https://tfhub.dev/google/experts/bert/wiki_books/2',\n    'talking-heads_base':\n        'https://tfhub.dev/tensorflow/talkheads_ggelu_bert_en_base/1',\n}\n\nmap_model_to_preprocess = {\n    'bert_en_uncased_L-12_H-768_A-12':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'bert_en_cased_L-12_H-768_A-12':\n        'https://tfhub.dev/tensorflow/bert_en_cased_preprocess/3',\n    'small_bert/bert_en_uncased_L-2_H-128_A-2':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-2_H-256_A-4':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-2_H-512_A-8':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-2_H-768_A-12':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-4_H-128_A-2':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-4_H-256_A-4':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-4_H-512_A-8':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-4_H-768_A-12':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-6_H-128_A-2':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-6_H-256_A-4':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-6_H-512_A-8':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-6_H-768_A-12':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-8_H-128_A-2':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-8_H-256_A-4':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-8_H-512_A-8':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-8_H-768_A-12':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-10_H-128_A-2':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-10_H-256_A-4':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-10_H-512_A-8':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-10_H-768_A-12':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-12_H-128_A-2':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-12_H-256_A-4':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-12_H-512_A-8':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-12_H-768_A-12':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'bert_multi_cased_L-12_H-768_A-12':\n        'https://tfhub.dev/tensorflow/bert_multi_cased_preprocess/3',\n    'albert_en_base':\n        'https://tfhub.dev/tensorflow/albert_en_preprocess/3',\n    'electra_small':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'electra_base':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'experts_pubmed':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'experts_wiki_books':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'talking-heads_base':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n}\n\n# here we choose the name manually\nbert_model_name = 'small_bert/bert_en_uncased_L-2_H-128_A-2'\n\ntfhub_handle_encoder = map_name_to_handle[bert_model_name]\ntfhub_handle_preprocess = map_model_to_preprocess[bert_model_name]\n\nprint(f'BERT model selected           : {tfhub_handle_encoder}')\nprint(f'Preprocess model auto-selected: {tfhub_handle_preprocess}')","metadata":{"execution":{"iopub.status.busy":"2022-07-09T13:26:51.982089Z","iopub.status.idle":"2022-07-09T13:26:51.982587Z","shell.execute_reply.started":"2022-07-09T13:26:51.982315Z","shell.execute_reply":"2022-07-09T13:26:51.98234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# THE important stuff","metadata":{}},{"cell_type":"code","source":"# this is THE pre process model\nsentence_features = ['premise', 'hypothesis']\n# sentence_features = ['premise_en', 'hypothesis_en']\nseq_length = 128\n\ninput_segments = [\n    tf.keras.layers.Input(shape=(), dtype=tf.string, name=ft) for ft in sentence_features]\n\n# Tokenize the text to word pieces.\nbert_preprocess = hub.load(tfhub_handle_preprocess)\ntokenizer = hub.KerasLayer(bert_preprocess.tokenize, name='tokenizer')\nsegments = [tokenizer(s) for s in input_segments]\n\npacker = hub.KerasLayer(bert_preprocess.bert_pack_inputs,\n                        arguments=dict(seq_length=seq_length),\n                        name='packer')\n\nmodel_inputs = packer(segments)\n\nbert_preprocess_model = tf.keras.Model(input_segments, model_inputs)\n\n################################################################\n################################################################\n# this is THE model\nclass Classifier(tf.keras.Model):\n    def __init__(self, num_classes):\n        super(Classifier, self).__init__(name=\"prediction\")\n        self.encoder = hub.KerasLayer(tfhub_handle_encoder, trainable=True)\n        self.dropout = tf.keras.layers.Dropout(0.1)\n        self.dense = tf.keras.layers.Dense(num_classes)\n        self.sig = tf.sigmoid\n        \n    def call(self, preprocessed_text):\n        encoder_outputs = self.encoder(preprocessed_text)\n        pooled_output = encoder_outputs[\"pooled_output\"]\n        x = self.dropout(pooled_output)\n        x = self.dense(x)\n        # x = self.sig(x)\n        # x = self.where(x == tf.reduce_max(x))[0][0]\n        return x\n    \n################################################################\n# The annoying data prep\ndef data_prep(data, batch_size, prep, features):\n    train = None\n    AUTOTUNE = tf.data.AUTOTUNE\n    if 'label' in data.columns:\n        train = True\n        data = data[[features[0], features[1], 'label']]\n    else:\n        data = data[[features[0],features[1]]]\n\n    length = len(data)\n    data = dict(data)\n    data = tf.data.Dataset.from_tensor_slices(data)\n\n    if train:\n        data = data.shuffle(length)\n        data = data.repeat()\n        data = data.batch(batch_size)\n        data = data.map(lambda ex: (prep(ex), ex['label']))\n    else:\n        data = data.batch(batch_size)\n        data = data.map(lambda ex: prep(ex))\n    \n    data = data.cache().prefetch(buffer_size=AUTOTUNE)\n    return data\n\n##################################################################################\nmodel = Classifier(3)\n\nmetrics = tf.keras.metrics.SparseCategoricalAccuracy('accuracy', dtype=tf.float32)\nloss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-09T13:26:51.984578Z","iopub.status.idle":"2022-07-09T13:26:51.985456Z","shell.execute_reply.started":"2022-07-09T13:26:51.985198Z","shell.execute_reply":"2022-07-09T13:26:51.985223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"let's pretrain with mnli","metadata":{}},{"cell_type":"markdown","source":"# preping the mnli dataset for pretrain using the tensorflow tutorial\n\nepochs = 3\nbatch_size = 32\ninit_lr = 2e-5\n\ntfds_name = 'glue/mnli' \n\ntfds_info = tfds.builder(tfds_name).info\n\nsentence_feature = list(tfds_info.features.keys())\nsentence_feature.remove('idx')\nsentence_feature.remove('label')\n\navailable_splits = list(tfds_info.splits.keys())\ntrain_split = 'train'\nvalidation_split = 'validation'\ntest_split = 'test'\nif tfds_name == 'glue/mnli':\n    validation_split = 'validation_matched'\n    test_split = 'test_matched'\n\nnum_classes = tfds_info.features['label'].num_classes\nnum_examples = tfds_info.splits.total_num_examples\n\nprint(f'Using {tfds_name} from TFDS')\nprint(f'This dataset has {num_examples} examples')\nprint(f'Number of classes: {num_classes}')\nprint(f'Features {sentence_features}')\nprint(f'Splits {available_splits}')\n\nwith tf.device('/job:localhost'):\n  # batch_size=-1 is a way to load the dataset into memory\n  in_memory_ds = tfds.load(tfds_name, batch_size=-1, shuffle_files=True)\n\n################################################################\nAUTOTUNE = tf.data.AUTOTUNE\n\n\ndef load_dataset_from_tfds(in_memory_ds, info, split, batch_size, bert_preprocess_model):\n    is_training = split.startswith('train')\n    dataset = tf.data.Dataset.from_tensor_slices(in_memory_ds[split])\n    num_examples = info.splits[split].num_examples\n\n    if is_training:\n        dataset = dataset.shuffle(num_examples)\n        dataset = dataset.repeat()\n    dataset = dataset.batch(batch_size)\n    dataset = dataset.map(lambda ex: (bert_preprocess_model(ex), ex['label']))\n    dataset = dataset.cache().prefetch(buffer_size=AUTOTUNE)\n    return dataset, num_examples\n\nwith strategy.scope():\n    mnli_train_dataset, train_data_size = load_dataset_from_tfds(\n        in_memory_ds, tfds_info, train_split, batch_size, bert_preprocess_model)\n    \n    steps_per_epoch = train_data_size // batch_size\n    num_train_steps = steps_per_epoch * epochs\n    num_warmup_steps = num_train_steps // 10\n\n    mnli_validation_dataset, validation_data_size = load_dataset_from_tfds(\n        in_memory_ds, tfds_info, validation_split, batch_size, bert_preprocess_model)\n    \n    validation_steps = validation_data_size // batch_size\n\n\n    optimizer = optimization.create_optimizer(\n        init_lr=init_lr,\n        num_train_steps=num_train_steps,\n        num_warmup_steps=num_warmup_steps,\n        optimizer_type='adamw')\n\n    model.compile(optimizer=optimizer, loss=loss, metrics=[metrics])\n\n    model.fit(\n        x=mnli_train_dataset,\n        validation_data=mnli_validation_dataset,\n        steps_per_epoch=steps_per_epoch,\n        epochs=epochs,\n        validation_steps=validation_steps)","metadata":{"execution":{"iopub.status.busy":"2022-07-09T13:26:51.986759Z","iopub.status.idle":"2022-07-09T13:26:51.98736Z","shell.execute_reply.started":"2022-07-09T13:26:51.987075Z","shell.execute_reply":"2022-07-09T13:26:51.987099Z"}}},{"cell_type":"code","source":"# splitting data for validation\nrawTrainData = train.sample(frac=0.8)\nrawValidationData = train.drop(rawTrainData.index)\n\nepochs = 25\nbatch_size = 32\ninit_lr = 2e-5\n\nwith strategy.scope():\n\n    train_data = data_prep(rawTrainData, batch_size, bert_preprocess_model, sentence_features)\n    validation_data = data_prep(rawValidationData, batch_size, bert_preprocess_model, sentence_features)\n    test_data = data_prep(test, batch_size, bert_preprocess_model, sentence_features)\n\n    train_data_size = len(rawTrainData)\n    validation_data_size = len(rawValidationData)\n\n    steps_per_epoch = train_data_size // batch_size\n\n    num_train_steps = steps_per_epoch * epochs\n    validation_steps = validation_data_size // batch_size\n    num_warmup_steps = num_train_steps // 10\n    \n    optimizer = optimization.create_optimizer(\n        init_lr=init_lr,\n        num_train_steps=num_train_steps,\n        num_warmup_steps=num_warmup_steps,\n        optimizer_type='adamw')\n\n    model.compile(optimizer=optimizer, loss=loss, metrics=[metrics])\n    history = model.fit(\n            x=train_data,\n            validation_data=validation_data,\n            steps_per_epoch=steps_per_epoch,\n            epochs=epochs,\n            validation_steps=validation_steps\n            )","metadata":{"execution":{"iopub.status.busy":"2022-07-09T13:26:51.988677Z","iopub.status.idle":"2022-07-09T13:26:51.989521Z","shell.execute_reply.started":"2022-07-09T13:26:51.98907Z","shell.execute_reply":"2022-07-09T13:26:51.989176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result = model.predict(test_data)\nresult = tf.sigmoid(result)\nresult = np.array(result)\n\na = pd.DataFrame(result, columns=['0','1','2'])\nsubmission['prediction'] = a.apply(lambda r: int(r[np.max(r) == r].index[0]), axis=1)\n\nsubmission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-09T13:26:51.990863Z","iopub.status.idle":"2022-07-09T13:26:51.991696Z","shell.execute_reply.started":"2022-07-09T13:26:51.991372Z","shell.execute_reply":"2022-07-09T13:26:51.99141Z"},"trusted":true},"execution_count":null,"outputs":[]}]}