{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Inference to models from experiment-8 (basecnn B0 models with original images and traditional image augmentation & transformation techniques)","metadata":{}},{"cell_type":"code","source":"import sys\nimport glob\nimport itertools\nimport os\nimport random\nimport gc\nimport numpy as np\nimport pandas as pd\nimport math\nimport pprint\nimport matplotlib.pylab as plt\nimport seaborn as sns\nsns.set(rc={\"axes.titlesize\":15, \"axes.labelsize\":9,\"axes.titlepad\":15,\n            \"axes.labelpad\":12, \"legend.fontsize\":9,\n            \"legend.title_fontsize\":9, \"figure.titlesize\":15,\n            \"axes.grid\":False})\n\nfrom sklearn.model_selection import train_test_split, GroupKFold\nfrom PIL import Image\nfrom tqdm.auto import tqdm\n\nimport tensorflow as tf\nimport tensorflow_hub as tfhub\nimport tensorflow.keras.backend as K\nfrom kaggle_datasets import KaggleDatasets\n\nprint('TF version:', tf.__version__)\nprint('Hub version:', tfhub.__version__)\nprint('Physical devices:', tf.config.list_physical_devices())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(SEED):\n    os.environ['PYTHONHASHSEED']=str(SEED)\n    random.seed(SEED)\n    np.random.seed(SEED)\n    tf.random.set_seed(SEED)\n    os.environ['TF_CUDNN_DETERMINISTIC'] = str(SEED)\n    \nclass Config:\n    seed = 2021\n    model_arch = \"efficientnetv2-b0\" #\"efficientnetv2-s-21k-ft1k\" ## Choose model architecture\n    hub_type = \"feature_vector\"\n    wandb_project = 'lmju-covid-inf8'\n    dataset = \"ljmusiimcovid19inf8\"\n    BATCH_SIZE = 32\n    TEST_IMAGE_DIMS = (256, 256)\n    trail_run = False\n    \n    seed_everything(seed)\n    \ncfg = Config()","metadata":{"execution":{"iopub.status.busy":"2021-08-26T07:06:27.097926Z","iopub.execute_input":"2021-08-26T07:06:27.098412Z","iopub.status.idle":"2021-08-26T07:06:27.107256Z","shell.execute_reply.started":"2021-08-26T07:06:27.098374Z","shell.execute_reply":"2021-08-26T07:06:27.105961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test_df = pd.read_csv('/kaggle/input/ljmusiimcovid19infv2/ljmu-siim-covid19-inf1-v2/test_df.csv')\ntest_df = pd.read_csv('/kaggle/input/ljmusiimcovid19inf8/ljmu-siim-covid19-inf8/test_df.csv')\n\ntest_df['Negative'] = 0\ntest_df.loc[test_df.Covid == 0, 'Negative'] = 1\ntest_df.head(3)","metadata":{"papermill":{"duration":5.695531,"end_time":"2021-07-17T19:03:16.105354","exception":false,"start_time":"2021-07-17T19:03:10.409823","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-14T18:55:55.261428Z","iopub.execute_input":"2021-08-14T18:55:55.261778Z","iopub.status.idle":"2021-08-14T18:55:55.298871Z","shell.execute_reply.started":"2021-08-14T18:55:55.261745Z","shell.execute_reply":"2021-08-14T18:55:55.297462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.shape","metadata":{"execution":{"iopub.status.busy":"2021-08-14T18:55:55.300812Z","iopub.execute_input":"2021-08-14T18:55:55.301149Z","iopub.status.idle":"2021-08-14T18:55:55.307839Z","shell.execute_reply.started":"2021-08-14T18:55:55.301114Z","shell.execute_reply":"2021-08-14T18:55:55.306681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if cfg.trail_run == True:\n    test_df = test_df.sample(10)\n    print(\"\\ntest_df\")\n    display(test_df.head(10))\n","metadata":{"papermill":{"duration":0.28084,"end_time":"2021-07-17T19:03:16.86864","exception":false,"start_time":"2021-07-17T19:03:16.5878","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-14T18:55:55.309292Z","iopub.execute_input":"2021-08-14T18:55:55.30964Z","iopub.status.idle":"2021-08-14T18:55:55.321018Z","shell.execute_reply.started":"2021-08-14T18:55:55.309609Z","shell.execute_reply":"2021-08-14T18:55:55.31982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %pip install -q wandb \n#!pip install --upgrade wandb\nimport wandb\nfrom kaggle_secrets import UserSecretsClient\n\nuser_secrets = UserSecretsClient()\nwandb_api = user_secrets.get_secret(\"wandb_apikey\") \nwandb.login(key=wandb_api)\nprint('wandb connected ...')","metadata":{"execution":{"iopub.status.busy":"2021-08-14T18:55:55.322698Z","iopub.execute_input":"2021-08-14T18:55:55.323022Z","iopub.status.idle":"2021-08-14T18:55:56.242591Z","shell.execute_reply.started":"2021-08-14T18:55:55.322985Z","shell.execute_reply":"2021-08-14T18:55:56.2409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<span style=\"color: #00857e; font-family: Segoe UI; font-size: 1.8em; font-weight: 300;\">Custom Wrapper for Loading TFHub Model trained in TPU</span>\n\nSince the EffNetV2 Classifier models were trained on a TPU with the `tfhub.KerasLayer` formed with the handle argument as a GCS path, while loading the saved model for inference, the method tries to download the pre-trained weights from the definition of the layer from training i.e a GCS path.\n\nSince, inference notebooks don't have GCS and internet access, it is not possible to load the model without the pretrained weights explicitly loaded from the local directory.\n\nIf the models were trained on a GPU, we can use the cache location method to load the pre-trained weights by storing them in a cache folder with the hashed key of the model location, as the folder name. I tried this method here but, it doesn't seem to work as the model was trained with a GCS path defined in the `tfhub.KerasLayer` and the method kept on hitting the GCS path rather than loading the weights from the cache location.\n\nThe only solution was to create a wrapper class to correct the handle argument to load the right pretrained weights explicitly from the local directory.","metadata":{"papermill":{"duration":0.096404,"end_time":"2021-07-17T19:03:21.416262","exception":false,"start_time":"2021-07-17T19:03:21.319858","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def build_decoder(with_labels=True, target_size=(300, 300), ext='jpg'):\n    def decode(path):\n        file_bytes = tf.io.read_file(path)\n        if ext == 'png':\n            img = tf.image.decode_png(file_bytes, channels=3)\n        elif ext in ['jpg', 'jpeg']:\n            img = tf.image.decode_jpeg(file_bytes, channels=3)\n        else:\n            raise ValueError(\"Image extension not supported\")\n\n        img = tf.cast(img, tf.float32) / 255.0\n        img = tf.image.resize(img, target_size)\n\n        return img\n\n    def decode_with_labels(path, label):\n        return decode(path), label\n\n    return decode_with_labels if with_labels else decode\n\ndef build_augmenter(with_labels=True):\n    def augment(img):\n        img = tf.image.random_flip_left_right(img)\n        img = tf.image.random_flip_up_down(img)\n        return img\n\n    def augment_with_labels(img, label):\n        return augment(img), label\n\n    return augment_with_labels if with_labels else augment\n\ndef build_dataset(paths, labels=None, bsize=32, cache=True,\n                  decode_fn=None, augment_fn=None,\n                  augment=True, repeat=True, shuffle=1024, \n                  cache_dir=\"\"):\n    if cache_dir != \"\" and cache is True:\n        os.makedirs(cache_dir, exist_ok=True)\n\n    if decode_fn is None:\n        decode_fn = build_decoder(labels is not None)\n\n    if augment_fn is None:\n        augment_fn = build_augmenter(labels is not None)\n\n    AUTO = tf.data.experimental.AUTOTUNE\n    slices = paths if labels is None else (paths, labels)\n\n    dset = tf.data.Dataset.from_tensor_slices(slices)\n    dset = dset.map(decode_fn, num_parallel_calls=AUTO)\n    dset = dset.cache(cache_dir) if cache else dset\n    dset = dset.map(augment_fn, num_parallel_calls=AUTO) if augment else dset\n    dset = dset.repeat() if repeat else dset\n    dset = dset.shuffle(shuffle) if shuffle else dset\n    dset = dset.batch(bsize).prefetch(AUTO)\n\n    return dset","metadata":{"execution":{"iopub.status.busy":"2021-08-14T18:55:56.244396Z","iopub.execute_input":"2021-08-14T18:55:56.244858Z","iopub.status.idle":"2021-08-14T18:55:56.26284Z","shell.execute_reply.started":"2021-08-14T18:55:56.244815Z","shell.execute_reply":"2021-08-14T18:55:56.261662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get the TensorFlow Hub model URL\n# MODEL_ARCH_PATH = f'/kaggle/input/efficientnetv2-tfhub-weight-files/tfhub_models/{cfg.model_arch}/{cfg.hub_type}'\nDS_GCS_PATH = KaggleDatasets().get_gcs_path(\"efficientnetv2-tfhub-weight-files\")\n# MODEL_ARCH_PATH = f'{DS_GCS_PATH}/tfhub_models/{cfg.model_arch}/{cfg.hub_type}'\nMODEL_ARCH_PATH = f'../input/efficientnetv2-tfhub-weight-files/tfhub_models/{cfg.model_arch}/{cfg.hub_type}'\n\n# Get the GCS path of the images from the Kaggle dataset\n# GCS_DS_PATH =  KaggleDatasets().get_gcs_path(cfg.dataset)+\"/ljmu-siim-covid19-inf1/test/\"\nGCS_DS_PATH =  f'../input/ljmusiimcovid19inf8/ljmu-siim-covid19-inf8/test/'\nMODEL_PATH = f'../input/ljmusiimcovid19inf8/ljmu-siim-covid19-inf8/models'\n\nprint(MODEL_ARCH_PATH)\nprint()\nprint(GCS_DS_PATH)\n\n# Custom wrapper class to load the right pretrained weights explicitly from the local directory\nclass KerasLayerWrapper(tfhub.KerasLayer):\n    def __init__(self, handle, **kwargs):\n        handle = tfhub.KerasLayer(tfhub.load(MODEL_ARCH_PATH))\n        super().__init__(handle, **kwargs)","metadata":{"papermill":{"duration":4.259738,"end_time":"2021-07-17T19:03:25.768808","exception":false,"start_time":"2021-07-17T19:03:21.50907","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-14T18:55:56.266594Z","iopub.execute_input":"2021-08-14T18:55:56.266925Z","iopub.status.idle":"2021-08-14T18:55:56.941348Z","shell.execute_reply.started":"2021-08-14T18:55:56.266895Z","shell.execute_reply":"2021-08-14T18:55:56.940097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<span style=\"color: #00857e; font-family: Segoe UI; font-size: 1.8em; font-weight: 300;\">Predict Study Level</span>","metadata":{"papermill":{"duration":0.087465,"end_time":"2021-07-17T19:03:25.943841","exception":false,"start_time":"2021-07-17T19:03:25.856376","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# strategy = auto_select_accelerator()\n# BATCH_SIZE = strategy.num_replicas_in_sync * 16\ntest_paths = [GCS_DS_PATH+jpg for jpg in test_df['Id'].tolist()]\ntest_paths[0]","metadata":{"execution":{"iopub.status.busy":"2021-08-14T18:55:56.943394Z","iopub.execute_input":"2021-08-14T18:55:56.943778Z","iopub.status.idle":"2021-08-14T18:55:56.953061Z","shell.execute_reply.started":"2021-08-14T18:55:56.943724Z","shell.execute_reply":"2021-08-14T18:55:56.951614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#for predictions\nlabel_cols = ['Covid', 'Negative']\npred_df = test_df[['Id', 'Covid', 'Negative']].copy()\npred_df[label_cols] = 0\n\ntest_decoder = build_decoder(with_labels=False,\n                             target_size=(cfg.TEST_IMAGE_DIMS[0],\n                                          cfg.TEST_IMAGE_DIMS[0]), ext='jpg')\ntest_dataset = build_dataset(\n    test_paths, bsize=cfg.BATCH_SIZE, repeat=False, \n    shuffle=False, augment=False, cache=False,\n    decode_fn=test_decoder\n)\n\n\nwith tf.device('/device:GPU:0'):\n    models = []\n    models0 = tf.keras.models.load_model(f'{MODEL_PATH}/model0.h5',\n                                         custom_objects={'KerasLayer': KerasLayerWrapper})\n    \n    models1 = tf.keras.models.load_model(f'{MODEL_PATH}/model1.h5',\n                                         custom_objects={'KerasLayer': KerasLayerWrapper})\n    models2 = tf.keras.models.load_model(f'{MODEL_PATH}/model2.h5',\n                                         custom_objects={'KerasLayer': KerasLayerWrapper})\n    models3 = tf.keras.models.load_model(f'{MODEL_PATH}/model3.h5',\n                                         custom_objects={'KerasLayer': KerasLayerWrapper})\n    models4 = tf.keras.models.load_model(f'{MODEL_PATH}/model4.h5',\n                                      custom_objects={'KerasLayer': KerasLayerWrapper})\n    models.append(models0)\n    models.append(models1)\n    models.append(models2)\n    models.append(models3)\n    models.append(models4)\n\npred_df[label_cols] = sum([model.predict(test_dataset, verbose=1) for model in models]) / len(models)\n\ndel models\ndel models0, models1, models2, models3, models4\ndel test_dataset, test_decoder\ngc.collect()\n\nprint(\"Prediction completed ...\")","metadata":{"execution":{"iopub.status.busy":"2021-08-14T18:55:56.954444Z","iopub.execute_input":"2021-08-14T18:55:56.954778Z","iopub.status.idle":"2021-08-14T19:04:39.140297Z","shell.execute_reply.started":"2021-08-14T18:55:56.954748Z","shell.execute_reply":"2021-08-14T19:04:39.13897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_df['Covid_Prob'] = pred_df['Covid']\npred_df['Negative_Prob'] = pred_df['Negative']\n# CHECK: sum of probs shouldn't be over 100\n#df_100 = pred_df[pred_df['Covid_Prob']+pred_df['Negative_Prob']>100]\n#df_100.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-14T19:04:39.144945Z","iopub.execute_input":"2021-08-14T19:04:39.145268Z","iopub.status.idle":"2021-08-14T19:04:39.158139Z","shell.execute_reply.started":"2021-08-14T19:04:39.145238Z","shell.execute_reply":"2021-08-14T19:04:39.156884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_df['Covid'] = pred_df.apply(lambda x: 1 if x['Covid_Prob'] == x['Negative_Prob'] else x['Covid'], axis=1)\npred_df['Negative'] = pred_df.apply(lambda x: 0 if x['Covid_Prob'] == x['Negative_Prob'] else x['Negative'], axis=1)\npred_df['Covid'] = pred_df.apply(lambda x: 0 if x['Covid_Prob'] < 0.50 else 1, axis=1)\npred_df['Negative'] = pred_df.apply(lambda x: 0 if x['Negative_Prob'] < 0.50 else 1, axis=1)\n\n#y_pred = (y_pred > 0.5) \npred_df.head(10)","metadata":{"execution":{"iopub.status.busy":"2021-08-14T19:04:39.159729Z","iopub.execute_input":"2021-08-14T19:04:39.160044Z","iopub.status.idle":"2021-08-14T19:04:39.241678Z","shell.execute_reply.started":"2021-08-14T19:04:39.160016Z","shell.execute_reply":"2021-08-14T19:04:39.240499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head(10)","metadata":{"execution":{"iopub.status.busy":"2021-08-14T19:04:39.243364Z","iopub.execute_input":"2021-08-14T19:04:39.243685Z","iopub.status.idle":"2021-08-14T19:04:39.256837Z","shell.execute_reply.started":"2021-08-14T19:04:39.243655Z","shell.execute_reply":"2021-08-14T19:04:39.255559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#summary reports\n#for metrics\nx_test = test_df[\"Id\"]\ny_true_covid = test_df[\"Covid\"]\ny_true_negative = test_df[\"Negative\"]\n\ny_pred_covid = pred_df[\"Covid\"]\ny_pred_negative = pred_df[\"Negative\"]\n\ny_pred_covid_prob = pred_df[\"Covid_Prob\"]\ny_pred_negative_prob = pred_df[\"Negative_Prob\"]\n\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import classification_report\n\n# confusion matrix\n# outcome values order in sklearn\n# classification report for precision, recall f1-score and accuracy\n\nprint(\"\\n ****** Covid Summary ****** \")\nmatrix = confusion_matrix(y_true_covid,y_pred_covid, labels=[1,0])\nprint('Confusion matrix : \\n',matrix)\ntp, fn, fp, tn = confusion_matrix(y_true_covid,y_pred_covid,labels=[1,0]).reshape(-1)\nprint('Outcome values : \\n', \"TP = \"+str(tp), \"FN = \"+str(fn), \"FP = \"+str(fp), \"TN = \"+str(tn))\nmatrix = classification_report(y_true_covid,y_pred_covid,labels=[1,0])\nprint('Classification report : \\n',matrix)\n\n#print(\"\\n ****** Negative Summary ****** \")\n#matrix = confusion_matrix(y_true_negative,y_pred_negative, labels=[1,0])\n#print('Confusion matrix : \\n',matrix)\n#tp, fn, fp, tn = confusion_matrix(y_true_negative,y_pred_negative,labels=[1,0]).reshape(-1)\n#print('Outcome values : \\n', \"TP = \"+str(tp), \"FN = \"+str(fn), \"FP = \"+str(fp), \"TN = \"+str(tn))\n#matrix = classification_report(y_true_negative,y_pred_negative,labels=[1,0])\n#print('Classification report : \\n',matrix)\n\nfrom sklearn.metrics import precision_recall_curve\nfrom sklearn.metrics import f1_score\nfrom sklearn.metrics import auc\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.metrics import precision_score\nfrom sklearn.metrics import recall_score\n\nlr_probs =  pred_df[[\"Covid_Prob\"]].to_numpy()\nlr_precision, lr_recall, _ = precision_recall_curve(y_true_covid, lr_probs)\nlr_f1, lr_auc = f1_score(y_true_covid, y_pred_covid), auc(lr_recall, lr_precision)\nlr_auc = roc_auc_score(y_true_covid, lr_probs)\nprecision = precision_score(y_true_covid, y_pred_covid, labels=[1,0], average='macro')\nrecall = recall_score(y_true_covid, y_pred_covid, average='macro')\n\nprint('Precision: %.3f' % precision)\nprint('Recall: %.3f' % recall)\nprint('ROC AUC: %.3f' % (lr_auc))\nprint('F1: %.3f AUC: %.3f' % (lr_f1, lr_auc))\n","metadata":{"execution":{"iopub.status.busy":"2021-08-14T19:28:47.672911Z","iopub.execute_input":"2021-08-14T19:28:47.673399Z","iopub.status.idle":"2021-08-14T19:28:47.718525Z","shell.execute_reply.started":"2021-08-14T19:28:47.673361Z","shell.execute_reply":"2021-08-14T19:28:47.717095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#plot graphs\nfrom matplotlib import pyplot\nfrom sklearn.metrics import roc_curve\n\nlr_fpr, lr_tpr, _ = roc_curve(y_true_covid, lr_probs)\npyplot.plot(lr_fpr, lr_tpr, marker='.', label=str('ROC: %.3f' % (lr_auc)))\npyplot.title(\"Receiver Operating Characteristic\")\n# axis labels\npyplot.xlabel('False Positive Rate')\npyplot.ylabel('True Positive Rate')\n# show the legend\npyplot.legend()\n# show the plot\npyplot.show()\n\npyplot.plot(lr_recall, lr_precision, marker='.', color='green', label=str('Precision: %.3f Recall: %.3f' % (precision, recall)))\npyplot.title(\"Precision-Recall\")\n# axis labels\npyplot.xlabel('Recall')\npyplot.ylabel('Precision')\n# show the legend\npyplot.legend()\n# show the plot\npyplot.show()\n\n\nfrom sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay\n\ncm = confusion_matrix(y_true_covid, y_pred_covid, labels=[1,0])\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=['Covid','Negative'])\nfig = pyplot.figure()\nax = fig.add_subplot(111)\n#disp.title(\"Confusion Matrix\")\ndisp = disp.plot(include_values=True, cmap='viridis',xticks_rotation='horizontal', ax=ax)\npyplot.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-14T19:36:53.724174Z","iopub.execute_input":"2021-08-14T19:36:53.72462Z","iopub.status.idle":"2021-08-14T19:36:54.324686Z","shell.execute_reply.started":"2021-08-14T19:36:53.724584Z","shell.execute_reply":"2021-08-14T19:36:54.323683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# upload CM to wandb\nconfig_dict = dict(vars(Config))\n#jobconfig = f\"inf1-Arch:{config_dict['model_arch']};seed:{config_dict['seed']};batchsize:{config_dict['BATCH_SIZE']};dataset:{config_dict['dataset']}\"\nwandb.init(project=cfg.wandb_project) #, name='inf1')#, config=config_dict)\n\nwandb.sklearn.plot_confusion_matrix(y_true_covid, y_pred_covid.to_numpy(), [\"Covid\",\"Negative\"])\n#wandb.sklearn.plot_confusion_matrix(y_true_negative, y_pred_negative, [\"Covid\",\"Negative\"])\n\n#get probability of positive only\n#prob_df = pred_df[[\"Covid_Prob\",\"Negative_Prob\"]]\n#prob_df = pred_df[[\"Covid_Prob\"]]\n\n#wandb.sklearn.plot_precision_recall(y_true_covid, prob_df.to_numpy(), [\"Covid\",\"Negative\"])\n#wandb.sklearn.plot_precision_recall(y_true_negative, y_pred_negative_prob, [\"Covid\",\"Negative\"])\n\n#wandb.sklearn.plot_roc(y_true_covid, prob_df.to_numpy(), [\"Covid\",\"Negative\"])\n#wandb.sklearn.plot_roc(y_true_negative, y_pred_negative_prob, [\"Covid\",\"Negative\"])\n\nwandb.finish()\n\nprint(\"Report completed ...\")","metadata":{"execution":{"iopub.status.busy":"2021-08-14T19:20:11.058267Z","iopub.execute_input":"2021-08-14T19:20:11.058671Z","iopub.status.idle":"2021-08-14T19:20:32.801775Z","shell.execute_reply.started":"2021-08-14T19:20:11.058635Z","shell.execute_reply":"2021-08-14T19:20:32.800416Z"},"trusted":true},"execution_count":null,"outputs":[]}]}