{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Experiment-8: Training basecnn B0 models with original images and traditional image augmentation & transformation techniques","metadata":{}},{"cell_type":"code","source":"import itertools\nimport os\nimport random\nimport gc\nimport numpy as np\nimport pandas as pd\nimport math\nimport pprint\nimport matplotlib.pylab as plt\nimport seaborn as sns\nsns.set(rc={\"axes.titlesize\":15, \"axes.labelsize\":9,\"axes.titlepad\":15,\n            \"axes.labelpad\":12, \"legend.fontsize\":9,\n            \"legend.title_fontsize\":9, \"figure.titlesize\":15,\n            \"axes.grid\":False})\n\nfrom sklearn.model_selection import train_test_split, GroupKFold\nimport tensorflow as tf\nimport tensorflow_hub as tfhub\nimport tensorflow.keras.backend as K\nfrom kaggle_datasets import KaggleDatasets\n\nprint('TF version:', tf.__version__)\nprint('Hub version:', tfhub.__version__)\nprint('Physical devices:', tf.config.list_physical_devices())","metadata":{"execution":{"iopub.status.busy":"2021-08-26T21:26:49.788441Z","iopub.execute_input":"2021-08-26T21:26:49.788877Z","iopub.status.idle":"2021-08-26T21:26:52.221542Z","shell.execute_reply.started":"2021-08-26T21:26:49.788774Z","shell.execute_reply":"2021-08-26T21:26:52.220485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<span style=\"color: #027fc1; font-family: Segoe UI; font-size: 1.9em; font-weight: 300;\">Data Loaders & Augmentations on TPU</span>\n\nThanks to [@cdeotte](https://www.kaggle.com/cdeotte) & [@xhlulu](https://www.kaggle.com/xhlulu)","metadata":{}},{"cell_type":"code","source":"SATURATION  = (0.9, 1.1)\nCONTRAST = (0.9, 1.1)\nBRIGHTNESS  =  0.1\nROTATION    = 10.0\nSHEAR    = 2.0\nHZOOM  = 8.0\nWZOOM  = 4.0\nHSHIFT = 4.0\nWSHIFT = 4.0\n\ndef get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear    = math.pi * shear    / 180.\n\n    def get_3x3_mat(lst):\n        return tf.reshape(tf.concat([lst],axis=0), [3,3])\n    \n    # ROTATION MATRIX\n    c1   = tf.math.cos(rotation)\n    s1   = tf.math.sin(rotation)\n    one  = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    \n    rotation_matrix = get_3x3_mat([c1,   s1,   zero, \n                                   -s1,  c1,   zero, \n                                   zero, zero, one])    \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)    \n    \n    shear_matrix = get_3x3_mat([one,  s2,   zero, \n                                zero, c2,   zero, \n                                zero, zero, one])        \n    # ZOOM MATRIX\n    zoom_matrix = get_3x3_mat([one/height_zoom, zero,           zero, \n                               zero,            one/width_zoom, zero, \n                               zero,            zero,           one])    \n    # SHIFT MATRIX\n    shift_matrix = get_3x3_mat([one,  zero, height_shift, \n                                zero, one,  width_shift, \n                                zero, zero, one])\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), \n                 K.dot(zoom_matrix,     shift_matrix))\n\n\ndef transform(image, label):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    DIM = cfg.input_dims[0]\n    XDIM = DIM%2\n    \n    rot = ROTATION * tf.random.normal([1], dtype='float32')\n    shr = SHEAR * tf.random.normal([1], dtype='float32')\n    h_zoom = 1.0 + tf.random.normal([1], dtype='float32') / HZOOM\n    w_zoom = 1.0 + tf.random.normal([1], dtype='float32') / WZOOM\n    h_shift = HSHIFT * tf.random.normal([1], dtype='float32')\n    w_shift = WSHIFT * tf.random.normal([1], dtype='float32')\n  \n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot, shr, h_zoom, w_zoom, h_shift, w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image,tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3]), label","metadata":{"execution":{"iopub.status.busy":"2021-08-26T21:26:52.222942Z","iopub.execute_input":"2021-08-26T21:26:52.223257Z","iopub.status.idle":"2021-08-26T21:26:52.243729Z","shell.execute_reply.started":"2021-08-26T21:26:52.223219Z","shell.execute_reply":"2021-08-26T21:26:52.24264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def auto_select_accelerator():\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        print(\"Running on TPU:\", tpu.master())\n    except ValueError:\n        strategy = tf.distribute.get_strategy()\n    print(f\"Running on {strategy.num_replicas_in_sync} replicas\")\n    \n    return strategy\n\n\ndef build_decoder(with_labels=True, target_size=(256, 256), ext='jpg'):\n    def decode(path):\n        file_bytes = tf.io.read_file(path)\n\n        if ext == 'png':\n            img = tf.image.decode_png(file_bytes, channels=3)\n        elif ext in ['jpg', 'jpeg']:\n            img = tf.image.decode_jpeg(file_bytes, channels=3)\n        else:\n            raise ValueError(\"Image extension not supported\")\n        img = tf.cast(img, tf.float32) / 255.0\n        img = tf.image.resize(img, target_size)\n\n        return img\n    \n    def decode_with_labels(path, label):\n        return decode(path), label\n    \n    return decode_with_labels if with_labels else decode\n\n\ndef build_augmenter(with_labels=True):\n    def augment(img):\n        img = tf.image.random_flip_left_right(img)\n        #img = tf.image.random_flip_up_down(img)\n        img = tf.image.random_saturation(img, SATURATION[0], SATURATION[1])\n        img = tf.image.random_contrast(img, CONTRAST[0], CONTRAST[1])\n        img = tf.image.random_brightness(img, BRIGHTNESS)\n        return img\n    \n    def augment_with_labels(img, label):\n        return augment(img), label\n    \n    return augment_with_labels if with_labels else augment\n\n\ndef build_dataset(paths, labels=None, bsize=128, cache=True,\n                  decode_fn=None, augment_fn=None,\n                  augment=True, repeat=True, shuffle=1024,\n                  seed=None, cache_dir=\"\"):\n    if cache_dir != \"\" and cache is True:\n        os.makedirs(cache_dir, exist_ok=True)\n    \n    if decode_fn is None:\n        decode_fn = build_decoder(labels is not None)\n    \n    if augment_fn is None:\n        augment_fn = build_augmenter(labels is not None)\n    \n    AUTO = tf.data.experimental.AUTOTUNE\n    slices = paths if labels is None else (paths, labels)\n    \n    dset = tf.data.Dataset.from_tensor_slices(slices)\n    dset = dset.map(decode_fn, num_parallel_calls=AUTO)\n    dset = dset.cache(cache_dir) if cache else dset\n    \n    # Map the functions to perform Augmentations\n    dset = dset.map(augment_fn, num_parallel_calls=AUTO) if augment else dset\n    dset = dset.map(transform, num_parallel_calls=AUTO) if augment else dset\n    dset = dset.repeat() if repeat else dset\n    dset = dset.shuffle(shuffle, seed=seed) if shuffle else dset\n    dset = dset.batch(bsize).prefetch(AUTO)\n    \n    return dset","metadata":{"execution":{"iopub.status.busy":"2021-08-26T21:26:52.245891Z","iopub.execute_input":"2021-08-26T21:26:52.246217Z","iopub.status.idle":"2021-08-26T21:26:52.265543Z","shell.execute_reply.started":"2021-08-26T21:26:52.24616Z","shell.execute_reply":"2021-08-26T21:26:52.264308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<span style=\"color: #027fc1; font-family: Segoe UI; font-size: 1.9em; font-weight: 300;\">Global Config & Seeding</span>","metadata":{}},{"cell_type":"code","source":"def seed_everything(SEED):\n    os.environ['PYTHONHASHSEED']=str(SEED)\n    random.seed(SEED)\n    np.random.seed(SEED)\n    tf.random.set_seed(SEED)\n    os.environ['TF_CUDNN_DETERMINISTIC'] = str(SEED)\n    \nclass Config:\n    seed = 2021\n    job = 1\n    num_classes = 2\n    input_dims = (256, 256)\n    model_arch = \"efficientnetv2-b0\" #\"efficientnetv2-s-21k-ft1k\" ## Choose model architecture\n    batch_size = 8*16\n    kfold = 5\n    n_epochs = 15\n    lr = 0.0001\n    loss_func = 'binary_crossentropy'\n    # Whether to finetune the whole model or just the top layer.\n    fine_tune = True\n    wandb_project = 'lmju-covid-exp8-aug-trans'\n    dataset = \"ljmusiimcovid19exp1\" #using same dataset from exp1\n    \n    seed_everything(seed)\n    \ncfg = Config()\n","metadata":{"execution":{"iopub.status.busy":"2021-08-26T21:26:52.267499Z","iopub.execute_input":"2021-08-26T21:26:52.267948Z","iopub.status.idle":"2021-08-26T21:26:52.281536Z","shell.execute_reply.started":"2021-08-26T21:26:52.267914Z","shell.execute_reply":"2021-08-26T21:26:52.28041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<span style=\"color: #027fc1; font-family: Segoe UI; font-size: 1.9em; font-weight: 300;\">KFold Split</span>","metadata":{}},{"cell_type":"code","source":"#study_df = pd.read_csv('../input/siim-covid19-detection/train_study_level.csv')\n\n# Has Id, Covid columns. \ntrain_df = pd.read_csv('../input/ljmusiimcovid19exp1/V2/train_df.csv')\ntrain_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T21:26:52.282878Z","iopub.execute_input":"2021-08-26T21:26:52.28329Z","iopub.status.idle":"2021-08-26T21:26:52.322285Z","shell.execute_reply.started":"2021-08-26T21:26:52.283257Z","shell.execute_reply":"2021-08-26T21:26:52.321129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# For one-hot encoding Covid label, add Negative column and flip the value. 1 0 <-- using this\n# For label encoding, just keep label column 1 or 0\n\ntrain_df.rename(columns={'label':'Covid'}, inplace=True)\ntrain_df.rename(columns={'id':'Id'}, inplace=True)\ntrain_df['Negative'] = 0\ntrain_df.loc[train_df.Covid == 0, 'Negative'] = 1\ntrain_df['Fold'] = -1\nlabel_cols = ['Covid', 'Negative']\n\ngkf  = GroupKFold(n_splits=cfg.kfold)\n\nfor fold, (train_idx, val_idx) in enumerate(gkf.split(train_df, groups=train_df.Id.tolist())):\n    train_df.loc[val_idx, 'Fold'] = fold\n    \ntrain_df.to_csv('train_df_with_fold.csv')\ntrain_df.sample(10)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T21:26:52.325278Z","iopub.execute_input":"2021-08-26T21:26:52.325595Z","iopub.status.idle":"2021-08-26T21:26:52.394269Z","shell.execute_reply.started":"2021-08-26T21:26:52.325565Z","shell.execute_reply":"2021-08-26T21:26:52.392969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<span style=\"color: #027fc1; font-family: Segoe UI; font-size: 1.9em; font-weight: 300;\">🚀 Training</span>\n\nWe first find the GCS path of the selected EffNetV2 architecture from the EffNetV2 weights Kaggle dataset.","metadata":{}},{"cell_type":"markdown","source":"from kaggle_secrets import UserSecretsClient\nuser_secrets = UserSecretsClient()\nuser_credential = user_secrets.get_gcloud_credential()\nuser_secrets.set_tensorflow_credential(user_credential)","metadata":{"execution":{"iopub.status.busy":"2021-08-13T20:51:15.477772Z","iopub.execute_input":"2021-08-13T20:51:15.478046Z","iopub.status.idle":"2021-08-13T20:51:16.811907Z","shell.execute_reply.started":"2021-08-13T20:51:15.478013Z","shell.execute_reply":"2021-08-13T20:51:16.810812Z"}}},{"cell_type":"code","source":"# Get the TensorFlow Hub model URL\nhub_type = 'feature_vector' # ['classification', 'feature_vector']\n# Get the GCS path of EfficientNet Models\nDS_GCS_PATH = KaggleDatasets().get_gcs_path(\"efficientnetv2-tfhub-weight-files\")\nMODEL_GCS_PATH = f'{DS_GCS_PATH}/tfhub_models/{cfg.model_arch}/{hub_type}'\n\n# Get the GCS path of the images from the Kaggle dataset\n#GCS_DS_PATH =  KaggleDatasets().get_gcs_path(cfg.dataset)+\"/ljmu-siim-covid19/exp1/train/\"\n\nGCS_DS_PATH =  KaggleDatasets().get_gcs_path(cfg.dataset)+\"/V2/train/\"\n\nprint(DS_GCS_PATH)\nprint()\nprint(GCS_DS_PATH)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T21:26:52.395472Z","iopub.execute_input":"2021-08-26T21:26:52.395886Z","iopub.status.idle":"2021-08-26T21:26:53.423734Z","shell.execute_reply.started":"2021-08-26T21:26:52.395852Z","shell.execute_reply":"2021-08-26T21:26:53.422585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Cloud Storage\nfrom google.cloud import storage\nstorage_client = storage.Client(project='YOUR PROJECT ID')","metadata":{}},{"cell_type":"code","source":"import wandb\nfrom kaggle_secrets import UserSecretsClient\n\nuser_secrets = UserSecretsClient()\nwandb_api = user_secrets.get_secret(\"wandb_apikey\") \nwandb.login(key=wandb_api)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T21:26:53.42574Z","iopub.execute_input":"2021-08-26T21:26:53.426039Z","iopub.status.idle":"2021-08-26T21:26:55.376868Z","shell.execute_reply.started":"2021-08-26T21:26:53.426004Z","shell.execute_reply":"2021-08-26T21:26:55.375744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"tf.compat.v1.enable_eager_execution()\n\ntrain_paths = GCS_DS_PATH + train_df[train_df['Fold']!=1]['Id'] \n\ntrain_labels = train_df[train_df['Fold']!=1][label_cols].values\n\ndecoder = build_decoder(with_labels=True, target_size=cfg.input_dims, ext='jpg')\n\n\n\ntrain_dataset1 = build_dataset(\n    train_paths, train_labels, bsize=cfg.batch_size, \n    decode_fn=decoder,repeat=False, shuffle=False, augment=True\n)\n\ncnt = train_dataset1.reduce(np.int64(0), lambda x, _: x + 1)\nc=cnt\n        \nprint(\"Before augmentation, len of training data:\"+ str(len(train_paths)) ,\"After augmentation, len of training data:\"+ str(c))            ","metadata":{"execution":{"iopub.status.busy":"2021-08-26T15:21:33.332873Z","iopub.execute_input":"2021-08-26T15:21:33.333522Z","iopub.status.idle":"2021-08-26T15:23:30.421063Z","shell.execute_reply.started":"2021-08-26T15:21:33.333485Z","shell.execute_reply":"2021-08-26T15:23:30.420149Z"}}},{"cell_type":"code","source":"DISPLAY_PLOT = True\noof_aucs = dict()\n\nAUG_COUNT=0\n\nfor fold in range(cfg.kfold):\n    \n    print(\"\\nFold:\", fold)\n    \n    # Define TPU strategy and clear TPU\n    strategy = auto_select_accelerator()\n    \n    # Converting global config class object to a dictionary to log using Wandb\n    config_dict = dict(vars(Config))\n    config_dict = {k:(v if type(v)==int else str(v)) for (k,v) in config_dict.items() if '__' not in k}\n    config_dict['fold'] = fold\n    config_dict['job_name'] = f\"{config_dict['model_arch']}_fold{fold}_job{config_dict['job']}\"\n    print(\"Train Job:\", config_dict['job_name'], \"\\nConfig\")\n    pprint.pprint(config_dict)\n\n    wandb.init(project=cfg.wandb_project, name=config_dict['job_name'], config=config_dict)\n\n    valid_paths = GCS_DS_PATH + train_df[train_df['Fold']==fold]['Id'] # 00abxyz.jpg\n    train_paths = GCS_DS_PATH + train_df[train_df['Fold']!=fold]['Id'] \n    valid_labels = train_df[train_df['Fold']==fold][label_cols].values\n    train_labels = train_df[train_df['Fold']!=fold][label_cols].values\n\n    decoder = build_decoder(with_labels=True, target_size=cfg.input_dims, ext='jpg')\n    test_decoder = build_decoder(with_labels=False, target_size=cfg.input_dims, ext='jpg')\n\n    train_dataset = build_dataset(\n        train_paths, train_labels, bsize=cfg.batch_size, decode_fn=decoder\n        #repeat=False, shuffle=False, augment=False\n    )\n\n    valid_dataset = build_dataset(\n        valid_paths, valid_labels, bsize=cfg.batch_size, decode_fn=decoder,\n        repeat=False, shuffle=False, augment=False\n    )\n\n    num_classes = cfg.num_classes if cfg.num_classes else train_labels.shape[1]\n\n    with strategy.scope():\n        model = tf.keras.Sequential([\n            # Explicitly define the input shape so the model can be properly\n            # loaded by the TFLiteConverter\n            tf.keras.layers.InputLayer(input_shape=[cfg.input_dims[0], cfg.input_dims[1], 3]),\n            tfhub.KerasLayer(MODEL_GCS_PATH, trainable=cfg.fine_tune),\n            tf.keras.layers.Dropout(rate=0.1),\n            tf.keras.layers.Dense(num_classes,\n                                  activation='sigmoid')\n        ])\n\n        model.build((None, cfg.input_dims[0], cfg.input_dims[1], 3))\n        model.summary()\n\n        metrics = ['accuracy', tf.keras.metrics.AUC(name='auc', multi_label=True)]\n    \n        model.compile(\n          optimizer=tf.keras.optimizers.Adam(learning_rate=cfg.lr),\n          loss=cfg.loss_func,\n          metrics=metrics)\n\n    steps_per_epoch = train_paths.shape[0] // cfg.batch_size\n    callbacks = [\n        tf.keras.callbacks.ModelCheckpoint(f'model{fold}.h5',\n                                           save_best_only=True,\n                                           monitor='val_loss',\n                                           mode='min'),\n        wandb.keras.WandbCallback(save_model=True, monitor='val_loss',  mode='min'),\n        tf.keras.callbacks.ReduceLROnPlateau(monitor=\"val_loss\",\n                                             patience=3,\n                                             min_lr=1e-6,\n                                             mode='min'),\n                ]\n\n    history = model.fit(\n        train_dataset, \n        epochs=cfg.n_epochs,\n        verbose=1,\n        callbacks=callbacks,\n        steps_per_epoch=steps_per_epoch,\n        validation_data=valid_dataset)\n\n    history_df = pd.DataFrame(history.history)\n    history_df.to_csv(f'history{fold}.csv')\n\n    del decoder, test_decoder, train_dataset, valid_dataset, model\n    gc.collect()\n\n    oof_aucs[fold] = float(np.max(history.history['val_auc']))\n    print(\"oof_aucs\", oof_aucs)\n    \n    wandb.finish()\n    \n    # Plot Training History\n    if DISPLAY_PLOT:\n        print (\"\\n\\n\")\n        plt.figure(figsize=(15,5))\n        plt.plot(np.arange(len(history.history['auc'])), history.history['auc'],\n                 '-o', label='Train auc', color='#ff7f0e')\n        plt.plot(np.arange(len(history.history['auc'])), history.history['val_auc'],\n                 '-o', label='Val auc', color='#1f77b4')\n        x = np.argmax(history.history['val_auc'])\n        y = np.max(history.history['val_auc'])\n        xdist = plt.xlim()[1] - plt.xlim()[0]\n        ydist = plt.ylim()[1] - plt.ylim()[0]\n        plt.scatter(x,y,s=200, color='#1f77b4')\n        plt.text(x-0.03*xdist, y-0.13*ydist, 'Max AUC\\n%.2f'%y, size=10)\n        plt.ylabel('auc', size=14)\n        plt.xlabel('Epoch', size=14)\n        plt.legend(loc=2)\n\n        plt2 = plt.gca().twinx()\n        plt2.plot(np.arange(len(history.history['auc'])),\n                  history.history['loss'],'-o', label='Train Loss', color='#2ca02c')\n        plt2.plot(np.arange(len(history.history['auc'])),\n                  history.history['val_loss'],'-o', label='Val Loss', color='#d62728')\n        x = np.argmin(history.history['val_loss'])\n        y = np.min(history.history['val_loss'])\n        ydist = plt.ylim()[1] - plt.ylim()[0]\n        plt.scatter(x, y, s=200, color='#d62728')\n        plt.text(x-0.03*xdist, y+0.05*ydist,'Min Loss', size=10)\n        plt.ylabel('Loss', size=14)\n        plt.title(f\"{config_dict['job_name']} Size - {cfg.input_dims}\")\n        plt.legend(loc=3)\n        plt.savefig(f'fig{fold}.png')\n        plt.show()\n        \nprint(\"\\n\"+\"-\"*40)\nprint(\"CV AUC:\", sum(list(oof_aucs.values()))/cfg.kfold)\nprint(\"-\"*40)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T21:26:55.378562Z","iopub.execute_input":"2021-08-26T21:26:55.378869Z","iopub.status.idle":"2021-08-26T21:35:11.621421Z","shell.execute_reply.started":"2021-08-26T21:26:55.378838Z","shell.execute_reply":"2021-08-26T21:35:11.616786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!rm -r /kaggle/working/wandb\nprint(\"Completed ...\")","metadata":{"execution":{"iopub.status.busy":"2021-08-26T21:35:11.622966Z","iopub.status.idle":"2021-08-26T21:35:11.62394Z"},"trusted":true},"execution_count":null,"outputs":[]}]}