{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"![](http://storage.googleapis.com/kaggle-competitions/kaggle/29653/logos/header.png?t=2021-07-07-17-26-56)\n# Interactive Task1 EDA\nIn this notebook you will find\n* How useful are Task1 scans for Task2 (this competition).\n* How to build interactive figures with Plotly.\n* That Task2 is not about finding tumors.\n","metadata":{}},{"cell_type":"code","source":"!pip install pyradiomics","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:43:55.618263Z","iopub.execute_input":"2021-08-26T13:43:55.618609Z","iopub.status.idle":"2021-08-26T13:44:10.497847Z","shell.execute_reply.started":"2021-08-26T13:43:55.618534Z","shell.execute_reply":"2021-08-26T13:44:10.496456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import radiomics\nimport SimpleITK as sitk\n\nimport tarfile\nimport plotly\nfrom plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot\n\ninit_notebook_mode(connected=True)\n\ndef extract_task1_files(root=\"./data\"):\n    tar = tarfile.open(\"../input/brats-2021-task1/BraTS2021_Training_Data.tar\")\n    tar.extractall(root)\n    tar.close()","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:44:10.500175Z","iopub.execute_input":"2021-08-26T13:44:10.500590Z","iopub.status.idle":"2021-08-26T13:44:11.154414Z","shell.execute_reply.started":"2021-08-26T13:44:10.500554Z","shell.execute_reply":"2021-08-26T13:44:11.153300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"extract_task1_files()","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:44:11.157769Z","iopub.execute_input":"2021-08-26T13:44:11.158260Z","iopub.status.idle":"2021-08-26T13:46:56.272980Z","shell.execute_reply.started":"2021-08-26T13:44:11.158189Z","shell.execute_reply":"2021-08-26T13:46:56.271670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. Introduction","metadata":{}},{"cell_type":"markdown","source":"RSNA-MICCAI Brain Tumor Radiogenomic Classification is a second part to the RSNA-ASNR-MICCAI BraTS 2021 challenge. First part focuses on multiclass segmentation, this one (Task 2) - on binary classification.\n\n> The RSNA-ASNR-MICCAI BraTS 2021 challenge utilizes ... mpMRI scans, and focuses on (**Task 1**) the evaluation of state-of-the-art methods for the **segmentation** of intrinsically heterogeneous **brain glioblastoma sub-regions** in mpMRI scans.\n>\n> Furthermore, this year's challenge also focuses on (**Task 2**) the evaluation of **classification** methods to predict the **MGMT promoter methylation status** at pre-operative baseline scans.\n\nTask 1 sub-regions are defined as\n\n> ... the **GD-enhancing tumor** (ET — label **4**), the peritumoral edematous/**invaded tissue** (ED — label **2**), and the **necrotic tumor core** (NCR — label **1**).\n\nBoth tasks share matching patient ids. Can task1 scans be used in task2?","metadata":{}},{"cell_type":"markdown","source":"# 2. Plotting 3D MRI scans","metadata":{}},{"cell_type":"markdown","source":"First, let's take a peek at how task1 scans look in 3D. To plot them, we need to rasterize stacked images into a point cloud with reduced dimensionality. Passing each scanned pixel into our visualization would net us more than a million points, so we need to 1) resize every image to 128x128 and 2) downsample space without tumor for brevity.","metadata":{}},{"cell_type":"code","source":"import nibabel as nib\nimport os\nimport albumentations as A\nimport numpy as np\n\n\nclass ImageReader:\n    def __init__(self, root:str, img_size:int=256, normalize:bool=False, single_class:bool=False):\n        pad_size = 256 if img_size > 256 else 224\n        self.resize = A.Compose(\n            [\n                A.PadIfNeeded(min_height=pad_size, min_width=pad_size, value=0),\n                A.Resize(img_size, img_size)\n            ]\n        )\n        self.normalize=normalize\n        self.single_class=single_class\n        self.root=root\n        \n    def read_file(self, path:str) -> dict:\n        scan_type = path.split('_')[-1]\n        raw_image = nib.load(path).get_fdata()\n        raw_mask = nib.load(path.replace(scan_type, 'seg.nii.gz')).get_fdata()\n        processed_frames, processed_masks = [], []\n        for frame_idx in range(raw_image.shape[2]):\n            frame = raw_image[:, :, frame_idx]\n            mask = raw_mask[:, :, frame_idx]\n            if self.normalize:\n                if frame.max() > 0:\n                    frame = frame/frame.max()\n                frame = frame.astype(np.float32)\n            else:\n                frame = frame.astype(np.uint8)\n            resized = self.resize(image=frame, mask=mask)\n            processed_frames.append(resized['image'])\n            processed_masks.append(1*(resized['mask'] > 0) if self.single_class else resized['mask'])\n        return {\n            'scan': np.stack(processed_frames, 0),\n            'segmentation': np.stack(processed_masks, 0),\n            'orig_shape': raw_image.shape\n        }\n    \n    def load_patient_scan(self, idx:int, scan_type:str='flair') -> dict:\n        patient_id = str(idx).zfill(5)\n        scan_filename = f'{self.root}/BraTS2021_{patient_id}/BraTS2021_{patient_id}_{scan_type}.nii.gz'\n        return self.read_file(scan_filename)\n            ","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2021-08-26T13:46:56.275437Z","iopub.execute_input":"2021-08-26T13:46:56.275871Z","iopub.status.idle":"2021-08-26T13:47:28.828311Z","shell.execute_reply.started":"2021-08-26T13:46:56.275834Z","shell.execute_reply":"2021-08-26T13:47:28.827063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"A 3D point cloud is visualized by utilizing the Plotly library. Generating a trace (plotly.graph_objects.Scatter3d) per tissue type allows us to simultaneously show different point clouds with different opacities on a single 3D graph (plotly.graph_objects.Figure).\nThe resulting figure is interactive, try to rotate it or disable overlaying tumor tissue types.","metadata":{}},{"cell_type":"code","source":"import plotly.graph_objects as go\nimport numpy as np\n\n\ndef generate_3d_scatter(\n    x:np.array, y:np.array, z:np.array, colors:np.array,\n    size:int=3, opacity:float=0.2, scale:str='Teal',\n    hover:str='skip', name:str='MRI'\n) -> go.Scatter3d:\n    return go.Scatter3d(\n        x=x, y=y, z=z,\n        mode='markers', hoverinfo=hover,\n        marker = dict(\n            size=size, opacity=opacity,\n            color=colors, colorscale=scale\n        ),\n        name=name\n    )\n\n\nclass ImageViewer3d():\n    def __init__(\n        self, reader:ImageReader, mri_downsample:int=10, mri_colorscale:str='Ice'\n    ) -> None:\n        self.reader = reader\n        self.mri_downsample = mri_downsample\n        self.mri_colorscale = mri_colorscale\n\n    def load_clean_mri(self, image:np.array, orig_dim:int) -> dict:\n        shape_offset = image.shape[1]/orig_dim\n        z, x, y = (image > 0).nonzero()\n        # only (1/mri_downsample) is sampled for the resulting image\n        x, y, z = x[::self.mri_downsample], y[::self.mri_downsample], z[::self.mri_downsample]\n        colors = image[z, x, y]\n        return dict(x=x/shape_offset, y=y/shape_offset, z=z, colors=colors)\n    \n    def load_tumor_segmentation(self, image:np.array, orig_dim:int) -> dict:\n        tumors = {}\n        shape_offset = image.shape[1]/orig_dim\n        # 1/1, 1/3 and 1/5 pixels for tumor tissue classes 1(core), 2(invaded) and 4(enhancing)\n        sampling = {\n            1: 1, 2: 3, 4: 5\n        }\n        for class_idx in sampling:\n            z, x, y = (image == class_idx).nonzero()\n            x, y, z = x[::sampling[class_idx]], y[::sampling[class_idx]], z[::sampling[class_idx]]\n            tumors[class_idx] = dict(\n                x=x/shape_offset, y=y/shape_offset, z=z,\n                colors=class_idx/4\n            )\n        return tumors\n    \n    def collect_patient_data(self, scan:dict) -> tuple:\n        clean_mri = self.load_clean_mri(scan['scan'], scan['orig_shape'][0])\n        tumors = self.load_tumor_segmentation(scan['segmentation'], scan['orig_shape'][0])\n        markers_created = clean_mri['x'].shape[0] + sum(tumors[class_idx]['x'].shape[0] for class_idx in tumors)\n        return [\n            generate_3d_scatter(**clean_mri, scale=self.mri_colorscale, opacity=0.3, hover='skip', name='Brain MRI'),\n            generate_3d_scatter(**tumors[1], opacity=0.8, hover='all', name='Necrotic tumor core'),\n            generate_3d_scatter(**tumors[2], opacity=0.4, hover='all', name='Peritumoral invaded tissue'),\n            generate_3d_scatter(**tumors[4], opacity=0.4, hover='all', name='GD-enhancing tumor'),\n        ], markers_created\n    \n    def get_3d_scan(self, patient_idx:int, scan_type:str='flair') -> go.Figure:\n        scan = self.reader.load_patient_scan(patient_idx, scan_type)\n        data, num_markers = self.collect_patient_data(scan)\n        fig = go.Figure(data=data)\n        fig.update_layout(\n            title=f\"[Patient id:{patient_idx}] brain MRI scan ({num_markers} points)\",\n            legend_title=\"Pixel class (click to enable/disable)\",\n            font=dict(\n                family=\"Courier New, monospace\",\n                size=14,\n            ),\n            margin=dict(\n                l=0,r=0,b=0,t=30\n            ),\n            legend=dict(itemsizing='constant')\n        )\n        return fig","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:47:28.830051Z","iopub.execute_input":"2021-08-26T13:47:28.830714Z","iopub.status.idle":"2021-08-26T13:47:28.877162Z","shell.execute_reply.started":"2021-08-26T13:47:28.830670Z","shell.execute_reply":"2021-08-26T13:47:28.876142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reader = ImageReader('./data', img_size=128, normalize=True, single_class=False)\nviewer = ImageViewer3d(reader, mri_downsample=25)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:47:28.878727Z","iopub.execute_input":"2021-08-26T13:47:28.879398Z","iopub.status.idle":"2021-08-26T13:47:28.885446Z","shell.execute_reply.started":"2021-08-26T13:47:28.879334Z","shell.execute_reply":"2021-08-26T13:47:28.883925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Positive scan: a tumor is present.","metadata":{}},{"cell_type":"code","source":"fig = viewer.get_3d_scan(0, 't1')\nplotly.offline.iplot(fig)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:47:28.887715Z","iopub.execute_input":"2021-08-26T13:47:28.888228Z","iopub.status.idle":"2021-08-26T13:47:30.996660Z","shell.execute_reply.started":"2021-08-26T13:47:28.888179Z","shell.execute_reply":"2021-08-26T13:47:30.995213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Negative scan: a tumor is present too.","metadata":{}},{"cell_type":"code","source":"fig = viewer.get_3d_scan(14, 'flair')\nplotly.offline.iplot(fig)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:47:31.031678Z","iopub.execute_input":"2021-08-26T13:47:31.036267Z","iopub.status.idle":"2021-08-26T13:47:32.248536Z","shell.execute_reply.started":"2021-08-26T13:47:31.036202Z","shell.execute_reply":"2021-08-26T13:47:32.246995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As you can see, we're not looking at whether a tumor is present on an MRI scan, but rather *classifying a type of this tumor* (with or without MGMT promoter methylation).","metadata":{}},{"cell_type":"markdown","source":"# 3. Feature engineering","metadata":{}},{"cell_type":"markdown","source":"Let's collect a simple set of features - centroids for tumor cores and overall tumor size relative to a full MRI scan.","metadata":{}},{"cell_type":"code","source":"from skimage.morphology import binary_closing\nimport plotly.express as px\n\ndata = reader.load_patient_scan(0)\n\nimage = data['scan'][40]\nmasked_image = 1 * (image > 0)\nfilled_image = 1 * binary_closing(image)\n\npx.imshow(\n    np.array([image, masked_image, filled_image]),\n    facet_col=0, title=\"Different image masking - none, threshold and binary closing\",\n)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:47:32.250567Z","iopub.execute_input":"2021-08-26T13:47:32.251000Z","iopub.status.idle":"2021-08-26T13:47:36.284988Z","shell.execute_reply.started":"2021-08-26T13:47:32.250957Z","shell.execute_reply":"2021-08-26T13:47:36.283476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Tumor to all tissue ratio can be (approximately) calculated as (sum of tumor pixels/sum of tissue pixels)","metadata":{}},{"cell_type":"code","source":"def get_approx_pixel_count(scan:np.array, close:bool=False, mask:bool=False, mask_idx:int=-1) -> int:\n    slice_areas = []\n    for slice_idx in range(scan.shape[0]):\n        if close:\n            mri = 1 * binary_closing(scan[slice_idx, :, :])\n        elif mask_idx >= 0:\n            mri = 1 * (scan[slice_idx, :, :] == mask_idx)\n        elif mask:\n            mri = 1 * (scan[slice_idx, :, :] > 0)\n        else:\n            raise ValueError('Masking mechanism should be specified')\n        mri_area = mri.sum()\n        slice_areas.append(mri_area)\n    return np.sum(slice_areas)\n\nget_approx_pixel_count(data['segmentation'], mask=True) / get_approx_pixel_count(data['scan'], mask=True)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:47:36.287209Z","iopub.execute_input":"2021-08-26T13:47:36.287618Z","iopub.status.idle":"2021-08-26T13:47:36.340039Z","shell.execute_reply.started":"2021-08-26T13:47:36.287575Z","shell.execute_reply":"2021-08-26T13:47:36.338866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_centroid(scan:np.array, mask_idx:int=1) -> list:\n    z, x, y = (scan == mask_idx).nonzero()\n    x, y, z = np.median(x), np.median(y), np.median(z)\n    return [x/scan.shape[1], y/scan.shape[2], z/scan.shape[0]]\n\nget_centroid(data['segmentation'], 4), get_centroid(data['segmentation'], 1)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:47:36.341922Z","iopub.execute_input":"2021-08-26T13:47:36.342943Z","iopub.status.idle":"2021-08-26T13:47:36.390419Z","shell.execute_reply.started":"2021-08-26T13:47:36.342877Z","shell.execute_reply":"2021-08-26T13:47:36.389404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Putting everything into one DataFrame.","metadata":{}},{"cell_type":"code","source":"import pandas as pd\ndf = pd.read_csv('../input/rsna-miccai-brain-tumor-radiogenomic-classification/train_labels.csv')\ntargets = dict(zip(df.BraTS21ID, df.MGMT_value))","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:47:36.392429Z","iopub.execute_input":"2021-08-26T13:47:36.393251Z","iopub.status.idle":"2021-08-26T13:47:36.432675Z","shell.execute_reply.started":"2021-08-26T13:47:36.393206Z","shell.execute_reply":"2021-08-26T13:47:36.431341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nfeatures = []\nfinal_targets = []\nfor patient_idx in targets:\n    try:\n        data = reader.load_patient_scan(patient_idx)\n\n        shape = radiomics.shape.RadiomicsShape(\n            sitk.GetImageFromArray(data[\"scan\"]), \n            sitk.GetImageFromArray(data[\"segmentation\"])\n        )\n        try:\n            patient_features = [\n                shape.getMeshVolumeFeatureValue(), \\\n                shape.getVoxelVolumeFeatureValue(), \\\n                shape.getSurfaceAreaFeatureValue(), \\\n                shape.getSurfaceVolumeRatioFeatureValue(), \\\n                shape.getSphericityFeatureValue(), \\\n                shape.getCompactness1FeatureValue(), \\\n                shape.getCompactness2FeatureValue(), \\\n                shape.getSphericalDisproportionFeatureValue(), \\\n                shape.getMaximum3DDiameterFeatureValue(), \\\n                shape.getMaximum2DDiameterSliceFeatureValue(), \\\n                shape.getMaximum2DDiameterColumnFeatureValue(), \\\n                shape.getMaximum2DDiameterRowFeatureValue(), \\\n                shape.getMajorAxisLengthFeatureValue(), \\\n                shape.getMinorAxisLengthFeatureValue(), \\\n                shape.getLeastAxisLengthFeatureValue(), \\\n                shape.getElongationFeatureValue(), \\\n                shape.getFlatnessFeatureValue()\n            ]\n            features.append(patient_features)\n            final_targets.append(targets[patient_idx])\n        except:\n            print(patient_idx)\n    except FileNotFoundError:\n        print(patient_idx)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:51:40.399794Z","iopub.execute_input":"2021-08-26T13:51:40.400249Z","iopub.status.idle":"2021-08-26T13:55:38.160909Z","shell.execute_reply.started":"2021-08-26T13:51:40.400195Z","shell.execute_reply":"2021-08-26T13:55:38.159880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame(\n    features,\n    columns=[\n        \"getMeshVolumeFeatureValue\", \\\n        \"getVoxelVolumeFeatureValue\", \\\n        \"getSurfaceAreaFeatureValue\", \\\n        \"getSurfaceVolumeRatioFeatureValue\", \\\n        \"getSphericityFeatureValue\", \\\n        \"getCompactness1FeatureValue\", \\\n        \"getCompactness2FeatureValue\", \\\n        \"getSphericalDisproportionFeatureValue\", \\\n        \"getMaximum3DDiameterFeatureValue\", \\\n        \"getMaximum2DDiameterSliceFeatureValue\", \\\n        \"getMaximum2DDiameterColumnFeatureValue\", \\\n        \"getMaximum2DDiameterRowFeatureValue\", \\\n        \"getMajorAxisLengthFeatureValue\", \\\n        \"getMinorAxisLengthFeatureValue\", \\\n        \"getLeastAxisLengthFeatureValue\", \\\n        \"getElongationFeatureValue\",\n        \"getFlatnessFeatureValue\"\n    ]\n)\ndf[\"target\"] = final_targets\ndf = df.fillna(0)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:55:38.162707Z","iopub.execute_input":"2021-08-26T13:55:38.163185Z","iopub.status.idle":"2021-08-26T13:55:38.177725Z","shell.execute_reply.started":"2021-08-26T13:55:38.163123Z","shell.execute_reply":"2021-08-26T13:55:38.176595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Is there a difference between 1 and 0 classes? Let's look at the tumor volume percent:","metadata":{}},{"cell_type":"markdown","source":"# 4. Models","metadata":{}},{"cell_type":"markdown","source":"If there is indeed such a difference then a simple model would pick it up. Let's build an ensemble of KNN, decision tree and logistic regression classifiers.","metadata":{}},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:55:38.179399Z","iopub.execute_input":"2021-08-26T13:55:38.180196Z","iopub.status.idle":"2021-08-26T13:55:38.229664Z","shell.execute_reply.started":"2021-08-26T13:55:38.180033Z","shell.execute_reply":"2021-08-26T13:55:38.228208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX, y = df.drop('target', axis=1).values, df['target'].values\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.15, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:55:38.231492Z","iopub.execute_input":"2021-08-26T13:55:38.232243Z","iopub.status.idle":"2021-08-26T13:55:38.247039Z","shell.execute_reply.started":"2021-08-26T13:55:38.232156Z","shell.execute_reply":"2021-08-26T13:55:38.245798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from catboost import Pool, CatBoostClassifier","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:55:38.248927Z","iopub.execute_input":"2021-08-26T13:55:38.249766Z","iopub.status.idle":"2021-08-26T13:55:38.435604Z","shell.execute_reply.started":"2021-08-26T13:55:38.249719Z","shell.execute_reply":"2021-08-26T13:55:38.434457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = CatBoostClassifier()","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:55:38.437333Z","iopub.execute_input":"2021-08-26T13:55:38.437898Z","iopub.status.idle":"2021-08-26T13:55:38.446487Z","shell.execute_reply.started":"2021-08-26T13:55:38.437828Z","shell.execute_reply":"2021-08-26T13:55:38.445068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_pool = Pool(X_train, y_train)\ntest_pool = Pool(X_test, y_test)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:55:38.448299Z","iopub.execute_input":"2021-08-26T13:55:38.448780Z","iopub.status.idle":"2021-08-26T13:55:38.472676Z","shell.execute_reply.started":"2021-08-26T13:55:38.448708Z","shell.execute_reply":"2021-08-26T13:55:38.471520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fit(\n    train_pool, \n    eval_set=test_pool,\n    verbose=200\n)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:55:38.477616Z","iopub.execute_input":"2021-08-26T13:55:38.478159Z","iopub.status.idle":"2021-08-26T13:55:44.233723Z","shell.execute_reply.started":"2021-08-26T13:55:38.478115Z","shell.execute_reply":"2021-08-26T13:55:44.232504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.eval_metrics(\n    test_pool,\n    \"AUC\"\n)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:55:44.235531Z","iopub.execute_input":"2021-08-26T13:55:44.236043Z","iopub.status.idle":"2021-08-26T13:55:44.383704Z","shell.execute_reply.started":"2021-08-26T13:55:44.235996Z","shell.execute_reply":"2021-08-26T13:55:44.382475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}