{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Use stacked images (3D) and Efficientnet3D model\n\nAcknowledgements:\n\n- https://www.kaggle.com/ihelon/brain-tumor-eda-with-animations-and-modeling\n- https://www.kaggle.com/furcifer/torch-efficientnet3d-for-mri-no-train\n- https://github.com/shijianjian/EfficientNet-PyTorch-3D\n    \n    \nUse models with only one MRI type, then ensemble the 4 models \n","metadata":{"papermill":{"duration":0.019891,"end_time":"2021-10-02T09:05:40.963289","exception":false,"start_time":"2021-10-02T09:05:40.943398","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import os\nimport sys \nimport json\nimport glob\nimport random\nimport collections\nimport time\n\nimport numpy as np\nimport pandas as pd\nimport pydicom\nimport cv2\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport torch\nfrom torch import nn\nfrom torch.utils import data as torch_data\nfrom sklearn import model_selection as sk_model_selection\nfrom torch.nn import functional as torch_functional\nimport torch.nn.functional as F\nfrom torch.optim import lr_scheduler\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import roc_auc_score","metadata":{"papermill":{"duration":2.407823,"end_time":"2021-10-02T09:05:43.389452","exception":false,"start_time":"2021-10-02T09:05:40.981629","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-10T12:05:42.186165Z","iopub.execute_input":"2021-10-10T12:05:42.186462Z","iopub.status.idle":"2021-10-10T12:05:47.700148Z","shell.execute_reply.started":"2021-10-10T12:05:42.186367Z","shell.execute_reply":"2021-10-10T12:05:47.6994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if os.path.exists(\"../input/rsna-miccai-brain-tumor-radiogenomic-classification\"):\n    data_directory = '../input/rsna-miccai-brain-tumor-radiogenomic-classification'\n    pytorch3dpath = \"../input/efficientnetpyttorch3d/EfficientNet-PyTorch-3D\"\nelse:\n    data_directory = '/media/roland/data/kaggle/rsna-miccai-brain-tumor-radiogenomic-classification'\n    pytorch3dpath = \"EfficientNet-PyTorch-3D\"\n    \n#mri_types = ['FLAIR','T1w','T1wCE','T2w']\n#mri_types = ['FLAIR','T2w','T1w','T1wCE']\nmri_types = ['T1wCE']\nSIZE = 256\nNUM_IMAGES = 64\n\n# sys.path.append(pytorch3dpath)\n# from efficientnet_pytorch_3d import EfficientNet3D","metadata":{"lines_to_end_of_cell_marker":2,"lines_to_next_cell":2,"papermill":{"duration":0.054057,"end_time":"2021-10-02T09:05:43.461994","exception":false,"start_time":"2021-10-02T09:05:43.407937","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-10T12:05:47.703811Z","iopub.execute_input":"2021-10-10T12:05:47.704011Z","iopub.status.idle":"2021-10-10T12:05:47.711361Z","shell.execute_reply.started":"2021-10-10T12:05:47.703987Z","shell.execute_reply":"2021-10-10T12:05:47.710595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Functions to load images","metadata":{"papermill":{"duration":0.01799,"end_time":"2021-10-02T09:05:43.498118","exception":false,"start_time":"2021-10-02T09:05:43.480128","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from fractions import Fraction\nimport shutil\ndef image_aug(n,kijyun = NUM_IMAGES):\n    #print('★n before:', n.shape)\n    im_x = n.shape[1]\n    im_y = n.shape[2]\n    while kijyun != n.shape[0]:\n        n_cnt = n.shape[0]\n        n_tmpo = kijyun/n_cnt\n        bunkatsu = Fraction(n_tmpo).limit_denominator(kijyun)\n        bunshi = bunkatsu.numerator\n        bunnbo = bunkatsu.denominator\n        #print('bunshi:',bunshi,' bunnbo:',bunnbo)\n        n_n = np.array([])\n        if bunshi == bunnbo:\n            #そのまま\n            break\n        elif bunshi > bunnbo:\n            #拡張\n            if (bunshi / bunnbo) >= 2:\n                baisu = int(bunshi / bunnbo) - 1\n                for w,n_x in enumerate(n):\n                    #print('w:',w)\n                    n_x = n_x.reshape(-1,im_x,im_y)\n                    if n.shape[0] == w + 1:\n                        n_y = n_x\n                    else:\n                        n_y = n[w+1].reshape(-1,im_x,im_y)\n                    \n                    if(n_n.shape[0] == 0):\n                        n_n = n_x\n                    else:\n                        n_n = np.concatenate([n_n,n_x])\n                    #print('baisu:',baisu)\n                    for baisu_cnt in range(baisu):\n                        baisu_cnt += 1\n                        n_x = n_x * (baisu_cnt / baisu)\n                        n_y = n_y * ((baisu - baisu_cnt) / baisu)\n                        n_z = np.concatenate([n_x,n_y])\n                        n_z = np.mean(n_z,axis=0).reshape(-1,im_x,im_y)\n                        n_n = np.concatenate([n_n,n_z])\n            else:\n                #ins_cnt = int(bunnbo / 2)\n                ins_cnt = int((kijyun) / (kijyun - n_cnt))\n                if ins_cnt == kijyun:\n                    ins_cnt -= 1\n                #print('n_cnt:',n_cnt)\n                #print('kijyun:',kijyun)\n                #print('ins_cnt:',ins_cnt)\n                #print('int(n.shape[0]/ins_cnt):',int(n.shape[0]/ins_cnt))\n                for w in range(int(n_cnt/ins_cnt)):\n                    #print('w:',w)\n                    w += 1\n                    w = (w * ins_cnt)\n                    #print('w:',w)\n                    #print('w-ins_cnt:',w-ins_cnt)\n                    \n                    n_x = n[w-ins_cnt:w]\n                    \n                    #print('n_cnt:',n_cnt)\n                    #print('w:',w)\n                    if n_cnt <= w + 1:\n                        #一番後ろの列を\n                        #print('n_x:',n_x.shape)\n                        #print('n_x[-1:]:',n_x[-1:].shape)\n                        n_x = np.concatenate([n_x,n_x[-1:]])\n                    else:\n                        n_y = n[w+1].reshape(-1,im_x,im_y)\n                        n_y = np.concatenate([n_x[-1:],n_y])\n                        n_y = np.mean(n_y,axis=0).reshape(-1,im_x,im_y)\n                        n_x = np.concatenate([n_x,n_y])\n                    if(n_n.shape[0] == 0):\n                        n_n = n_x\n                    else:\n                        n_n = np.concatenate([n_n,n_x])\n                n_n = np.concatenate([n_n,n[w:]])\n        elif bunshi < bunnbo:\n            #縮小\n            #print('n:',n.shape)\n            img_size = n.shape[0]\n            if int(bunnbo / bunshi) >= 2:\n                small_size = int(bunnbo / bunshi)\n                \n                img_size_current = 0\n                while img_size > img_size_current:\n                    #small_sizeに指定されている画像を１つにまとめていく\n                    if img_size >= (img_size_current + small_size):\n                        n_x = np.expand_dims(n[img_size_current:img_size_current + small_size].mean(axis=0),0)\n                    else:\n                        n_x = np.expand_dims(n[img_size_current:img_size].mean(axis=0),0)\n                    \n                    if(n_n.shape[0] == 0):\n                        n_n = n_x\n                    else:\n                        n_n = np.concatenate([n_n,n_x])\n                    img_size_current = img_size_current + small_size\n            else:\n                while bunnbo >= kijyun: \n                    bunnbo = int(bunnbo / 2)\n                    bunshi = int(bunshi / 2)\n                del_cnt = int(bunnbo / 2)\n                i = 0\n                #print(del_cnt)\n                for n_x in n:\n                    i += 1\n                    if i % del_cnt != 0:\n                        n_x = n_x.reshape(-1,n_x.shape[0],n_x.shape[1])\n                        if(n_n.shape[0] == 0):\n                            n_n = n_x\n                        else:\n                            n_n = np.concatenate([n_n,n_x])\n                    else:\n                        del_cnt = del_cnt + bunnbo\n        n = n_n\n        #print('n.shape:',n.shape)\n    return n","metadata":{"execution":{"iopub.status.busy":"2021-10-10T12:05:47.712822Z","iopub.execute_input":"2021-10-10T12:05:47.713136Z","iopub.status.idle":"2021-10-10T12:05:47.739177Z","shell.execute_reply.started":"2021-10-10T12:05:47.713102Z","shell.execute_reply":"2021-10-10T12:05:47.738391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def load_dicom_image(path, img_size=SIZE):\n#     dicom = pydicom.read_file(path)\n#     data = dicom.pixel_array\n#     if np.min(data)==np.max(data):\n#         data = np.zeros((img_size,img_size))\n#         return data\n#     data = data - np.min(data)\n#     if np.max(data) != 0:\n#         data = data / np.max(data)\n    \n#     #data = (data * 255).astype(np.uint8)\n#     data = cv2.resize(data, (img_size, img_size))\n#     return data\n\n# def load_dicom_images_3d(scan_id, num_imgs=NUM_IMAGES, img_size=SIZE, mri_type=\"FLAIR\", split=\"train\"):\n\n#     files = sorted(glob.glob(f\"{data_directory}/{split}/{scan_id}/{mri_type}/*.dcm\"))\n    \n#     middle = len(files)//2\n#     num_imgs2 = num_imgs//2\n#     p1 = max(0, middle - num_imgs2)\n#     p2 = min(len(files), middle + num_imgs2)\n#     img3d = np.stack([load_dicom_image(f) for f in files[p1:p2]]).T \n#     if img3d.shape[-1] < num_imgs:\n#         n_zero = np.zeros((img_size, img_size, num_imgs - img3d.shape[-1]))\n#         img3d = np.concatenate((img3d,  n_zero), axis = -1)\n            \n#     return np.expand_dims(img3d,0)\n\n# load_dicom_images_3d(\"00000\").shape\ndef load_dicom_image(path, img_size=SIZE):\n    #print(path)\n    dicom = pydicom.read_file(path)\n    data = dicom.pixel_array\n    #print(np.array(data).shape)\n    \n#     if np.min(data)==np.max(data):\n#         data = np.zeros((img_size,img_size))\n#         return data\n#     data = data - np.min(data)\n#     if np.max(data) != 0:\n#         data = data / np.max(data)\n    \n    #data = (data * 255).astype(np.uint8)\n    data = cv2.resize(data, (img_size, img_size))\n    #print('(np.array(data).reshape(-1)!=0).sum():',(np.array(data).reshape(-1)!=0).sum())\n    if (np.array(data).reshape(-1)!=0).sum() >= 1000:\n        return data\n    else:\n        return False\n\ndef load_dicom_images_3d(scan_id, num_imgs=NUM_IMAGES, img_size=SIZE, mri_type=\"FLAIR\", split=\"train\"):\n    #print('scan_id:',scan_id)\n    if os.path.exists(f\"/tmp_np/{split}/{scan_id}/{mri_type}\"):\n        img3d = np.load(f\"/tmp_np/{split}/{scan_id}/{mri_type}/np_sa.npy\")\n        return img3d\n    else:\n        files = sorted(glob.glob(f\"{data_directory}/{split}/{scan_id}/{mri_type}/*.dcm\"))\n\n    #     middle = len(files)//2\n    #     num_imgs2 = num_imgs//2\n    #     p1 = max(0, middle - num_imgs2)\n    #     p2 = min(len(files), middle + num_imgs2)\n        #img3d = np.stack([load_dicom_image(f) for f in files[p1:p2]]).T\n        np_stack = []\n        for f in files:\n            f_np = load_dicom_image(f)\n            if type(f_np) == np.ndarray:\n                np_stack.append(f_np)\n        if len(np_stack) == 0:\n            img3d = np.zeros((1,img_size, img_size))\n        else:\n            img3d = np.stack(np_stack)\n        #img3d = np.stack([load_dicom_image(f) for f in files])\n        img3d = image_aug(img3d)\n\n        if img3d.shape[-1] < num_imgs:\n            n_zero = np.zeros((img_size, img_size, num_imgs - img3d.shape[-1]))\n            img3d = np.concatenate((img3d,  n_zero), axis = -1)\n\n        img3d = img3d - np.min(img3d)\n        if np.max(img3d) != 0:\n            img3d = img3d / np.max(img3d)\n        img3d = img3d.transpose(1,2,0)\n        img3d = np.expand_dims(img3d,0)\n        os.makedirs(f\"/tmp_np/{split}/{scan_id}/{mri_type}\")\n        np.save(f\"/tmp_np/{split}/{scan_id}/{mri_type}/np_sa.npy\",img3d)\n        return img3d\n\nload_dicom_images_3d(\"00000\").shape","metadata":{"papermill":{"duration":0.868774,"end_time":"2021-10-02T09:05:44.385083","exception":false,"start_time":"2021-10-02T09:05:43.516309","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-10T12:05:47.741632Z","iopub.execute_input":"2021-10-10T12:05:47.742Z","iopub.status.idle":"2021-10-10T12:05:52.850498Z","shell.execute_reply.started":"2021-10-10T12:05:47.741947Z","shell.execute_reply":"2021-10-10T12:05:52.844149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed_all(seed)\n        torch.backends.cudnn.deterministic = True\n\nset_seed(42)","metadata":{"papermill":{"duration":0.069733,"end_time":"2021-10-02T09:05:44.4759","exception":false,"start_time":"2021-10-02T09:05:44.406167","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-10T12:05:52.851653Z","iopub.execute_input":"2021-10-10T12:05:52.851915Z","iopub.status.idle":"2021-10-10T12:05:52.915655Z","shell.execute_reply.started":"2021-10-10T12:05:52.851882Z","shell.execute_reply":"2021-10-10T12:05:52.913321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## train / test splits","metadata":{"papermill":{"duration":0.018419,"end_time":"2021-10-02T09:05:44.513274","exception":false,"start_time":"2021-10-02T09:05:44.494855","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train_df = pd.read_csv(f\"{data_directory}/train_labels.csv\")\ndisplay(train_df)\n\ndf_train, df_valid = sk_model_selection.train_test_split(\n    train_df, \n    test_size=0.2, \n    random_state=42, \n    stratify=train_df[\"MGMT_value\"],\n)\n","metadata":{"papermill":{"duration":0.054423,"end_time":"2021-10-02T09:05:44.586322","exception":false,"start_time":"2021-10-02T09:05:44.531899","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-10T12:05:52.920843Z","iopub.execute_input":"2021-10-10T12:05:52.921154Z","iopub.status.idle":"2021-10-10T12:05:52.971576Z","shell.execute_reply.started":"2021-10-10T12:05:52.921117Z","shell.execute_reply":"2021-10-10T12:05:52.970589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.tail()","metadata":{"papermill":{"duration":0.030068,"end_time":"2021-10-02T09:05:44.637813","exception":false,"start_time":"2021-10-02T09:05:44.607745","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-10T12:05:52.975963Z","iopub.execute_input":"2021-10-10T12:05:52.976317Z","iopub.status.idle":"2021-10-10T12:05:52.996611Z","shell.execute_reply.started":"2021-10-10T12:05:52.976274Z","shell.execute_reply":"2021-10-10T12:05:52.992667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model and training classes","metadata":{"papermill":{"duration":0.031538,"end_time":"2021-10-02T09:05:44.69935","exception":false,"start_time":"2021-10-02T09:05:44.667812","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# class Dataset(torch_data.Dataset):\n#     def __init__(self, paths, targets=None, mri_type=None, label_smoothing=0.01, split=\"train\"):\n#         self.paths = paths\n#         self.targets = targets\n#         self.mri_type = mri_type\n#         self.label_smoothing = label_smoothing\n#         self.split = split\n          \n#     def __len__(self):\n#         return len(self.paths)\n    \n#     def __getitem__(self, index):\n#         scan_id = self.paths[index]\n#         if self.targets is None:\n#             data = load_dicom_images_3d(str(scan_id).zfill(5), mri_type=self.mri_type[index], split=self.split)\n#         else:\n#             data = load_dicom_images_3d(str(scan_id).zfill(5), mri_type=self.mri_type[index], split=\"train\")\n\n#         if self.targets is None:\n#             return {\"X\": torch.tensor(data).float(), \"id\": scan_id}\n#         else:\n#             y = torch.tensor(abs(self.targets[index]-self.label_smoothing), dtype=torch.float)\n#             return {\"X\": torch.tensor(data).float(), \"y\": y}\n","metadata":{"papermill":{"duration":0.045599,"end_time":"2021-10-02T09:05:44.782106","exception":false,"start_time":"2021-10-02T09:05:44.736507","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-10T12:05:53.00067Z","iopub.execute_input":"2021-10-10T12:05:53.000926Z","iopub.status.idle":"2021-10-10T12:05:53.009022Z","shell.execute_reply.started":"2021-10-10T12:05:53.000892Z","shell.execute_reply":"2021-10-10T12:05:53.008275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Dataset_LSTM(torch_data.Dataset):\n    def __init__(self, paths, targets=None, mri_type=None, label_smoothing=0.01, split=\"train\"):\n        self.paths = paths\n        self.targets = targets\n        self.mri_type = mri_type\n        self.label_smoothing = label_smoothing\n        self.split = split\n          \n    def __len__(self):\n        return len(self.paths)\n    \n    def __getitem__(self, index):\n        scan_id = self.paths[index]\n        if self.targets is None:\n            data = load_dicom_images_3d(str(scan_id).zfill(5), mri_type=self.mri_type[index], split=self.split)\n        else:\n            data = load_dicom_images_3d(str(scan_id).zfill(5), mri_type=self.mri_type[index], split=\"train\")\n        \n        \n        data = torch.tensor(data).permute(3,0,1,2).squeeze()\n        data = data.reshape([data.shape[0],-1])\n        \n        \n        if self.targets is None:\n            return {\"X\": data.float(), \"id\": scan_id}\n        else:\n            y = torch.tensor(abs(self.targets[index]-self.label_smoothing), dtype=torch.float)\n            return {\"X\": data.float(), \"y\": y}","metadata":{"papermill":{"duration":0.060276,"end_time":"2021-10-02T09:05:44.879221","exception":false,"start_time":"2021-10-02T09:05:44.818945","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-10T12:05:53.012727Z","iopub.execute_input":"2021-10-10T12:05:53.014368Z","iopub.status.idle":"2021-10-10T12:05:53.032989Z","shell.execute_reply.started":"2021-10-10T12:05:53.014331Z","shell.execute_reply":"2021-10-10T12:05:53.032015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# class Model(nn.Module):\n#     def __init__(self):\n#         super().__init__()\n#         #モデルを修正 efficientnet-b0→efficientnet-b3\n#         #self.net = EfficientNet3D.from_name(\"efficientnet-b0\", override_params={'num_classes': 2}, in_channels=1)\n#         self.net = EfficientNet3D.from_name(\"efficientnet-b3\", override_params={'num_classes': 2}, in_channels=1)\n#         n_features = self.net._fc.in_features\n#         self.net._fc = nn.Linear(in_features=n_features, out_features=1, bias=True)\n    \n#     def forward(self, x):\n#         out = self.net(x)\n#         return out","metadata":{"papermill":{"duration":0.039262,"end_time":"2021-10-02T09:05:44.951052","exception":false,"start_time":"2021-10-02T09:05:44.91179","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-10T12:05:53.040517Z","iopub.execute_input":"2021-10-10T12:05:53.046639Z","iopub.status.idle":"2021-10-10T12:05:53.054235Z","shell.execute_reply.started":"2021-10-10T12:05:53.046602Z","shell.execute_reply":"2021-10-10T12:05:53.053357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SwishImplementation2(torch.autograd.Function):\n    @staticmethod\n    def forward(ctx, i):\n        result = i * torch.sigmoid(i)\n        ctx.save_for_backward(i)\n        return result\n\n    @staticmethod\n    def backward(ctx, grad_output):\n        #i = ctx.saved_variables[0]\n        i = ctx.saved_tensors[0]\n        sigmoid_i = torch.sigmoid(i)\n        return grad_output * (sigmoid_i * (1 + i * (1 - sigmoid_i)))\n\nclass MemoryEfficientSwish2(nn.Module):\n    def forward(self, x):\n        return SwishImplementation2.apply(x)","metadata":{"papermill":{"duration":0.044029,"end_time":"2021-10-02T09:05:45.027004","exception":false,"start_time":"2021-10-02T09:05:44.982975","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-10T12:05:53.059961Z","iopub.execute_input":"2021-10-10T12:05:53.065027Z","iopub.status.idle":"2021-10-10T12:05:53.077086Z","shell.execute_reply.started":"2021-10-10T12:05:53.064991Z","shell.execute_reply":"2021-10-10T12:05:53.076123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Model_LSTM(nn.Module):\n    def __init__(self):\n        super().__init__()\n        \n#         self.grubat=nn.BatchNorm2d(64)\n        self.xh   = torch.nn.GRU(256*256, 1024)\n#         self.xh1  = torch.nn.GRU(256*256, 1024)\n        #self.xh1  = torch.nn.GRU(2048, 1024)\n        \n        self.drop = nn.Dropout(0.1)\n        self.hy   = torch.nn.Linear(1024, 1024)\n        self.bat  = nn.BatchNorm1d(1024)\n        #self.swift= MemoryEfficientSwish2()\n        self.swift= nn.ELU()\n        \n        self.drop1= nn.Dropout(0.1)\n        self.hy1  = torch.nn.Linear(1024, 1024)\n        self.bat1 = nn.BatchNorm1d(1024)\n        #self.swift1= MemoryEfficientSwish2()\n        self.swift1= nn.ELU()\n        \n        self.drop3= nn.Dropout(0.1)\n        self.hy3  = torch.nn.Linear(1024, 1024)\n        self.bat3 = nn.BatchNorm1d(1024)\n        #self.swift1= MemoryEfficientSwish2()\n        self.swift3= nn.ELU()\n        \n        self.drop4= nn.Dropout(0.1)\n        self.hy4  = torch.nn.Linear(1024, 1024)\n        self.bat4 = nn.BatchNorm1d(1024)\n        #self.swift1= MemoryEfficientSwish2()\n        self.swift4= nn.ELU()\n        \n        self.drop2= nn.Dropout(0.1)\n        self.bat2 = nn.BatchNorm1d(4096)\n        self.hy2  = torch.nn.Linear(4096, 1024)\n        self.elu  = nn.ELU()\n        \n        self.drop5= nn.Dropout(0.1)\n        self.bat5 = nn.BatchNorm1d(1024)\n        self.hy5  = torch.nn.Linear(1024, 1)\n        self.elu5 = nn.ELU()\n        \n        \n        #self.sig  = nn.Sigmoid()\n        #self.swish= MemoryEfficientSwish2()\n        \n#         self._init_weights(self.hy)\n#         self._init_weights(self.hy1)\n#         self._init_weights(self.hy3)\n#         self._init_weights(self.hy4)\n#         self._init_weights(self.hy2)\n        \n    def _init_weights(self, module):\n        if isinstance(module, nn.Linear):\n            self._std = np.sqrt(2/(module.in_features + module.out_features))\n            print('self._std:',self._std)\n            #module.weight.data.normal_(mean=0.0, std=0.02)\n            module.weight.data.normal_(mean=0.0, std=self._std)\n            if module.bias is not None:\n                module.bias.data.zero_()\n        elif isinstance(module, nn.LayerNorm):\n            module.bias.data.zero_()\n            module.weight.data.fill_(1.0)\n    \n    def forward(self, x):\n# #         print('x:',x.shape)\n#         x = x.reshape([-1,64,256,256])\n# #         print('x:',x.shape)\n#         x = self.grubat(x)\n# #         print('x:',x.shape)\n#         x = x.reshape([-1,64,256*256])\n\n        x1,x2 = self.xh(x)\n        #x1,x2 = self.xh1(x1)\n        \n        out = self.drop(x1[:,-1,:])\n        out = self.hy(out)\n        #out = self.swift(self.bat(out))\n        out = self.bat(out)\n        out = self.swift(out)\n        \n        out1= self.drop1(x1[:,32,:])\n        out1= self.hy1(out1)\n        #out1= self.swift1(self.bat1(out1))\n        out1= self.bat1(out1)\n        out1= self.swift1(out1)\n        \n        out2= self.drop3(x1[:,45,:])\n        out2= self.hy3(out2)\n        #out1= self.swift1(self.bat1(out1))\n        out2= self.bat3(out2)\n        out2= self.swift3(out2)\n        \n        out3= self.drop4(x1[:,15,:])\n        out3= self.hy4(out3)\n        #out1= self.swift1(self.bat1(out1))\n        out3= self.bat4(out3)\n        out3= self.swift4(out3)\n        \n#         x1,x2 = self.xh(x)\n\n#         out2= self.drop3(x1[:,15,:])\n#         out2= self.hy3(out2)\n#         #out1= self.swift1(self.bat1(out1))\n#         out2= self.bat3(out2)\n#         out2= self.swift3(out2)\n        \n#         out3= self.drop4(x1[:,-1,:])\n#         out3= self.hy4(out3)\n#         #out1= self.swift1(self.bat1(out1))\n#         out3= self.bat4(out3)\n#         out3= self.swift4(out3)\n        \n#         x3,x4 = self.xh1(x[:,32:64,:])\n        \n#         out = self.drop(x3[:,-1,:])\n#         out = self.hy(out)\n#         #out = self.swift(self.bat(out))\n#         out = self.bat(out)\n#         out = self.swift(out)\n        \n#         out1= self.drop1(x3[:,15,:])\n#         out1= self.hy1(out1)\n#         #out1= self.swift1(self.bat1(out1))\n#         out1= self.bat1(out1)\n#         out1= self.swift1(out1)\n\n        out =  torch.cat([out,out1,out2,out3], axis=1)\n        \n        out = self.drop2(out)\n        out = self.bat2(out)\n        out = self.hy2(out)\n        out = self.elu(out)\n        \n        out = self.drop5(out)\n        out = self.bat5(out)\n        out = self.hy5(out)\n\n        out = self.elu5(out)\n        \n        \n        #out = self.sig(out)\n        #out = self.nn_softmax(out)\n        return out","metadata":{"papermill":{"duration":0.05009,"end_time":"2021-10-02T09:05:45.109267","exception":false,"start_time":"2021-10-02T09:05:45.059177","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-10T12:05:53.083732Z","iopub.execute_input":"2021-10-10T12:05:53.087809Z","iopub.status.idle":"2021-10-10T12:05:53.118524Z","shell.execute_reply.started":"2021-10-10T12:05:53.087774Z","shell.execute_reply":"2021-10-10T12:05:53.117889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Trainer:\n    def __init__(\n        self, \n        model, \n        device, \n        optimizer, \n        criterion\n    ):\n        self.model = model\n        self.device = device\n        self.optimizer = optimizer\n        self.criterion = criterion\n\n        self.best_valid_score = np.inf\n        self.n_patience = 0\n        self.n_scheduler = 0\n        self.lastmodel = None\n        \n    def fit(self, epochs, train_loader, valid_loader, save_path, patience): \n        train_count = train_loader.__len__()\n        self.scheduler = lr_scheduler.OneCycleLR(optimizer=self.optimizer\n                                            , pct_start=0.3\n                                            , div_factor=100\n                                            , max_lr=0.0001\n                                            , epochs=epochs\n                                            , steps_per_epoch=train_count)\n        for n_epoch in range(1, epochs + 1):\n            self.info_message(\"EPOCH: {}\", n_epoch)\n            \n            train_loss, train_time = self.train_epoch(train_loader)\n            valid_loss, valid_auc, valid_time = self.valid_epoch(valid_loader)\n            \n            self.info_message(\n                \"[Epoch Train: {}] loss: {:.4f}, time: {:.2f} s            \",\n                n_epoch, train_loss, train_time\n            )\n            \n            self.info_message(\n                \"[Epoch Valid: {}] loss: {:.4f}, auc: {:.4f}, time: {:.2f} s\",\n                n_epoch, valid_loss, valid_auc, valid_time\n            )\n\n            # if True:\n            # if self.best_valid_score < valid_auc: \n            if self.best_valid_score > valid_loss: \n                self.save_model(n_epoch, save_path, valid_loss, valid_auc)\n                self.info_message(\n                     \"auc improved from {:.4f} to {:.4f}. Saved model to '{}'\", \n                    self.best_valid_score, valid_loss, self.lastmodel\n                )\n                self.best_valid_score = valid_loss\n                self.n_patience = 0\n                self.n_scheduler= 0\n            else:\n                self.n_patience += 1\n                self.n_scheduler+= 1\n                if self.n_scheduler >= 3:\n                    self.scheduler = lr_scheduler.OneCycleLR(optimizer=self.optimizer\n                                                        , pct_start=0.3\n                                                        , div_factor=1000\n                                                        , max_lr=0.001\n                                                        , epochs=epochs\n                                                        , steps_per_epoch=train_count)\n                    self.n_scheduler= 0\n            \n            if self.n_patience >= patience:\n                self.info_message(\"\\nValid auc didn't improve last {} epochs.\", patience)\n                break\n            \n    def train_epoch(self, train_loader):\n        self.model.train()\n        t = time.time()\n        sum_loss = 0\n\n        for step, batch in enumerate(train_loader, 1):\n            X = batch[\"X\"].to(self.device)\n            targets = batch[\"y\"].to(self.device)\n            self.optimizer.zero_grad()\n            outputs = self.model(X).squeeze(1)\n            \n            loss = self.criterion(outputs, targets)\n            loss.backward()\n\n            sum_loss += loss.detach().item()\n\n            self.optimizer.step()\n            self.scheduler.step()\n            _lr = self.scheduler.get_last_lr()[0]\n            message = 'Train Step {}/{}, train_loss: {:.4f}, get_lr: {:.7f}'\n            self.info_message(message, step, len(train_loader), sum_loss/step,_lr, end=\"\\r\")\n        \n        return sum_loss/len(train_loader), int(time.time() - t)\n    \n    def valid_epoch(self, valid_loader):\n        self.model.eval()\n        t = time.time()\n        sum_loss = 0\n        y_all = []\n        outputs_all = []\n\n        for step, batch in enumerate(valid_loader, 1):\n            with torch.no_grad():\n                X = batch[\"X\"].to(self.device)\n                targets = batch[\"y\"].to(self.device)\n\n                outputs = self.model(X).squeeze(1)\n                loss = self.criterion(outputs, targets)\n\n                sum_loss += loss.detach().item()\n                y_all.extend(batch[\"y\"].tolist())\n                outputs_all.extend(outputs.tolist())\n\n            message = 'Valid Step {}/{}, valid_loss: {:.4f}'\n            self.info_message(message, step, len(valid_loader), sum_loss/step, end=\"\\r\")\n            \n        y_all = [1 if x > 0.5 else 0 for x in y_all]\n        auc = roc_auc_score(y_all, outputs_all)\n        \n        return sum_loss/len(valid_loader), auc, int(time.time() - t)\n    \n    def save_model(self, n_epoch, save_path, loss, auc):\n        if self.lastmodel != None:\n            os.remove(self.lastmodel)\n        self.lastmodel = f\"{save_path}-e{n_epoch}-loss{loss:.3f}-auc{auc:.3f}.pth\"\n        torch.save(\n            {\n                \"model_state_dict\": self.model.state_dict(),\n                \"optimizer_state_dict\": self.optimizer.state_dict(),\n                \"best_valid_score\": self.best_valid_score,\n                \"n_epoch\": n_epoch,\n            },\n            self.lastmodel,\n        )\n    \n    @staticmethod\n    def info_message(message, *args, end=\"\\n\"):\n        print(message.format(*args), end=end)","metadata":{"papermill":{"duration":0.066229,"end_time":"2021-10-02T09:05:45.207548","exception":false,"start_time":"2021-10-02T09:05:45.141319","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-10T12:05:53.122894Z","iopub.execute_input":"2021-10-10T12:05:53.123464Z","iopub.status.idle":"2021-10-10T12:05:53.16155Z","shell.execute_reply.started":"2021-10-10T12:05:53.12343Z","shell.execute_reply":"2021-10-10T12:05:53.160745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## train models","metadata":{"papermill":{"duration":0.01988,"end_time":"2021-10-02T09:05:45.251604","exception":false,"start_time":"2021-10-02T09:05:45.231724","status":"completed"},"tags":[]}},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\ndef train_mri_type(df_train, df_valid, mri_type):\n    if mri_type==\"all\":\n        train_list = []\n        valid_list = []\n        for mri_type in mri_types:\n            df_train.loc[:,\"MRI_Type\"] = mri_type\n            train_list.append(df_train.copy())\n            df_valid.loc[:,\"MRI_Type\"] = mri_type\n            valid_list.append(df_valid.copy())\n\n        df_train = pd.concat(train_list)\n        df_valid = pd.concat(valid_list)\n    else:\n        df_train.loc[:,\"MRI_Type\"] = mri_type\n        df_valid.loc[:,\"MRI_Type\"] = mri_type\n\n    print(df_train.shape, df_valid.shape)\n    display(df_train.head())\n    \n    train_data_retriever = Dataset_LSTM(\n        df_train[\"BraTS21ID\"].values, \n        df_train[\"MGMT_value\"].values, \n        df_train[\"MRI_Type\"].values\n    )\n\n    valid_data_retriever = Dataset_LSTM(\n        df_valid[\"BraTS21ID\"].values, \n        df_valid[\"MGMT_value\"].values,\n        df_valid[\"MRI_Type\"].values\n    )\n\n    train_loader = torch_data.DataLoader(\n        train_data_retriever,\n        batch_size=4,\n        shuffle=True,\n        num_workers=8,\n    )\n\n    valid_loader = torch_data.DataLoader(\n        valid_data_retriever, \n        batch_size=4,\n        shuffle=False,\n        num_workers=8,\n    )\n\n    model = Model_LSTM()\n    model.to(device)\n\n    #checkpoint = torch.load(\"best-model-all-auc0.555.pth\")\n    #model.load_state_dict(checkpoint[\"model_state_dict\"])\n\n    #print(model)\n\n    #★学習率を修正0.001→0.0001\n    optimizer = torch.optim.Adam(model.parameters(), lr=0.0001)\n    #optimizer = torch.optim.SGD(model.parameters(), lr=0.001, momentum=0.9)\n\n    criterion = torch_functional.binary_cross_entropy_with_logits\n\n    trainer = Trainer(\n        model, \n        device, \n        optimizer, \n        criterion\n    )\n\n    history = trainer.fit(\n        15, \n        train_loader, \n        valid_loader, \n        f\"{mri_type}\", \n        10,\n    )\n    \n    #★numpyをファイル保存する対応の際に追加\n    shutil.rmtree(f'/tmp_np/')\n    \n    return trainer.lastmodel\n\nmodelfiles = None\n\nif not modelfiles:\n    modelfiles = [train_mri_type(df_train, df_valid, m) for m in mri_types]\n    print(modelfiles)","metadata":{"lines_to_next_cell":2,"papermill":{"duration":7089.581815,"end_time":"2021-10-02T11:03:54.852825","exception":false,"start_time":"2021-10-02T09:05:45.27101","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-10T12:05:53.166235Z","iopub.execute_input":"2021-10-10T12:05:53.168093Z","iopub.status.idle":"2021-10-10T12:08:02.439553Z","shell.execute_reply.started":"2021-10-10T12:05:53.16806Z","shell.execute_reply":"2021-10-10T12:08:02.43741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Predict function","metadata":{"papermill":{"duration":1.966634,"end_time":"2021-10-02T11:03:58.793052","exception":false,"start_time":"2021-10-02T11:03:56.826418","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# def predict(modelfile, df, mri_type, split):\n#     print(\"Predict:\", modelfile, mri_type, df.shape)\n#     df.loc[:,\"MRI_Type\"] = mri_type\n#     data_retriever = Dataset_LSTM(\n#         df.index.values, \n#         mri_type=df[\"MRI_Type\"].values,\n#         split=split\n#     )\n\n#     data_loader = torch_data.DataLoader(\n#         data_retriever,\n#         batch_size=4,\n#         shuffle=False,\n#         num_workers=8,\n#     )\n   \n#     model = Model_LSTM()\n#     model.to(device)\n    \n#     checkpoint = torch.load(modelfile)\n#     model.load_state_dict(checkpoint[\"model_state_dict\"])\n#     model.eval()\n    \n#     y_pred = []\n#     ids = []\n\n#     for e, batch in enumerate(data_loader,1):\n#         print(f\"{e}/{len(data_loader)}\", end=\"\\r\")\n#         with torch.no_grad():\n#             tmp_pred = torch.sigmoid(model(batch[\"X\"].to(device))).cpu().numpy().squeeze()\n#             #tmp_pred = model(batch[\"X\"].to(device)).cpu().numpy().astype(np.float64).squeeze()\n#             #tmp_pred = np.where(tmp_pred>1.0,1.0,np.where(tmp_pred<0.0,0.0,tmp_pred))\n#             #print(\"tmp_pred:\",tmp_pred)\n#             if tmp_pred.size == 1:\n#                 y_pred.append(tmp_pred)\n#             else:\n#                 y_pred.extend(tmp_pred.tolist())\n#             ids.extend(batch[\"id\"].numpy().tolist())\n            \n#     preddf = pd.DataFrame({\"BraTS21ID\": ids, \"MGMT_value\": y_pred}) \n#     preddf = preddf.set_index(\"BraTS21ID\")\n#     return preddf","metadata":{"papermill":{"duration":2.01394,"end_time":"2021-10-02T11:04:02.824998","exception":false,"start_time":"2021-10-02T11:04:00.811058","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-10T12:08:02.440936Z","iopub.status.idle":"2021-10-10T12:08:02.441363Z","shell.execute_reply.started":"2021-10-10T12:08:02.441138Z","shell.execute_reply":"2021-10-10T12:08:02.441162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Ensemble for validation","metadata":{"papermill":{"duration":2.034737,"end_time":"2021-10-02T11:04:07.079672","exception":false,"start_time":"2021-10-02T11:04:05.044935","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# modelfiles = [\"../input/gru-net-training-data/FLAIR-e12-loss0.654-auc0.692.pth\"\n#              ,\"../input/gru-net-training-data/T1w-e9-loss0.685-auc0.578.pth\"\n#              ,\"../input/gru-net-training-data/T1wCE-e15-loss0.679-auc0.606.pth\"\n#              ,\"../input/gru-net-training-data/T2w-e6-loss0.657-auc0.698.pth\"]","metadata":{"papermill":{"duration":1.996541,"end_time":"2021-10-02T11:04:11.155266","exception":false,"start_time":"2021-10-02T11:04:09.158725","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-10T12:08:02.442556Z","iopub.status.idle":"2021-10-10T12:08:02.442967Z","shell.execute_reply.started":"2021-10-10T12:08:02.442737Z","shell.execute_reply":"2021-10-10T12:08:02.442758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_valid = df_valid.set_index(\"BraTS21ID\")\n# df_valid[\"MGMT_pred\"] = 0\n# for m, mtype in zip(modelfiles,  mri_types):\n#     pred = predict(m, df_valid, mtype, \"train\")\n#     df_valid[\"MGMT_pred\"] += pred[\"MGMT_value\"]\n# df_valid[\"MGMT_pred\"] /= len(modelfiles)\n# auc = roc_auc_score(df_valid[\"MGMT_value\"], df_valid[\"MGMT_pred\"])\n# print(f\"Validation ensemble AUC: {auc:.4f}\")\n# sns.displot(df_valid[\"MGMT_pred\"])","metadata":{"papermill":{"duration":2.582513,"end_time":"2021-10-02T11:04:15.968956","exception":false,"start_time":"2021-10-02T11:04:13.386443","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-10T12:08:02.444265Z","iopub.status.idle":"2021-10-10T12:08:02.444811Z","shell.execute_reply.started":"2021-10-10T12:08:02.444574Z","shell.execute_reply":"2021-10-10T12:08:02.444597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_valid = df_valid.set_index(\"BraTS21ID\")\n# df_valid[\"MGMT_pred\"] = 0\n# for m, mtype in zip(modelfiles,  mri_types):\n#     pred = predict(m, df_valid, mtype, \"train\")\n#     df_valid[\"MGMT_pred\" + mtype] = pred[\"MGMT_value\"].astype(np.float64)\n# # df_valid[\"MGMT_pred\"] /= len(modelfiles)\n# for mtype in mri_types:\n#     auc = roc_auc_score(df_valid[\"MGMT_value\"], df_valid[\"MGMT_pred\" + mtype])\n#     print(f\"Validation ensemble AUC: {auc:.4f}\")\n#     sns.displot(df_valid[\"MGMT_pred\" + mtype].astype(np.float64))","metadata":{"papermill":{"duration":367.689203,"end_time":"2021-10-02T11:10:25.636606","exception":false,"start_time":"2021-10-02T11:04:17.947403","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-10T12:08:02.446157Z","iopub.status.idle":"2021-10-10T12:08:02.44698Z","shell.execute_reply.started":"2021-10-10T12:08:02.446739Z","shell.execute_reply":"2021-10-10T12:08:02.446762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":2.227802,"end_time":"2021-10-02T11:10:29.849733","exception":false,"start_time":"2021-10-02T11:10:27.621931","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Ensemble for submission","metadata":{"papermill":{"duration":2.019426,"end_time":"2021-10-02T11:10:33.896161","exception":false,"start_time":"2021-10-02T11:10:31.876735","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# submission = pd.read_csv(f\"{data_directory}/sample_submission.csv\", index_col=\"BraTS21ID\")\n\n# submission[\"MGMT_value\"] = 0\n# for m, mtype in zip(modelfiles, mri_types):\n#     pred = predict(m, submission, mtype, split=\"test\")\n#     submission[\"MGMT_value\"] += pred[\"MGMT_value\"]\n\n# submission[\"MGMT_value\"] /= len(modelfiles)\n# submission[\"MGMT_value\"].to_csv(\"submission.csv\")","metadata":{"papermill":{"duration":2.034373,"end_time":"2021-10-02T11:10:37.915731","exception":false,"start_time":"2021-10-02T11:10:35.881358","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-10T12:08:02.447891Z","iopub.status.idle":"2021-10-10T12:08:02.448584Z","shell.execute_reply.started":"2021-10-10T12:08:02.448342Z","shell.execute_reply":"2021-10-10T12:08:02.448365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# submission = pd.read_csv(f\"{data_directory}/sample_submission.csv\", index_col=\"BraTS21ID\")\n\n# submission[\"MGMT_value\"] = 0\n# for m, mtype in zip(modelfiles, mri_types):\n#     pred = predict(m, submission, mtype, split=\"test\")\n#     submission[\"MGMT_value\"+mtype] = pred[\"MGMT_value\"]\n\n# submission[\"MGMT_value\"] += submission[\"MGMT_valueFLAIR\"] * 0.5\n# submission[\"MGMT_value\"] += submission[\"MGMT_valueT1w\"] * 0.00\n# submission[\"MGMT_value\"] += submission[\"MGMT_valueT1wCE\"] * 0.00\n# submission[\"MGMT_value\"] += submission[\"MGMT_valueT2w\"] * 0.5\n\n# submission[\"MGMT_value\"].to_csv(\"submission.csv\")","metadata":{"papermill":{"duration":345.959557,"end_time":"2021-10-02T11:16:26.097875","exception":false,"start_time":"2021-10-02T11:10:40.138318","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-10T12:08:02.449651Z","iopub.status.idle":"2021-10-10T12:08:02.450495Z","shell.execute_reply.started":"2021-10-10T12:08:02.450235Z","shell.execute_reply":"2021-10-10T12:08:02.450264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# submission","metadata":{"papermill":{"duration":2.050501,"end_time":"2021-10-02T11:16:30.167806","exception":false,"start_time":"2021-10-02T11:16:28.117305","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-10T12:08:02.451622Z","iopub.status.idle":"2021-10-10T12:08:02.452034Z","shell.execute_reply.started":"2021-10-10T12:08:02.451806Z","shell.execute_reply":"2021-10-10T12:08:02.451827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sns.displot(submission[\"MGMT_value\"])","metadata":{"papermill":{"duration":2.30695,"end_time":"2021-10-02T11:16:34.835578","exception":false,"start_time":"2021-10-02T11:16:32.528628","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-10T12:08:02.453026Z","iopub.status.idle":"2021-10-10T12:08:02.453443Z","shell.execute_reply.started":"2021-10-10T12:08:02.453205Z","shell.execute_reply":"2021-10-10T12:08:02.453226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":2.061063,"end_time":"2021-10-02T11:16:38.908618","exception":false,"start_time":"2021-10-02T11:16:36.847555","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}