{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Use stacked images (3D) and simple 3D CNN model\n\nUse models with only one MRI type, then ensemble the 4 models \n\nThe models were trained using this notebook: https://www.kaggle.com/rluethy/simple-3d-cnn-with-one-mri-type\n\nAcknowledgements:\n\n- https://www.kaggle.com/ihelon/brain-tumor-eda-with-animations-and-modeling\n- https://www.kaggle.com/furcifer/torch-efficientnet3d-for-mri-no-train\n- https://www.kaggle.com/davidbroberts/adjusting-contrast-on-mr-images\n    ","metadata":{"papermill":{"duration":0.018361,"end_time":"2021-08-08T19:41:50.969168","exception":false,"start_time":"2021-08-08T19:41:50.950807","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import os\nimport sys \nimport json\nimport glob\nimport random\nimport collections\nimport time\nimport re\nimport numpy as np\nimport pandas as pd\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nimport cv2\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport torch\nfrom torch import nn\nfrom torch.utils import data as torch_data\nfrom sklearn import model_selection as sk_model_selection\nfrom torch.nn import functional as torch_functional\nimport torch.nn.functional as F\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import roc_auc_score, log_loss","metadata":{"papermill":{"duration":2.441444,"end_time":"2021-08-08T19:41:53.425983","exception":false,"start_time":"2021-08-08T19:41:50.984539","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-14T17:28:57.304032Z","iopub.execute_input":"2021-08-14T17:28:57.304435Z","iopub.status.idle":"2021-08-14T17:28:57.31304Z","shell.execute_reply.started":"2021-08-14T17:28:57.304403Z","shell.execute_reply":"2021-08-14T17:28:57.312255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Functions to load images","metadata":{"papermill":{"duration":0.014835,"end_time":"2021-08-08T19:41:53.496831","exception":false,"start_time":"2021-08-08T19:41:53.481996","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Load and process images\n\nimport os\nimport glob\nimport re\nimport pickle\n\nimport numpy as np\nimport pandas as pd\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nimport cv2\nimport matplotlib.pyplot as plt\nfrom IPython.display import HTML\nfrom base64 import b64encode\nimport matplotlib.animation as animation\n\n# Settings\n\nif os.path.exists(\"../input/rsna-miccai-brain-tumor-radiogenomic-classification\"):\n    data_directory = '../input/rsna-miccai-brain-tumor-radiogenomic-classification'\n    working_dir = \"/tmp/rsna\"\n    modelpath = \"../input/brain-tumor-models\"\nelse:\n    data_directory = '/media/roland/data/kaggle/rsna-miccai-brain-tumor-radiogenomic-classification'\n    working_dir = \"/tmp/rsna\"\n\nif not os.path.exists(working_dir):\n    os.mkdir(working_dir)\n\nmri_types = ['FLAIR', 'T1w', 'T1wCE', 'T2w']\nSIZE = 256\nNUM_IMAGES = 32\nUSE_IMAGE_CACHE = False\nUSE_LUT_CONTRAST = None  # {\"window_width\": 2000, \"window_level\": 2000}\nUSE_VOI_LUT = True\n\n\ndef find_crop_area(imgfiles):\n    x1 = 1000\n    x2 = 0\n    y1 = 1000\n    y2 = 0\n\n    for f in imgfiles:\n        dicom = pydicom.read_file(f)\n        data = dicom.pixel_array\n        # bb = None\n        if np.max(data) > np.min(data):\n            data = data - np.min(data)\n            data = data / np.max(data)\n            data = (data * 255).astype(np.uint8)\n            bb = cv2.boundingRect(data)\n            if (bb[2] > 0) and (bb[3] > 0):\n                if bb[0] < x1:\n                    x1 = bb[0]\n                if bb[1] < y1:\n                    y1 = bb[1]\n                if bb[0] + bb[2] > x2:\n                    x2 = bb[0] + bb[2]\n                if bb[1] + bb[3] > y2:\n                    y2 = bb[1] + bb[3]\n\n        # print(bb, x1, x2, y1, y2)\n\n    if (x2 > x1) and (y2 > y2):\n        return x1, y1, x2 - x1, y2 - y1\n    else:\n        return 0, 0, data.shape[0], data.shape[1]\n\n\ndef load_dicom_image(path, img_size=SIZE, crop_area=None, voi_lut=USE_VOI_LUT, rotate=0):\n    dicom = pydicom.read_file(path)\n    # print(path[-10:], dicom.InstanceNumber, np.min(data), np.max(data), end=\" \")\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n        # print(\"lut\", np.min(data), np.max(data))\n    else:\n        data = dicom.pixel_array\n        # print()\n    if crop_area:\n        cropped = data[crop_area[1]:crop_area[1] + crop_area[3], crop_area[0]:crop_area[0] + crop_area[2]]\n        data = cropped\n    if rotate > 0:\n        rot_choices = [0, cv2.ROTATE_90_CLOCKWISE, cv2.ROTATE_90_COUNTERCLOCKWISE, cv2.ROTATE_180]\n        data = cv2.rotate(data, rot_choices[rotate])\n\n    try:\n        data = cv2.resize(data, (img_size, img_size))\n    except Exception as exc:\n        print(exc)\n        print(path)\n        print(crop_area)\n        raise (exc)\n    return data\n\n\ndef get_image_plane(img_path):\n    # Ref:\n    # https://www.kaggle.com/davidbroberts/determining-mr-image-planes\n\n    dicom = pydicom.read_file(img_path)\n    loc = dicom.ImageOrientationPatient\n\n    row_x = round(loc[0])\n    row_y = round(loc[1])\n    row_z = round(loc[2])\n    col_x = round(loc[3])\n    col_y = round(loc[4])\n    col_z = round(loc[5])\n\n    if row_x == 1 and row_y == 0 and col_x == 0 and col_y == 0:\n        return 0, \"Coronal\"\n\n    if row_x == 0 and row_y == 1 and col_x == 0 and col_y == 0:\n        return 1, \"Sagittal\"\n\n    if row_x == 1 and row_y == 0 and col_x == 0 and col_y == 1:\n        return 2, \"Axial\"\n\n    return \"Unknown\"\n\n\ndef load_dicom_images_3d(scan_id, num_imgs=NUM_IMAGES, img_size=SIZE, mri_type=\"FLAIR\",\n                         split=\"train\", lut_contrast=USE_LUT_CONTRAST,  # {\"window_width\": 1000, \"window_level\": 2000},\n                         offset=0, voi_lut=USE_VOI_LUT, rotate=0, use_cache=USE_IMAGE_CACHE,\n                         threshold=0):\n    cfn = f\"{working_dir}/{scan_id}_{mri_type}_{SIZE}_{num_imgs}_{offset}_{rotate}{'L' if voi_lut else ''}{'C' if lut_contrast else ''}.pkl\"\n    if use_cache and os.path.exists(cfn):\n        img3d = pickle.load(open(cfn, \"rb\"))\n    else:\n        files = sorted(glob.glob(f\"{data_directory}/{split}/{scan_id}/{mri_type}/*.dcm\"),\n                       key=lambda var: [int(x) if x.isdigit() else x for x in re.findall(r'[^0-9]|[0-9]+', var)])\n\n        assert len(files) > 0, f\"no image files for {data_directory}/{split}/{scan_id}/{mri_type}/*.dcm\"\n        middle = len(files) // 2 + offset\n        # print(\"A \",scan_id, \"n\", len(files), offset, \"m\", middle)\n        if (middle <= 5) or (middle >= (len(files) - 5)):\n            middle = len(files) // 2\n        num_imgs2 = num_imgs // 2\n        p1 = max(0, middle - num_imgs2)\n        p2 = min(len(files), middle + num_imgs2)\n        # print(\"B \",\"n\",len(files), offset, \"m\", middle, \"s\",p1, \"e\", p2)\n\n        crop_area = find_crop_area(files[p1:p2])\n        img3d = np.stack([load_dicom_image(f, SIZE, crop_area, voi_lut, rotate) for f in files[p1:p2]]).T\n        # print(np.min(img3d), np.max(img3d), np.mean(img3d), np.median(img3d))\n        if img3d.shape[-1] < num_imgs:\n            n_zero = np.zeros((img_size, img_size, num_imgs - img3d.shape[-1]))\n            img3d = np.concatenate((img3d, n_zero), axis=-1)\n\n        # print(np.min(img3d), np.max(img3d), np.mean(img3d), np.median(img3d))\n        if lut_contrast:\n            img3d = img3d - np.min(img3d)\n            img3d = img3d / np.max(img3d)\n            img3d = lut_contrast[\"window_level\"] * img3d\n            # print(np.min(img3d), np.max(img3d), np.mean(img3d), np.median(img3d))\n            lut = make_lut(img3d, windowWidth=lut_contrast[\"window_width\"], windowLevel=lut_contrast[\"window_level\"])\n            img3d = np.reshape(apply_lut(img3d, lut), (img3d.shape[0], img3d.shape[1], img3d.shape[2]))\n            # print(np.min(img3d), np.max(img3d), np.mean(img3d), np.median(img3d))\n        if np.min(img3d) < np.max(img3d):\n            img3d = img3d - np.min(img3d)\n            img3d = img3d / np.max(img3d)\n            #img3d = 2*img3d - 1\n\n        if threshold > 0:\n            idx = img3d < threshold\n            img3d[idx] = 0\n\n        if use_cache:\n            pickle.dump(img3d, open(cfn, \"wb\"))\n\n    # print(np.min(img3d), np.max(img3d), np.mean(img3d), np.median(img3d))\n    return np.expand_dims(img3d, 0)\n\n\n# Adjusting Contrast on MR Images\n# https://www.kaggle.com/davidbroberts/adjusting-contrast-on-mr-images\n# Make a simple linear VOI LUT from the raw (stored) pixel data\ndef make_lut(storedPixels, windowWidth, windowLevel, p_i=\"MONOCHROME2\"):\n    # Slope and Intercept set to 1 and 0 for MR. Get these from DICOM tags instead if using\n    # on a modality that requires them (CT, PT etc)\n    slope = 1.0\n    intercept = 0.0\n    minPixel = int(np.amin(storedPixels))\n    maxPixel = int(np.amax(storedPixels))\n\n    # Make an empty array for the LUT the size of the pixel 'width' in the raw pixel data\n    lut = [0] * (maxPixel + 1)\n\n    # Invert pixels and windowLevel for MONOCHROME1. We invert the specified windowLevel so that \n    # increasing the level value makes the images brighter regardless of photometric intrepretation\n    invert = False\n    if p_i == \"MONOCHROME1\":\n        invert = True\n    else:\n        windowLevel = (maxPixel - minPixel) - windowLevel\n\n    # Loop through the pixels and calculate each LUT value\n    for storedValue in range(minPixel, maxPixel):\n        modalityLutValue = storedValue * slope + intercept\n        voiLutValue = (((modalityLutValue - windowLevel) / windowWidth + 0.5) * 255.0)\n        clampedValue = min(max(voiLutValue, 0), 255)\n        if invert:\n            lut[storedValue] = round(255 - clampedValue)\n        else:\n            lut[storedValue] = round(clampedValue)\n\n    return lut\n\n\n# Apply the LUT to a pixel array\ndef apply_lut(pixels_in, lut):\n    pixels_in = pixels_in.flatten()\n    pixels_out = [0] * len(pixels_in)\n\n    for i in range(0, len(pixels_in)):\n        pixel = int(pixels_in[i])\n        pixels_out[i] = int(lut[pixel])\n\n    return pixels_out\n\n\n# Save images as video\ndef play(filename):\n    html = ''\n    video = open(filename, 'rb').read()\n    src = 'data:video/mp4;base64,' + b64encode(video).decode()\n    html += '<video width=500 controls autoplay loop><source src=\"%s\" type=\"video/mp4\"></video>' % src\n    return HTML(html)\n\n\ndef create_video(imgs, output=f'{working_dir}/vis_video.mp4', frame_delay=200):\n    fig, ax = plt.subplots(figsize=(15, 10))\n    ims = []\n    n_imgs = imgs.shape[-1]\n    for i in range(n_imgs):\n        # print(i,np.min(imgs[0,:,:,i]), np.max(imgs[0,:,:,i]))\n        im = ax.imshow(imgs[0, :, :, i], animated=True, cmap='gray')\n        ims.append([im])\n    plt.close(fig)\n    # print(len(ims))\n    ani = animation.ArtistAnimation(fig, ims, interval=frame_delay, blit=True, repeat_delay=1000)\n\n    ani.save(output)\n    return output\n\n\nif __name__ == \"__main__\":\n    a = load_dicom_images_3d(\"00000\")\n    print(a.shape)\n    print(np.min(a), np.max(a), np.mean(a), np.median(a))\n\nTHRESHOLD = 0.5\n\n! rm {working_dir}/*.pkl\na = load_dicom_images_3d(\"00144\", offset=-2, threshold=THRESHOLD)\nprint(a.shape)\nprint(np.min(a), np.max(a), np.mean(a), np.median(a))\n\nplay(create_video(a))","metadata":{"papermill":{"duration":1.105527,"end_time":"2021-08-08T19:41:54.616738","exception":false,"start_time":"2021-08-08T19:41:53.511211","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-14T17:28:57.318002Z","iopub.execute_input":"2021-08-14T17:28:57.318344Z","iopub.status.idle":"2021-08-14T17:29:02.698044Z","shell.execute_reply.started":"2021-08-14T17:28:57.318319Z","shell.execute_reply":"2021-08-14T17:29:02.697198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed_all(seed)\n        torch.backends.cudnn.deterministic = True\n\nset_seed(42)","metadata":{"papermill":{"duration":0.063443,"end_time":"2021-08-08T19:41:54.698288","exception":false,"start_time":"2021-08-08T19:41:54.634845","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-14T17:29:02.699817Z","iopub.execute_input":"2021-08-14T17:29:02.700201Z","iopub.status.idle":"2021-08-14T17:29:02.706982Z","shell.execute_reply.started":"2021-08-14T17:29:02.700144Z","shell.execute_reply":"2021-08-14T17:29:02.706226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset class","metadata":{"papermill":{"duration":0.017285,"end_time":"2021-08-08T19:41:54.73169","exception":false,"start_time":"2021-08-08T19:41:54.714405","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class Dataset(torch_data.Dataset):\n    def __init__(self, paths, targets=None, mri_type=None, label_smoothing=0.01, split=\"train\", augment=False):\n        self.paths = paths\n        self.targets = targets\n        self.mri_type = mri_type\n        self.label_smoothing = label_smoothing\n        self.split = split\n        self.augment = augment\n          \n    def __len__(self):\n        return len(self.paths)\n    \n    def __getitem__(self, index):\n        scan_id = self.paths[index]\n        if self.targets is None:\n            data = load_dicom_images_3d(str(scan_id).zfill(5), mri_type=self.mri_type[index], \n                                        split=self.split, threshold=THRESHOLD)\n        else:\n            offset = 0\n            if self.augment:\n                # offset = np.random.randint(-10,10)\n                rotation = np.random.randint(0,4)\n            else:\n                rotation = 0\n            data = load_dicom_images_3d(str(scan_id).zfill(5), mri_type=self.mri_type[index], \n                                        split=\"train\", offset=offset, rotate=rotation, threshold=THRESHOLD)\n            #if self.augment:\n            #    if np.random.random() > 0.5:\n            #        data = np.transpose(data, [0, 2,1,3])\n\n        if self.targets is None:\n            return {\"X\": torch.tensor(data).float(), \"id\": scan_id}\n        else:\n            y = torch.tensor(abs(self.targets[index]-self.label_smoothing), dtype=torch.float)\n            return {\"X\": torch.tensor(data).float(), \"y\": y}\n\n\n","metadata":{"papermill":{"duration":0.026991,"end_time":"2021-08-08T19:41:54.776291","exception":false,"start_time":"2021-08-08T19:41:54.7493","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-14T17:29:02.708937Z","iopub.execute_input":"2021-08-14T17:29:02.709578Z","iopub.status.idle":"2021-08-14T17:29:02.720863Z","shell.execute_reply.started":"2021-08-14T17:29:02.709539Z","shell.execute_reply":"2021-08-14T17:29:02.720122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model class","metadata":{}},{"cell_type":"code","source":"class Model(nn.Module):\n    \n    def __init__(self, num_classes=1, num_channels=1):\n        super(Model, self).__init__()\n        \n        self.conv_layers = nn.ModuleList([self._conv_layer_set(num_channels, 8)])\n        d1 = int((SIZE-2)/2)\n        d2 = int((NUM_IMAGES-2)/2)\n        #print(d1,d1,d2)\n        self.conv_layers.append(self._conv_layer_set(8, 16))\n        d1 = int((d1-2)/2)\n        d2 = int((d2-2)/2)\n        #print(d1,d1,d2)\n        self.conv_layers.append(self._conv_layer_set(16, 32))\n        d1 = int((d1-2)/2)\n        d2 = int((d2-2)/2)\n        #print(d1,d1,d2)\n        #self.conv_layer4 = self._conv_layer_set(32, 16)\n        #print(d1,d1,d2)\n        self.fc1 = nn.Linear(32*d1*d1*d2, 128)\n        #self.fc1 = nn.Linear(16*14*14*2, 128)\n        self.fc_final = nn.Linear(128, num_classes)\n        self.activation1 = nn.LeakyReLU()\n        self.batchnorm1 = nn.BatchNorm1d(128)\n        #self.drop1 = nn.Dropout(p=0.15)        \n        \n    def _conv_layer_set(self, in_c, out_c):\n        conv_layer = nn.Sequential(\n        nn.Conv3d(in_c, out_c, kernel_size=(3, 3, 3), padding=0),\n        nn.LeakyReLU(),\n        nn.MaxPool3d((2, 2, 2)),\n        )\n        return conv_layer\n    \n\n    def forward(self, x):\n        #print(x.shape)\n        out = self.conv_layers[0](x)\n        #print(out.shape)\n        for l in self.conv_layers[1:]:\n            out = l(out)\n            #print(out.shape)\n        out = out.view(out.size(0), -1)\n        #print(out.shape)\n        out = self.fc1(out)\n        #print(out.shape)\n        out = self.activation1(out)\n        #print(out.shape)\n        out = self.batchnorm1(out)\n        #out = self.drop1(out)\n        out = self.fc_final(out)\n        #print(out.shape)\n        \n        return out","metadata":{"papermill":{"duration":0.027463,"end_time":"2021-08-08T19:41:54.818899","exception":false,"start_time":"2021-08-08T19:41:54.791436","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-14T17:29:02.724246Z","iopub.execute_input":"2021-08-14T17:29:02.724545Z","iopub.status.idle":"2021-08-14T17:29:02.738744Z","shell.execute_reply.started":"2021-08-14T17:29:02.724521Z","shell.execute_reply":"2021-08-14T17:29:02.737874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nuseCV = True\n\ntrain_df = pd.read_csv(f\"{data_directory}/train_labels.csv\")\nprint(train_df.shape)\ntrain_df = train_df[~train_df[\"BraTS21ID\"].isin([109,123,709])]\nprint(train_df.shape)\n\ndisplay(train_df)\n\nmodelfiles = ['FLAIR_fold1-e9-loss0.673-auc0.593.pth',\n                 'T1w_fold1-e16-loss0.685-auc0.612.pth',\n                 'T1wCE_fold1-e8-loss0.689-auc0.493.pth',\n                 'T2w_fold1-e7-loss0.692-auc0.522.pth',\n                 'FLAIR_fold2-e17-loss0.667-auc0.641.pth',\n                 'T1w_fold2-e4-loss0.670-auc0.635.pth',\n                 'T1wCE_fold2-e3-loss0.678-auc0.596.pth',\n                 'T2w_fold2-e15-loss0.658-auc0.655.pth',\n                 'FLAIR_fold3-e2-loss0.659-auc0.676.pth',\n                 'T1w_fold3-e20-loss0.669-auc0.632.pth',\n                 'T1wCE_fold3-e8-loss0.654-auc0.642.pth',\n                 'T2w_fold3-e5-loss0.655-auc0.658.pth',\n                 'FLAIR_fold4-e7-loss0.694-auc0.522.pth',\n                 'T1w_fold4-e10-loss0.688-auc0.555.pth',\n                 'T1wCE_fold4-e16-loss0.686-auc0.564.pth',\n                 'T2w_fold4-e16-loss0.686-auc0.562.pth',\n                 'FLAIR_fold5-e6-loss0.685-auc0.585.pth',\n                 'T1w_fold5-e15-loss0.660-auc0.646.pth',\n                 'T1wCE_fold5-e16-loss0.673-auc0.611.pth',\n                 'T2w_fold5-e4-loss0.708-auc0.453.pth']\n\n","metadata":{"lines_to_end_of_cell_marker":2,"lines_to_next_cell":2,"papermill":{"duration":10673.834499,"end_time":"2021-08-08T22:39:48.804194","exception":false,"start_time":"2021-08-08T19:41:54.969695","status":"completed"},"scrolled":true,"tags":[],"execution":{"iopub.status.busy":"2021-08-14T17:29:02.73983Z","iopub.execute_input":"2021-08-14T17:29:02.740246Z","iopub.status.idle":"2021-08-14T17:29:02.765932Z","shell.execute_reply.started":"2021-08-14T17:29:02.740206Z","shell.execute_reply":"2021-08-14T17:29:02.764991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Predict function","metadata":{"papermill":{"duration":2.400041,"end_time":"2021-08-08T22:39:53.646738","exception":false,"start_time":"2021-08-08T22:39:51.246697","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def predict(modelfile, df, mri_type, split):\n    print(\"Predict:\", modelfile, mri_type, df.shape)\n    df.loc[:,\"MRI_Type\"] = mri_type\n    data_retriever = Dataset(\n        df.index.values, \n        mri_type=df[\"MRI_Type\"].values,\n        split=split\n    )\n\n    data_loader = torch_data.DataLoader(\n        data_retriever,\n        batch_size=4,\n        shuffle=False,\n        num_workers=4, pin_memory = True\n    )\n   \n    model = Model()\n    model.to(device)\n    \n    checkpoint = torch.load(f\"{modelpath}/{modelfile}\")\n    model.load_state_dict(checkpoint[\"model_state_dict\"])\n    model.eval()\n    \n    y_pred = []\n    ids = []\n\n    for e, batch in enumerate(data_loader,1):\n        print(f\"{e}/{len(data_loader)}\", end=\"\\r\")\n        with torch.no_grad():\n            tmp_pred = torch.sigmoid(model(batch[\"X\"].to(device))).cpu().numpy().squeeze()\n            if tmp_pred.size == 1:\n                y_pred.append(tmp_pred)\n            else:\n                y_pred.extend(tmp_pred.tolist())\n            ids.extend(batch[\"id\"].numpy().tolist())\n            \n    preddf = pd.DataFrame({\"BraTS21ID\": ids, \"MGMT_value\": y_pred}) \n    preddf = preddf.set_index(\"BraTS21ID\")\n    return preddf","metadata":{"papermill":{"duration":2.447248,"end_time":"2021-08-08T22:39:58.706438","exception":false,"start_time":"2021-08-08T22:39:56.25919","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-14T17:29:02.767211Z","iopub.execute_input":"2021-08-14T17:29:02.767555Z","iopub.status.idle":"2021-08-14T17:29:02.777263Z","shell.execute_reply.started":"2021-08-14T17:29:02.76752Z","shell.execute_reply":"2021-08-14T17:29:02.77613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Ensemble for validation","metadata":{"papermill":{"duration":2.388853,"end_time":"2021-08-08T22:40:03.511843","exception":false,"start_time":"2021-08-08T22:40:01.12299","status":"completed"},"tags":[]}},{"cell_type":"code","source":"if useCV:\n    skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=1234)\n    splits = {}\n    fold = 0\n    for trn_index, val_index in skf.split(train_df, train_df[\"MGMT_value\"]):\n        fold+=1\n        trn_idx = trn_index\n        val_idx = val_index\n        splits[fold] = (trn_idx, val_idx)\n    print(splits.keys())\n    train_df[\"MGMT_pred\"] = 0\n    for m in modelfiles:\n        mtype = m.split(\"_\")[0].split(\"/\")[-1]\n        fold = int(m.split(\"-\")[0][-1])\n        val_df0 = train_df.iloc[splits[fold][1]]\n        val_df = val_df0.set_index(\"BraTS21ID\")\n        #print(m, mtype, fold, val_df.shape)\n        pred = predict(m, val_df, mtype, \"train\")\n        tmp = train_df.loc[val_df0.index,\"MGMT_pred\"] + pred[\"MGMT_value\"].values\n        train_df.loc[val_df0.index,\"MGMT_pred\"] = tmp\n    train_df[\"MGMT_pred\"] /= 4\n    auc = roc_auc_score(train_df[\"MGMT_value\"], train_df[\"MGMT_pred\"])\n    loss = log_loss(train_df[\"MGMT_value\"], train_df[\"MGMT_pred\"])\nelse:\n    df_valid = df_valid.set_index(\"BraTS21ID\")\n    df_valid[\"MGMT_pred\"] = 0\n    for m, mtype in zip(modelfiles,  mri_types):\n        pred = predict(m, df_valid, mtype, \"train\")\n        df_valid[\"MGMT_pred\"] += pred[\"MGMT_value\"]\n    df_valid[\"MGMT_pred\"] /= len(modelfiles)\n    auc = roc_auc_score(df_valid[\"MGMT_value\"], df_valid[\"MGMT_pred\"])\n    loss = log_loss(df_valid[\"MGMT_value\"], df_valid[\"MGMT_pred\"])\nprint(f\"Validation ensemble loss {loss:.4f}, AUC: {auc:.4f}\")","metadata":{"papermill":{"duration":132.038359,"end_time":"2021-08-08T22:42:18.258835","exception":false,"start_time":"2021-08-08T22:40:06.220476","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-14T17:29:02.77882Z","iopub.execute_input":"2021-08-14T17:29:02.7792Z","iopub.status.idle":"2021-08-14T17:29:23.344311Z","shell.execute_reply.started":"2021-08-14T17:29:02.779143Z","shell.execute_reply":"2021-08-14T17:29:23.341517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if useCV:\n    sns.displot(train_df[\"MGMT_pred\"])\n    plt.title(f'lut_contrast={USE_LUT_CONTRAST}, voi_lut={USE_VOI_LUT}, 5 fold CV, val AUC {auc:.3f}')\nelse:\n    sns.displot(df_valid[\"MGMT_pred\"])\n    plt.title(f'lut_contrast={USE_LUT_CONTRAST}, voi_lut={USE_VOI_LUT} val AUC {auc:.3f}')","metadata":{"papermill":{"duration":2.813903,"end_time":"2021-08-08T22:42:23.546136","exception":false,"start_time":"2021-08-08T22:42:20.732233","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-14T17:29:23.345329Z","iopub.status.idle":"2021-08-14T17:29:23.345726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Ensemble for submission","metadata":{"papermill":{"duration":2.427829,"end_time":"2021-08-08T22:42:28.435019","exception":false,"start_time":"2021-08-08T22:42:26.00719","status":"completed"},"tags":[]}},{"cell_type":"code","source":"submission = pd.read_csv(f\"{data_directory}/sample_submission.csv\", index_col=\"BraTS21ID\")\n\nsubmission[\"MGMT_value\"] = 0\nfor m in modelfiles:\n    mtype = m.split(\"_\")[0].split(\"/\")[-1]\n    print(m, mtype)\n    pred = predict(m, submission, mtype, split=\"test\")\n    submission[\"MGMT_value\"] += pred[\"MGMT_value\"]\n\nsubmission[\"MGMT_value\"] /= len(modelfiles)\nsubmission[\"MGMT_value\"].to_csv(\"submission.csv\")","metadata":{"papermill":{"duration":104.994465,"end_time":"2021-08-08T22:44:16.138279","exception":false,"start_time":"2021-08-08T22:42:31.143814","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-14T17:29:23.346682Z","iopub.status.idle":"2021-08-14T17:29:23.347235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"papermill":{"duration":2.694003,"end_time":"2021-08-08T22:44:21.722607","exception":false,"start_time":"2021-08-08T22:44:19.028604","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-14T17:29:23.348388Z","iopub.status.idle":"2021-08-14T17:29:23.348956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.displot(submission[\"MGMT_value\"])","metadata":{"papermill":{"duration":2.741607,"end_time":"2021-08-08T22:44:26.94996","exception":false,"start_time":"2021-08-08T22:44:24.208353","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-14T17:29:23.350072Z","iopub.status.idle":"2021-08-14T17:29:23.350659Z"},"trusted":true},"execution_count":null,"outputs":[]}]}