{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Use stacked images (3D) and Efficientnet3D model\n\nUse models with only one MRI type, then ensemble the 4 models \n","metadata":{}},{"cell_type":"code","source":"import os\nimport sys \nimport glob\nimport random\nimport time\n\nimport numpy as np\nimport pandas as pd\nimport pydicom\nimport cv2\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport torch\nfrom torch import nn\nfrom torch.utils import data as torch_data\nfrom sklearn import model_selection as sk_model_selection\nfrom torch.nn import functional as torch_functional\nimport torch.nn.functional as F\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import roc_auc_score\n\nimport imgaug as ia\nimport imgaug.augmenters as iaa","metadata":{"papermill":{"duration":1.048295,"end_time":"2021-07-14T20:26:46.309722","exception":false,"start_time":"2021-07-14T20:26:45.261427","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-24T13:55:57.646776Z","iopub.execute_input":"2022-03-24T13:55:57.647110Z","iopub.status.idle":"2022-03-24T13:56:00.961212Z","shell.execute_reply.started":"2022-03-24T13:55:57.647080Z","shell.execute_reply":"2022-03-24T13:56:00.960385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = 'test'\nseries_names = ['FLAIR','T1w','T1wCE','T2w']\ndirectory = '../input/rsna-miccai-brain-tumor-radiogenomic-classification'\n# This function gets called by get_middle_images function\n#Returns the list of all images present in a particular series(modality) of a given patient\ndef get_series_list(dataset, study_id, series_name):\n\n    series_list = []\n\n    for subdirs, dirs, files in os.walk(directory + '/' + dataset + '/' + study_id + \"/\" + series_name):\n        series_list = os.listdir(directory + '/' + dataset + '/' + study_id + '/' + series_name)\n    return series_list\n\n\ndef get_middle_images(study_id):\n    \n    middle_images = []\n    \n    # Iterate through each of the four series directories and get the files \n    for ser in series_names:\n        series_files = get_series_list(dataset, study_id, ser)\n        series_df = pd.DataFrame(columns = ['image','instance_number'])\n\n        # Get the DICOM InstanceNumber tag to order the images since we can't rely on the filenames to be in order\n        for s in series_files:\n            img = pydicom.dcmread(directory + \"/\" + dataset + \"/\" + study_id + \"/\" + ser + \"/\" + s)\n            series_df.loc[len(series_df.index)] = [s, img[0x0020,0x0013].value]\n            \n            # 0x0020,0x0013 refers to image number, comes from Dicom dictionary (https://imagej.nih.gov/nih-image/download/nih-image_spin-offs/NucMed_Image/DICOM%20Dictionary)\n \n        series_df['instance_number'] = pd.to_numeric(series_df['instance_number'])\n\n        # Sort the image list by InstanceNumber\n        series_df = series_df.sort_values(by=['instance_number'])\n        \n        # Find the image in the middle of the list\n        middle_index = int(series_df.shape[0] / 2)\n        middle_image = series_df.iloc[middle_index]['image']\n\n        middle_images.append(ser + \"/\" + middle_image)\n\n    return middle_images\n\n\n#Given the image orientation, returns the image plane \ndef get_image_plane(loc):\n\n    row_x = round(loc[0])\n    row_y = round(loc[1])\n    row_z = round(loc[2])\n    col_x = round(loc[3])\n    col_y = round(loc[4])\n    col_z = round(loc[5])\n\n    if row_x == 1 and row_y == 0 and col_x == 0 and col_y == 0:\n        return \"Coronal\"\n\n    if row_x == 0 and row_y == 1 and col_x == 0 and col_y == 0:\n        return \"Sagittal\"\n\n    return 'Axial'\n\n#for getting the image plane corresponding to the middle images of all the series of a particular patient\ndef plot_images(images, image_id):\n    result = []\n    for img in images:\n        image = pydicom.dcmread(directory + \"/\" + dataset + \"/\" + image_id + \"/\" + img)\n        # 0x0020,0x0037 in dicom dictionary refers to \"Image Orientation (Patient)\"\n        image_orientation_patient = image[0x0020,0x0037]\n        plane = get_image_plane(image_orientation_patient)\n        result.append(plane)\n        \n    return result\n\ndf_sub = pd.read_csv(\"../input/rsna-miccai-brain-tumor-radiogenomic-classification/sample_submission.csv\")\n# x:05 converts columns to have a fixed width(to include leading zeroes)\ndf_sub['BraTS21ID'] = df_sub['BraTS21ID'].apply(lambda x: f\"{x:05}\")\nimage_plane_mid_images = df_sub['BraTS21ID'].apply(lambda x: plot_images(get_middle_images(x), x))\nprint(image_plane_mid_images.head())\nprint('--------------------------------------')\ndata = pd.DataFrame.from_dict(dict(zip(image_plane_mid_images.index, image_plane_mid_images.values))).T\nprint(data.head())\nprint('--------------------------------------')\n\ndf_sub[['FLAIR', 'T1w', 'T1wCE', 'T2w']] = 0\ndf_sub[['FLAIR', 'T1w', 'T1wCE', 'T2w']] = data\nprint(df_sub.head())","metadata":{"execution":{"iopub.status.busy":"2022-03-24T15:43:08.076438Z","iopub.execute_input":"2022-03-24T15:43:08.076821Z","iopub.status.idle":"2022-03-24T15:54:11.023475Z","shell.execute_reply.started":"2022-03-24T15:43:08.076782Z","shell.execute_reply":"2022-03-24T15:54:11.022631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = 'train'\nseries_names = ['FLAIR','T1w','T1wCE','T2w']\ndirectory = '../input/rsna-miccai-brain-tumor-radiogenomic-classification'\n# This function gets called by get_middle_images function\n#Returns the list of all images present in a particular series(modality) of a given patient\ndef get_series_list(dataset, study_id, series_name):\n\n    series_list = []\n\n    for subdirs, dirs, files in os.walk(directory + '/' + dataset + '/' + study_id + \"/\" + series_name):\n        series_list = os.listdir(directory + '/' + dataset + '/' + study_id + '/' + series_name)\n    return series_list\n\n\ndef get_middle_images(study_id):\n    \n    middle_images = []\n    \n    # Iterate through each of the four series directories and get the files \n    for ser in series_names:\n        series_files = get_series_list(dataset, study_id, ser)\n        series_df = pd.DataFrame(columns = ['image','instance_number'])\n\n        # Get the DICOM InstanceNumber tag to order the images since we can't rely on the filenames to be in order\n        for s in series_files:\n            img = pydicom.dcmread(directory + \"/\" + dataset + \"/\" + study_id + \"/\" + ser + \"/\" + s)\n            series_df.loc[len(series_df.index)] = [s, img[0x0020,0x0013].value]\n            \n            # 0x0020,0x0013 refers to image number, comes from Dicom dictionary (https://imagej.nih.gov/nih-image/download/nih-image_spin-offs/NucMed_Image/DICOM%20Dictionary)\n \n        series_df['instance_number'] = pd.to_numeric(series_df['instance_number'])\n\n        # Sort the image list by InstanceNumber\n        series_df = series_df.sort_values(by=['instance_number'])\n        \n        # Find the image in the middle of the list\n        middle_index = int(series_df.shape[0] / 2)\n        middle_image = series_df.iloc[middle_index]['image']\n\n        middle_images.append(ser + \"/\" + middle_image)\n\n    return middle_images\n\n\n#Given the image orientation, returns the image plane \ndef get_image_plane(loc):\n\n    row_x = round(loc[0])\n    row_y = round(loc[1])\n    row_z = round(loc[2])\n    col_x = round(loc[3])\n    col_y = round(loc[4])\n    col_z = round(loc[5])\n\n    if row_x == 1 and row_y == 0 and col_x == 0 and col_y == 0:\n        return \"Coronal\"\n\n    if row_x == 0 and row_y == 1 and col_x == 0 and col_y == 0:\n        return \"Sagittal\"\n\n    return 'Axial'\n\n#for getting the image plane corresponding to the middle images of all the series of a particular patient\ndef plot_images(images, image_id):\n    result = []\n    for img in images:\n        image = pydicom.dcmread(directory + \"/\" + dataset + \"/\" + image_id + \"/\" + img)\n        # 0x0020,0x0037 in dicom dictionary refers to \"Image Orientation (Patient)\"\n        image_orientation_patient = image[0x0020,0x0037]\n        plane = get_image_plane(image_orientation_patient)\n        result.append(plane)\n        \n    return result\n\ndf_train = pd.read_csv('../input/rsna-miccai-brain-tumor-radiogenomic-classification/train_labels.csv')\ndf_train['BraTS21ID'] = df_train['BraTS21ID'].apply(lambda x: f\"{x:05}\")\nimage_plane_mid_images = df_train['BraTS21ID'].apply(lambda x: plot_images(get_middle_images(x), x))\ndata = pd.DataFrame.from_dict(dict(zip(image_plane_mid_images.index, image_plane_mid_images.values))).T\nprint(data.head())\ndf_train[['FLAIR', 'T1w', 'T1wCE', 'T2w']] = 0\ndf_train[['FLAIR', 'T1w', 'T1wCE', 'T2w']] = data\nprint(df_train.head())\n","metadata":{"execution":{"iopub.status.busy":"2022-03-24T13:56:03.523947Z","iopub.execute_input":"2022-03-24T13:56:03.524271Z","iopub.status.idle":"2022-03-24T15:30:51.236119Z","shell.execute_reply.started":"2022-03-24T13:56:03.524241Z","shell.execute_reply":"2022-03-24T15:30:51.235232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle\n\ndf_train.to_pickle('train')\ndf_sub.to_pickle('sub')\n","metadata":{"execution":{"iopub.status.busy":"2022-03-24T15:59:59.321427Z","iopub.execute_input":"2022-03-24T15:59:59.321796Z","iopub.status.idle":"2022-03-24T15:59:59.327388Z","shell.execute_reply.started":"2022-03-24T15:59:59.321764Z","shell.execute_reply":"2022-03-24T15:59:59.326482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_directory = '../input/rsna-miccai-brain-tumor-radiogenomic-classification'\npytorch3dpath = \"../input/efficientnetpyttorch3d/EfficientNet-PyTorch-3D\"\n    \nmri_types = ['FLAIR','T1w','T1wCE','T2w']\nangle_types = [ 'Sagittal', 'Axial', 'Coronal']\nSIZE = 256\nNUM_IMAGES = 64\n\nsys.path.append(pytorch3dpath)\nfrom efficientnet_pytorch_3d import EfficientNet3D","metadata":{"lines_to_end_of_cell_marker":2,"lines_to_next_cell":2,"papermill":{"duration":0.05565,"end_time":"2021-07-14T20:26:46.486521","exception":false,"start_time":"2021-07-14T20:26:46.430871","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-24T19:28:42.772726Z","iopub.execute_input":"2022-03-24T19:28:42.773098Z","iopub.status.idle":"2022-03-24T19:28:42.778688Z","shell.execute_reply.started":"2022-03-24T19:28:42.773064Z","shell.execute_reply":"2022-03-24T19:28:42.777807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Functions to load images","metadata":{}},{"cell_type":"code","source":"from pydicom.pixel_data_handlers import apply_voi_lut\ndef load_dicom_image(path, img_size=SIZE):\n    dicom = pydicom.read_file(path)\n    #VOI LUT is used to transform raw DICOM data to \"human-friendly\" view\n    #pixel_array returns a numpy.ndarray containing the pixel data\n\n    data = apply_voi_lut(dicom.pixel_array, dicom)\n    # depending on this value, X-ray may look inverted - fix that:\n    if dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n    data = data - np.min(data)\n    if np.max(data) != 0:\n        data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    data = cv2.resize(data, (img_size, img_size))\n    return data\n\ndef load_dicom_images_3d(scan_id, num_imgs=NUM_IMAGES, img_size=SIZE, mri_type=\"FLAIR\", split=\"train\"):\n\n    files = sorted(glob.glob(f\"{data_directory}/{split}/{scan_id}/{mri_type}/*.dcm\"))\n    \n    middle = len(files)//2\n    num_imgs2 = num_imgs//2\n    p1 = max(0, middle - num_imgs2)\n    p2 = min(len(files), middle + num_imgs2)\n    img3d = np.stack([load_dicom_image(f) for f in files[p1:p2]]).T \n    if img3d.shape[-1] < num_imgs:\n        n_zero = np.zeros((img_size, img_size, num_imgs - img3d.shape[-1]))\n        img3d = np.concatenate((img3d,  n_zero), axis = -1)\n            \n    return np.expand_dims(img3d,0)\n\na = load_dicom_images_3d(\"00000\")\na.shape","metadata":{"papermill":{"duration":0.035761,"end_time":"2021-07-14T20:26:46.726756","exception":false,"start_time":"2021-07-14T20:26:46.690995","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-24T19:28:45.225469Z","iopub.execute_input":"2022-03-24T19:28:45.225844Z","iopub.status.idle":"2022-03-24T19:28:45.690058Z","shell.execute_reply.started":"2022-03-24T19:28:45.225804Z","shell.execute_reply":"2022-03-24T19:28:45.689197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed_all(seed)\n        torch.backends.cudnn.deterministic = True\n\nset_seed(42)","metadata":{"papermill":{"duration":0.668331,"end_time":"2021-07-14T20:27:48.114522","exception":false,"start_time":"2021-07-14T20:27:47.446191","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-24T19:28:45.691634Z","iopub.execute_input":"2022-03-24T19:28:45.692156Z","iopub.status.idle":"2022-03-24T19:28:45.698929Z","shell.execute_reply.started":"2022-03-24T19:28:45.692113Z","shell.execute_reply":"2022-03-24T19:28:45.697969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## train / test splits\n","metadata":{}},{"cell_type":"code","source":"# train_df = pd.read_csv(f\"{data_directory}/train_labels.csv\")\ndisplay(df_train)\n\ndf_train, df_valid = sk_model_selection.train_test_split(\n    df_train, \n    test_size=0.2, \n    random_state=10, \n    stratify=df_train[\"MGMT_value\"],\n)","metadata":{"execution":{"iopub.status.busy":"2022-03-24T19:28:46.120897Z","iopub.execute_input":"2022-03-24T19:28:46.121228Z","iopub.status.idle":"2022-03-24T19:28:46.143205Z","shell.execute_reply.started":"2022-03-24T19:28:46.121196Z","shell.execute_reply":"2022-03-24T19:28:46.142270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model and training classes","metadata":{}},{"cell_type":"code","source":"\n#Label smoothing is a regularization technique for classification problems to prevent the \n#model from predicting the labels too confidently during training and generalizing poorly\n\nclass Dataset(torch_data.Dataset):\n    def __init__(self, paths, targets=None, mri_type=None, label_smoothing=0.01, split=\"train\"):\n        self.paths = paths\n        self.targets = targets\n        self.mri_type = mri_type\n        self.label_smoothing = label_smoothing\n        self.split = split\n          \n    def __len__(self):\n        return len(self.paths)\n    \n    def __getitem__(self, index):\n        scan_id = self.paths[index]\n        if self.targets is None:\n            data = load_dicom_images_3d(str(scan_id).zfill(5), mri_type=self.mri_type[index], split=self.split)\n        else:\n            data = load_dicom_images_3d(str(scan_id).zfill(5), mri_type=self.mri_type[index], split=\"train\")\n\n        if self.targets is None:\n            return {\"X\": torch.tensor(data).float(), \"id\": scan_id}\n        else:\n            y = torch.tensor(abs(self.targets[index]-self.label_smoothing), dtype=torch.float)\n            return {\"X\": torch.tensor(data).float(), \"y\": y}\n","metadata":{"papermill":{"duration":0.634322,"end_time":"2021-07-14T20:27:50.594701","exception":false,"start_time":"2021-07-14T20:27:49.960379","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-24T19:28:47.060048Z","iopub.execute_input":"2022-03-24T19:28:47.060471Z","iopub.status.idle":"2022-03-24T19:28:47.068636Z","shell.execute_reply.started":"2022-03-24T19:28:47.060425Z","shell.execute_reply":"2022-03-24T19:28:47.067768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Model(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.net = EfficientNet3D.from_name(\"efficientnet-b0\", override_params={'num_classes': 2}, in_channels=1)\n        n_features = self.net._fc.in_features\n        self.net._fc = nn.Linear(in_features=n_features, out_features=1, bias=True)\n    \n    def forward(self, x):\n        out = self.net(x)\n        return out","metadata":{"papermill":{"duration":0.825458,"end_time":"2021-07-14T20:27:55.604161","exception":false,"start_time":"2021-07-14T20:27:54.778703","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-24T19:28:48.035120Z","iopub.execute_input":"2022-03-24T19:28:48.035442Z","iopub.status.idle":"2022-03-24T19:28:48.042670Z","shell.execute_reply.started":"2022-03-24T19:28:48.035412Z","shell.execute_reply":"2022-03-24T19:28:48.041494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Trainer:\n    def __init__(\n        self, \n        model, \n        device, \n        optimizer, \n        criterion\n    ):\n        self.model = model\n        self.device = device\n        self.optimizer = optimizer\n        self.criterion = criterion\n\n        self.best_valid_score = np.inf\n        self.n_patience = 0\n        self.lastmodel = None\n        \n    def fit(self, epochs, train_loader, valid_loader, save_path, patience):        \n        for n_epoch in range(1, epochs + 1):\n            self.info_message(\"EPOCH: {}\", n_epoch)\n            \n            train_loss, train_time = self.train_epoch(train_loader)\n            valid_loss, valid_auc, valid_time = self.valid_epoch(valid_loader)\n            \n            self.info_message(\n                \"[Epoch Train: {}] loss: {:.4f}, time: {:.2f} s            \",\n                n_epoch, train_loss, train_time\n            )\n            \n            self.info_message(\n                \"[Epoch Valid: {}] loss: {:.4f}, auc: {:.4f}, time: {:.2f} s\",\n                n_epoch, valid_loss, valid_auc, valid_time\n            )\n\n            # if True:\n            # if self.best_valid_score < valid_auc: \n            if self.best_valid_score > valid_loss: \n                self.save_model(n_epoch, save_path, valid_loss, valid_auc)\n                self.info_message(\n                     \"valid loss decreased from {:.4f} to {:.4f}. Saved model to '{}'\", \n                    self.best_valid_score, valid_loss, self.lastmodel\n                )\n                self.best_valid_score = valid_loss\n                self.n_patience = 0\n            else:\n                self.n_patience += 1\n            \n            if self.n_patience >= patience:\n                self.info_message(\"\\nValid loss didn't improve in last {} epochs.\", patience)\n                break\n            \n    def train_epoch(self, train_loader):\n        self.model.train()\n        t = time.time()\n        sum_loss = 0\n\n        for step, batch in enumerate(train_loader, 1):\n            X = batch[\"X\"].to(self.device)\n            targets = batch[\"y\"].to(self.device)\n            self.optimizer.zero_grad()\n            outputs = self.model(X).squeeze(1)\n            \n            loss = self.criterion(outputs, targets)\n            loss.backward()\n\n            sum_loss += loss.detach().item()\n\n            self.optimizer.step()\n            \n            message = 'Train Step {}/{}, train_loss: {:.4f}'\n            self.info_message(message, step, len(train_loader), sum_loss/step, end=\"\\r\")\n        \n        return sum_loss/len(train_loader), int(time.time() - t)\n    \n    def valid_epoch(self, valid_loader):\n        self.model.eval()\n        t = time.time()\n        sum_loss = 0\n        y_all = []\n        outputs_all = []\n\n        for step, batch in enumerate(valid_loader, 1):\n            with torch.no_grad():\n                X = batch[\"X\"].to(self.device)\n                targets = batch[\"y\"].to(self.device)\n\n                outputs = self.model(X).squeeze(1)\n                loss = self.criterion(outputs, targets)\n\n                sum_loss += loss.detach().item()\n                y_all.extend(batch[\"y\"].tolist())\n                outputs_all.extend(outputs.tolist())\n\n            message = 'Valid Step {}/{}, valid_loss: {:.4f}'\n            self.info_message(message, step, len(valid_loader), sum_loss/step, end=\"\\r\")\n            \n        y_all = [1 if x > 0.5 else 0 for x in y_all]\n        auc = roc_auc_score(y_all, outputs_all)\n        \n        return sum_loss/len(valid_loader), auc, int(time.time() - t)\n    \n    def save_model(self, n_epoch, save_path, loss, auc):\n        self.lastmodel = f\"{save_path}-e{n_epoch}-loss{loss:.3f}-auc{auc:.3f}.pth\"\n        torch.save(\n            {\n                \"model_state_dict\": self.model.state_dict(),\n                \"optimizer_state_dict\": self.optimizer.state_dict(),\n                \"best_valid_score\": self.best_valid_score,\n                \"n_epoch\": n_epoch,\n            },\n            self.lastmodel,\n        )\n    \n    @staticmethod\n    def info_message(message, *args, end=\"\\n\"):\n        print(message.format(*args), end=end)","metadata":{"execution":{"iopub.status.busy":"2022-03-24T19:28:49.078182Z","iopub.execute_input":"2022-03-24T19:28:49.078498Z","iopub.status.idle":"2022-03-24T19:28:49.097780Z","shell.execute_reply.started":"2022-03-24T19:28:49.078467Z","shell.execute_reply":"2022-03-24T19:28:49.096590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## train models","metadata":{}},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\ndef train_mri_type(df_train, df_valid, mri_type, angle_type):\n\n    df_train = df_train[df_train[mri_type]==angle_type]\n    df_valid = df_valid[df_valid[mri_type]==angle_type]\n    df_train.loc[:,\"MRI_Type\"] = mri_type\n    df_valid.loc[:,\"MRI_Type\"] = mri_type\n\n    print(df_train.shape, df_valid.shape)\n    display(df_train.head())\n    \n    train_data_retriever = Dataset(\n        df_train[\"BraTS21ID\"].values, \n        df_train[\"MGMT_value\"].values, \n        df_train[\"MRI_Type\"].values,\n    )\n\n    valid_data_retriever = Dataset(\n        df_valid[\"BraTS21ID\"].values, \n        df_valid[\"MGMT_value\"].values,\n        df_valid[\"MRI_Type\"].values\n    )\n\n    train_loader = torch_data.DataLoader(\n        train_data_retriever,\n        batch_size=4,\n        shuffle=True,\n        num_workers=8,\n    )\n\n    valid_loader = torch_data.DataLoader(\n        valid_data_retriever, \n        batch_size=4,\n        shuffle=False,\n        num_workers=8,\n    )\n\n    model = Model()\n    model.to(device)\n\n    optimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n\n    criterion = torch_functional.binary_cross_entropy_with_logits\n\n    trainer = Trainer(\n        model, \n        device, \n        optimizer, \n        criterion\n    )\n\n    history = trainer.fit(\n        50, \n        train_loader, \n        valid_loader, \n        f\"{mri_type}-{angle_type}\", \n        10,\n    )\n    \n    return trainer.lastmodel\n\nmodelfiles = None\n\nif not modelfiles:\n    for m in mri_types:\n        for a in angle_types:\n            modelfiles = train_mri_type(df_train, df_valid, m,a)\nprint(modelfiles)","metadata":{"execution":{"iopub.status.busy":"2022-03-24T21:06:30.940070Z","iopub.execute_input":"2022-03-24T21:06:30.940416Z","iopub.status.idle":"2022-03-24T21:06:30.947641Z","shell.execute_reply.started":"2022-03-24T21:06:30.940378Z","shell.execute_reply":"2022-03-24T21:06:30.946796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Predict function","metadata":{}},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"lines_to_next_cell":2,"papermill":{"duration":447.387602,"end_time":"2021-07-14T20:35:26.110421","exception":false,"start_time":"2021-07-14T20:27:58.722819","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-24T21:08:58.610256Z","iopub.execute_input":"2022-03-24T21:08:58.610677Z","iopub.status.idle":"2022-03-24T21:08:58.616190Z","shell.execute_reply.started":"2022-03-24T21:08:58.610631Z","shell.execute_reply":"2022-03-24T21:08:58.615248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"modeldict = {\n    'FLAIR': {\n        'Axial': './FLAIR-Axial-e5-loss0.684-auc0.574.pth',\n        'Coronal': '../input/effnets3d/FLAIR-Coronal-e6-loss0.624-auc0.556.pth',\n        'Sagittal': '../input/effnets3d/FLAIR-Sagittal-e1-loss0.694-auc0.425.pth',\n    },\n    \n    'T1w': {\n        'Axial': '../input/effnets3d/T1w-Axial-e16-loss0.680-auc0.595.pth',\n        'Coronal': '../input/effnets3d/T1w-Coronal-e1-loss0.693-auc0.500.pth',\n        'Sagittal': '../input/effnets3d/T1w-Sagittal-e1-loss0.696-auc0.500.pth',\n    },\n    \n    'T1wCE': {\n        'Axial': '../input/effnets3d/T1wCE-Axial-e3-loss0.693-auc0.623.pth',\n        'Coronal': '../input/effnets3d/T1wCE-Coronal-e15-loss0.680-auc0.583.pth',\n        'Sagittal': '../input/effnets3d/T1wCE-Sagittal-e13-loss0.671-auc0.375.pth',\n    },\n    \n    'T2w': {\n        'Axial': '../input/effnets3d/T2w-Axial-e10-loss0.686-auc0.604.pth',\n        'Coronal': '../input/effnets3d/T2w-Coronal-e24-loss0.663-auc0.500.pth',\n        'Sagittal': '../input/effnets3d/T2w-Sagittal-e1-loss0.693-auc0.475.pth',\n    }\n}","metadata":{"execution":{"iopub.status.busy":"2022-03-24T21:08:54.708715Z","iopub.execute_input":"2022-03-24T21:08:54.709075Z","iopub.status.idle":"2022-03-24T21:08:54.714249Z","shell.execute_reply.started":"2022-03-24T21:08:54.709043Z","shell.execute_reply":"2022-03-24T21:08:54.713274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict(modelfile, df, mri_type, angle_type, split):\n    print(\"Predict:\", modelfile, mri_type, angle_type, df.shape)\n    \n    df = df[df[mri_type]==angle_type]\n    df.loc[:,\"MRI_Type\"] = mri_type\n    data_retriever = Dataset(\n        df.BraTS21ID.values, \n        mri_type=df[\"MRI_Type\"].values,\n        split=split\n    )\n\n    data_loader = torch_data.DataLoader(\n        data_retriever,\n        batch_size=4,\n        shuffle=False,\n        num_workers=2,\n    )\n   \n    model = Model()\n    model.to(device)\n    \n    checkpoint = torch.load(modelfile, map_location=device)\n    model.load_state_dict(checkpoint[\"model_state_dict\"])\n    model.eval()\n    \n    y_pred = []\n    ids = []\n\n    for e, batch in enumerate(data_loader,1):\n        print(f\"{e}/{len(data_loader)}\", end=\"\\r\")\n        with torch.no_grad():\n            tmp_pred = torch.sigmoid(model(batch[\"X\"].to(device))).cpu().numpy().squeeze()\n            if tmp_pred.size == 1:\n                y_pred.append(tmp_pred)\n            else:\n                y_pred.extend(tmp_pred.tolist())\n            ids.extend(batch[\"id\"])\n            \n    preddf = pd.DataFrame({\"BraTS21ID\": ids, \"MGMT_value\": y_pred}) \n    preddf = preddf.set_index(\"BraTS21ID\")\n    return preddf","metadata":{"execution":{"iopub.status.busy":"2022-03-24T21:09:09.509068Z","iopub.execute_input":"2022-03-24T21:09:09.509456Z","iopub.status.idle":"2022-03-24T21:09:09.519750Z","shell.execute_reply.started":"2022-03-24T21:09:09.509382Z","shell.execute_reply":"2022-03-24T21:09:09.518817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Ensemble for submission","metadata":{}},{"cell_type":"code","source":"submission = df_sub.copy()\nsubmission[\"MGMT_value\"] = 0\nfor mtype in mri_types:\n    for atype in angle_types:\n        m = modeldict[mtype][atype]\n        try:\n            pred = predict(m, submission, mtype, atype, split=\"test\")\n        except ValueError:\n            continue\n        submission = pd.merge(submission, pred, how='left', on='BraTS21ID').fillna(0)\n        submission['MGMT_value'] = submission['MGMT_value_x'] + submission['MGMT_value_y']\n        submission = submission[['BraTS21ID', 'MGMT_value', 'FLAIR', 'T1w', 'T1wCE', 'T2w']]\n\nsubmission = submission[['BraTS21ID', 'MGMT_value']]\nsubmission[\"MGMT_value\"] /= len(mri_types)","metadata":{"papermill":{"duration":0.990911,"end_time":"2021-07-14T20:35:30.482254","exception":false,"start_time":"2021-07-14T20:35:29.491343","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-24T21:09:11.965174Z","iopub.execute_input":"2022-03-24T21:09:11.965520Z","iopub.status.idle":"2022-03-24T21:09:47.231949Z","shell.execute_reply.started":"2022-03-24T21:09:11.965487Z","shell.execute_reply":"2022-03-24T21:09:47.230935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<!-- submission.to_csv(\"submission.csv\", index=False) -->","metadata":{}},{"cell_type":"code","source":"submission.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-03-24T21:09:47.233714Z","iopub.execute_input":"2022-03-24T21:09:47.234056Z","iopub.status.idle":"2022-03-24T21:09:47.244755Z","shell.execute_reply.started":"2022-03-24T21:09:47.234017Z","shell.execute_reply":"2022-03-24T21:09:47.243872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv('./submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-03-24T21:12:11.879836Z","iopub.execute_input":"2022-03-24T21:12:11.880167Z","iopub.status.idle":"2022-03-24T21:12:11.889599Z","shell.execute_reply.started":"2022-03-24T21:12:11.880137Z","shell.execute_reply":"2022-03-24T21:12:11.888727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2022-03-24T21:12:17.113986Z","iopub.execute_input":"2022-03-24T21:12:17.114310Z","iopub.status.idle":"2022-03-24T21:12:17.129624Z","shell.execute_reply.started":"2022-03-24T21:12:17.114279Z","shell.execute_reply":"2022-03-24T21:12:17.128707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}