{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# !pip install torchio","metadata":{"execution":{"iopub.status.busy":"2021-08-05T12:39:58.811337Z","iopub.execute_input":"2021-08-05T12:39:58.811737Z","iopub.status.idle":"2021-08-05T12:39:58.815991Z","shell.execute_reply.started":"2021-08-05T12:39:58.811702Z","shell.execute_reply":"2021-08-05T12:39:58.814928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport sys \nimport json\nimport glob\nimport random\nimport collections\nimport time\n\nimport numpy as np\nimport pandas as pd\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nimport cv2\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport torch\nfrom torch import nn\nfrom torch.utils import data as torch_data\nfrom sklearn import model_selection as sk_model_selection\nfrom torch.nn import functional as torch_functional\nimport torch.nn.functional as F\nfrom torchvision import transforms, utils\n\nfrom sklearn import model_selection\nfrom sklearn import metrics\nfrom skimage import exposure\n\nfrom albumentations import Resize, Normalize, Compose\nfrom albumentations.pytorch import ToTensorV2\nimport albumentations as album\n# import torchio as tio\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nplt.style.use(\"dark_background\")\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import roc_auc_score","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-08-05T12:39:58.81792Z","iopub.execute_input":"2021-08-05T12:39:58.818369Z","iopub.status.idle":"2021-08-05T12:39:58.832068Z","shell.execute_reply.started":"2021-08-05T12:39:58.818321Z","shell.execute_reply":"2021-08-05T12:39:58.83088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(f\"Using {device} device\")","metadata":{"execution":{"iopub.status.busy":"2021-08-05T12:39:58.834579Z","iopub.execute_input":"2021-08-05T12:39:58.835059Z","iopub.status.idle":"2021-08-05T12:39:58.851088Z","shell.execute_reply.started":"2021-08-05T12:39:58.835007Z","shell.execute_reply":"2021-08-05T12:39:58.849966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pip install --upgrade batchgenerators","metadata":{"execution":{"iopub.status.busy":"2021-08-05T12:39:58.853081Z","iopub.execute_input":"2021-08-05T12:39:58.853441Z","iopub.status.idle":"2021-08-05T12:39:58.865027Z","shell.execute_reply.started":"2021-08-05T12:39:58.85338Z","shell.execute_reply":"2021-08-05T12:39:58.863704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_data_retriever = Dataset(df_train[\"BraTS21ID\"].values, df_train[\"MGMT_value\"].values)\n\n# train_loader = torch_data.DataLoader(train_data_retriever, batch_size=4, shuffle=True, num_workers=1,)\n# batch = next(iter(train_loader))","metadata":{"execution":{"iopub.status.busy":"2021-08-05T12:39:58.866616Z","iopub.execute_input":"2021-08-05T12:39:58.867018Z","iopub.status.idle":"2021-08-05T12:39:58.879272Z","shell.execute_reply.started":"2021-08-05T12:39:58.866985Z","shell.execute_reply":"2021-08-05T12:39:58.87812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def plot_batch(batch):\n#     batch_size = batch['X'].shape[0]\n#     plt.figure(figsize=(16, 10))\n#     for i in range(batch_size):\n#         plt.subplot(1, batch_size, i+1)\n#         plt.imshow(batch['X'][i, 0, 0, :, :], cmap=\"gray\") # only grayscale image here\n#     plt.show()\n\n# plot_batch(batch)","metadata":{"execution":{"iopub.status.busy":"2021-08-05T12:39:58.881438Z","iopub.execute_input":"2021-08-05T12:39:58.88178Z","iopub.status.idle":"2021-08-05T12:39:58.891476Z","shell.execute_reply.started":"2021-08-05T12:39:58.881747Z","shell.execute_reply":"2021-08-05T12:39:58.890411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# array = batch['X'][0, 0, :, :, :].numpy().astype(np.float32)","metadata":{"execution":{"iopub.status.busy":"2021-08-05T12:39:58.89278Z","iopub.execute_input":"2021-08-05T12:39:58.893487Z","iopub.status.idle":"2021-08-05T12:39:58.903678Z","shell.execute_reply.started":"2021-08-05T12:39:58.893445Z","shell.execute_reply":"2021-08-05T12:39:58.90255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from batchgenerators.transforms.color_transforms import ContrastAugmentationTransform\n# from batchgenerators.transforms.spatial_transforms import MirrorTransform\n# from batchgenerators.transforms.abstract_transforms import Compose\n# from batchgenerators.dataloading.multi_threaded_augmenter import MultiThreadedAugmenter\n\n# my_transforms = []\n# brightness_transform = ContrastAugmentationTransform((0.3, 3.), preserve_range=True)\n# my_transforms.append(brightness_transform)\n# mirror_transform = MirrorTransform(axes=(0, 1))\n# my_transforms.append(mirror_transform)\n\n# all_transforms = Compose(my_transforms)\n\n# multithreaded_generator = MultiThreadedAugmenter(array, all_transforms, 4, 1, seeds=None)\n# plot_batch(next(iter(multithreaded_generator)))\n# multithreaded_generator._finish()\n","metadata":{"execution":{"iopub.status.busy":"2021-08-05T12:39:58.905197Z","iopub.execute_input":"2021-08-05T12:39:58.905649Z","iopub.status.idle":"2021-08-05T12:39:58.916962Z","shell.execute_reply.started":"2021-08-05T12:39:58.905608Z","shell.execute_reply":"2021-08-05T12:39:58.915621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if os.path.exists(\"../input/rsna-miccai-brain-tumor-radiogenomic-classification\"):\n    data_directory = '../input/rsna-miccai-brain-tumor-radiogenomic-classification'\n    pytorch3dpath = \"../input/efficientnetpyttorch3d/EfficientNet-PyTorch-3D\"\nelse:\n    data_directory = '/media/roland/data/kaggle/rsna-miccai-brain-tumor-radiogenomic-classification'\n    pytorch3dpath = \"EfficientNet-PyTorch-3D\"\n    \nmri_types = ['FLAIR','T1w','T1wCE','T2w']\nSIZE = 128\nNUM_IMAGES = 64\n\nsys.path.append(pytorch3dpath)\nfrom efficientnet_pytorch_3d import EfficientNet3D","metadata":{"execution":{"iopub.status.busy":"2021-08-05T12:39:58.919072Z","iopub.execute_input":"2021-08-05T12:39:58.919696Z","iopub.status.idle":"2021-08-05T12:39:58.935211Z","shell.execute_reply.started":"2021-08-05T12:39:58.91965Z","shell.execute_reply":"2021-08-05T12:39:58.934194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_dicom_image(path, img_size=SIZE):\n    dicom = pydicom.read_file(path)\n    data = dicom.pixel_array\n    \n    if np.min(data)==np.max(data):\n        data = np.zeros((img_size,img_size))\n        return data\n    \n#     data = exposure.equalize_adapthist(data, clip_limit=0.08)\n    data = apply_voi_lut(dicom.pixel_array, dicom)\n    if dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n        \n    data = cv2.resize(data, (img_size, img_size))\n    \n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n\n    return data.astype(np.uint8)","metadata":{"execution":{"iopub.status.busy":"2021-08-05T12:39:58.936688Z","iopub.execute_input":"2021-08-05T12:39:58.937349Z","iopub.status.idle":"2021-08-05T12:39:58.946047Z","shell.execute_reply.started":"2021-08-05T12:39:58.937314Z","shell.execute_reply":"2021-08-05T12:39:58.944877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_dicom_images_3d(scan_id, num_imgs=NUM_IMAGES, img_size=SIZE, mri_type=\"FLAIR\", split=\"train\"):\n    files = sorted(glob.glob(f\"{data_directory}/{split}/{scan_id}/{mri_type}/*.dcm\"))\n    \n    middle = len(files) // 2\n    num_imgs2 = num_imgs // 2\n    p1 = max(0, middle - num_imgs2)\n    p2 = min(len(files), middle + num_imgs2)\n    img3d = np.stack([load_dicom_image(f) for f in files[p1:p2]]).T \n    \n    if img3d.shape[-1] < num_imgs:\n        n_zero = np.zeros((img_size, img_size, num_imgs - img3d.shape[-1]))\n        img3d = np.concatenate((img3d,  n_zero), axis = -1)\n            \n    return img3d","metadata":{"execution":{"iopub.status.busy":"2021-08-05T12:39:58.947341Z","iopub.execute_input":"2021-08-05T12:39:58.947929Z","iopub.status.idle":"2021-08-05T12:39:58.958837Z","shell.execute_reply.started":"2021-08-05T12:39:58.947893Z","shell.execute_reply":"2021-08-05T12:39:58.957694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images = []\nfor i in mri_types:\n    images.append(load_dicom_images_3d(scan_id=\"00000\", mri_type=i))\nfour_channel_pack = np.stack(images)\nfour_channel_pack = np.transpose(four_channel_pack, (0, 3, 1, 2))\n\nprint(four_channel_pack.shape)\nplt.imshow(four_channel_pack[0, 20, :, :])","metadata":{"execution":{"iopub.status.busy":"2021-08-05T12:39:58.960361Z","iopub.execute_input":"2021-08-05T12:39:58.960978Z","iopub.status.idle":"2021-08-05T12:40:00.79911Z","shell.execute_reply.started":"2021-08-05T12:39:58.960923Z","shell.execute_reply":"2021-08-05T12:40:00.798249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed_all(seed)\n        torch.backends.cudnn.deterministic = True\n\nset_seed(42)","metadata":{"execution":{"iopub.status.busy":"2021-08-05T12:40:00.801163Z","iopub.execute_input":"2021-08-05T12:40:00.801651Z","iopub.status.idle":"2021-08-05T12:40:00.808928Z","shell.execute_reply.started":"2021-08-05T12:40:00.801615Z","shell.execute_reply":"2021-08-05T12:40:00.808003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(f\"{data_directory}/train_labels.csv\")\ndisplay(train_df)\n\ndf_train, df_valid = sk_model_selection.train_test_split(\n    train_df, \n    test_size=0.1, \n    random_state=42, \n    stratify=train_df[\"MGMT_value\"],\n)","metadata":{"execution":{"iopub.status.busy":"2021-08-05T12:40:00.810674Z","iopub.execute_input":"2021-08-05T12:40:00.811148Z","iopub.status.idle":"2021-08-05T12:40:00.840762Z","shell.execute_reply.started":"2021-08-05T12:40:00.811114Z","shell.execute_reply":"2021-08-05T12:40:00.839671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3D Augmenatation and Transformation","metadata":{}},{"cell_type":"code","source":"def get_training_augmentation():\n    train_transform = [\n        album.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),\n    ]\n    return album.Compose(train_transform,)","metadata":{"execution":{"iopub.status.busy":"2021-08-05T12:40:00.841919Z","iopub.execute_input":"2021-08-05T12:40:00.842377Z","iopub.status.idle":"2021-08-05T12:40:00.847982Z","shell.execute_reply.started":"2021-08-05T12:40:00.842343Z","shell.execute_reply":"2021-08-05T12:40:00.846718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# _transforms = {\n# #     tio.RandomFlip(axes=['LR', 'AP', 'IS'], p=0.8),\n#     tio.RandomElasticDeformation(p=0.5),\n#     tio.ZNormalization(masking_method=tio.ZNormalization.mean, p=1),\n# #     tio.RandomAffine(degrees=20, p=0.5),\n    \n# #     tio.OneOf([\n# # #         tio.RandomMotion(p=0.2),\n# #         tio.RandomBiasField(p=0.3),\n# # #         tio.RandomNoise(p=0.5),\n# #     ]),\n    \n# #     tio.OneOf([\n# #         tio.ZNormalization(masking_method=tio.ZNormalization.mean, p=0.5),\n# #         tio.RescaleIntensity(out_min_max=(0, 1), p=0.5),  \n# #     ])\n# }\n\n# outer_transforms = _transforms\n# transform = tio.Compose(outer_transforms)","metadata":{"execution":{"iopub.status.busy":"2021-08-05T12:40:00.849932Z","iopub.execute_input":"2021-08-05T12:40:00.850358Z","iopub.status.idle":"2021-08-05T12:40:00.860139Z","shell.execute_reply.started":"2021-08-05T12:40:00.850321Z","shell.execute_reply":"2021-08-05T12:40:00.858878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Dataset and Dataloader","metadata":{}},{"cell_type":"code","source":"class Dataset(torch_data.Dataset):\n    def __init__(self, paths, labels=None, mri_type=None, label_smoothing=0.01, augmentation=None, transformation=None, split=\"train\"):\n        self.paths = paths\n        self.labels = labels\n        self.mri_type = mri_type\n        self.label_smoothing = label_smoothing\n        self.augmentation = augmentation\n        self.transformation = transformation\n        self.split = split\n          \n    def __len__(self):\n        return len(self.paths)\n    \n    def __getitem__(self, index):\n        scan_id = self.paths[index]\n        label = self.labels[index]\n        \n#         inner_transform = transforms.Compose([\n#             transforms.ToTensor(), \n# #             transforms.Normalize((0.5,) * NUM_IMAGES, (0.5,) * NUM_IMAGES)\n#         ])\n    \n\n        images = []\n        for i in mri_types:\n            image_3d = load_dicom_images_3d(scan_id=str(scan_id).zfill(5), mri_type=i)\n            \n            # normalization\n#             if self.augmentation:\n#                 for i in range(image_3d.shape[-1]):\n#                     temp_img = image_3d[:, :, i].astype(np.uint8)\n#                     temp_img = cv2.cvtColor(temp_img, cv2.COLOR_BGR2RGB)\n#                     temp_img = self.augmentation(image=temp_img)['image'][:, :, 0]\n#                     image_3d[:, :, i] = temp_img\n            images.append(image_3d)\n        four_channel_pack = np.stack(images)\n        \n        # transformation\n        if self.transformation:\n            four_channel_pack = self.transformation(four_channel_pack)\n        \n        four_channel_pack = np.transpose(four_channel_pack, (0, 3, 2, 1))\n        y = self.labels[index]\n        \n        return {\"X\": torch.tensor(four_channel_pack).float(), \"y\": y}","metadata":{"execution":{"iopub.status.busy":"2021-08-05T12:40:00.861987Z","iopub.execute_input":"2021-08-05T12:40:00.862334Z","iopub.status.idle":"2021-08-05T12:40:00.876143Z","shell.execute_reply.started":"2021-08-05T12:40:00.862305Z","shell.execute_reply":"2021-08-05T12:40:00.874847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Test Transformations and Normalization","metadata":{}},{"cell_type":"code","source":"train_data_retriever = Dataset(\n    paths=df_train[\"BraTS21ID\"].values, \n    labels=df_train[\"MGMT_value\"].values,\n    augmentation=get_training_augmentation(),\n#     transformation=transform,\n)\n\ntrain_loader = torch_data.DataLoader(train_data_retriever, batch_size=4, shuffle=False, num_workers=8,)\na = train_data_retriever[5][\"X\"]","metadata":{"execution":{"iopub.status.busy":"2021-08-05T12:40:00.877892Z","iopub.execute_input":"2021-08-05T12:40:00.878258Z","iopub.status.idle":"2021-08-05T12:40:02.105245Z","shell.execute_reply.started":"2021-08-05T12:40:00.87822Z","shell.execute_reply":"2021-08-05T12:40:02.104104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(a[0, 16, :, :])\nprint(a[0, 14, :, :].max(), a[0, 14, :, :].min())","metadata":{"execution":{"iopub.status.busy":"2021-08-05T12:40:02.106748Z","iopub.execute_input":"2021-08-05T12:40:02.10715Z","iopub.status.idle":"2021-08-05T12:40:02.327881Z","shell.execute_reply.started":"2021-08-05T12:40:02.107106Z","shell.execute_reply":"2021-08-05T12:40:02.326535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model and Training","metadata":{}},{"cell_type":"code","source":"class Model(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.net = EfficientNet3D.from_name(\"efficientnet-b0\", override_params={'num_classes': 2}, in_channels=4)\n        n_features = self.net._fc.in_features\n#         self.net._fc = nn.Linear(in_features=n_features, out_features=2, bias=True)\n\n        self.net._fc = nn.Sequential(nn.Linear(n_features, 128),\n                                        nn.SELU(),\n                                        nn.Dropout(p=0.5),\n                                        nn.Linear(128, 64),\n                                        nn.SELU(),\n                                        nn.Dropout(p=0.4),\n                                        nn.Linear(64, 2),\n                                        nn.LogSigmoid()\n                                )\n    \n    def forward(self, x):\n        out = self.net(x)\n        return out","metadata":{"execution":{"iopub.status.busy":"2021-08-05T12:40:02.329682Z","iopub.execute_input":"2021-08-05T12:40:02.330246Z","iopub.status.idle":"2021-08-05T12:40:02.341653Z","shell.execute_reply.started":"2021-08-05T12:40:02.330187Z","shell.execute_reply":"2021-08-05T12:40:02.340292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Trainer:\n    def __init__(\n        self, \n        model, \n        device, \n        optimizer, \n        criterion\n    ):\n        self.model = model\n        self.device = device\n        self.optimizer = optimizer\n        self.criterion = criterion\n\n        self.best_valid_score = np.inf\n        self.n_patience = 0\n        self.lastmodel = None\n        \n    def fit(self, epochs, train_loader, valid_loader, save_path, patience):        \n        for n_epoch in range(1, epochs + 1):\n            self.info_message(\"EPOCH: {}\", n_epoch)\n            \n            train_acc, train_loss, train_time = self.train_epoch(train_loader)\n            valid_acc, valid_loss, valid_time = self.valid_epoch(valid_loader)\n            \n            self.info_message(\n                \"[Epoch Train: {}] accuracy: {:.4f}, loss: {:.4f}, time: {:.2f} s            \",\n                n_epoch, train_acc, train_loss, train_time\n            )\n            \n            self.info_message(\n                \"[Epoch Valid: {}] accuracy: {:.4f}, loss: {:.4f}, time: {:.2f} s\",\n                n_epoch, valid_acc, valid_loss, valid_time\n            )\n\n            # if True:\n            # if self.best_valid_score < valid_auc: \n            if self.best_valid_score > valid_loss: \n                self.save_model(n_epoch, save_path, valid_loss)\n                self.info_message(\n                     \"loss has decresed from {:.4f} to {:.4f}. Saved model to '{}'\", \n                    self.best_valid_score, valid_loss, self.lastmodel\n                )\n                self.best_valid_score = valid_loss\n                self.n_patience = 0\n            else:\n                self.n_patience += 1\n            \n            if self.n_patience >= patience:\n                self.info_message(\"\\nValid auc didn't improve last {} epochs.\", patience)\n                break\n            \n    def train_epoch(self, train_loader):\n        self.model.train()\n        t = time.time()\n        sum_loss = 0\n        train_acc = 0.0\n\n        for step, batch in enumerate(train_loader, 1):\n            X = batch[\"X\"].to(self.device)\n            targets = batch[\"y\"].to(self.device)\n            \n            self.optimizer.zero_grad()\n            outputs = self.model(X).squeeze(1)\n#             print(torch.nn.functional.softmax(outputs, dim=1)[0] * 100)\n#             print(outputs.argmax())\n        \n            loss = self.criterion(outputs, targets)\n            loss.backward()\n            sum_loss += loss.detach().item()\n            self.optimizer.step()\n            \n            _, pred = torch.max(outputs, dim=1)\n#             print(pred, targets)\n            train_acc += sum((pred == targets).cpu().numpy())\n            \n            message = 'Train Step {}/{}, train_loss: {:.4f}'\n            self.info_message(message, step, len(train_loader), sum_loss/step, end=\"\\r\")\n#         print(train_acc / len(train_loader.dataset))\n        return train_acc / len(train_loader.dataset), sum_loss / len(train_loader), int(time.time() - t)\n    \n    def valid_epoch(self, valid_loader):\n        self.model.train()\n        t = time.time()\n        sum_loss = 0\n        y_all = []\n        outputs_all = []\n        \n        valid_acc = 0.0\n\n        for step, batch in enumerate(valid_loader, 1):\n            with torch.no_grad():\n                X = batch[\"X\"].to(self.device)\n                targets = batch[\"y\"].to(self.device)\n\n                outputs = self.model(X).squeeze(1)\n#                 print(torch.nn.functional.softmax(outputs, dim=1)[0] * 100)\n#                 print(outputs.argmax())\n                \n                loss = self.criterion(outputs, targets)\n                sum_loss += loss.detach().item()\n                y_all.extend(batch[\"y\"].tolist())\n                outputs_all.extend(outputs.tolist())\n                \n                _, pred = torch.max(outputs, dim=1)\n#                 print(pred, targets)\n                valid_acc += sum((pred == targets).cpu().numpy())\n\n            message = 'Valid Step {}/{}, valid_loss: {:.4f}'\n            self.info_message(message, step, len(valid_loader), sum_loss/step, end=\"\\r\")\n            \n#         y_all = [1 if x > 0.5 else 0 for x in y_all]\n#         auc = roc_auc_score(y_all, outputs_all)\n#         print(valid_acc / len(valid_loader.dataset))\n        return valid_acc / len(valid_loader.dataset), sum_loss/len(valid_loader), int(time.time() - t)\n    \n    def save_model(self, n_epoch, save_path, loss):\n        self.lastmodel = f\"{save_path}-e{n_epoch}-loss{loss:.3f}.pth\"\n        torch.save(\n            {\n                \"model_state_dict\": self.model.state_dict(),\n                \"optimizer_state_dict\": self.optimizer.state_dict(),\n                \"best_valid_score\": self.best_valid_score,\n                \"n_epoch\": n_epoch,\n            },\n            self.lastmodel,\n        )\n    \n    @staticmethod\n    def info_message(message, *args, end=\"\\n\"):\n        print(message.format(*args), end=end)","metadata":{"execution":{"iopub.status.busy":"2021-08-05T12:40:02.343788Z","iopub.execute_input":"2021-08-05T12:40:02.34423Z","iopub.status.idle":"2021-08-05T12:40:02.374155Z","shell.execute_reply.started":"2021-08-05T12:40:02.344183Z","shell.execute_reply":"2021-08-05T12:40:02.372677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n# torch.cuda.empty_cache()\n\n# def train_mri_type(df_train, df_valid):\n    \n#     train_data_retriever = Dataset(\n#         paths=df_train[\"BraTS21ID\"].values, \n#         labels=df_train[\"MGMT_value\"].values,\n#         augmentation=get_training_augmentation(),\n#         transformation=transform,\n#     )\n#     valid_data_retriever = Dataset(\n#         paths=df_valid[\"BraTS21ID\"].values, \n#         labels=df_valid[\"MGMT_value\"].values,\n#         augmentation=get_training_augmentation(),\n#         transformation=transform,\n#     )\n\n#     train_loader = torch_data.DataLoader(train_data_retriever, batch_size=8, shuffle=True, num_workers=8,)\n#     valid_loader = torch_data.DataLoader(valid_data_retriever, batch_size=8, shuffle=True, num_workers=8,)\n\n#     model = Model()\n#     model.to(device)\n\n# #     checkpoint = torch.load(\"../input/3dbrainmrimodels/classification_model-e5-loss0.683.pth\")\n# #     model.load_state_dict(checkpoint[\"model_state_dict\"])\n\n#     #print(model)\n\n#     optimizer = torch.optim.Adam(model.parameters(), lr=3e-4, weight_decay=1e-5)\n# #     optimizer = torch.optim.SGD(model.parameters(), lr=0.001, momentum=0.9)\n# #     criterion = torch_functional.binary_cross_entropy_with_logits\n#     criterion = nn.CrossEntropyLoss()\n# #     criterion = nn.BCEWithLogitsLoss()\n\n#     trainer = Trainer(\n#         model, \n#         device, \n#         optimizer, \n#         criterion\n#     )\n\n#     history = trainer.fit(\n#         10, \n#         train_loader, \n#         valid_loader, \n#         \"classification_model\",\n# #         f\"{mri_type}\", \n#         10,\n#     )\n    \n#     return trainer.lastmodel\n\n# train_mri_type(df_train, df_valid)","metadata":{"execution":{"iopub.status.busy":"2021-08-05T12:40:02.376475Z","iopub.execute_input":"2021-08-05T12:40:02.376974Z","iopub.status.idle":"2021-08-05T12:40:02.394252Z","shell.execute_reply.started":"2021-08-05T12:40:02.37692Z","shell.execute_reply":"2021-08-05T12:40:02.392843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Testing the model","metadata":{}},{"cell_type":"code","source":"df_train = df_train.set_index(\"BraTS21ID\")\ndf_train[\"MGMT_pred\"] = 0","metadata":{"execution":{"iopub.status.busy":"2021-08-05T12:40:02.396329Z","iopub.execute_input":"2021-08-05T12:40:02.396845Z","iopub.status.idle":"2021-08-05T12:40:02.412149Z","shell.execute_reply.started":"2021-08-05T12:40:02.396784Z","shell.execute_reply":"2021-08-05T12:40:02.410806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_valid = df_valid.set_index(\"BraTS21ID\")\ndf_valid[\"MGMT_pred\"] = 0","metadata":{"execution":{"iopub.status.busy":"2021-08-05T12:40:02.414266Z","iopub.execute_input":"2021-08-05T12:40:02.414745Z","iopub.status.idle":"2021-08-05T12:40:02.426845Z","shell.execute_reply.started":"2021-08-05T12:40:02.414685Z","shell.execute_reply":"2021-08-05T12:40:02.425714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# modelfile = \"./classification_model-e5-loss0.683.pth\"\nmodelfile = \"../input/3dbrainmrimodels/classification_model-e2-loss0.647.pth\"\nmodel = Model()\nmodel.to(device)\n\nif torch.cuda.is_available():\n    checkpoint = torch.load(modelfile)\nelse:\n    checkpoint = torch.load(modelfile, map_location=torch.device('cpu'))    \nmodel.load_state_dict(checkpoint[\"model_state_dict\"])","metadata":{"execution":{"iopub.status.busy":"2021-08-05T12:40:02.431799Z","iopub.execute_input":"2021-08-05T12:40:02.432352Z","iopub.status.idle":"2021-08-05T12:40:02.776113Z","shell.execute_reply.started":"2021-08-05T12:40:02.432297Z","shell.execute_reply":"2021-08-05T12:40:02.772068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TestDataset(torch_data.Dataset):\n    def __init__(self, paths, labels=None, mri_type=None, label_smoothing=0.01, augmentation=None, split=\"train\"):\n        self.paths = paths\n        self.labels = labels\n        self.mri_type = mri_type\n        self.label_smoothing = label_smoothing\n        self.augmentation = augmentation\n        self.split = split\n          \n    def __len__(self):\n        return len(self.paths)\n    \n    def __getitem__(self, index):\n        scan_id = self.paths[index]\n        label = self.labels[index]\n        \n        transform = transforms.Compose([\n            transforms.ToTensor(), \n            transforms.Normalize((0.5,) * NUM_IMAGES, (0.5,) * NUM_IMAGES)\n        ])\n        \n        images = []\n        for i in mri_types:\n            image_3d = load_dicom_images_3d(scan_id=str(scan_id).zfill(5), mri_type=i)\n            image_3d = transform(image_3d)\n            images.append(image_3d)\n            \n        four_channel_pack = np.stack(images)        \n        \n        y = self.labels[index]\n        return torch.tensor(four_channel_pack).float(), y","metadata":{"execution":{"iopub.status.busy":"2021-08-05T12:40:02.777998Z","iopub.execute_input":"2021-08-05T12:40:02.778355Z","iopub.status.idle":"2021-08-05T12:40:02.789037Z","shell.execute_reply.started":"2021-08-05T12:40:02.77832Z","shell.execute_reply":"2021-08-05T12:40:02.787794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data_retriever = TestDataset(\n    df_train.index.values, \n    df_train[\"MGMT_value\"].values,\n    augmentation=get_training_augmentation(),\n    split=\"test\",\n)\n\ntest_data_loader = torch_data.DataLoader(\n    test_data_retriever,\n    batch_size=16,\n    shuffle=False,\n    num_workers=8,\n)","metadata":{"execution":{"iopub.status.busy":"2021-08-05T12:40:02.790614Z","iopub.execute_input":"2021-08-05T12:40:02.790953Z","iopub.status.idle":"2021-08-05T12:40:02.805259Z","shell.execute_reply.started":"2021-08-05T12:40:02.790915Z","shell.execute_reply":"2021-08-05T12:40:02.804087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a, b = test_data_retriever[0]\nplt.imshow(a[0, 10, :, :])","metadata":{"execution":{"iopub.status.busy":"2021-08-05T12:40:02.806926Z","iopub.execute_input":"2021-08-05T12:40:02.80757Z","iopub.status.idle":"2021-08-05T12:40:03.71959Z","shell.execute_reply.started":"2021-08-05T12:40:02.807519Z","shell.execute_reply":"2021-08-05T12:40:03.718482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fig, axis = plt.subplots(4, 4, figsize=(8, 10))\n# with torch.no_grad():\n#     model.eval()\n#     for i, ax in enumerate(axis.flat):\n#         image, label = test_fetures[i].to(device), test_labels[i].to(device)\n#         ax.imshow(image[0, 20, :, :].cpu())\n        \n# #         image_tensor = image.unsqueeze_(0)\n#         output_ = model(image).squeeze(1)\n#         _, pred = torch.max(output_, dim=1)\n#         print(pred, label)\n        \n\n#         print(\n#             torch.nn.functional.softmax(model(image), dim=1)[0] * 100,\n#         )\n#         _, index = torch.max(output_, 1)\n        \n#         percentage = torch.nn.functional.softmax(output_, dim=1)[0] * 100\n#         result = output_.argmax()\n        \n#         print(index, percentage[0], percentage[1])\n#         print(sum([percentage[0], percentage[1]]), \"\\n\")\n#         ax.set(title = f\"actual:{label}\\nprediction:{result}\")","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2021-08-05T12:40:03.721134Z","iopub.execute_input":"2021-08-05T12:40:03.721483Z","iopub.status.idle":"2021-08-05T12:40:03.725895Z","shell.execute_reply.started":"2021-08-05T12:40:03.721442Z","shell.execute_reply":"2021-08-05T12:40:03.724852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### AUC Score","metadata":{}},{"cell_type":"code","source":"data_retriever = Dataset(\n    df_valid.index.values, \n    df_valid[\"MGMT_value\"].values,\n    split=\"test\",\n)\n\ndata_loader = torch_data.DataLoader(\n    data_retriever,\n    batch_size=1,\n    shuffle=False,\n    num_workers=8,\n)\n\ny_preds = []\ny = []\n\nfor e, batch in enumerate(data_loader):\n    print(f\"{e}/{len(data_loader)}\", end=\"\\r\")\n    with torch.no_grad():\n#         model.eval()\n        image, label = batch[\"X\"].to(device), batch[\"y\"]\n        \n#         plt.imshow(image.cpu()[0, 0, 30, :, :])\n#         plt.show()\n        \n        output_ = model(image).squeeze(1)\n        _, pred = torch.max(output_, dim=1)\n\n        _, index = torch.max(output_, 1)\n        percentage = torch.nn.functional.softmax(output_, dim=1)[0]\n#         print(percentage * 100)\n#         tmp_pred = torch.nn.functional.softmax(model(image), dim=1)[0] * 100\n    \n    \n        label = label.detach().cpu().numpy()[0]\n        prediction = float(percentage[1].detach().cpu().numpy())\n        y.append(label)\n        y_preds.append(prediction)","metadata":{"execution":{"iopub.status.busy":"2021-08-05T12:40:03.727336Z","iopub.execute_input":"2021-08-05T12:40:03.727695Z","iopub.status.idle":"2021-08-05T12:40:40.767926Z","shell.execute_reply.started":"2021-08-05T12:40:03.727664Z","shell.execute_reply":"2021-08-05T12:40:40.766543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# score\ny = np.array(y)\ny_preds = np.array(y_preds)\n\nfpr, tpr, thresholds = metrics.roc_curve(y, y_preds, pos_label=1)\nroc_auc = metrics.auc(fpr, tpr)\n\nprint(f\"AUC score is: {roc_auc}\")","metadata":{"execution":{"iopub.status.busy":"2021-08-05T12:40:40.769839Z","iopub.execute_input":"2021-08-05T12:40:40.770168Z","iopub.status.idle":"2021-08-05T12:40:40.782171Z","shell.execute_reply.started":"2021-08-05T12:40:40.770132Z","shell.execute_reply":"2021-08-05T12:40:40.780874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr_fpr, lr_tpr, _ = metrics.roc_curve(y, y_preds)\nplt.plot(lr_fpr, lr_tpr, marker='.', label='abc')\nplt.xlabel('False Positive Rate')\nplt.ylabel('True Positive Rate')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-05T12:40:40.784498Z","iopub.execute_input":"2021-08-05T12:40:40.784901Z","iopub.status.idle":"2021-08-05T12:40:40.955225Z","shell.execute_reply.started":"2021-08-05T12:40:40.784848Z","shell.execute_reply":"2021-08-05T12:40:40.954015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SUBMISSIONDataset(torch_data.Dataset):\n    def __init__(self, \n                 augmentation=None, \n                 preprocessing=None,\n                ):\n        self.indexes = sorted(os.listdir(\"../input/rsna-miccai-brain-tumor-radiogenomic-classification/test\"))\n        self.augmentation = augmentation\n        self.preprocessing = preprocessing\n          \n    def __len__(self):\n        return len(self.indexes)\n    \n    def __getitem__(self, index):\n        scan_id = self.indexes[index]\n        four_channel_pack = None\n#         label = self.labels[index]\n\n        self.indexes.append(\"09129\")\n    \n        transform = transforms.Compose([\n            transforms.ToTensor(), \n            transforms.Normalize((0.5,) * NUM_IMAGES, (0.5,) * NUM_IMAGES)\n        ])\n        images = []\n        \n        try:\n            for i in mri_types:\n                try:\n                    image_3d = load_dicom_images_3d(scan_id=str(scan_id).zfill(5), split=\"test\", mri_type=i)\n                    image_3d = transform(image_3d)\n                    images.append(image_3d)\n                except:\n                    pass\n                four_channel_pack = np.stack(images)\n        except:\n            pass\n\n#         y = self.labels[index]\n        return {\"X\": torch.tensor(four_channel_pack).float(), \"id\": scan_id}","metadata":{"execution":{"iopub.status.busy":"2021-08-04T17:20:52.62502Z","iopub.execute_input":"2021-08-04T17:20:52.625462Z","iopub.status.idle":"2021-08-04T17:20:52.64192Z","shell.execute_reply.started":"2021-08-04T17:20:52.625422Z","shell.execute_reply":"2021-08-04T17:20:52.641018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv(\"../input/rsna-miccai-brain-tumor-radiogenomic-classification/sample_submission.csv\")\n\ntest_dataset = SUBMISSIONDataset(\n#     augmentation=get_validation_augmentation(),\n)\n\ndata_loader = torch_data.DataLoader(\n    test_dataset,\n    batch_size=1,\n    shuffle=False,\n    num_workers=8,\n)","metadata":{"execution":{"iopub.status.busy":"2021-08-04T17:20:54.538363Z","iopub.execute_input":"2021-08-04T17:20:54.538737Z","iopub.status.idle":"2021-08-04T17:20:54.551172Z","shell.execute_reply.started":"2021-08-04T17:20:54.538705Z","shell.execute_reply":"2021-08-04T17:20:54.550239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ids = []\npreds = []\n\nfor e, batch in enumerate(data_loader):\n    print(f\"{e}/{len(data_loader)}\", end=\"\\r\")\n    with torch.no_grad():\n#         model.eval()\n        image, id = batch[\"X\"].to(device), str(batch[\"id\"][0])\n        \n        try:\n            output_ = model(image).squeeze(1)\n            percentage = torch.nn.functional.softmax(output_, dim=1)[0]\n\n            prediction = float(percentage[1].detach().cpu().numpy())\n        except:\n            prediction = 0.5\n            \n        preds.append(prediction)\n        ids.append(id)","metadata":{"execution":{"iopub.status.busy":"2021-08-04T17:21:49.773753Z","iopub.execute_input":"2021-08-04T17:21:49.774126Z","iopub.status.idle":"2021-08-04T17:23:05.789284Z","shell.execute_reply.started":"2021-08-04T17:21:49.774093Z","shell.execute_reply":"2021-08-04T17:23:05.78822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame({\"BraTS21ID\": ids, \"MGMT_value\": preds})\n# df['MGMT_value'] = preds\ndf[['BraTS21ID', 'MGMT_value']].to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-08-04T17:23:05.791483Z","iopub.execute_input":"2021-08-04T17:23:05.791876Z","iopub.status.idle":"2021-08-04T17:23:05.809243Z","shell.execute_reply.started":"2021-08-04T17:23:05.791842Z","shell.execute_reply":"2021-08-04T17:23:05.808165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2021-08-04T17:23:05.811132Z","iopub.execute_input":"2021-08-04T17:23:05.811442Z","iopub.status.idle":"2021-08-04T17:23:05.829449Z","shell.execute_reply.started":"2021-08-04T17:23:05.811414Z","shell.execute_reply":"2021-08-04T17:23:05.828291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.displot(df[\"MGMT_value\"])","metadata":{"execution":{"iopub.status.busy":"2021-08-04T17:23:05.831499Z","iopub.execute_input":"2021-08-04T17:23:05.83235Z","iopub.status.idle":"2021-08-04T17:23:06.192727Z","shell.execute_reply.started":"2021-08-04T17:23:05.83219Z","shell.execute_reply":"2021-08-04T17:23:06.191877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}