{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Use stacked images (3D) and Vit model\n\nRefernece:\n\n- https://www.kaggle.com/rluethy/efficientnet3d-with-one-mri-type\n- https://github.com/lucidrains/vit-pytorch\n\n\n#### Many thanks to @ROLAND LUETHY\n* I use @ROLAND LUETHY great notebook, and change the efficient3D model to VIT 3D model, just want to try Vit on 3D \n* Training log, please check Version1\n    ","metadata":{"execution":{"iopub.status.busy":"2023-02-27T04:42:00.255034Z","iopub.execute_input":"2023-02-27T04:42:00.255491Z","iopub.status.idle":"2023-02-27T04:42:00.265194Z","shell.execute_reply.started":"2023-02-27T04:42:00.255459Z","shell.execute_reply":"2023-02-27T04:42:00.263286Z"}}},{"cell_type":"code","source":"pip install einops","metadata":{"execution":{"iopub.status.busy":"2023-04-06T18:46:49.423649Z","iopub.execute_input":"2023-04-06T18:46:49.424032Z","iopub.status.idle":"2023-04-06T18:47:00.324313Z","shell.execute_reply.started":"2023-04-06T18:46:49.424000Z","shell.execute_reply":"2023-04-06T18:47:00.322697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport sys \nimport json\nimport glob\nimport random\nimport collections\nimport time\nimport re\n\nimport numpy as np\nimport pandas as pd\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nimport cv2\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n\nimport torch\nfrom torch import nn\nfrom torch.utils import data as torch_data\nfrom sklearn import model_selection as sk_model_selection\nfrom torch.nn import functional as torch_functional\nimport torch.nn.functional as F\nfrom einops import rearrange, repeat\nfrom einops.layers.torch import Rearrange\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.metrics import roc_curve,auc,precision_recall_curve,average_precision_score","metadata":{"papermill":{"duration":1.048295,"end_time":"2021-07-14T20:26:46.309722","exception":false,"start_time":"2021-07-14T20:26:45.261427","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-06T18:47:00.328577Z","iopub.execute_input":"2023-04-06T18:47:00.328904Z","iopub.status.idle":"2023-04-06T18:47:02.209795Z","shell.execute_reply.started":"2023-04-06T18:47:00.328869Z","shell.execute_reply":"2023-04-06T18:47:02.208727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install efficientnet-pytorch","metadata":{"execution":{"iopub.status.busy":"2023-04-06T18:47:02.211209Z","iopub.execute_input":"2023-04-06T18:47:02.212660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed_all(seed)\n        torch.backends.cudnn.deterministic = True\n\nset_seed(12)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if os.path.exists(\"../input/rsna-miccai-brain-tumor-radiogenomic-classification\"):\n    data_directory = '../input/rsna-miccai-brain-tumor-radiogenomic-classification'\n    pytorch3dpath = \"../input/efficientnetpyttorch3d/EfficientNet-PyTorch-3D\"\nelse:\n    data_directory = '/media/roland/data/kaggle/rsna-miccai-brain-tumor-radiogenomic-classification'\n    pytorch3dpath = \"EfficientNet-PyTorch-3D\"\n    \nmri_types = ['FLAIR','T1w','T1wCE','T2w']\nSIZE = 256\nNUM_IMAGES = 64\n\nsys.path.append(pytorch3dpath)\nfrom efficientnet_pytorch_3d import EfficientNet3D","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(f\"{data_directory}/train_labels.csv\")\ndisplay(train_df)\n\ndf_train, df_valid = sk_model_selection.train_test_split(\n    train_df, \n    test_size=0.2, \n    random_state=12, \n    stratify=train_df[\"MGMT_value\"],\n)\n\ndf_train.tail()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Dataset(torch_data.Dataset):\n    def __init__(self, paths, targets=None, mri_type=None, label_smoothing=0.01, split=\"train\", augment=True):\n        self.paths = paths\n        self.targets = targets\n        self.mri_type = mri_type\n        self.label_smoothing = label_smoothing\n        self.split = split\n        self.augment = augment\n          \n    def __len__(self):\n        return len(self.paths)\n    \n    def __getitem__(self, index):\n        scan_id = self.paths[index]\n        if self.targets is None:\n            data = load_dicom_images_3d(str(scan_id).zfill(5), mri_type=self.mri_type[index], split=self.split)\n        else:\n            if self.augment:\n                rotation = np.random.randint(0,4)\n            else:\n                rotation = 0\n\n            data = load_dicom_images_3d(str(scan_id).zfill(5), mri_type=self.mri_type[index], split=\"train\", rotate=rotation)\n\n        if self.targets is None:\n            return {\"X\": torch.tensor(data).float(), \"id\": scan_id}\n        else:\n            y = torch.tensor(abs(self.targets[index]-self.label_smoothing), dtype=torch.float)\n            return {\"X\": torch.tensor(data).float(), \"y\": y}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Functions to load images","metadata":{}},{"cell_type":"code","source":"def load_dicom_image(path, img_size=SIZE, voi_lut=True, rotate=0):\n    dicom = pydicom.read_file(path)\n    data = dicom.pixel_array\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n        \n    if rotate > 0:\n        rot_choices = [0, cv2.ROTATE_90_CLOCKWISE, cv2.ROTATE_90_COUNTERCLOCKWISE, cv2.ROTATE_180]\n        data = cv2.rotate(data, rot_choices[rotate])\n        \n    data = cv2.resize(data, (img_size, img_size))\n    return data\n\ndef load_dicom_images_3d(scan_id, num_imgs=NUM_IMAGES, img_size=SIZE, mri_type=\"FLAIR\", split=\"train\", rotate=0):\n\n    files = sorted(glob.glob(f\"{data_directory}/{split}/{scan_id}/{mri_type}/*.dcm\"), \n               key=lambda var:[int(x) if x.isdigit() else x for x in re.findall(r'[^0-9]|[0-9]+', var)])\n\n    middle = len(files)//2\n    num_imgs2 = num_imgs//2\n    p1 = max(0, middle - num_imgs2)\n    p2 = min(len(files), middle + num_imgs2)\n    img3d = np.stack([load_dicom_image(f, rotate=rotate) for f in files[p1:p2]]).T \n    if img3d.shape[-1] < num_imgs:\n        n_zero = np.zeros((img_size, img_size, num_imgs - img3d.shape[-1]))\n        img3d = np.concatenate((img3d,  n_zero), axis = -1)\n        \n    if np.min(img3d) < np.max(img3d):\n        img3d = img3d - np.min(img3d)\n        img3d = img3d / np.max(img3d)\n            \n    return np.expand_dims(img3d,0)\n\na = load_dicom_images_3d(\"00000\")\nprint(a.shape)\nprint(np.min(a), np.max(a), np.mean(a), np.median(a))","metadata":{"papermill":{"duration":0.035761,"end_time":"2021-07-14T20:26:46.726756","exception":false,"start_time":"2021-07-14T20:26:46.690995","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predict function","metadata":{}},{"cell_type":"code","source":"def predict(modelfile, df, mri_type, split):\n    print(\"Predict:\", modelfile, mri_type, df.shape)\n    df.loc[:,\"MRI_Type\"] = mri_type\n    \n    data_retriever = Dataset(\n        df.index.values, \n        mri_type=df[\"MRI_Type\"].values,\n        split=split\n    )\n    data_loader = torch_data.DataLoader(\n        data_retriever,\n        batch_size=4,\n        shuffle=False,\n        num_workers=8,\n    )\n   \n    model = Model(\n        image_size = 256,\n        patch_size = 32,\n        num_classes = 1,\n        dim = 1024,\n        depth = 2,\n        heads = 16,\n        mlp_dim = 2048,\n        channels = 1,\n        dropout = 0.1,\n        emb_dropout = 0.1\n    )\n    model.to(device)\n    \n    checkpoint = torch.load(modelfile)\n    model.load_state_dict(checkpoint[\"model_state_dict\"])\n    model.eval()\n    \n    y_pred = []\n    ids = []\n\n    for e, batch in enumerate(data_loader,1):\n        print(f\"{e}/{len(data_loader)}\", end=\"\\r\")\n        with torch.no_grad():\n            tmp_pred = torch.sigmoid(model(batch[\"X\"].to(device))).cpu().numpy().squeeze()\n            if tmp_pred.size == 1:\n                y_pred.append(tmp_pred)\n            else:\n                y_pred.extend(tmp_pred.tolist())\n            ids.extend(batch[\"id\"].numpy().tolist())\n            \n    preddf = pd.DataFrame({\"BraTS21ID\": ids, \"MGMT_value\": y_pred}) \n    preddf = preddf.set_index(\"BraTS21ID\")\n    return preddf","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_roc_pr_curve(y_true, y_pred):\n\n    # 绘制模型在验证集上的ROC曲线\n    fpr, tpr, _ = roc_curve(y_true, y_pred)\n    roc_auc = auc(fpr, tpr)\n    plt.plot(fpr, tpr, color='darkorange', lw=2, label='ROC curve (area = %0.2f)' % roc_auc)\n    plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')\n    plt.xlim([0.0, 1.0])\n    plt.ylim([0.0, 1.05])\n    plt.xlabel('False Positive Rate')\n    plt.ylabel('True Positive Rate')\n    plt.title('Receiver operating characteristic')\n    plt.legend(loc=\"lower right\")\n    plt.show()\n\n    # 绘制训练集和验证集上的PR曲线\n    precision, recall, _ = precision_recall_curve(y_true, y_pred)\n    average_precision = average_precision_score(y_true, y_pred)\n    plt.plot(recall, precision, lw=2, label='PR curve (area = %0.2f)' % average_precision)\n    plt.xlabel('Recall')\n    plt.ylabel('Precision')\n    plt.title('Precision-Recall curve')\n    plt.legend(loc=\"lower right\")\n    plt.show()\n    \n    \nclass Trainer:\n    def __init__(\n        self, \n        model, \n        device, \n        optimizer, \n        criterion\n    ):\n        self.model = model\n        self.device = device\n        self.optimizer = optimizer\n        self.criterion = criterion\n\n        self.best_valid_score = np.inf\n        self.n_patience = 0\n        self.lastmodel = None\n        \n    def fit(self, epochs, train_loader, valid_loader, save_path, patience):\n        global train_loss_history,valid_loss_history,valid_auc_history\n        train_loss_history=[]\n        valid_loss_history=[]\n        valid_auc_history=[]\n        train_auc_history=[]\n\n        for n_epoch in range(1, 100):\n            self.info_message(\"EPOCH: {}\", n_epoch)\n            \n            train_loss, train_time ,train_auc,y_all_train, outputs_all_train= self.train_epoch(train_loader)\n            valid_loss, valid_auc, valid_time,y_all_valid, outputs_all_valid = self.valid_epoch(valid_loader)\n            \n            train_loss_history.append(train_loss)\n            valid_loss_history.append(valid_loss)\n            valid_auc_history.append(valid_auc)\n            train_auc_history.append(train_auc)\n            \n            self.info_message(\n                \"[Epoch Train: {}] loss: {:.4f}, time: {:.2f} s            \",\n                n_epoch, train_loss, train_time\n            )\n            \n            self.info_message(\n                \"[Epoch Valid: {}] loss: {:.4f}, auc: {:.4f}, time: {:.2f} s\",\n                n_epoch, valid_loss, valid_auc, valid_time\n            )\n\n\n            if self.best_valid_score > valid_loss: \n                self.save_model(n_epoch, save_path, valid_loss, valid_auc)\n                self.info_message(\n                     \"auc improved from {:.4f} to {:.4f}. Saved model to '{}'\", \n                    self.best_valid_score, valid_loss, self.lastmodel\n                )\n                self.best_valid_score = valid_loss\n                self.n_patience = 0\n            \n        plt.plot(train_loss_history, label='Train Loss')\n        plt.plot(valid_loss_history, label='Valid Loss')\n        plt.legend()\n        plt.xlabel('Epoch')\n        plt.ylabel('Loss')\n        plt.title('Training and Validation Loss')\n        plt.show()\n\n        plt.plot(valid_auc_history, label='Valid AUC')\n        plt.plot(train_auc_history, label='Train AUC')\n        plt.legend()\n        plt.xlabel('Epoch')\n        plt.ylabel('AUC')\n        plt.title('Training and Validation AUC')\n        plt.show()\n        \n        plot_roc_pr_curve(y_all_valid, outputs_all_valid)#测试集的ROC和PR曲线\n            \n    def train_epoch(self, train_loader):\n        self.model.train()\n        t = time.time()\n        sum_loss = 0\n        y_all = []\n        outputs_all = []\n\n        for step, batch in enumerate(train_loader, 1):\n            X = batch[\"X\"].to(self.device)\n            targets = batch[\"y\"].to(self.device)\n            self.optimizer.zero_grad()\n            outputs = self.model(X).squeeze(1)\n            \n            loss = self.criterion(outputs, targets)\n            loss.backward()\n\n            sum_loss += loss.detach().item()\n\n            self.optimizer.step()\n            \n            y_all.extend(batch[\"y\"].tolist())\n            outputs_all.extend(torch.sigmoid(outputs).tolist())\n            \n            message = 'Train Step {}/{}, train_loss: {:.4f}'\n            self.info_message(message, step, len(train_loader), sum_loss/step, end=\"\\r\")\n            \n        y_all = [1 if x > 0.5 else 0 for x in y_all]\n        auc = roc_auc_score(y_all, outputs_all)\n        \n        return sum_loss/len(train_loader), int(time.time() - t),auc,y_all, outputs_all\n    \n    def valid_epoch(self, valid_loader):\n        self.model.eval()\n        t = time.time()\n        sum_loss = 0\n        y_all = []\n        outputs_all = []\n\n        for step, batch in enumerate(valid_loader, 1):\n            with torch.no_grad():\n                X = batch[\"X\"].to(self.device)\n                targets = batch[\"y\"].to(self.device)\n\n                outputs = self.model(X).squeeze(1)\n                loss = self.criterion(outputs, targets)\n\n                sum_loss += loss.detach().item()\n                y_all.extend(batch[\"y\"].tolist())\n                outputs_all.extend(torch.sigmoid(outputs).tolist())\n\n            message = 'Valid Step {}/{}, valid_loss: {:.4f}'\n            self.info_message(message, step, len(valid_loader), sum_loss/step, end=\"\\r\")\n            \n        y_all = [1 if x > 0.5 else 0 for x in y_all]\n        auc = roc_auc_score(y_all, outputs_all)\n        \n        return sum_loss/len(valid_loader), auc, int(time.time() - t),y_all, outputs_all\n    \n    def save_model(self, n_epoch, save_path, loss, auc):\n        self.lastmodel = f\"{save_path}-best.pth\"\n        torch.save(\n            {\n                \"model_state_dict\": self.model.state_dict(),\n                \"optimizer_state_dict\": self.optimizer.state_dict(),\n                \"best_valid_score\": self.best_valid_score,\n                \"n_epoch\": n_epoch,\n            },\n            self.lastmodel,\n        )\n    \n    @staticmethod\n    def info_message(message, *args, end=\"\\n\"):\n        print(message.format(*args), end=end)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model and training classes","metadata":{}},{"cell_type":"code","source":"!pip install ../input/einops-030/einops-0.3.0-py2.py3-none-any.whl","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pair(t):\n    return t if isinstance(t, tuple) else (t, t)\n\n# classes\n\nclass PreNorm(nn.Module):\n    def __init__(self, dim, fn):\n        super().__init__()\n        self.norm = nn.LayerNorm(dim)\n        self.fn = fn\n    def forward(self, x, **kwargs):\n        return self.fn(self.norm(x), **kwargs)\n\nclass FeedForward(nn.Module):\n    def __init__(self, dim, hidden_dim, dropout = 0.):\n        super().__init__()\n        self.net = nn.Sequential(\n            nn.Linear(dim, hidden_dim),\n            nn.GELU(),\n            nn.Dropout(dropout),\n            nn.Linear(hidden_dim, dim),\n            nn.Dropout(dropout)\n        )\n    def forward(self, x):\n        return self.net(x)\n\nclass Attention(nn.Module):\n    def __init__(self, dim, heads = 8, dim_head = 64, dropout = 0.):\n        super().__init__()\n        inner_dim = dim_head *  heads\n        project_out = not (heads == 1 and dim_head == dim)\n\n        self.heads = heads\n        self.scale = dim_head ** -0.5\n\n        self.attend = nn.Softmax(dim = -1)\n        self.to_qkv = nn.Linear(dim, inner_dim * 3, bias = False)\n\n        self.to_out = nn.Sequential(\n            nn.Linear(inner_dim, dim),\n            nn.Dropout(dropout)\n        ) if project_out else nn.Identity()\n\n    def forward(self, x):\n        qkv = self.to_qkv(x).chunk(3, dim = -1)\n        q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> b h n d', h = self.heads), qkv)\n\n        dots = torch.matmul(q, k.transpose(-1, -2)) * self.scale\n\n        attn = self.attend(dots)\n\n        out = torch.matmul(attn, v)\n        out = rearrange(out, 'b h n d -> b n (h d)')\n        return self.to_out(out)\n\nclass Transformer(nn.Module):\n    def __init__(self, dim, depth, heads, dim_head, mlp_dim, dropout = 0.):\n        super().__init__()\n        self.layers = nn.ModuleList([])\n        mlp_dim = 2048\n        for _ in range(depth):\n            self.layers.append(nn.ModuleList([\n                PreNorm(dim, Attention(dim, heads = heads, dim_head = dim_head, dropout = dropout)),\n                PreNorm(dim, FeedForward(dim, mlp_dim, dropout = dropout))\n            ]))\n    def forward(self, x):\n        for attn, ff in self.layers:\n            x = attn(x) + x\n            x = ff(x) + x\n        return x\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Vit Model change to 3D\n\nreference:\nhttps://github.com/lucidrains/vit-pytorch","metadata":{}},{"cell_type":"code","source":"class Model(nn.Module):\n    def __init__(self, *, image_size, patch_size, num_classes, dim, depth, heads, mlp_dim, channels = 3, dropout = 0., emb_dropout = 0.):\n        super().__init__()\n        assert image_size % patch_size == 0, 'image dimensions must be divisible by the patch size'\n        num_patches = (image_size // patch_size) *(image_size // patch_size)* 2\n        patch_dim = channels * patch_size ** 3\n\n        self.patch_size = patch_size\n\n        self.pos_embedding = nn.Parameter(torch.randn(1, num_patches + 1, dim))\n        self.patch_to_embedding = nn.Linear(patch_dim, dim)\n        self.cls_token = nn.Parameter(torch.randn(1, 1, dim))\n        self.dropout = nn.Dropout(emb_dropout)\n        self.transformer = Transformer(dim, depth, heads, mlp_dim, dropout)\n        self.to_cls_token = nn.Identity()\n\n        self.mlp_head = nn.Sequential(\n            nn.LayerNorm(dim),\n            nn.Linear(dim, mlp_dim),\n            nn.GELU(),\n            nn.Dropout(dropout),\n            nn.Linear(mlp_dim, num_classes),\n            nn.Dropout(dropout)\n        )\n\n    def forward(self, img, mask = None):\n        p = self.patch_size\n        x = rearrange(img, 'b c (h p1) (w p2) (d p3) -> b (h w d) (p1 p2 p3 c)', p1 = p, p2 = p, p3 = p)\n        x = self.patch_to_embedding(x)\n        cls_tokens = self.cls_token.expand(img.shape[0], -1, -1)\n        x = torch.cat((cls_tokens, x), dim=1)\n        x += self.pos_embedding\n        x = self.dropout(x)\n\n        x = self.transformer(x)\n\n        x = self.to_cls_token(x[:, 0])\n        return self.mlp_head(x)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# class Model(nn.Module):\n#     def __init__(self):\n#         super().__init__()\n#         self.net = EfficientNet3D.from_name(\"efficientnet-b0\", override_params={'num_classes': 2}, in_channels=1)\n#         n_features = self.net._fc.in_features\n#         self.net._fc = nn.Linear(in_features=n_features, out_features=1, bias=True)\n    \n#     def forward(self, x):\n#         out = self.net(x)\n#         return out\n    ","metadata":{"papermill":{"duration":0.825458,"end_time":"2021-07-14T20:27:55.604161","exception":false,"start_time":"2021-07-14T20:27:54.778703","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"def plot_roc_pr_curve(y_true, y_pred):\n\n    # 绘制模型在验证集上的ROC曲线\n    fpr, tpr, _ = roc_curve(y_true, y_pred)\n    roc_auc = auc(fpr, tpr)\n    plt.plot(fpr, tpr, color='darkorange', lw=2, label='ROC curve (area = %0.2f)' % roc_auc)\n    plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')\n    plt.xlim([0.0, 1.0])\n    plt.ylim([0.0, 1.05])\n    plt.xlabel('False Positive Rate')\n    plt.ylabel('True Positive Rate')\n    plt.title('Receiver operating characteristic')\n    plt.legend(loc=\"lower right\")\n    plt.show()\n\n    # 绘制训练集和验证集上的PR曲线\n    precision, recall, _ = precision_recall_curve(y_true, y_pred)\n    average_precision = average_precision_score(y_true, y_pred)\n    plt.plot(recall, precision, lw=2, label='PR curve (area = %0.2f)' % average_precision)\n    plt.xlabel('Recall')\n    plt.ylabel('Precision')\n    plt.title('Precision-Recall curve')\n    plt.legend(loc=\"lower right\")\n    plt.show()\n    \n    \nclass Trainer:\n    def __init__(\n        self, \n        model, \n        device, \n        optimizer, \n        criterion\n    ):\n        self.model = model\n        self.device = device\n        self.optimizer = optimizer\n        self.criterion = criterion\n\n        self.best_valid_score = np.inf\n        self.n_patience = 0\n        self.lastmodel = None\n        \n    def fit(self, epochs, train_loader, valid_loader, save_path, patience):\n        global train_loss_history,valid_loss_history,valid_auc_history\n        train_loss_history=[]\n        valid_loss_history=[]\n        valid_auc_history=[]\n        train_auc_history=[]\n\n        for n_epoch in range(1, epochs + 1):\n            self.info_message(\"EPOCH: {}\", n_epoch)\n            \n            train_loss, train_time ,train_auc,y_all_train, outputs_all_train= self.train_epoch(train_loader)\n            valid_loss, valid_auc, valid_time,y_all_valid, outputs_all_valid = self.valid_epoch(valid_loader)\n            \n            train_loss_history.append(train_loss)\n            valid_loss_history.append(valid_loss)\n            valid_auc_history.append(valid_auc)\n            train_auc_history.append(train_auc)\n            \n            self.info_message(\n                \"[Epoch Train: {}] loss: {:.4f}, time: {:.2f} s            \",\n                n_epoch, train_loss, train_time\n            )\n            \n            self.info_message(\n                \"[Epoch Valid: {}] loss: {:.4f}, auc: {:.4f}, time: {:.2f} s\",\n                n_epoch, valid_loss, valid_auc, valid_time\n            )\n\n\n            if self.best_valid_score > valid_loss: \n                self.save_model(n_epoch, save_path, valid_loss, valid_auc)\n                self.info_message(\n                     \"auc improved from {:.4f} to {:.4f}. Saved model to '{}'\", \n                    self.best_valid_score, valid_loss, self.lastmodel\n                )\n                self.best_valid_score = valid_loss\n                self.n_patience = 0\n            else:\n                self.n_patience += 1\n            \n            if self.n_patience >= patience:\n                self.info_message(\"\\nValid auc didn't improve last {} epochs.\", patience)\n                break\n                \n        plt.plot(train_loss_history, label='Train Loss')\n        plt.plot(valid_loss_history, label='Valid Loss')\n        plt.legend()\n        plt.xlabel('Epoch')\n        plt.ylabel('Loss')\n        plt.title('Training and Validation Loss')\n        plt.show()\n\n        plt.plot(valid_auc_history, label='Valid AUC')\n        plt.plot(train_auc_history, label='Train AUC')\n        plt.legend()\n        plt.xlabel('Epoch')\n        plt.ylabel('AUC')\n        plt.title('Training and Validation AUC')\n        plt.show()\n        \n        plot_roc_pr_curve(y_all_valid, outputs_all_valid)#测试集的ROC和PR曲线\n            \n    def train_epoch(self, train_loader):\n        self.model.train()\n        t = time.time()\n        sum_loss = 0\n        y_all = []\n        outputs_all = []\n\n        for step, batch in enumerate(train_loader, 1):\n            X = batch[\"X\"].to(self.device)\n            targets = batch[\"y\"].to(self.device)\n            self.optimizer.zero_grad()\n            outputs = self.model(X).squeeze(1)\n            \n            loss = self.criterion(outputs, targets)\n            loss.backward()\n\n            sum_loss += loss.detach().item()\n\n            self.optimizer.step()\n            \n            y_all.extend(batch[\"y\"].tolist())\n            outputs_all.extend(torch.sigmoid(outputs).tolist())\n            \n            message = 'Train Step {}/{}, train_loss: {:.4f}'\n            self.info_message(message, step, len(train_loader), sum_loss/step, end=\"\\r\")\n            \n        y_all = [1 if x > 0.5 else 0 for x in y_all]\n        auc = roc_auc_score(y_all, outputs_all)\n        \n        return sum_loss/len(train_loader), int(time.time() - t),auc,y_all, outputs_all\n    \n    def valid_epoch(self, valid_loader):\n        self.model.eval()\n        t = time.time()\n        sum_loss = 0\n        y_all = []\n        outputs_all = []\n\n        for step, batch in enumerate(valid_loader, 1):\n            with torch.no_grad():\n                X = batch[\"X\"].to(self.device)\n                targets = batch[\"y\"].to(self.device)\n\n                outputs = self.model(X).squeeze(1)\n                loss = self.criterion(outputs, targets)\n\n                sum_loss += loss.detach().item()\n                y_all.extend(batch[\"y\"].tolist())\n                outputs_all.extend(torch.sigmoid(outputs).tolist())\n\n            message = 'Valid Step {}/{}, valid_loss: {:.4f}'\n            self.info_message(message, step, len(valid_loader), sum_loss/step, end=\"\\r\")\n            \n        y_all = [1 if x > 0.5 else 0 for x in y_all]\n        auc = roc_auc_score(y_all, outputs_all)\n        \n        return sum_loss/len(valid_loader), auc, int(time.time() - t),y_all, outputs_all\n    \n    def save_model(self, n_epoch, save_path, loss, auc):\n        self.lastmodel = f\"{save_path}-best.pth\"\n        torch.save(\n            {\n                \"model_state_dict\": self.model.state_dict(),\n                \"optimizer_state_dict\": self.optimizer.state_dict(),\n                \"best_valid_score\": self.best_valid_score,\n                \"n_epoch\": n_epoch,\n            },\n            self.lastmodel,\n        )\n    \n    @staticmethod\n    def info_message(message, *args, end=\"\\n\"):\n        print(message.format(*args), end=end)","metadata":{"papermill":{"duration":0.637077,"end_time":"2021-07-14T20:27:58.09407","exception":false,"start_time":"2021-07-14T20:27:57.456993","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-02T17:10:14.875606Z","iopub.execute_input":"2023-04-02T17:10:14.876477Z","iopub.status.idle":"2023-04-02T17:10:14.905482Z","shell.execute_reply.started":"2023-04-02T17:10:14.876436Z","shell.execute_reply":"2023-04-02T17:10:14.904163Z"}}},{"cell_type":"markdown","source":"acc_list = []\nfor epoch in range(num_epochs):\n    num_epochs = 10\n    train_acc, train_loss = train_epoch(model, train_iterator, optimizer, criterion)\n    valid_acc, valid_loss = evaluate(model, valid_iterator, criterion)\n    acc_list.append(valid_acc) ","metadata":{"execution":{"iopub.status.busy":"2023-04-01T20:05:08.200312Z","iopub.execute_input":"2023-04-01T20:05:08.201128Z","iopub.status.idle":"2023-04-01T20:05:08.22095Z","shell.execute_reply.started":"2023-04-01T20:05:08.201089Z","shell.execute_reply":"2023-04-01T20:05:08.219341Z"}}},{"cell_type":"markdown","source":"## train models","metadata":{}},{"cell_type":"markdown","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\ndef calculate_accuracy(model, data_loader):\n    model.eval()\n    correct = 0\n    total = 0\n    with torch.no_grad():\n        for images, labels in data_loader:\n            images = [float(i) for i in images]\n            labels = [int(l) for l in labels]\n            images = torch.tensor(images).to(device)\n            labels = torch.tensor(labels).to(device)\n            outputs = model(images)\n            predicted = torch.max(outputs, 1)\n            total += labels.size(0)\n            correct += (predicted == labels).sum().item()\n    accuracy = correct / total\n    return accuracy","metadata":{"execution":{"iopub.status.busy":"2023-04-01T22:04:18.4274Z","iopub.execute_input":"2023-04-01T22:04:18.428007Z","iopub.status.idle":"2023-04-01T22:04:18.440796Z","shell.execute_reply.started":"2023-04-01T22:04:18.427903Z","shell.execute_reply":"2023-04-01T22:04:18.439693Z"}}},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\ndef train_mri_type(df_train, df_valid, mri_type):\n    if mri_type==\"all\":\n        train_list = []\n        valid_list = []\n        for mri_type in mri_types:\n            df_train.loc[:,\"MRI_Type\"] = mri_type\n            train_list.append(df_train.copy())\n            df_valid.loc[:,\"MRI_Type\"] = mri_type\n            valid_list.append(df_valid.copy())\n\n        df_train = pd.concat(train_list)\n        df_valid = pd.concat(valid_list)\n    else:\n        df_train.loc[:,\"MRI_Type\"] = mri_type\n        df_valid.loc[:,\"MRI_Type\"] = mri_type\n\n    print(df_train.shape, df_valid.shape)\n    display(df_train.head())\n    \n    train_data_retriever = Dataset(\n        df_train[\"BraTS21ID\"].values, \n        df_train[\"MGMT_value\"].values, \n        df_train[\"MRI_Type\"].values,\n        augment=True\n    )\n\n    valid_data_retriever = Dataset(\n        df_valid[\"BraTS21ID\"].values, \n        df_valid[\"MGMT_value\"].values,\n        df_valid[\"MRI_Type\"].values\n    )\n\n    train_loader = torch_data.DataLoader(\n        train_data_retriever,\n        batch_size=4,\n        shuffle=True,\n        num_workers=8,pin_memory = True\n    )\n\n    valid_loader = torch_data.DataLoader(\n        valid_data_retriever, \n        batch_size=4,\n        shuffle=False,\n        num_workers=8,pin_memory = True\n    )\n\n    model = Model(\n        image_size = 256,\n        patch_size = 32,\n        num_classes = 1,\n        dim = 1024,\n        depth = 2,\n        heads = 16,\n        mlp_dim = 2048,\n        channels = 1,\n        dropout = 0.1,\n        emb_dropout = 0.1\n    )\n    model.to(device)\n\n    optimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n\n    criterion = torch_functional.binary_cross_entropy_with_logits\n\n    trainer = Trainer(\n        model, \n        device, \n        optimizer, \n        criterion\n    )\n\n    history = trainer.fit(\n            2, \n            train_loader, \n            valid_loader, \n            mri_type, \n            10,\n    )\n    \n    return trainer.lastmodel\n\nmodelfiles = None\n\nif not modelfiles:\n    modelfiles = [train_mri_type(df_train[:30], df_valid[:30], m) for m in mri_types[:]]\n    print(modelfiles)","metadata":{"lines_to_next_cell":2,"papermill":{"duration":447.387602,"end_time":"2021-07-14T20:35:26.110421","exception":false,"start_time":"2021-07-14T20:27:58.722819","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Ensemble for validation","metadata":{}},{"cell_type":"code","source":"df_valid = df_valid.set_index(\"BraTS21ID\")\ndf_valid[\"MGMT_pred\"] = 0\nfor m, mtype in zip(modelfiles,  mri_types):\n    pred = predict(m, df_valid, mtype, \"train\")\n    df_valid[\"MGMT_pred\"] += pred[\"MGMT_value\"]\ndf_valid[\"MGMT_pred\"] /= len(modelfiles)\nauc = roc_auc_score(df_valid[\"MGMT_value\"], df_valid[\"MGMT_pred\"])\nprint(f\"Validation ensemble AUC: {auc:.4f}\")\nsns.displot(df_valid[\"MGMT_pred\"])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Ensemble for submission","metadata":{}},{"cell_type":"code","source":"submission = pd.read_csv(f\"{data_directory}/sample_submission.csv\", index_col=\"BraTS21ID\")\n\nsubmission[\"MGMT_value\"] = 0\nfor m, mtype in zip(modelfiles, mri_types):\n    pred = predict(m, submission, mtype, split=\"test\")\n    submission[\"MGMT_value\"] += pred[\"MGMT_value\"]\n\nsubmission[\"MGMT_value\"] /= len(modelfiles)\nsubmission[\"MGMT_value\"].to_csv(\"submission.csv\")","metadata":{"papermill":{"duration":0.990911,"end_time":"2021-07-14T20:35:30.482254","exception":false,"start_time":"2021-07-14T20:35:29.491343","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"papermill":{"duration":0.972383,"end_time":"2021-07-14T20:35:41.939464","exception":false,"start_time":"2021-07-14T20:35:40.967081","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.displot(submission[\"MGMT_value\"])","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}