{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Combining \"net_3D\" and \"net_0\" models\n\n## This notebook takes the first efficientnet3D model we downloaded and combines the prediction with the efficientnet0 model. \n\n## \"net_3D\" takes stacked 3D images and the Efficientnet3D architecture, runs each MRI type (4) and then equally weight ensembles those models. This takes ~ 4 hours to train. \n\n## \"net_0\" takes ...\n\n## This notebook ensembles these 5 models\n\n","metadata":{}},{"cell_type":"markdown","source":"# \"net_3D\"\n\n## Use stacked images (3D) and Efficientnet3D model\n\n# Taken online from leaderboard for eval\n\nAcknowledgements:\n\n- https://www.kaggle.com/ihelon/brain-tumor-eda-with-animations-and-modeling\n- https://www.kaggle.com/furcifer/torch-efficientnet3d-for-mri-no-train\n- https://github.com/shijianjian/EfficientNet-PyTorch-3D\n    \n    \nUse models with only one MRI type, then ensemble the 4 models \n\nV14: add image rotation augmentation","metadata":{}},{"cell_type":"code","source":"import os\nimport sys \nimport json\nimport glob\nimport random\nimport collections\nimport time\nimport re\n\nimport numpy as np\nimport pandas as pd\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nimport cv2\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport torch\nfrom torch import nn\nfrom torch.utils import data as torch_data\nfrom sklearn import model_selection as sk_model_selection\nfrom torch.nn import functional as torch_functional\nimport torch.nn.functional as F\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import roc_auc_score","metadata":{"papermill":{"duration":1.048295,"end_time":"2021-07-14T20:26:46.309722","exception":false,"start_time":"2021-07-14T20:26:45.261427","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-24T21:51:48.935859Z","iopub.execute_input":"2021-08-24T21:51:48.936254Z","iopub.status.idle":"2021-08-24T21:51:51.824366Z","shell.execute_reply.started":"2021-08-24T21:51:48.936164Z","shell.execute_reply":"2021-08-24T21:51:51.823383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if os.path.exists(\"../input/rsna-miccai-brain-tumor-radiogenomic-classification\"):\n    data_directory = '../input/rsna-miccai-brain-tumor-radiogenomic-classification'\n    pytorch3dpath = \"../input/efficientnetpyttorch3d/EfficientNet-PyTorch-3D\"\nelse:\n    data_directory = '/media/roland/data/kaggle/rsna-miccai-brain-tumor-radiogenomic-classification'\n    pytorch3dpath = \"EfficientNet-PyTorch-3D\"\n    \nmri_types = ['FLAIR']#,'T1w']#,'T1wCE','T2w']\nSIZE = 256\nNUM_IMAGES = 64\n\nsys.path.append(pytorch3dpath)\nfrom efficientnet_pytorch_3d import EfficientNet3D","metadata":{"lines_to_end_of_cell_marker":2,"lines_to_next_cell":2,"papermill":{"duration":0.05565,"end_time":"2021-07-14T20:26:46.486521","exception":false,"start_time":"2021-07-14T20:26:46.430871","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-24T21:51:51.825951Z","iopub.execute_input":"2021-08-24T21:51:51.826336Z","iopub.status.idle":"2021-08-24T21:51:51.864373Z","shell.execute_reply.started":"2021-08-24T21:51:51.826295Z","shell.execute_reply":"2021-08-24T21:51:51.863451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Functions to load images","metadata":{}},{"cell_type":"code","source":"def load_dicom_image(path, img_size=SIZE, voi_lut=True, rotate=0):\n    dicom = pydicom.read_file(path)\n    data = dicom.pixel_array\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n        \n    if rotate > 0:\n        rot_choices = [0, cv2.ROTATE_90_CLOCKWISE, cv2.ROTATE_90_COUNTERCLOCKWISE, cv2.ROTATE_180]\n        data = cv2.rotate(data, rot_choices[rotate])\n        \n    data = cv2.resize(data, (img_size, img_size))\n    return data\n\n\ndef load_dicom_images_3d(scan_id, num_imgs=NUM_IMAGES, img_size=SIZE, mri_type=\"FLAIR\", split=\"train\", rotate=0):\n\n    files = sorted(glob.glob(f\"{data_directory}/{split}/{scan_id}/{mri_type}/*.dcm\"), \n               key=lambda var:[int(x) if x.isdigit() else x for x in re.findall(r'[^0-9]|[0-9]+', var)])\n\n    middle = len(files)//2\n    num_imgs2 = num_imgs//2\n    p1 = max(0, middle - num_imgs2)\n    p2 = min(len(files), middle + num_imgs2)\n    img3d = np.stack([load_dicom_image(f, rotate=rotate) for f in files[p1:p2]]).T \n    if img3d.shape[-1] < num_imgs:\n        n_zero = np.zeros((img_size, img_size, num_imgs - img3d.shape[-1]))\n        img3d = np.concatenate((img3d,  n_zero), axis = -1)\n        \n    if np.min(img3d) < np.max(img3d):\n        img3d = img3d - np.min(img3d)\n        img3d = img3d / np.max(img3d)\n            \n    return np.expand_dims(img3d,0)\n\na = load_dicom_images_3d(\"00019\")\nprint(a.shape)\nprint(np.min(a), np.max(a), np.mean(a), np.median(a))","metadata":{"papermill":{"duration":0.035761,"end_time":"2021-07-14T20:26:46.726756","exception":false,"start_time":"2021-07-14T20:26:46.690995","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-24T21:51:51.868277Z","iopub.execute_input":"2021-08-24T21:51:51.868591Z","iopub.status.idle":"2021-08-24T21:51:52.838648Z","shell.execute_reply.started":"2021-08-24T21:51:51.868547Z","shell.execute_reply":"2021-08-24T21:51:52.837593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed_all(seed)\n        torch.backends.cudnn.deterministic = True\n\nset_seed(12)","metadata":{"papermill":{"duration":0.668331,"end_time":"2021-07-14T20:27:48.114522","exception":false,"start_time":"2021-07-14T20:27:47.446191","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-24T21:51:52.840291Z","iopub.execute_input":"2021-08-24T21:51:52.840840Z","iopub.status.idle":"2021-08-24T21:51:52.905710Z","shell.execute_reply.started":"2021-08-24T21:51:52.840797Z","shell.execute_reply":"2021-08-24T21:51:52.904729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## train / test splits","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import KFold\nfrom sklearn.model_selection import StratifiedKFold\ntrain_df = pd.read_csv(f\"{data_directory}/train_labels.csv\")\n# kf = KFold(n_splits=5, random_state=42) #5 fold \nkf = StratifiedKFold(n_splits=5, random_state=42) #5 fold \n\nindex_list = list(kf.split(train_df, train_df[\"MGMT_value\"]))\n#to access n-th fold, change the n in the index_list[n] below: \ntrain_idx, val_idx = index_list[0] \ndf_train = train_df.iloc[train_idx] \ndf_valid = train_df.iloc[val_idx]\nprint(df_train.shape, df_valid.shape)","metadata":{"execution":{"iopub.status.busy":"2021-08-24T22:25:52.400871Z","iopub.execute_input":"2021-08-24T22:25:52.401269Z","iopub.status.idle":"2021-08-24T22:25:52.417265Z","shell.execute_reply.started":"2021-08-24T22:25:52.401236Z","shell.execute_reply":"2021-08-24T22:25:52.416062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-24T22:25:55.547303Z","iopub.execute_input":"2021-08-24T22:25:55.547631Z","iopub.status.idle":"2021-08-24T22:25:55.557031Z","shell.execute_reply.started":"2021-08-24T22:25:55.547601Z","shell.execute_reply":"2021-08-24T22:25:55.556071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # train_df = pd.read_csv(f\"{data_directory}/train_labels.csv\")\n# # display(train_df)\n\n# df_train, df_valid = sk_model_selection.train_test_split(\n#     train_df, \n#     test_size=0.2, \n#     random_state=12, \n#     stratify=train_df[\"MGMT_value\"],\n# )\n","metadata":{"papermill":{"duration":0.633753,"end_time":"2021-07-14T20:27:49.350524","exception":false,"start_time":"2021-07-14T20:27:48.716771","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-24T21:51:52.907112Z","iopub.execute_input":"2021-08-24T21:51:52.907508Z","iopub.status.idle":"2021-08-24T21:51:52.955913Z","shell.execute_reply.started":"2021-08-24T21:51:52.907468Z","shell.execute_reply":"2021-08-24T21:51:52.954942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_valid.tail()","metadata":{"execution":{"iopub.status.busy":"2021-08-24T22:26:35.046339Z","iopub.execute_input":"2021-08-24T22:26:35.046698Z","iopub.status.idle":"2021-08-24T22:26:35.058045Z","shell.execute_reply.started":"2021-08-24T22:26:35.046668Z","shell.execute_reply":"2021-08-24T22:26:35.056734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model and training classes","metadata":{}},{"cell_type":"code","source":"class Dataset(torch_data.Dataset):\n    def __init__(self, paths, targets=None, mri_type=None, label_smoothing=0.01, split=\"train\", augment=False):\n        self.paths = paths\n        self.targets = targets\n        self.mri_type = mri_type\n        self.label_smoothing = label_smoothing\n        self.split = split\n        self.augment = augment\n          \n    def __len__(self):\n        return len(self.paths)\n    \n    def __getitem__(self, index):\n        scan_id = self.paths[index]\n        if self.targets is None:\n            data = load_dicom_images_3d(str(scan_id).zfill(5), mri_type=self.mri_type[index], split=self.split)\n        else:\n            if self.augment:\n                rotation = np.random.randint(0,4)\n            else:\n                rotation = 0\n\n            data = load_dicom_images_3d(str(scan_id).zfill(5), mri_type=self.mri_type[index], split=\"train\", rotate=rotation)\n\n        if self.targets is None:\n            return {\"X\": torch.tensor(data).float(), \"id\": scan_id}\n        else:\n            y = torch.tensor(abs(self.targets[index]-self.label_smoothing), dtype=torch.float)\n            return {\"X\": torch.tensor(data).float(), \"y\": y}\n","metadata":{"papermill":{"duration":0.634322,"end_time":"2021-07-14T20:27:50.594701","exception":false,"start_time":"2021-07-14T20:27:49.960379","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-24T21:51:58.872468Z","iopub.execute_input":"2021-08-24T21:51:58.872805Z","iopub.status.idle":"2021-08-24T21:51:58.884187Z","shell.execute_reply.started":"2021-08-24T21:51:58.872775Z","shell.execute_reply":"2021-08-24T21:51:58.883102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Model(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.net = EfficientNet3D.from_name(\"efficientnet-b0\", override_params={'num_classes': 2}, in_channels=1)\n        n_features = self.net._fc.in_features\n        self.net._fc = nn.Linear(in_features=n_features, out_features=1, bias=True)\n    \n    def forward(self, x):\n        out = self.net(x)\n        return out\n    ","metadata":{"papermill":{"duration":0.825458,"end_time":"2021-07-14T20:27:55.604161","exception":false,"start_time":"2021-07-14T20:27:54.778703","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-24T21:51:59.579868Z","iopub.execute_input":"2021-08-24T21:51:59.580268Z","iopub.status.idle":"2021-08-24T21:51:59.587240Z","shell.execute_reply.started":"2021-08-24T21:51:59.580237Z","shell.execute_reply":"2021-08-24T21:51:59.586169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Trainer:\n    def __init__(\n        self, \n        model, \n        device, \n        optimizer, \n        criterion\n    ):\n        self.model = model\n        self.device = device\n        self.optimizer = optimizer\n        self.criterion = criterion\n\n        self.best_valid_score = np.inf\n        self.n_patience = 0\n        self.lastmodel = None\n        \n    def fit(self, epochs, train_loader, valid_loader, save_path, patience):        \n        for n_epoch in range(1, epochs + 1):\n            self.info_message(\"EPOCH: {}\", n_epoch)\n            \n            train_loss, train_time = self.train_epoch(train_loader)\n            valid_loss, valid_auc, valid_time = self.valid_epoch(valid_loader)\n            \n            self.info_message(\n                \"[Epoch Train: {}] loss: {:.4f}, time: {:.2f} s            \",\n                n_epoch, train_loss, train_time\n            )\n            \n            self.info_message(\n                \"[Epoch Valid: {}] loss: {:.4f}, auc: {:.4f}, time: {:.2f} s\",\n                n_epoch, valid_loss, valid_auc, valid_time\n            )\n\n            # if True:\n            # if self.best_valid_score < valid_auc: \n            if self.best_valid_score > valid_loss: \n                self.save_model(n_epoch, save_path, valid_loss, valid_auc)\n                self.info_message(\n                     \"auc improved from {:.4f} to {:.4f}. Saved model to '{}'\", \n                    self.best_valid_score, valid_loss, self.lastmodel\n                )\n                self.best_valid_score = valid_loss\n                self.n_patience = 0\n            else:\n                self.n_patience += 1\n            \n            if self.n_patience >= patience:\n                self.info_message(\"\\nValid auc didn't improve last {} epochs.\", patience)\n                break\n            \n    def train_epoch(self, train_loader):\n        self.model.train()\n        t = time.time()\n        sum_loss = 0\n\n        for step, batch in enumerate(train_loader, 1):\n            X = batch[\"X\"].to(self.device)\n            targets = batch[\"y\"].to(self.device)\n            self.optimizer.zero_grad()\n            outputs = self.model(X).squeeze(1)\n            \n            loss = self.criterion(outputs, targets)\n            loss.backward()\n\n            sum_loss += loss.detach().item()\n\n            self.optimizer.step()\n            \n            message = 'Train Step {}/{}, train_loss: {:.4f}'\n            self.info_message(message, step, len(train_loader), sum_loss/step, end=\"\\r\")\n        \n        return sum_loss/len(train_loader), int(time.time() - t)\n    \n    def valid_epoch(self, valid_loader):\n        self.model.eval()\n        t = time.time()\n        sum_loss = 0\n        y_all = []\n        outputs_all = []\n\n        for step, batch in enumerate(valid_loader, 1):\n            with torch.no_grad():\n                X = batch[\"X\"].to(self.device)\n                targets = batch[\"y\"].to(self.device)\n\n                outputs = self.model(X).squeeze(1)\n                loss = self.criterion(outputs, targets)\n\n                sum_loss += loss.detach().item()\n                y_all.extend(batch[\"y\"].tolist())\n                outputs_all.extend(outputs.tolist())\n\n            message = 'Valid Step {}/{}, valid_loss: {:.4f}'\n            self.info_message(message, step, len(valid_loader), sum_loss/step, end=\"\\r\")\n            \n        y_all = [1 if x > 0.5 else 0 for x in y_all]\n        auc = roc_auc_score(y_all, outputs_all)\n        \n        return sum_loss/len(valid_loader), auc, int(time.time() - t)\n    \n    def save_model(self, n_epoch, save_path, loss, auc):\n        self.lastmodel = f\"{save_path}-e{n_epoch}-loss{loss:.3f}-auc{auc:.3f}.pth\"\n        torch.save(\n            {\n                \"model_state_dict\": self.model.state_dict(),\n                \"optimizer_state_dict\": self.optimizer.state_dict(),\n                \"best_valid_score\": self.best_valid_score,\n                \"n_epoch\": n_epoch,\n            },\n            self.lastmodel,\n        )\n    \n    @staticmethod\n    def info_message(message, *args, end=\"\\n\"):\n        print(message.format(*args), end=end)","metadata":{"papermill":{"duration":0.637077,"end_time":"2021-07-14T20:27:58.09407","exception":false,"start_time":"2021-07-14T20:27:57.456993","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-24T21:52:00.400204Z","iopub.execute_input":"2021-08-24T21:52:00.400545Z","iopub.status.idle":"2021-08-24T21:52:00.423095Z","shell.execute_reply.started":"2021-08-24T21:52:00.400516Z","shell.execute_reply":"2021-08-24T21:52:00.421927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## train models","metadata":{}},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\ndef train_mri_type(df_train, df_valid, mri_type):\n    if mri_type==\"all\":\n        train_list = []\n        valid_list = []\n        for mri_type in mri_types:\n            df_train.loc[:,\"MRI_Type\"] = mri_type\n            train_list.append(df_train.copy())\n            df_valid.loc[:,\"MRI_Type\"] = mri_type\n            valid_list.append(df_valid.copy())\n\n        df_train = pd.concat(train_list)\n        df_valid = pd.concat(valid_list)\n    else:\n        df_train.loc[:,\"MRI_Type\"] = mri_type\n        df_valid.loc[:,\"MRI_Type\"] = mri_type\n\n    print(df_train.shape, df_valid.shape)\n    display(df_train.head())\n    \n    train_data_retriever = Dataset(\n        df_train[\"BraTS21ID\"].values, \n        df_train[\"MGMT_value\"].values, \n        df_train[\"MRI_Type\"].values,\n        augment=True\n    )\n\n    valid_data_retriever = Dataset(\n        df_valid[\"BraTS21ID\"].values, \n        df_valid[\"MGMT_value\"].values,\n        df_valid[\"MRI_Type\"].values\n    )\n\n    train_loader = torch_data.DataLoader(\n        train_data_retriever,\n        batch_size=4,\n        shuffle=True,\n        num_workers=8,\n    )\n\n    valid_loader = torch_data.DataLoader(\n        valid_data_retriever, \n        batch_size=4,\n        shuffle=False,\n        num_workers=8,\n    )\n\n    model = Model()\n    model.to(device)\n\n    #checkpoint = torch.load(\"best-model-all-auc0.555.pth\")\n    #model.load_state_dict(checkpoint[\"model_state_dict\"])\n\n    #print(model)\n\n    optimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n    #optimizer = torch.optim.SGD(model.parameters(), lr=0.001, momentum=0.9)\n\n    criterion = torch_functional.binary_cross_entropy_with_logits\n\n    trainer = Trainer(\n        model, \n        device, \n        optimizer, \n        criterion\n    )\n\n    history = trainer.fit(\n        10, \n        train_loader, \n        valid_loader, \n        f\"{mri_type}\", \n        10,\n    )\n    \n    return trainer.lastmodel\n\nmodelfiles = None\n\nif not modelfiles:\n    modelfiles = [train_mri_type(df_train, df_valid, m) for m in mri_types]\n    print(modelfiles)","metadata":{"lines_to_next_cell":2,"papermill":{"duration":447.387602,"end_time":"2021-07-14T20:35:26.110421","exception":false,"start_time":"2021-07-14T20:27:58.722819","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-24T21:52:02.835569Z","iopub.execute_input":"2021-08-24T21:52:02.835919Z","iopub.status.idle":"2021-08-24T21:58:53.427128Z","shell.execute_reply.started":"2021-08-24T21:52:02.835888Z","shell.execute_reply":"2021-08-24T21:58:53.424125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Predict function","metadata":{}},{"cell_type":"code","source":"def predict(modelfile, df, mri_type, split):\n    print(\"Predict:\", modelfile, mri_type, df.shape)\n    df.loc[:,\"MRI_Type\"] = mri_type\n    data_retriever = Dataset(\n        df.index.values, \n        mri_type=df[\"MRI_Type\"].values,\n        split=split\n    )\n\n    data_loader = torch_data.DataLoader(\n        data_retriever,\n        batch_size=4,\n        shuffle=False,\n        num_workers=8,\n    )\n   \n    model = Model()\n    model.to(device)\n    \n    checkpoint = torch.load(modelfile)\n    model.load_state_dict(checkpoint[\"model_state_dict\"])\n    model.eval()\n    \n    y_pred = []\n    ids = []\n\n    for e, batch in enumerate(data_loader,1):\n        print(f\"{e}/{len(data_loader)}\", end=\"\\r\")\n        with torch.no_grad():\n            tmp_pred = torch.sigmoid(model(batch[\"X\"].to(device))).cpu().numpy().squeeze()\n            if tmp_pred.size == 1:\n                y_pred.append(tmp_pred)\n            else:\n                y_pred.extend(tmp_pred.tolist())\n            ids.extend(batch[\"id\"].numpy().tolist())\n            \n    preddf = pd.DataFrame({\"BraTS21ID\": ids, \"MGMT_value\": y_pred}) \n    preddf = preddf.set_index(\"BraTS21ID\")\n    return preddf","metadata":{"papermill":{"duration":0.990911,"end_time":"2021-07-14T20:35:30.482254","exception":false,"start_time":"2021-07-14T20:35:29.491343","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-24T21:58:53.428422Z","iopub.status.idle":"2021-08-24T21:58:53.428878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Ensemble for validation - \"net_3D\"","metadata":{}},{"cell_type":"code","source":"df_valid = df_valid.set_index(\"BraTS21ID\")\ndf_valid[\"MGMT_pred\"] = 0\nfor m, mtype in zip(modelfiles,  mri_types):\n    pred = predict(m, df_valid, mtype, \"train\")\n    df_valid[\"MGMT_pred\"] += pred[\"MGMT_value\"]\ndf_valid[\"MGMT_pred\"] /= len(modelfiles)\nauc = roc_auc_score(df_valid[\"MGMT_value\"], df_valid[\"MGMT_pred\"])\nprint(f\"Validation ensemble AUC: {auc:.4f}\")\nsns.displot(df_valid[\"MGMT_pred\"])","metadata":{"execution":{"iopub.status.busy":"2021-08-24T21:58:53.430101Z","iopub.status.idle":"2021-08-24T21:58:53.430661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv(f\"{data_directory}/sample_submission.csv\", index_col=\"BraTS21ID\")\n\nsubmission[\"MGMT_value\"] = 0\nfor m, mtype in zip(modelfiles, mri_types):\n    pred = predict(m, submission, mtype, split=\"test\")\n    submission[\"MGMT_value\"] += pred[\"MGMT_value\"]","metadata":{"execution":{"iopub.status.busy":"2021-08-18T22:27:19.021536Z","iopub.execute_input":"2021-08-18T22:27:19.021897Z","iopub.status.idle":"2021-08-18T22:27:50.529546Z","shell.execute_reply.started":"2021-08-18T22:27:19.021854Z","shell.execute_reply":"2021-08-18T22:27:50.528558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# \"net_0\"","metadata":{}},{"cell_type":"code","source":"import os\nimport json\nimport glob\nimport random\nimport collections\n\nimport numpy as np\nimport pandas as pd\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nimport cv2\nimport matplotlib.pyplot as plt\nimport seaborn as sns\npackage_path = \"../input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master/\"\nimport sys \nsys.path.append(package_path)\n\nimport time\n\nimport torch\nfrom torch import nn\nfrom torch.utils import data as torch_data\nfrom sklearn import model_selection as sk_model_selection\nfrom torch.nn import functional as torch_functional\nimport efficientnet_pytorch\n\nfrom sklearn.model_selection import StratifiedKFold","metadata":{"execution":{"iopub.status.busy":"2021-08-23T17:41:35.189226Z","iopub.execute_input":"2021-08-23T17:41:35.189643Z","iopub.status.idle":"2021-08-23T17:41:35.197506Z","shell.execute_reply.started":"2021-08-23T17:41:35.189585Z","shell.execute_reply":"2021-08-23T17:41:35.196156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(f\"{data_directory}/train_labels.csv\")\ndisplay(train_df)\n\ndf_train, df_valid = sk_model_selection.train_test_split(\n    train_df, \n    test_size=0.2, \n    random_state=12,  #change the random state here\n    stratify=train_df[\"MGMT_value\"],\n)","metadata":{"execution":{"iopub.status.busy":"2021-08-23T17:41:36.821183Z","iopub.execute_input":"2021-08-23T17:41:36.821595Z","iopub.status.idle":"2021-08-23T17:41:36.870313Z","shell.execute_reply.started":"2021-08-23T17:41:36.821561Z","shell.execute_reply":"2021-08-23T17:41:36.869294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DataRetriever(torch_data.Dataset):\n    def __init__(self, paths, targets):\n        self.paths = paths\n        self.targets = targets\n          \n    def __len__(self):\n        return len(self.paths)\n    \n    def __getitem__(self, index):\n        _id = self.paths[index]\n        patient_path = f\"../input/rsna-miccai-brain-tumor-radiogenomic-classification/train/{str(_id).zfill(5)}/\"\n        channels = []\n        for t in (\"FLAIR\", \"T1w\",  \"T1wCE\"): #,): # \"T2w\" \n            t_paths = sorted(\n                glob.glob(os.path.join(patient_path, t, \"*\")), \n                key=lambda x: int(x[:-4].split(\"-\")[-1]),\n            )\n            # start, end = int(len(t_paths) * 0.475), int(len(t_paths) * 0.525)\n            x = len(t_paths)\n            if x < 10:\n                r = range(x)\n            else:\n                d = x // 10\n                r = range(d, x - d, d)\n                \n            channel = []\n            # for i in range(start, end + 1):\n            for i in r:\n                channel.append(cv2.resize(load_dicom(t_paths[i]), (256, 256)) / 255)\n            channel = np.mean(channel, axis=0)\n            channels.append(channel)\n            \n        y = torch.tensor(self.targets[index], dtype=torch.float)\n        \n        return {\"X\": torch.tensor(channels).float(), \"y\": y}\n    \n    #df_train = train_df\n\ntrain_data_retriever = DataRetriever(\n    df_train[\"BraTS21ID\"].values, \n    df_train[\"MGMT_value\"].values, \n)\n\nvalid_data_retriever = DataRetriever(\n    df_valid[\"BraTS21ID\"].values, \n    df_valid[\"MGMT_value\"].values,\n)\n\nclass Model(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.net = efficientnet_pytorch.EfficientNet.from_name(\"efficientnet-b2\") #b0\n        checkpoint = torch.load(\"../input/efficientnet-pytorch/efficientnet-b2-27687264.pth\")\n        #checkpoint = torch.load(\"../input/efficientnet-pytorch/efficientnet-b0-08094119.pth\")\n        self.net.load_state_dict(checkpoint)\n        n_features = self.net._fc.in_features\n        self.net._fc = nn.Linear(in_features=n_features, out_features=1, bias=True)\n    \n    def forward(self, x):\n        out = self.net(x)\n        return out\n    \nclass LossMeter:\n    def __init__(self):\n        self.avg = 0\n        self.n = 0\n\n    def update(self, val):\n        self.n += 1\n        # incremental update\n        self.avg = val / self.n + (self.n - 1) / self.n * self.avg\n\n        \nclass AccMeter:\n    def __init__(self):\n        self.avg = 0\n        self.n = 0\n        \n    def update(self, y_true, y_pred):\n        y_true = y_true.cpu().numpy().astype(int)\n        y_pred = y_pred.cpu().numpy() >= 0\n        last_n = self.n\n        self.n += len(y_true)\n        true_count = np.sum(y_true == y_pred)\n        # incremental update\n        self.avg = true_count / self.n + last_n / self.n * self.avg\n        \nclass Trainer:\n    def __init__(\n        self, \n        model, \n        device, \n        optimizer, \n        criterion, \n        loss_meter, \n        score_meter\n    ):\n        self.model = model\n        self.device = device\n        self.optimizer = optimizer\n        self.criterion = criterion\n        self.loss_meter = loss_meter\n        self.score_meter = score_meter\n        \n        self.best_valid_score = -np.inf\n        self.n_patience = 0\n        \n        self.messages = {\n            \"epoch\": \"[Epoch {}: {}] loss: {:.5f}, score: {:.5f}, time: {} s\",\n            \"checkpoint\": \"The score improved from {:.5f} to {:.5f}. Save model to '{}'\",\n            \"patience\": \"\\nValid score didn't improve last {} epochs.\"\n        }\n    \n    def fit(self, epochs, train_loader, valid_loader, save_path, patience):        \n        for n_epoch in range(1, epochs + 1):\n            self.info_message(\"EPOCH: {}\", n_epoch)\n            \n            train_loss, train_score, train_time = self.train_epoch(train_loader)\n            valid_loss, valid_score, valid_time = self.valid_epoch(valid_loader)\n            \n            self.info_message(\n                self.messages[\"epoch\"], \"Train\", n_epoch, train_loss, train_score, train_time\n            )\n            \n            self.info_message(\n                self.messages[\"epoch\"], \"Valid\", n_epoch, valid_loss, valid_score, valid_time\n            )\n\n            if True:\n#             if self.best_valid_score < valid_score:\n                self.info_message(\n                    self.messages[\"checkpoint\"], self.best_valid_score, valid_score, save_path\n                )\n                self.best_valid_score = valid_score\n                self.save_model(n_epoch, save_path)\n                self.n_patience = 0\n            else:\n                self.n_patience += 1\n            \n            if self.n_patience >= patience:\n                self.info_message(self.messages[\"patience\"], patience)\n                break\n            \n    def train_epoch(self, train_loader):\n        self.model.train()\n        t = time.time()\n        train_loss = self.loss_meter()\n        train_score = self.score_meter()\n        \n        for step, batch in enumerate(train_loader, 1):\n            X = batch[\"X\"].to(self.device)\n            targets = batch[\"y\"].to(self.device)\n            self.optimizer.zero_grad()\n            outputs = self.model(X).squeeze(1)\n            \n            loss = self.criterion(outputs, targets)\n            loss.backward()\n\n            train_loss.update(loss.detach().item())\n            train_score.update(targets, outputs.detach())\n\n            self.optimizer.step()\n            \n            _loss, _score = train_loss.avg, train_score.avg\n            message = 'Train Step {}/{}, train_loss: {:.5f}, train_score: {:.5f}'\n            self.info_message(message, step, len(train_loader), _loss, _score, end=\"\\r\")\n        \n        return train_loss.avg, train_score.avg, int(time.time() - t)\n    \n    def valid_epoch(self, valid_loader):\n        self.model.eval()\n        t = time.time()\n        valid_loss = self.loss_meter()\n        valid_score = self.score_meter()\n\n        for step, batch in enumerate(valid_loader, 1):\n            with torch.no_grad():\n                X = batch[\"X\"].to(self.device)\n                targets = batch[\"y\"].to(self.device)\n\n                outputs = self.model(X).squeeze(1)\n                loss = self.criterion(outputs, targets)\n\n                valid_loss.update(loss.detach().item())\n                valid_score.update(targets, outputs)\n                \n            _loss, _score = valid_loss.avg, valid_score.avg\n            message = 'Valid Step {}/{}, valid_loss: {:.5f}, valid_score: {:.5f}'\n            self.info_message(message, step, len(valid_loader), _loss, _score, end=\"\\r\")\n        \n        return valid_loss.avg, valid_score.avg, int(time.time() - t)\n    \n    def save_model(self, n_epoch, save_path):\n        torch.save(\n            {\n                \"model_state_dict\": self.model.state_dict(),\n                \"optimizer_state_dict\": self.optimizer.state_dict(),\n                \"best_valid_score\": self.best_valid_score,\n                \"n_epoch\": n_epoch,\n            },\n            save_path,\n        )\n    \n    @staticmethod\n    def info_message(message, *args, end=\"\\n\"):\n        print(message.format(*args), end=end)\n\ndef load_dicom(path):\n    dicom = pydicom.read_file(path)\n    data = dicom.pixel_array\n    data = data - np.min(data)\n    if np.max(data) != 0:\n        data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    return data","metadata":{"execution":{"iopub.status.busy":"2021-08-23T17:41:40.626837Z","iopub.execute_input":"2021-08-23T17:41:40.627237Z","iopub.status.idle":"2021-08-23T17:41:40.669742Z","shell.execute_reply.started":"2021-08-23T17:41:40.627204Z","shell.execute_reply":"2021-08-23T17:41:40.668588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\ntrain_data_retriever = DataRetriever(\n    df_train[\"BraTS21ID\"].values, \n    df_train[\"MGMT_value\"].values, \n)\n\nvalid_data_retriever = DataRetriever(\n    df_valid[\"BraTS21ID\"].values, \n    df_valid[\"MGMT_value\"].values,\n)\n\ntrain_loader = torch_data.DataLoader(\n    train_data_retriever,\n    batch_size=8,\n    shuffle=True,\n    num_workers=8,\n)\n\nvalid_loader = torch_data.DataLoader(\n    valid_data_retriever, \n    batch_size=8,\n    shuffle=False,\n    num_workers=8,\n)\n\nmodel = Model()\nmodel.to(device)\n\noptimizer = torch.optim.Adam(model.parameters(), lr=0.001)\ncriterion = torch_functional.binary_cross_entropy_with_logits\n\ntrainer = Trainer(\n    model, \n    device, \n    optimizer, \n    criterion, \n    LossMeter, \n    AccMeter\n)\n\nhistory = trainer.fit(\n    2, \n    train_loader, \n    valid_loader, \n    f\"best-model-0.pth\", \n    100,\n)\n","metadata":{"execution":{"iopub.status.busy":"2021-08-23T14:30:51.018913Z","iopub.execute_input":"2021-08-23T14:30:51.019234Z","iopub.status.idle":"2021-08-23T14:33:08.510929Z","shell.execute_reply.started":"2021-08-23T14:30:51.019202Z","shell.execute_reply":"2021-08-23T14:33:08.509751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = []\nfor i in range(1):\n    model = Model()\n    model.to(device)\n    \n    checkpoint = torch.load(f\"best-model-{i}.pth\")\n    model.load_state_dict(checkpoint[\"model_state_dict\"])\n    model.eval()\n    \n    models.append(model)","metadata":{"execution":{"iopub.status.busy":"2021-08-23T14:37:37.03346Z","iopub.execute_input":"2021-08-23T14:37:37.033836Z","iopub.status.idle":"2021-08-23T14:37:37.459138Z","shell.execute_reply.started":"2021-08-23T14:37:37.033798Z","shell.execute_reply":"2021-08-23T14:37:37.458242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DataRetriever(torch_data.Dataset):\n    def __init__(self, paths):\n        self.paths = paths\n          \n    def __len__(self):\n        return len(self.paths)\n    \n    def __getitem__(self, index):\n        _id = self.paths[index]\n        patient_path = f\"../input/rsna-miccai-brain-tumor-radiogenomic-classification/test/{str(_id).zfill(5)}/\"\n        channels = []\n        for t in (\"FLAIR\", \"T1w\", \"T1wCE\"): # \"T2w\"\n            t_paths = sorted(\n                glob.glob(os.path.join(patient_path, t, \"*\")), \n                key=lambda x: int(x[:-4].split(\"-\")[-1]),\n            )\n            # start, end = int(len(t_paths) * 0.475), int(len(t_paths) * 0.525)\n            x = len(t_paths)\n            if x < 10:\n                r = range(x)\n            else:\n                d = x // 10\n                r = range(d, x - d, d)\n                \n            channel = []\n            # for i in range(start, end + 1):\n            for i in r:\n                channel.append(cv2.resize(load_dicom(t_paths[i]), (256, 256)) / 255)\n            channel = np.mean(channel, axis=0)\n            channels.append(channel)\n        \n        return {\"X\": torch.tensor(channels).float(), \"id\": _id}","metadata":{"execution":{"iopub.status.busy":"2021-08-23T14:37:44.27981Z","iopub.execute_input":"2021-08-23T14:37:44.280133Z","iopub.status.idle":"2021-08-23T14:37:44.288847Z","shell.execute_reply.started":"2021-08-23T14:37:44.280102Z","shell.execute_reply":"2021-08-23T14:37:44.287881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_net0 = pd.read_csv(\"../input/rsna-miccai-brain-tumor-radiogenomic-classification/sample_submission.csv\")\n\ntest_data_retriever = DataRetriever(\n    submission_net0[\"BraTS21ID\"].values, \n)\n\ntest_loader = torch_data.DataLoader(\n    test_data_retriever,\n    batch_size=4,\n    shuffle=False,\n    num_workers=8,\n)","metadata":{"execution":{"iopub.status.busy":"2021-08-23T14:37:48.969955Z","iopub.execute_input":"2021-08-23T14:37:48.970264Z","iopub.status.idle":"2021-08-23T14:37:48.986263Z","shell.execute_reply.started":"2021-08-23T14:37:48.970234Z","shell.execute_reply":"2021-08-23T14:37:48.985454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = []\nids = []\n\nfor e, batch in enumerate(test_loader):\n    print(f\"{e}/{len(test_loader)}\", end=\"\\r\")\n    with torch.no_grad():\n        tmp_pred = np.zeros((batch[\"X\"].shape[0], ))\n        for model in models:\n            tmp_res = torch.sigmoid(model(batch[\"X\"].to(device))).cpu().numpy().squeeze()\n            tmp_pred += tmp_res\n        y_pred.extend(tmp_pred)\n        ids.extend(batch[\"id\"].numpy().tolist())\n        \nsubmission_net0 = pd.DataFrame({\"BraTS21ID\": ids, \"MGMT_value\": y_pred})\n#submission.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-08-23T14:37:51.011717Z","iopub.execute_input":"2021-08-23T14:37:51.012039Z","iopub.status.idle":"2021-08-23T14:38:01.320937Z","shell.execute_reply.started":"2021-08-23T14:37:51.012008Z","shell.execute_reply":"2021-08-23T14:38:01.319953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Ensemble for submission - \"Model_3D\" + \"Model_net0\"","metadata":{}},{"cell_type":"code","source":"#submission_ensemble = submission_net0\n#submission_ensemble.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-08-18T22:30:10.274856Z","iopub.execute_input":"2021-08-18T22:30:10.275224Z","iopub.status.idle":"2021-08-18T22:30:10.279777Z","shell.execute_reply.started":"2021-08-18T22:30:10.275179Z","shell.execute_reply":"2021-08-18T22:30:10.278422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_ensemble = submission_net0\n\ntemp = np.zeros((len(submission)))\nfor i in range(1,len(submission)):\n    temp[i] = (submission_net0.MGMT_value[i] + submission.MGMT_value[submission_net0.BraTS21ID[i]]) / 2\n    #submission_ensemble = pd.DataFrame({\"BraTS21ID\": ids, \"MGMT_value\": y_pred})\n\nsubmission_ensemble['MGMT_value'] = temp\n\nsubmission_ensemble.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-08-18T22:30:10.284223Z","iopub.execute_input":"2021-08-18T22:30:10.284585Z","iopub.status.idle":"2021-08-18T22:30:10.306007Z","shell.execute_reply.started":"2021-08-18T22:30:10.284559Z","shell.execute_reply":"2021-08-18T22:30:10.305236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2021-08-18T22:35:56.528192Z","iopub.execute_input":"2021-08-18T22:35:56.528672Z","iopub.status.idle":"2021-08-18T22:35:56.555113Z","shell.execute_reply.started":"2021-08-18T22:35:56.52863Z","shell.execute_reply":"2021-08-18T22:35:56.554124Z"},"trusted":true},"execution_count":null,"outputs":[]}]}