{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport sys \nimport json\nimport glob\nimport random\nimport collections\nimport time\nimport re\n\nimport numpy as np\nimport pandas as pd\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nimport cv2\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport torch\nfrom torch import nn\nfrom torch.utils import data as torch_data\nfrom sklearn import model_selection as sk_model_selection\nfrom torch.nn import functional as torch_functional\nimport torch.nn.functional as F\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import roc_auc_score","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-10-15T20:34:19.133071Z","iopub.execute_input":"2021-10-15T20:34:19.133623Z","iopub.status.idle":"2021-10-15T20:34:24.610825Z","shell.execute_reply.started":"2021-10-15T20:34:19.133514Z","shell.execute_reply":"2021-10-15T20:34:24.609846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_monaipath = \"../input/rsnamonai\"\nmonaipath = \"/kaggle/tmp/monai/\"","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:34:24.615932Z","iopub.execute_input":"2021-10-15T20:34:24.616413Z","iopub.status.idle":"2021-10-15T20:34:24.624157Z","shell.execute_reply.started":"2021-10-15T20:34:24.616373Z","shell.execute_reply":"2021-10-15T20:34:24.623435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p {monaipath}\n!cp -r {input_monaipath}/* {monaipath}","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:34:35.014235Z","iopub.execute_input":"2021-10-15T20:34:35.014734Z","iopub.status.idle":"2021-10-15T20:34:38.509170Z","shell.execute_reply.started":"2021-10-15T20:34:35.014695Z","shell.execute_reply":"2021-10-15T20:34:38.508009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mri_types = ['FLAIR', 'T1w', 'T1wCE', 'T2w']\nSIZE = 256\nNUM_IMAGES = 64\nBATCH_SIZE = 2\nN_EPOCHS = 12\nSEED = 42\nLEARNING_RATE = 0.0005\nLR_DECAY = 0.9\n\nsys.path.append(monaipath)\n\nfrom monai.networks.nets.densenet import DenseNet121","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:34:38.511103Z","iopub.execute_input":"2021-10-15T20:34:38.511367Z","iopub.status.idle":"2021-10-15T20:34:40.855962Z","shell.execute_reply.started":"2021-10-15T20:34:38.511329Z","shell.execute_reply":"2021-10-15T20:34:40.855123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if os.path.exists(\"../input/rsna-miccai-brain-tumor-radiogenomic-classification\"):\n    data_directory = '../input/rsna-miccai-voxel-128-dataset/voxel'\n    pytorch3dpath = \"../input/efficientnetpyttorch3d/EfficientNet-PyTorch-3D\"\nelse:\n    data_directory = '/media/roland/data/kaggle/rsna-miccai-brain-tumor-radiogenomic-classification'\n    pytorch3dpath = \"EfficientNet-PyTorch-3D\"\n    \nmri_types = ['FLAIR','T1w','T1wCE','T2w']\nSIZE = 256\nNUM_IMAGES = 64\n\nsys.path.append(pytorch3dpath)\nfrom efficientnet_pytorch_3d import EfficientNet3D","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:34:40.857427Z","iopub.execute_input":"2021-10-15T20:34:40.857723Z","iopub.status.idle":"2021-10-15T20:34:40.899790Z","shell.execute_reply.started":"2021-10-15T20:34:40.857685Z","shell.execute_reply":"2021-10-15T20:34:40.899157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Effficentnet","metadata":{}},{"cell_type":"code","source":"def load_images_processed(scan_id, num_imgs=NUM_IMAGES, img_size=SIZE, mri_type=\"FLAIR\", split=\"train\", rotate=0):\n  \n  if os.path.exists(f\"{data_directory}/{split}/{scan_id}/{mri_type}.npy\"):\n    file = f\"{data_directory}/{split}/{scan_id}/{mri_type}.npy\"\n    array = np.load(file) \n    data = array\n  else:\n        data =  np.zeros( (128,128,128) , dtype=int)\n  \n  return np.expand_dims(data,0)","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:34:47.774451Z","iopub.execute_input":"2021-10-15T20:34:47.774964Z","iopub.status.idle":"2021-10-15T20:34:47.781663Z","shell.execute_reply.started":"2021-10-15T20:34:47.774924Z","shell.execute_reply":"2021-10-15T20:34:47.780778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed_all(seed)\n        torch.backends.cudnn.deterministic = True\n\nset_seed(3480)","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:34:49.992699Z","iopub.execute_input":"2021-10-15T20:34:49.993389Z","iopub.status.idle":"2021-10-15T20:34:50.041481Z","shell.execute_reply.started":"2021-10-15T20:34:49.993355Z","shell.execute_reply":"2021-10-15T20:34:50.040760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/rsna-miccai-brain-tumor-radiogenomic-classification/train_labels.csv')\n\ndf_train, df_valid = sk_model_selection.train_test_split(\n    train_df, \n    test_size=0.2, \n    random_state=24, \n    stratify=train_df[\"MGMT_value\"],\n)","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:34:51.261229Z","iopub.execute_input":"2021-10-15T20:34:51.262130Z","iopub.status.idle":"2021-10-15T20:34:51.287617Z","shell.execute_reply.started":"2021-10-15T20:34:51.262090Z","shell.execute_reply":"2021-10-15T20:34:51.286957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#EFFICIENTNET3D\nclass Dataset(torch_data.Dataset):\n    def __init__(self, paths, targets=None, mri_type=None, label_smoothing=0.01, split=\"train\", augment=False):\n        self.paths = paths\n        self.targets = targets\n        self.mri_type = mri_type\n        self.label_smoothing = label_smoothing\n        self.split = split\n        self.augment = augment\n          \n    def __len__(self):\n        return len(self.paths)\n    \n    def __getitem__(self, index):\n        scan_id = self.paths[index]\n        if self.targets is None:\n            data = load_images_processed(str(scan_id).zfill(5), mri_type=self.mri_type[index], split=self.split)\n        else:\n            if self.augment:\n                rotation = np.random.randint(0,4)\n            else:\n                rotation = 0\n            data = load_images_processed(str(scan_id).zfill(5), mri_type=self.mri_type[index], split=\"train\", rotate=rotation)\n\n        if self.targets is None:\n            return {\"X\": torch.tensor(data).float(), \"id\": scan_id}\n        else:\n            y = torch.tensor(abs(self.targets[index]-self.label_smoothing), dtype=torch.float)\n            return {\"X\": torch.tensor(data).float(), \"y\": y}","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:34:51.915222Z","iopub.execute_input":"2021-10-15T20:34:51.915787Z","iopub.status.idle":"2021-10-15T20:34:51.925018Z","shell.execute_reply.started":"2021-10-15T20:34:51.915752Z","shell.execute_reply":"2021-10-15T20:34:51.924342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Model(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.net = EfficientNet3D.from_name(\"efficientnet-b0\", override_params={'num_classes': 2}, in_channels=1)\n        n_features = self.net._fc.in_features\n        self.net._fc = nn.Linear(in_features=n_features, out_features=1, bias=True)\n    \n    def forward(self, x):\n        out = self.net(x)\n        return out","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:34:52.566251Z","iopub.execute_input":"2021-10-15T20:34:52.566725Z","iopub.status.idle":"2021-10-15T20:34:52.573448Z","shell.execute_reply.started":"2021-10-15T20:34:52.566690Z","shell.execute_reply":"2021-10-15T20:34:52.572564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Trainer:\n    def __init__(\n        self, \n        model, \n        device, \n        optimizer, \n        criterion\n    ):\n        self.model = model\n        self.device = device\n        self.optimizer = optimizer\n        self.criterion = criterion\n\n        self.best_valid_score = np.inf\n        self.n_patience = 0\n        self.lastmodel = None\n        \n    def fit(self, epochs, train_loader, valid_loader, save_path, patience):        \n        for n_epoch in range(1, epochs + 1):\n            self.info_message(\"EPOCH: {}\", n_epoch)\n            \n            train_loss, train_time = self.train_epoch(train_loader)\n            valid_loss, valid_auc, valid_time = self.valid_epoch(valid_loader)\n            \n            self.info_message(\n                \"[Epoch Train: {}] loss: {:.4f}, time: {:.2f} s            \",\n                n_epoch, train_loss, train_time\n            )\n            \n            self.info_message(\n                \"[Epoch Valid: {}] loss: {:.4f}, auc: {:.4f}, time: {:.2f} s\",\n                n_epoch, valid_loss, valid_auc, valid_time\n            )\n\n            # if True:\n            # if self.best_valid_score < valid_auc: \n            if self.best_valid_score > valid_loss: \n                self.save_model(n_epoch, save_path, valid_loss, valid_auc)\n                self.info_message(\n                     \"auc improved from {:.4f} to {:.4f}. Saved model to '{}'\", \n                    self.best_valid_score, valid_loss, self.lastmodel\n                )\n                self.best_valid_score = valid_loss\n                self.n_patience = 0\n            else:\n                self.n_patience += 1\n            \n            if self.n_patience >= patience:\n                self.info_message(\"\\nValid auc didn't improve last {} epochs.\", patience)\n                break\n            \n    def train_epoch(self, train_loader):\n        self.model.train()\n        t = time.time()\n        sum_loss = 0\n\n        for step, batch in enumerate(train_loader, 1):\n            X = batch[\"X\"].to(self.device)\n            targets = batch[\"y\"].to(self.device)\n            self.optimizer.zero_grad()\n            outputs = self.model(X).squeeze(1)\n            \n            loss = self.criterion(outputs, targets)\n            loss.backward()\n\n            sum_loss += loss.detach().item()\n\n            self.optimizer.step()\n            \n            message = 'Train Step {}/{}, train_loss: {:.4f}'\n            self.info_message(message, step, len(train_loader), sum_loss/step, end=\"\\r\")\n        \n        return sum_loss/len(train_loader), int(time.time() - t)\n    \n    def valid_epoch(self, valid_loader):\n        self.model.eval()\n        t = time.time()\n        sum_loss = 0\n        y_all = []\n        outputs_all = []\n\n        for step, batch in enumerate(valid_loader, 1):\n            with torch.no_grad():\n                X = batch[\"X\"].to(self.device)\n                targets = batch[\"y\"].to(self.device)\n\n                outputs = self.model(X).squeeze(1)\n                loss = self.criterion(outputs, targets)\n\n                sum_loss += loss.detach().item()\n                y_all.extend(batch[\"y\"].tolist())\n                outputs_all.extend(outputs.tolist())\n\n            message = 'Valid Step {}/{}, valid_loss: {:.4f}'\n            self.info_message(message, step, len(valid_loader), sum_loss/step, end=\"\\r\")\n            \n        y_all = [1 if x > 0.5 else 0 for x in y_all]\n        auc = roc_auc_score(y_all, outputs_all)\n        \n        return sum_loss/len(valid_loader), auc, int(time.time() - t)\n    \n    def save_model(self, n_epoch, save_path, loss, auc):\n        self.lastmodel = f\"{save_path}-e{n_epoch}-loss{loss:.3f}-auc{auc:.3f}.pth\"\n        torch.save(\n            {\n                \"model_state_dict\": self.model.state_dict(),\n                \"optimizer_state_dict\": self.optimizer.state_dict(),\n                \"best_valid_score\": self.best_valid_score,\n                \"n_epoch\": n_epoch,\n            },\n            self.lastmodel,\n        )\n    \n    @staticmethod\n    def info_message(message, *args, end=\"\\n\"):\n        print(message.format(*args), end=end)","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:34:53.225264Z","iopub.execute_input":"2021-10-15T20:34:53.225530Z","iopub.status.idle":"2021-10-15T20:34:53.248752Z","shell.execute_reply.started":"2021-10-15T20:34:53.225500Z","shell.execute_reply":"2021-10-15T20:34:53.247850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict(modelfile, df, mri_type, split):\n    print(\"Predict:\", modelfile, mri_type, df.shape)\n    df.loc[:,\"MRI_Type\"] = mri_type\n    data_retriever = Dataset(\n        df.index.values, \n        mri_type=df[\"MRI_Type\"].values,\n        split=split\n    )\n\n    data_loader = torch_data.DataLoader(\n        data_retriever,\n        batch_size=4,\n        shuffle=False,\n        num_workers=8,\n    )\n   \n    model = Model()\n    model.to(device)\n    \n    checkpoint = torch.load(modelfile)\n    model.load_state_dict(checkpoint[\"model_state_dict\"])\n    model.eval()\n    \n    y_pred = []\n    ids = []\n\n    for e, batch in enumerate(data_loader,1):\n        print(f\"{e}/{len(data_loader)}\", end=\"\\r\")\n        with torch.no_grad():\n            tmp_pred = torch.sigmoid(model(batch[\"X\"].to(device))).cpu().numpy().squeeze()\n            if tmp_pred.size == 1:\n                y_pred.append(tmp_pred)\n            else:\n                y_pred.extend(tmp_pred.tolist())\n            ids.extend(batch[\"id\"].numpy().tolist())\n            \n    preddf = pd.DataFrame({\"BraTS21ID\": ids, \"MGMT_value\": y_pred}) \n    preddf = preddf.set_index(\"BraTS21ID\")\n    return preddf","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:34:53.976628Z","iopub.execute_input":"2021-10-15T20:34:53.976902Z","iopub.status.idle":"2021-10-15T20:34:53.989016Z","shell.execute_reply.started":"2021-10-15T20:34:53.976871Z","shell.execute_reply":"2021-10-15T20:34:53.988290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"efficientnet3dfiles = ['../input/efficientnet3d128modelfiles/FLAIR-e5-loss0.686-auc0.485.pth',\n            '../input/efficientnet3d128modelfiles/T1w-e4-loss0.683-auc0.570.pth',\n            '../input/efficientnet3d128modelfiles/T1wCE-e3-loss0.689-auc0.539.pth',\n            '../input/efficientnet3d128modelfiles/T2w-e9-loss0.692-auc0.509.pth']","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:34:54.640340Z","iopub.execute_input":"2021-10-15T20:34:54.641047Z","iopub.status.idle":"2021-10-15T20:34:54.647956Z","shell.execute_reply.started":"2021-10-15T20:34:54.641009Z","shell.execute_reply":"2021-10-15T20:34:54.647127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:34:57.582164Z","iopub.execute_input":"2021-10-15T20:34:57.582433Z","iopub.status.idle":"2021-10-15T20:34:57.587004Z","shell.execute_reply.started":"2021-10-15T20:34:57.582404Z","shell.execute_reply":"2021-10-15T20:34:57.586164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_valid = df_valid.set_index(\"BraTS21ID\")\ndf_valid[\"MGMT_pred\"] = 0\nfor m, mtype in zip(efficientnet3dfiles,  mri_types):\n    pred = predict(m, df_valid, mtype, \"train\")\n    df_valid[\"MGMT_pred\"] += pred[\"MGMT_value\"]\n","metadata":{"execution":{"iopub.status.busy":"2021-10-15T19:59:16.099890Z","iopub.execute_input":"2021-10-15T19:59:16.100146Z","iopub.status.idle":"2021-10-15T19:59:47.885423Z","shell.execute_reply.started":"2021-10-15T19:59:16.100122Z","shell.execute_reply":"2021-10-15T19:59:47.884623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_valid[\"MGMT_pred\"] /= len(efficientnet3dfiles)\nauc = roc_auc_score(df_valid[\"MGMT_value\"], df_valid[\"MGMT_pred\"])\nprint(f\"Validation ensemble AUC: {auc:.4f}\")\nsns.displot(df_valid[\"MGMT_pred\"])","metadata":{"execution":{"iopub.status.busy":"2021-10-15T19:59:47.887231Z","iopub.execute_input":"2021-10-15T19:59:47.887501Z","iopub.status.idle":"2021-10-15T19:59:49.543277Z","shell.execute_reply.started":"2021-10-15T19:59:47.887465Z","shell.execute_reply":"2021-10-15T19:59:49.542562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv('../input/rsna-miccai-brain-tumor-radiogenomic-classification/sample_submission.csv', index_col=\"BraTS21ID\")\n\nsubmission[\"MGMT_value\"] = 0\nfor m, mtype in zip(efficientnet3dfiles, mri_types):\n    pred = predict(m, submission, mtype, split=\"test\")\n    submission[\"MGMT_value\"] += pred[\"MGMT_value\"]\n\nsubmission[\"MGMT_value\"] /= len(efficientnet3dfiles)\nsns.displot(submission[\"MGMT_value\"])\ns1 = submission[\"MGMT_value\"]","metadata":{"execution":{"iopub.status.busy":"2021-10-15T19:59:49.545833Z","iopub.execute_input":"2021-10-15T19:59:49.546232Z","iopub.status.idle":"2021-10-15T20:00:08.213371Z","shell.execute_reply.started":"2021-10-15T19:59:49.546194Z","shell.execute_reply":"2021-10-15T20:00:08.212635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Densenet","metadata":{}},{"cell_type":"code","source":"if os.path.exists(\"../input/rsna-miccai-brain-tumor-radiogenomic-classification\"):\n    data_directory = '../input/rsna-miccai-voxel-64-dataset/voxel'\n    pytorch3dpath = \"../input/efficientnetpyttorch3d/EfficientNet-PyTorch-3D\"\nelse:\n    data_directory = '/media/roland/data/kaggle/rsna-miccai-brain-tumor-radiogenomic-classification'\n    pytorch3dpath = \"EfficientNet-PyTorch-3D\"\n    \nmri_types = ['FLAIR','T1w','T1wCE','T2w']\nSIZE = 256\nNUM_IMAGES = 64\n\nsys.path.append(pytorch3dpath)\nfrom efficientnet_pytorch_3d import EfficientNet3D","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:00:08.215081Z","iopub.execute_input":"2021-10-15T20:00:08.215337Z","iopub.status.idle":"2021-10-15T20:00:08.221592Z","shell.execute_reply.started":"2021-10-15T20:00:08.215304Z","shell.execute_reply":"2021-10-15T20:00:08.220829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_images_processed(scan_id, num_imgs=NUM_IMAGES, img_size=SIZE, mri_type=\"FLAIR\", split=\"train\", rotate=0):\n  \n  if os.path.exists(f\"{data_directory}/{split}/{scan_id}/{mri_type}.npy\"):\n    file = f\"{data_directory}/{split}/{scan_id}/{mri_type}.npy\"\n    array = np.load(file) \n    data = array\n  else:\n        data =  np.zeros( (64,64,64) , dtype=int)\n  \n  return np.expand_dims(data,0)","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:00:08.223150Z","iopub.execute_input":"2021-10-15T20:00:08.223406Z","iopub.status.idle":"2021-10-15T20:00:08.234346Z","shell.execute_reply.started":"2021-10-15T20:00:08.223372Z","shell.execute_reply":"2021-10-15T20:00:08.233596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Dataset(torch_data.Dataset):\n    def __init__(self, paths, targets=None, mri_type=None, split=\"train\"):\n        self.paths = paths\n        self.targets = targets\n        self.mri_type = mri_type\n        self.split = split\n          \n    def __len__(self):\n        return len(self.paths)\n    \n    def __getitem__(self, index):\n        scan_id = self.paths[index]\n        if self.targets is None:\n            data = load_images_processed(str(scan_id).zfill(5), mri_type=self.mri_type[index], split=self.split)\n        else:\n            data = load_images_processed(str(scan_id).zfill(5), mri_type=self.mri_type[index], split=\"train\")\n            \n        if self.targets is None:\n            return {\"X\": data, \"id\": scan_id}\n        else:\n            return {\"X\": data, \"y\": torch.tensor(self.targets[index], dtype=torch.float)}\n","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:00:08.235388Z","iopub.execute_input":"2021-10-15T20:00:08.235972Z","iopub.status.idle":"2021-10-15T20:00:08.244862Z","shell.execute_reply.started":"2021-10-15T20:00:08.235937Z","shell.execute_reply":"2021-10-15T20:00:08.244002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_model():\n    model = DenseNet121(spatial_dims=3, in_channels=1, out_channels=1)\n    return model  ","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:00:08.246181Z","iopub.execute_input":"2021-10-15T20:00:08.246895Z","iopub.status.idle":"2021-10-15T20:00:08.255981Z","shell.execute_reply.started":"2021-10-15T20:00:08.246857Z","shell.execute_reply":"2021-10-15T20:00:08.255286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Trainer:\n    def __init__(\n        self, \n        model, \n        device, \n        optimizer, \n        criterion\n    ):\n        self.model = model\n        self.device = device\n        self.optimizer = optimizer\n        self.lr_scheduler = torch.optim.lr_scheduler.ExponentialLR(self.optimizer, gamma=LR_DECAY)\n        self.criterion = criterion\n\n        self.best_valid_score = .0\n        self.n_patience = 0\n        self.lastmodel = None\n        \n        self.val_losses = []\n        self.train_losses = []\n        self.val_auc = []\n        \n    def fit(self, epochs, train_loader, valid_loader, save_path, patience):      \n        for n_epoch in range(1, epochs + 1):\n            self.info_message(\"EPOCH: {}\", n_epoch)\n            \n            train_loss, train_time = self.train_epoch(train_loader)\n            valid_loss, valid_auc, valid_time = self.valid_epoch(valid_loader)\n            \n            self.train_losses.append(train_loss)\n            self.val_losses.append(valid_loss)\n            self.val_auc.append(valid_auc)\n            \n            self.info_message(\n                \"[Epoch Train: {}] loss: {:.4f}, time: {:.2f} s\",\n                n_epoch, train_loss, train_time\n            )\n            \n            self.info_message(\n                \"[Epoch Valid: {}] loss: {:.4f}, auc: {:.4f}, time: {:.2f} s\",\n                n_epoch, valid_loss, valid_auc, valid_time\n            )\n\n            if self.best_valid_score < valid_auc: \n                self.save_model(n_epoch, save_path, valid_loss, valid_auc)\n                self.info_message(\n                     \"auc improved from {:.4f} to {:.4f}. Saved model to '{}'\", \n                    self.best_valid_score, valid_auc, self.lastmodel\n                )\n                self.best_valid_score = valid_auc\n                self.n_patience = 0\n            else:\n                self.n_patience += 1\n            \n            if self.n_patience >= patience:\n                self.info_message(\"\\nValid auc didn't improve last {} epochs.\", patience)\n                break\n            \n    def train_epoch(self, train_loader):\n        self.model.train()\n        t = time.time()\n        sum_loss = 0\n\n        for step, batch in enumerate(train_loader, 1):\n            X = torch.tensor(batch[\"X\"]).float().to(self.device)\n            targets = batch[\"y\"].to(self.device)\n            self.optimizer.zero_grad()\n            outputs = self.model(X).squeeze(1)\n            loss = self.criterion(outputs, targets)\n                \n            loss.backward()\n\n            sum_loss += loss.detach().item()\n            \n            self.optimizer.step()\n            \n            message = 'Train Step {}/{}, train_loss: {:.4f}'\n            self.info_message(message, step, len(train_loader), sum_loss/step, end=\"\\r\")\n            \n        self.lr_scheduler.step()\n        \n        return sum_loss/len(train_loader), int(time.time() - t)\n    \n    def valid_epoch(self, valid_loader):\n        self.model.eval()\n        t = time.time()\n        sum_loss = 0\n        y_all = []\n        outputs_all = []\n\n        for step, batch in enumerate(valid_loader, 1):\n            with torch.no_grad():\n                targets = batch[\"y\"].to(self.device)\n\n                output = torch.sigmoid(self.model(torch.tensor(batch[\"X\"]).float().to(self.device)).squeeze(1))\n                loss = self.criterion(output, targets)\n                sum_loss += loss.detach().item()\n\n                y_all.extend(batch[\"y\"].tolist())\n                outputs_all.extend(output.tolist())\n\n            message = 'Valid Step {}/{}, valid_loss: {:.4f}'\n            self.info_message(message, step, len(valid_loader), sum_loss/step, end=\"\\r\")\n            \n        auc = roc_auc_score(y_all, outputs_all)\n        \n        return sum_loss/len(valid_loader), auc, int(time.time() - t)\n    \n    def save_model(self, n_epoch, save_path, loss, auc):\n        self.lastmodel = f\"{save_path}-e{n_epoch}-loss{loss:.3f}-auc{auc:.3f}.pth\"\n        torch.save(\n            {\n                \"model_state_dict\": self.model.state_dict(),\n                \"optimizer_state_dict\": self.optimizer.state_dict(),\n                \"best_valid_score\": self.best_valid_score,\n                \"n_epoch\": n_epoch,\n            },\n            self.lastmodel,\n        )\n        \n    def display_plots(self, mri_type):\n        plt.figure(figsize=(10,5))\n        plt.title(\"{}: Training and Validation Loss\")\n        plt.plot(self.val_losses,label=\"val\")\n        plt.plot(self.train_losses,label=\"train\")\n        plt.xlabel(\"iterations\")\n        plt.ylabel(\"Loss\")\n        plt.legend()\n        plt.show()\n        plt.close()\n        \n        plt.figure(figsize=(10,5))\n        plt.title(\"{}: Validation AUC-ROC\")\n        plt.plot(self.val_auc,label=\"val\")\n        plt.xlabel(\"iterations\")\n        plt.ylabel(\"AUC\")\n        plt.legend()\n        plt.show()\n        plt.close()\n    \n    @staticmethod\n    def info_message(message, *args, end=\"\\n\"):\n        print(message.format(*args), end=end)","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:00:08.257372Z","iopub.execute_input":"2021-10-15T20:00:08.257693Z","iopub.status.idle":"2021-10-15T20:00:08.283401Z","shell.execute_reply.started":"2021-10-15T20:00:08.257661Z","shell.execute_reply":"2021-10-15T20:00:08.282575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"densenet121files = ['../input/monaitrain/FLAIR-e10-loss0.702-auc0.662.pth',\n            '../input/monaitrain/T1w-e6-loss0.693-auc0.599.pth',\n            '../input/monaitrain/T1wCE-e12-loss0.706-auc0.599.pth',\n            '../input/monaitrain/T2w-e1-loss0.722-auc0.620.pth']","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:00:08.284742Z","iopub.execute_input":"2021-10-15T20:00:08.284983Z","iopub.status.idle":"2021-10-15T20:00:08.294827Z","shell.execute_reply.started":"2021-10-15T20:00:08.284954Z","shell.execute_reply":"2021-10-15T20:00:08.294108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict(modelfile, df, mri_type, split):\n    print(\"Predict:\", modelfile, mri_type, df.shape)\n    df.loc[:,\"MRI_Type\"] = mri_type\n    data_retriever = Dataset(\n        df.index.values, \n        mri_type=df[\"MRI_Type\"].values,\n        split=split\n    )\n\n    data_loader = torch_data.DataLoader(\n        data_retriever,\n        batch_size=1,\n        shuffle=False,\n        num_workers=8,\n    )\n   \n    model = build_model()\n    model.to(device)\n    \n    checkpoint = torch.load(modelfile)\n    model.load_state_dict(checkpoint[\"model_state_dict\"])\n    model.eval()\n    \n    y_pred = []\n    ids = []\n\n    for e, batch in enumerate(data_loader,1):\n        print(f\"{e}/{len(data_loader)}\", end=\"\\r\")\n        with torch.no_grad():\n            tmp_pred = torch.sigmoid(model(torch.tensor(batch[\"X\"]).float().to(device)).squeeze(1)).cpu().numpy().squeeze()\n            if tmp_pred.size == 1:\n                y_pred.append(tmp_pred)\n            else:\n                y_pred.extend(tmp_pred.tolist())\n            ids.extend(batch[\"id\"].numpy().tolist())\n            \n    preddf = pd.DataFrame({\"BraTS21ID\": ids, \"MGMT_value\": y_pred}) \n    preddf = preddf.set_index(\"BraTS21ID\")\n    return preddf","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:00:08.295947Z","iopub.execute_input":"2021-10-15T20:00:08.296719Z","iopub.status.idle":"2021-10-15T20:00:08.308407Z","shell.execute_reply.started":"2021-10-15T20:00:08.296684Z","shell.execute_reply":"2021-10-15T20:00:08.307615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:00:08.309498Z","iopub.execute_input":"2021-10-15T20:00:08.309888Z","iopub.status.idle":"2021-10-15T20:00:08.316705Z","shell.execute_reply.started":"2021-10-15T20:00:08.309853Z","shell.execute_reply":"2021-10-15T20:00:08.316033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/rsna-miccai-brain-tumor-radiogenomic-classification/train_labels.csv')\n\ndf_train, df_valid = sk_model_selection.train_test_split(\n    train_df, \n    test_size=0.2, \n    random_state=24, \n    stratify=train_df[\"MGMT_value\"],\n)","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:00:08.318020Z","iopub.execute_input":"2021-10-15T20:00:08.318317Z","iopub.status.idle":"2021-10-15T20:00:08.330591Z","shell.execute_reply.started":"2021-10-15T20:00:08.318250Z","shell.execute_reply":"2021-10-15T20:00:08.329773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_pred = df_valid.set_index(\"BraTS21ID\")\ndf_pred[\"MGMT_pred\"] = 0\nfor m, mtype in zip(densenet121files,  mri_types):\n    pred = predict(m, df_pred, mtype, \"train\")\n    df_pred[\"MGMT_pred\"] += pred[\"MGMT_value\"]\ndf_pred[\"MGMT_pred\"] /= len(densenet121files)\nauc = roc_auc_score(df_pred[\"MGMT_value\"], df_pred[\"MGMT_pred\"])\nprint(f\"Validation ensemble AUC: {auc:.4f}\")\nsns.displot(df_pred[\"MGMT_pred\"])","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:00:08.331575Z","iopub.execute_input":"2021-10-15T20:00:08.331815Z","iopub.status.idle":"2021-10-15T20:00:41.037933Z","shell.execute_reply.started":"2021-10-15T20:00:08.331785Z","shell.execute_reply":"2021-10-15T20:00:41.037173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv(\"../input/rsna-miccai-brain-tumor-radiogenomic-classification/sample_submission.csv\", index_col=\"BraTS21ID\")\n\nsubmission[\"MGMT_value\"] = 0\nfor m, mtype in zip(densenet121files, mri_types):\n    pred = predict(m, submission, mtype, split=\"test\")\n    submission[\"MGMT_value\"] += pred[\"MGMT_value\"]\n\nsubmission[\"MGMT_value\"] /= len(densenet121files)\ns2 = submission[\"MGMT_value\"]","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:00:41.042752Z","iopub.execute_input":"2021-10-15T20:00:41.042958Z","iopub.status.idle":"2021-10-15T20:00:59.355553Z","shell.execute_reply.started":"2021-10-15T20:00:41.042932Z","shell.execute_reply":"2021-10-15T20:00:59.354758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## KERAS Turner","metadata":{}},{"cell_type":"code","source":"import os\nimport glob\n\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'\n\nimport pandas as pd\nimport numpy as np\nfrom pathlib import Path\n\nimport random\nfrom tqdm.notebook import tqdm\nimport pydicom \n\nimport cv2  \n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn import model_selection\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.initializers import RandomUniform\n","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:00:59.357257Z","iopub.execute_input":"2021-10-15T20:00:59.357515Z","iopub.status.idle":"2021-10-15T20:01:03.554313Z","shell.execute_reply.started":"2021-10-15T20:00:59.357481Z","shell.execute_reply":"2021-10-15T20:01:03.553560Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dir = Path('../input/rsna-miccai-brain-tumor-radiogenomic-classification/')\n\nmri_types = [\"FLAIR\", \"T1w\", \"T2w\", \"T1wCE\"]\nexcluded_images = [109, 123, 709] # Bad images\n\ntrain_df = pd.read_csv(data_dir / \"train_labels.csv\")\ntest_df = pd.read_csv(data_dir / \"sample_submission.csv\")\nsample_submission = pd.read_csv(data_dir / \"sample_submission.csv\")\n\ntrain_df = train_df[~train_df.BraTS21ID.isin(excluded_images)].reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:01:03.555721Z","iopub.execute_input":"2021-10-15T20:01:03.555993Z","iopub.status.idle":"2021-10-15T20:01:03.573834Z","shell.execute_reply.started":"2021-10-15T20:01:03.555960Z","shell.execute_reply":"2021-10-15T20:01:03.573190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_folds(data, num_splits):\n    data[\"kfold\"] = -1\n    kf = model_selection.KFold(n_splits=num_splits, shuffle=True, random_state=42)\n    for f, (t, v) in enumerate(kf.split(X=data)):\n        data.loc[v, \"kfold\"] = f\n    return data\nk = 5\ntrain_df = create_folds(train_df, k)","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:01:03.575153Z","iopub.execute_input":"2021-10-15T20:01:03.575413Z","iopub.status.idle":"2021-10-15T20:01:03.588415Z","shell.execute_reply.started":"2021-10-15T20:01:03.575379Z","shell.execute_reply":"2021-10-15T20:01:03.587717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_dicom(path, size = 224):\n    dicom = pydicom.read_file(path)\n    data = dicom.pixel_array\n    # transform data into black and white scale / grayscale\n#     data = data - np.min(data)\n    if np.max(data) != 0:\n        data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    return cv2.resize(data, (size, size))\n\ndef get_all_image_paths(brats21id, image_type, folder='train'): \n    '''\n    Returns an arry of all the images of a particular type for a particular patient ID\n    '''\n    assert(image_type in mri_types)\n    \n    patient_path = os.path.join(\n        \"../input/rsna-miccai-brain-tumor-radiogenomic-classification/%s/\" % folder, \n        str(brats21id).zfill(5),\n    )\n        \n    paths = sorted(\n        glob.glob(os.path.join(patient_path, image_type, \"*\")), \n        key=lambda x: int(x[:-4].split(\"-\")[-1]),\n    )\n    \n    num_images = len(paths)\n    \n    start = int(num_images * 0.25)\n    end = int(num_images * 0.75)\n\n    interval = 3\n    \n    if num_images < 10: \n        interval = 1\n    \n    return np.array(paths[start:end:interval])\n\ndef get_all_images(brats21id, image_type, folder='train', size=225):\n    return [load_dicom(path, size) for path in get_all_image_paths(brats21id, image_type, folder)]\n\ndef get_all_data_for_train(image_type, image_size=32):\n    global train_df\n    \n    X = []\n    y = []\n    train_ids = []\n\n    for i in tqdm(train_df.index):\n        x = train_df.loc[i]\n        images = get_all_images(int(x['BraTS21ID']), image_type, 'train', image_size)\n        label = x['MGMT_value']\n\n        X += images\n        y += [label] * len(images)\n        train_ids += [int(x['BraTS21ID'])] * len(images)\n        assert(len(X) == len(y))\n    return np.array(X), np.array(y), np.array(train_ids)\n\ndef get_all_data_for_test(image_type, image_size=32):\n    global test_df\n    \n    X = []\n    test_ids = []\n\n    for i in tqdm(test_df.index):\n        x = test_df.loc[i]\n        images = get_all_images(int(x['BraTS21ID']), image_type, 'test', image_size)\n        X += images\n        test_ids += [int(x['BraTS21ID'])] * len(images)\n\n    return np.array(X), np.array(test_ids)","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:01:03.589639Z","iopub.execute_input":"2021-10-15T20:01:03.590093Z","iopub.status.idle":"2021-10-15T20:01:03.606529Z","shell.execute_reply.started":"2021-10-15T20:01:03.590053Z","shell.execute_reply":"2021-10-15T20:01:03.605916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X, y, trainidt = get_all_data_for_train('T1wCE', image_size=32)\nX_test, testidt = get_all_data_for_test('T1wCE', image_size=32)","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:01:03.607825Z","iopub.execute_input":"2021-10-15T20:01:03.608081Z","iopub.status.idle":"2021-10-15T20:03:23.233480Z","shell.execute_reply.started":"2021-10-15T20:01:03.608050Z","shell.execute_reply":"2021-10-15T20:03:23.232522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_valid, y_train, y_valid, trainidt_train, trainidt_valid = train_test_split(X, y, trainidt, test_size=0.2, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:03:23.235112Z","iopub.execute_input":"2021-10-15T20:03:23.235387Z","iopub.status.idle":"2021-10-15T20:03:23.262880Z","shell.execute_reply.started":"2021-10-15T20:03:23.235352Z","shell.execute_reply":"2021-10-15T20:03:23.262038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = tf.expand_dims(X_train, axis=-1)\nX_valid = tf.expand_dims(X_valid, axis=-1)\ny_train = to_categorical(y_train)\ny_valid = to_categorical(y_valid)","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:03:23.264137Z","iopub.execute_input":"2021-10-15T20:03:23.264943Z","iopub.status.idle":"2021-10-15T20:03:25.038114Z","shell.execute_reply.started":"2021-10-15T20:03:23.264902Z","shell.execute_reply":"2021-10-15T20:03:25.037281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SineDenseLayer(keras.layers.Layer):\n    def __init__(self, features,\n                 is_first=False, omega_0=30):\n        super().__init__()\n        self.omega_0 = omega_0\n        self.is_first = is_first\n        \n        self.features = features\n        \n        if self.is_first:\n            initializer = RandomUniform(-1 / self.features, 1 / self.features)   \n            self.linear = keras.layers.Dense(features, kernel_initializer=initializer)\n    \n        else:\n            initializer = RandomUniform(-np.sqrt(6 / self.features) / self.omega_0, np.sqrt(6 / self.features) / self.omega_0)\n            self.linear = keras.layers.Dense(features, kernel_initializer=initializer)\n     \n\n    def call(self, input):\n        return tf.math.sin(self.omega_0 * self.linear(input))\n \n\nclass SineConvLayer(keras.layers.Layer):\n    \n    def __init__(self, features, kernel_size,\n                 is_first=False, omega_0=30):\n        super().__init__()\n        self.omega_0 = omega_0\n        self.is_first = is_first\n        \n        self.features = features\n        \n        if self.is_first:\n            initializer = RandomUniform(-1 / self.features, 1 / self.features)            \n            self.conv = keras.layers.Conv2D(features, kernel_size, kernel_initializer=initializer)\n            \n        else:\n            initializer = RandomUniform(-np.sqrt(6 / self.features) / self.omega_0, np.sqrt(6 / self.features) / self.omega_0)\n            self.conv = keras.layers.Conv2D(features, kernel_size, kernel_initializer=initializer)\n            \n\n    def call(self, input):\n        return tf.math.sin(self.omega_0 * self.conv(input))\n    \n#     def forward_with_intermediate(self, input): \n#         # For visualization of activation distributions\n#         intermediate = self.omega_0 * self.linear(input)\n#         return tf.math.sin(intermediate), intermediate\n\n","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:03:25.039621Z","iopub.execute_input":"2021-10-15T20:03:25.039908Z","iopub.status.idle":"2021-10-15T20:03:25.054709Z","shell.execute_reply.started":"2021-10-15T20:03:25.039873Z","shell.execute_reply":"2021-10-15T20:03:25.053806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import keras_tuner as kt\n\n\ndef make_model(hp):\n    inputs = keras.Input(shape=X_train.shape[1:])\n    \n    x = keras.layers.experimental.preprocessing.Rescaling(1.0 / 255)(inputs)\n\n#     num_block = hp.Int('num_block', min_value=2, max_value=5, step=1)\n#     num_filters = hp.Int('num_filters', min_value=32, max_value=128, step=32)\n    \n#     x = keras.layers.Conv2D(64, kernel_size=(4, 4), activation=\"relu\", name=\"Conv_1\")(x)\n    x = keras.layers.Conv2D(filters=hp.Int('units_Conv_1_' + str(0),\n                                            min_value=64,\n                                            max_value=256,\n                                            step=32),\n                            kernel_size=(4, 4),\n                            activation=\"relu\", \n                            name=\"Conv_1\")(x)\n\n    x = keras.layers.MaxPool2D(pool_size=(2, 2))(x)\n\n#     x = keras.layers.Conv2D(32, kernel_size=(2, 2), activation=\"relu\", name=\"Conv_2\")(x)\n    x = keras.layers.Conv2D(filters=hp.Int('units_conv2_' + str(1),\n                                            min_value=16,\n                                            max_value=128,\n                                            step=16),\n                            kernel_size=(2, 2),\n                            activation=\"relu\",\n                            name=\"Conv_2\")(x)\n\n    x = keras.layers.MaxPool2D(pool_size=(1, 1))(x)\n    \n#     for i in range(num_block):\n#         x = keras.layers.Conv2D(num_filters, \n#                                 kernel_size=(4, 4),\n#                                 activation=\"relu\",\n#                                 )(x)\n    \n#         x = keras.layers.MaxPool2D(pool_size=(2, 2))(x)\n\n#     x = keras.layers.Conv2D(32, kernel_size=(2, 2), activation=\"relu\", name=\"Conv_2\")(x)\n#     x = keras.layers.MaxPool2D(pool_size=(1, 1))(x)\n\n#     h = keras.layers.Dropout(0.1)(h)\n    x = layers.Dropout(\n        hp.Float('dense_dropout', min_value=0., max_value=0.7)\n    )(x)\n    x = keras.layers.Flatten()(x)\n#     reduction_type = hp.Choice('reduction_type', ['flatten', 'avg'])\n#     if reduction_type == 'flatten':\n#         x = layers.Flatten()(x)\n#     else:\n#         x = layers.GlobalAveragePooling2D()(x)\n        \n#     x = keras.layers.Dense(32, activation=\"relu\")(x)\n    x = layers.Dense(\n        units=hp.Int('num_dense_units', min_value=16, max_value=64, step=8),\n        activation='relu'\n    )(x)\n\n    outputs = keras.layers.Dense(2, activation=\"softmax\")(x)\n\n    model = keras.Model(inputs, outputs)\n\n    roc_auc = tf.keras.metrics.AUC(name='roc_auc', curve='ROC')\n\n    model.compile(\n        loss=\"categorical_crossentropy\", optimizer=\"adam\", metrics=[roc_auc]\n    )\n    model.summary()\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:03:25.056525Z","iopub.execute_input":"2021-10-15T20:03:25.056895Z","iopub.status.idle":"2021-10-15T20:03:25.118131Z","shell.execute_reply.started":"2021-10-15T20:03:25.056855Z","shell.execute_reply":"2021-10-15T20:03:25.117353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_model_augmented(hp):\n    input_shape = (32, 32, 1)\n    classes = 10\n\n    # Create a data augmentation stage with horizontal flipping, rotations, zooms\n#     data_augmentation = keras.Sequential(\n#         [\n#             layers.experimental.preprocessing.RandomFlip(\"horizontal\"),\n#             layers.experimental.preprocessing.RandomRotation(0.1),\n#             layers.experimental.preprocessing.RandomZoom(0.1),\n#         ]\n#     )\n    \n    data_augmentation = keras.Sequential(\n    [\n        layers.experimental.preprocessing.RandomFlip(\"horizontal\"),\n        layers.experimental.preprocessing.RandomRotation(0.1),\n    ]\n)\n\n    shape=X_train.shape[1:]\n    print(f\"shape={shape}\") # shape=(32, 32, 1)\n    \n    inputs = keras.Input(shape=input_shape)\n    x = data_augmentation(inputs)\n\n    x = keras.layers.experimental.preprocessing.Rescaling(1.0 / 255)(x)\n#     x = layers.experimental.preprocessing.RandomFlip(\"horizontal\")(x),\n#     x = layers.experimental.preprocessing.RandomRotation(0.1)(x),\n#     x = layers.experimental.preprocessing.RandomZoom(\n#         height_factor = 0.2,\n#         width_factor = -0.3,\n#         fill_mode = \"constant\",\n#         interpolation = \"bilinear\",\n#         seed = 42\n#     )(x),\n#     num_block = hp.Int('num_block', min_value=2, max_value=5, step=1)\n#     num_filters = hp.Int('num_filters', min_value=32, max_value=128, step=32)\n    \n#     x = keras.layers.Conv2D(64, kernel_size=(4, 4), activation=\"relu\", name=\"Conv_1\")(x)\n    x = keras.layers.Conv2D(filters=hp.Int('units_Conv_1_' + str(0),\n                                            min_value=64,\n                                            max_value=256,\n                                            step=32),\n                            kernel_size=(4, 4),\n                            activation=\"relu\", \n                            name=\"Conv_1\")(x)\n\n    x = keras.layers.MaxPool2D(pool_size=(2, 2))(x)\n\n#     x = keras.layers.Conv2D(32, kernel_size=(2, 2), activation=\"relu\", name=\"Conv_2\")(x)\n    x = keras.layers.Conv2D(filters=hp.Int('units_conv2_' + str(1),\n                                            min_value=16,\n                                            max_value=128,\n                                            step=16),\n                            kernel_size=(2, 2),\n                            activation=\"relu\",\n                            name=\"Conv_2\")(x)\n\n    x = keras.layers.MaxPool2D(pool_size=(1, 1))(x)\n    \n#     for i in range(num_block):\n#         x = keras.layers.Conv2D(num_filters, \n#                                 kernel_size=(4, 4),\n#                                 activation=\"relu\",\n#                                 )(x)\n    \n#         x = keras.layers.MaxPool2D(pool_size=(2, 2))(x)\n\n#     x = keras.layers.Conv2D(32, kernel_size=(2, 2), activation=\"relu\", name=\"Conv_2\")(x)\n#     x = keras.layers.MaxPool2D(pool_size=(1, 1))(x)\n\n#     h = keras.layers.Dropout(0.1)(h)\n    x = layers.Dropout(\n        hp.Float('dense_dropout', min_value=0., max_value=0.7)\n    )(x)\n    x = keras.layers.Flatten()(x)\n#     reduction_type = hp.Choice('reduction_type', ['flatten', 'avg'])\n#     if reduction_type == 'flatten':\n#         x = layers.Flatten()(x)\n#     else:\n#         x = layers.GlobalAveragePooling2D()(x)\n        \n#     x = keras.layers.Dense(32, activation=\"relu\")(x)\n    x = layers.Dense(\n        units=hp.Int('num_dense_units', min_value=16, max_value=64, step=8),\n        activation='relu'\n    )(x)\n\n    outputs = keras.layers.Dense(2, activation=\"softmax\")(x)\n\n    model = keras.Model(inputs, outputs)\n\n    roc_auc = tf.keras.metrics.AUC(name='roc_auc', curve='ROC')\n\n    model.compile(\n        loss=\"categorical_crossentropy\", optimizer=\"adam\", metrics=[roc_auc]\n    )\n    model.summary()\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:03:25.119313Z","iopub.execute_input":"2021-10-15T20:03:25.119993Z","iopub.status.idle":"2021-10-15T20:03:25.134927Z","shell.execute_reply.started":"2021-10-15T20:03:25.119963Z","shell.execute_reply":"2021-10-15T20:03:25.134170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import keras_tuner as kt\n\n\ndef make_model_siren(hp):\n    inputs = keras.Input(shape=X_train.shape[1:])\n    \n    x = keras.layers.experimental.preprocessing.Rescaling(1.0 / 255)(inputs)\n\n    x = SineConvLayer(features=hp.Int('features_conv_1', min_value=64, max_value=256, step=32),\n                      kernel_size=hp.Int('kernel_conv_1', min_value=2, max_value=7, step=1),\n                      is_first=True, \n                      omega_0=hp.Int('omega_0_conv_1', min_value=10, max_value=50, step=5))(x)\n    \n    x = keras.layers.MaxPool2D(pool_size=(2, 2))(x)\n\n    x = SineConvLayer(features=hp.Int('features_conv_2', min_value=16, max_value=128, step=16),\n                      kernel_size=hp.Int('kernel_conv_2', min_value=2, max_value=7, step=1),\n                      is_first=False, \n                      omega_0=hp.Int('omega_0_conv_2', min_value=10, max_value=50, step=5))(x)\n\n    x = keras.layers.MaxPool2D(pool_size=(1, 1))(x)\n    \n    x = layers.Dropout(\n        hp.Float('dense_dropout', min_value=0., max_value=0.7)\n    )(x)\n    x = keras.layers.Flatten()(x)\n    x = SineDenseLayer(features=hp.Int('features_dense_1', min_value=64, max_value=256, step=32),\n                      is_first=False, \n                      omega_0=hp.Int('omega_0_dense_1', min_value=10, max_value=50, step=5))(x)\n\n    outputs = keras.layers.Dense(2, activation=\"softmax\")(x)\n\n    model = keras.Model(inputs, outputs)\n\n    roc_auc = tf.keras.metrics.AUC(name='roc_auc', curve='ROC')\n\n    model.compile(\n        loss=\"categorical_crossentropy\", optimizer=\"adam\", metrics=[roc_auc]\n    )\n    model.summary()\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:03:25.136461Z","iopub.execute_input":"2021-10-15T20:03:25.137315Z","iopub.status.idle":"2021-10-15T20:03:25.150897Z","shell.execute_reply.started":"2021-10-15T20:03:25.137278Z","shell.execute_reply":"2021-10-15T20:03:25.150145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tuner = kt.tuners.BayesianOptimization(\n#     make_model_siren,\n#     make_model,\n    make_model_augmented,\n    objective='val_loss',\n    max_trials=5,  # Set to 5 to run quicker, but need 100+ for good results\n    overwrite=True)\n\ncallbacks=[keras.callbacks.EarlyStopping(monitor='val_roc_acc', mode='max', patience=3, baseline=0.9)]\n\ntuner.search(X_train, y_train, validation_split=0.2, callbacks=callbacks, verbose=1, epochs=20)","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:03:25.152014Z","iopub.execute_input":"2021-10-15T20:03:25.152339Z","iopub.status.idle":"2021-10-15T20:07:19.416365Z","shell.execute_reply.started":"2021-10-15T20:03:25.152306Z","shell.execute_reply":"2021-10-15T20:07:19.415648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_hp = tuner.get_best_hyperparameters()[0]\nbest_model = make_model(best_hp)","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:07:19.417592Z","iopub.execute_input":"2021-10-15T20:07:19.417995Z","iopub.status.idle":"2021-10-15T20:07:19.507635Z","shell.execute_reply.started":"2021-10-15T20:07:19.417957Z","shell.execute_reply":"2021-10-15T20:07:19.506838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_model.save(\"best_model\")","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:07:19.511434Z","iopub.execute_input":"2021-10-15T20:07:19.511806Z","iopub.status.idle":"2021-10-15T20:07:20.552297Z","shell.execute_reply.started":"2021-10-15T20:07:19.511772Z","shell.execute_reply":"2021-10-15T20:07:20.551504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = best_model.fit(X_train, y_train, validation_split=0.2, epochs=50)","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:07:20.555487Z","iopub.execute_input":"2021-10-15T20:07:20.555756Z","iopub.status.idle":"2021-10-15T20:08:54.218712Z","shell.execute_reply.started":"2021-10-15T20:07:20.555712Z","shell.execute_reply":"2021-10-15T20:08:54.218019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = best_model.predict(X_valid)\n\npred = np.argmax(y_pred, axis=1)\n\nresult = pd.DataFrame(trainidt_valid)\nresult[1] = pred\n\nresult.columns = [\"BraTS21ID\", \"MGMT_value\"]\nresult2 = result.groupby(\"BraTS21ID\", as_index=False).mean()\nresult2","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:08:54.220178Z","iopub.execute_input":"2021-10-15T20:08:54.220985Z","iopub.status.idle":"2021-10-15T20:08:54.504012Z","shell.execute_reply.started":"2021-10-15T20:08:54.220927Z","shell.execute_reply":"2021-10-15T20:08:54.503276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result2 = result2.merge(train_df, on=\"BraTS21ID\")\n","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:08:54.505129Z","iopub.execute_input":"2021-10-15T20:08:54.505361Z","iopub.status.idle":"2021-10-15T20:08:54.520346Z","shell.execute_reply.started":"2021-10-15T20:08:54.505329Z","shell.execute_reply":"2021-10-15T20:08:54.519668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"auc = roc_auc_score(\n    result2.MGMT_value_y,\n    result2.MGMT_value_x,\n)\nprint(f\"Validation AUC={auc}\")","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:08:54.521340Z","iopub.execute_input":"2021-10-15T20:08:54.521596Z","iopub.status.idle":"2021-10-15T20:08:54.530290Z","shell.execute_reply.started":"2021-10-15T20:08:54.521554Z","shell.execute_reply":"2021-10-15T20:08:54.529410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = best_model.predict(X_test)\n\npred = np.argmax(y_pred, axis=1) #\n\nresult = pd.DataFrame(testidt)\nresult[1] = pred\npred","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:08:54.531828Z","iopub.execute_input":"2021-10-15T20:08:54.532088Z","iopub.status.idle":"2021-10-15T20:08:54.730884Z","shell.execute_reply.started":"2021-10-15T20:08:54.532055Z","shell.execute_reply":"2021-10-15T20:08:54.730208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result.columns=['BraTS21ID','MGMT_value']\n\nresult2 = result.groupby('BraTS21ID',as_index=False).mean()\nresult2['BraTS21ID'] = sample_submission['BraTS21ID']\n\nresult2","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:08:54.732254Z","iopub.execute_input":"2021-10-15T20:08:54.732497Z","iopub.status.idle":"2021-10-15T20:08:54.749615Z","shell.execute_reply.started":"2021-10-15T20:08:54.732466Z","shell.execute_reply":"2021-10-15T20:08:54.748824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Rounding... 0.907866 -> 0.9\nresult2['MGMT_value'] = result2['MGMT_value'].apply(lambda x:round(x*10)/10)\n# result2['MGMT_value'] = result2['MGMT_value'] # No rounding\nresult2.to_csv('submission.csv',index=False)\ns3 = result2[\"MGMT_value\"]","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:08:54.750973Z","iopub.execute_input":"2021-10-15T20:08:54.751235Z","iopub.status.idle":"2021-10-15T20:08:54.761770Z","shell.execute_reply.started":"2021-10-15T20:08:54.751198Z","shell.execute_reply":"2021-10-15T20:08:54.761003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1 = s1.reset_index()\ndf2 = s2.reset_index()\ndf3 = s3.reset_index()","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:08:54.763176Z","iopub.execute_input":"2021-10-15T20:08:54.763418Z","iopub.status.idle":"2021-10-15T20:08:54.771154Z","shell.execute_reply.started":"2021-10-15T20:08:54.763388Z","shell.execute_reply":"2021-10-15T20:08:54.770219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv(\"../input/rsna-miccai-brain-tumor-radiogenomic-classification/sample_submission.csv\", index_col=\"BraTS21ID\")\nFinalsubmission = submission.copy()\nFinalsubmission['MGMT_value'] =\\\ndf1['MGMT_value'].values*0.32+\\\ndf2['MGMT_value'].values*0.27+\\\ndf3['MGMT_value'].values*0.41","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:08:54.772564Z","iopub.execute_input":"2021-10-15T20:08:54.773042Z","iopub.status.idle":"2021-10-15T20:08:54.785404Z","shell.execute_reply.started":"2021-10-15T20:08:54.773006Z","shell.execute_reply":"2021-10-15T20:08:54.784718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Finalsubmission = Finalsubmission.reset_index()\n","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:11:26.454817Z","iopub.execute_input":"2021-10-15T20:11:26.455745Z","iopub.status.idle":"2021-10-15T20:11:26.461962Z","shell.execute_reply.started":"2021-10-15T20:11:26.455684Z","shell.execute_reply":"2021-10-15T20:11:26.460790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Finalsubmission","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:11:33.326178Z","iopub.execute_input":"2021-10-15T20:11:33.326447Z","iopub.status.idle":"2021-10-15T20:11:33.337757Z","shell.execute_reply.started":"2021-10-15T20:11:33.326420Z","shell.execute_reply":"2021-10-15T20:11:33.337113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Finalsubmission['BraTS21ID'] = Finalsubmission['BraTS21ID'].apply(lambda x: str(x).zfill(5))\nFsubmission = Finalsubmission.set_index('BraTS21ID')\nFsubmissionDict = Fsubmission['MGMT_value'].to_dict()\n\nlistOfStudyPaths = glob.glob('../input/rsna-miccai-brain-tumor-radiogenomic-classification/test/*')\nlistOfStudies = [eachPath.split('/')[-1] for eachPath in listOfStudyPaths]\n\npredList = []\nfor eachStudy in listOfStudies:\n    if eachStudy not in FsubmissionDict:\n        predList.append('0.500')\n    else:\n        score = float(FsubmissionDict[eachStudy])\n        predList.append(score)\n        \nsubmission = pd.DataFrame({'BraTS21ID':listOfStudies,'MGMT_value':predList})","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:11:39.256720Z","iopub.execute_input":"2021-10-15T20:11:39.257337Z","iopub.status.idle":"2021-10-15T20:11:39.277791Z","shell.execute_reply.started":"2021-10-15T20:11:39.257298Z","shell.execute_reply":"2021-10-15T20:11:39.277026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission=submission.sort_values(by=['BraTS21ID'], ascending=True)\nsubmission=submission.reset_index(drop=True)\nsubmission.head(5)","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:11:44.440210Z","iopub.execute_input":"2021-10-15T20:11:44.440480Z","iopub.status.idle":"2021-10-15T20:11:44.457260Z","shell.execute_reply.started":"2021-10-15T20:11:44.440450Z","shell.execute_reply":"2021-10-15T20:11:44.456459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:11:49.087653Z","iopub.execute_input":"2021-10-15T20:11:49.088250Z","iopub.status.idle":"2021-10-15T20:11:49.094799Z","shell.execute_reply.started":"2021-10-15T20:11:49.088212Z","shell.execute_reply":"2021-10-15T20:11:49.093847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''df1 = s1.reset_index()\ndf2 = s2.reset_index()\ndf3 = s3.reset_index()\n\nsubmission = pd.read_csv(\"../input/rsna-miccai-brain-tumor-radiogenomic-classification/sample_submission.csv\", index_col=\"BraTS21ID\")\nsubmission[\"MGMT_value\"] = 0\n\nMGMT_value=[]\nfor i in range(0,len(submission)):\n    value=(df1['MGMT_value'][i]*0.285+df2['MGMT_value'][i]*0.205 + df3['MGMT_value'][i]*0.475)\n    MGMT_value.append(value)\nsubmission[\"MGMT_value\"]=MGMT_value\nsubmission[\"MGMT_value\"].to_csv(\"submission.csv\")'''","metadata":{"execution":{"iopub.status.busy":"2021-10-15T20:08:54.892364Z","iopub.status.idle":"2021-10-15T20:08:54.892974Z","shell.execute_reply.started":"2021-10-15T20:08:54.892702Z","shell.execute_reply":"2021-10-15T20:08:54.892738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}