{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import sys \nimport os\nimport glob\nimport random\nimport time\nimport math\nimport numpy as np\nimport pandas as pd\nimport pydicom\nimport cv2\nfrom tqdm import tqdm\nimport torch\nimport torchvision\nfrom torch import nn\nfrom torch.nn import functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.cuda.amp import autocast, GradScaler\nfrom sklearn.model_selection import StratifiedKFold, train_test_split\nfrom sklearn.metrics import roc_auc_score","metadata":{"execution":{"iopub.status.busy":"2021-07-22T07:44:31.998242Z","iopub.execute_input":"2021-07-22T07:44:31.998626Z","iopub.status.idle":"2021-07-22T07:44:34.902041Z","shell.execute_reply.started":"2021-07-22T07:44:31.998550Z","shell.execute_reply":"2021-07-22T07:44:34.900852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CFG = {\n    'fold_num': 5,\n    'seed': 42,\n    'model': 'resnet10',\n    'epochs': 20,\n    'train_bs': 8, \n    'valid_bs': 8,\n    'lr': 1e-5, \n    'num_workers': 4,\n    'device': 0,\n    'fp16': False,\n}","metadata":{"execution":{"iopub.status.busy":"2021-07-22T07:44:34.906865Z","iopub.execute_input":"2021-07-22T07:44:34.907267Z","iopub.status.idle":"2021-07-22T07:44:34.916866Z","shell.execute_reply.started":"2021-07-22T07:44:34.907230Z","shell.execute_reply":"2021-07-22T07:44:34.915774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\nseed_everything(CFG['seed'])","metadata":{"execution":{"iopub.status.busy":"2021-07-22T07:44:34.923460Z","iopub.execute_input":"2021-07-22T07:44:34.925757Z","iopub.status.idle":"2021-07-22T07:44:34.940125Z","shell.execute_reply.started":"2021-07-22T07:44:34.925718Z","shell.execute_reply":"2021-07-22T07:44:34.939128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.cuda.set_device(CFG['device'])\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"execution":{"iopub.status.busy":"2021-07-22T07:44:34.945037Z","iopub.execute_input":"2021-07-22T07:44:34.947294Z","iopub.status.idle":"2021-07-22T07:44:35.040298Z","shell.execute_reply.started":"2021-07-22T07:44:34.947250Z","shell.execute_reply":"2021-07-22T07:44:35.038981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(\"../input/rsna-miccai-brain-tumor-radiogenomic-classification/train_labels.csv\")\ntest_df = pd.read_csv(\"../input/rsna-miccai-brain-tumor-radiogenomic-classification/sample_submission.csv\")\n\ntest_df","metadata":{"execution":{"iopub.status.busy":"2021-07-22T07:44:35.045641Z","iopub.execute_input":"2021-07-22T07:44:35.048372Z","iopub.status.idle":"2021-07-22T07:44:35.099911Z","shell.execute_reply.started":"2021-07-22T07:44:35.048328Z","shell.execute_reply":"2021-07-22T07:44:35.099098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MyDataset(Dataset):\n    def __init__(self, df):\n        self.df = df.reset_index()\n        self.input_D=64\n        self.input_H=128\n        self.input_W=128\n          \n    def __len__(self):\n        return len(self.df)\n    \n    def load_dicom(self, path):\n        dicom = pydicom.read_file(path)\n        data = dicom.pixel_array\n        data = data - np.min(data)\n        if np.max(data) != 0:\n            data = data / np.max(data)\n        return data\n    \n    def resize(self, data):\n        [depth, height, width] = data.shape\n        scale = [self.input_D*1.0/depth, self.input_H*1.0/height, self.input_W*1.0/width]  \n        data = ndimage.interpolation.zoom(data, scale, order=0)\n\n        return data\n    \n    def __getitem__(self, index):\n        _id = self.df[\"BraTS21ID\"].values[index]\n        patient_path = f\"../input/rsna-miccai-brain-tumor-radiogenomic-classification/test/{str(_id).zfill(5)}/\"\n        \n        channels = []\n        for t in (\"FLAIR\", \"T1w\", \"T1wCE\"):  \n            t_paths = sorted(\n                glob.glob(os.path.join(patient_path, t, \"*\")), \n                key=lambda x: int(x[:-4].split(\"-\")[-1]),\n            )\n            \n#             channel = []\n#             for i in range(len(t_paths)):\n#                 channel.append((self.load_dicom(t_paths[i])).astype(np.float32))\n#             channel = self.resize(np.array(channel))\n\n            length = len(t_paths)\n            n = 64 + 1\n            r = [math.floor(i / n * length) for i in range(1, n)]\n            channel = []\n            for i in r:\n                channel.append(cv2.resize(self.load_dicom(t_paths[i]), (128, 128)).astype(np.float32))\n            \n            channels.append(channel)\n        image = np.array(channels)\n        \n        y = self.df[\"MGMT_value\"].values[index]\n        \n        return image, _id","metadata":{"execution":{"iopub.status.busy":"2021-07-22T07:44:35.103862Z","iopub.execute_input":"2021-07-22T07:44:35.106027Z","iopub.status.idle":"2021-07-22T07:44:35.125616Z","shell.execute_reply.started":"2021-07-22T07:44:35.105988Z","shell.execute_reply":"2021-07-22T07:44:35.124558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MyDataset(test_df)[0][0].shape","metadata":{"execution":{"iopub.status.busy":"2021-07-22T07:44:35.130492Z","iopub.execute_input":"2021-07-22T07:44:35.133666Z","iopub.status.idle":"2021-07-22T07:44:36.661500Z","shell.execute_reply.started":"2021-07-22T07:44:35.133611Z","shell.execute_reply":"2021-07-22T07:44:36.660520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.autograd import Variable\nfrom functools import partial\n\n__all__ = [\n    'resnet10', 'resnet18', 'resnet34', 'resnet50', 'resnet101',\n    'resnet152', 'resnet200'\n]\n\ndef conv3x3x3(in_planes, out_planes, stride=1, dilation=1):\n    # 3x3x3 convolution with padding\n    return nn.Conv3d(\n        in_planes,\n        out_planes,\n        kernel_size=3,\n        dilation=dilation,\n        stride=stride,\n        padding=dilation,\n        bias=False)\n\ndef downsample_basic_block(x, planes, stride, no_cuda=False):\n    out = F.avg_pool3d(x, kernel_size=1, stride=stride)\n    zero_pads = torch.Tensor(out.size(0), planes - out.size(1), out.size(2), out.size(3), out.size(4)).zero_()\n    if not no_cuda:\n        if isinstance(out.data, torch.cuda.FloatTensor):\n            zero_pads = zero_pads.cuda()\n\n    out = Variable(torch.cat([out.data, zero_pads], dim=1))\n    return out\n\nclass BasicBlock(nn.Module):\n    expansion = 1\n\n    def __init__(self, inplanes, planes, stride=1, dilation=1, downsample=None):\n        super(BasicBlock, self).__init__()\n        self.conv1 = conv3x3x3(inplanes, planes, stride=stride, dilation=dilation)\n        self.bn1 = nn.BatchNorm3d(planes)\n        self.relu = nn.ReLU(inplace=True)\n        self.conv2 = conv3x3x3(planes, planes, dilation=dilation)\n        self.bn2 = nn.BatchNorm3d(planes)\n        self.downsample = downsample\n        self.stride = stride\n        self.dilation = dilation\n\n    def forward(self, x):\n        residual = x\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n        out = self.conv2(out)\n        out = self.bn2(out)\n\n        if self.downsample is not None:\n            residual = self.downsample(x)\n\n        out += residual\n        out = self.relu(out)\n        return out\n\n\nclass Bottleneck(nn.Module):\n    expansion = 4\n\n    def __init__(self, inplanes, planes, stride=1, dilation=1, downsample=None):\n        super(Bottleneck, self).__init__()\n        self.conv1 = nn.Conv3d(inplanes, planes, kernel_size=1, bias=False)\n        self.bn1 = nn.BatchNorm3d(planes)\n        self.conv2 = nn.Conv3d(\n            planes, planes, kernel_size=3, stride=stride, dilation=dilation, padding=dilation, bias=False)\n        self.bn2 = nn.BatchNorm3d(planes)\n        self.conv3 = nn.Conv3d(planes, planes * 4, kernel_size=1, bias=False)\n        self.bn3 = nn.BatchNorm3d(planes * 4)\n        self.relu = nn.ReLU(inplace=True)\n        self.downsample = downsample\n        self.stride = stride\n        self.dilation = dilation\n\n    def forward(self, x):\n        residual = x\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n\n        out = self.conv2(out)\n        out = self.bn2(out)\n        out = self.relu(out)\n\n        out = self.conv3(out)\n        out = self.bn3(out)\n\n        if self.downsample is not None:\n            residual = self.downsample(x)\n\n        out += residual\n        out = self.relu(out)\n        return out\n\nclass ResNet3D(nn.Module):\n\n    def __init__(self,\n                 block,\n                 layers,\n                 shortcut_type='B',\n                 num_class = 2,\n                 no_cuda=False):\n\n        self.inplanes = 64\n        self.no_cuda = no_cuda\n        super(ResNet3D, self).__init__()\n\n        # 3D conv net\n        self.conv1 = nn.Conv3d(3, 64, kernel_size=7, stride=(2, 2, 2), padding=(3, 3, 3), bias=False)\n        # self.conv1 = nn.Conv3d(1, 64, kernel_size=7, stride=(2, 2, 2), padding=(3, 3, 3), bias=False)\n        self.bn1 = nn.BatchNorm3d(64)\n        self.relu = nn.ReLU(inplace=True)\n        self.maxpool = nn.MaxPool3d(kernel_size=(3, 3, 3), stride=2, padding=1)\n        self.layer1 = self._make_layer(block, 64, layers[0], shortcut_type)\n        self.layer2 = self._make_layer(\n            block, 64*2, layers[1], shortcut_type, stride=2)\n        self.layer3 = self._make_layer(\n            block, 128*2, layers[2], shortcut_type, stride=1, dilation=2)\n        self.layer4 = self._make_layer(\n            block, 256*2, layers[3], shortcut_type, stride=1, dilation=4)\n\n        self.fea_dim = 256*2 * block.expansion\n        self.dropout = nn.Dropout(0.5)\n        self.fc = nn.Sequential(nn.Linear(self.fea_dim, num_class, bias=True))\n\n        for m in self.modules():\n            if isinstance(m, nn.Conv3d):\n                m.weight = nn.init.kaiming_normal_(m.weight, mode='fan_out')\n            elif isinstance(m, nn.BatchNorm3d):\n                m.weight.data.fill_(1)\n                m.bias.data.zero_()\n\n    def _make_layer(self, block, planes, blocks, shortcut_type, stride=1, dilation=1):\n        downsample = None\n        if stride != 1 or self.inplanes != planes * block.expansion:\n\n            if shortcut_type == 'A':\n                downsample = partial(\n                    downsample_basic_block,\n                    planes=planes * block.expansion,\n                    stride=stride,\n                    no_cuda=self.no_cuda)\n            else:\n                downsample = nn.Sequential(\n                    nn.Conv3d(\n                        self.inplanes,\n                        planes * block.expansion,\n                        kernel_size=1,\n                        stride=stride,\n                        bias=False), nn.BatchNorm3d(planes * block.expansion))\n\n        layers = []\n        layers.append(block(self.inplanes, planes, stride=stride, dilation=dilation, downsample=downsample))\n        self.inplanes = planes * block.expansion\n        for i in range(1, blocks):\n            layers.append(block(self.inplanes, planes, dilation=dilation))\n\n        return nn.Sequential(*layers)\n\n    def forward(self, x):\n        x = self.conv1( x)\n        x = self.bn1(x)\n        x = self.relu(x)\n        x = self.maxpool(x)\n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = self.layer3(x)\n        x = self.layer4(x)\n\n        x = F.adaptive_avg_pool3d(x, (1, 1, 1))\n        emb_3d = x.view((-1, self.fea_dim))\n        emb_3d = self.dropout(emb_3d)\n        out = self.fc(emb_3d)\n        return out\n\n\ndef resnet10(**kwargs):\n    \"\"\"Constructs a ResNet-18 model.\n    \"\"\"\n    model = ResNet3D(BasicBlock, [1, 1, 1, 1],**kwargs)\n    return model\n\ndef resnet3d_10(**kwargs):\n    \"\"\"Constructs a ResNet-18 model.\n    \"\"\"\n    model = ResNet3D(BasicBlock, [1, 1, 1, 1], **kwargs)\n    return model\n\ndef resnet18(**kwargs):\n    \"\"\"Constructs a ResNet-18 model.\n    \"\"\"\n    model = ResNet3D(BasicBlock, [2, 2, 2, 2], **kwargs)\n    return model\n\ndef resnet34(**kwargs):\n    \"\"\"Constructs a ResNet-34 model.\n    \"\"\"\n    model = ResNet3D(BasicBlock, [3, 4, 6, 3], **kwargs)\n    return model\n\ndef resnet50(**kwargs):\n    \"\"\"Constructs a ResNet-50 model.\n    \"\"\"\n    model = ResNet3D(Bottleneck, [3, 4, 6, 3], **kwargs)\n    return model\n\ndef resnet101(**kwargs):\n    \"\"\"Constructs a ResNet-101 model.\n    \"\"\"\n    model = ResNet3D(Bottleneck, [3, 4, 23, 3], **kwargs)\n    return model\n\ndef resnet152(**kwargs):\n    \"\"\"Constructs a ResNet-101 model.\n    \"\"\"\n    model = ResNet3D(Bottleneck, [3, 8, 36, 3], **kwargs)\n    return model\n\ndef resnet200(**kwargs):\n    \"\"\"Constructs a ResNet-101 model.\n    \"\"\"\n    model = ResNet3D(Bottleneck, [3, 24, 36, 3], **kwargs)\n    return model","metadata":{"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2021-07-22T07:44:36.664413Z","iopub.execute_input":"2021-07-22T07:44:36.664670Z","iopub.status.idle":"2021-07-22T07:44:36.705603Z","shell.execute_reply.started":"2021-07-22T07:44:36.664644Z","shell.execute_reply":"2021-07-22T07:44:36.704778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generate_model(opt):\n    assert opt.model in [\n        'resnet'\n    ]\n\n    print('model depth: ',opt.model_depth)\n\n    if opt.model == 'resnet':\n        assert opt.model_depth in [10, 18, 34, 50, 101, 152, 200]\n        \n        if opt.model_depth == 10:\n            model = resnet10(\n                # sample_input_W=opt.input_W,\n                # sample_input_H=opt.input_H,\n                # sample_input_D=opt.input_D,\n                shortcut_type=opt.resnet_shortcut,\n                no_cuda=opt.no_cuda,\n                # num_seg_classes=opt.n_seg_classes,\n            )\n        elif opt.model_depth == 18:\n            model = resnet.resnet18(\n                sample_input_W=opt.input_W,\n                sample_input_H=opt.input_H,\n                sample_input_D=opt.input_D,\n                shortcut_type=opt.resnet_shortcut,\n                no_cuda=opt.no_cuda,\n                num_seg_classes=opt.n_seg_classes)\n        elif opt.model_depth == 34:\n            model = resnet.resnet34(\n                sample_input_W=opt.input_W,\n                sample_input_H=opt.input_H,\n                sample_input_D=opt.input_D,\n                shortcut_type=opt.resnet_shortcut,\n                no_cuda=opt.no_cuda,\n                num_seg_classes=opt.n_seg_classes)\n        elif opt.model_depth == 50:\n            model = resnet.resnet50(\n                sample_input_W=opt.input_W,\n                sample_input_H=opt.input_H,\n                sample_input_D=opt.input_D,\n                shortcut_type=opt.resnet_shortcut,\n                no_cuda=opt.no_cuda,\n                num_seg_classes=opt.n_seg_classes)\n        elif opt.model_depth == 101:\n            model = resnet.resnet101(\n                sample_input_W=opt.input_W,\n                sample_input_H=opt.input_H,\n                sample_input_D=opt.input_D,\n                shortcut_type=opt.resnet_shortcut,\n                no_cuda=opt.no_cuda,\n                num_seg_classes=opt.n_seg_classes)\n        elif opt.model_depth == 152:\n            model = resnet.resnet152(\n                sample_input_W=opt.input_W,\n                sample_input_H=opt.input_H,\n                sample_input_D=opt.input_D,\n                shortcut_type=opt.resnet_shortcut,\n                no_cuda=opt.no_cuda,\n                num_seg_classes=opt.n_seg_classes)\n        elif opt.model_depth == 200:\n            model = resnet.resnet200(\n                sample_input_W=opt.input_W,\n                sample_input_H=opt.input_H,\n                sample_input_D=opt.input_D,\n                shortcut_type=opt.resnet_shortcut,\n                no_cuda=opt.no_cuda,\n                num_seg_classes=opt.n_seg_classes)\n    \n    if not opt.no_cuda:\n            model = model.cuda()\n            net_dict = model.state_dict()\n    else:\n        net_dict = model.state_dict()\n    \n    # load pretrain\n    if opt.pretrain_path:\n        print ('loading pretrained model {}'.format(opt.pretrain_path))\n        pretrain = torch.load(opt.pretrain_path)\n        pretrain_dict = {k: v for k, v in pretrain['state_dict'].items() if k in net_dict.keys() and 'conv1' not in k}\n        print(pretrain_dict.keys())\n\n        net_dict.update(pretrain_dict)\n        model.load_state_dict(net_dict)\n\n        new_parameters = []\n        for pname, p in model.named_parameters():\n            for layer_name in opt.new_layer_names:\n                if pname.find(layer_name) >= 0:\n                    new_parameters.append(p)\n                    break\n\n        new_parameters_id = list(map(id, new_parameters))\n        base_parameters = list(filter(lambda p: id(p) not in new_parameters_id, model.parameters()))\n        parameters = {'base_parameters': base_parameters,\n                      'new_parameters': new_parameters}\n\n        return model, parameters\n\n    return model, model.parameters()","metadata":{"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2021-07-22T07:44:36.708356Z","iopub.execute_input":"2021-07-22T07:44:36.708800Z","iopub.status.idle":"2021-07-22T07:44:36.725816Z","shell.execute_reply.started":"2021-07-22T07:44:36.708760Z","shell.execute_reply":"2021-07-22T07:44:36.724883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config :\n    n_seg_classes=2\n    input_D=64\n    input_H=128\n    input_W=128\n    pretrain_path= None\n    new_layer_names=['conv_seg']\n    no_cuda = False\n    model='resnet'\n    model_depth=10\n    resnet_shortcut = 'B'","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-07-22T07:44:36.727600Z","iopub.execute_input":"2021-07-22T07:44:36.728246Z","iopub.status.idle":"2021-07-22T07:44:36.736597Z","shell.execute_reply.started":"2021-07-22T07:44:36.728206Z","shell.execute_reply":"2021-07-22T07:44:36.735789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_set = MyDataset(test_df)\ntest_loader = DataLoader(test_set, batch_size=CFG['valid_bs'], shuffle=False, num_workers=CFG['num_workers'])","metadata":{"execution":{"iopub.status.busy":"2021-07-22T07:44:36.737819Z","iopub.execute_input":"2021-07-22T07:44:36.738184Z","iopub.status.idle":"2021-07-22T07:44:36.746182Z","shell.execute_reply.started":"2021-07-22T07:44:36.738152Z","shell.execute_reply":"2021-07-22T07:44:36.745340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ensemble_pred = []\n\nfor fold in range(5):\n    model, parameters = generate_model(Config())\n    model.load_state_dict(torch.load('../input/fork-of-brain-tumor-train/{}_fold{}.pt'.format(CFG['model'], fold)))\n    model.eval()\n\n    y_pred = []\n    ids = []\n\n    for e, batch in tqdm(enumerate(test_loader), total=len(test_loader)):\n        with torch.no_grad():\n            img, _id = [x.to(device) for x in batch]\n            pred = model(img).softmax(1)[:,1].cpu().numpy()   \n            y_pred.extend(pred)\n            ids.extend(_id.cpu().numpy().tolist())\n    \n    ensemble_pred.append(np.array(y_pred))\n    \nensemble_pred = np.mean(ensemble_pred, 0)","metadata":{"execution":{"iopub.status.busy":"2021-07-22T07:44:36.747517Z","iopub.execute_input":"2021-07-22T07:44:36.747869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({\"BraTS21ID\": ids, \"MGMT_value\": y_pred})\nsubmission.to_csv(\"submission.csv\", index=False)\nsubmission","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}