{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# !pip install --upgrade torch torchvision\nimport sys\nsys.path.insert(1, '../input/utility-scripts')\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\n\nimport DenseNet3D\nimport Simple_3dCNN\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport re\nimport os\nimport random\nimport time\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn import model_selection as sk_model_selection\n\nimport transforms3D\n","metadata":{"execution":{"iopub.status.busy":"2021-10-14T05:52:28.807523Z","iopub.execute_input":"2021-10-14T05:52:28.807744Z","iopub.status.idle":"2021-10-14T05:52:28.81663Z","shell.execute_reply.started":"2021-10-14T05:52:28.807708Z","shell.execute_reply":"2021-10-14T05:52:28.81588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# class FocalLoss(nn.Module):\n\n#     def __init__(self, gamma=0, eps=1e-7):\n#         super(FocalLoss, self).__init__()\n#         self.gamma = gamma\n#         self.eps = eps\n\n#     def forward(self, input, target):\n#         logit = F.sigmoid(input)\n#         logit = logit.clamp(self.eps, 1. - self.eps)\n#         pt = logit * target + (1- logit) * (1 - target)\n\n#         loss = -1 * ((1-pt) ** self.gamma) * torch.log(pt) # cross entropy\n\n#         return loss.mean()","metadata":{"execution":{"iopub.status.busy":"2021-10-14T05:52:28.81778Z","iopub.execute_input":"2021-10-14T05:52:28.818Z","iopub.status.idle":"2021-10-14T05:52:28.831913Z","shell.execute_reply.started":"2021-10-14T05:52:28.817959Z","shell.execute_reply":"2021-10-14T05:52:28.831157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# reproduceable\n\ntorch.manual_seed(0)\nnp.random.seed(0)\nrandom.seed(0)","metadata":{"collapsed":false,"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-10-14T05:52:31.509424Z","iopub.execute_input":"2021-10-14T05:52:31.50969Z","iopub.status.idle":"2021-10-14T05:52:31.516026Z","shell.execute_reply.started":"2021-10-14T05:52:31.509663Z","shell.execute_reply":"2021-10-14T05:52:31.515377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_csv = pd.read_csv('../input/rsna-miccai-brain-tumor-radiogenomic-classification/train_labels.csv', )\nlabel_csv = label_csv[(label_csv.BraTS21ID!=109) & (label_csv.BraTS21ID!=123) & (label_csv.BraTS21ID!=709)]\n\ntraincsv_path = './train_label.csv'\nvalcsv_path = './test_label.csv'\ntrain_portion = 0.8\n\nmaskdilate_kernelsize=5\n\nimg_dir = '../input/imagemask-voxel/train_voxel/train_voxel/'\nmask_dir = '../input/imagemask-voxel/trainmask_voxel/trainmask_voxel/'\n\nsample_indexs = np.arange(len(label_csv))\nnp.random.shuffle(sample_indexs)\ntrain_indexs = sample_indexs[:int(len(label_csv) * train_portion)]\nval_indexs = sample_indexs[int(len(label_csv) * train_portion):]\nprint(len(set(train_indexs)), len(set(val_indexs)), len(set(np.concatenate((train_indexs, val_indexs)))))\n\n#%%\n\ntrain_csv = label_csv.iloc[np.array(train_indexs)]\nval_csv = label_csv.iloc[np.array(val_indexs)]\n\ntrain_csv.to_csv(traincsv_path, index=False)\nval_csv.to_csv(valcsv_path, index=False)","metadata":{"execution":{"iopub.status.busy":"2021-10-14T05:52:32.211901Z","iopub.execute_input":"2021-10-14T05:52:32.212783Z","iopub.status.idle":"2021-10-14T05:52:32.250015Z","shell.execute_reply.started":"2021-10-14T05:52:32.212741Z","shell.execute_reply":"2021-10-14T05:52:32.249193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # build model\n# device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n# # model = DenseNet3D.densenet121(pretrained=False, progress=True, ).to(device)\n# model = DenseNet3D.DenseNet(growth_rate = 32, block_config = (6, 12, 24, 16), num_init_features = 64, bn_size = 4, drop_rate = 0).to(device)\n# print(torch.cuda.device_count())\n# model.train()","metadata":{"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2021-10-10T16:58:38.845563Z","iopub.execute_input":"2021-10-10T16:58:38.84584Z","iopub.status.idle":"2021-10-10T16:58:44.566075Z","shell.execute_reply.started":"2021-10-10T16:58:38.845808Z","shell.execute_reply":"2021-10-10T16:58:44.564202Z"},"jupyter":{"outputs_hidden":true},"collapsed":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n# model = DenseNet3D.densenet121(pretrained=False, progress=True, ).to(device)\nmodel = Simple_3dCNN.CNN3D().to(device)\nprint(torch.cuda.device_count())\nmodel.train()","metadata":{"execution":{"iopub.status.busy":"2021-10-14T05:52:34.553988Z","iopub.execute_input":"2021-10-14T05:52:34.554283Z","iopub.status.idle":"2021-10-14T05:52:34.620691Z","shell.execute_reply.started":"2021-10-14T05:52:34.554242Z","shell.execute_reply":"2021-10-14T05:52:34.619865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # model initialize with ImageNet training result\n\n# model_urls = {\n#     'densenet121': 'https://download.pytorch.org/models/densenet121-a639ec97.pth',\n#     'densenet169': 'https://download.pytorch.org/models/densenet169-b2777c0a.pth',\n#     'densenet201': 'https://download.pytorch.org/models/densenet201-c1103571.pth',\n#     'densenet161': 'https://download.pytorch.org/models/densenet161-8d451a50.pth',\n# }\n# state_dict = torch.hub.load_state_dict_from_url(model_urls['densenet121'], progress=True, map_location=device)\n\n# pattern = re.compile(\n#     r'^(.*denselayer\\d+\\.(?:norm|relu|conv))\\.((?:[12])\\.(?:weight|bias|running_mean|running_var))$')\n\n# for key in list(state_dict.keys()):\n#     res = pattern.match(key)\n#     if res:\n#         new_key = res.group(1) + res.group(2)\n#         if len(state_dict[key].shape) == 4:\n#             state_3d = torch.unsqueeze(state_dict[key], dim=2)\n#             if state_3d.shape[-1] != 1 :\n#                 state_3d = torch.cat(((state_3d,)*state_3d.shape[-1]), dim=2)\n#             state_dict[new_key] = state_3d.to(device)\n#             print(f'key{new_key} shape changed from {state_dict[key].shape} to{state_dict[new_key].shape}')\n#         else:\n#             state_dict[new_key] = state_dict[key].to(device)\n#         del state_dict[key]\n\n#     elif 'conv' in key:\n#         if len(state_dict[key].shape) == 4:\n#             state_3d = torch.unsqueeze(state_dict[key], dim=2)\n#             if state_3d.shape[-1] != 1 :\n#                 state_3d = torch.cat(((state_3d,)*state_3d.shape[-1]), dim=2)\n#             print(f'key{key} shape changed from {state_dict[key].shape} to{state_3d.shape}')\n#             state_dict[key] = state_3d.to(device)\n\n# featureused_RGB = 0\n# conv0_weight = torch.unsqueeze(state_dict['features.conv0.weight'][:, featureused_RGB, : ,:, :], dim=1).to(device)\n# print(f\"key features.conv0.weight shape changed from {state_dict['features.conv0.weight'].shape}\", end='')\n# state_dict['features.conv0.weight'] = torch.cat((conv0_weight, ) * 4, dim=1).to(device)\n# print(f\"to {state_dict['features.conv0.weight'].shape} duplicating original feature map {featureused_RGB}\")\n\n# classifierused = 0\n# classifier_weight = torch.unsqueeze(state_dict['classifier.weight'][classifierused, :], dim=0).to(device)\n# print(f\"key fclassifier.weight shape changed from {state_dict['classifier.weight'].shape}\", end='')\n# state_dict['classifier.weight'] = classifier_weight.to(device)\n# print(f\"to {state_dict['classifier.weight'].shape} taking one dimension of  original feature map {featureused_RGB}\")\n\n# classifierused = 0\n# classifier_bias = torch.tensor([state_dict['classifier.bias'][classifierused]]).to(device)\n# print(f\"key classifier.bias shape changed from {state_dict['classifier.bias'].shape}\", end='')\n# state_dict['classifier.bias'] = classifier_bias.to(device)\n# print(f\"to {state_dict['classifier.bias'].shape} taking one dimension of original feature map {featureused_RGB}\")\n\n\n# model.load_state_dict(state_dict)","metadata":{"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2021-10-10T16:59:19.55001Z","iopub.execute_input":"2021-10-10T16:59:19.550356Z","iopub.status.idle":"2021-10-10T16:59:21.660352Z","shell.execute_reply.started":"2021-10-10T16:59:19.55032Z","shell.execute_reply":"2021-10-10T16:59:21.659716Z"},"jupyter":{"outputs_hidden":true,"source_hidden":true},"collapsed":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def plot3view_mask_4C(image_C1, image_C2, image_C3, image_C4, mask=None, title=''):\n\n#         fig = plt.figure(figsize=(12, 25))\n#         fig.suptitle(f\"{title}\", fontsize=24)\n#         if mask is not None:\n#             ax = fig.add_subplot(10 ,3, 1)\n#             ax.set_title(f'mask x-y{mask.shape}')\n#             ax.imshow(mask[mask.shape[0] // 2], cmap='gray')\n#             ax = fig.add_subplot(10 ,3, 2)\n#             ax.set_title('mask y-z')\n#             ax.imshow(mask[:, mask.shape[1] // 2], cmap='gray')\n#             ax = fig.add_subplot(10 ,3, 3)\n#             ax.set_title('mask z-x')\n#             ax.imshow(mask[:, :, mask.shape[2] // 2], cmap='gray')\n\n#         ax = fig.add_subplot(10 ,3, 4)\n#         ax.set_title(f'FLAIR x-y{image_C1.shape}')\n#         ax.imshow(image_C1[image_C1.shape[0] // 2], cmap='gray')\n#         ax = fig.add_subplot(10 ,3, 5)\n#         ax.set_title('FLAIR y-z')\n#         ax.imshow(image_C1[:, image_C1.shape[1] // 2], cmap='gray')\n#         ax = fig.add_subplot(10 ,3, 6)\n#         ax.set_title('FLAIR z-x')\n#         ax.imshow(image_C1[:, :, image_C1.shape[2] // 2], cmap='gray')\n\n#         ax = fig.add_subplot(10 ,3, 7)\n#         ax.set_title(f'T1w x-y{image_C1.shape}')\n#         ax.imshow(image_C2[image_C1.shape[0] // 2], cmap='gray')\n#         ax = fig.add_subplot(10 ,3, 8)\n#         ax.set_title('T1w y-z')\n#         ax.imshow(image_C2[:, image_C1.shape[1] // 2], cmap='gray')\n#         ax = fig.add_subplot(10 ,3, 9)\n#         ax.set_title('T1w z-x')\n#         ax.imshow(image_C2[:, :, image_C1.shape[2] // 2], cmap='gray')\n\n#         ax = fig.add_subplot(10 ,3, 10)\n#         ax.set_title(f'FLAIR x-y{image_C1.shape}')\n#         ax.imshow(image_C3[image_C1.shape[0] // 2], cmap='gray')\n#         ax = fig.add_subplot(10 ,3, 11)\n#         ax.set_title('FLAIR y-z')\n#         ax.imshow(image_C3[:, image_C1.shape[1] // 2], cmap='gray')\n#         ax = fig.add_subplot(10 ,3, 12)\n#         ax.set_title('FLAIR z-x')\n#         ax.imshow(image_C3[:, :, image_C1.shape[2] // 2], cmap='gray')\n\n#         ax = fig.add_subplot(10 ,3, 13)\n#         ax.set_title(f'FLAIR x-y{image_C1.shape}')\n#         ax.imshow(image_C4[image_C1.shape[0] // 2], cmap='gray')\n#         ax = fig.add_subplot(10 ,3, 14)\n#         ax.set_title('FLAIR y-z')\n#         ax.imshow(image_C4[:, image_C1.shape[1] // 2], cmap='gray')\n#         ax = fig.add_subplot(10 ,3, 15)\n#         ax.set_title('FLAIR z-x')\n#         ax.imshow(image_C4[:, :, image_C1.shape[2] // 2], cmap='gray')","metadata":{"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2021-10-10T16:59:26.193238Z","iopub.execute_input":"2021-10-10T16:59:26.193492Z","iopub.status.idle":"2021-10-10T16:59:26.210485Z","shell.execute_reply.started":"2021-10-10T16:59:26.193464Z","shell.execute_reply":"2021-10-10T16:59:26.209833Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dataloader initialize\n\nclass Image4CDataSet(Dataset):\n    def __init__(self, label_csv, img_dir, mask_dir=None, maskdilate_kernelsize=3, transform=None, transform_mask=None,\n                 target_transform=None, Training = True):\n        \n        self.img_labels = pd.read_csv(label_csv, )\n        self.mask_dir = mask_dir\n        self.img_dir_C1 = os.path.join(img_dir, 'FLAIR')\n        self.img_dir_C2 = os.path.join(img_dir, 'T1w')\n        self.img_dir_C3 = os.path.join(img_dir, 'T1wCE')\n        self.img_dir_C4 = os.path.join(img_dir, 'T2w')\n\n        self.maskdilate_kernelsize = maskdilate_kernelsize\n\n        self.transform = transform\n        self.transform_mask = transform_mask\n        self.target_transform = target_transform\n\n        self.Training = Training\n    def __len__(self):\n        return len(self.img_labels)\n\n    def __getitem__(self, idx):\n#         print(f'__getitem__ called for id: {idx}')\n              \n        img_path_C1 = os.path.join(self.img_dir_C1, f'{self.img_labels.iloc[idx, 0]:05}.npy')\n        img_path_C2 = os.path.join(self.img_dir_C2, f'{self.img_labels.iloc[idx, 0]:05}.npy')\n        img_path_C3 = os.path.join(self.img_dir_C3, f'{self.img_labels.iloc[idx, 0]:05}.npy')\n        img_path_C4 = os.path.join(self.img_dir_C4, f'{self.img_labels.iloc[idx, 0]:05}.npy')\n\n        image_C1 = torch.from_numpy(np.load(img_path_C1)).type(torch.float)\n        image_C2 = torch.from_numpy(np.load(img_path_C2)).type(torch.float)\n        image_C3 = torch.from_numpy(np.load(img_path_C3)).type(torch.float)\n        image_C4 = torch.from_numpy(np.load(img_path_C4)).type(torch.float)\n        \n\n        image4C = torch.cat((torch.unsqueeze(image_C1, 0),\n                     torch.unsqueeze(image_C2, 0),\n                     torch.unsqueeze(image_C3, 0),\n                     torch.unsqueeze(image_C4, 0),), dim=0)\n#         plot3view_mask_4C(image4C[0, ...], image4C[1, ...], image4C[2, ...], image4C[3, ...],\n#                               None, title={f'id: {idx}; operation done: maske dilate, cutoutl, rotate, Gaussian'})\n\n        if self.Training:\n            label = torch.tensor(self.img_labels.iloc[idx, 1] *1.).type(torch.float)\n        else:\n            label = f'{self.img_labels.iloc[idx, 0]:05}'\n\n        if self.transform:\n            mask_path = os.path.join(self.mask_dir, f'{self.img_labels.iloc[idx, 0]:05}.npy')\n            mask = torch.from_numpy(np.load(mask_path)).type(torch.float)\n\n            assert self.maskdilate_kernelsize % 2 == 1, 'maskdilate_kernelsize must be odd'\n            mask = torch.nn.functional.max_pool3d(input = mask.unsqueeze(0), kernel_size=self.maskdilate_kernelsize,\n                                              stride=1, padding=int((self.maskdilate_kernelsize-1)/2), dilation=1,\n                                              return_indices=False, ceil_mode=False).squeeze(0)\n            \n#             plot3view_mask_4C(image4C[0, ...], image4C[1, ...], image4C[2, ...], image4C[3, ...],\n#                               mask, title={f'id: {idx}; operation done: None'})\n#             print('None:')\n#             print(torch.max(image4C[0, ...]), torch.max(image4C[1, ...]), torch.max(image4C[2, ...]), torch.max(image4C[3, ...]))\n#             print(torch.min(image4C[0, ...]), torch.min(image4C[1, ...]), torch.min(image4C[2, ...]), torch.min(image4C[3, ...]))\n            \n            image4C = self.transform_mask(image4C, mask)\n#             plot3view_mask_4C(image4C[0, ...], image4C[1, ...], image4C[2, ...], image4C[3, ...],\n#                               mask, title={f'id: {idx}; operation done: maske dilate, cutout'})\n            \n            \n#             print('Cutout:')\n#             print(torch.max(image4C[0, ...]), torch.max(image4C[1, ...]), torch.max(image4C[2, ...]), torch.max(image4C[3, ...]))\n#             print(torch.min(image4C[0, ...]), torch.min(image4C[1, ...]), torch.min(image4C[2, ...]), torch.min(image4C[3, ...]))\n            image4C = self.transform[0](image4C)\n#             plot3view_mask_4C(image4C[0, ...], image4C[1, ...], image4C[2, ...], image4C[3, ...],\n#                               mask, title={f'id: {idx}; operation done: maske dilate, cutoutl, rotate'})\n\n#             print('cutoutl, rotate:')\n#             print(torch.max(image4C[0, ...]), torch.max(image4C[1, ...]), torch.max(image4C[2, ...]), torch.max(image4C[3, ...]))\n#             print(torch.min(image4C[0, ...]), torch.min(image4C[1, ...]), torch.min(image4C[2, ...]), torch.min(image4C[3, ...]))\n            image4C = self.transform[1](image4C)\n#             image4C = self.transform[2](image4C)\n\n#             print(torch.max(image4C[0, ...]), torch.max(image4C[1, ...]), torch.max(image4C[2, ...]), torch.max(image4C[3, ...]))\n#             print(torch.min(image4C[0, ...]), torch.min(image4C[1, ...]), torch.min(image4C[2, ...]), torch.min(image4C[3, ...]))\n#             plot3view_mask_4C(image4C[0, ...], image4C[1, ...], image4C[2, ...], image4C[3, ...],\n#                               mask, title={f'id: {idx}; operation done: maske dilate, cutoutl, rotate, Gaussian'})\n        if self.target_transform:\n            label = self.target_transform(label)\n            \n\n        return image4C/255, label","metadata":{"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2021-10-14T05:52:38.143451Z","iopub.execute_input":"2021-10-14T05:52:38.144051Z","iopub.status.idle":"2021-10-14T05:52:38.163938Z","shell.execute_reply.started":"2021-10-14T05:52:38.144Z","shell.execute_reply":"2021-10-14T05:52:38.16312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#transform tuple\n\ntransform_mask = transforms3D.RandomCutout_withmask(num_cut=50, percent_cut=0.25,)\n# transform_train = transforms3D.Compose([\n#     transforms3D.RandomRotation(degrees=15),\n#     transforms3D.GaussianBlur(kernel_size=3, sigma=(0.1, 2.0)),\n# ])\ntransform_train = [transforms3D.RandomRotation(degrees=15), transforms3D.GaussianBlur(kernel_size=[1, 3, 3], sigma=(0.1, 3.0))]#, transforms3D.GammaAdjust(gamma_range=(0.8, 1.2), gain_range=(0.8, 1.2))]\n# !pip install elasticdeform\n# import elasticdeform\n\n# transform_train = transforms3D.GammaAdjust(gamma_range=(0.8, 1.2), gain_range=(0.8, 1.2))","metadata":{"collapsed":false,"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-10-14T05:52:39.734893Z","iopub.execute_input":"2021-10-14T05:52:39.735488Z","iopub.status.idle":"2021-10-14T05:52:39.741242Z","shell.execute_reply.started":"2021-10-14T05:52:39.73545Z","shell.execute_reply":"2021-10-14T05:52:39.740662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def worker_init_fn(worker_id):                                                          \n    np.random.seed(np.random.get_state()[1][0] + worker_id)","metadata":{"execution":{"iopub.status.busy":"2021-10-14T05:52:41.6027Z","iopub.execute_input":"2021-10-14T05:52:41.603132Z","iopub.status.idle":"2021-10-14T05:52:41.607387Z","shell.execute_reply.started":"2021-10-14T05:52:41.60309Z","shell.execute_reply":"2021-10-14T05:52:41.606542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 15\n\ntrain_dataset = Image4CDataSet(label_csv = traincsv_path,\n                                 img_dir = img_dir,\n                                 mask_dir = mask_dir,\n                                 maskdilate_kernelsize=maskdilate_kernelsize, transform=transform_train, transform_mask=transform_mask, Training = True)\n\nval_dataset = Image4CDataSet(label_csv = valcsv_path,\n                                 img_dir = img_dir,\n                                 Training = True)\n\n# train_dataset, val_dataset = torch.utils.data.random_split(image4c_dataset,\n#         [int(train_ratio * len(image4c_dataset)), len(image4c_dataset) - int(train_ratio * len(image4c_dataset))],) #generator=torch.Generator().manual_seed(0))\n\n# train_dataset, val_dataset = torch.utils.data.random_split(image4c_dataset, [6, 4],)\n\nprint(f'training set len:{len(train_dataset)} \\n validation set len:{len(val_dataset)}')\n\ntrain_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, drop_last = True, num_workers = 2, worker_init_fn=worker_init_fn)\nval_dataloader = DataLoader(val_dataset, batch_size=1, shuffle=False, drop_last = True, num_workers = 1, worker_init_fn=worker_init_fn)\n\nprint(f'with batch size {batch_size} \\ntraining loader len:{len(train_dataloader)} \\n validation loader len:{len(val_dataloader)}')\nimage, label = train_dataset[0]\nprint(image.shape, label)\nimage, label = train_dataset[1]\nprint(image.shape, label)\nimage, label = val_dataset[0]\nprint(image.shape, label)\n","metadata":{"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2021-10-14T05:52:46.390943Z","iopub.execute_input":"2021-10-14T05:52:46.391276Z","iopub.status.idle":"2021-10-14T05:52:51.65159Z","shell.execute_reply.started":"2021-10-14T05:52:46.391243Z","shell.execute_reply":"2021-10-14T05:52:51.650756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Trainer:\n    def __init__(\n            self,\n            model,\n            device,\n            optimizer,\n            loss_fn\n    ):\n        self.model = model.to(device)\n        self.device = device\n        self.optimizer = optimizer\n        self.lr_scheduler = torch.optim.lr_scheduler.ExponentialLR(self.optimizer, gamma=LR_DECAY)\n        self.loss_fn = loss_fn\n\n        self.best_valid_score = .0\n        self.n_patience = 0\n        self.lastmodel = None\n\n        self.val_losses = []\n        self.train_losses = []\n        self.val_auc = []\n        self.train_auc = []\n\n    def fit(self, epochs, train_dataloader, valid_dataloader, mrimodule, patience, model_savepath):\n        for n_epoch in range(1, epochs + 1):\n            np.random.seed(2021 + n_epoch)\n            self.info_message(\"EPOCH: {}\", n_epoch)\n\n            train_loss, train_auc, train_time = self.train_epoch(train_dataloader)\n            valid_loss, valid_auc, valid_time = self.valid_epoch(valid_dataloader)\n\n            self.train_losses.append(train_loss)\n            self.train_auc.append(train_auc)\n            self.val_losses.append(valid_loss)\n            self.val_auc.append(valid_auc)\n\n            self.info_message(\n                \"[Epoch Train: {}] loss: {:.4f}, auc: {:.4f}, time: {:.2f} s\",\n                n_epoch, train_loss, train_auc, train_time\n            )\n\n            self.info_message(\n                \"[Epoch Valid: {}] loss: {:.4f}, auc: {:.4f}, time: {:.2f} s\",\n                n_epoch, valid_loss, valid_auc, valid_time\n            )\n\n            if self.best_valid_score < valid_auc:\n                self.save_model(model_savepath, n_epoch, mrimodule, valid_loss, valid_auc)\n                self.info_message(\n                    \"auc improved from {:.4f} to {:.4f}. Saved model to '{}'\",\n                    self.best_valid_score, valid_auc, self.lastmodel\n                )\n                self.best_valid_score = valid_auc\n                self.n_patience = 0\n            else:\n                self.n_patience += 1\n\n            if self.n_patience >= patience:\n                self.info_message(\"\\nValid auc didn't improve last {} epochs.\", patience)\n                break\n\n    def train_epoch(self, train_dataloader):\n        # print('in trainer')\n        self.model.train()\n        t = time.time()\n        sum_loss = 0\n\n        y_all = []\n        outputs_all = []\n\n        for step, (X, Y) in enumerate(train_dataloader, 1):\n            # print(step)\n            X = X.to(self.device)\n            Y = Y.to(self.device)\n            self.optimizer.zero_grad()\n            pred = self.model(X).squeeze(1)\n#             print(X.is_cuda, Y.is_cuda, next(model.parameters()).is_cuda)\n            loss = self.loss_fn(pred, Y)\n\n            loss.backward()\n\n            sum_loss += loss.detach().item()\n\n            self.optimizer.step()\n\n            y_all.extend(Y.tolist())\n            outputs_all.extend(torch.sigmoid(pred).tolist())\n\n            message = 'Train Step {}/{}, train_loss: {:.4f}'\n            self.info_message(message, step, len(train_dataloader), sum_loss/step, end=\"\\r\")\n\n        self.lr_scheduler.step()\n        auc = roc_auc_score(y_all, outputs_all)\n\n        return sum_loss/len(train_dataloader), auc, int(time.time() - t)\n\n    def valid_epoch(self, valid_dataloader):\n        # print('in valdier')\n        self.model.eval()\n        t = time.time()\n        sum_loss = 0\n\n        y_all = []\n        outputs_all = []\n\n        with torch.no_grad():\n            for step, (X, Y) in enumerate(valid_dataloader, 1):\n                X = X.to(self.device)\n                Y = Y.to(self.device)\n\n                pred = self.model(X).squeeze(1)\n                loss = self.loss_fn(pred, Y)\n                sum_loss += loss.detach().item()\n\n                y_all.extend(Y.tolist())\n                outputs_all.extend(torch.sigmoid(pred).tolist())\n\n                message = 'Valid Step {}/{}, valid_loss: {:.4f}'\n                self.info_message(message, step, len(valid_dataloader), sum_loss/step, end=\"\\r\")\n\n        auc = roc_auc_score(y_all, outputs_all)\n\n        return sum_loss/len(valid_dataloader), auc, int(time.time() - t)\n\n    def save_model(self, model_savepath, n_epoch, mrimodule, loss, auc):\n        self.lastmodel = f\"{model_savepath}/{mrimodule}-e{n_epoch}-loss{loss:.3f}-auc{auc:.3f}.pth\"\n        torch.save(\n            {\n                \"model_state_dict\": self.model.state_dict(),\n                \"optimizer_state_dict\": self.optimizer.state_dict(),\n                \"best_valid_score\": self.best_valid_score,\n                \"n_epoch\": n_epoch,\n            },\n            self.lastmodel,\n        )\n\n    def display_plots(self, mrimodule):\n        plt.figure(figsize=(10,5))\n        plt.title(\"{}: Training and Validation Loss\".format(mrimodule))\n        plt.plot(self.val_losses,label=\"val\")\n        plt.plot(self.train_losses,label=\"train\")\n        plt.xlabel(\"iterations\")\n        plt.ylabel(\"Loss\")\n        plt.legend()\n        plt.show()\n        plt.close()\n\n        plt.figure(figsize=(10,5))\n        plt.title(\"{}: Training and Validation AUC-ROC\".format(mrimodule))\n        plt.plot(self.val_auc,label=\"val\")\n        plt.plot(self.train_auc,label=\"train\")\n        plt.xlabel(\"iterations\")\n        plt.ylabel(\"AUC\")\n        plt.legend()\n        plt.show()\n        plt.close()\n\n    @staticmethod\n    def info_message(message, *args, end=\"\\n\"):\n        print(message.format(*args), end=end)","metadata":{"collapsed":false,"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-10-14T05:52:51.653347Z","iopub.execute_input":"2021-10-14T05:52:51.6536Z","iopub.status.idle":"2021-10-14T05:52:51.686771Z","shell.execute_reply.started":"2021-10-14T05:52:51.65357Z","shell.execute_reply":"2021-10-14T05:52:51.685596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N_EPOCHS = 20\nLEARNING_RATE = 0.0005\nLR_DECAY = 0.9\n\nmodel_savepath = './'\n# train, plot and save\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\noptimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE)\nloss_fn = torch.nn.BCEWithLogitsLoss()\n# loss_fn = FocalLoss(gamma=2, eps=1e-7)\n# loss_fn_valid = torch.nn.BCEWithLogitsLoss(weight=torch.ones(1).to(device))\n\n\nmodel.to(device)\n\ntrainer = Trainer(model, device, optimizer, loss_fn)\n\ntrainer.fit(N_EPOCHS, train_dataloader=train_dataloader, valid_dataloader=val_dataloader,\n            mrimodule=\"4C\", patience=10 , model_savepath=model_savepath)\n\ntrainer.display_plots(\"4C\")\n\nprint(f'best module achieved: {trainer.lastmodel}')\n","metadata":{"collapsed":false,"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-10-14T05:52:51.688426Z","iopub.execute_input":"2021-10-14T05:52:51.688756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}