{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport json\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport cv2\nfrom sklearn.model_selection import train_test_split","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-13T13:47:51.277395Z","iopub.execute_input":"2023-07-13T13:47:51.277755Z","iopub.status.idle":"2023-07-13T13:47:51.284285Z","shell.execute_reply.started":"2023-07-13T13:47:51.277723Z","shell.execute_reply":"2023-07-13T13:47:51.283287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(\"/kaggle/input/hubmap-hacking-the-human-vasculature/polygons.jsonl\", \"r\") as f:\n    train_data = [json.loads(line) for line in f]\ntrain_df = pd.DataFrame(train_data)","metadata":{"execution":{"iopub.status.busy":"2023-07-13T13:47:51.286802Z","iopub.execute_input":"2023-07-13T13:47:51.287525Z","iopub.status.idle":"2023-07-13T13:47:53.821280Z","shell.execute_reply.started":"2023-07-13T13:47:51.287472Z","shell.execute_reply":"2023-07-13T13:47:53.820297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create dataset from json\nfrom torch.utils.data import Dataset, DataLoader\nimport torch\nimport random\n\n# set seed for same output with albumentations\nrandom.seed(0)\n\nclass HuBMAPDataset(Dataset):\n    def __init__(self, json_df:pd.DataFrame, phase:str='train', img_path:str=\"/kaggle/input/hubmap-hacking-the-human-vasculature/train\", transforms:object=None):\n        # test, valで読み込む画像を分けられるように、DataFrame形式データを引数にとる\n        self.json_df = json_df\n        self.img_path = img_path\n        self.transforms = transforms\n        self.phase = phase\n        self.classes_dict = {\n            \"background\": 0,\n            \"blood_vessel\": 1,\n            \"glomerulus\": 2,\n            \"unsure\": 3,\n        }\n        \n        if transforms:\n            # min_area:指定したピクセル数より低いbboxの値は返さない\n            # min_visibility:加工前のbbox領域を1とし、加工後のbbox領域が指定した領域面積割合より低ければ値を返さない\n            self.bbox_params = {\n                'format':\"pascal_voc\",\n                'min_area':0,\n                'min_visibility':0,\n                'label_fields':['category_id']\n            }\n            if phase == 'test':\n                self.transforms = A.Compose(transforms)\n            else:\n                self.transforms = A.Compose(transforms, bbox_params=self.bbox_params)\n        \n    def __len__(self):\n        # 画像枚数を返す\n        return len(self.json_df)\n    \n    def __getitem__(self, idx):\n        sample = self.json_df['id'].iloc[idx]\n        sample = sample + '.tif'\n        # print(sample)\n        img = cv2.imread(os.path.join(self.img_path, sample))\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        \n        # testの場合,正解データが存在しないため、処理を分ける\n        if self.phase == 'test':\n            if self.transforms:\n                img = self.transform(self.phase, img)\n            return img\n            \n        target_data = self.json_df.iloc[idx]\n        target = self._get_target(target_data)\n        # print(target)\n        target['image_id'] = torch.tensor([idx])\n        \n        if self.transforms:\n            img, target = self.transform(self.phase, img, target)\n        \n        # print(target)\n        \n        return img, target\n    \n    def transform(self, phase, image, target=None):\n        transformed = self.transforms(image=image, masks=target['masks'], bboxes=target['boxes'], category_id=target['labels'])\n        image = transformed['image']\n        if phase == 'test':\n            return image\n        \n        target['masks'] = torch.as_tensor(np.array(list(map(np.array, transformed['masks'])), dtype=np.uint8))\n        target['labels'] = torch.tensor(transformed['category_id'])\n        target['boxes'] = torch.as_tensor(transformed['bboxes'], dtype=torch.float32)\n        target['area'] = self._get_area(target['boxes'])\n        \n        return image, target\n    \n    def _get_target(self, target_data):\n        target = self._get_target_sample()\n        annotations = target_data['annotations']\n        annotations = np.array(annotations)\n        \n        # print(annotations)\n        for annotation in annotations:\n            label = int(self.classes_dict[annotation['type']])\n            mask = self._get_mask(label, annotation['coordinates'])\n            box = self._get_box(mask)\n            \n            target['labels'].append(label)\n            target['masks'].append(mask)\n            target['boxes'].append(box)\n            \n        num_obj = len(target['labels'])\n        target['iscrowd'] = torch.zeros((num_obj,), dtype=torch.int64)\n        \n        return target\n    \n    @staticmethod\n    def _get_mask(label, coodinate):\n        mask = np.zeros((512,512), dtype=np.uint8)\n        # fillPolyを動かすためにnp.array型に変換 (1, N, 2)じゃないと動かない\n        pts = np.array(coodinate)\n        return cv2.fillPoly(mask, pts=pts, color=(1, 1, 1))\n    \n    @staticmethod\n    def _get_box(mask):\n        pos = np.nonzero(mask)\n        xmin = np.min(pos[1])\n        xmax = np.max(pos[1])\n        ymin = np.min(pos[0])\n        ymax = np.max(pos[0])\n        \n        return [xmin, ymin, xmax, ymax]\n    \n    @staticmethod\n    def _get_target_sample():\n        return {\n            \"boxes\": [],\n            \"masks\": [],\n            \"area\": [],\n            \"labels\": [],\n            \"iscrowd\": None,\n            \"image_id\": None\n        }\n    \n    @staticmethod\n    def _get_area(boxes):\n        # tensorが空の時がある\n        # print(boxes)\n        return (boxes[:, 3] - boxes[:, 1]) * (boxes[:, 2] - boxes[:, 0])\n\ndataset = HuBMAPDataset(train_df)\nimage, target = dataset.__getitem__(0)\nimage.shape\n# b680700d5e57:1155\ntarget['masks'][0].shape","metadata":{"execution":{"iopub.status.busy":"2023-07-13T13:49:08.646769Z","iopub.execute_input":"2023-07-13T13:49:08.647259Z","iopub.status.idle":"2023-07-13T13:49:08.726901Z","shell.execute_reply.started":"2023-07-13T13:49:08.647219Z","shell.execute_reply":"2023-07-13T13:49:08.725854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom sklearn.model_selection import train_test_split\n\ndef fix_normalize(image, mean, std):\n    for i, m in enumerate(zip(mean, std)):\n        image[:,:,i] = image[:,:,i]*m[1]+m[0]\n    return image\n\ndef visualize(image, bbox, label):\n    x_min, y_min, x_max, y_max = bbox\n    cv2.rectangle(image, (int(x_min), int(y_min)), (int(x_max), int(y_max)), color=(255,0,0), thickness=2)\n    return image\n\ndef plot_img(image, target, mean, std):\n    img = image.numpy()\n    img = img.transpose(1,2,0)\n    img = fix_normalize(img, mean, std)\n    img = img.copy()\n    for bbox, label in zip(target['boxes'], target['labels']):\n        img = visualize(img, bbox, label)\n    \n    plt.figure(figsize=(12,12))\n    plt.imshow(img)\n    plt.show()\n\n# create transform\n# decrare coco's dataset mean and std\nmean=[123.675, 116.28, 103.53]\nstd=[58.395, 57.12, 57.375]\n\ntrain_transform = [\n    A.Resize(512, 512, p=1), \n    A.HorizontalFlip(),\n    A.VerticalFlip(),\n    # A.ShiftScaleRotate(),\n    A.RandomBrightnessContrast(p=0.4),\n    \n    A.OneOf([\n        # 被写体に動きによるブレを生じさせているように加工するフィルター\n        A.MotionBlur(),\n        # カーネル内の画素平均で対象ピクセルを塗りつぶす\n        A.Blur(blur_limit=3),\n        # メディアンフィルター、デフォルトカーネルサイズは7\n        A.MedianBlur(blur_limit=3),\n        # ガウスノイズを加える、チャンネル毎にノイズを乗せる\n        A.GaussNoise()\n    ], p=0.1),\n    A.Normalize(mean, std),\n    ToTensorV2()\n]\n\nvalidation_transform = [\n    A.Resize(512, 512, p=1), \n    A.Normalize(mean, std),\n    ToTensorV2()\n]\n\ntest_transform = [\n    A.Resize(512, 512, p=1), \n    A.Normalize(mean, std),\n    ToTensorV2()\n]\n\ntrain, val = train_test_split(train_df, test_size=0.2, random_state=1)\ntrain_dataset = HuBMAPDataset(train, transforms=train_transform)\nval_dataset = HuBMAPDataset(val, phase='val', transforms=validation_transform)\ntrain_dl = DataLoader(train_dataset, batch_size=5, shuffle=True, pin_memory=True, collate_fn=lambda x:tuple(zip(*x)))\nval_dl = DataLoader(val_dataset, batch_size=5, shuffle=False, pin_memory=True, collate_fn=lambda x:tuple(zip(*x)))\n\ntrain_img, train_target = train_dataset.__getitem__(0)\nval_img, val_target = val_dataset.__getitem__(0)\n\nplot_img(train_img, train_target, mean, std)\nplot_img(val_img, val_target, mean, std)\nprint(train_target['masks'][0])","metadata":{"execution":{"iopub.status.busy":"2023-07-13T13:50:09.004801Z","iopub.execute_input":"2023-07-13T13:50:09.005428Z","iopub.status.idle":"2023-07-13T13:50:11.100946Z","shell.execute_reply.started":"2023-07-13T13:50:09.005386Z","shell.execute_reply":"2023-07-13T13:50:11.100127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create mask r-cnn\nimport torchvision\nfrom torchvision.models.detection.faster_rcnn import FastRCNNPredictor\nfrom torchvision.models.detection.mask_rcnn import MaskRCNNPredictor, MaskRCNN_ResNet50_FPN_V2_Weights\n\ndef get_model(num_classes:int):\n    model = torchvision.models.detection.maskrcnn_resnet50_fpn_v2(weights=MaskRCNN_ResNet50_FPN_V2_Weights.COCO_V1)\n    in_features = model.roi_heads.box_predictor.cls_score.in_features\n    model.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)\n    \n    in_features_mask = model.roi_heads.mask_predictor.conv5_mask.in_channels\n    hidden_layer = 512\n    model.roi_heads.mask_predictor = MaskRCNNPredictor(in_features_mask, hidden_layer, num_classes)\n\n    return model\n\nmodel = get_model(4)\nmodel","metadata":{"execution":{"iopub.status.busy":"2023-07-13T13:47:54.098272Z","iopub.status.idle":"2023-07-13T13:47:54.098661Z","shell.execute_reply.started":"2023-07-13T13:47:54.098489Z","shell.execute_reply":"2023-07-13T13:47:54.098506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#import wandb\n\n#wandb.login()\n#run = wandb.init(\n#    project=\"HuBMAP-Mask-RCNNV2\",\n#    config = {\n#        'learning_rate':0.0035,\n#        'epoch':15,\n#    }\n#)","metadata":{"execution":{"iopub.status.busy":"2023-07-13T13:47:54.100454Z","iopub.status.idle":"2023-07-13T13:47:54.101241Z","shell.execute_reply.started":"2023-07-13T13:47:54.100997Z","shell.execute_reply":"2023-07-13T13:47:54.101020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm.notebook import tqdm\nimport torch.nn as nn\nimport torch.optim as optim\n\nclass Trainer:\n    def __init__(self, \n                 model:nn.Module, \n                 train_dl:DataLoader, \n                 val_dl:DataLoader, \n                 early_stop:dict={\"monitor\":'loss_mask', 'patience':5}, \n                 save_every_epoch:int=1, \n                 save_dirpath:str='/kaggle/working/runs'\n                ):\n        \n        # callbacks:Early stopping and model checkpoint\n        self.patience = early_stop['patience']\n        self.monitor = early_stop['monitor']\n        self.tracked_list = []\n        self.save_every_epoch = save_every_epoch\n        self.save_dirpath = save_dirpath\n        \n        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n        self.train_dl = train_dl\n        self.val_dl = val_dl\n        self.train_batches = len(self.train_dl)\n        self.val_batches = len(self.val_dl)\n        self.model = model\n        self.setup_model()\n        self.optim_dict = self.configure_optimizers()\n        self.optimizer = self.optim_dict['optimizer']\n        self.lr_scheduler = self.optim_dict['lr_scheduler']\n        \n        self.step_outputs = {\n            'loss_objectness':0,\n            'loss_mask':0,\n            'loss_classifier':0,\n            'loss_rpn_box_reg':0,\n            'loss_box_reg':0,\n            'loss':0\n        }\n    \n    def configure_optimizers(self):\n        params = [\n            p for p in self.model.parameters() if p.requires_grad\n        ]\n        \n        # optimizer = optim.SGD(params, lr=0.00018, momentum=0.938, weight_decay=0.00053)\n        optimizer = optim.AdamW(params, lr=0.0001)\n        lr_scheduler = optim.lr_scheduler.StepLR(\n            optimizer,\n            step_size=3,\n            gamma=0.1\n        )\n        \n        return {'optimizer':optimizer, 'lr_scheduler':lr_scheduler}\n    \n    def setup_model(self):\n        for param in self.model.parameters():\n            param.requires_grad = True\n        self.model.to(self.device)\n        self.model.train()\n        \n    def train(self, max_epoch:int):\n        for epoch in range(1, max_epoch + 1):\n            print('{}/{}'.format(epoch, max_epoch+1))\n            print('-'*10)\n            \n            for batch_idx, batch in tqdm(enumerate(self.train_dl, 1), desc='Training', total=self.train_batches):\n                outputs = self.training_step(batch)\n                for key, value in outputs.items():\n                    self.step_outputs[key] += float(value.detach().cpu().numpy()) / self.train_batches\n            self.on_train_epoch_end(epoch)\n            \n            for batch_idx, batch in tqdm(enumerate(self.val_dl, 1), desc='Validation', total=self.val_batches):\n                outputs = self.validation_step(batch)\n                for key, value in outputs.items():\n                    self.step_outputs[key] += float(value.detach().cpu().numpy()) / self.val_batches\n            \n            tracked_loss, patience = self.on_validation_epoch_end(epoch)\n            \n            if epoch % self.save_every_epoch == 0:\n                if not os.path.exists(self.save_dirpath):\n                    os.mkdir(self.save_dirpath)\n                path = os.path.join(self.save_dirpath, f\"epoch_{epoch}_{self.monitor}_{tracked_loss:.2f}.pth\")\n                torch.save(model.state_dict(), path) \n                print(\"\\nThe model passed the save checkpoint successfully!\\n\")\n            \n            if patience == 0:\n                print(\"Our patience has run out! Model training stopped beforehand.\")\n                break\n            \n    def training_step(self, batch):\n        images, targets = self.to_device(batch)\n        self.optimizer.zero_grad()\n        outputs = self.model(images, targets)\n        loss = sum([loss for loss in outputs.values()])\n        outputs['loss'] = loss\n        loss.backward()\n        self.optimizer.step()\n        return outputs\n    \n    def validation_step(self, batch):\n        images, targets = self.to_device(batch)\n        with torch.no_grad():\n            outputs = self.model(images, targets)\n            loss = sum([loss for loss in outputs.values()])\n            outputs['loss'] = loss\n        return outputs\n        \n    def to_device(self, batch):\n        images, targets = batch\n        images = list(image.to(self.device) for image in images)\n        \n        targets = [\n            {key:value.to(self.device) for key, value in target.items()}\n            for target in targets\n        ]\n        \n        return images, targets\n    \n    def on_train_epoch_end(self, epoch:int):\n        return self.shared_epoch_end(stage='train', epoch=epoch)\n    \n    def on_validation_epoch_end(self, epoch:int):\n        tracked_loss = self.shared_epoch_end(stage='val', epoch=epoch)\n        patience = 0\n        \n        if epoch > self.patience:\n            last_tracked = list(reversed(self.tracked_list))[:self.patience]\n            for i in last_tracked:\n                if i <= tracked_loss:\n                    patience += 1\n                    \n        self.tracked_list.append(tracked_loss)\n        \n        return tracked_loss, (self.patience - patience)\n    \n    def shared_epoch_end(self, stage:str, epoch:int):\n        tracked_loss = self.step_outputs[self.monitor]\n        loss_objectness = self.step_outputs['loss_objectness']\n        loss_mask = self.step_outputs['loss_mask']\n        loss_classifier = self.step_outputs['loss_classifier']\n        loss_rpn_box_reg = self.step_outputs['loss_rpn_box_reg']\n        loss_box_reg = self.step_outputs['loss_box_reg']\n        loss = self.step_outputs['loss']\n        \n        #wandb.log({\n        #    f\"{stage}_loss_objectness\": loss_objectness,\n        #    f\"{stage}_loss_mask\": loss_mask,\n        #    f\"{stage}_loss_classifier\": loss_classifier,\n        #    f\"{stage}_loss_rpn_box_reg\": loss_rpn_box_reg,\n        #    f\"{stage}_loss_box_reg\": loss_box_reg,\n        #    f\"{stage}_loss\": loss  \n        #})\n        \n        print(\n            f\"\"\"\n            || End {epoch} {stage} epoch ||\n            loss_objectness: {loss_objectness:.2f}\n            loss_mask: {loss_mask:.2f}\n            loss_classifier: {loss_classifier:.2f}\n            loss_rpn_box_reg: {loss_rpn_box_reg:.2f}\n            loss_box_reg: {loss_box_reg:.2f} \n            loss: {loss:.2f}\\n\n            \"\"\"\n        )\n        \n        self.step_outputs = self.step_outputs.fromkeys(self.step_outputs, 0)\n        if stage == 'train':\n            self.lr_scheduler.step()\n            \n        if stage == 'val':\n            return tracked_loss\n                \ntrainer = Trainer(\n    model=model,\n    train_dl=train_dl,\n    val_dl=val_dl,\n    early_stop = {\"monitor\": \"loss_mask\", \"patience\": 5},\n    save_every_epoch=1\n)","metadata":{"execution":{"iopub.status.busy":"2023-07-13T13:47:54.102788Z","iopub.status.idle":"2023-07-13T13:47:54.103590Z","shell.execute_reply.started":"2023-07-13T13:47:54.103343Z","shell.execute_reply":"2023-07-13T13:47:54.103367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainer.train(max_epoch=15)","metadata":{"execution":{"iopub.status.busy":"2023-07-13T13:47:54.104997Z","iopub.status.idle":"2023-07-13T13:47:54.105784Z","shell.execute_reply.started":"2023-07-13T13:47:54.105536Z","shell.execute_reply":"2023-07-13T13:47:54.105560Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}