{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# LET'S START! \n참고: https://deep-learning-study.tistory.com/568","metadata":{}},{"cell_type":"markdown","source":"**1. Importing packages**","metadata":{}},{"cell_type":"code","source":"# import pakages\nimport torch\nfrom torch import nn\nfrom torch import optim\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nfrom torch.utils.data import Dataset\nfrom torch.utils.data import DataLoader\nfrom torchvision import utils\n# from torchsummary import summary\nimport torchvision.transforms.functional as TF\nfrom torchvision.transforms.functional import to_pil_image\nfrom PIL import Image, ImageDraw, ImageFont\nimport matplotlib.pyplot as plt\nimport cv2\nimport os\nimport copy\nimport numpy as np\nimport pandas as pd\nimport random\nimport albumentations as A\nimport ast\nfrom albumentations.pytorch import ToTensorV2\n%matplotlib inline\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(device)","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2022-01-16T14:05:15.372888Z","iopub.execute_input":"2022-01-16T14:05:15.373432Z","iopub.status.idle":"2022-01-16T14:05:15.386347Z","shell.execute_reply.started":"2022-01-16T14:05:15.373398Z","shell.execute_reply":"2022-01-16T14:05:15.385566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"b = [1, 2, 3, 45623, 3, 13]\n\nprint(b[:2])","metadata":{"execution":{"iopub.status.busy":"2022-01-16T14:05:15.388435Z","iopub.execute_input":"2022-01-16T14:05:15.389016Z","iopub.status.idle":"2022-01-16T14:05:15.402379Z","shell.execute_reply.started":"2022-01-16T14:05:15.388979Z","shell.execute_reply":"2022-01-16T14:05:15.401422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**2. Classes Name**","metadata":{}},{"cell_type":"code","source":"classes = [\n    \"STARFISH\",\n]","metadata":{"execution":{"iopub.status.busy":"2022-01-16T14:05:15.403696Z","iopub.execute_input":"2022-01-16T14:05:15.404028Z","iopub.status.idle":"2022-01-16T14:05:15.411159Z","shell.execute_reply.started":"2022-01-16T14:05:15.403913Z","shell.execute_reply":"2022-01-16T14:05:15.410288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**3. CUSTOM DATASET**","metadata":{}},{"cell_type":"code","source":"'''\nGBRDataset (Great-Barrier-Reef Dataset)\n'''\n\nclass GBRDataset(Dataset):\n    def __init__(self, csv_file, img_dir, transform=None, trans_params=None, is_test = False, is_train=True, idx=0):\n        self.img_dir = img_dir\n        self.transform = transform\n        self.trans_params = trans_params\n        self.is_train = is_train\n        \n        csv_dataset = pd.read_csv(csv_file)\n        if not is_test:\n            self.annotations = csv_dataset[:idx] if is_train else csv_dataset[idx:] # idx: train/test boundary index\n        else:\n            self.annotations = csv_dataset\n\n    def __len__(self):\n        return len(self.annotations)\n\n    def __getitem__(self, index):\n        img_id = self.annotations.iloc[index, 4].strip('-') # 0-24\n        coordinates = ast.literal_eval(self.annotations.iloc[index, 5])\n        label_data = np.array([list(coord.values()) + [0] for coord in coordinates], dtype = 'float32') if (len(coordinates) > 0) else np.array([])\n        img_subpath = '/video_' + img_id[0] + '/' + img_id[2:] + '.jpg'\n        img_path = self.img_dir + img_subpath # /tensorflow-great-barrier-reef/train_images/video_{}/{}.jpg\n        image = np.array(Image.open(img_path).convert(\"RGB\")) # albumentation을 적용하기 위해 np.array로 변환합니다.\n        \n        if label_data != np.array([]): # normalization\n            label_data[:, 2] = label_data[:, 2] / 1280 # normalization of w\n            label_data[:, 3] = label_data[:, 3] / 1280 # normalization of h\n            label_data[:, 0] = label_data[:, 0] / 1280 + label_data[:, 2] / 2 # normalization of x\n            label_data[:, 1] = label_data[:, 1] / 1280 + label_data[:, 3] / 2 + 280 / 1280 # normalization of y (considering padding 280)\n\n    \n        if self.transform:\n            # apply albumentations\n            augmentations = self.transform(image=image, bboxes=label_data)\n            image = augmentations['image']\n            targets = augmentations['bboxes']\n            \n            # for DataLoader\n            # lables: ndarray -> tensor\n            # dimension: [batch, cx, cy, w, h, class]\n            if targets is not None:\n                targets = torch.zeros((len(label_data), 6))\n                targets[:, 1:] = torch.tensor(label_data).reshape(-1, 5) \n        else:\n            targets = label_data\n\n        return image, targets","metadata":{"execution":{"iopub.status.busy":"2022-01-16T14:05:15.414459Z","iopub.execute_input":"2022-01-16T14:05:15.414661Z","iopub.status.idle":"2022-01-16T14:05:15.430970Z","shell.execute_reply.started":"2022-01-16T14:05:15.414637Z","shell.execute_reply":"2022-01-16T14:05:15.430130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Test**","metadata":{}},{"cell_type":"code","source":"\nidx = 11\nanno = pd.read_csv('../input/tensorflow-great-barrier-reef/train.csv').iloc[0:5]\nprint(anno)\n                                     ","metadata":{"execution":{"iopub.status.busy":"2022-01-16T14:05:15.434777Z","iopub.execute_input":"2022-01-16T14:05:15.435213Z","iopub.status.idle":"2022-01-16T14:05:15.475302Z","shell.execute_reply.started":"2022-01-16T14:05:15.435182Z","shell.execute_reply":"2022-01-16T14:05:15.474604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"4. Generating Training Dataset, Validating Dataset","metadata":{}},{"cell_type":"code","source":"# dataset 생성하기\ndataset_csv_file = '../input/tensorflow-great-barrier-reef/train.csv' # train + val 동시에 생성 \nimg_dir = '../input/tensorflow-great-barrier-reef/train_images'\n\ntrain_ratio = 0.9\nidx = int(train_ratio * len(pd.read_csv(dataset_csv_file)))\n          \ntrain_ds = GBRDataset(dataset_csv_file, img_dir, is_train = True, idx = idx)\nval_ds = GBRDataset(dataset_csv_file, img_dir, is_train = False, idx = idx)\n\nimg, labels = train_ds[31]\n\n\nprint('number of train data : {}, val data : {}, total data : {}'.format(len(train_ds), len(val_ds), len(train_ds) + len(val_ds)))\nprint('image size:', img.shape, type(img)) # 720 x 1280 x 3\nprint('labels shape:', labels.shape, type(labels))  # x1,y1,x2,y2\nprint('lables \\n', labels)","metadata":{"execution":{"iopub.status.busy":"2022-01-16T14:05:15.476506Z","iopub.execute_input":"2022-01-16T14:05:15.476745Z","iopub.status.idle":"2022-01-16T14:05:15.590788Z","shell.execute_reply.started":"2022-01-16T14:05:15.476713Z","shell.execute_reply":"2022-01-16T14:05:15.589964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**val dataset 생성하기 -> 안 해도 될 듯**","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:25:27.748244Z","iopub.status.idle":"2022-01-09T14:25:27.748904Z","shell.execute_reply.started":"2022-01-09T14:25:27.74863Z","shell.execute_reply":"2022-01-09T14:25:27.748658Z"}}},{"cell_type":"markdown","source":"**5. Data Augmentation Using Albumentation**","metadata":{}},{"cell_type":"code","source":"# transforms 정의하기\nIMAGE_COL, IMAGE_ROW = 1280, 1280\nIMAGE_SIZE = max(IMAGE_COL, IMAGE_ROW)\nscale = 1.0\n\n# for train\ntrain_transforms = A.Compose([\n        # 이미지의 maxsize를 max_size로 rescale합니다. aspect ratio는 유지합니다.\n        A.LongestMaxSize(max_size=int(IMAGE_SIZE * scale)),\n        # min_size보다 작으면 pad\n        A.PadIfNeeded(min_height=int(IMAGE_SIZE * scale), min_width=int(IMAGE_SIZE * scale), border_mode=cv2.BORDER_CONSTANT),\n        # random crop\n        A.RandomCrop(width=IMAGE_SIZE, height=IMAGE_SIZE),\n        # brightness, contrast, saturation을 무작위로 변경합니다.\n        A.ColorJitter(brightness=0.6, contrast=0.6, saturation=0.6, hue=0.6, p=0.4),\n        # 수평 뒤집기\n        A.HorizontalFlip(p=0.5),\n        # blur\n        A.Blur(p=0.1),\n        # Contrast Limited Adaptive Histogram Equalization 적용\n        A.CLAHE(p=0.1),\n        # 각 채널의 bit 감소\n        A.Posterize(p=0.1),\n        # grayscale로 변환\n        A.ToGray(p=0.1),\n        # 무작위로 channel을 섞기\n        A.ChannelShuffle(p=0.05),\n        # normalize\n        A.Normalize(mean=[0,0,0], std=[1,1,1], max_pixel_value=255),\n        ToTensorV2()\n        ],\n        # (x1, y1, x2, y2) -> (cx, cy, w, h)\n        bbox_params=A.BboxParams(format='yolo', min_visibility=0.4, label_fields=[])\n        )\n\ntrain_transforms_check = A.Compose([\n        # 이미지의 maxsize를 max_size로 rescale합니다. aspect ratio는 유지합니다.\n        A.LongestMaxSize(max_size=int(IMAGE_SIZE * scale)),\n        # min_size보다 작으면 pad\n        A.PadIfNeeded(min_height=int(IMAGE_SIZE * scale), min_width=int(IMAGE_SIZE * scale), border_mode=cv2.BORDER_CONSTANT),\n        A.Normalize(mean=[0, 0, 0], std=[1, 1, 1], max_pixel_value=255),\n        ToTensorV2()\n        ], \n        bbox_params=A.BboxParams(format='yolo', min_visibility=0.4, label_fields=[])\n        )\n\n'''\n생략\n     # transforms 중 하나를 선택해 적용합니다.\n        A.OneOf([\n                 # shift, scale, rotate 를 무작위로 적용합니다.\n                 A.ShiftScaleRotate(rotate_limit=20, p=0.5, border_mode=cv2.BORDER_CONSTANT),\n                 # affine 변환\n                 A.IAAAffine(shear=15, p=0.5, mode='constant')\n        ], p=1.0),\n'''\n# for validation\nval_transforms = A.Compose([\n        A.LongestMaxSize(max_size=int(IMAGE_SIZE * scale)),\n        A.PadIfNeeded(min_height=int(IMAGE_SIZE * scale), min_width=int(IMAGE_SIZE * scale), border_mode=cv2.BORDER_CONSTANT),\n        A.Normalize(mean=[0, 0, 0], std=[1, 1, 1], max_pixel_value=255),\n        ToTensorV2(),\n        ],\n        bbox_params=A.BboxParams(format='yolo', min_visibility=0.4, label_fields=[])\n        )","metadata":{"execution":{"iopub.status.busy":"2022-01-16T14:05:15.595642Z","iopub.execute_input":"2022-01-16T14:05:15.597693Z","iopub.status.idle":"2022-01-16T14:05:15.623224Z","shell.execute_reply.started":"2022-01-16T14:05:15.597650Z","shell.execute_reply":"2022-01-16T14:05:15.621417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 데이터셋에 transforms 적용하기\ntrain_ds.transform = train_transforms\nval_ds.transform = val_transforms","metadata":{"execution":{"iopub.status.busy":"2022-01-16T14:05:15.624372Z","iopub.execute_input":"2022-01-16T14:05:15.627712Z","iopub.status.idle":"2022-01-16T14:05:15.641586Z","shell.execute_reply.started":"2022-01-16T14:05:15.627666Z","shell.execute_reply":"2022-01-16T14:05:15.639978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**6. Applying Transforms into Images**","metadata":{}},{"cell_type":"code","source":"# 정규화된 x,y,w,h를 이미지 크기에 맞게 변경\ndef rescale_bbox(bb, W, H):\n    x,y,w,h = bb\n    return [x*W, y*H, w*W, h*H]\n\n# 바운딩 박스 색상\nCOLORS = np.random.randint(0, 255, size=(80,3), dtype='uint8')\n\n# image 출력 함수 정의\ndef show_img_bbox(img, targets, classes=classes):\n    if torch.is_tensor(img):\n        img=to_pil_image(img)\n    if torch.is_tensor(targets):\n        targets=targets.numpy()[:,1:]\n    \n    H, W = 1280, 1280 # resized to maxsize\n    draw = ImageDraw.Draw(img)\n\n    for tg in targets:\n        id_=int(tg[-1])\n        bbox=tg[:4]\n        bbox=rescale_bbox(bbox,W,H)\n        xc,yc,w,h = bbox\n\n        color = [int(c) for c in COLORS[id_]]\n        name=classes[id_]\n\n        draw.rectangle(((xc-w/2, yc-h/2), (xc+w/2, yc+h/2)), outline=tuple(color), width=3)\n        draw.text((xc-w/2, yc-h/2), name, fill=(255,255,255,0))\n    plt.imshow(np.array(img))","metadata":{"execution":{"iopub.status.busy":"2022-01-16T14:05:15.642838Z","iopub.execute_input":"2022-01-16T14:05:15.644317Z","iopub.status.idle":"2022-01-16T14:05:15.665964Z","shell.execute_reply.started":"2022-01-16T14:05:15.644279Z","shell.execute_reply":"2022-01-16T14:05:15.665050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# transforms가 적용된 sample image 확인\nnp.random.seed(25)\n\ngrid_size = 2\nrnd_ind = np.random.randint(0, len(train_ds), grid_size)\nprint('image indices:',rnd_ind)\n\n# train_transform\nplt.figure(figsize=(20, 20))\nfor i, indice in enumerate(rnd_ind):\n    img, label = train_ds[indice]\n    plt.subplot(1, grid_size, i+1)\n    show_img_bbox(img, label)\n\n\n# train_transforms_check for checking bboxes (completed!)\ntrain_ds.transform = train_transforms_check\nplt.figure(figsize=(20, 20))\nfor i, indice in enumerate(rnd_ind):\n    img, label = train_ds[indice]\n    plt.subplot(2, grid_size, i+1)\n    show_img_bbox(img, label)\n","metadata":{"execution":{"iopub.status.busy":"2022-01-16T14:05:15.670777Z","iopub.execute_input":"2022-01-16T14:05:15.671710Z","iopub.status.idle":"2022-01-16T14:05:18.029866Z","shell.execute_reply.started":"2022-01-16T14:05:15.671665Z","shell.execute_reply":"2022-01-16T14:05:18.029179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**7. Collate_fn**","metadata":{}},{"cell_type":"code","source":"# collate_fn 를 정의합니다.\n# collate_fn은 DataLoader의 인자로 사용되며, batch 단위로 imgs와 targets를 묶습니다.\ndef collate_fn(batch):\n    imgs, targets = list(zip(*batch))\n    # 빈 박스 제거하기\n    targets = [boxes for boxes in targets if boxes is not None]\n    # index 설정하기\n    for b_i, boxes in enumerate(targets):\n        boxes[:, 0] = b_i\n    targets = torch.cat(targets, 0)\n    imgs = torch.stack([img for img in imgs])\n    return imgs, targets","metadata":{"execution":{"iopub.status.busy":"2022-01-16T14:05:18.032369Z","iopub.execute_input":"2022-01-16T14:05:18.032813Z","iopub.status.idle":"2022-01-16T14:05:18.040592Z","shell.execute_reply.started":"2022-01-16T14:05:18.032776Z","shell.execute_reply":"2022-01-16T14:05:18.039355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make DataLoader\ntrain_dl = DataLoader(train_ds, batch_size=1, shuffle=True, collate_fn=collate_fn)\nval_dl = DataLoader(val_ds, batch_size=1, shuffle=True, collate_fn=collate_fn)","metadata":{"execution":{"iopub.status.busy":"2022-01-16T14:05:18.041909Z","iopub.execute_input":"2022-01-16T14:05:18.042232Z","iopub.status.idle":"2022-01-16T14:05:18.050711Z","shell.execute_reply.started":"2022-01-16T14:05:18.042197Z","shell.execute_reply":"2022-01-16T14:05:18.049760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**8. YOLO Model**","metadata":{}},{"cell_type":"code","source":"class BasicConv(nn.Module):\n    def __init__(self, in_channels, out_channels, kernel_size, stride, padding):\n        super().__init__()\n\n        self.conv = nn.Sequential(\n            nn.Conv2d(in_channels, out_channels, kernel_size, stride, padding, bias=False),\n            nn.BatchNorm2d(out_channels),\n            nn.LeakyReLU(0.1)\n        )\n\n    def forward(self, x):\n        return self.conv(x)","metadata":{"execution":{"iopub.status.busy":"2022-01-16T14:05:18.052266Z","iopub.execute_input":"2022-01-16T14:05:18.052528Z","iopub.status.idle":"2022-01-16T14:05:18.061180Z","shell.execute_reply.started":"2022-01-16T14:05:18.052493Z","shell.execute_reply":"2022-01-16T14:05:18.059475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ResidualBlock(nn.Module): # image size : conservative\n    def __init__(self, channels):\n        super().__init__()\n\n        self.residual = nn.Sequential(\n            BasicConv(channels, channels//2, 1, stride=1, padding=0),\n            BasicConv(channels//2, channels, 3, stride=1, padding=1)\n        )\n\n        self.shortcut = nn.Sequential()\n\n    def forward(self, x):\n        x_shortcut = self.shortcut(x)\n        x_residual = self.residual(x)\n\n        return x_shortcut + x_residual","metadata":{"execution":{"iopub.status.busy":"2022-01-16T14:05:18.062306Z","iopub.execute_input":"2022-01-16T14:05:18.063192Z","iopub.status.idle":"2022-01-16T14:05:18.070918Z","shell.execute_reply.started":"2022-01-16T14:05:18.063167Z","shell.execute_reply":"2022-01-16T14:05:18.070213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# FPN의 Top_down layer 입니다.\n# lateral connection과 Upsampling이 concatate 한 뒤에 수행합니다.\nclass Top_down(nn.Module):\n    def __init__(self, in_channels, out_channels):\n        super().__init__()\n\n        self.conv = nn.Sequential(\n            BasicConv(in_channels, out_channels, 1, stride=1, padding=0),\n            BasicConv(out_channels, out_channels*2, 3, stride=1, padding=1),\n            BasicConv(out_channels*2, out_channels, 1, stride=1, padding=0),\n            BasicConv(out_channels, out_channels*2, 3, stride=1, padding=1),\n            BasicConv(out_channels*2, out_channels, 1, stride=1, padding=0)\n        )\n\n    def forward(self, x):\n        return self.conv(x)","metadata":{"execution":{"iopub.status.busy":"2022-01-16T14:05:18.072469Z","iopub.execute_input":"2022-01-16T14:05:18.073001Z","iopub.status.idle":"2022-01-16T14:05:18.083300Z","shell.execute_reply.started":"2022-01-16T14:05:18.072962Z","shell.execute_reply":"2022-01-16T14:05:18.082448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# YOLO Layer를 정의합니다.\n# YOLO Layer는 40x40, 80x80, 160x160 피쳐맵에서 예측을 수행합니다.\nclass YOLOLayer(nn.Module):\n    def __init__(self, channels, anchors, num_classes=1, img_dim=1280):\n        super().__init__()\n        self.anchors = anchors # three anchors per YOLO Layer\n        self.num_anchors = len(anchors) # 3\n        self.num_classes = num_classes # VOC classes 1\n        self.img_dim = img_dim # 입력 이미지 크기 1280\n        self.grid_size = 0\n\n        # 예측을 수행하기 전, smooth conv layer 입니다.\n        self.conv = nn.Sequential(\n            BasicConv(channels, channels*2, 3, stride=1, padding=1),\n            nn.Conv2d(channels*2, 18, 1, stride=1, padding=0)\n        )\n\n    def forward(self, x):\n        x = self.conv(x)\n\n        # prediction\n        # x: batch, channels, W, H\n        batch_size = x.size(0)\n        grid_size = x.size(2) # S = 40 or 80 or 160\n        device = x.device\n\n        prediction = x.view(batch_size, self.num_anchors, self.num_classes + 5,\n                            grid_size, grid_size) # shape = (batch, 3, 6, S, S)\n        \n        # shape change (batch, 3, 6, S, S) -> (batch, 3, S, S, 6)\n        prediction = prediction.permute(0, 1, 3, 4, 2)\n        prediction = prediction.contiguous() # continuous data address\n\n        obj_score = torch.sigmoid(prediction[..., 4]) # Confidence: 1 if object, else 0\n        pred_cls = torch.sigmoid(prediction[..., 5:]) # 바운딩 박스 좌표\n\n        # grid_size 갱신\n        if grid_size != self.grid_size:\n            # grid_size를 갱신하고, transform_outputs 함수를 위해 anchor 박스를 전처리 합니다.\n            self.compute_grid_offsets(grid_size, cuda=x.is_cuda)\n\n        # calculate bounding box coordinates\n        pred_boxes = self.transform_outputs(prediction)\n\n        # output shape(batch, num_anchors x S x S, 25)\n        # ex) at 13x13 -> [batch, 507, 25], at 26x26 -> [batch, 2028, 25], at 52x52 -> [batch, 10647, 25]\n        # 최종적으로 YOLO는 10647개의 바운딩박스를 예측합니다.\n        output = torch.cat((pred_boxes.view(batch_size, -1, 4),\n                    obj_score.view(batch_size, -1, 1),\n                    pred_cls.view(batch_size, -1, self.num_classes)), -1)\n        return output\n\n\n    # grid_size를 갱신하고, transform_outputs 함수를 위해 anchor 박스를 전처리 합니다.\n    def compute_grid_offsets(self, grid_size, cuda=True):\n        self.grid_size = grid_size # ex) 16, 32, 64\n        self.stride = self.img_dim / self.grid_size # ex) 80, 40, 20\n\n        # cell index 생성\n        # transform_outputs 함수에서 바운딩 박스의 x, y좌표를 예측할 때 사용합니다.\n        # 1, 1, S, S\n        self.grid_x = torch.arange(grid_size, device=device).repeat(1, 1, grid_size, 1).type(torch.float32)\n        # 1, 1, S, S\n        self.grid_y = torch.arange(grid_size, device=device).repeat(1, 1, grid_size, 1).transpose(3,2).type(torch.float32)\n\n        # anchors를 feature map 크기로 정규화, [0~1] 범위\n        scaled_anchors = [(a_w / self.stride, a_h / self.stride) for a_w, a_h in self.anchors]\n        # tensor로 변환\n        self.scaled_anchors = torch.tensor(scaled_anchors, device=device)\n\n        # transform_outputs 함수에서 바운딩 박스의 w, h를 예측할 때 사용합니다.\n        # shape=(3,2) -> (1, 3, 1, 1)\n        self.anchor_w = self.scaled_anchors[:, 0:1].view(1, self.num_anchors, 1, 1)\n        self.anchor_h = self.scaled_anchors[:, 1:2].view(1, self.num_anchors, 1, 1)\n\n\n    # 예측한 바운딩 박스 좌표를 계산하는 함수입니다.\n    def transform_outputs(self, prediction):\n        # prediction = (batch, num_anchors, S, S, coordinates + classes)\n        device = prediction.device\n        x = torch.sigmoid(prediction[..., 0]) # sigmoid(box x), 예측값을 sigmoid로 감싸서 [0~1] 범위\n        y = torch.sigmoid(prediction[..., 1]) # sigmoid(box y), 예측값을 sigmoid로 감싸서 [0~1] 범위\n        w = prediction[..., 2] # 예측한 바운딩 박스 너비\n        h = prediction[..., 3] # 예측한 바운딩 박스 높이\n\n        pred_boxes = torch.zeros_like(prediction[..., :4]).to(device)\n        pred_boxes[..., 0] = x.data + self.grid_x # sigmoid(box x) + cell x 좌표\n        pred_boxes[..., 1] = y.data + self.grid_y # sigmoid(box y) + cell y 좌표\n        pred_boxes[..., 2] = torch.exp(w.data) * self.anchor_w\n        pred_boxes[..., 3] = torch.exp(h.data) * self.anchor_h\n\n        return pred_boxes * self.stride","metadata":{"execution":{"iopub.status.busy":"2022-01-16T14:05:18.084826Z","iopub.execute_input":"2022-01-16T14:05:18.085422Z","iopub.status.idle":"2022-01-16T14:05:18.237460Z","shell.execute_reply.started":"2022-01-16T14:05:18.085386Z","shell.execute_reply":"2022-01-16T14:05:18.236676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DarkNet(nn.Module):\n    def __init__(self, anchors, num_blocks=[1,2,8,8,4], num_classes=20):\n        super().__init__()\n\n        # feature extractor\n        self.conv1 = BasicConv(3, 32, 3, stride=1, padding=1)\n        self.res_block_1 = self._make_residual_block(64, num_blocks[0]) # 208x208\n        self.res_block_2 = self._make_residual_block(128, num_blocks[1]) # 104x104\n        self.res_block_3 = self._make_residual_block(256, num_blocks[2]) # 52x52, FPN lateral connection\n        self.res_block_4 = self._make_residual_block(512, num_blocks[3]) # 26x26, FPN lateral connection\n        self.res_block_5 = self._make_residual_block(1024, num_blocks[4]) # 13x13, Top layer\n\n        # FPN Top down, conv + upsampling을 수행합니다.\n        self.topdown_1 = Top_down(1024, 512)\n        self.topdown_2 = Top_down(768, 256)\n        self.topdown_3 = Top_down(384, 128)\n\n        # FPN lateral connection\n        # 차원 축소를 위해 사용합니다.\n        self.lateral_1 = BasicConv(512, 256, 1, stride=1, padding=0)\n        self.lateral_2 = BasicConv(256, 128, 1, stride=1, padding=0)\n\n        # prediction, 13x13, 26x26, 52x52 피쳐맵에서 예측을 수행합니다.\n        self.yolo_1 = YOLOLayer(512, anchors=anchors[2]) # 13x13\n        self.yolo_2 = YOLOLayer(256, anchors=anchors[1]) # 26x26\n        self.yolo_3 = YOLOLayer(128, anchors=anchors[0]) # 52x52\n\n        self.upsample = nn.Upsample(scale_factor=2)\n\n\n    def forward(self, x):\n        # feature extractor\n        x = self.conv1(x)\n        c1 = self.res_block_1(x)\n        c2 = self.res_block_2(c1)\n        c3 = self.res_block_3(c2)\n        c4 = self.res_block_4(c3)\n        c5 = self.res_block_5(c4)\n\n        # FPN Top-downm, Upsample and lateral connection \n        p5 = self.topdown_1(c5)\n        p4 = self.topdown_2(torch.cat((self.upsample(p5), self.lateral_1(c4)), 1))\n        p3 = self.topdown_3(torch.cat((self.upsample(p4), self.lateral_2(c3)), 1))\n\n        # prediction\n        yolo_1 = self.yolo_1(p5)\n        yolo_2 = self.yolo_2(p4)\n        yolo_3 = self.yolo_3(p3)\n\n        return torch.cat((yolo_1, yolo_2, yolo_3), 1), [yolo_1, yolo_2, yolo_3]\n\n    def _make_residual_block(self,in_channels, num_block):\n        blocks = []\n\n        # down sample\n        blocks.append(BasicConv(in_channels//2, in_channels, 3, stride=2, padding=1))\n\n        for i in range(num_block):\n            blocks.append(ResidualBlock(in_channels))\n        \n        return nn.Sequential(*blocks)","metadata":{"execution":{"iopub.status.busy":"2022-01-16T14:05:18.239242Z","iopub.execute_input":"2022-01-16T14:05:18.239495Z","iopub.status.idle":"2022-01-16T14:05:18.256590Z","shell.execute_reply.started":"2022-01-16T14:05:18.239460Z","shell.execute_reply":"2022-01-16T14:05:18.255814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**9. Checking Model**","metadata":{}},{"cell_type":"code","source":"anchors = [[(10,13),(16,30),(33,23)],[(30,61),(62,45),(59,119)],[(116,90),(156,198),(373,326)]]\nx = torch.randn(1, 3, 1280, 1280).to(device=device)\nwith torch.no_grad():\n    model = DarkNet(anchors).to(device=device)\n    output_cat , output = model(x)\n    print(output_cat.size())\n    print(output[0].size(), output[1].size(), output[2].size())\n","metadata":{"execution":{"iopub.status.busy":"2022-01-16T14:05:18.258083Z","iopub.execute_input":"2022-01-16T14:05:18.258538Z","iopub.status.idle":"2022-01-16T14:05:18.336778Z","shell.execute_reply.started":"2022-01-16T14:05:18.258504Z","shell.execute_reply":"2022-01-16T14:05:18.335484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**10. Loss Function**","metadata":{}},{"cell_type":"code","source":"def get_loss_batch(output,targets, params_loss, opt=None):\n    ignore_thres=params_loss[\"ignore_thres\"]\n    scaled_anchors= params_loss[\"scaled_anchors\"] # 정규화된 anchor   \n    mse_loss= params_loss[\"mse_loss\"] # nn.MSELoss\n    bce_loss= params_loss[\"bce_loss\"] # nn.BCELoss, 이진 분류에서 사용\n    \n    num_yolos=params_loss[\"num_yolos\"] # 3\n    num_anchors= params_loss[\"num_anchors\"] # 3\n    obj_scale= params_loss[\"obj_scale\"] # 1\n    noobj_scale= params_loss[\"noobj_scale\"] # 100\n\n    loss = 0.0\n\n    for yolo_ind in range(num_yolos):\n        yolo_out = output[yolo_ind] # yolo_out: batch, num_boxes, class+coordinates\n        batch_size, num_bbxs, _ = yolo_out.shape\n\n        # get grid size\n        gz_2 = num_bbxs/num_anchors # ex) at 40x40, 4800 / 3\n        grid_size=int(np.sqrt(gz_2))\n\n        # (batch, num_boxes, class+coordinates) -> (batch, num_anchors, S, S, class+coordinates)\n        yolo_out = yolo_out.view(batch_size, num_anchors, grid_size, grid_size, -1)\n\n        pred_boxes = yolo_out[:,:,:,:,:4] # get box coordinates\n        x,y,w,h = transform_bbox(pred_boxes, scaled_anchors[yolo_ind]) # cell 내에서 x,y 좌표와  \n        pred_conf = yolo_out[:,:,:,:,4] # get confidence\n        pred_cls_prob = yolo_out[:,:,:,:,5:]\n\n        yolo_targets = get_yolo_targets({\n            'pred_cls_prob':pred_cls_prob,\n            'pred_boxes':pred_boxes,\n            'targets':targets,\n            'anchors':scaled_anchors[yolo_ind],\n            'ignore_thres':ignore_thres,\n        })\n\n        obj_mask=yolo_targets[\"obj_mask\"]        \n        noobj_mask=yolo_targets[\"noobj_mask\"]            \n        tx=yolo_targets[\"tx\"]                \n        ty=yolo_targets[\"ty\"]                    \n        tw=yolo_targets[\"tw\"]                        \n        th=yolo_targets[\"th\"]                            \n        tcls=yolo_targets[\"tcls\"]                                \n        t_conf=yolo_targets[\"t_conf\"]\n\n        loss_x = mse_loss(x[obj_mask], tx[obj_mask])\n        loss_y = mse_loss(y[obj_mask], ty[obj_mask])\n        loss_w = mse_loss(w[obj_mask], tw[obj_mask])\n        loss_h = mse_loss(h[obj_mask], th[obj_mask])\n        \n        loss_conf_obj = bce_loss(pred_conf[obj_mask], t_conf[obj_mask])\n        loss_conf_noobj = bce_loss(pred_conf[noobj_mask], t_conf[noobj_mask])\n        loss_conf = obj_scale * loss_conf_obj + noobj_scale * loss_conf_noobj\n        loss_cls = bce_loss(pred_cls_prob[obj_mask], tcls[obj_mask])\n        loss += loss_x + loss_y + loss_w + loss_h + loss_conf + loss_cls\n        \n    if opt is not None:\n        opt.zero_grad()\n        loss.backward()\n        opt.step()\n        \n    return loss.item()","metadata":{"execution":{"iopub.status.busy":"2022-01-16T14:05:18.337882Z","iopub.status.idle":"2022-01-16T14:05:18.338434Z","shell.execute_reply.started":"2022-01-16T14:05:18.338207Z","shell.execute_reply":"2022-01-16T14:05:18.338231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def transform_bbox(bbox, anchors):\n    # bbox: predicted bbox coordinates\n    # anchors: scaled anchors\n\n    x = bbox[:,:,:,:,0]\n    y = bbox[:,:,:,:,1]\n    w = bbox[:,:,:,:,2]\n    h = bbox[:,:,:,:,3]\n    anchor_w = anchors[:,0].view((1,3,1,1))\n    anchor_h = anchors[:,1].view((1,3,1,1))\n\n    x=x-x.floor() # 전체 이미지의 x 좌표에서 셀 내의 x좌표로 변경\n    y=y-y.floor() # 전체 이미지의 y 좌표에서 셀 내의 y좌표로 변경\n    w=torch.log(w / anchor_w + 1e-16)\n    h=torch.log(h / anchor_h + 1e-16)\n    return x, y, w, h","metadata":{"execution":{"iopub.status.busy":"2022-01-16T14:05:18.339538Z","iopub.status.idle":"2022-01-16T14:05:18.340094Z","shell.execute_reply.started":"2022-01-16T14:05:18.339848Z","shell.execute_reply":"2022-01-16T14:05:18.339872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_yolo_targets(params):\n    pred_boxes = params['pred_boxes']\n    pred_cls_prob = params['pred_cls_prob']\n    target = params['targets'] # batchsize, cls, cx, cy, w, h\n    anchors = params['anchors']\n    ignore_thres = params['ignore_thres']\n\n    batch_size = pred_boxes.size(0)\n    num_anchors = pred_boxes.size(1)\n    grid_size = pred_boxes.size(2)\n    num_cls = pred_cls_prob.size(-1)\n\n\n    sizeT = batch_size, num_anchors, grid_size, grid_size\n    obj_mask = torch.zeros(sizeT, device=device, dtype=torch.uint8)\n    noobj_mask = torch.ones(sizeT, device=device, dtype=torch.uint8)\n    tx = torch.zeros(sizeT, device=device, dtype=torch.float32)\n    ty = torch.zeros(sizeT, device=device, dtype=torch.float32)\n    tw = torch.zeros(sizeT, device=device, dtype=torch.float32)\n    th = torch.zeros(sizeT, device=device, dtype=torch.float32)\n\n    sizeT = batch_size, num_anchors, grid_size, grid_size, num_cls\n    tcls = torch.zeros(sizeT, device=device, dtype=torch.float32)\n\n    # target = batch, cx, cy, w, h, class\n    target_bboxes = target[:, 1:5] * grid_size\n    t_xy = target_bboxes[:, :2]\n    t_wh = target_bboxes[:, 2:]\n    t_x, t_y = t_xy.t() # .t(): 전치\n    t_w, t_h = t_wh.t() # .t(): 전치\n\n    grid_i, grid_j = t_xy.long().t() # .long(): int로 변환\n\n    # anchor와 target의 iou 계산\n    iou_with_anchors = [get_iou_WH(anchor, t_wh) for anchor in anchors]\n    iou_with_anchors = torch.stack(iou_with_anchors)\n    best_iou_wa, best_anchor_ind = iou_with_anchors.max(0) # iou가 가장 높은 anchor 추출\n\n    batch_inds, target_labels = target[:, 0].long(), target[:, 5].long()\n    obj_mask[batch_inds, best_anchor_ind, grid_j, grid_i] = 1 # iou가 가장 높은 anchor 할당\n    noobj_mask[batch_inds, best_anchor_ind, grid_j, grid_i] = 0\n\n    # threshold 보다 높은 iou를 지닌 anchor\n    # iou가 가장 높은 anchor만 할당하면 되기 때문입니다.\n    for ind, iou_wa in enumerate(iou_with_anchors.t()):\n        noobj_mask[batch_inds[ind], iou_wa > ignore_thres, grid_j[ind], grid_i[ind]] = 0\n\n    # cell 내에서 x,y로 변환\n    tx[batch_inds, best_anchor_ind, grid_j, grid_i] = t_x - t_x.float()\n    ty[batch_inds, best_anchor_ind, grid_j, grid_i] = t_y - t_y.float()\n\n    anchor_w = anchors[best_anchor_ind][:, 0]\n    tw[batch_inds, best_anchor_ind, grid_j, grid_i] = torch.log(t_w / anchor_w + 1e-16)\n\n    anchor_h = anchors[best_anchor_ind][:, 1]\n    th[batch_inds, best_anchor_ind, grid_j, grid_i] = torch.log(t_h / anchor_h + 1e-16)\n\n    tcls[batch_inds, best_anchor_ind, grid_j, grid_i, target_labels] = 1\n\n    output = {\n        'obj_mask': obj_mask,\n        'noobj_mask': noobj_mask,\n        'tx': tx,\n        'ty': ty,\n        'tw': tw,\n        'th': th,\n        'tcls': tcls,\n        't_conf': obj_mask.float(),\n    }\n    return output","metadata":{"execution":{"iopub.status.busy":"2022-01-16T14:05:18.341223Z","iopub.status.idle":"2022-01-16T14:05:18.341765Z","shell.execute_reply.started":"2022-01-16T14:05:18.341528Z","shell.execute_reply":"2022-01-16T14:05:18.341552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# anchor와 target box의 iou 계산하는 함수입니다.\ndef get_iou_WH(wh1, wh2):\n    wh2 = wh2.t()\n    w1, h1 = wh1[0], wh1[1]\n    w2, h2 = wh2[0], wh2[1]\n    inter_area = torch.min(w1, w2) * torch.min(h1, h2)\n    union_area = (w1 * h1 + 1e-16) + w2 * h2 - inter_area\n    return inter_area / union_area","metadata":{"execution":{"iopub.status.busy":"2022-01-16T14:05:18.342846Z","iopub.status.idle":"2022-01-16T14:05:18.343415Z","shell.execute_reply.started":"2022-01-16T14:05:18.343176Z","shell.execute_reply":"2022-01-16T14:05:18.343201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**10. Model Learning**","metadata":{}},{"cell_type":"code","source":"# 현재 lr 계산하는 함수\ndef get_lr(opt):\n    for param_group in opt.param_groups:\n        return param_group['lr']","metadata":{"execution":{"iopub.status.busy":"2022-01-16T14:05:18.344494Z","iopub.status.idle":"2022-01-16T14:05:18.345058Z","shell.execute_reply.started":"2022-01-16T14:05:18.344816Z","shell.execute_reply":"2022-01-16T14:05:18.344839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# epoch당 loss 계산하는 함수\ndef loss_epoch(model,params_loss,dataset_dl,sanity_check=False,opt=None):\n    running_loss=0.0\n    len_data=len(dataset_dl.dataset)\n    running_metrics= {}\n    \n    for img, target in dataset_dl:\n        target=target.to(device)\n        _,output=model(img.to(device))\n        loss_b=get_loss_batch(output,target, params_loss,opt)\n        running_loss+=loss_b\n        if sanity_check is True:\n            break \n    loss=running_loss/float(len_data)\n    return loss","metadata":{"execution":{"iopub.status.busy":"2022-01-16T14:05:18.346131Z","iopub.status.idle":"2022-01-16T14:05:18.346665Z","shell.execute_reply.started":"2022-01-16T14:05:18.346429Z","shell.execute_reply":"2022-01-16T14:05:18.346453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\ndef train_val(model, params):\n    num_epochs=params[\"num_epochs\"] # 3\n    params_loss=params[\"params_loss\"] # params_loss\n    opt=params[\"optimizer\"] # opt == Adam\n    train_dl=params[\"train_dl\"] # train_dl\n    val_dl=params[\"val_dl\"] # val_dl\n    sanity_check=params[\"sanity_check\"]\n    lr_scheduler=params[\"lr_scheduler\"]\n    path2weights=params[\"path2weights\"] # ./models/weights.pt\n    \n    \n    loss_history={\n        \"train\": [],\n        \"val\": [],\n    }\n    \n    best_model_wts = copy.deepcopy(model.state_dict())\n    best_loss=float('inf') \n    \n    start_time = time.time()\n    for epoch in range(num_epochs):\n        current_lr=get_lr(opt)\n        print('Epoch {}/{}, current lr={}'.format(epoch, num_epochs - 1, current_lr))\n        model.to(device)\n        model.train()\n        train_loss=loss_epoch(model,params_loss,train_dl,sanity_check,opt)\n        loss_history[\"train\"].append(train_loss)  \n        \n        model.eval()\n        with torch.no_grad():\n            val_loss=loss_epoch(model,params_loss,val_dl,sanity_check)\n        loss_history[\"val\"].append(val_loss)\n        \n        if val_loss < best_loss:\n            best_loss = val_loss\n            best_model_wts = copy.deepcopy(model.state_dict())\n            torch.save(model.state_dict(), path2weights)\n            print(\"Copied best model weights!\")\n            print('Get best val loss')\n            \n        lr_scheduler.step(val_loss)\n        if current_lr != get_lr(opt):\n            print(\"Loading best model weights!\")\n            model.load_state_dict(best_model_wts) \n        print(\"train loss: %.6f, val loss: %.6f, time: %.4f min\" %(train_loss, val_loss, (time.time()-start_time)/60))\n        print(\"-\"*10) \n    model.load_state_dict(best_model_wts)\n    return model, loss_history","metadata":{"execution":{"iopub.status.busy":"2022-01-16T14:05:18.347757Z","iopub.status.idle":"2022-01-16T14:05:18.348306Z","shell.execute_reply.started":"2022-01-16T14:05:18.348081Z","shell.execute_reply":"2022-01-16T14:05:18.348105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path2models= \"./models/\"\nif not os.path.exists(path2models):\n        os.mkdir(path2models)\n\nanchors = [[[10,13],  [16,30],  [33,23]], [[30,61],  [62,45],  [59,119]], [[116,90],  [156,198],  [373,32]]]\nscaled_anchors = [[[10/40,13/40], [16/40,30/40], [33/40,23/40]], [[30/80,61/80], [62/80,45/80], [59/80,119/80]], [[116/160,90/160], [156/160,198/160], [373/160,32/160]]]\nmodel = DarkNet(anchors)\n\n'''\nscaled_anchors=[model.module_list[82][0].scaled_anchors,\n                model.module_list[94][0].scaled_anchors,\n                model.module_list[106][0].scaled_anchors]\n'''\n\nmse_loss = nn.MSELoss(reduction=\"sum\")\nbce_loss = nn.BCELoss(reduction=\"sum\")\nparams_loss={\n    \"scaled_anchors\" : scaled_anchors,\n    \"ignore_thres\": 0.5,\n    \"mse_loss\": mse_loss,\n    \"bce_loss\": bce_loss,\n    \"num_yolos\": 3,\n    \"num_anchors\": 3,\n    \"obj_scale\": 1,\n    \"noobj_scale\": 100,\n}","metadata":{"execution":{"iopub.status.busy":"2022-01-16T14:05:18.349375Z","iopub.status.idle":"2022-01-16T14:05:18.349911Z","shell.execute_reply.started":"2022-01-16T14:05:18.349686Z","shell.execute_reply":"2022-01-16T14:05:18.349709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"opt = optim.Adam(model.parameters(), lr=1e-3)\nlr_scheduler = ReduceLROnPlateau(opt, mode='min',factor=0.5, patience=20, verbose=1)","metadata":{"execution":{"iopub.status.busy":"2022-01-16T14:05:18.350987Z","iopub.status.idle":"2022-01-16T14:05:18.351540Z","shell.execute_reply.started":"2022-01-16T14:05:18.351302Z","shell.execute_reply":"2022-01-16T14:05:18.351328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params_train={\n    \"num_epochs\": 3,\n    \"optimizer\": opt, # Adam\n    \"params_loss\": params_loss,\n    \"train_dl\": train_dl, \n    \"val_dl\": val_dl,\n    \"sanity_check\": True,\n    \"lr_scheduler\": lr_scheduler,\n    \"path2weights\": path2models+\"weights.pt\",\n}\n\nmodel, loss_hist = train_val(model, params_train)","metadata":{"execution":{"iopub.status.busy":"2022-01-16T14:05:18.352649Z","iopub.status.idle":"2022-01-16T14:05:18.353181Z","shell.execute_reply.started":"2022-01-16T14:05:18.352959Z","shell.execute_reply":"2022-01-16T14:05:18.352982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Prediction through Model**","metadata":{}},{"cell_type":"code","source":"import greatbarrierreef\nenv = greatbarrierreef.make_env()   # initialize the environment\niter_test = env.iter_test()    # an iterator which loops over the test set and sample submission\nfor (pixel_array, sample_prediction_df) in iter_test:\n    pred_output = model(pixel_array)\n    sample_prediction_df['annotations'] = '0.2 0 0 100 100'  # make your predictions here\n    env.predict(sample_prediction_df)   # register your predictions","metadata":{"execution":{"iopub.status.busy":"2022-01-16T14:05:18.354236Z","iopub.status.idle":"2022-01-16T14:05:18.354770Z","shell.execute_reply.started":"2022-01-16T14:05:18.354534Z","shell.execute_reply":"2022-01-16T14:05:18.354558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}