{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom PIL import Image, ImageDraw\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nimport torchvision as tv","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-02T21:36:30.920418Z","iopub.execute_input":"2022-02-02T21:36:30.921138Z","iopub.status.idle":"2022-02-02T21:36:32.518384Z","shell.execute_reply.started":"2022-02-02T21:36:30.921065Z","shell.execute_reply":"2022-02-02T21:36:32.517432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Faster R-CNN Implementation** - in PyTorch\nThomas Hopkins\n\nWe will start by reading in the training and testing metadata.","metadata":{}},{"cell_type":"code","source":"BASE_DIR = '/kaggle/input/tensorflow-great-barrier-reef'\ntraining_data = pd.read_csv(f'{BASE_DIR}/train.csv')\ntesting_data = pd.read_csv(f'{BASE_DIR}/test.csv')\ntraining_data.info()","metadata":{"execution":{"iopub.status.busy":"2022-02-02T21:36:32.520437Z","iopub.execute_input":"2022-02-02T21:36:32.520819Z","iopub.status.idle":"2022-02-02T21:36:32.615961Z","shell.execute_reply.started":"2022-02-02T21:36:32.520775Z","shell.execute_reply":"2022-02-02T21:36:32.615005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now we can view what a single image looks like with the correct bounding box.\n\n*Note*: For some reason `im.show()` is not working for Firefox on Ubuntu.","metadata":{}},{"cell_type":"code","source":"example = training_data.iloc[5100, :]\nvideo_id = example['video_id']\nvideo_frame = example['video_frame']\nsequence = example['sequence']\nsequence_frame = example['sequence_frame']\nimage_id = example['image_id']\nannotations = eval(example['annotations'])[0]\nprint(f'Image ID: {image_id}')\nxy = [annotations['x'], annotations['y'], annotations['x'] + annotations['width'], annotations['y'] + annotations['height']]\nwith Image.open(f'{BASE_DIR}/train_images/video_{video_id}/{video_frame}.jpg') as im:\n    draw = ImageDraw.Draw(im)\n    draw.rectangle(xy, outline='red', width=2)\n    im.save('fig.jpg')\nImage.open('fig.jpg')","metadata":{"execution":{"iopub.status.busy":"2022-02-02T21:36:32.617311Z","iopub.execute_input":"2022-02-02T21:36:32.617605Z","iopub.status.idle":"2022-02-02T21:36:33.218141Z","shell.execute_reply.started":"2022-02-02T21:36:32.617564Z","shell.execute_reply":"2022-02-02T21:36:33.217484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's also see what the input size of the image looks like.","metadata":{}},{"cell_type":"code","source":"print(f'Number of images: {len(training_data)}')\nwith Image.open(f'{BASE_DIR}/train_images/video_{video_id}/{video_frame}.jpg') as im:\n    transform = tv.transforms.PILToTensor()\n    tensor = transform(im)\nprint(f'Channels: {tensor.shape[0]}\\nHeight: {tensor.shape[1]}\\nWidth: {tensor.shape[2]}')","metadata":{"execution":{"iopub.status.busy":"2022-02-02T21:36:33.220517Z","iopub.execute_input":"2022-02-02T21:36:33.220972Z","iopub.status.idle":"2022-02-02T21:36:33.269335Z","shell.execute_reply.started":"2022-02-02T21:36:33.220917Z","shell.execute_reply":"2022-02-02T21:36:33.268561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now that we see what kind of data we are working with and the problem we are trying to solve, we can implement a neural network that will solve this problem. The network architecture I chose is from the paper *Faster R-CNN: Towards Real-Time Object Detection with Region Proposal Networks*. I believe this should yield decent results without too much tweaking necessary. Here is my implementation of this architecture in PyTorch:","metadata":{}},{"cell_type":"code","source":"class VGG(nn.Module):\n    ''' VGG-16 Network with only convolutions '''\n    def __init__(self):\n        super().__init__()\n        self.feature_extractor = nn.Sequential(\n            nn.Conv2d(3, 64, 3, padding=1),\n            nn.ReLU(),\n            nn.Conv2d(64, 64, 3, padding=1),\n            nn.ReLU(),\n            nn.MaxPool2d(2),\n            nn.Conv2d(64, 128, 3, padding=1),\n            nn.ReLU(),\n            nn.Conv2d(128, 128, 3, padding=1),\n            nn.ReLU(),\n            nn.MaxPool2d(2),\n            nn.Conv2d(128, 256, 3, padding=1),\n            nn.ReLU(),\n            nn.Conv2d(256, 256, 3, padding=1),\n            nn.ReLU(),\n            nn.Conv2d(256, 256, 3, padding=1),\n            nn.ReLU(),\n            nn.MaxPool2d(2),\n            nn.Conv2d(256, 512, 3, padding=1),\n            nn.ReLU(),\n            nn.Conv2d(512, 512, 3, padding=1),\n            nn.ReLU(),\n            nn.Conv2d(512, 512, 3, padding=1),\n            nn.ReLU(),\n            nn.MaxPool2d(2),\n            nn.Conv2d(512, 512, 3, padding=1),\n            nn.ReLU(),\n            nn.Conv2d(512, 512, 3, padding=1),\n            nn.ReLU(),\n            nn.Conv2d(512, 512, 3, padding=1),\n            nn.ReLU(),\n            nn.MaxPool2d(2))\n        \n    def forward(self, x):\n        return self.feature_extractor(x)\n\n\nclass RPN(nn.Module):\n    ''' Region Proposal Network '''\n    def __init__(self, k=9):\n        self.preprocess = nn.Sequential(\n            nn.Conv2d(512, 512, 3),\n            nn.ReLU())\n        self.classifier = nn.Conv2d(512, 2 * k, 1)\n        self.regressor = nn.Conv2d(512, 4 * k, 1)\n        \n    def transform(self, bbox_preds):\n        ''' Converts predictions into [x1, y1, x2, y2] coordinates '''\n        pass\n    \n    def forward(self, x):\n        x = self.preprocess(x)\n        obj_preds = self.classifier(x)\n        bbox_preds = self.regressor(x)\n        # TODO: need to convert classification and regression to\n        return obj_preds, bbox_preds\n    \n\nclass FastRCNN(nn.Module):\n    ''' Fast R-CNN Network '''\n    def __init__(self, crop_size=7):\n        self.feature_extractor = nn.Sequential(\n            nn.Flatten(),\n            nn.Linear(512 * crop_size * crop_size, 1024),\n            nn.ReLU(),\n            nn.Linear(1024, 1024),\n            nn.ReLU())\n        self.crop_size = crop_size\n        # 4 is refined bounding boxes for detecting starfish\n        self.regressor = nn.Linear(1024, 4)\n        # 2 is for whether the box contains the starfish or not\n        # might not need this since RPN detects object or not\n        self.classifier = nn.Linear(1024, 2)\n         \n    def forward(self, x, proposals):\n        x = tv.ops.roi_pool(x, proposals, output_size=self.crop_size)\n        x = self.feature_extractor(x)\n        bbox_preds = self.regressor(x)\n        class_preds = self.classifier(x)\n        return class_preds, bbox_preds\n\n\nclass FasterRCNN(nn.Module):\n    def __init__(self):\n        self.VGG = VGG()\n        self.RPN = RPN()\n        self.FastRCNN = FastRCNN()\n        \n    def forward(self, x):\n        x = self.VGG(x)\n        proposals = self.RPN(x)\n        outputs = self.FastRCNN(x, proposals)\n        return proposals, outputs","metadata":{"execution":{"iopub.status.busy":"2022-02-02T22:03:51.984903Z","iopub.execute_input":"2022-02-02T22:03:51.985207Z","iopub.status.idle":"2022-02-02T22:03:52.007501Z","shell.execute_reply.started":"2022-02-02T22:03:51.985175Z","shell.execute_reply":"2022-02-02T22:03:52.006920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"With that out of the way, we can now define the multi-task loss functions that we will use to optimize the network.\n\nThis will be a combination of smooth $L_1$ (for bounding box prediction) and binary cross-entropy (for classification).","metadata":{}},{"cell_type":"code","source":"class RPNLoss(nn.Module):\n    ''' Computes combined loss for classification and regression '''\n    def __init__(self, n_cls=256, n_reg=2400, l=10):\n        ''' Defines regularization parameters (usually batch size and total # of anchors) '''\n        self.n_cls = n_cls\n        self.n_reg = n_reg\n        self.l = l\n    \n    def forward(self, predictions, targets):\n        bbox_preds = predictions[0]\n        class_preds = predictions[1]\n        bbox_truth = targets[0]\n        class_truth = targets[1]\n        return (1 / self.n_cls) * F.binary_cross_entropy_with_logits(class_preds, class_truth) \\\n                + (self.l / self.n_reg) * (class_truth * F.smooth_l1_loss(bbox_preds, bbox_truth))","metadata":{"execution":{"iopub.status.busy":"2022-02-02T22:03:52.550911Z","iopub.execute_input":"2022-02-02T22:03:52.551329Z","iopub.status.idle":"2022-02-02T22:03:52.558241Z","shell.execute_reply.started":"2022-02-02T22:03:52.551291Z","shell.execute_reply":"2022-02-02T22:03:52.557530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now, finally we can perform the 4-step training process which proceeds as follows:\n1. train the RPN end-to-end, using pre-trained ImageNet model for VGG-16 (?).\n2. train the Fast R-CNN using proposals generated by RPN (RPN is fixed now), VGG-16 is re-initialized with pre-trained ImageNet model (?). \n3. share VGG-16 with RPN and fine-tune only the RPN parameters.\n4. keeping VGG-16 fixed, fine-tune Fast R-CNN parameters.\n\n(?) means I am unsure that I need to do this.","metadata":{}},{"cell_type":"code","source":"def convert_annotations(annotations):\n    ''' Convert annotations from string to array '''\n    if annotations == '[]':\n        return torch.tensor([])\n    list_of_bbox = eval(annotations)\n    return torch.tensor([[d['x'] + d['width'] // 2, d['y'] + d['height'] // 2,\n                          d['width'], d['height']]\n                         for d in list_of_bbox])\n    \n\ndef get_anchor_boxes(width, height, \n                     aspect_ratios=[[1, 1], [1, 2], [2, 1]],\n                     scales=[128, 256, 512]):\n    ''' Generate the set of possible anchor boxes (size (W * H * k) x 4) '''\n    \n    '''\n    TODO: continue working on generating all possible (valid) anchor boxes\n    The size output of VGG is (1x512x22x40) with an input image of size (1x3x720x1280)\n    So we need to figure out the anchor points (x, y) that each coordinate of 22x40 maps\n    to in 720x1280. Then we can generate the set of possible boxes and check for validity.\n    Valid boxes do not overlap with the image boundaries.\n    '''\n    W = torch.arange(1, width)\n    H = torch.arange(1, height)\n    ratios = torch.tensor(aspect_ratios).flatten()\n    scales = torch.tensor(scales)\n    \n    print(torch.outer(scales, ratios))\n    \n\ndef sample_annotations(true_annotations, sample_size,\n                       iou_positive_thresh=0.7, iou_negative_thresh=0.3,\n                       aspect_ratios=[[1, 1], [1, 2], [2, 1]], scales=[128, 256, 512]):\n    ''' Randomly generates positive/negative anchor boxes '''\n    print(sample_size)\n    print(true_annotations)\n    \n\ndef get_images(dataframe, transform=tv.transforms.PILToTensor(),\n               num_annotations=256):\n    ''' Generator which loads an image for training '''\n    ''' \n    TODO: Look into generating a batch from a single image.\n        This would be something like a batch size of 8 where there are\n        positive and negative bounding boxes. I.E. bounding boxes that\n        contain a starfish are positive, bounding boxes that do not are\n        negative. Striking a balance in positive and negative samples\n        here is important. The data has many images without any bounding boxes.\n        Not sure how to handle this just yet.\n    '''\n    data = dataframe.sample()\n    with Image.open(f'{BASE_DIR}/train_images/video_{data.iloc[0][\"video_id\"]}/{data.iloc[0][\"video_frame\"]}.jpg') as im:\n        x = transform(im)\n    y = convert_annotations(data.iloc[0]['annotations'])\n    yield x, sample_annotations(x, y, num_annotations)","metadata":{"execution":{"iopub.status.busy":"2022-02-02T22:03:52.997424Z","iopub.execute_input":"2022-02-02T22:03:52.998092Z","iopub.status.idle":"2022-02-02T22:03:53.009616Z","shell.execute_reply.started":"2022-02-02T22:03:52.998057Z","shell.execute_reply":"2022-02-02T22:03:53.009079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform = tv.transforms.Compose([\n    tv.transforms.PILToTensor(),\n    tv.transforms.ConvertImageDtype(torch.float)\n])\nim, annotations = next(iter(get_images(training_data, transform=transform)))\nprint(im.shape)\ntest_net = VGG()\nprint(test_net(im.unsqueeze(0)).shape)","metadata":{"execution":{"iopub.status.busy":"2022-02-02T22:06:50.939539Z","iopub.execute_input":"2022-02-02T22:06:50.940046Z","iopub.status.idle":"2022-02-02T22:06:57.028468Z","shell.execute_reply.started":"2022-02-02T22:06:50.939983Z","shell.execute_reply":"2022-02-02T22:06:57.027637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}