{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **About author: I'm a beginner in this field trying to learn and discovering the enjoyment of Data Science.**\n### Note1: This notebook is a copy version plus some editing and experimenting for my own understanding and learning.\n### Note2: If this notebook is useful for you in anyway, please give an upvote or commenting your gratitude on the notebook in the reference section. ","metadata":{}},{"cell_type":"markdown","source":"# **Import**","metadata":{}},{"cell_type":"code","source":"# Very few imports. This is a pure torch solution!\nimport cv2\nimport time\n\nimport numpy as np\nimport pandas as pd\nfrom matplotlib import pyplot as plt\n\nimport albumentations as A\nfrom albumentations.pytorch.transforms import ToTensorV2\n\nimport torch\nimport torchvision\nfrom torch.utils.data import DataLoader, Dataset\nfrom torchvision.models.detection.faster_rcnn import FastRCNNPredictor\nfrom torchvision.models.detection import FasterRCNN","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEVICE = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\nBASE_DIR = \"../input/tensorflow-great-barrier-reef/train_images/\"\n\nNUM_EPOCHS = 20","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Load df** \n# from [CoTS - CV strategy: subsequences](https://www.kaggle.com/warotjanpinitrat/cv-strategy-subsequences)","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(\"../input/reef-cv-strategy-subsequences-dataframes/train-validation-split/train-0.1.csv\")\n\n# Turn annotations from strings into lists of dictionaries\ndf['annotations'] = df['annotations'].apply(eval)\n\n# Create the image path for the row\ndf['image_path'] = \"video_\" + df['video_id'].astype(str) + \"/\" + df['video_frame'].astype(str) + \".jpg\"\n\ndf.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train, df_val = df[df['is_train']], df[~df['is_train']]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The model doesn't support images with no annotations\n# It raises an error that suggest that it just doesn't support them:\n# V    alueError: No ground-truth boxes available for one of the images during training\n# I'm dropping those images for now\n# https://discuss.pytorch.org/t/fasterrcnn-images-with-no-objects-present-cause-an-error/117974/3\ndf_train = df_train[df_train.annotations.str.len() > 0 ].reset_index(drop=True)\ndf_val = df_val[df_val.annotations.str.len() > 0 ].reset_index(drop=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.shape[0], df_val.shape[0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Dataset Class**","metadata":{}},{"cell_type":"code","source":"class ReefDataset:\n\n    def __init__(self, df, transforms=None):\n        self.df = df\n        self.transforms = transforms\n\n    def can_augment(self, boxes):\n        \"\"\" Check if bounding boxes are OK to augment\n        \n        \n        For example: image_id 1-490 has a bounding box that is partially outside of the image\n        It breaks albumentation\n        Here we check the margins are within the image to make sure the augmentation can be applied\n        \"\"\"\n        \n        box_outside_image = ((boxes[:, 0] < 0).any() or (boxes[:, 1] < 0).any() \n                             or (boxes[:, 2] > 1280).any() or (boxes[:, 3] > 720).any())\n        return not box_outside_image\n\n    def get_boxes(self, row):\n        \"\"\"Returns the bboxes for a given row as a 3D matrix with format [x_min, y_min, x_max, y_max]\"\"\"\n        \n        boxes = pd.DataFrame(row['annotations'], columns=['x', 'y', 'width', 'height']).astype(float).values\n        \n        # Change from [x_min, y_min, w, h] to [x_min, y_min, x_max, y_max]\n        boxes[:, 2] = boxes[:, 0] + boxes[:, 2]\n        boxes[:, 3] = boxes[:, 1] + boxes[:, 3]\n        return boxes\n    \n    def get_image(self, row):\n        \"\"\"Gets the image for a given row\"\"\"\n        \n        image = cv2.imread(f'{BASE_DIR}/{row[\"image_path\"]}', cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n        image /= 255.0\n        return image\n    \n    def __getitem__(self, i):\n\n        row = self.df.iloc[i]\n        image = self.get_image(row)\n        boxes = self.get_boxes(row)\n        \n        n_boxes = boxes.shape[0]\n        \n        # Calculate the area\n        area = (boxes[:, 3] - boxes[:, 1]) * (boxes[:, 2] - boxes[:, 0])\n        \n        \n        target = {\n            'boxes': torch.as_tensor(boxes, dtype=torch.float32),\n            'area': torch.as_tensor(area, dtype=torch.float32),\n            \n            'image_id': torch.tensor([i]),\n            \n            # There is only one class\n            'labels': torch.ones((n_boxes,), dtype=torch.int64),\n            \n            # Suppose all instances are not crowd\n            'iscrowd': torch.zeros((n_boxes,), dtype=torch.int64)            \n        }\n\n        if self.transforms and self.can_augment(boxes):\n            sample = {\n                'image': image,\n                'bboxes': target['boxes'],\n                'labels': target['labels']\n            }\n            sample = self.transforms(**sample)\n            image = sample['image']\n            \n            if n_boxes > 0:\n                target['boxes'] = torch.stack(tuple(map(torch.tensor, zip(*sample['bboxes'])))).permute(1, 0)\n        else:\n            image = ToTensorV2(p=1.0)(image=image)['image']\n\n        return image, target\n\n    def __len__(self):\n        return len(self.df)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Augmentation**","metadata":{}},{"cell_type":"code","source":"def get_train_transform():\n    return A.Compose([\n        A.Flip(0.5),\n        ToTensorV2(p=1.0)\n    ], bbox_params={'format': 'pascal_voc', 'label_fields': ['labels']})\n\n\ndef get_valid_transform():\n    return A.Compose([\n        ToTensorV2(p=1.0)\n    ], bbox_params={'format': 'pascal_voc', 'label_fields': ['labels']})","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define datasets\nds_train = ReefDataset(df_train, get_train_transform())\nds_val = ReefDataset(df_val, get_valid_transform())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's get an interesting one ;)\ndf_train[df_train.annotations.str.len() > 12].head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image, targets = ds_train[2200]\nimage","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"boxes = targets['boxes'].cpu().numpy().astype(np.int32)\nimg = image.permute(1,2,0).cpu().numpy()\nfig, ax = plt.subplots(1, 1, figsize=(16, 8))\n\nfor box in boxes:\n    cv2.rectangle(img,\n                  (box[0], box[1]),\n                  (box[2], box[3]),\n                  (220, 0, 0), 3)\n    \nax.set_axis_off()\nax.imshow(img);","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **DataLoader**","metadata":{}},{"cell_type":"code","source":"def collate_fn(batch):\n    return tuple(zip(*batch))\n\ndl_train = DataLoader(ds_train, batch_size=8, shuffle=False, num_workers=4, collate_fn=collate_fn)\ndl_val = DataLoader(ds_val, batch_size=8, shuffle=False, num_workers=4, collate_fn=collate_fn)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Model**","metadata":{}},{"cell_type":"markdown","source":"### For understanding and learning purpose, I will leave the link to model document [click here](https://pytorch.org/vision/stable/_modules/torchvision/models/detection/faster_rcnn.html)","metadata":{}},{"cell_type":"code","source":"def get_model():\n    # load a model; pre-trained on COCO\n    model = torchvision.models.detection.fasterrcnn_resnet50_fpn(pretrained=True)\n\n    num_classes = 2  # 1 class (starfish) + background\n\n    # get number of input features for the classifier\n    in_features = model.roi_heads.box_predictor.cls_score.in_features\n\n    # replace the pre-trained head with a new one\n    model.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)\n\n    model.to(DEVICE)\n    return model\n\nmodel = get_model()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = [p for p in model.parameters() if p.requires_grad]\noptimizer = torch.optim.SGD(params, lr=0.0015, momentum=0.9, weight_decay=0.0005)\n# lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=3, gamma=0.1)\nlr_scheduler = None\n\nn_batches, n_batches_val = len(dl_train), len(dl_val)\nvalidation_losses = []\n\n\nfor epoch in range(NUM_EPOCHS):\n    time_start = time.time()\n    loss_accum = 0\n    \n    for batch_idx, (images, targets) in enumerate(dl_train, 1):\n        \n        images = list(image.float().to(DEVICE) for image in images)\n        targets = [{k: v.to(torch.float32).to(DEVICE) if \"box\" in k else v.to(DEVICE) for k, v in t.items()} for t in targets]\n\n        # Predict\n        loss_dict = model(images, targets)\n        losses = sum(loss for loss in loss_dict.values())\n        loss_value = losses.item()\n\n        loss_accum += loss_value\n\n        # Back-prop\n        optimizer.zero_grad()\n        losses.backward()\n        optimizer.step()\n\n    \n    # update the learning rate\n    if lr_scheduler is not None:\n        lr_scheduler.step()\n\n    # Validation \n    val_loss_accum = 0\n        \n    with torch.no_grad():\n        for batch_idx, (images, targets) in enumerate(dl_val, 1):\n            images = list(image.float().to(DEVICE) for image in images)\n            targets = [{k: v.to(torch.float32).to(DEVICE) if \"box\" in k else v.to(DEVICE) for k, v in t.items()} for t in targets]\n            \n            val_loss_dict = model(images, targets)\n            val_batch_loss = sum(loss for loss in val_loss_dict.values())\n            val_loss_accum += val_batch_loss.item()\n    \n    # Logging\n    val_loss = val_loss_accum / n_batches_val\n    train_loss = loss_accum / n_batches\n    validation_losses.append(val_loss)\n    \n    # Save model\n    chk_name = f'fasterrcnn_resnet50_fpn-e{epoch}.bin'\n    torch.save(model.state_dict(), chk_name)\n    \n    \n    elapsed = time.time() - time_start\n    \n    print(f\"[Epoch {epoch+1:2d} / {NUM_EPOCHS:2d}] Train loss: {train_loss:.3f}. Val loss: {val_loss:.3f} --> {chk_name}  [{elapsed:.0f} secs]\")   ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation_losses","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.argmin(validation_losses)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Check result**","metadata":{}},{"cell_type":"code","source":"# idx = 0\n\n# images, targets = next(iter(dl_val))\n# images = list(img.to(DEVICE) for img in images)\n# targets = [{k: v.to(DEVICE) for k, v in t.items()} for t in targets]\n\n# boxes = targets[idx]['boxes'].cpu().numpy().astype(np.int32)\n# sample = images[idx].permute(1,2,0).cpu().numpy()\n\n# model.eval()\n\n# outputs = model(images)\n# outputs = [{k: v.detach().cpu().numpy() for k, v in t.items()} for t in outputs]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fig, ax = plt.subplots(1, 1, figsize=(16, 8))\n\n# # Red for ground truth\n# for box in boxes:\n#     cv2.rectangle(sample,\n#                   (box[0], box[1]),\n#                   (box[2], box[3]),\n#                   (220, 0, 0), 3)\n\n    \n# # Green for predictions\n# # Print the first 5\n# for box in outputs[idx]['boxes'][:5]:\n#     cv2.rectangle(sample,\n#                   (box[0], box[1]),\n#                   (box[2], box[3]),\n#                   (0, 220, 0), 3)\n\n# ax.set_axis_off()\n# ax.imshow(sample);","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Reference**\n###  [🐠 Reef- Starter Torch FasterRCNN Train [LB=0.416]](https://www.kaggle.com/julian3833/reef-starter-torch-fasterrcnn-train-lb-0-416)\n###  [Pytorch Starter - FasterRCNN Train](https://www.kaggle.com/pestipeti/pytorch-starter-fasterrcnn-train)","metadata":{}},{"cell_type":"markdown","source":"#### You can criticize my work or give your suggestion, your comment is a treasure of knowledge for me\n##### P.S. sorry for a poor grammar","metadata":{}}]}