{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# PyTorch Lightning Faster R-CNN Inference LB=0.358\nRun inference with trained model from [Starter PyTorch Lightning Faster R-CNN Finetuning](https://www.kaggle.com/clemchris/starter-pytorch-lightning-faster-r-cnn-finetuning).\n\nInference notebooks cannot use the internet. Therefore, we need to upload the required wheels that are not included in Kaggle's Docker image, pretrained weights, and model checkpoints ourselves using Kaggle Datasets.\n\n## Sources and Inspirations\n- [Reef- Starter Torch FasterRCNN Infer [LB=0.416]](https://www.kaggle.com/julian3833/reef-starter-torch-fasterrcnn-infer-lb-0-416)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"## Imports","metadata":{}},{"cell_type":"code","source":"import math\nfrom pathlib import Path\n\nimport greatbarrierreef\nimport numpy as np\nimport pytorch_lightning as pl\nimport torch\nimport torchmetrics\nfrom torchvision.models.detection import fasterrcnn_resnet50_fpn\nfrom torchvision.models.detection.faster_rcnn import FastRCNNPredictor\nfrom torchvision.models.detection.retinanet import retinanet_resnet50_fpn","metadata":{"execution":{"iopub.status.busy":"2021-11-27T16:08:00.250879Z","iopub.execute_input":"2021-11-27T16:08:00.251375Z","iopub.status.idle":"2021-11-27T16:08:00.257266Z","shell.execute_reply.started":"2021-11-27T16:08:00.251335Z","shell.execute_reply":"2021-11-27T16:08:00.256549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Paths","metadata":{}},{"cell_type":"code","source":"INPUT_DIR = Path(\"../input\")\n\nCHECKPOINTS_DIR = INPUT_DIR / \"fasterrcnn-checkpoints\"\n\nPRETRAINED_WEIGHTS_PATH = CHECKPOINTS_DIR / \"fasterrcnn_resnet50_fpn_coco-258fb6c6.pth\"\nFINETUNED_CHECKPOINT_PATH = CHECKPOINTS_DIR / \"model.ckpt\"\n","metadata":{"execution":{"iopub.status.busy":"2021-11-27T16:08:00.258768Z","iopub.execute_input":"2021-11-27T16:08:00.259389Z","iopub.status.idle":"2021-11-27T16:08:00.270765Z","shell.execute_reply.started":"2021-11-27T16:08:00.259346Z","shell.execute_reply":"2021-11-27T16:08:00.270036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Settings","metadata":{}},{"cell_type":"code","source":"DETECTION_THRESHOLD = 0.66\n\nDEVICE = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')","metadata":{"execution":{"iopub.status.busy":"2021-11-27T16:08:00.271794Z","iopub.execute_input":"2021-11-27T16:08:00.273054Z","iopub.status.idle":"2021-11-27T16:08:00.280116Z","shell.execute_reply.started":"2021-11-27T16:08:00.273021Z","shell.execute_reply":"2021-11-27T16:08:00.2794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Lightning Module Class","metadata":{}},{"cell_type":"code","source":"class GBRModule(pl.LightningModule):\n    \"\"\"LightningModule class to finetune torchvision's Faster R-CNN model.\"\"\"\n    \n    def __init__(self, pretrained_weights_path=None):\n        super().__init__()\n\n        self.model = self._create_model(pretrained_weights_path)\n\n        # self.val_map = torchmetrics.MAP()\n\n    def _create_model(self, pretrained_weights_path):\n        \"\"\"Creates finetunable Faster R-CNN model.\"\"\"\n        if pretrained_weights_path is None:\n            model = fasterrcnn_resnet50_fpn(pretrained=True)\n        else:\n            model = fasterrcnn_resnet50_fpn(pretrained=False, pretrained_backbone=False)\n            model.load_state_dict(torch.load(pretrained_weights_path))\n\n        in_features = model.roi_heads.box_predictor.cls_score.in_features\n        model.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes=2)\n\n        return model\n\n    def forward(self, image):\n        \"\"\"Runs inference.\"\"\"\n        self.model.eval()\n        output = self.model(image)\n\n        return output\n\n    def training_step(self, batch, batch_idx):\n        image, target = batch\n        loss_dict = self.model(image, target)\n        losses = sum(loss for loss in loss_dict.values())\n\n        batch_size = len(batch[0])\n        self.log_dict(loss_dict, batch_size=batch_size)\n        self.log(\"train_loss\", losses, batch_size=batch_size)\n\n        return losses\n\n    def validation_step(self, batch, batch_idx):\n        image, target = batch\n        output = self.model(image)\n\n        #val_map = self.val_map(output, target)\n        #self.log(\"val_map\", val_map[\"map\"])\n\n    def configure_optimizers(self):\n        params = [p for p in self.model.parameters() if p.requires_grad]\n        optimizer = torch.optim.SGD(params, lr=0.005, momentum=0.9, weight_decay=0.0005)\n\n        lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=3, gamma=0.1)\n\n        return [optimizer], [lr_scheduler]","metadata":{"execution":{"iopub.status.busy":"2021-11-27T16:08:00.283463Z","iopub.execute_input":"2021-11-27T16:08:00.28396Z","iopub.status.idle":"2021-11-27T16:08:00.297545Z","shell.execute_reply.started":"2021-11-27T16:08:00.283929Z","shell.execute_reply":"2021-11-27T16:08:00.296893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Helper Functions","metadata":{}},{"cell_type":"code","source":"def get_eval_gbr_module(checkpoint_path, pretrained_weights_path, device):\n    gbr_module = GBRModule.load_from_checkpoint(checkpoint_path, pretrained_weights_path=pretrained_weights_path)\n    gbr_module.eval()\n    \n    gbr_module = gbr_module.to(device)\n    \n    return gbr_module","metadata":{"execution":{"iopub.status.busy":"2021-11-27T16:08:00.300682Z","iopub.execute_input":"2021-11-27T16:08:00.30088Z","iopub.status.idle":"2021-11-27T16:08:00.308436Z","shell.execute_reply.started":"2021-11-27T16:08:00.300855Z","shell.execute_reply":"2021-11-27T16:08:00.3077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_prediction_string(boxes, scores):\n    pred_strings = []\n    for box, score in zip(boxes, scores):\n        x, y, w, h = box\n        pred_string = f\"{score:.2f} {x} {y} {w} {h}\"\n        pred_strings.append(pred_string)\n    \n    pred_string = \" \".join(pred_strings)\n    return pred_string","metadata":{"execution":{"iopub.status.busy":"2021-11-27T16:08:00.310296Z","iopub.execute_input":"2021-11-27T16:08:00.310741Z","iopub.status.idle":"2021-11-27T16:08:00.320115Z","shell.execute_reply.started":"2021-11-27T16:08:00.310704Z","shell.execute_reply":"2021-11-27T16:08:00.319356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# From: https://www.kaggle.com/julian3833/reef-starter-torch-fasterrcnn-infer-lb-0-416\ndef predict(model, pixel_array, detection_threshold):\n    # Convert to tensor\n    pixel_array = pixel_array.astype(np.float32) / 255.\n    image = torch.from_numpy(pixel_array.transpose(2, 0, 1)).unsqueeze(0)\n    \n    with torch.no_grad():\n        outputs = model(image.to(model.device))[0]\n    \n    # Move predictions to cpu and numpy\n    boxes = outputs['boxes'].data.cpu().numpy()\n    scores = outputs['scores'].data.cpu().numpy()\n    \n    # Filter predictions with low score\n    boxes = boxes[scores >= detection_threshold].astype(np.int32)\n    scores = scores[scores >= detection_threshold]\n    \n    # Go back from x_min, y_min, x_max, y_max to x_min, y_min, w, h\n    boxes[:, 2] = boxes[:, 2] - boxes[:, 0]\n    boxes[:, 3] = boxes[:, 3] - boxes[:, 1]\n  \n    prediction_string = create_prediction_string(boxes, scores)\n    return prediction_string","metadata":{"execution":{"iopub.status.busy":"2021-11-27T16:08:00.321569Z","iopub.execute_input":"2021-11-27T16:08:00.322088Z","iopub.status.idle":"2021-11-27T16:08:00.330613Z","shell.execute_reply.started":"2021-11-27T16:08:00.322053Z","shell.execute_reply":"2021-11-27T16:08:00.32994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create Submission env","metadata":{}},{"cell_type":"code","source":"env = greatbarrierreef.make_env()\niter_test = env.iter_test() ","metadata":{"execution":{"iopub.status.busy":"2021-11-27T16:08:00.331871Z","iopub.execute_input":"2021-11-27T16:08:00.332177Z","iopub.status.idle":"2021-11-27T16:08:00.520266Z","shell.execute_reply.started":"2021-11-27T16:08:00.332141Z","shell.execute_reply":"2021-11-27T16:08:00.519331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Predict","metadata":{}},{"cell_type":"code","source":"eval_gbr_module = get_eval_gbr_module(FINETUNED_CHECKPOINT_PATH, PRETRAINED_WEIGHTS_PATH, DEVICE)","metadata":{"execution":{"iopub.status.busy":"2021-11-27T16:08:00.521356Z","iopub.status.idle":"2021-11-27T16:08:00.521927Z","shell.execute_reply.started":"2021-11-27T16:08:00.521684Z","shell.execute_reply":"2021-11-27T16:08:00.521707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for (pixel_array, df_pred) in iter_test:  \n    prediction_string = predict(eval_gbr_module, pixel_array, DETECTION_THRESHOLD)\n    print(prediction_string)\n    df_pred['annotations'] = prediction_string\n    env.predict(df_pred)","metadata":{"execution":{"iopub.status.busy":"2021-11-27T16:08:00.523053Z","iopub.status.idle":"2021-11-27T16:08:00.523609Z","shell.execute_reply.started":"2021-11-27T16:08:00.523354Z","shell.execute_reply":"2021-11-27T16:08:00.523387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}