{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Section 1: Exploratory Analysis\n","metadata":{}},{"cell_type":"code","source":"import cv2\nfrom matplotlib import pyplot as plt\nimport pandas as pd \n","metadata":{"execution":{"iopub.status.busy":"2022-02-03T14:08:12.886794Z","iopub.execute_input":"2022-02-03T14:08:12.887240Z","iopub.status.idle":"2022-02-03T14:08:13.260191Z","shell.execute_reply.started":"2022-02-03T14:08:12.887156Z","shell.execute_reply":"2022-02-03T14:08:13.259090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_metadata = pd.read_csv(\"../input/tensorflow-great-barrier-reef/train.csv\")\ntrain_metadata[train_metadata.annotations!=\"[]\"]","metadata":{"execution":{"iopub.status.busy":"2022-02-03T14:08:13.263456Z","iopub.execute_input":"2022-02-03T14:08:13.263714Z","iopub.status.idle":"2022-02-03T14:08:13.348506Z","shell.execute_reply.started":"2022-02-03T14:08:13.263686Z","shell.execute_reply":"2022-02-03T14:08:13.347004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from ast import literal_eval\n\ndef load_image(video_id, image_id):\n    path = f\"../input/tensorflow-great-barrier-reef/train_images/video_{video_id}/{image_id}.jpg\"\n    img = cv2.imread(path)\n    return img\n\ndef plot_image(img):\n    img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)\n    plt.imshow(img)\n    return plt.show()\n\ndef parse_annotations(annotations):\n    return literal_eval(annotations)\n\ndef load_image_with_annotations(video_id, image_id, annotations):\n    img = load_image(video_id, image_id)\n    for ret in parse_annotations(annotations):\n        cv2.rectangle(img,\n                      (ret['x'], ret['y']),\n                      (ret['x'] + ret['width'], ret['y'] + ret['height']),\n                      (0,0,255),\n                      2)\n    return img\n\nplot_image(load_image(0, 16))\nimg = load_image_with_annotations(\n                            train_metadata.video_id.iloc[16],\n                            train_metadata.video_frame.iloc[16],\n                            train_metadata.annotations.iloc[16],\n                            )\nfigsize = (16,8)\nplt.figure(figsize=figsize)\nplot_image(img)\n","metadata":{"execution":{"iopub.status.busy":"2022-02-03T14:08:39.136974Z","iopub.execute_input":"2022-02-03T14:08:39.137374Z","iopub.status.idle":"2022-02-03T14:08:40.034303Z","shell.execute_reply.started":"2022-02-03T14:08:39.137342Z","shell.execute_reply":"2022-02-03T14:08:40.033385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import clear_output\n\nplt.figure(figsize=(16,8))\nfor i in train_metadata[train_metadata.annotations!=\"[]\"].index:\n    img = load_image_with_annotations(\n                            train_metadata.video_id.iloc[i],\n                            train_metadata.video_frame.iloc[i],\n                            train_metadata.annotations.iloc[i],\n                            )\n    plot_image(img)\n    clear_output(wait=True)","metadata":{"execution":{"iopub.status.busy":"2022-02-03T14:18:51.349608Z","iopub.execute_input":"2022-02-03T14:18:51.349956Z","iopub.status.idle":"2022-02-03T14:19:19.285345Z","shell.execute_reply.started":"2022-02-03T14:18:51.349916Z","shell.execute_reply":"2022-02-03T14:19:19.283451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shapes = []\n\nfor i in train_metadata.index:\n    img = load_image(train_metadata.video_id.iloc[i], train_metadata.video_frame.iloc[i])\n    shapes.append(img.shape)\n    \nprint(set(shapes))","metadata":{"execution":{"iopub.status.busy":"2022-02-03T14:11:49.349657Z","iopub.execute_input":"2022-02-03T14:11:49.349961Z","iopub.status.idle":"2022-02-03T14:17:52.762008Z","shell.execute_reply.started":"2022-02-03T14:11:49.349931Z","shell.execute_reply":"2022-02-03T14:17:52.760408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Section 2: Simple model","metadata":{}},{"cell_type":"code","source":"import torchvision\nfrom torchvision.models.detection.faster_rcnn import FastRCNNPredictor\n\n# load a model pre-trained on COCO\nmodel = torchvision.models.detection.fasterrcnn_resnet50_fpn(pretrained=True)","metadata":{"execution":{"iopub.status.busy":"2022-02-03T14:34:37.455040Z","iopub.execute_input":"2022-02-03T14:34:37.455367Z","iopub.status.idle":"2022-02-03T14:34:47.309417Z","shell.execute_reply.started":"2022-02-03T14:34:37.455330Z","shell.execute_reply":"2022-02-03T14:34:47.308784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model","metadata":{"execution":{"iopub.status.busy":"2022-02-03T14:34:51.660162Z","iopub.execute_input":"2022-02-03T14:34:51.660570Z","iopub.status.idle":"2022-02-03T14:34:51.672673Z","shell.execute_reply.started":"2022-02-03T14:34:51.660539Z","shell.execute_reply":"2022-02-03T14:34:51.671805Z"},"jupyter":{"source_hidden":true,"outputs_hidden":true},"collapsed":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# replace the classifier with a new one, that has\n# num_classes which is user-defined\nnum_classes = 2  # 1 class (starfish) + background\n# get number of input features for the classifier\nin_features = model.roi_heads.box_predictor.cls_score.in_features\n# replace the pre-trained head with a new one\nmodel.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)","metadata":{"execution":{"iopub.status.busy":"2022-02-03T14:36:26.081994Z","iopub.execute_input":"2022-02-03T14:36:26.082397Z","iopub.status.idle":"2022-02-03T14:36:26.086623Z","shell.execute_reply.started":"2022-02-03T14:36:26.082366Z","shell.execute_reply":"2022-02-03T14:36:26.085991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import Dataset\nfrom PIL import Image\nimport numpy as np\nimport torch\nimport os\n\nclass StarfishDataset(torch.utils.data.Dataset):\n    def __init__(self, transforms=None):\n        self.transforms = transforms\n        # load all image files, sorting them to\n        # ensure that they are aligned\n        self.imgs = list(sorted(os.listdir(\"../input/tensorflow-great-barrier-reef/train_images/video_0\")))\n        #self.masks = list(sorted(os.listdir(os.path.join(root, \"PedMasks\"))))\n\n    def __getitem__(self, idx):\n        # load images and masks\n        img_path = os.path.join(\"../input/tensorflow-great-barrier-reef/train_images/video_0\", self.imgs[idx])\n        annotation = parse_annotations(train_metadata[\"annotations\"].iloc[idx])\n        print(annotation)\n        img = Image.open(img_path).convert(\"RGB\")\n        #img = torch.as_tensor(img, dtype=torch.float32)\n        # get bounding box coordinates for each mask\n        num_objs = len(annotation)\n        boxes = []\n        for annotations in annotation:\n            boxes.append((annotations[\"x\"], annotations[\"y\"], annotations[\"x\"]+annotations[\"width\"], annotations[\"y\"]+annotations[\"height\"]))\n\n        # convert everything into a torch.Tensor\n        if len(annotation) == 0:\n            boxes = [(1, 1, 2, 2)]\n            labels = torch.zeros((1,), dtype=torch.int64)\n        else:\n            labels = torch.ones((num_objs,), dtype=torch.int64)\n        \n        boxes = torch.as_tensor(boxes, dtype=torch.float32)\n        boxes = boxes.squeeze()\n        labels = labels.squeeze()\n        # there is only one class\n\n        image_id = torch.tensor([idx])\n        #area = (boxes[:, 3] - boxes[:, 1]) * (boxes[:, 2] - boxes[:, 0])\n\n\n        target = {}\n        target[\"boxes\"] = boxes\n        target[\"labels\"] = labels\n        #target[\"masks\"] = masks\n        target[\"image_id\"] = image_id\n        #target[\"area\"] = area\n\n        if self.transforms is not None:\n            img = self.transforms(img)\n        print(img)\n        print(target)\n        return img, target\n\n    def __len__(self):\n        return len(self.imgs)\n    \n","metadata":{"execution":{"iopub.status.busy":"2022-02-03T15:38:06.695253Z","iopub.execute_input":"2022-02-03T15:38:06.695562Z","iopub.status.idle":"2022-02-03T15:38:06.710056Z","shell.execute_reply.started":"2022-02-03T15:38:06.695527Z","shell.execute_reply":"2022-02-03T15:38:06.709172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torchvision.transforms as T\n\ndef get_transform(train):\n    transforms = []\n    transforms.append(T.ToTensor())\n#     if train:\n#         transforms.append(T.RandomHorizontalFlip(0.5))\n    return T.Compose(transforms)","metadata":{"execution":{"iopub.status.busy":"2022-02-03T15:38:07.011064Z","iopub.execute_input":"2022-02-03T15:38:07.012173Z","iopub.status.idle":"2022-02-03T15:38:07.017506Z","shell.execute_reply.started":"2022-02-03T15:38:07.012117Z","shell.execute_reply":"2022-02-03T15:38:07.016679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = StarfishDataset(transforms=get_transform(train=True))\ndata_loader = torch.utils.data.DataLoader(\n dataset, batch_size=1, shuffle=True, num_workers=0)\n\n# For Training\nimages,targets = next(iter(data_loader))\nimages = list(image for image in images)\n\ntargets = [{k: v for k, v in targets.items()}]\noutput = model(images,targets)   # Returns losses and detections\n# For inference\nmodel.eval()\nx = [torch.rand(3, 300, 400), torch.rand(3, 500, 400)]\npredictions = model(x)           # Returns predictions","metadata":{"execution":{"iopub.status.busy":"2022-02-03T15:38:07.326439Z","iopub.execute_input":"2022-02-03T15:38:07.326927Z","iopub.status.idle":"2022-02-03T15:38:27.520135Z","shell.execute_reply.started":"2022-02-03T15:38:07.326874Z","shell.execute_reply":"2022-02-03T15:38:27.516647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"metadata","metadata":{}},{"cell_type":"code","source":"predictions","metadata":{"execution":{"iopub.status.busy":"2022-02-03T15:38:36.848180Z","iopub.execute_input":"2022-02-03T15:38:36.849052Z","iopub.status.idle":"2022-02-03T15:38:36.867629Z","shell.execute_reply.started":"2022-02-03T15:38:36.848998Z","shell.execute_reply":"2022-02-03T15:38:36.866698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls ../input/tensorflow-great-barrier-reef/train_images/vide","metadata":{"execution":{"iopub.status.busy":"2022-02-03T15:11:30.350490Z","iopub.execute_input":"2022-02-03T15:11:30.350848Z","iopub.status.idle":"2022-02-03T15:11:31.164863Z","shell.execute_reply.started":"2022-02-03T15:11:30.350807Z","shell.execute_reply":"2022-02-03T15:11:31.163930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_with_annotations = train_metadata[train_metadata.annotations!=\"[]\"]","metadata":{"execution":{"iopub.status.busy":"2022-02-03T14:43:05.081403Z","iopub.execute_input":"2022-02-03T14:43:05.081978Z","iopub.status.idle":"2022-02-03T14:43:05.089619Z","shell.execute_reply.started":"2022-02-03T14:43:05.081931Z","shell.execute_reply":"2022-02-03T14:43:05.088804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\ndata = []\nall_labels = []\nall_bboxes = []\nimage_id = []\n# loop over the rows\nfor idx, row in train_with_annotations.iterrows():\n    bboxes = []\n    labels = []\n\n    image = load_image(row[\"video_id\"], row[\"video_frame\"])\n    (h, w) = image.shape[:2]\n    # scale the bounding box coordinates relative to the spatial\n    # dimensions of the input image\n#     startX = float(startX) / w\n#     startY = float(startY) / h\n#     endX = float(endX) / w\n#     endY = float(endY) / h\n    # load the image and preprocess it\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    image = cv2.resize(image, (224, 224))\n    # update our list of data, class labels, bounding boxes, and\n    # image paths\n    data.append(image)\n    annotations = parse_annotations(row[\"annotations\"]) \n\n    for annotation in annotations:\n        labels.append(1)\n        bboxes.append((annotations[\"x\"], annotations[\"y\"], annotations[\"x\"]+annotations[\"width\"], annotations[\"y\"]+annotations[\"height\"]))\n    \n    image_id.append(row[\"image_id\"])\n    all_bboxes.append(bboxes)\n    all_labels.append(labels)\n    imagePaths.append(imagePath)\n    break","metadata":{"execution":{"iopub.status.busy":"2022-02-03T14:52:48.660906Z","iopub.execute_input":"2022-02-03T14:52:48.661203Z","iopub.status.idle":"2022-02-03T14:52:48.711384Z","shell.execute_reply.started":"2022-02-03T14:52:48.661164Z","shell.execute_reply":"2022-02-03T14:52:48.710461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = np.array(data, dtype=\"float32\")\nlabels = np.array(labels)\nbboxes = np.array(bboxes, dtype=\"float32\")\nimagePaths = np.array(imagePaths)\n\n(trainImages, testImages) = torch.tensor(trainImages), torch.tensor(testImages)\n\n\n# create data loaders\ntrainDS = CustomTensorDataset((trainImages, trainLabels, trainBBoxes))\nbatch_size = 8\n\n","metadata":{"execution":{"iopub.status.busy":"2022-02-03T14:48:09.431664Z","iopub.execute_input":"2022-02-03T14:48:09.432268Z","iopub.status.idle":"2022-02-03T14:48:09.450015Z","shell.execute_reply.started":"2022-02-03T14:48:09.432227Z","shell.execute_reply":"2022-02-03T14:48:09.449171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Section 3: Submit Test predictions","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}