{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# THis solution is implemented only using Pytorch, here are the imports that are necessary.\nimport os\nimport cv2\nimport time\nimport random\nimport numpy as np\nimport pandas as pd\nfrom matplotlib import pyplot as plt\nimport albumentations as A\nfrom albumentations.pytorch.transforms import ToTensorV2\nimport torch\nimport torchvision\n\nfrom torch.utils.data import DataLoader\nfrom torchvision.models.detection.faster_rcnn import FastRCNNPredictor\nfrom torchvision.models.detection import FasterRCNN\n\n\n# Fix randomness, https://pytorch.org/docs/stable/notes/randomness.html\n# we try to limit the number of sources of nondeterministic beahviour so that most operations, given the same inputs, produce the same result\n\ndef fix_all_seeds(seed):\n    np.random.seed(seed)\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    \nfix_all_seeds(42)","metadata":{"execution":{"iopub.status.busy":"2022-04-19T02:41:52.338319Z","iopub.execute_input":"2022-04-19T02:41:52.338682Z","iopub.status.idle":"2022-04-19T02:41:52.346602Z","shell.execute_reply.started":"2022-04-19T02:41:52.338651Z","shell.execute_reply":"2022-04-19T02:41:52.345962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEVICE = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n\nBASE_DIR = \"../input/tensorflow-great-barrier-reef/train_images/\"\n\n# Configuration for the Optimizer\nLEARNING_RATE = 0.0025\nMOMENTUM = 0.9\nWEIGHT_DECAY = 0.0005\n\n# Number of epochs\nNUM_EPOCHS = 12\n\nBATCH_SIZE = 8","metadata":{"execution":{"iopub.status.busy":"2022-04-18T23:34:49.708253Z","iopub.execute_input":"2022-04-18T23:34:49.708508Z","iopub.status.idle":"2022-04-18T23:34:49.773468Z","shell.execute_reply.started":"2022-04-18T23:34:49.708479Z","shell.execute_reply":"2022-04-18T23:34:49.772745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"../input/reef-cv-strategy-subsequences-dataframes/train-validation-split/train-0.1.csv\")\n\n# Turn annotations from strings into lists of dictionaries\ndf['annotations'] = df['annotations'].apply(eval)\n\n# Create the image path for the row\ndf['image_path'] = \"video_\" + df['video_id'].astype(str) + \"/\" + df['video_frame'].astype(str) + \".jpg\"\n\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-18T23:34:53.222697Z","iopub.execute_input":"2022-04-18T23:34:53.223417Z","iopub.status.idle":"2022-04-18T23:34:53.800010Z","shell.execute_reply.started":"2022-04-18T23:34:53.223359Z","shell.execute_reply":"2022-04-18T23:34:53.799353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#From the result, we can see that there are a lot of images with no annotations, of about 80% of the original images provided within the competition dataset.\n(df['annotations'].str.len() > 0).value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-04-19T02:45:22.854970Z","iopub.execute_input":"2022-04-19T02:45:22.855432Z","iopub.status.idle":"2022-04-19T02:45:22.870068Z","shell.execute_reply.started":"2022-04-19T02:45:22.855394Z","shell.execute_reply":"2022-04-19T02:45:22.869206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(df['annotations'].str.len() > 0).value_counts(normalize=True).round(2)","metadata":{"execution":{"iopub.status.busy":"2022-04-19T02:45:22.957693Z","iopub.execute_input":"2022-04-19T02:45:22.957970Z","iopub.status.idle":"2022-04-19T02:45:22.971609Z","shell.execute_reply.started":"2022-04-19T02:45:22.957940Z","shell.execute_reply":"2022-04-19T02:45:22.970820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We shall drop images with no annotation to speed up the training process\ndf = df[df['annotations'].str.len() > 0].reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-04-18T23:35:02.510766Z","iopub.execute_input":"2022-04-18T23:35:02.511446Z","iopub.status.idle":"2022-04-18T23:35:02.531684Z","shell.execute_reply.started":"2022-04-18T23:35:02.511406Z","shell.execute_reply":"2022-04-18T23:35:02.530809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#here we create the training and validation split, and show the number of images within each portion.\ndf_train = df[df['is_train']].reset_index(drop=True)\ndf_val = df[~df['is_train']].reset_index(drop=True)\n\ndf_train.shape[0], df_val.shape[0]","metadata":{"execution":{"iopub.status.busy":"2022-04-18T23:35:04.992804Z","iopub.execute_input":"2022-04-18T23:35:04.993366Z","iopub.status.idle":"2022-04-18T23:35:05.003786Z","shell.execute_reply.started":"2022-04-18T23:35:04.993327Z","shell.execute_reply":"2022-04-18T23:35:05.002881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ReefDataset:\n\n    def __init__(self, df, transforms=None):\n        self.df = df\n        self.transforms = transforms\n\n    def get_boxes(self, row):\n        # this function returns the bboxes for a given row as a 3D matrix with format [x_min, y_min, x_max, y_max]\n        \n        boxes = pd.DataFrame(row['annotations'], columns=['x', 'y', 'width', 'height']).astype(float).values\n        \n        # Transformation from [x_min, y_min, w, h] to [x_min, y_min, x_max, y_max]\n        boxes[:, 2] = boxes[:, 0] + boxes[:, 2]\n        boxes[:, 3] = boxes[:, 1] + boxes[:, 3]\n        \n        boxes[:, 2] = np.clip(boxes[:, 2], 0, 1280)\n        boxes[:, 3] = np.clip(boxes[:, 3], 0, 720)\n        \n        return boxes\n    \n    def get_image(self, row):\n        #Gets the image for a given row\n        \n        image = cv2.imread(f'{BASE_DIR}/{row[\"image_path\"]}', cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n        image /= 255.0\n        return image\n    \n    def __getitem__(self, i):\n\n        row = self.df.iloc[i]\n        image = self.get_image(row)\n        boxes = self.get_boxes(row)\n        \n        n_boxes = boxes.shape[0]\n        \n        # Calculate the area\n        area = (boxes[:, 3] - boxes[:, 1]) * (boxes[:, 2] - boxes[:, 0])\n        \n        \n        target = {\n            'boxes': torch.as_tensor(boxes, dtype=torch.float32),\n            'area': torch.as_tensor(area, dtype=torch.float32),\n            \n            'image_id': torch.tensor([i]),\n            \n            # There is only one class\n            'labels': torch.ones((n_boxes,), dtype=torch.int64),\n            \n            'iscrowd': torch.zeros((n_boxes,), dtype=torch.int64)            \n        }\n\n        \n        sample = {\n            'image': image,\n            'bboxes': target['boxes'],\n            'labels': target['labels']\n        }\n        sample = self.transforms(**sample)\n        image = sample['image']\n\n        if n_boxes > 0:\n            target['boxes'] = torch.stack(tuple(map(torch.tensor, zip(*sample['bboxes'])))).permute(1, 0)\n\n        return image, target\n\n    def __len__(self):\n        return len(self.df)","metadata":{"execution":{"iopub.status.busy":"2022-04-18T23:35:07.801829Z","iopub.execute_input":"2022-04-18T23:35:07.802363Z","iopub.status.idle":"2022-04-18T23:35:07.817748Z","shell.execute_reply.started":"2022-04-18T23:35:07.802324Z","shell.execute_reply":"2022-04-18T23:35:07.816938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# here we experiment image augmentation using simple transformations \n\ndef get_train_transform():\n    return A.Compose([\n        A.Flip(0.5),\n        ToTensorV2(p=1.0)\n    ], bbox_params={'format': 'pascal_voc', 'label_fields': ['labels']})\n\n\ndef get_valid_transform():\n    return A.Compose([\n        ToTensorV2(p=1.0)\n    ], bbox_params={'format': 'pascal_voc', 'label_fields': ['labels']})","metadata":{"execution":{"iopub.status.busy":"2022-04-19T02:48:55.745671Z","iopub.execute_input":"2022-04-19T02:48:55.745977Z","iopub.status.idle":"2022-04-19T02:48:55.751582Z","shell.execute_reply.started":"2022-04-19T02:48:55.745949Z","shell.execute_reply":"2022-04-19T02:48:55.750687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# defining the data sets to be used after augmentation\nds_train = ReefDataset(df_train, get_train_transform())\nds_val = ReefDataset(df_val, get_valid_transform())","metadata":{"execution":{"iopub.status.busy":"2022-04-18T23:35:15.126455Z","iopub.execute_input":"2022-04-18T23:35:15.127086Z","iopub.status.idle":"2022-04-18T23:35:15.130796Z","shell.execute_reply.started":"2022-04-18T23:35:15.127048Z","shell.execute_reply":"2022-04-18T23:35:15.130126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# testing whether the augmentations work on an image and checking output\ntry:\n    idx = df_train[df_train.annotations.str.len() > 12].iloc[0].name\nexcept:\n    idx = 0\n    \n\nimage, targets = ds_train[idx]\n\n\nboxes = targets['boxes'].cpu().numpy().astype(np.int32)\nimg = image.permute(1,2,0).cpu().numpy()\nfig, ax = plt.subplots(1, 1, figsize=(16, 8))\n\nfor box in boxes:\n    cv2.rectangle(img,\n                  (box[0], box[1]),\n                  (box[2], box[3]),\n                  (220, 0, 0), 3)\n    \nax.set_axis_off()\nax.imshow(img);","metadata":{"execution":{"iopub.status.busy":"2022-04-18T23:35:17.669590Z","iopub.execute_input":"2022-04-18T23:35:17.670021Z","iopub.status.idle":"2022-04-18T23:35:18.417723Z","shell.execute_reply.started":"2022-04-18T23:35:17.669982Z","shell.execute_reply":"2022-04-18T23:35:18.417095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#load the data for the model\ndef collate_fn(batch):\n    return tuple(zip(*batch))\n\ndl_train = DataLoader(ds_train, batch_size=BATCH_SIZE, shuffle=False, num_workers=4, collate_fn=collate_fn)\ndl_val = DataLoader(ds_val, batch_size=BATCH_SIZE, shuffle=False, num_workers=4, collate_fn=collate_fn)","metadata":{"execution":{"iopub.status.busy":"2022-04-18T23:35:22.417187Z","iopub.execute_input":"2022-04-18T23:35:22.417433Z","iopub.status.idle":"2022-04-18T23:35:22.426598Z","shell.execute_reply.started":"2022-04-18T23:35:22.417404Z","shell.execute_reply":"2022-04-18T23:35:22.425834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model():\n    # load a model that has been pre-trained from  pytorch libraries\n    model = torchvision.models.detection.fasterrcnn_resnet50_fpn(pretrained=True)\n\n    num_classes = 2  # 1st class is that of the starfish while the second is that of the background\n\n    # get number of input features for the classifier\n    in_features = model.roi_heads.box_predictor.cls_score.in_features\n\n    # replace the pre-trained head with a new one\n    model.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)\n\n    model.to(DEVICE)\n    return model\n\nmodel = get_model()","metadata":{"execution":{"iopub.status.busy":"2022-04-18T23:35:35.615659Z","iopub.execute_input":"2022-04-18T23:35:35.615933Z","iopub.status.idle":"2022-04-18T23:35:44.999296Z","shell.execute_reply.started":"2022-04-18T23:35:35.615883Z","shell.execute_reply":"2022-04-18T23:35:44.998466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = [p for p in model.parameters() if p.requires_grad]\noptimizer = torch.optim.SGD(params, lr=LEARNING_RATE, momentum=MOMENTUM, weight_decay=WEIGHT_DECAY)\n\nn_batches, n_batches_val = len(dl_train), len(dl_val)\nval_losses = []\nval_box_losses = []\n\nfor epoch in range(NUM_EPOCHS):\n    \n    model.train()\n    \n    time_start = time.time()\n    loss_accum = 0\n    loss_box_accum = 0\n    \n    for batch_idx, (images, targets) in enumerate(dl_train, 1):\n        images = list(image.float().to(DEVICE) for image in images)\n        targets = [{k: v.to(torch.float32).to(DEVICE) if \"box\" in k else v.to(DEVICE) for k, v in t.items()} for t in targets]\n\n        # This dict has the following keys:\n        #    loss_classifier, loss_box_reg, loss_objectness, loss_rpn_box_reg\n        loss_dict = model(images, targets)\n        \n        losses = sum(loss for loss in loss_dict.values())\n        loss_value = losses.item()\n\n        loss_accum += loss_value\n        loss_box_accum += loss_dict['loss_box_reg'].item()\n        \n        optimizer.zero_grad()\n        losses.backward()\n        optimizer.step()\n\n        \n    # Validation step!\n    val_loss_accum = 0\n    val_loss_box_accum = 0\n    \n    with torch.no_grad():\n        for batch_idx, (images, targets) in enumerate(dl_val, 1):\n            images = list(image.float().to(DEVICE) for image in images)\n            targets = [{k: v.to(torch.float32).to(DEVICE) if \"box\" in k else v.to(DEVICE) for k, v in t.items()} for t in targets]\n            \n            val_loss_dict = model(images, targets)\n            val_batch_loss = sum(loss for loss in val_loss_dict.values())\n            \n            val_loss_accum += val_batch_loss.item()\n            val_loss_box_accum += val_loss_dict['loss_box_reg'].item()\n\n    \n    # Calculate epoch losses\n    val_loss = val_loss_accum / n_batches_val\n    val_loss_box = val_loss_box_accum / n_batches_val\n    \n    train_loss = loss_accum / n_batches\n    train_loss_box = loss_box_accum / n_batches\n    \n    val_losses.append(val_loss)\n    val_box_losses.append(val_loss_box)\n    \n    # Save model\n    chk_name = f'pytorch_model-e{epoch}.bin'\n    torch.save(model.state_dict(), chk_name)\n    \n    \n    # Logging\n    elapsed = time.time() - time_start\n    \n    prefix = f\"[Epoch {epoch+1:2d} / {NUM_EPOCHS:2d}]\"\n    print()\n    print(f\"{prefix} Train loss: {train_loss:.3f}.  Train loss (bbox only): {train_loss_box:.3f}.  Val loss (bbox only): {val_loss_box:.3f}\")   \n    print(prefix)\n    print(f\"{prefix} Saved to  : {chk_name}  [{elapsed:.0f} secs]\")\n    print(f\"{prefix} Val loss  : {val_loss:.3f}\")","metadata":{"execution":{"iopub.status.busy":"2022-04-18T23:35:46.592720Z","iopub.execute_input":"2022-04-18T23:35:46.593083Z","iopub.status.idle":"2022-04-19T02:03:47.372529Z","shell.execute_reply.started":"2022-04-18T23:35:46.593048Z","shell.execute_reply":"2022-04-19T02:03:47.371657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Best model based on lowest validation loss\nnp.argmin(val_losses)","metadata":{"execution":{"iopub.status.busy":"2022-04-19T02:03:59.758009Z","iopub.execute_input":"2022-04-19T02:03:59.758309Z","iopub.status.idle":"2022-04-19T02:03:59.765188Z","shell.execute_reply.started":"2022-04-19T02:03:59.758277Z","shell.execute_reply":"2022-04-19T02:03:59.764237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Best model based on lowest bbox loss\nnp.argmin(val_box_losses)","metadata":{"execution":{"iopub.status.busy":"2022-04-19T02:04:02.262705Z","iopub.execute_input":"2022-04-19T02:04:02.262975Z","iopub.status.idle":"2022-04-19T02:04:02.268253Z","shell.execute_reply.started":"2022-04-19T02:04:02.262943Z","shell.execute_reply":"2022-04-19T02:04:02.267305Z"},"trusted":true},"execution_count":null,"outputs":[]}]}