{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This is the second notebook on the TensorFlow Great Barrier Reef Competition. The first one can be found here: https://www.kaggle.com/scr0ll0/great-barrier-reef-eda-animation","metadata":{}},{"cell_type":"markdown","source":"# Part 2. Modeling","metadata":{}},{"cell_type":"markdown","source":"Our goal this time is to create a baseline model. To do this, we'll use a FasterRCNN model for training and StratifiedKFolds for validation.","metadata":{}},{"cell_type":"code","source":"import cv2\nimport time\nimport numpy as np\nimport pandas as pd\nfrom matplotlib import pyplot as plt\nimport albumentations as A\nfrom albumentations.pytorch.transforms import ToTensorV2\nimport torch\nimport torchvision\nfrom torch.utils.data import DataLoader, Dataset\nfrom torchvision.models.detection.faster_rcnn import FastRCNNPredictor\nfrom torchvision.models.detection import FasterRCNN\nimport ast\nfrom sklearn.model_selection import train_test_split, StratifiedKFold","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEVICE = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\nBASE_DIR = \"../input/tensorflow-great-barrier-reef/train_images/\"\nNUM_EPOCHS = 3","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/tensorflow-great-barrier-reef/train.csv')\ntest = pd.read_csv('../input/tensorflow-great-barrier-reef/test.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Set annotations to list instead of String\ntrain['annotations'] = train['annotations'].apply(lambda x: ast.literal_eval(x))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Set video path in training data.\ntrain['image_path'] = \"video_\" + train['video_id'].astype(str) + \"/\" + train['video_frame'].astype(str) + \".jpg\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Before loading up a model, we need to split the training data into training and validation data, which is not as easy as just dividing up every image into two sets because the images are tied to videos. In addition, we know that there are 20 sequences, but every sequence has vastly different lengths, making splitting by sequence not ideal also.\n\nInstead, we'll do the following:\n\n1. Define a subsequence as a set of frames that continuously have or don't have annotations.\n2. Number all of the different subsequences.\n3. Use Stratified K-Folds to split the data into training and validation sets. \n\nCredit to Julián Peller for coming up with this subsequence idea. https://www.kaggle.com/julian3833/reef-a-cv-strategy-subsequences?scriptVersionId=80623179","metadata":{}},{"cell_type":"code","source":"def subsequence(train = train):\n    start = 0\n    num_annotations = train['annotations'].str.len()\n    annotation = (num_annotations == 0) & (num_annotations.shift(1) != 0)\n    no_annotation = (num_annotations != 0) & (num_annotations.shift(1) == 0)\n    new_sequence = train['sequence'] != train['sequence'].shift(1)\n    final = train.index == len(train) - 1\n    cuts = annotation | no_annotation | new_sequence | final\n    for sub_id, end in enumerate(cuts[cuts == True].index):\n        train.loc[start:end, 'subsequence_number'] = sub_id\n        start = end\n    train['subsequence_number'] = train['subsequence_number'].astype(int)\n    train['has_annotations'] = train['annotations'].str.len() > 0\n    return train","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = subsequence(train)\ntrain.tail()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_split  = train.groupby(\"subsequence_number\").agg({'has_annotations': 'max', 'video_frame': 'count'}).astype(int).reset_index()\ntrain_split","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kf = StratifiedKFold(n_splits=10, shuffle=True)\nfor fold, (train_idx, val_idx) in enumerate(kf.split(train_split['subsequence_number'], train_split['has_annotations'])):\n    train.loc[train['subsequence_number'].isin(train_split['subsequence_number'].iloc[val_idx]), 'fold'] = fold\ntrain['fold'] = train['fold'].astype(int)\ntrain.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"All good. Now to deal with training.","metadata":{}},{"cell_type":"markdown","source":"Credit to Julián Peller again for most of the code that is to follow; this code uses the FasterRCNN Model to train and evaluate the data; I will mainly be trying to adapt this code for K-Folding. Note that this code will not work on more recent environments as it will treat certain tensors as Doubles when Floats are expected. https://www.kaggle.com/julian3833/reef-starter-torch-fasterrcnn-train-lb-0-416/notebook","metadata":{}},{"cell_type":"code","source":"class ReefDataset:\n    def __init__(self, df, transforms=None):\n        self.df = df\n        self.transforms = transforms\n    def can_augment(self, boxes):\n        box_outside_image = ((boxes[:, 0] < 0).any() or (boxes[:, 1] < 0).any() or (boxes[:, 2] > 1280).any() or (boxes[:, 3] > 720).any())\n        return not box_outside_image\n    def get_boxes(self, row):\n        boxes = pd.DataFrame(row['annotations'], columns=['x', 'y', 'width', 'height']).astype(float).values\n        boxes[:, 2] = boxes[:, 0] + boxes[:, 2]\n        boxes[:, 3] = boxes[:, 1] + boxes[:, 3]\n        return boxes\n    def get_image(self, row):\n        image = cv2.imread(f'{BASE_DIR}/{row[\"image_path\"]}', cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n        image /= 255.0\n        return image\n    def __getitem__(self, i):\n        row = self.df.iloc[i]\n        image = self.get_image(row)\n        boxes = self.get_boxes(row)\n        n_boxes = boxes.shape[0]\n        area = (boxes[:, 3] - boxes[:, 1]) * (boxes[:, 2] - boxes[:, 0])\n        target = {\n            'boxes': torch.as_tensor(boxes, dtype=torch.float32),\n            'area': torch.as_tensor(area, dtype=torch.float32),\n            'image_id': torch.tensor([i]),\n            'labels': torch.ones((n_boxes,), dtype=torch.int64),\n            'iscrowd': torch.zeros((n_boxes,), dtype=torch.int64)            \n        }\n        if self.transforms and self.can_augment(boxes):\n            sample = {\n                'image': image,\n                'bboxes': target['boxes'],\n                'labels': target['labels']\n            }\n            sample = self.transforms(**sample)\n            image = sample['image']\n            if n_boxes > 0:\n                target['boxes'] = torch.stack(tuple(map(torch.tensor, zip(*sample['bboxes'])))).permute(1, 0)\n        else:\n            image = ToTensorV2(p=1.0)(image=image)['image']\n        return image, target\n    def __len__(self):\n        return len(self.df)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_train_transform():\n    return A.Compose([\n        A.Flip(0.5),\n        ToTensorV2(p=1.0)\n    ], bbox_params={'format': 'pascal_voc', 'label_fields': ['labels']})\n\n\ndef get_valid_transform():\n    return A.Compose([\n        ToTensorV2(p=1.0)\n    ], bbox_params={'format': 'pascal_voc', 'label_fields': ['labels']})","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def collate_fn(batch):\n    return tuple(zip(*batch))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model():\n    model = torchvision.models.detection.fasterrcnn_resnet50_fpn(pretrained=True)\n    model.roi_heads.box_predictor = FastRCNNPredictor(model.roi_heads.box_predictor.cls_score.in_features, 2)\n    model.to(DEVICE)\n    return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = get_model()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(fold):\n    params = [p for p in model.parameters() if p.requires_grad]\n    optimizer = torch.optim.SGD(params, lr=0.0025, momentum=0.9, weight_decay=0.0005)\n    # lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=3, gamma=0.1)\n    lr_scheduler = None\n    n_batches, n_batches_val = len(dl_train), len(dl_val)\n    validation_losses = []\n    for epoch in range(NUM_EPOCHS):\n        time_start = time.time()\n        loss_accum = 0\n        for batch_idx, (images, targets) in enumerate(dl_train, 1):\n            images = list(image.to(DEVICE) for image in images)\n            targets = [{k: v.to(DEVICE) for k, v in t.items()} for t in targets]\n            loss_dict = model(images, targets)\n            losses = sum(loss for loss in loss_dict.values())\n            loss_value = losses.item()\n            loss_accum += loss_value\n            optimizer.zero_grad()\n            losses.backward()\n            optimizer.step()\n        if lr_scheduler is not None:\n            lr_scheduler.step()\n        val_loss_accum = 0\n        with torch.no_grad():\n            for batch_idx, (images, targets) in enumerate(dl_val, 1):\n                images = list(image.to(DEVICE) for image in images)\n                targets = [{k: v.to(DEVICE) for k, v in t.items()} for t in targets]\n                val_loss_dict = model(images, targets)\n                val_batch_loss = sum(loss for loss in val_loss_dict.values())\n                val_loss_accum += val_batch_loss.item()\n        val_loss = val_loss_accum / n_batches_val\n        train_loss = loss_accum / n_batches\n        validation_losses.append(val_loss)\n        chk_name = f'fasterrcnn_resnet50_fpn-e{epoch}-fold{fold}.bin'\n        torch.save(model.state_dict(), chk_name)\n        elapsed = time.time() - time_start\n        print(f\"[Epoch {epoch+1:2d} / {NUM_EPOCHS:2d}] Train loss: {train_loss:.3f}. Val loss: {val_loss:.3f} --> {chk_name}  [{elapsed:.0f} secs]\")   ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(10):\n    train_split = train[train['fold'] != i]\n    val_split = train[train['fold'] == i]\n    train_split = train_split[train_split.annotations.str.len() > 0 ].reset_index(drop=True)\n    val_split = val_split[val_split.annotations.str.len() > 0 ].reset_index(drop=True)\n    ds_train = ReefDataset(train_split, get_train_transform())\n    ds_val = ReefDataset(val_split, get_valid_transform())\n    dl_train = DataLoader(ds_train, batch_size=8, shuffle=False, num_workers=4, collate_fn=collate_fn)\n    dl_val = DataLoader(ds_val, batch_size=8, shuffle=False, num_workers=4, collate_fn=collate_fn)\n    train_model(i)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I'll be moving to a separate notebook where the predictions will be made.","metadata":{}}]}