{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## RetinaNet in Pytorch - Tensorflow Great Barrier Reef\n\nI wrote this notebook using following two notebooks\n\n1. https://www.kaggle.com/jainamshah17/gwd-retinanet-pytorch-train\n2. https://www.kaggle.com/julian3833/reef-starter-torch-fasterrcnn-train-lb-0-416/notebook\n","metadata":{}},{"cell_type":"code","source":"!git clone https://github.com/yhenon/pytorch-retinanet.git\n!cp -r /kaggle/working/pytorch-retinanet/retinanet ./\n!pip install -q pycocotools","metadata":{"execution":{"iopub.status.busy":"2022-01-24T18:29:25.307406Z","iopub.execute_input":"2022-01-24T18:29:25.307852Z","iopub.status.idle":"2022-01-24T18:30:03.873706Z","shell.execute_reply.started":"2022-01-24T18:29:25.307744Z","shell.execute_reply":"2022-01-24T18:30:03.872573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport re\nimport cv2\nimport time\nimport numpy as np\nimport pandas as pd\n\nimport albumentations as A\nfrom albumentations.pytorch.transforms import ToTensorV2\n\nimport torch\nimport torch.optim as optim\nimport torchvision.transforms as T\nfrom torchvision.utils import make_grid \nfrom torch.utils.data import DataLoader, Dataset\n\nfrom retinanet import model\nfrom retinanet.dataloader import collater, Resizer, Augmenter, Normalizer, UnNormalizer\n\nimport seaborn as sns\nfrom matplotlib import pyplot as plt\n%matplotlib inline\n\nBASE_DIR = \"../input/tensorflow-great-barrier-reef/train_images\"\nDEVICE = torch.device('cuda' if torch.cuda.is_available() else torch.device('cpu'))\nNUM_EPOCHS = 5","metadata":{"execution":{"iopub.status.busy":"2022-01-24T18:30:03.875753Z","iopub.execute_input":"2022-01-24T18:30:03.876119Z","iopub.status.idle":"2022-01-24T18:30:08.265077Z","shell.execute_reply.started":"2022-01-24T18:30:03.876084Z","shell.execute_reply":"2022-01-24T18:30:08.264097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(r'../input/reef-a-cv-strategy-subsequences/train-validation-split/train-0.1.csv')\n\ndisplay(df)\n\n# Turn annotations from strings into lists of dictionaries\ndf['annotations'] = df['annotations'].apply(eval)\n# Create the image path for the row\ndf['image_path'] = \"video_\" + df['video_id'].astype(str) + \"/\" + df['video_frame'].astype(str) + \".jpg\"\n\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-24T18:30:08.266567Z","iopub.execute_input":"2022-01-24T18:30:08.266933Z","iopub.status.idle":"2022-01-24T18:30:08.747146Z","shell.execute_reply.started":"2022-01-24T18:30:08.266891Z","shell.execute_reply":"2022-01-24T18:30:08.745454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train, df_val = df[df['is_train']], df[~df['is_train']]","metadata":{"execution":{"iopub.status.busy":"2022-01-24T18:30:08.749157Z","iopub.execute_input":"2022-01-24T18:30:08.749397Z","iopub.status.idle":"2022-01-24T18:30:08.757439Z","shell.execute_reply.started":"2022-01-24T18:30:08.74937Z","shell.execute_reply":"2022-01-24T18:30:08.756376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Removing the instances with no target\ndf_train = df_train[df_train.annotations.str.len() > 0 ].reset_index(drop=True)\ndf_val = df_val[df_val.annotations.str.len() > 0 ].reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-01-24T18:30:08.758734Z","iopub.execute_input":"2022-01-24T18:30:08.759458Z","iopub.status.idle":"2022-01-24T18:30:08.789283Z","shell.execute_reply.started":"2022-01-24T18:30:08.759411Z","shell.execute_reply":"2022-01-24T18:30:08.788627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# remove later\ndf_train.shape[0], df_val.shape[0]","metadata":{"execution":{"iopub.status.busy":"2022-01-24T18:30:08.790577Z","iopub.execute_input":"2022-01-24T18:30:08.791058Z","iopub.status.idle":"2022-01-24T18:30:08.797954Z","shell.execute_reply.started":"2022-01-24T18:30:08.791024Z","shell.execute_reply":"2022-01-24T18:30:08.797113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ReefDataset:\n\n    def __init__(self, df, transforms=None):\n        self.df = df\n        self.transforms = transforms\n\n    def get_boxes(self, row):\n        \"\"\"Returns the bboxes for a given row as a 3D matrix with format [x_min, y_min, x_max, y_max]\"\"\"\n        \n        records = pd.DataFrame(row['annotations'])\n        boxes = np.zeros((records.shape[0], 5))\n        \n        boxes[:, 0:4] = records[['x', 'y', 'width', 'height']].values\n        # Change from [x_min, y_min, w, h] to [x_min, y_min, x_max, y_max]\n        boxes[:, 2] = boxes[:, 0] + boxes[:, 2]\n        boxes[:, 3] = boxes[:, 1] + boxes[:, 3]\n        \n        # to correct out of box annotations\n        boxes[:, 0] = np.maximum(0, boxes[:, 0])\n        boxes[:, 1] = np.maximum(0, boxes[:, 1])\n        boxes[:, 2] = np.minimum(1280, boxes[:, 2])\n        boxes[:, 3] = np.minimum(720, boxes[:, 3])\n        return boxes\n    \n    def get_image(self, row):\n        \"\"\"Gets the image for a given row\"\"\"\n        \n        image = cv2.imread(f'{BASE_DIR}/{row[\"image_path\"]}', cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n        image /= 255.0\n        return image\n    \n    def __getitem__(self, i):\n\n        row = self.df.iloc[i]\n        image = self.get_image(row)\n        boxes = self.get_boxes(row)\n        \n        n_boxes = boxes.shape[0]\n        \n        # Calculate the area\n        area = (boxes[:, 3] - boxes[:, 1]) * (boxes[:, 2] - boxes[:, 0])\n        \n        sample = {\n                'img': image,\n                'annot': boxes,\n            }\n    \n        if self.transforms :\n            sample = self.transforms(sample)\n        \n\n        return sample\n\n    def __len__(self):\n        return len(self.df)","metadata":{"execution":{"iopub.status.busy":"2022-01-24T18:42:59.765763Z","iopub.execute_input":"2022-01-24T18:42:59.766768Z","iopub.status.idle":"2022-01-24T18:42:59.787883Z","shell.execute_reply.started":"2022-01-24T18:42:59.766716Z","shell.execute_reply":"2022-01-24T18:42:59.787178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_train_transform():\n    return T.Compose([Augmenter(), Normalizer(), Resizer()])\n\ndef get_valid_transform():\n    return T.Compose([Normalizer(), Resizer()])","metadata":{"execution":{"iopub.status.busy":"2022-01-24T18:43:00.350698Z","iopub.execute_input":"2022-01-24T18:43:00.351079Z","iopub.status.idle":"2022-01-24T18:43:00.357323Z","shell.execute_reply.started":"2022-01-24T18:43:00.351038Z","shell.execute_reply":"2022-01-24T18:43:00.35638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define datasets\nds_train = ReefDataset(df_train, get_train_transform())\nds_val = ReefDataset(df_val, get_valid_transform())","metadata":{"execution":{"iopub.status.busy":"2022-01-24T18:43:01.153956Z","iopub.execute_input":"2022-01-24T18:43:01.155079Z","iopub.status.idle":"2022-01-24T18:43:01.160026Z","shell.execute_reply.started":"2022-01-24T18:43:01.155023Z","shell.execute_reply":"2022-01-24T18:43:01.159148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define dataloaders\ndl_train = DataLoader(ds_train, batch_size=8, shuffle=False, num_workers=4, collate_fn=collater)\ndl_val = DataLoader(ds_val, batch_size=8, shuffle=False, num_workers=4, collate_fn=collater)","metadata":{"execution":{"iopub.status.busy":"2022-01-24T18:43:01.809355Z","iopub.execute_input":"2022-01-24T18:43:01.809954Z","iopub.status.idle":"2022-01-24T18:43:01.817249Z","shell.execute_reply.started":"2022-01-24T18:43:01.809901Z","shell.execute_reply":"2022-01-24T18:43:01.816031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Set Up for training","metadata":{}},{"cell_type":"code","source":"retinanet = model.resnet50(num_classes = 2, pretrained = True)\nretinanet.to(DEVICE)\noptimizer = torch.optim.Adam(retinanet.parameters(), lr = 0.0001)","metadata":{"execution":{"iopub.status.busy":"2022-01-24T18:30:45.853088Z","iopub.execute_input":"2022-01-24T18:30:45.853533Z","iopub.status.idle":"2022-01-24T18:30:47.560546Z","shell.execute_reply.started":"2022-01-24T18:30:45.853485Z","shell.execute_reply":"2022-01-24T18:30:47.559905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm.notebook import tqdm\n\ndef train_one_epoch(epoch_num, train_data_loader):\n    retinanet.train()\n\n    epoch_loss = []\n\n    for iter_num, data in tqdm(enumerate(train_data_loader)):\n                \n        # Reseting gradients after each iter\n        optimizer.zero_grad()\n            \n        # Forward\n        classification_loss, regression_loss = retinanet([data['img'].to(DEVICE).float(), data['annot'].to(DEVICE).float()])\n                \n        # Calculating Loss\n        classification_loss = classification_loss.mean()\n        regression_loss = regression_loss.mean()\n\n        loss = classification_loss + regression_loss\n\n        if bool(loss == 0):\n            continue\n                \n        # Calculating Gradients\n        loss.backward()\n\n        # Gradient Clipping\n        torch.nn.utils.clip_grad_norm_(retinanet.parameters(), 0.1)\n           \n        # Updating Weights\n        optimizer.step()\n\n        epoch_loss.append(float(loss))\n\n            \n        print(\n            'Epoch: {} | Iteration: {} | Classification loss: {:1.5f} | Regression loss: {:1.5f} | Running loss: {:1.5f}'.format(\n                epoch_num, iter_num, float(classification_loss), float(regression_loss), np.mean(epoch_loss)))\n\n        del classification_loss\n        del regression_loss","metadata":{"execution":{"iopub.status.busy":"2022-01-24T18:30:47.561736Z","iopub.execute_input":"2022-01-24T18:30:47.562243Z","iopub.status.idle":"2022-01-24T18:30:47.571777Z","shell.execute_reply.started":"2022-01-24T18:30:47.562208Z","shell.execute_reply":"2022-01-24T18:30:47.570705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def valid_one_epoch(epoch_num, valid_data_loader):\n    \n    epoch_loss = []\n\n    for iter_num, data in tqdm(enumerate(valid_data_loader)):\n                \n        with torch.no_grad():\n            \n            # Forward\n            classification_loss, regression_loss = retinanet([data['img'].to(DEVICE).float(), data['annot'].to(DEVICE).float()])\n\n            # Calculating Loss\n            classification_loss = classification_loss.mean()\n            regression_loss = regression_loss.mean()\n            loss = classification_loss + regression_loss\n\n            #Epoch Loss\n            epoch_loss.append(float(loss))\n\n            print(\n                'Epoch: {} | Iteration: {} | Classification loss: {:1.5f} | Regression loss: {:1.5f} | Running loss: {:1.5f}'.format(\n                    epoch_num, iter_num, float(classification_loss), float(regression_loss), np.mean(epoch_loss)))\n\n            del classification_loss\n            del regression_loss\n        \n    # Save Model after each epoch\n    torch.save(retinanet, f\"retinanet_barrier_reef_epoch{epoch_num}.pt\")\n    \n  ","metadata":{"execution":{"iopub.status.busy":"2022-01-24T18:30:48.743528Z","iopub.execute_input":"2022-01-24T18:30:48.743889Z","iopub.status.idle":"2022-01-24T18:30:48.753609Z","shell.execute_reply.started":"2022-01-24T18:30:48.743851Z","shell.execute_reply":"2022-01-24T18:30:48.752668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Training Loop\nfor epoch in tqdm(range(NUM_EPOCHS)):\n    print(\"Epoch - {} Started\".format(epoch))    \n    train_one_epoch(epoch, dl_train)\n    valid_one_epoch(epoch, dl_val)","metadata":{"execution":{"iopub.status.busy":"2022-01-24T18:30:49.477577Z","iopub.execute_input":"2022-01-24T18:30:49.477943Z","iopub.status.idle":"2022-01-24T18:34:54.454681Z","shell.execute_reply.started":"2022-01-24T18:30:49.4779Z","shell.execute_reply":"2022-01-24T18:34:54.453065Z"},"trusted":true},"execution_count":null,"outputs":[]}]}