{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"#
SIIM-FISABIO-RSNA COVID-19 Detection using PyTorch
FasterRCNNDetector
","metadata":{}},{"cell_type":"markdown","source":"### Table of contents\n[1. Import libraries](#sec1)
\n[2. Datasets preparation](#sec2)
\n[3. Custom Datasets and Transforms](#sec3)
\n[4. Retrieve the Model](#sec4)
\n[5. Evaluation Metrics](#sec5)
\n[6.Training the Model](#sec6)
\n[7. Simple inference](#sec7)
\n[8. References](#sec8)
","metadata":{}},{"cell_type":"markdown","source":"# 1. Import libraries","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n# torch\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\n\n# torchvision\nimport torchvision\nfrom torchvision.models.detection.faster_rcnn import FastRCNNPredictor, FasterRCNN\nfrom torchvision.models.detection.backbone_utils import resnet_fpn_backbone\n\n# Albumenatations\nimport albumentations as A\nfrom albumentations.pytorch.transforms import ToTensorV2\n\n# sklearn\nfrom sklearn.model_selection import StratifiedKFold\n\nimport os \nfrom glob import glob\nimport numpy as np\nimport pandas as ps\nfrom datetime import datetime\nimport time\nimport random\nfrom tqdm import tqdm_notebook as tqdm # progress bar\nimport matplotlib.pyplot as plt\n\nimport cv2\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2021-09-23T16:57:21.967310Z","iopub.execute_input":"2021-09-23T16:57:21.967681Z","iopub.status.idle":"2021-09-23T16:57:21.975819Z","shell.execute_reply.started":"2021-09-23T16:57:21.967653Z","shell.execute_reply":"2021-09-23T16:57:21.974762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Datasets preparation","metadata":{}},{"cell_type":"code","source":"def get_train_file_path(image_id):\n return \"../input/siim-covid19-resized-to-256px-jpg/train/{}.jpg\".format(image_id)\ndef get_test_file_path(image_id):\n return \"../input/siim-covid19-resized-to-256px-jpg/test/{}.jpg\".format(image_id)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T16:57:21.977608Z","iopub.execute_input":"2021-09-23T16:57:21.977987Z","iopub.status.idle":"2021-09-23T16:57:21.991046Z","shell.execute_reply.started":"2021-09-23T16:57:21.977951Z","shell.execute_reply":"2021-09-23T16:57:21.990276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nupdated_train_labels = pd.read_csv('../input/siim-covid19-updated-train-labels/updated_train_labels.csv')\n\nupdated_train_labels['jpg_path'] = updated_train_labels['id'].apply(get_train_file_path)\ntrain = updated_train_labels.copy()\ntrain.head(4)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T16:57:21.992890Z","iopub.execute_input":"2021-09-23T16:57:21.993230Z","iopub.status.idle":"2021-09-23T16:57:22.078060Z","shell.execute_reply.started":"2021-09-23T16:57:21.993197Z","shell.execute_reply":"2021-09-23T16:57:22.077115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Configuration","metadata":{}},{"cell_type":"code","source":"class DefaultConfig:\n n_folds: int = 5\n #seed: int = 2021\n num_classes: int = 4 # \"negative\", \"typical\", \"indeterminate\", \"atypical\"\n img_size: int = 256\n fold_num: int = 0\n device: str = 'cuda:0'","metadata":{"execution":{"iopub.status.busy":"2021-09-23T16:57:22.079789Z","iopub.execute_input":"2021-09-23T16:57:22.080133Z","iopub.status.idle":"2021-09-23T16:57:22.084992Z","shell.execute_reply.started":"2021-09-23T16:57:22.080097Z","shell.execute_reply":"2021-09-23T16:57:22.083834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if torch.cuda.is_available():\n print('using device: cuda')\n device = torch.device(DefaultConfig.device) \nelse:\n print('We are using device: cpu')\n device = torch.device('cpu')\n ","metadata":{"execution":{"iopub.status.busy":"2021-09-23T16:57:22.086334Z","iopub.execute_input":"2021-09-23T16:57:22.086735Z","iopub.status.idle":"2021-09-23T16:57:22.096642Z","shell.execute_reply.started":"2021-09-23T16:57:22.086699Z","shell.execute_reply":"2021-09-23T16:57:22.095598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed):\n random.seed(seed)\n os.environ['PYTHONHASHSEED'] = str(seed)\n np.random.seed(seed)\n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n torch.backends.cudnn.deterministic = True\n torch.backends.cudnn.benchmark = True\n \n#seed_everything(DefaultConfig.seed)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T16:57:22.097896Z","iopub.execute_input":"2021-09-23T16:57:22.098241Z","iopub.status.idle":"2021-09-23T16:57:22.107728Z","shell.execute_reply.started":"2021-09-23T16:57:22.098206Z","shell.execute_reply":"2021-09-23T16:57:22.106510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Split train set","metadata":{}},{"cell_type":"code","source":"df_folds = train.copy()\nseed = 41\nskf = StratifiedKFold(n_splits=5, shuffle=True, random_state=41)\nfor n, (train_index, val_index) in enumerate(skf.split(X=df_folds.index, y=df_folds.integer_label)):\n df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = int(n)\ndf_folds['fold'] = df_folds['fold'].astype(int)\nprint(df_folds.groupby(['fold', df_folds.integer_label]).size())","metadata":{"execution":{"iopub.status.busy":"2021-09-23T16:57:22.109282Z","iopub.execute_input":"2021-09-23T16:57:22.109741Z","iopub.status.idle":"2021-09-23T16:57:22.139270Z","shell.execute_reply.started":"2021-09-23T16:57:22.109708Z","shell.execute_reply":"2021-09-23T16:57:22.138354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Custom Datasets and Transforms\n### Albumentations","metadata":{}},{"cell_type":"code","source":"def get_train_transforms():\n return A.Compose([\n #A.HorizontalFlip(p=0.5), \n #A.VerticalFlip(p=0.5),\n A.OneOf([\n A.HueSaturationValue(hue_shift_limit=0.1, sat_shift_limit=0.1,\n val_shift_limit=0.1, p=0.3), \n A.RandomBrightnessContrast(brightness_limit=0.2, \n contrast_limit=0.2, p=0.5),\n ], p=0.2),\n A.Resize(height=DefaultConfig.img_size, width=DefaultConfig.img_size, p=1.0),\n #A.Normalize(mean=(0, 0, 0), std=(1, 1, 1), max_pixel_value=255.0, p=1.0),\n ToTensorV2(p=1.0),\n ],\n p=1.0, bbox_params={'format': 'pascal_voc', 'label_fields': ['labels']})\n\ndef get_valid_transforms():\n return A.Compose([\n A.Resize(height=DefaultConfig.img_size, width=DefaultConfig.img_size, p=1.0),\n #A.Normalize(mean=(0, 0, 0), std=(1, 1, 1), max_pixel_value=255.0, p=1.0),\n ToTensorV2(p=1.0)\n ], bbox_params={'format': 'pascal_voc', 'label_fields': ['labels']})","metadata":{"execution":{"iopub.status.busy":"2021-09-23T16:57:22.142002Z","iopub.execute_input":"2021-09-23T16:57:22.142333Z","iopub.status.idle":"2021-09-23T16:57:22.150222Z","shell.execute_reply.started":"2021-09-23T16:57:22.142298Z","shell.execute_reply":"2021-09-23T16:57:22.149174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Custom Dataset","metadata":{}},{"cell_type":"code","source":"class CustomDataset(Dataset):\n\n def __init__(self, image_ids, df, transforms=None, test=False):\n super().__init__()\n\n self.image_ids = image_ids\n self.df = df\n self.file_names = df['jpg_path'].values\n self.transforms = transforms\n self.test = test\n\n def __getitem__(self, index: int):\n image_id = self.image_ids[index]\n \n image, boxes, labels = self.load_image_and_boxes(index)\n \n target = {}\n target['boxes'] = boxes\n target['labels'] = torch.tensor(labels)\n target['image_id'] = torch.tensor([index])\n\n if self.transforms:\n for i in range(10):\n sample = self.transforms(**{\n 'image': image,\n 'bboxes': target['boxes'],\n 'labels': labels\n })\n if len(sample['bboxes']) > 0:\n image = sample['image']\n target['boxes'] = torch.stack(tuple(map(torch.tensor, zip(*sample['bboxes'])))).permute(1, 0)\n break\n return image, target, image_id\n\n def __len__(self) -> int:\n return self.image_ids.shape[0]\n\n def load_image_and_boxes(self, index):\n image_id = self.image_ids[index]\n image = cv2.imread(self.file_names[index], cv2.IMREAD_COLOR).copy().astype(np.float32)\n image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n image /= 255.0\n records = self.df[self.df['id'] == image_id] \n boxes = []\n for bbox in records[['frac_xmin', 'frac_ymin', 'frac_xmax', 'frac_ymax']].values:\n bbox = np.clip(bbox, 0, 1.0)\n temp = A.convert_bbox_from_albumentations(bbox, 'pascal_voc', image.shape[0], image.shape[0]) \n boxes.append(temp)\n '''\n [0: 'atypical', 1: 'indeterminate', 2: 'negative', 3: 'typical']\n '''\n labels = records['integer_label'].values\n return image, boxes, labels\ndf_folds = df_folds.set_index('id')\n\ndef get_train_dataset(fold_number): \n return CustomDataset(\n image_ids = df_folds[df_folds['fold'] != fold_number].index.values,\n df = train,\n transforms = get_train_transforms()\n )\n\ndef get_validation_dataset(fold_number):\n return CustomDataset(\n image_ids = df_folds[df_folds['fold'] == fold_number].index.values,\n df = train,\n transforms = get_valid_transforms()\n )\n\n","metadata":{"execution":{"iopub.status.busy":"2021-09-23T16:57:22.152683Z","iopub.execute_input":"2021-09-23T16:57:22.153106Z","iopub.status.idle":"2021-09-23T16:57:22.172726Z","shell.execute_reply.started":"2021-09-23T16:57:22.153067Z","shell.execute_reply":"2021-09-23T16:57:22.171808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_train_dataset(fold_number): \n return CustomDataset(\n image_ids = df_folds[df_folds['fold'] != fold_number].index.values,\n df = train,\n transforms = get_train_transforms()\n )\n\ndef get_validation_dataset(fold_number):\n return CustomDataset(\n image_ids = df_folds[df_folds['fold'] == fold_number].index.values,\n df = train,\n transforms = get_valid_transforms()\n )","metadata":{"execution":{"iopub.status.busy":"2021-09-23T16:57:22.174013Z","iopub.execute_input":"2021-09-23T16:57:22.174483Z","iopub.status.idle":"2021-09-23T16:57:22.181970Z","shell.execute_reply.started":"2021-09-23T16:57:22.174406Z","shell.execute_reply":"2021-09-23T16:57:22.181212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Instantiate these classes","metadata":{}},{"cell_type":"code","source":"#create the train dataset \ntrain_dataset = get_train_dataset(0)\n\nimage, target, image_id = train_dataset[3]\nboxes = target['boxes'].cpu().numpy().astype(np.int32)\n\nnumpy_image = image.permute(1,2,0).cpu().numpy()\n\nfig, ax = plt.subplots(1, 1, figsize=(10, 6))\n\nfor box in boxes:\n cv2.rectangle(numpy_image, (box[0], box[1]), (box[2], box[3]), (0, 255, 0), 2)\n \nax.set_axis_off()\nax.imshow(numpy_image);","metadata":{"execution":{"iopub.status.busy":"2021-09-23T16:57:22.183212Z","iopub.execute_input":"2021-09-23T16:57:22.183641Z","iopub.status.idle":"2021-09-23T16:57:22.315391Z","shell.execute_reply.started":"2021-09-23T16:57:22.183594Z","shell.execute_reply":"2021-09-23T16:57:22.314412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Create DataLoaders","metadata":{}},{"cell_type":"code","source":"def get_train_data_loader(train_dataset, batch_size=16):\n return DataLoader(\n train_dataset,\n batch_size = batch_size,\n shuffle = False,\n num_workers = 4,\n collate_fn = collate_fn\n )\n\ndef get_validation_data_loader(valid_dataset, batch_size=16):\n return DataLoader(\n valid_dataset,\n batch_size = batch_size,\n shuffle = False,\n num_workers = 4,\n collate_fn = collate_fn\n ) \n\ndef collate_fn(batch):\n return tuple(zip(*batch))","metadata":{"execution":{"iopub.status.busy":"2021-09-23T16:57:22.316816Z","iopub.execute_input":"2021-09-23T16:57:22.317141Z","iopub.status.idle":"2021-09-23T16:57:22.323578Z","shell.execute_reply.started":"2021-09-23T16:57:22.317108Z","shell.execute_reply":"2021-09-23T16:57:22.322493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_rows=3\nn_cols=3\n\n# Create a dataloader\ntrain_dataset = get_train_dataset(fold_number=DefaultConfig.fold_num)\ntrain_data_loader = get_train_data_loader(train_dataset, batch_size=9)\n\nimages, targets, image_ids = next(iter(train_data_loader))\n\nimages = list(image.to(device) for image in images)\ntargets = [{k: v.to(device) for k, v in t.items()} for t in targets]\n\n# plot some augmentations!\nfig, ax = plt.subplots(figsize=(20, 20), nrows=n_rows, ncols=n_cols)\nfor i in range (n_rows*n_cols): \n boxes = targets[i]['boxes'].cpu().numpy().astype(np.int32)\n sample = images[i].permute(1,2,0).cpu().numpy()\n for box in boxes:\n cv2.rectangle(sample,\n (box[0], box[1]),\n (box[2], box[3]),\n (0, 255, 0), 3)\n \n ax[i // n_rows][i % n_cols].imshow(sample) ","metadata":{"execution":{"iopub.status.busy":"2021-09-23T16:57:22.325007Z","iopub.execute_input":"2021-09-23T16:57:22.325386Z","iopub.status.idle":"2021-09-23T16:57:24.248333Z","shell.execute_reply.started":"2021-09-23T16:57:22.325329Z","shell.execute_reply":"2021-09-23T16:57:24.247258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. Retrieve the Model","metadata":{}},{"cell_type":"code","source":"class FasterRCNNDetector(torch.nn.Module):\n def __init__(self, pretrained=False, **kwargs):\n super(FasterRCNNDetector, self).__init__()\n # load pre-trained model incl. head\n self.model = torchvision.models.detection.fasterrcnn_resnet50_fpn(pretrained=pretrained, pretrained_backbone=pretrained)\n \n # get number of input features for the classifier custom head\n in_features = self.model.roi_heads.box_predictor.cls_score.in_features\n \n # replace the pre-trained head with a new one\n self.model.roi_heads.box_predictor = FastRCNNPredictor(in_features, DefaultConfig.num_classes)\n \n def forward(self, images, targets=None):\n return self.model(images, targets)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T16:57:24.249903Z","iopub.execute_input":"2021-09-23T16:57:24.250297Z","iopub.status.idle":"2021-09-23T16:57:24.258199Z","shell.execute_reply.started":"2021-09-23T16:57:24.250254Z","shell.execute_reply":"2021-09-23T16:57:24.257143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\n\ndef get_model(checkpoint_path=None, pretrained=False):\n model = FasterRCNNDetector(pretrained=pretrained)\n \n # Load the trained weights\n if checkpoint_path is not None:\n checkpoint = torch.load(checkpoint_path)\n model.load_state_dict(checkpoint['model_state_dict'])\n\n del checkpoint\n gc.collect()\n \n return model.cuda()\n\nnet = get_model(pretrained=True)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T16:57:24.259803Z","iopub.execute_input":"2021-09-23T16:57:24.260367Z","iopub.status.idle":"2021-09-23T16:57:25.017613Z","shell.execute_reply.started":"2021-09-23T16:57:24.260253Z","shell.execute_reply":"2021-09-23T16:57:25.016662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5. Evaluation Metrics\nHere I use Intersection Over Union (IOU) which is a measure that evaluates the overlap between two bounding boxes. The code is from this [notebooke](https://www.kaggle.com/pestipeti/competition-metric-details-script). You can learn more about evaluation metrics [here](https://github.com/rafaelpadilla/Object-Detection-Metrics#different-competitions-different-metrics).\n\n","metadata":{}},{"cell_type":"code","source":"#https://www.kaggle.com/pestipeti/competition-metric-details-script\nfrom numba import jit\n@jit(nopython=True)\ndef calculate_iou(gt, pr, form='pascal_voc') -> float:\n \"\"\"Calculates the Intersection over Union.\n\n Args:\n gt: (np.ndarray[Union[int, float]]) coordinates of the ground-truth box\n pr: (np.ndarray[Union[int, float]]) coordinates of the prdected box\n form: (str) gt/pred coordinates format\n - pascal_voc: [xmin, ymin, xmax, ymax]\n - coco: [xmin, ymin, w, h]\n Returns:\n (float) Intersection over union (0.0 <= iou <= 1.0)\n \"\"\"\n if form == 'coco':\n gt = gt.copy()\n pr = pr.copy()\n\n gt[2] = gt[0] + gt[2]\n gt[3] = gt[1] + gt[3]\n pr[2] = pr[0] + pr[2]\n pr[3] = pr[1] + pr[3]\n\n # Calculate overlap area\n dx = min(gt[2], pr[2]) - max(gt[0], pr[0]) + 1\n \n if dx < 0:\n return 0.0\n \n dy = min(gt[3], pr[3]) - max(gt[1], pr[1]) + 1\n\n if dy < 0:\n return 0.0\n\n overlap_area = dx * dy\n\n # Calculate union area\n union_area = (\n (gt[2] - gt[0] + 1) * (gt[3] - gt[1] + 1) +\n (pr[2] - pr[0] + 1) * (pr[3] - pr[1] + 1) -\n overlap_area\n )\n\n return overlap_area / union_area\n\n@jit(nopython=True)\ndef find_best_match(gts, pred, pred_idx, threshold = 0.5, form = 'pascal_voc', ious=None) -> int:\n \"\"\"Returns the index of the 'best match' between the\n ground-truth boxes and the prediction. The 'best match'\n is the highest IoU. (0.0 IoUs are ignored).\n\n Args:\n gts: (List[List[Union[int, float]]]) Coordinates of the available ground-truth boxes\n pred: (List[Union[int, float]]) Coordinates of the predicted box\n pred_idx: (int) Index of the current predicted box\n threshold: (float) Threshold\n form: (str) Format of the coordinates\n ious: (np.ndarray) len(gts) x len(preds) matrix for storing calculated ious.\n\n Return:\n (int) Index of the best match GT box (-1 if no match above threshold)\n \"\"\"\n best_match_iou = -np.inf\n best_match_idx = -1\n\n for gt_idx in range(len(gts)):\n \n if gts[gt_idx][0] < 0:\n # Already matched GT-box\n continue\n \n iou = -1 if ious is None else ious[gt_idx][pred_idx]\n\n if iou < 0:\n iou = calculate_iou(gts[gt_idx], pred, form=form)\n \n if ious is not None:\n ious[gt_idx][pred_idx] = iou\n\n if iou < threshold:\n continue\n\n if iou > best_match_iou:\n best_match_iou = iou\n best_match_idx = gt_idx\n\n return best_match_idx\n\n@jit(nopython=True)\ndef calculate_precision(gts, preds, threshold = 0.5, form = 'coco', ious=None) -> float:\n \"\"\"Calculates precision for GT - prediction pairs at one threshold.\n\n Args:\n gts: (List[List[Union[int, float]]]) Coordinates of the available ground-truth boxes\n preds: (List[List[Union[int, float]]]) Coordinates of the predicted boxes,\n sorted by confidence value (descending)\n threshold: (float) Threshold\n form: (str) Format of the coordinates\n ious: (np.ndarray) len(gts) x len(preds) matrix for storing calculated ious.\n\n Return:\n (float) Precision\n \"\"\"\n n = len(preds)\n tp = 0\n fp = 0\n \n # for pred_idx, pred in enumerate(preds_sorted):\n for pred_idx in range(n):\n\n best_match_gt_idx = find_best_match(gts, preds[pred_idx], pred_idx,\n threshold=threshold, form=form, ious=ious)\n\n if best_match_gt_idx >= 0:\n # True positive: The predicted box matches a gt box with an IoU above the threshold.\n tp += 1\n # Remove the matched GT box\n gts[best_match_gt_idx] = -1\n\n else:\n # No match\n # False positive: indicates a predicted box had no associated gt box.\n fp += 1\n\n # False negative: indicates a gt box had no associated predicted box.\n fn = (gts.sum(axis=1) > 0).sum()\n\n return tp / (tp + fp + fn)\n\n\n@jit(nopython=True)\ndef calculate_image_precision(gts, preds, thresholds = (0.5, ), form = 'coco') -> float:\n \"\"\"Calculates image precision.\n The mean average precision at different intersection over union (IoU) thresholds.\n\n Args:\n gts: (List[List[Union[int, float]]]) Coordinates of the available ground-truth boxes\n preds: (List[List[Union[int, float]]]) Coordinates of the predicted boxes,\n sorted by confidence value (descending)\n thresholds: (float) Different thresholds\n form: (str) Format of the coordinates\n\n Return:\n (float) Precision\n \"\"\"\n n_threshold = len(thresholds)\n image_precision = 0.0\n \n ious = np.ones((len(gts), len(preds))) * -1\n # ious = None\n\n for threshold in thresholds:\n precision_at_threshold = calculate_precision(gts.copy(), preds, threshold=threshold,\n form=form, ious=ious)\n image_precision += precision_at_threshold / n_threshold\n\n return image_precision","metadata":{"execution":{"iopub.status.busy":"2021-09-23T16:57:25.019077Z","iopub.execute_input":"2021-09-23T16:57:25.019457Z","iopub.status.idle":"2021-09-23T16:57:25.046682Z","shell.execute_reply.started":"2021-09-23T16:57:25.019417Z","shell.execute_reply":"2021-09-23T16:57:25.045745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Helper functions","metadata":{}},{"cell_type":"code","source":"class AverageMeter(object):\n \"\"\"Computes and stores the average and current value\"\"\"\n def __init__(self):\n self.reset()\n\n def reset(self):\n self.val = 0\n self.avg = 0\n self.sum = 0\n self.count = 0\n\n def update(self, val, n=1):\n self.val = val\n self.sum += val * n\n self.count += n\n self.avg = self.sum / self.count","metadata":{"execution":{"iopub.status.busy":"2021-09-23T16:57:25.048848Z","iopub.execute_input":"2021-09-23T16:57:25.049405Z","iopub.status.idle":"2021-09-23T16:57:25.055625Z","shell.execute_reply.started":"2021-09-23T16:57:25.049370Z","shell.execute_reply":"2021-09-23T16:57:25.054450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"iou_thresholds = [0.5]\n\nclass EvalMeter(object):\n \"\"\"Computes and stores the average and current value\"\"\"\n def __init__(self):\n self.reset()\n\n def reset(self):\n self.image_precision = 0\n self.avg = 0\n self.sum = 0\n self.count = 0\n\n def update(self, gt_boxes, pred_boxes, n=1): \n \"\"\" pred_boxes : need to be sorted.\"\"\"\n \n self.image_precision = calculate_image_precision(pred_boxes,\n gt_boxes,\n thresholds=iou_thresholds,\n form='pascal_voc')\n self.count += n\n self.sum += self.image_precision * n\n self.avg = self.sum / self.count","metadata":{"execution":{"iopub.status.busy":"2021-09-23T16:57:25.057179Z","iopub.execute_input":"2021-09-23T16:57:25.057644Z","iopub.status.idle":"2021-09-23T16:57:25.065026Z","shell.execute_reply.started":"2021-09-23T16:57:25.057609Z","shell.execute_reply":"2021-09-23T16:57:25.064172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 6. Training the Model","metadata":{}},{"cell_type":"code","source":"class TrainGlobalConfig:\n num_workers: int = 4\n batch_size: int = 16\n n_epochs: int = 2 \n lr: float = 0.0002\n\n img_size = DefaultConfig.img_size\n \n #folder = '/kaggle/working/' #folder_name\n folder = './'\n\n # -------------------\n verbose = True\n verbose_step = 1\n # -------------------\n\n # --------------------\n step_scheduler = False # do scheduler.step after optimizer.step\n validation_scheduler = False # do scheduler.step after validation stage loss\n\n# SchedulerClass = torch.optim.lr_scheduler.OneCycleLR\n# scheduler_params = dict(\n# max_lr=0.001,\n# epochs=n_epochs,\n# steps_per_epoch=int(len(train_dataset) / batch_size),\n# pct_start=0.1,\n# anneal_strategy='cos', \n# final_div_factor=10**5\n# )\n \n SchedulerClass = torch.optim.lr_scheduler.ReduceLROnPlateau\n scheduler_params = dict(\n mode='min',\n factor=0.5,\n patience=1,\n verbose=False, \n threshold=0.0001,\n threshold_mode='abs',\n cooldown=0, \n min_lr=1e-8,\n eps=1e-08\n )","metadata":{"execution":{"iopub.status.busy":"2021-09-23T16:57:25.066215Z","iopub.execute_input":"2021-09-23T16:57:25.066754Z","iopub.status.idle":"2021-09-23T16:57:25.076134Z","shell.execute_reply.started":"2021-09-23T16:57:25.066717Z","shell.execute_reply":"2021-09-23T16:57:25.075094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Choose your optimizers:\nAdam = False\nif Adam: \n Adam_config = {\"lr\" : 0.001, \"betas\" : (0.9, 0.999), \"eps\" : 1e-08}\nelse:\n SGD_config = {\"lr\" : 0.001, \"momentum\" : 0.9, \"weight_decay\" : 0.001}","metadata":{"execution":{"iopub.status.busy":"2021-09-23T16:57:25.077377Z","iopub.execute_input":"2021-09-23T16:57:25.077744Z","iopub.status.idle":"2021-09-23T16:57:25.085939Z","shell.execute_reply.started":"2021-09-23T16:57:25.077708Z","shell.execute_reply":"2021-09-23T16:57:25.085137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Fitter:\n def __init__(self, model, device, config):\n self.config = config\n self.epoch = 0\n\n self.base_dir = f'./{config.folder}'\n if not os.path.exists(self.base_dir):\n os.makedirs(self.base_dir)\n \n self.log_path = f'{self.base_dir}/log.txt'\n self.best_summary_loss = 10**5\n self.best_score = 0\n\n self.model = model\n self.device = device\n\n param_optimizer = list(self.model.named_parameters())\n no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n optimizer_grouped_parameters = [\n {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},\n {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}\n ]\n \n # get the configured optimizer\n if Adam:\n self.optimizer = torch.optim.Adam(self.model.parameters(), **Adam_config)\n else:\n self.optimizer = torch.optim.SGD(self.model.parameters(), **SGD_config)\n\n self.scheduler = config.SchedulerClass(self.optimizer, **config.scheduler_params)\n self.log(f'Fitter prepared. Device is {self.device}')\n self.log(f'Fold num is {DefaultConfig.fold_num}')\n\n def fit(self, train_loader, validation_loader):\n for e in range(self.config.n_epochs):\n if self.config.verbose:\n lr = self.optimizer.param_groups[0]['lr']\n timestamp = datetime.utcnow().isoformat()\n self.log(f'\\n{timestamp}\\nLR: {lr}')\n\n t = time.time()\n summary_loss = self.train_one_epoch(train_loader)\n \n if e == 0:\n self.best_summary_loss = summary_loss.avg\n\n self.log(f'[RESULT]: Train. Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, time: {(time.time() - t):.5f}')\n self.save(f'{self.base_dir}/last-checkpoint.bin')\n\n t = time.time()\n _, eval_scores = self.validation(validation_loader)\n\n self.log(f'[RESULT]: Val. Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, image_precision: {eval_scores.avg:.5f}, time: {(time.time() - t):.5f}')\n \n #if summary_loss.avg < self.best_summary_loss:\n if eval_scores.avg > self.best_score:\n self.best_summary_loss = summary_loss.avg\n self.best_score = eval_scores.avg\n self.model.eval()\n self.save(f'{self.base_dir}/best-checkpoint-{str(self.epoch).zfill(3)}epoch.bin')\n for path in sorted(glob(f'{self.base_dir}/best-checkpoint-*epoch.bin'))[:-3]:\n os.remove(path)\n\n if self.config.validation_scheduler:\n self.scheduler.step(metrics=eval_scores.avg)\n #self.scheduler.step(metrics=summary_loss.avg)\n\n self.epoch += 1\n\n def validation(self, val_loader):\n self.model.eval()\n \n # model.eval() mode --> it will return boxes and scores.\n # in this part, just print train_loss\n summary_loss = AverageMeter()\n summary_loss.update(self.best_summary_loss, self.config.batch_size)\n \n eval_scores = EvalMeter()\n validation_image_precisions = []\n \n t = time.time()\n for step, (images, targets, image_ids) in enumerate(val_loader):\n if self.config.verbose:\n if step % self.config.verbose_step == 0:\n print(\n f'Val Step {step}/{len(val_loader)}, ' + \\\n f'summary_loss: {summary_loss.avg:.5f}, ' + \\\n f'val_precision: {eval_scores.avg:.5f}, ' + \\\n f'time: {(time.time() - t):.5f}', end='\\r'\n )\n with torch.no_grad():\n images = torch.stack(images)\n batch_size = images.shape[0]\n images = images.to(self.device).float()\n labels = [target['labels'].float() for target in targets]\n\n \"\"\"\n In model.train() mode, model(images) is returning losses.\n We are using model.eval() mode --> it will return boxes and scores. \n \"\"\"\n outputs = self.model(images) \n \n for i, image in enumerate(images): \n gt_boxes = targets[i]['boxes'].data.cpu().numpy()\n boxes = outputs[i]['boxes'].data.cpu().numpy()\n scores = outputs[i]['scores'].detach().cpu().numpy()\n \n preds_sorted_idx = np.argsort(scores)[::-1]\n preds_sorted_boxes = boxes[preds_sorted_idx]\n\n eval_scores.update(pred_boxes=preds_sorted_boxes, gt_boxes=gt_boxes)\n\n return summary_loss, eval_scores\n\n def train_one_epoch(self, train_loader):\n self.model.train()\n summary_loss = AverageMeter()\n\n t = time.time()\n for step, (images, targets, image_ids) in enumerate(train_loader):\n if self.config.verbose:\n if step % self.config.verbose_step == 0:\n print(\n f'Train Step {step}/{len(train_loader)}, ' + \\\n f'summary_loss: {summary_loss.avg:.5f}, ' + \\\n f'time: {(time.time() - t):.5f}', end='\\r'\n )\n \n images = torch.stack(images)\n images = images.to(self.device).float()\n batch_size = images.shape[0]\n boxes = [target['boxes'].to(self.device).float() for target in targets]\n labels = [target['labels'].to(self.device).float() for target in targets]\n targets = [{k: v.to(device) for k, v in t.items()} for t in targets] \n\n self.optimizer.zero_grad()\n \n outputs = self.model(images, targets)\n \n loss = sum(loss for loss in outputs.values())\n \n loss.backward()\n\n summary_loss.update(loss.detach().item(), batch_size)\n\n self.optimizer.step()\n\n if self.config.step_scheduler:\n self.scheduler.step()\n\n return summary_loss\n \n def save(self, path):\n self.model.eval()\n torch.save({\n 'model_state_dict': self.model.state_dict(), #'model_state_dict': self.model.model.state_dict(),\n 'optimizer_state_dict': self.optimizer.state_dict(),\n 'scheduler_state_dict': self.scheduler.state_dict(),\n 'best_summary_loss': self.best_summary_loss,\n 'epoch': self.epoch,\n }, path)\n\n def load(self, path):\n checkpoint = torch.load(path)\n self.model.load_state_dict(checkpoint['model_state_dict'])\n self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])\n self.scheduler.load_state_dict(checkpoint['scheduler_state_dict'])\n self.best_summary_loss = checkpoint['best_summary_loss']\n self.epoch = checkpoint['epoch'] + 1\n \n def log(self, message):\n if self.config.verbose:\n print(message)\n with open(self.log_path, 'a+') as logger:\n logger.write(f'{message}\\n')","metadata":{"execution":{"iopub.status.busy":"2021-09-23T16:57:25.087148Z","iopub.execute_input":"2021-09-23T16:57:25.087462Z","iopub.status.idle":"2021-09-23T16:57:25.119836Z","shell.execute_reply.started":"2021-09-23T16:57:25.087437Z","shell.execute_reply":"2021-09-23T16:57:25.118960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def run_training(fold=0):\n net.to(device)\n \n train_dataset = get_train_dataset(fold_number=fold)\n train_data_loader = get_train_data_loader(\n train_dataset,\n batch_size=TrainGlobalConfig.batch_size\n )\n \n validation_dataset = get_validation_dataset(fold_number=fold)\n validation_data_loader = get_validation_data_loader(\n validation_dataset, \n batch_size=TrainGlobalConfig.batch_size\n )\n\n fitter = Fitter(model=net, device=device, config=TrainGlobalConfig)\n fitter.fit(train_data_loader, validation_data_loader)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T16:57:25.123661Z","iopub.execute_input":"2021-09-23T16:57:25.123957Z","iopub.status.idle":"2021-09-23T16:57:25.133919Z","shell.execute_reply.started":"2021-09-23T16:57:25.123932Z","shell.execute_reply":"2021-09-23T16:57:25.132876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"run_training(fold=DefaultConfig.fold_num)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T16:57:25.135572Z","iopub.execute_input":"2021-09-23T16:57:25.136084Z","iopub.status.idle":"2021-09-23T17:42:07.203723Z","shell.execute_reply.started":"2021-09-23T16:57:25.136047Z","shell.execute_reply":"2021-09-23T17:42:07.202638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file = open('log.txt', 'r')\nfor line in file.readlines():\n print(line[:-1])\nfile.close()","metadata":{"execution":{"iopub.status.busy":"2021-09-23T17:42:07.205522Z","iopub.execute_input":"2021-09-23T17:42:07.205898Z","iopub.status.idle":"2021-09-23T17:42:07.217743Z","shell.execute_reply.started":"2021-09-23T17:42:07.205866Z","shell.execute_reply":"2021-09-23T17:42:07.214160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 7. Simple Inference","metadata":{}},{"cell_type":"code","source":"validation_dataset = get_validation_dataset(fold_number=DefaultConfig.fold_num)\nvalidation_data_loader = get_validation_data_loader(\n validation_dataset, \n batch_size=TrainGlobalConfig.batch_size\n)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T17:42:07.219034Z","iopub.execute_input":"2021-09-23T17:42:07.219428Z","iopub.status.idle":"2021-09-23T17:42:07.229457Z","shell.execute_reply.started":"2021-09-23T17:42:07.219394Z","shell.execute_reply":"2021-09-23T17:42:07.228670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images, targets, image_id = next(iter(validation_data_loader))\n\nimages = list(img.to(device) for img in images)\ntargets = [{k: v.to(device) for k, v in t.items()} for t in targets]\n\nboxes = targets[1]['boxes'].cpu().numpy().astype(np.int32)\nsample = images[1].permute(1,2,0).cpu().numpy()","metadata":{"execution":{"iopub.status.busy":"2021-09-23T17:42:07.230737Z","iopub.execute_input":"2021-09-23T17:42:07.231240Z","iopub.status.idle":"2021-09-23T17:42:08.024210Z","shell.execute_reply.started":"2021-09-23T17:42:07.231204Z","shell.execute_reply":"2021-09-23T17:42:08.023140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"net = get_model('./best-checkpoint-001epoch.bin')","metadata":{"execution":{"iopub.status.busy":"2021-09-23T17:44:30.799244Z","iopub.execute_input":"2021-09-23T17:44:30.799635Z","iopub.status.idle":"2021-09-23T17:44:31.919653Z","shell.execute_reply.started":"2021-09-23T17:44:30.799604Z","shell.execute_reply":"2021-09-23T17:44:31.918770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"net.eval()\ncpu_device = torch.device(\"cpu\")\n\noutputs = net(images)\noutputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs]","metadata":{"execution":{"iopub.status.busy":"2021-09-23T17:44:37.604870Z","iopub.execute_input":"2021-09-23T17:44:37.605202Z","iopub.status.idle":"2021-09-23T17:44:38.523071Z","shell.execute_reply.started":"2021-09-23T17:44:37.605172Z","shell.execute_reply":"2021-09-23T17:44:38.522118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize=(16, 8))\n\nfor box in boxes:\n cv2.rectangle(sample,\n (box[0], box[1]),\n (box[2], box[3]),\n (220, 0, 0), 2)\n \nax.set_axis_off()\nax.imshow(sample)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T17:44:43.201409Z","iopub.execute_input":"2021-09-23T17:44:43.201732Z","iopub.status.idle":"2021-09-23T17:44:43.364201Z","shell.execute_reply.started":"2021-09-23T17:44:43.201704Z","shell.execute_reply":"2021-09-23T17:44:43.363245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 8. References\n\nAll credits of this notebook go to [Heroseo](https://www.kaggle.com/piantic/train-siim-covid-19-detection-fasterrcnn).
\nThe updated labels are from [this dataset](https://www.kaggle.com/dschettler8845/siim-covid19-updated-train-labels).
\nResized images were downloaded from [here](https://www.kaggle.com/xhlulu/siim-covid19-resized-to-256px-jpg).","metadata":{}}]}