{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Pneumonia Detection\n\n## Task:\n\nIn this competition we have set of radiographic images. Our task is to detect a visual signal for pneumonia. Along with images we have bounding box coordinates for our training set. Since we have bounding boxes we will create a model for object detection.\n\n## Model:\n\nThe model that we are going to use is [Faster R-CNN](https://arxiv.org/abs/1506.01497) (Region-based Convolutional Neural Network). For convenient training, evaluation and visualization we will use [Torchvision](https://pytorch.org/vision/stable/index.html) package.","metadata":{}},{"cell_type":"code","source":"import os\nimport glob\nimport pydicom\n\nimport PIL\nimport torch\nimport torchvision\nimport numpy as np \nimport pandas as pd \nimport matplotlib.pyplot as plt\nimport seaborn as sns\nplt.rcParams[\"axes.grid\"] = False\n\nseed = 42\nnum_classes = 2\nbatch_size = 10\ntrain_img_size = 256 \norigin_img_size = 1024\nscale_factor = train_img_size / origin_img_size\nnp.random.seed(seed)\nTRAIN_DIR = \"stage_2_train_images\"\nTEST_DIR = \"stage_2_test_images\"\nROOT_DIR = \"../input/rsna-pneumonia-detection-challenge\"\nLABELS_FILE = \"stage_2_train_labels.csv\"\nSUBMISSION_FILE = \"stage_2_sample_submission.csv\"\nrcnn_losses = [\"loss_objectness\", \"loss_box_reg\", \"loss_rpn_box_reg\"]","metadata":{"execution":{"iopub.status.busy":"2022-08-17T13:31:42.810064Z","iopub.execute_input":"2022-08-17T13:31:42.810780Z","iopub.status.idle":"2022-08-17T13:31:45.227812Z","shell.execute_reply.started":"2022-08-17T13:31:42.810666Z","shell.execute_reply":"2022-08-17T13:31:45.226775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_imgs = os.listdir(os.path.join(ROOT_DIR, TRAIN_DIR))\ntest_imgs =  [patienId + \".dcm\" for patienId in pd.read_csv(os.path.join(ROOT_DIR, SUBMISSION_FILE)).patientId]","metadata":{"execution":{"iopub.status.busy":"2022-08-17T13:31:45.229608Z","iopub.execute_input":"2022-08-17T13:31:45.230138Z","iopub.status.idle":"2022-08-17T13:31:45.758876Z","shell.execute_reply.started":"2022-08-17T13:31:45.230100Z","shell.execute_reply":"2022-08-17T13:31:45.757944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's split our training dataset into training and validation sets. We leave a third of training samples for validation.","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\ntrain_imgs, valid_imgs = train_test_split(train_imgs, test_size=0.33, random_state=seed)\n\nprint(f\"Number of training samples: {len(train_imgs)}\")\nprint(f\"Number of validation samples: {len(valid_imgs)}\")","metadata":{"execution":{"iopub.status.busy":"2022-08-17T13:31:45.761248Z","iopub.execute_input":"2022-08-17T13:31:45.761973Z","iopub.status.idle":"2022-08-17T13:31:45.898404Z","shell.execute_reply.started":"2022-08-17T13:31:45.761934Z","shell.execute_reply":"2022-08-17T13:31:45.897340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Preprocessing and Visualization","metadata":{}},{"cell_type":"markdown","source":"Pneumonia bounding boxes coordinates are sored in stage_2_train_labels.csv file. There are a lot of lines with zero target value and NaN value in box columns. These lines represent images without bounding box.","metadata":{}},{"cell_type":"code","source":"train_label_df = pd.read_csv(os.path.join(ROOT_DIR, LABELS_FILE))\ntrain_label_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-17T13:31:45.901402Z","iopub.execute_input":"2022-08-17T13:31:45.901742Z","iopub.status.idle":"2022-08-17T13:31:45.970485Z","shell.execute_reply.started":"2022-08-17T13:31:45.901709Z","shell.execute_reply":"2022-08-17T13:31:45.969609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Our model require box coordinates in format X0, X1. We also multiply bounding box coordinates with scale_factor since we will train our model on images of size 256 instead of their original size 1024. R-CNN also require area of bounding box as input.","metadata":{}},{"cell_type":"code","source":"isna_count = len(train_label_df[train_label_df.Target == 0]) # number of images without bounding box\ntrain_label_df = train_label_df[train_label_df.Target == 1]\ntrain_label_df.rename(columns={\"x\": \"X0\", \"y\": \"Y0\"}, inplace=True)\ntrain_label_df[\"X1\"] = train_label_df[\"X0\"] + train_label_df[\"width\"]\ntrain_label_df[\"Y1\"] = train_label_df[\"Y0\"] + train_label_df[\"height\"]\ntrain_label_df[[\"X0\", \"X1\", \"Y0\", \"Y1\"]] = train_label_df[[\"X0\", \"X1\", \"Y0\", \"Y1\"]] * scale_factor\ntrain_label_df[\"area\"] = train_label_df[\"width\"] * scale_factor * train_label_df[\"height\"] * scale_factor\ntrain_label_df.drop([\"width\", \"height\"], axis=1, inplace=True)\ntrain_label_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-17T13:31:45.971780Z","iopub.execute_input":"2022-08-17T13:31:45.972106Z","iopub.status.idle":"2022-08-17T13:31:46.001667Z","shell.execute_reply.started":"2022-08-17T13:31:45.972072Z","shell.execute_reply":"2022-08-17T13:31:46.000816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now let´s check how many images have zero, one or more bounding boxes.","metadata":{}},{"cell_type":"code","source":"from collections import Counter\n\ncnt = Counter(train_label_df.patientId)\nsample_batch = [sample[0] + \".dcm\" for sample in cnt.most_common(2)] # We will take two imgs with 4 boxes to display\ncounts = pd.Series(cnt.values()).value_counts()\ncounts[0] = isna_count\nplt.title(\"Box frequency\")\nplt.ylabel(\"Count\")\nplt.xticks(counts.index)\nplt.bar(counts.index, counts)\nplt.show()\ndel cnt","metadata":{"execution":{"iopub.status.busy":"2022-08-17T13:31:46.004267Z","iopub.execute_input":"2022-08-17T13:31:46.004595Z","iopub.status.idle":"2022-08-17T13:31:46.332149Z","shell.execute_reply.started":"2022-08-17T13:31:46.004569Z","shell.execute_reply":"2022-08-17T13:31:46.331173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"counts.sort_values(ascending=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-17T13:31:46.333557Z","iopub.execute_input":"2022-08-17T13:31:46.334107Z","iopub.status.idle":"2022-08-17T13:31:46.344225Z","shell.execute_reply.started":"2022-08-17T13:31:46.334067Z","shell.execute_reply":"2022-08-17T13:31:46.343082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see that most of images do not have any bounding box. There is also a handful of images with three and four bounding boxes. Previously we calculated bounding boxes area so we can now disply area distribution.","metadata":{}},{"cell_type":"code","source":"fig, axs = plt.subplots(1, 2, figsize=(18, 5))\nsns.histplot(data=train_label_df, x=\"area\", ax=axs[0])\nsns.boxplot(y=train_label_df[\"area\"])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-17T13:31:46.345899Z","iopub.execute_input":"2022-08-17T13:31:46.346732Z","iopub.status.idle":"2022-08-17T13:31:46.699626Z","shell.execute_reply.started":"2022-08-17T13:31:46.346701Z","shell.execute_reply":"2022-08-17T13:31:46.698690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see that most of boxes have area smaller than 7000. There are also outliers with area greater than 15000. Now we will define some helper functions to display samples batch along with boxes.","metadata":{}},{"cell_type":"code","source":"from torchvision.utils import draw_bounding_boxes, make_grid\n\n\ndef find_boxes(img_name, label_df): \n    patient_id = img_name.split(\".\")[0]\n    boxes = label_df[label_df.patientId == patient_id]\n    boxes_coord = boxes[[\"X0\", \"Y0\", \"X1\", \"Y1\"]].to_numpy()\n    \n    return boxes_coord, boxes.area.to_numpy()\n\n\ndef read_images(img_names, label_df, resize):\n    batch = []\n    for img_name in img_names:\n        img_path = os.path.join(ROOT_DIR, TRAIN_DIR, img_name)\n        img = PIL.Image.fromarray(pydicom.dcmread(img_path).pixel_array).convert(\"RGB\")\n        img = img.resize(resize)\n        img = np.array(img)\n       \n        boxes, _ = find_boxes(img_name, label_df)\n\n        img = torch.tensor(img, dtype=torch.uint8)\n        boxes = torch.tensor(boxes, dtype=torch.int32)\n        batch.append((img, boxes))\n    return batch\n\n\ndef plot_samples(batch, n_rows=3, n_cols=5, box_color=\"red\", titles=None, box_width=3, fig_size=(20,12)):\n    assert len(batch) >= n_rows * n_cols, f\"Not enough samples to display, required at least {n_rows * n_cols} samples\"\n    fig, axs = plt.subplots(n_rows, 1, figsize=fig_size)\n    fig.tight_layout()\n    imgs_with_boxes = []\n    for sample in batch[:n_rows * n_cols]:\n        scores = None\n        if len(sample) == 2:\n            image, boxes = sample\n        else:\n            image, boxes, scores = sample \n            scores = [f\"Score: {score:.2f}\" for score in scores]\n            \n        img = draw_bounding_boxes(image=image.permute(2, 0, 1), \n                                  boxes=boxes, \n                                  colors=box_color, \n                                  labels=scores,\n                                  width=box_width)\n        imgs_with_boxes.append(img)\n        \n    for i, ax in enumerate(axs): \n        if titles:\n            ax.set_title(titles[i], fontsize=18)\n        img_with_boxes = make_grid(imgs_with_boxes[i * n_cols: (i + 1) * n_cols]).numpy()       \n        ax.imshow(np.transpose(img_with_boxes, (1, 2, 0)))\n    plt.show()\n       \n    \nsample_batch = np.concatenate([sample_batch, np.random.choice(train_imgs, size=13)])\nsample_batch = read_images(sample_batch, train_label_df, (train_img_size, train_img_size))\nplot_samples(sample_batch)","metadata":{"execution":{"iopub.status.busy":"2022-08-17T13:31:46.701871Z","iopub.execute_input":"2022-08-17T13:31:46.702357Z","iopub.status.idle":"2022-08-17T13:31:48.110497Z","shell.execute_reply.started":"2022-08-17T13:31:46.702312Z","shell.execute_reply":"2022-08-17T13:31:48.109626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Some bounding boxes covers small part of the lung while others covers most of the lung. ","metadata":{}},{"cell_type":"markdown","source":"## Image augmentation","metadata":{}},{"cell_type":"markdown","source":"Now we will use a nifty library for image augmentation [imgaug](https://github.com/aleju/imgaug). The library can perform geometric transformations along with bounding boxes, masks and key points. In addition we add some noice and blur to image.","metadata":{}},{"cell_type":"code","source":"import imgaug\nfrom imgaug import augmenters as iaa \nfrom imgaug.augmentables.bbs import BoundingBox, BoundingBoxesOnImage\n\nclass Augmentation:\n    def __init__(self, \n                 x_translation=(-0.05, 0.05), \n                 rotate=(-5, 5),\n                 scale=(0.9, 1.1),\n                 noice=(0, 10),\n                 blur=(0, 0.1)):\n        \n        self.transform = iaa.Sequential([            \n            iaa.Affine(translate_percent={\"x\": x_translation}, scale=scale, rotate=rotate),\n            iaa.AdditiveGaussianNoise(scale=noice),\n            iaa.GaussianBlur(sigma=blur)])\n    \n    def __call__(self, image, boxes): \n        bbs = BoundingBoxesOnImage([BoundingBox(x1=box[0], y1=box[1], x2=box[2], y2=box[3]) \n                                    for box in boxes], shape=image.shape)\n        \n        image, bbs = self.transform(image=image, bounding_boxes=bbs)\n        return image, bbs.to_xyxy_array()\n\n    def worker_init_fn(self, worker_id):\n        \"\"\"Set unique seed for each worker.\"\"\"\n        imgaug.seed(np.random.get_state()[1][0] + worker_id)","metadata":{"execution":{"iopub.status.busy":"2022-08-17T13:31:48.114469Z","iopub.execute_input":"2022-08-17T13:31:48.115091Z","iopub.status.idle":"2022-08-17T13:31:48.946355Z","shell.execute_reply.started":"2022-08-17T13:31:48.115053Z","shell.execute_reply":"2022-08-17T13:31:48.944877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torchvision.transforms import PILToTensor, ConvertImageDtype\n\nclass PneumoniaDataset(torch.utils.data.Dataset):\n    def __init__(self, root, transforms, img_names, train_img_size, labels=None):\n        self.root = root\n        self.transforms = transforms\n        self.img_names = img_names\n        self.resize = (train_img_size, train_img_size)       \n        self.labels = labels\n        \n    def __getitem__(self, idx):\n        \"\"\"Load image, boxes and process them\"\"\"\n        img_name = self.img_names[idx]\n       \n        img_path = os.path.join(self.root, self.img_names[idx])\n        img = PIL.Image.fromarray(pydicom.dcmread(img_path).pixel_array).convert(\"RGB\")\n        img = img.resize(self.resize)\n        img = np.array(img)\n                 \n        if self.labels is None:           \n            return torch.tensor(img / 255.).permute(2, 0, 1).float() \n        \n        boxes, area = find_boxes(img_name, self.labels)\n        n_objects = len(boxes) \n        if self.transforms is not None:            \n            img, boxes = self.transforms(img, boxes)\n            \n        img = torch.tensor(img / 255.).permute(2, 0, 1).float() \n               \n        target = {}\n        target[\"image_id\"] = torch.tensor([idx])\n        target[\"boxes\"] = torch.as_tensor(boxes, dtype=torch.float32)\n        target[\"labels\"] = torch.ones(n_objects, dtype=torch.int64)\n        target[\"area\"] = torch.as_tensor(area, dtype=torch.float32)\n        target[\"iscrowd\"] = torch.zeros(n_objects, dtype=torch.int32)\n        return img, target\n        \n    def __len__(self):\n        return len(self.img_names)\n    \n    def collate_fn(self, batch):\n        return tuple(zip(*batch))    ","metadata":{"execution":{"iopub.status.busy":"2022-08-17T13:31:48.947681Z","iopub.execute_input":"2022-08-17T13:31:48.948044Z","iopub.status.idle":"2022-08-17T13:31:48.960790Z","shell.execute_reply.started":"2022-08-17T13:31:48.948009Z","shell.execute_reply":"2022-08-17T13:31:48.959722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import DataLoader\n\naugmentation = Augmentation()\ntrain_ds = PneumoniaDataset(os.path.join(ROOT_DIR, TRAIN_DIR), augmentation, train_imgs, train_img_size, train_label_df)\nvalid_ds = PneumoniaDataset(os.path.join(ROOT_DIR, TRAIN_DIR), None, valid_imgs, train_img_size, train_label_df)\ntest_ds = PneumoniaDataset(os.path.join(ROOT_DIR, TEST_DIR), None, test_imgs, train_img_size)\n\ntrain_loader = DataLoader(train_ds, batch_size=batch_size, shuffle=True, num_workers=2, collate_fn=train_ds.collate_fn,  \n                          worker_init_fn=augmentation.worker_init_fn)\nvalid_loader = DataLoader(valid_ds, batch_size=batch_size, shuffle=False, num_workers=2, collate_fn=valid_ds.collate_fn) \ntest_loader = DataLoader(test_ds, batch_size=batch_size, shuffle=False, num_workers=2) ","metadata":{"execution":{"iopub.status.busy":"2022-08-17T13:31:48.962606Z","iopub.execute_input":"2022-08-17T13:31:48.963960Z","iopub.status.idle":"2022-08-17T13:31:48.976166Z","shell.execute_reply.started":"2022-08-17T13:31:48.963921Z","shell.execute_reply":"2022-08-17T13:31:48.975215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Since we have image augmentation pipeline we can display some augmented images and compare them with originals.","metadata":{}},{"cell_type":"code","source":"def process_batch(idx, sample):\n    img, boxes = sample\n            \n    augmentation = Augmentation()\n    augmentation.worker_init_fn(idx)\n    img, boxes = augmentation(img.numpy(), boxes.numpy())\n    img = torch.tensor(img, dtype=torch.uint8)\n    boxes = torch.tensor(boxes, dtype=torch.float32)\n    return img, boxes\n\n\naugmented_batch = sample_batch[:5] + [process_batch(i, sample) for i, sample in enumerate(sample_batch[:5])] \nplot_samples(augmented_batch, titles=[\"Original images\", \"Augmented images\"], fig_size=(20, 10), n_rows=2)\ndel sample_batch, augmented_batch","metadata":{"execution":{"iopub.status.busy":"2022-08-17T13:31:48.977914Z","iopub.execute_input":"2022-08-17T13:31:48.978467Z","iopub.status.idle":"2022-08-17T13:31:49.778852Z","shell.execute_reply.started":"2022-08-17T13:31:48.978422Z","shell.execute_reply":"2022-08-17T13:31:49.777868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model","metadata":{}},{"cell_type":"markdown","source":"We will fine-tune a model trained on COCO dataset. For this purpose we have to replace model classifier with a new one adapted to our task. Our model will exploit [Resnet50](https://arxiv.org/pdf/1512.03385v1.pdf) as backbone. We incorporate our model into [PyTorch lightning](https://www.pytorchlightning.ai/) module. This way we can easily train and validate our model. ","metadata":{}},{"cell_type":"code","source":"import pytorch_lightning as pl\nfrom torchvision.ops import nms\nfrom torchvision.models.detection.faster_rcnn import FastRCNNPredictor\n\n\nclass LitRCNN(pl.LightningModule):    \n    def __init__(self, num_classes):\n        super().__init__()\n\n        model = torchvision.models.detection.fasterrcnn_resnet50_fpn(pretrained=True)\n        in_features = model.roi_heads.box_predictor.cls_score.in_features\n        model.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)\n               \n        self.model = model\n        \n    def forward(self, x):\n        self.model.eval()\n        \n        outputs =  self.model(x)\n        \n        preds = []\n        for output in outputs:       \n            boxes = output[\"boxes\"]\n            scores = output[\"scores\"]\n            idx = nms(boxes,scores, 0.05)\n            preds.append({\"boxes\": boxes[idx].cpu().detach().numpy(), \"scores\": scores[idx].cpu().detach().numpy()})\n        \n        return preds\n        \n    def training_step(self, batch, batch_idx):   \n        images, targets = batch   \n               \n        losses = self.model(images, targets)  \n      \n        loss = sum(loss for loss in losses.values())        \n        self.log_losses(loss, losses)   \n        \n        return loss\n    \n    def validation_step(self, batch, batch_idx):     \n        self.model.train()\n        \n        images, targets = batch \n        losses = self.model(images, targets)\n      \n        loss = sum(loss for loss in losses.values())   \n        self.log_losses(loss, losses, mode=\"val\")\n        \n    def predict_step(self, batch, batch_idx):   \n        return self.forward(batch)    \n       \n    def configure_optimizers(self):\n        optimizer = torch.optim.SGD(self.parameters(),  lr=0.005, momentum=0.9, weight_decay=0.0005)\n        scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=0.3)\n        \n        return {\n            \"optimizer\": optimizer,\n            \"lr_scheduler\": {\n                \"scheduler\": scheduler,  \n                \"interval\": \"epoch\",\n                \"frequency\": 1}}\n    \n    def log_losses(self, loss, losses, mode=\"train\"): \n        self.log_dict({f\"{mode}_loss\": loss,                   \n                       f\"{mode}_loss_box_reg\": losses['loss_box_reg'],\n                       f\"{mode}_loss_objectness\": losses['loss_objectness'],\n                       f\"{mode}_loss_rpn_box_reg\": losses['loss_rpn_box_reg']}, on_step=False, on_epoch=True, batch_size=batch_size)\n\nmodel = LitRCNN(num_classes)","metadata":{"execution":{"iopub.status.busy":"2022-08-17T13:31:49.780773Z","iopub.execute_input":"2022-08-17T13:31:49.781588Z","iopub.status.idle":"2022-08-17T13:32:05.675764Z","shell.execute_reply.started":"2022-08-17T13:31:49.781550Z","shell.execute_reply":"2022-08-17T13:32:05.674607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We define some callbacks for logging and model checkpoints.","metadata":{}},{"cell_type":"code","source":"from pytorch_lightning import Callback\nfrom pytorch_lightning.callbacks import ModelCheckpoint\n\n\nclass MetricsCallback(Callback):\n    \"\"\"PyTorch Lightning metric callback.\"\"\"\n\n    def __init__(self, metrics):\n        super().__init__()\n        self.metrics = metrics      \n        self.training = {}\n        self.validations = {}\n        \n    def on_train_epoch_end(self, trainer, pl_module):\n        self.training[trainer.current_epoch] = {metric: trainer.callback_metrics[\"train_\" + metric] for metric in self.metrics}\n\n    def on_validation_end(self, trainer, pl_module):        \n        self.validations[trainer.current_epoch] = {metric: trainer.callback_metrics[\"val_\" + metric] for metric in self.metrics}\n        \n\ncheckpoint_callback = ModelCheckpoint(dirpath='checkpoints',\n                                      filename='{epoch}-{val_loss:.4f}',\n                                      every_n_epochs=1,\n                                      monitor='val_loss',\n                                      save_top_k=1,\n                                      mode='min')\n\ncallbacks = [MetricsCallback([\"loss\"] + rcnn_losses), checkpoint_callback]","metadata":{"execution":{"iopub.status.busy":"2022-08-17T13:32:05.677530Z","iopub.execute_input":"2022-08-17T13:32:05.678586Z","iopub.status.idle":"2022-08-17T13:32:05.692041Z","shell.execute_reply.started":"2022-08-17T13:32:05.678540Z","shell.execute_reply":"2022-08-17T13:32:05.691006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"device = \"gpu\" if torch.cuda.is_available() else \"cpu\"\ntrainer = pl.Trainer(accelerator=device, max_epochs=6, callbacks=callbacks) \ntrainer.fit(model=model, train_dataloaders=train_loader, val_dataloaders=valid_loader)","metadata":{"execution":{"iopub.status.busy":"2022-08-17T13:32:05.694121Z","iopub.execute_input":"2022-08-17T13:32:05.694866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = LitRCNN.load_from_checkpoint(trainer.checkpoint_callback.best_model_path, num_classes=num_classes)\nprint(trainer.checkpoint_callback.best_model_path)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Evaluation","metadata":{}},{"cell_type":"markdown","source":"Now we can check some predictions on validation set.","metadata":{}},{"cell_type":"code","source":"sample_batch_idx = np.random.choice(len(valid_imgs), size=5)\nsample_batch = read_images(np.array(valid_ds.img_names)[sample_batch_idx], train_label_df, (train_img_size, train_img_size))\n\npreds = model([valid_ds[i][0] for i in sample_batch_idx])\nfor i, pred in enumerate(preds):\n    sample_batch.append((sample_batch[i][0], \n                         torch.tensor(pred[\"boxes\"], dtype=torch.int32),\n                         pred[\"scores\"]))\n    \nplot_samples(sample_batch, titles=[\"Validation\", \"Predictions\"], fig_size=(20, 10), n_rows=2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set_style(\"whitegrid\")\ntraining = pd.DataFrame.from_dict(callbacks[0].training, orient=\"index\").applymap(lambda x: x.cpu().numpy())\nvalidations = pd.DataFrame.from_dict(callbacks[0].validations, orient=\"index\").applymap(lambda x: x.cpu().numpy())\nfig, axs = plt.subplots(2, 2, figsize=(16, 8))\n\nfor ax, metric in zip(axs.flat, callbacks[0].metrics):    \n    ax.set_title(metric)\n    \n    g = sns.lineplot(data=training, x=training.index, y=metric, ax=ax)\n    g = sns.lineplot(data=validations, x=validations.index, y=metric, ax=ax)\n    g.set(xlabel=\"Epoch\", ylabel=None)\n    g.set_xticks(training.index)\n    ax.legend(labels=[\"Training\",\"Validation\"])\n  \nfig.tight_layout()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = trainer.validate(model, dataloaders=valid_loader, verbose=False)\npd.DataFrame(history, index=[\"Validation\"]).T","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submission","metadata":{}},{"cell_type":"code","source":"preds = trainer.predict(model, test_loader)\n\noutputs = []\nfor batch_pred in preds:\n    for sample_pred in batch_pred:\n        scores, boxes = sample_pred[\"scores\"], sample_pred[\"boxes\"]\n        if len(scores) == 0:\n            outputs.append(np.nan)\n        else:\n            label = \"\"\n            boxes = boxes / scale_factor\n            for score, box in zip(scores, boxes):            \n                label += f\"{score:.2f} {box[0]:.1f} {box[1]:.1f} {(box[2]-box[0]):.1f} {(box[3]-box[1]):.1f} \"\n            outputs.append(label.strip())\n    \nsubmission = pd.read_csv(os.path.join(ROOT_DIR, SUBMISSION_FILE))\nsubmission.PredictionString = outputs\nsubmission.to_csv(\"submission.csv\", header=True, index=False)\nsubmission","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}