{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"effnet_path = '../input/efficientnet-pytorch/'\niterstrat_path = '../input/iterative-stratification/iterative-stratification-master'\nimport sys\nsys.path.append(effnet_path)\nsys.path.append(iterstrat_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport sys\nimport time\nimport math\nimport glob\nimport ast\nimport random\nfrom albumentations.augmentations.transforms import Resize\n\nimport matplotlib.pyplot as plt \nimport numpy as np\nimport pandas as pd\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport pytorch_lightning as pl\nfrom torchvision import transforms\nimport torchvision.transforms.functional as TF\nfrom efficientnet_pytorch import EfficientNet\n\nfrom sklearn.metrics import roc_auc_score\n\nimport PIL\nimport cv2\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom matplotlib_venn import venn2, venn3, venn3_circles\n\nfrom iterstrat.ml_stratifiers import MultilabelStratifiedKFold\n\nimport cv2\nimport albumentations as A\nfrom torchvision.transforms.transforms import RandomHorizontalFlip\nfrom albumentations.pytorch import ToTensorV2\n\nfrom joblib import Parallel, delayed\n\nfrom PIL import Image\nfrom PIL import ImageFile\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nfrom tqdm import tqdm_notebook as tqdm\nfrom tqdm import tqdm\n\nfrom torch.utils.tensorboard import SummaryWriter","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Parameters\n\nIMAGE_SIZE = (512, 512)\n\nPIL.ImageFile.LOAD_TRUNCATED_IMAGES = True\n\nIMAGE_BACKEND = 'cv2'\n\nFOLD = 0\n\ndef set_seed(seed=0):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    \nset_seed()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# path related variable\ndata_dir = \"../input/ranzcr-clip-catheter-line-classification/\"\n# checkpoint directory path\npath_checkpoints_dir = \"./checkpoints\"\n# submissions directory path\npath_submissions_dir = \"./\"\n# model path\npath_trained_models = \"../input/000-010-sgdr-ensemble-4\"\n\n\npath_test_dir= os.path.join(data_dir, 'test')\npath_sample_submission_file= os.path.join(data_dir, 'sample_submission.csv')\n\nsubmission_file = pd.read_csv(path_sample_submission_file)\npath_test_images = [os.path.join(path_test_dir, i + \".jpg\") for i in submission_file.StudyInstanceUID.values]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def resize_one_image(input_path, output_path, image_size):\n    image = cv2.imread(input_path)\n    image = cv2.resize(image, image_size)\n    cv2.imwrite(output_path, image)\n\ndef resize_image_batch(input_dir, output_dir, image_size):\n    \"\"\"multiprocessing image resize function\"\"\"\n    if not os.path.isdir(output_dir):\n        os.mkdir(output_dir)\n    input_paths = [os.path.join(input_dir, image_name) for image_name in os.listdir(input_dir)]\n    output_paths = [os.path.join(output_dir, image_name) for image_name in os.listdir(input_dir)]\n    image_sizes = [image_size]*len(input_paths)\n    \n    _ = Parallel(n_jobs=-1, verbose=3)(delayed(resize_one_image)(ipath, opath, img_size) for ipath, opath, img_size in zip(input_paths, output_paths, image_sizes))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path_resized_test_image_dir = os.path.join('./', \"test_resized\")\nprint(path_resized_test_image_dir)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_file = pd.read_csv(path_sample_submission_file)\npath_test_images = [os.path.join(path_resized_test_image_dir, i + \".jpg\") for i in submission_file.StudyInstanceUID.values]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path_test_dir = \"../input/ranzcr-clip-catheter-line-classification/test\"\nresize_image_batch(path_test_dir, path_resized_test_image_dir, IMAGE_SIZE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ImageDataset:\n    def __init__(\n        self,\n        image_paths,\n        targets=None,\n        augmentations=None,\n        backend=\"cv2\",\n        channel_first=True,\n        grayscale=False,\n        grayscale_as_rgb=False,\n    ):\n        \"\"\"\n        :param image_paths: list of paths to images\n        :param targets: numpy array\n        :param augmentations: albumentations augmentations\n        :param backend: 'pil' or 'cv2'\n        :param channel_first: f True Images in (C,H,W) format else (H,W,C) format\n        :param grayscale: grayscale flag\n        :grayscale_as_rgb: load grayscale images as RGB images for transfer learning purpose\n        \"\"\"\n        if grayscale is False and grayscale_as_rgb is True:\n            raise Exception(\"Invalid combination of \"                 \"arguments 'grayscale=False' and 'grayscale_as_rgb=True'\")\n        self.image_paths = image_paths\n        self.targets = targets\n        self.augmentations = augmentations\n        self.backend = backend\n        self.channel_first = channel_first\n        self.grayscale = grayscale\n        self.grayscale_as_rgb = grayscale_as_rgb\n\n    def __len__(self):\n        return len(self.image_paths)\n\n    def __getitem__(self, item):\n        # TODO: add test loader logic\n        if self.backend == \"pil\":\n            image = Image.open(self.image_paths[item])\n            if self.grayscale is True and self.grayscale_as_rgb is True:\n                image = image.convert('RGB')\n            image = np.array(image)\n            if self.augmentations is not None:\n                augmented = self.augmentations(image=image)\n                image = augmented[\"image\"]\n        elif self.backend == \"cv2\":\n            if self.grayscale is False or self.grayscale_as_rgb is True: \n                image = cv2.imread(self.image_paths[item])\n                image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n            else:\n                image = cv2.imread(self.image_paths[item], cv2.IMREAD_GRAYSCALE)\n            if self.augmentations is not None:\n                augmented = self.augmentations(image=image)\n                image = augmented[\"image\"]\n        else:\n            raise Exception(\"Backend not implemented\")\n            \n        if not isinstance(image, torch.Tensor):\n            if self.channel_first is True and image.ndim == 3:\n                image = np.transpose(image, (2, 0, 1)).astype(np.float32)\n                image = torch.tensor(image)\n                \n        if len(image.size()) == 2:\n            image = image.unsqueeze(0)\n            \n        if self.targets is not None:\n            targets = self.targets[item]\n            targets = torch.tensor(targets)\n        else: \n            targets = torch.tensor([])\n            \n        return {\n            \"image\": image,\n            \"targets\": targets,\n        }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# image augmentation\n\ntest_augmentation = A.Compose([\n    A.CLAHE(p=1),\n    A.Normalize(\n        mean=(0.485, 0.456, 0.406), \n        std=(0.229, 0.224, 0.225), \n        max_pixel_value=255.0, \n        always_apply=True,\n        ),\n    ToTensorV2(),\n    ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntest_dataset = ImageDataset(\n    path_test_images,  \n    None,  \n    augmentations=test_augmentation,  \n    backend=IMAGE_BACKEND,  \n    channel_first=True,  \n    grayscale=True,  \n    grayscale_as_rgb=True,\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class DataModule(object):\n    def __init__(self, train_dataset, valid_dataset, test_dataset):\n        self.train_dataset = train_dataset\n        self.valid_dataset = valid_dataset\n        self.test_dataset = test_dataset\n        \n    def get_train_dataloader(self, **kwargs):\n        return torch.utils.data.DataLoader(self.train_dataset, **kwargs)\n    \n    def get_valid_dataloader(self, **kwargs):\n        return torch.utils.data.DataLoader(self.valid_dataset, **kwargs)\n    \n    def get_test_dataloader(self, **kwargs):\n        return torch.utils.data.DataLoader(self.test_dataset, **kwargs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nclass EfficientNetModel(torch.nn.Module):\n    def __init__(self, num_labels=11, pretrained=True):\n        super().__init__()\n        self.num_labels = num_labels\n        if pretrained:\n            self.backbone = EfficientNet.from_pretrained(\"efficientnet-b5\",)\n        else:\n            self.backbone = EfficientNet.from_name(\"efficientnet-b5\",)\n\n        self.dropout = torch.nn.Dropout(p=0.5)\n        self.avgpool = torch.nn.AdaptiveAvgPool2d(1)\n        self.fc = torch.nn.Linear(2048, num_labels)\n        self.sigmoid = torch.nn.Sigmoid()\n    \n    def forward(self, image, targets=None):\n        image = self.backbone.extract_features(image)\n        image = self.avgpool(image).squeeze(-1).squeeze(-1)\n        image = self.dropout(image)\n        image = self.fc(image)\n        loss = None\n        if targets is not None and targets.size(1) == self.num_labels:\n            loss = torch.nn.BCEWithLogitsLoss()(image, targets.type_as(image))\n\n        with torch.no_grad():\n            y_pred = self.sigmoid(image)\n\n        return y_pred, loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Trainer:\n    def __init__(self, model, data_module, experiment_id, optimizer=None, scheduler=None, device='cuda'):\n        self.model = model\n        self.data_module = data_module\n        self.optimizer = optimizer\n        self.scheduler = scheduler\n        self.device = device\n        self.fp16 = False\n        self.step_scheduler_after = None\n        self.n_epochs = None\n        self.metrics = {}\n        self.metrics['train'] = []\n        self.metrics['valid'] = []\n        self.current_epoch = 0\n        self.current_batch = 0\n        self.scaler = None\n        # early stopping related variables\n        self._best_score = -np.inf\n        self._delta = None\n        self._current_score = None\n        self._counter = 0\n        self._patience = None\n        # variable related to model checkpoints\n        self.experiment_id=experiment_id\n        \n    def configure_trainer(self):\n        if self.optimizer is None:\n            self.configure_optimizers()\n        if self.scheduler is None:\n            self.configure_schedulers()\n        \n        if next(self.model.parameters()).device != self.device:\n            self.model.to(self.device)\n            \n        if self.fp16:\n            self.scaler = torch.cuda.amp.GradScaler()\n            \n    def configure_optimizers(self):\n        self.optimizer = torch.optim.Adam(self.model.parameters(), lr=self.lr)\n    \n    def configure_schedulers(self, **kwargs):\n        self.scheduler = None\n        \n    def set_params(self, **kwargs):\n        for parameter, value in kwargs.items():\n             setattr(self, parameter, value)\n\n    def compute_metrics(self, batch_outputs_collection, batch_targets_collection):\n        y_pred = np.concatenate(batch_outputs_collection, axis=0) \n        y_true = np.concatenate(batch_targets_collection, axis=0) \n        assert y_pred.shape == y_true.shape, \"shape mismatch\"\n        scores = []\n        for i in range(y_true.shape[1]):\n            try:\n                score = roc_auc_score(y_true[:,i], y_pred[:,i])\n                scores.append(score)\n            except ValueError:\n                raise(\"Behaviour Not expected: FIXME\")\n        avg_score = np.mean(scores)\n        return avg_score\n    \n    \n    def model_forward_pass(self, data):\n        \"\"\"forward pass of model\"\"\"\n        for key, value in data.items():\n            data[key] = value.to(self.device)\n            \n        if self.fp16:\n            with torch.cuda.amp.autocast():\n                output, loss = self.model(**data)\n        else:\n            output, loss = self.model(**data)\n                \n        return output, loss\n        \n    \n    def train_one_batch(self, data):\n        self.optimizer.zero_grad()\n        output, loss = self.model_forward_pass(data)\n        with torch.set_grad_enabled(True):\n            if self.fp16:\n                with torch.cuda.amp.autocast():\n                    self.scaler.scale(loss).backward()\n                    self.scaler.step(self.optimizer)\n                    self.scaler.update()\n            else:\n                loss.backward()\n                self.optimizer.step()\n            if self.scheduler:\n                if self.step_scheduler_after == \"batch\":\n                    if self.step_scheduler_metric is None:\n                        self.scheduler.step()\n                    else:\n                        pass\n                        # step_metric = self.name_to_metric(self.step_scheduler_metric)\n                        # self.scheduler.step(step_metric)\n        return output, loss\n        \n    def train_one_epoch(self, dataloader):\n        self.model.train()\n        all_outputs = []\n        all_targets = []\n        all_losses = []\n        tk0 = tqdm(enumerate(dataloader, 1), total=len(dataloader))\n        for batch_id, data in tk0:\n            batch_outputs, batch_loss= self.train_one_batch(data)\n            all_outputs.append(batch_outputs.detach().cpu().numpy())\n            all_targets.append(data['targets'].detach().cpu().numpy())\n            all_losses.append(batch_loss.detach().cpu().item())\n            tk0.set_postfix(loss=np.array(all_losses).mean(), stage=\"train\", epoch=self.current_epoch)\n        tk0.close()\n\n        # compute metrics\n        # compute average loss\n        avg_auc = self.compute_metrics(all_outputs, all_targets)\n        avg_loss = np.array(all_losses).mean()\n        self.metrics['train'].append({'epoch': self.current_epoch, \n        'avg_loss': avg_loss, 'auc_score': avg_auc})\n        print(self.metrics['train'][self.current_epoch -1])\n    \n    def validate_one_batch(self, data):\n        output, loss = self.model_forward_pass(data)\n        return output, loss\n \n\n    def validate_one_epoch(self, dataloader):\n        self.model.eval()\n        all_outputs = []\n        all_targets = []\n        all_losses = []\n        tk0 = tqdm(enumerate(dataloader, 1), total=len(dataloader))\n        for batch_id, data in tk0:\n            batch_outputs, batch_loss= self.validate_one_batch(data)\n            all_outputs.append(batch_outputs.detach().cpu().numpy())\n            all_targets.append(data['targets'].detach().cpu().numpy())\n            all_losses.append(batch_loss.detach().cpu().item())\n            tk0.set_postfix(loss=np.array(all_losses).mean(), stage=\"validate\", epoch=self.current_epoch)\n        tk0.close()\n        # compute metrics\n        # compute average loss\n        avg_auc = self.compute_metrics(all_outputs, all_targets)\n        avg_loss = np.array(all_losses).mean()\n        self.metrics['valid'].append({'epoch': self.current_epoch, \n        'avg_loss': avg_loss, 'auc_score': avg_auc})\n        print(self.metrics['valid'][-1])\n    \n    def early_stoping(self):\n        \"\"\"early stoping function\"\"\"\n        self._current_score = self.metrics['valid'][-1]['auc_score']\n        if (self._current_score - self._best_score) > self._delta:\n            self._best_score = self._current_score\n            self._counter = 0\n            self.save_checkpoint()\n            print(\"early stopping counter reset to 0\")\n        else:\n            self._counter += 1\n            print(f\"early stopping counter {self._counter} out of {self._patience}\")\n        if self._counter == self._patience:\n            return True\n        return False\n    \n    def save_checkpoint(self):\n        \"\"\"save model and optimizer state for resuming training\"\"\"\n        if not os.path.isdir(path_checkpoints_dir):\n            os.mkdir(path_checkpoints_dir)\n        model_path = os.path.join(path_checkpoints_dir, f\"{self.experiment_id}.pth\")\n        print(f\"saved the model at {model_path}\") \n        model_state_dict = self.model.state_dict()\n        if self.optimizer is not None:\n            opt_state_dict = self.optimizer.state_dict()\n        else:\n            opt_state_dict = None\n        if self.scheduler is not None:\n            sch_state_dict = self.scheduler.state_dict()\n        else:\n            sch_state_dict = None\n        model_dict = {}\n        model_dict[\"state_dict\"] = model_state_dict\n        model_dict[\"optimizer\"] = opt_state_dict\n        model_dict[\"scheduler\"] = sch_state_dict\n        model_dict[\"epoch\"] = self.current_epoch\n        model_dict[\"fp16\"] = self.fp16\n        model_dict['lr'] = self.lr\n        model_dict['metrics'] = self.metrics\n        model_dict['best_score'] = self._best_score\n        model_dict['patience'] = self._patience\n        model_dict['delta'] = self._delta\n        model_dict['train_batch_size'] = self.train_batch_size\n        model_dict['validation_batch_size'] = self.validation_batch_size\n        torch.save(model_dict, model_path)\n    \n    def load(self, model_path, device=None):\n        \"\"\"Load the saved model to resume training and inference\"\"\"\n        if device:\n            self.device = device\n        checkpoint = torch.load(model_path)\n        if self.model:\n            self.model.load_state_dict(checkpoint['state_dict'])\n            self.model.to(self.device)\n        if self.optimizer:\n            self.optimizer.load_state_dict(checkpoint['optimizer'])\n        if self.scheduler:\n            self.scheduler.load_state_dict(checkpoint['scheduler'])\n        #self.current_epoch = checkpoint['epoch']\n        self.fp16 = True\n        #self.lr = checkpoint['lr']\n        #self.metrics = checkpoint['metrics']\n        #self._best_score = checkpoint['best_score']\n        #self._patience = checkpoint['patience']\n        #self._delta = checkpoint['delta']\n        #self.train_batch_size = checkpoint['train_batch_size']\n        #self.validation_batch_size = checkpoint['validation_batch_size']\n    \n    def predict(self, test_batch_size=64, device='cuda', load=False, model_path=None, dataloader_num_workers=4, save_prediction=True):\n        \"\"\"make predictions on test images\"\"\"\n        self.model.eval()\n        self.device = device\n        self.test_batch_size = test_batch_size\n        if load:\n            if model_path:\n                self.load(model_path, device=self.device)\n            else:\n                model_path = os.path.join(path_checkpoints_dir, f\"{self.experiment_id}.pth\")\n                print(f\"loaded model={model_path}\")\n                self.load(model_path, device=self.device)\n        if self.model is None:\n            raise Exception(\"model cannot be None. Load or train the model before inference\")\n        dataloader = self.data_module.get_test_dataloader(batch_size=self.test_batch_size, shuffle=False, num_workers=dataloader_num_workers)\n        all_outputs = []\n        tk0 = tqdm(enumerate(dataloader, 1), total=len(dataloader))\n        for batch_id, data in tk0:\n            for key, value in data.items():\n                data[key] = value.to(self.device)\n            # batch_outputs, batch_loss = self.model(**data)\n            batch_outputs, batch_loss= self.validate_one_batch(data)\n            all_outputs.append(batch_outputs.detach().cpu().numpy())\n        predictions = np.concatenate(all_outputs, axis=0)\n        if save_prediction:\n            submission = pd.read_csv(path_sample_submission_file)\n            assert submission.shape[0] == predictions.shape[0], \"unexpected behavior.code fix required\"\n            submission.iloc[:, 1:] = predictions\n\n            if not os.path.isdir(path_submissions_dir):\n                os.mkdir(path_submissions_dir)\n            submission.to_csv(os.path.join(path_submissions_dir, f\"{self.experiment_id}.csv\"), index=False)\n        tk0.close()\n        return predictions\n\n    def fit(self,\n            n_epochs=100, \n            lr=1e-3, \n            step_scheduler_after='epoch', \n            device='cuda', \n            fp16=False,\n            train_batch_size = 64,\n            validation_batch_size=64,\n            dataloader_shuffle=True,\n            dataloader_num_workers=4,\n            tensorboard_writer = None,\n            es_delta=1e-4,\n            es_patience=3,\n           ):\n        \"\"\"fit method to train the model\"\"\"\n        self.n_epochs = n_epochs\n        self.step_scheduler_after = step_scheduler_after\n        self.device = device\n        self.fp16 = fp16\n        self.lr = lr\n        self._delta = es_delta\n        self._patience = es_patience\n        # self.experiment_id = experiment_id\n        self.train_batch_size = train_batch_size\n        self.validation_batch_size = validation_batch_size\n        self.configure_trainer()\n        # self.set_params(**kwargs)\n        for i in range(1, self.n_epochs+1):\n            self.current_epoch = i\n            # train\n            train_dataloader = self.data_module.get_train_dataloader(\n                batch_size=train_batch_size, \n                shuffle=dataloader_shuffle, \n                num_workers=dataloader_num_workers,\n                pin_memory=True)\n            self.train_one_epoch(train_dataloader)\n            # validate \n            validation_dataloader = self.data_module.get_valid_dataloader(\n                batch_size=validation_batch_size, \n                shuffle=dataloader_shuffle, \n                num_workers=dataloader_num_workers, \n                pin_memory=True\n            )\n            self.validate_one_epoch(validation_dataloader)\n            es_flag = self.early_stoping()\n            if es_flag:\n                print(f\"early stopping at epoch={i} out of {n_epochs}\")\n                break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def ensemble_models(model_paths, output_file, **kwargs):\n    \"\"\"combine different models to create the ensemble\"\"\"\n    model = EfficientNetModel(pretrained=False)\n    data_module = DataModule(None, None, test_dataset)\n    preds_list = []\n    num_models = len(model_paths)\n    print(f\"number of models to ensemble={num_models}\")\n    for mpath in model_paths:\n        print(mpath)\n        trainer = Trainer(model, data_module, None)\n        prediction = trainer.predict(load=True, model_path=mpath, save_prediction=False, **kwargs)\n        preds_list.append(prediction)\n    mean_prediction = np.stack(preds_list, axis=-1).mean(axis=-1)\n    print(f\"mean prediction array shape={mean_prediction.shape}\")\n    submission = pd.read_csv(path_sample_submission_file)\n    assert submission.shape[0] == mean_prediction.shape[0], \"unexpected behavior.code fix required\"\n    submission.iloc[:, 1:] = mean_prediction\n    if not os.path.isdir(path_submissions_dir):\n        os.mkdir(path_submissions_dir)\n    submission.to_csv(os.path.join(path_submissions_dir, f\"{output_file}.csv\"), index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_paths = os.listdir(path_trained_models,)\nmodel_paths = [os.path.join(path_trained_models, mpath) for mpath in model_paths]\nensemble_models(model_paths, \"submission\", test_batch_size=16)\nprint(\"done\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!rm -rf ./test_resized/","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}