{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install timm==0.4.5\n!pip install torch==1.7.0\n!pip install pytorch_lightning==1.2.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import timm\nimport torch\nimport pytorch_lightning\n\nassert timm.__version__ == '0.4.5'\nassert torch.__version__ == '1.7.0'\nassert pytorch_lightning.__version__ == '1.2.0'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# CONSTANTS"},{"metadata":{"trusted":true},"cell_type":"code","source":"from pathlib import Path\n\nIMAGE_SIZE = 512\n\nTARGET_COLS = ['ETT - Abnormal', 'ETT - Borderline',\n       'ETT - Normal', 'NGT - Abnormal', 'NGT - Borderline',\n       'NGT - Incompletely Imaged', 'NGT - Normal', 'CVC - Abnormal',\n       'CVC - Borderline', 'CVC - Normal', 'Swan Ganz Catheter Present',\n       ]\n\nCOLS = ['StudyInstanceUID','ETT - Abnormal', 'ETT - Borderline', 'ETT - Normal',\n                 'NGT - Abnormal', 'NGT - Borderline', 'NGT - Incompletely Imaged', 'NGT - Normal', \n                 'CVC - Abnormal', 'CVC - Borderline', 'CVC - Normal',\n                 'Swan Ganz Catheter Present']\n\nCOLOR_MAP = {'ETT - Abnormal': (255, 0, 0),\n             'ETT - Borderline': (0, 255, 0),\n             'ETT - Normal': (0, 0, 255),\n             'NGT - Abnormal': (255, 255, 0),\n             'NGT - Borderline': (255, 0, 255),\n             'NGT - Incompletely Imaged': (0, 255, 255),\n             'NGT - Normal': (128, 0, 0),\n             'CVC - Abnormal': (0, 128, 0),\n             'CVC - Borderline': (0, 0, 128),\n             'CVC - Normal': (128, 128, 0),\n             'Swan Ganz Catheter Present': (128, 0, 128),\n            }\n\nDATADIR = Path('../input/ranzcr-clip-catheter-line-classification')\n\nTRAINDIR = DATADIR.joinpath('train')\n\nTESTDIR = '../input/ranzcr-clip-catheter-line-classification/test'\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# clean-up previous logs:\n\nimport os\nimport shutil\n\noutput_files = os.listdir('/kaggle/working')\nprint(output_files)\n\nif 'lightning_logs' in output_files:\n    shutil.rmtree('/kaggle/working/lightning_logs')\n\noutput_files = os.listdir('/kaggle/working')\n\nassert not 'lightning_logs' in output_files","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# UTILS"},{"metadata":{"trusted":true},"cell_type":"code","source":"from albumentations import (\n    Compose, OneOf, Normalize, Resize, RandomResizedCrop, RandomCrop, HorizontalFlip, VerticalFlip, \n    RandomBrightness, RandomContrast, RandomBrightnessContrast, Rotate, ShiftScaleRotate, Cutout, \n    IAAAdditiveGaussianNoise, Transpose, HueSaturationValue, CoarseDropout\n    )\nfrom albumentations.pytorch import ToTensorV2\nfrom albumentations import ImageOnlyTransform","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def check_exists(path):\n    if os.path.exists(path):\n        raise ValueError(\"This path already exists\")\n        \n        \ndef get_transforms(*, data):\n    \n    if data == 'train':\n        return Compose([\n            #Resize(IMAGE_SIZE, ),\n            RandomResizedCrop(IMAGE_SIZE, IMAGE_SIZE, scale=(0.85, 1.0)),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            RandomBrightnessContrast(p=0.2, brightness_limit=(-0.2, 0.2), contrast_limit=(-0.2, 0.2)),\n            HueSaturationValue(p=0.2, hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2),\n            ShiftScaleRotate(p=0.2, shift_limit=0.0625, scale_limit=0.2, rotate_limit=20),\n            CoarseDropout(p=0.2),\n            Cutout(p=0.2, max_h_size=16, max_w_size=16, fill_value=(0., 0., 0.), num_holes=16),\n            Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225],\n            ),\n            ToTensorV2(),\n        ], additional_targets={'image_annot': 'image'})\n\n    elif data == 'valid' or data == 'test':\n        return Compose([\n            Resize(IMAGE_SIZE, IMAGE_SIZE),\n            Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225],\n            ),\n            ToTensorV2(),\n        ])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# DATASET"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport numpy as np\nimport random\nimport ast\nimport cv2\nimport torch\nfrom torch.utils.data import Dataset\nfrom torchvision import transforms\nfrom PIL import Image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class S1_Dataset(Dataset):\n    \"\"\"\n    Y.Nakama https://www.kaggle.com/yasufuminakama\n    \"\"\"\n    def __init__(self, df, df_annotations, annot_size=50, transform=None):\n        self.df = df\n        self.df_annotations = df_annotations\n        self.annot_size = annot_size\n        self.file_names = df['StudyInstanceUID'].values\n        self.labels = df[TARGET_COLS].values\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        file_name = self.file_names[idx]\n        file_path = f'{TRAINDIR}/{file_name}.jpg'\n        image = cv2.imread(file_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        query_string = f\"StudyInstanceUID == '{file_name}'\"\n        df = self.df_annotations.query(query_string)\n        for i, row in df.iterrows():\n            label = row[\"label\"]\n            data = np.array(ast.literal_eval(row[\"data\"]))\n            for d in data:\n                image[d[1]-self.annot_size//2:d[1]+self.annot_size//2,\n                      d[0]-self.annot_size//2:d[0]+self.annot_size//2,\n                      :] = COLOR_MAP[label]\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n        label = torch.tensor(self.labels[idx]).float()\n        #print('\\tS1 image shape:', image.shape)\n        return file_name, image, label\n    \n    \nclass S2_Dataset(Dataset):\n    \"\"\"\n    Y.Nakama https://www.kaggle.com/yasufuminakama\n    \"\"\"\n    def __init__(self, df, df_annotations, use_annot=False, annot_size=50, transform=None):\n        self.df = df\n        self.df_annotations = df_annotations\n        self.use_annot = use_annot\n        self.annot_size = annot_size\n        self.file_names = df['StudyInstanceUID'].values\n        self.labels = df[TARGET_COLS].values\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        file_name = self.file_names[idx]\n        file_path = f'{TRAINDIR}/{file_name}.jpg'\n        image = cv2.imread(file_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        labels = torch.tensor(self.labels[idx]).float()\n        if self.use_annot:\n            image_annot = image.copy()\n            query_string = f\"StudyInstanceUID == '{file_name}'\"\n            df = self.df_annotations.query(query_string)\n            for i, row in df.iterrows():\n                label = row[\"label\"]\n                data = np.array(ast.literal_eval(row[\"data\"]))\n                for d in data:\n                    image_annot[d[1]-self.annot_size//2:d[1]+self.annot_size//2,\n                                d[0]-self.annot_size//2:d[0]+self.annot_size//2,\n                                :] = COLOR_MAP[label]\n            if self.transform:\n                augmented = self.transform(image=image, image_annot=image_annot)\n                image = augmented['image']\n                image_annot = augmented['image_annot']\n            return file_name, image, image_annot, labels\n        else:\n            if self.transform:\n                augmented = self.transform(image=image)\n                image = augmented['image']\n            return file_name, image, labels\n    \n\nclass S3_Dataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.file_names = df['StudyInstanceUID'].values\n        self.labels = df[TARGET_COLS].values\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        file_name = self.file_names[idx]\n        file_path = f'{TRAINDIR}/{file_name}.jpg'\n        image = cv2.imread(file_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n        label = torch.tensor(self.labels[idx]).float()\n        return file_name, image, label\n    \nclass TestDataset(Dataset):\n    \n    def __init__(self, transform):\n        self.images = np.array(os.listdir(TESTDIR))\n        self.transform = transform\n                                \n    def __getitem__(self, index):\n        ID = self.images[index]\n        file_name = os.path.join(TESTDIR, ID)\n        image = cv2.imread(file_name)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n        return ID, image\n    \n    def __len__(self):\n        return len(self.images)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# DataModule"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport subprocess\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nfrom torch.optim import Adam\nfrom torch.utils.data import DataLoader\n\nimport torch.nn.functional as F\nimport pytorch_lightning as pl\n\nimport timm\nimport gc\n\n#from IPython import embed","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CLiP_DataModule(pl.LightningDataModule):\n    \n    def __init__(self, hparams, stage):\n        super().__init__()\n        self.hparams = hparams\n        self.expected_files = ['train', 'train_annotations.csv', 'test']\n        self.stage = stage\n        \n    def prepare_data(self):\n        files_on_disk = os.listdir(DATADIR)\n        if not all(file in files_on_disk for file in self.expected_files):\n            kaggle.api.authenticate()\n            kaggle.api.dataset_download_files(\n                'ranzcr-clip-catheter-line-classification', path=DATADIR, unzip=True)\n        if not 'resnet200d_ra2-bdba9bf9.pth' in files_on_disk:\n            subprocess.run(['wget', '-P', DATADIR, 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/resnet200d_ra2-bdba9bf9.pth'])\n\n    def setup(self, step):\n        df = pd.read_csv(DATADIR.joinpath('train.csv'))\n        \n        annots_df = pd.read_csv(DATADIR.joinpath('train_annotations.csv'))\n        \n        train_annots = annots_df.sample(frac=.7, random_state=22117).reset_index(drop=True)\n        \n        val_annots = annots_df[~annots_df['StudyInstanceUID'].isin(train_annots['StudyInstanceUID'])]\n        \n        train_df = df[df['StudyInstanceUID'].isin(train_annots['StudyInstanceUID'])]\n        \n        val_df = df[df['StudyInstanceUID'].isin(val_annots['StudyInstanceUID'])]\n        \n        if self.stage == 1:    \n            self.train_ds = S1_Dataset(train_df, train_annots, transform=get_transforms(data='train'))\n            self.val_ds = S1_Dataset(val_df, val_annots, transform=get_transforms(data='valid'))\n        \n        elif self.stage == 2:\n            self.train_ds = S2_Dataset(train_df, train_annots, use_annot=True, transform=get_transforms(data='train'))\n            self.val_ds = S2_Dataset(val_df, val_annots, use_annot=False, transform=get_transforms(data='valid'))\n            \n        elif self.stage == 3:\n            self.train_ds = S3_Dataset(train_df, transform=get_transforms(data='train'))\n            self.val_ds = S3_Dataset(val_df, transform=get_transforms(data='valid'))\n        \n        self.test_ds = TestDataset(transform=get_transforms(data='test'))\n\n    def train_dataloader(self):\n        return DataLoader(self.train_ds, batch_size=self.hparams.batch_size, num_workers=self.hparams.num_workers)\n    \n    def val_dataloader(self):\n        return DataLoader(self.val_ds, batch_size=self.hparams.batch_size, num_workers=self.hparams.num_workers)\n    \n    def test_dataloader(self):\n        return DataLoader(self.test_ds, batch_size=self.hparams.batch_size, num_workers=self.hparams.num_workers) ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Stage1"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CustomResNet200D(nn.Module):\n    \n    def __init__(self, hparams, pretraining, model_name='resnet200d'):\n        super().__init__()\n        self.hparams = hparams\n        self.model = timm.create_model(model_name)\n        if pretraining == 'image-net':\n            pretrained_path = '../input/resnet200d-pretrained-weight/resnet200d_ra2-bdba9bf9.pth'\n            self.model.load_state_dict(torch.load(pretrained_path))\n            print(f'load {model_name} pretrained model')\n        if pretraining == 'Nakama':\n            pretrained_path = hparams.restore\n            self.model.load_state_dict(torch.load(pretrained_path), strict=False)\n        n_features = self.model.fc.in_features\n        self.model.global_pool = nn.Identity()\n        self.model.fc = nn.Identity()\n        self.pooling = nn.AdaptiveAvgPool2d(1)\n        self.fc = nn.Linear(n_features, 11)\n\n    def forward(self, x):\n        bs = x.size(0)\n        features = self.model(x)\n        pooled_features = self.pooling(features).view(bs, -1)\n        output = self.fc(pooled_features)\n        return features, pooled_features, output\n    \n    \nclass Teacher(pl.LightningModule):\n    \n    def __init__(self, hparams):\n        super(Teacher, self).__init__()\n        self.hparams = hparams\n        self.teacher = CustomResNet200D(hparams, pretraining='image-net')\n        self.sig = nn.Sigmoid()\n\n    def forward(self, x):      \n        fts, pooled_fts, output = self.teacher.forward(x)\n        return fts, pooled_fts, output\n    \n    def training_step(self, batch, batch_idx):\n        p, xa, y = batch\n        assert xa.shape[1] == 3\n        fts, pooled_fts, y_hat = self.forward(xa)\n        train_loss = self.loss(y_hat, y)\n        self.log('train_loss', train_loss, prog_bar=True, logger=True, on_step=True, on_epoch=True)\n        return train_loss\n        \n    def validation_step(self, batch, batch_idx):\n        p, xa, y = batch\n        assert xa.shape[1] == 3\n        fts, pooled_fts, y_hat = self.forward(xa)\n        val_loss = self.loss(y_hat, y)\n        self.log('val_loss', val_loss, prog_bar=True, logger=True, on_step=True, on_epoch=True)\n        gc.collect() #pytorch/issues/40911\n        return val_loss\n    \n    def loss(self, y_hat, y):\n        loss = F.binary_cross_entropy_with_logits(y_hat, y)\n        return loss\n    \n    def configure_optimizers(self):\n        optimiser = Adam(\n                self.parameters(), lr=self.hparams.lr,\n                betas=(0.9, 0.999), eps=1e-8, \n                #weight_decay=self.hparams.weight_decay,\n                amsgrad=False\n                )\n        \n        scheduler = {\n                            'scheduler': torch.optim.lr_scheduler.ReduceLROnPlateau(\n                                    optimiser,\n                                    mode='min',\n                                    factor=0.1, \n                                    patience=2, \n                                    verbose=True,\n                                    threshold=1e-04,\n                                    threshold_mode='rel',\n                                    cooldown=0, \n                                    min_lr=1e-12,\n                                    eps=1e-13),\n                            'monitor': 'val_loss_epoch'\n                            }\n        return [optimiser], [scheduler]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from datetime import datetime\nfrom torch.utils.data import DataLoader, random_split\n\nimport pytorch_lightning as pl\nfrom pytorch_lightning import Trainer\nfrom pytorch_lightning.callbacks import ModelCheckpoint, EarlyStopping\nfrom pytorch_lightning.utilities.parsing import AttributeDict\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def s1(hparams):\n    \n    early_stop_callback = EarlyStopping(\n            monitor='val_loss_epoch',\n            min_delta=1,\n            patience=4,\n            verbose=True,\n            mode='min',\n            strict=False\n            )\n        \n    date_time = datetime.now().strftime(\"%Y-%m-%d\")\n    \n    ckpt_callback = ModelCheckpoint(\n            dirpath=None,\n            monitor='val_loss_epoch',\n            verbose=1,\n            save_top_k=5,\n            save_weights_only=True,\n            mode='min',\n            period=1,\n            filename='s1_{epoch}-{train_loss_epoch:.3f}-{val_loss_epoch:.3f}'\n            )   \n    \n    trainer = Trainer(\n        accelerator=hparams.accel,\n        accumulate_grad_batches=hparams.grad_cum,\n        amp_backend='native',\n        auto_lr_find=hparams.autolr,\n        auto_scale_batch_size=hparams.auto_scale_batch_size,\n        benchmark=True,\n        callbacks=[ckpt_callback, early_stop_callback],\n        check_val_every_n_epoch=hparams.check_val_n,\n        gpus=hparams.gpus,\n        max_epochs=hparams.max_epochs,\n        \n        overfit_batches=6,\n        \n        precision=hparams.precision,\n        progress_bar_refresh_rate=50,\n        )\n    \n    clip_data = CLiP_DataModule(hparams, stage=1)\n\n    model = Teacher(hparams)\n    \n    trainer.fit(model, clip_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"hparams = AttributeDict(\n        {\n            'accel': None,\n            'autolr': True, \n            'auto_scale_batch_size': 'binsearch',\n            'batch_size': 1,\n            'check_val_n': 1,\n            'dev': False,\n            'gpus': -1, # change to -1 if gpus enabled\n            'grad_cum': 16,\n            'lr': 0.0001,\n            'max_epochs': 6,\n            'num_workers': 0,\n            'train_path': TRAINDIR,\n            'pl_ver': pl.__version__,\n            'precision': 16,\n            'seed': 22117,\n            'stage': 1,\n            #'weight_decay': 1e-07\n            })\n    \ns1(hparams)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# STAGE 2"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Student(pl.LightningModule):\n    \n    def __init__(self, hparams):\n        super(Student, self).__init__()\n        self.hparams = hparams\n        self.teacher = CustomResNet200D(hparams, pretraining='Nakama')            \n        self.student = CustomResNet200D(hparams, pretraining='image-net')\n\n    def forward(self, x, x_annot):   \n        teacher_fts, _, teacher_output = self.teacher.forward(x_annot)\n        student_fts, _, student_output = self.student.forward(x)\n        return teacher_fts, teacher_output, student_fts, student_output\n    \n    def training_step(self, batch, batch_idx):\n        p, x, x_annot, y = batch\n        teacher_fts, teacher_ouput, student_fts, student_output = self.forward(x, x_annot)\n        train_loss = self.student_teacher_loss(student_output, y, student_fts, teacher_fts)\n        self.log('train_loss', train_loss, prog_bar=True, logger=True, on_step=True, on_epoch=True)\n        return train_loss\n        \n    def validation_step(self, batch, batch_idx):\n        #embed()\n        p, x, y = batch\n        student_fts, _, student_output = self.student.forward(x)\n        val_loss = self.bce_loss(student_output, y)\n        self.log('val_loss', val_loss, prog_bar=True, logger=True, on_step=True, on_epoch=True)\n        gc.collect() #pytorch/issues/40911\n        return val_loss\n    \n    def student_teacher_loss(self, y_hat, y, student_fts, teacher_fts):\n        mse_loss = F.mse_loss(student_fts.view(-1), teacher_fts.view(-1))\n        bce_loss = F.binary_cross_entropy_with_logits(y_hat, y) \n        loss = self.hparams.loss_w[0] * mse_loss + self.hparams.loss_w[1] * bce_loss\n        return loss\n    \n    def bce_loss(self, y_hat, y):\n        loss = F.binary_cross_entropy_with_logits(y_hat, y)\n        return loss\n    \n    def configure_optimizers(self):\n        optimiser = Adam(\n                self.parameters(), lr=self.hparams.lr,\n                betas=(0.9, 0.999), eps=1e-8, \n                #weight_decay=self.hparams.weight_decay,\n                amsgrad=False\n                )\n        \n        scheduler = {\n                            'scheduler': torch.optim.lr_scheduler.ReduceLROnPlateau(\n                                    optimiser,\n                                    mode='min',\n                                    factor=0.1, \n                                    patience=2, \n                                    verbose=True,\n                                    threshold=1e-04,\n                                    threshold_mode='rel',\n                                    cooldown=0, \n                                    min_lr=1e-12,\n                                    eps=1e-13),\n                            'monitor': 'val_loss_epoch'\n                            }\n        return [optimiser], [scheduler]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"last_ckpt = os.listdir('./lightning_logs/version_0/checkpoints')[-1]\ns1_ckpt_path = os.path.join('./lightning_logs/version_0/checkpoints', last_ckpt)\nprint(s1_ckpt_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def s2(hparams):\n    \n    early_stop_callback = EarlyStopping(\n            monitor='val_loss_epoch',\n            min_delta=1,\n            patience=4,\n            verbose=True,\n            mode='min',\n            strict=False\n            )\n        \n    date_time = datetime.now().strftime(\"%Y-%m-%d\")\n    \n    ckpt_callback = ModelCheckpoint(\n            dirpath=None,\n            monitor='val_loss_epoch',\n            verbose=1,\n            save_top_k=5,\n            save_weights_only=True,\n            mode='min',\n            period=1,\n            filename='s2_{epoch}-{train_loss_epoch:.3f}-{val_loss_epoch:.3f}'\n            ) \n    \n    trainer = Trainer(\n        accelerator=hparams.accel,\n        accumulate_grad_batches=hparams.grad_cum,\n        amp_backend='native',\n        auto_lr_find=hparams.autolr,\n        auto_scale_batch_size=hparams.auto_scale_batch_size,\n        benchmark=True,\n        callbacks=[ckpt_callback, early_stop_callback],\n        check_val_every_n_epoch=hparams.check_val_n,\n        gpus=hparams.gpus,\n        max_epochs=hparams.max_epochs,\n        \n        overfit_batches=6,\n\n        precision=hparams.precision,\n        progress_bar_refresh_rate=50,\n        )\n    \n    clip_data = CLiP_DataModule(hparams, stage=2)\n\n    model = Student(hparams)\n     \n    trainer.fit(model, clip_data)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"hparams = AttributeDict(\n        {\n            'accel': None,\n            'autolr': True,\n            'auto_scale_batch_size': 'binsearch',\n            'batch_size': 1,\n            'check_val_n': 1,\n            'dev': False,\n            'gpus': -1, # change to GPU if enabled\n            'grad_cum': 16,\n            'loss_w': (0.5, 1),\n            'lr': 0.0001,\n            'max_epochs': 6,\n            'num_workers': 0,\n            'train_path': TRAINDIR,\n            'pl_ver': pl.__version__,\n            'precision': 16,\n            'restore': s1_ckpt_path,\n            'seed': 22117,\n            'stage': 2,\n            #'weight_decay': 1e-07\n            })\n\ns2(hparams)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# STAGE 3"},{"metadata":{"trusted":true},"cell_type":"code","source":"class YasufumiNet(pl.LightningModule):\n    \n    def __init__(self, hparams):\n        super(YasufumiNet, self).__init__()\n        # save parameters\n        self.hparams = hparams\n        self.net = CustomResNet200D(hparams, pretraining='Nakama')\n\n    def forward(self, x):      \n        x = self.net(x)\n        return x \n    \n    def training_step(self, batch, batch_idx):\n        p, x, y = batch\n        fts, pooled_fts, y_hat = self.forward(x)\n        train_loss = self.loss(y_hat, y)\n        self.log('train_loss', train_loss, prog_bar=True, logger=True, on_step=True, on_epoch=True)\n        return train_loss\n        \n    def validation_step(self, batch, batch_idx):\n        p, x, y = batch\n        fts, pooled_fts, y_hat = self.forward(x)\n        val_loss = self.loss(y_hat, y)\n        self.log('val_loss', val_loss, prog_bar=True, logger=True, on_step=True, on_epoch=True)\n        gc.collect() #pytorch/issues/40911\n        return val_loss\n    \n    def loss(self, y_hat, y):\n        loss = F.binary_cross_entropy_with_logits(y_hat, y)\n        return loss\n    \n    def configure_optimizers(self):\n        optimiser = Adam(\n                self.parameters(), lr=self.hparams.lr,\n                betas=(0.9, 0.999), eps=1e-8, \n                #weight_decay=self.hparams.weight_decay,\n                amsgrad=False\n                )\n        \n        scheduler = {\n                            'scheduler': torch.optim.lr_scheduler.ReduceLROnPlateau(\n                                    optimiser,\n                                    mode='min',\n                                    factor=0.1, \n                                    patience=2, \n                                    verbose=True,\n                                    threshold=1e-04,\n                                    threshold_mode='rel',\n                                    cooldown=0, \n                                    min_lr=1e-12,\n                                    eps=1e-13),\n                            'monitor': 'val_loss_epoch'\n                            }\n        return [optimiser], [scheduler]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"last_ckpt = os.listdir('./lightning_logs/version_1/checkpoints')[-1]\ns2_ckpt_path = os.path.join('./lightning_logs/version_1/checkpoints', last_ckpt)\nprint(s2_ckpt_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def s3(hparams):\n    \n    early_stop_callback = EarlyStopping(\n            monitor='val_loss_epoch',\n            min_delta=1,\n            patience=4,\n            verbose=True,\n            mode='min',\n            strict=False\n            )\n        \n    date_time = datetime.now().strftime(\"%Y-%m-%d\")\n    \n    ckpt_callback = ModelCheckpoint(\n            dirpath=None,\n            monitor='val_loss_epoch',\n            verbose=1,\n            save_top_k=5,\n            save_weights_only=True,\n            mode='min',\n            period=1,\n            filename='s3_{epoch}-{train_loss_epoch:.3f}-{val_loss_epoch:.3f}'\n            ) \n    \n    trainer = Trainer(\n        accelerator=hparams.accel,\n        accumulate_grad_batches=hparams.grad_cum,\n        amp_backend='native',\n        auto_lr_find=hparams.autolr,\n        auto_scale_batch_size=hparams.auto_scale_batch_size,\n        benchmark=True,\n        callbacks=[ckpt_callback, early_stop_callback],\n        check_val_every_n_epoch=hparams.check_val_n,\n        gpus=hparams.gpus,\n        max_epochs=hparams.max_epochs,\n        \n        overfit_batches=10,\n        \n        precision=hparams.precision,\n        profiler=False,\n        progress_bar_refresh_rate=50,\n        )\n    \n    clip_data = CLiP_DataModule(hparams, stage=3)\n\n    model = YasufumiNet(hparams)\n     \n    trainer.fit(model, clip_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"hparams = AttributeDict(\n        {\n            'accel': None,\n            'autolr': True,\n            'auto_scale_batch_size': 'binsearch',\n            'batch_size': 1,\n            'check_val_n': 1,\n            'dev': False,\n            'gpus': -1,\n            'grad_cum': 16,\n            'loss_w': (0.5, 1),\n            'lr': 0.0001,\n            'max_epochs': 6,\n            'num_workers': 0,\n            'train_path': TRAINDIR,\n            'pl_ver': pl.__version__,\n            'precision': 16,\n            'restore': s2_ckpt_path,\n            'seed': 22117,\n            'stage': 3,\n            #'weight_decay': 1e-07\n            })\n    \ns3(hparams)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# INFERENCE"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport gc\nfrom tqdm import tqdm\n#from IPython import embed\n\ndef load_ensemble(path, n):\n    paths = sorted([os.path.join(path, x) for x in os.listdir(path)]) \n    keys = []\n    for i in range(n):\n        key = 'model' + str(i+1)\n        keys.append(key)\n        \n    d = dict.fromkeys(keys)\n    for checkpoint_path, key in zip(paths, d.keys()):\n        hparams = AttributeDict(\n            {'restore': checkpoint_path})\n        model = YasufumiNet(hparams)\n        d[key] = model\n    return d\n\n\ndef inference(args):\n    \"\"\"\n    Target vector:\n    # 'ETT - Abnormal',\n    # 'ETT - Borderline',\n    # 'ETT - Normal',\n    # 'NGT - Abnormal',\n    # 'NGT - Borderline',\n    # 'NGT - Incompletely Imaged',\n    # 'NGT - Normal',\n    # 'CVC - Abnormal',\n    # 'CVC - Borderline',\n    # 'CVC - Normal',\n    # 'Swan Ganz Catheter Present',\n    \"\"\"\n    \n    csv_path = os.path.join(args.csv_out_path, args.version)\n    #check_exists(csv_path)\n    if not os.path.exists(args.csv_out_path):\n        os.makedirs(args.csv_out_path)\n            \n    ensemble_dic = load_ensemble(args.ckpt_dir, args.best_n)\n    \n    test_loader = DataLoader(TestDataset(transform=get_transforms(data='test')), batch_size=1, num_workers=0)\n        \n    sigmoid = torch.nn.Sigmoid()\n    \n    df = pd.DataFrame(columns=COLS)\n    \n    for batch in tqdm(test_loader, desc='infering ensemble'):\n        ID, x = batch\n        y_hats = []\n        \n        for model in ensemble_dic.values():\n            model.eval()\n            _, _, output = model(x)\n            positive_probability = sigmoid(output)\n            y_hats.append(positive_probability.clone().detach())\n        \n        if args.voting == 'mean':\n            probs = torch.cat(y_hats).mean(axis=0)\n            \n        elif args.voting == 'max':\n            probs = torch.cat(y_hats).max(-2)\n            \n        preds = probs.detach().apply_(lambda x: x > .5).type(torch.int).tolist()\n        \n        d = dict({'StudyInstanceUID': ID[0]})\n        for k, v in zip(COLS[1:], preds):\n            d[k] = v\n            \n        df = df.append(d, ignore_index=True)\n    \n        df.to_csv(csv_path, index=False)\n        gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"parent_path = './lightning_logs/version_2/checkpoints'\ncheckpoint = sorted(os.listdir(parent_path))[-1]\npath = os.path.join('./lightning_logs/version_2/checkpoints', checkpoint)\nprint(path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"args = AttributeDict(\n    {\n        'best_n': 3,\n        'ckpt_dir': parent_path,\n        'csv_out_path': './',\n        'version': 'submission.csv',\n        'voting': 'mean',\n        'test_dir': TESTDIR\n        }\n)\n    \ninference(args)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}