{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"#!pip install '../input/python-packages/timm-0.3.4-py3-none-any (1).whl'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install '../input/effdet-latestvinbigdata-wbf-fused/timm-0.3.4-py3-none-any.whl'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import timm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import random\nimport os\nimport pandas as pd\nimport numpy as np\n\n\n#from efficientnet_pytorch import EfficientNet\n\n\n\n\nimport cv2\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom torch.cuda.amp import autocast, GradScaler\nfrom torch.nn.modules.loss import _WeightedLoss\nimport torch.nn.functional as F\n\nimport torchvision.models as models\n\nimport torch\nfrom torch import nn\nimport torch.nn.functional as F\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, CosineAnnealingLR, ReduceLROnPlateau\nfrom torch.optim import Adam, SGD\n\n\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import GroupKFold, StratifiedKFold\n\n\nfrom albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize\n)\n\nfrom albumentations.pytorch import ToTensorV2\nfrom albumentations import ImageOnlyTransform\nfrom tqdm import tqdm\nimport time\nfrom sklearn.metrics import log_loss\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\nBASE_DIR = \"../input/ranzcr-clip-catheter-line-classification\"\n\n\ntrain = pd.read_csv(os.path.join(BASE_DIR,'train.csv'))\ntrain.head()\n\ntarget_cols = ['ETT - Abnormal', 'ETT - Borderline', 'ETT - Normal', 'NGT - Abnormal', \n               'NGT - Borderline', 'NGT - Incompletely Imaged', 'NGT - Normal', 'CVC - Abnormal',\n               'CVC - Borderline', 'CVC - Normal', 'Swan Ganz Catheter Present']\n\n\n\nCFG = {\n    'fold_num': 5,\n    'scheduler':'CosineAnnealingLR',\n    'T_max':6,\n    'seed': 719,\n    'model_name': 'resnet200d',\n    'gradient_accumulation_steps':1,\n    'max_grad_norm':1000,\n    'img_size': 256,\n    'epochs': 1,\n    'train_bs': 16,\n    'valid_bs': 32,\n    'T_0': 10,\n    'lr': 1e-4,\n    'min_lr': 1e-6,\n    'weight_decay':1e-6,\n    'num_workers': 4,\n    'accum_iter': 2,\n    'verbose_step': 1,\n    'device': 'cuda:0',\n    'use_amp' : True\n}\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n    \ndef macro_multilabel_auc(label, pred):\n    aucs = []\n    for i in range(len(target_cols)):\n        aucs.append(roc_auc_score(label[:, i], pred[:, i]))\n    print(np.round(aucs, 4))\n    return np.mean(aucs)\n    \ndef get_score(y_true, y_pred):\n    scores = []\n    for i in range(y_true.shape[1]):\n        score = roc_auc_score(y_true[:,i], y_pred[:,i])\n        scores.append(score)\n    avg_score = np.mean(scores)\n    return avg_score, scores\n    \n    \ndef get_transforms(*, data):\n    \n    if data == 'train':\n        return Compose([\n            RandomResizedCrop(CFG['img_size'], CFG['img_size']),\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            ShiftScaleRotate(p=0.5),\n            HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n            RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            CoarseDropout(p=0.5),\n            Cutout(p=0.5),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n    \n\n    elif data == 'valid':\n        return Compose([\n            CenterCrop(CFG['img_size'], CFG['img_size'], p=1.),\n            Resize(CFG['img_size'], CFG['img_size']),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n    \n    \n    elif data == 'test':\n        return Compose([\n            CenterCrop(CFG['img_size'], CFG['img_size'], p=1.),\n            Resize(CFG['img_size'], CFG['img_size']),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n        \n\n    \n    \nclass TrainDataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.file_names = df['StudyInstanceUID'].values\n        self.labels = df[target_cols].values\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        file_name = self.file_names[idx]\n        file_path = f'{BASE_DIR}/train/{file_name}.jpg'\n        image = cv2.imread(file_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n        label = torch.tensor(self.labels[idx]).float()\n        return image, label\n    \n\nclass TestDataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.file_names = df['StudyInstanceUID'].values\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        file_name = self.file_names[idx]\n        file_path = f'{BASE_DIR}/test/{file_name}.jpg'\n        image = cv2.imread(file_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n        return image\n    \n\n\ndef prepare_dataloader(df, trn_idx, val_idx):\n    train_ = df.loc[trn_idx,:].reset_index(drop=True)\n    valid_ = df.loc[val_idx,:].reset_index(drop=True)\n        \n    train_ds = TrainDataset(train_, transform=get_transforms(data='train'))\n    valid_ds = TrainDataset(valid_, transform=get_transforms(data='valid'))\n    \n    train_loader = torch.utils.data.DataLoader(\n        train_ds,\n        batch_size=CFG['train_bs'],\n        pin_memory=False,\n        drop_last=False,\n        shuffle=True,        \n        num_workers=CFG['num_workers'],\n    )\n    val_loader = torch.utils.data.DataLoader(\n        valid_ds, \n        batch_size=CFG['valid_bs'],\n        num_workers=CFG['num_workers'],\n        shuffle=False,\n        pin_memory=False,\n    )\n    return train_loader, val_loader\n\nclass Classifier(nn.Module):\n    def __init__(self, model_name, n_class, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained)\n        n_features = self.model.fc.in_features\n        self.model.global_pool = nn.Identity()\n        self.model.fc = nn.Identity()\n        self.pooling = nn.AdaptiveAvgPool2d(1)\n        self.fc = nn.Linear(n_features, n_class)\n\n    def forward(self, x):\n        bs = x.size(0)\n        features = self.model(x)\n        pooled_features = self.pooling(features).view(bs, -1)\n        output = self.fc(pooled_features)\n        return output\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"   \ndef train_one_epoch(train_loader, model, criterion, optimizer, epoch, scheduler, device):\n    print('Training')\n    model.train()\n    scaler = GradScaler()\n    losses = []\n    for i, (image, label) in enumerate(tqdm(train_loader)):\n        image, target = image.to(device).float(), label.to(device).float()\n        if CFG['use_amp']:\n            with torch.cuda.amp.autocast():\n                logits = model(image)\n                loss = criterion(logits, target)\n            scaler.scale(loss).backward()\n            scaler.step(optimizer)\n            scaler.update()\n            optimizer.zero_grad()\n        else:\n            logits = model(image)\n            loss = criterion(logits, target)\n            loss.backward()\n            optimizer.step()\n            optimizer.zero_grad()\n        losses.append(loss.item()) \n        smooth_loss = np.mean(losses[-30:])\n\n    loss_train = np.mean(losses)\n    return loss_train\n\n\n        \ndef valid_one_epoch(val_loader, model, criterion, device):\n    print('Validating')\n    model.eval()\n    bar = tqdm(val_loader)\n\n    PROB = []\n    TARGETS = []\n    losses = []\n    PREDS = []\n    \n    with torch.no_grad():\n        for batch_idx, (images, targets) in enumerate(bar):\n\n            images, targets = images.to(device), targets.to(device)\n            logits = model(images)\n            PREDS += [logits.sigmoid()]\n            TARGETS += [targets.detach().cpu()]\n            loss = criterion(logits, targets)\n            print(loss.item())\n            losses.append(loss.item())\n            smooth_loss = np.mean(losses[-30:])\n            print(smooth_loss)\n            bar.set_description(f'loss: {loss.item():.5f}, smth: {smooth_loss:.5f}')\n            \n    PREDS = torch.cat(PREDS).cpu().numpy()\n    TARGETS = torch.cat(TARGETS).cpu().numpy()\n    roc_auc = macro_multilabel_auc(TARGETS, PREDS)\n    loss_valid = np.mean(losses)\n    return loss_valid, roc_auc\n\n\ndef main():\n        \n    seed_everything(CFG['seed'])\n    \n    folds = StratifiedKFold(n_splits=CFG['fold_num'], shuffle=True, random_state=CFG['seed']).split(np.arange(train.shape[0]), train['PatientID'].values)\n    \n    \n    for fold, (trn_idx, val_idx) in enumerate(folds):\n        print('Training with {} started'.format(fold))\n        print(len(trn_idx), len(val_idx))\n        valid_ = train.loc[val_idx,:].reset_index(drop=True)\n        valid_labels = valid_[target_cols].values\n        train_loader, val_loader = prepare_dataloader(train, trn_idx, val_idx)\n\n        def get_scheduler(optimizer):\n            if CFG['scheduler']=='ReduceLROnPlateau':\n                scheduler = ReduceLROnPlateau(optimizer, mode='min', factor=CFG['factor'], patience=CFG['patience'], verbose=True, eps=CFG['eps'])\n            elif CFG['scheduler']=='CosineAnnealingLR':\n                scheduler = CosineAnnealingLR(optimizer, T_max=CFG['T_max'], eta_min=CFG['min_lr'], last_epoch=-1)\n            elif CFG['scheduler']=='CosineAnnealingWarmRestarts':\n                scheduler = CosineAnnealingWarmRestarts(optimizer, T_0=CFG['T_0'], T_mult=1, eta_min=CFG['min_lr'], last_epoch=-1)\n            return scheduler\n\n        # ====================================================\n        # model & optimizer\n        # ====================================================\n\n        model = Classifier(CFG['model_name'],len(target_cols), pretrained=False)\n        model.to(device)\n\n        optimizer = Adam(model.parameters(), lr=CFG['lr'], weight_decay=CFG['weight_decay'], amsgrad=False)\n        scheduler = get_scheduler(optimizer)\n\n        # ====================================================\n        # loop\n        # ====================================================\n        criterion = nn.BCEWithLogitsLoss()\n\n        best_score = 0.\n        best_loss = np.inf\n\n        for epoch in range(CFG['epochs']):\n\n            start_time = time.time()\n\n            # train\n            avg_loss = train_one_epoch(train_loader, model, criterion, optimizer, epoch, scheduler, device)\n\n            # eval\n            avg_val_loss, roc_auc = valid_one_epoch(val_loader, model, criterion, device)\n\n            if isinstance(scheduler, ReduceLROnPlateau):\n                scheduler.step(avg_val_loss)\n            elif isinstance(scheduler, CosineAnnealingLR):\n                scheduler.step()\n            elif isinstance(scheduler, CosineAnnealingWarmRestarts):\n                scheduler.step()\n            \n            print(avg_val_loss)\n            # scoring\n            #score, scores = get_score(valid_labels, preds)\n\n            elapsed = time.time() - start_time\n\n            #LOGGER.info(f'Epoch {epoch+1} - avg_train_loss: {avg_loss:.4f}  avg_val_loss: {avg_val_loss:.4f}  time: {elapsed:.0f}s')\n            #LOGGER.info(f'Epoch {epoch+1} - Score: {score:.4f}  Scores: {np.round(scores, decimals=4)}')\n\n            \"\"\"\n            if score > best_score:\n                best_score = score\n                LOGGER.info(f'Epoch {epoch+1} - Save Best Score: {best_score:.4f} Model')\n                torch.save({'model': model.state_dict(), \n                            'preds': preds},\n                            OUTPUT_DIR+f'{CFG.model_name}_fold{fold}_best.pth')\n            \"\"\"\n\n            if avg_val_loss < best_loss:\n                best_loss = avg_val_loss\n                #LOGGER.info(f'Epoch {epoch+1} - Save Best Loss: {best_loss:.4f} Model')\n                torch.save(model.state_dict(),f\"{CFG['model_name']}_fold{fold}_best.pth\")\n\n        #check_point = torch.load(f\"{CFG['model_name']}_fold{fold}_best.pth\")\n        \n\n\n''' \n\nmodel = EfficientNetB5()\nmodel.load_state_dict(torch.load(MODEL_PATH)['model'])\nmodel.eval()\nmodels = [model.to(device)]\n\n\ntest_dataset = TestDataset(test, transform=get_transforms())\ntest_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False, \n                         num_workers=4 , pin_memory=True)\npredictions = inference(models, test_loader, device)\n\ntarget_cols = test.iloc[:, 1:12].columns.tolist()\ntest[target_cols] = predictions\ntest[['StudyInstanceUID'] + target_cols].to_csv('submission.csv', index=False)\ntest.head()\n\n'''\n\nif __name__ == '__main__':\n    main()\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('../input/ranzcr-clip-catheter-line-classification/sample_submission.csv')\n#print(test.shape)\n#test.head()\ntest_dataset = TestDataset(test, transform=get_transforms(data='test'))\ntest_loader = torch.utils.data.DataLoader(test_dataset, batch_size=CFG['valid_bs'], shuffle=False,  num_workers=24)\ndef inference_func(test_loader):\n    model.eval()\n    bar = tqdm(test_loader)\n    LOGITS = []\n    PREDS = []\n\n    with torch.no_grad():\n        for batch_idx, images in enumerate(bar):\n            x = images.to(device)\n            logits = model(x)\n            LOGITS.append(logits.cpu())\n            PREDS += [logits.sigmoid().detach().cpu()]\n        PREDS = torch.cat(PREDS).cpu().numpy()\n        LOGITS = torch.cat(LOGITS).cpu().numpy()\n    return PREDS\n\nmodel_path = ['../input/pretrainedefficient/resnet200d_fold0_best.pth',\n              '../input/pretrainedefficient/resnet200d_fold1_best.pth',\n              '../input/pretrainedefficient/resnet200d_fold2_best.pth',\n              '../input/pretrainedefficient/resnet200d_fold3_best.pth',\n              '../input/pretrainedefficient/resnet200d_fold4_best.pth'\n             ]\ntest_preds_1 = []\nfor i in range(5):\n    print('resnet200d loaded')\n    model = Classifier(CFG['model_name'], len(target_cols))\n    model = model.to(device)\n    model.load_state_dict(torch.load(model_path[i], map_location='cuda:0'))                            \n    test_preds_1 += [inference_func(test_loader)]\n\n\n    \n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\ndef inference(model, test_loader, device):\n    model.eval()\n    tk0 = tqdm(enumerate(test_loader), total=len(test_loader))\n    probs = []\n    for i, (image) in tk0:\n        images = image.to(device)\n        with torch.no_grad():\n            y_pred = model(images)\n            pred = y_pred.sigmoid().detach().cpu()\n            probs.append(pred)\n    probs = np.concatenate(probs)\n    return probs\n\nmodel = Classifier(CFG['model_name'],len(target_cols),pretrained=False)\nmodel = model.to(device)\nmodel.load_state_dict(torch.load('../input/pretrainedefficient/resnet200d_fold0_best.pth'))\ntest_dataset = TestDataset(test, transform=get_transforms(data='valid'))\ntest_loader = DataLoader(test_dataset,\n                         batch_size=CFG['valid_bs'],\n                         shuffle=False, \n                         num_workers=CFG['num_workers'], pin_memory=True)\n\npredictions = inference(model,test_loader, device)\n\ntest[target_cols] = predictions\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/ranzcr-clip-catheter-line-classification/sample_submission.csv')\nsubmission[target_cols] = np.mean(test_preds_1, axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}