{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import sys\nsys.path.append('../input/pretrainedmodels/pretrainedmodels-0.7.4')\nsys.path.append('../input/timm-pytorch-image-models/pytorch-image-models-master')\nimport os\nimport math\nimport time\nimport random\nimport shutil\nfrom pathlib import Path\nfrom contextlib import contextmanager\nfrom collections import defaultdict, Counter\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\nfrom sklearn import preprocessing\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold, KFold\nfrom tqdm.auto import tqdm\nfrom functools import partial\nimport cv2\nfrom PIL import Image\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import Adam, SGD\nimport torchvision.models as models\nfrom torch.nn.parameter import Parameter\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau, CosineAnnealingLR, CosineAnnealingWarmRestarts\nfrom albumentations import *\nfrom albumentations.pytorch import ToTensorV2\nfrom albumentations import ImageOnlyTransform\nimport timm\nfrom torch.cuda.amp import autocast, GradScaler\nfrom matplotlib import pyplot as plt\nimport warnings\nimport pretrainedmodels\nimport albumentations\n\n\nwarnings.filterwarnings('ignore')\n\npath = \"/kaggle/input/ranzcr-clip-catheter-line-classification/\"\npretrained_path = ''\ndevice = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\ntrain = pd.read_csv(os.path.join(path, \"train.csv\"))\n\nimage_size = 512\nclass PARAMETER:\n    debug = False\n    debug_size = 15000\n    batch_size = 32\n    n_fold = 3\n    epochs = 3\n    seed = 242\n    size = 512\n    num_workers = 4\n    model_name = 'resnext50_32x4d' #'resnet200d_320'\n    lr = 1e-5\n    weight_decay = 1e-6\n    scheduler = 'ReduceLROnPlateau'  # ['ReduceLROnPlateau', 'CosineAnnealingLR', 'CosineAnnealingWarmRestarts']\n    factor = 0.2     # ReduceLROnPlateau\n    patience = 4     # ReduceLROnPlateau\n    eps = 1e-6       # ReduceLROnPlateau\n    T_max = 6        # CosineAnnealingLR\n    T_0 = 6          # CosineAnnealingWarmRestarts\n    target_size = 11\n    target_cols = ['ETT - Abnormal', 'ETT - Borderline', 'ETT - Normal',\n                   'NGT - Abnormal', 'NGT - Borderline', 'NGT - Incompletely Imaged', 'NGT - Normal',\n                   'CVC - Abnormal', 'CVC - Borderline', 'CVC - Normal',\n                   'Swan Ganz Catheter Present']\n\n\nif PARAMETER.debug:\n    PARAMETER.epochs = 1\n    train = train.sample(n=PARAMETER.debug_size, random_state=PARAMETER.seed).reset_index(drop=True)\n\n\ndef get_transforms(*, data):\n    if data == 'train':\n        return Compose([\n            # Resize(CFG.size, CFG.size),\n   albumentations.RandomResizedCrop(image_size, image_size, scale=(0.9, 1), p=1), \n   albumentations.HorizontalFlip(p=0.5),\n   albumentations.ShiftScaleRotate(p=0.5),\n   albumentations.HueSaturationValue(hue_shift_limit=10, sat_shift_limit=10, val_shift_limit=10, p=0.7),\n   albumentations.RandomBrightnessContrast(brightness_limit=(-0.2,0.2), contrast_limit=(-0.2, 0.2), p=0.7),\n   albumentations.CLAHE(clip_limit=(1,4), p=0.5),\n   albumentations.OneOf([\n       albumentations.OpticalDistortion(distort_limit=1.0),\n       albumentations.GridDistortion(num_steps=5, distort_limit=1.),\n       albumentations.ElasticTransform(alpha=3),\n   ], p=0.2),\n   albumentations.OneOf([\n       albumentations.GaussNoise(var_limit=[10, 50]),\n       albumentations.GaussianBlur(),\n       albumentations.MotionBlur(),\n       albumentations.MedianBlur(),\n   ], p=0.2),\n  albumentations.Resize(image_size, image_size),\n  albumentations.OneOf([\n      JpegCompression(),\n      Downscale(scale_min=0.1, scale_max=0.15),\n  ], p=0.2),\n  IAAPiecewiseAffine(p=0.2),\n  IAASharpen(p=0.2),\n  albumentations.Cutout(max_h_size=int(image_size * 0.1), max_w_size=int(image_size * 0.1), num_holes=5, p=0.5),\n            Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225],\n            ),\n            ToTensorV2(),\n        ])\n\n    elif data == 'valid':\n        return Compose([\n            Resize(PARAMETER.size, PARAMETER.size),\n            Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225],\n            ),\n            ToTensorV2(),\n        ])\n\n\nclass Pre_trained_model(torch.nn.Module):\n    def __init__(self, model_name='inceptionv4', pretrained='imagenet'):\n        super(Pre_trained_model, self).__init__()\n        # define structure of the network here\n        self.model = pretrainedmodels.__dict__[model_name](num_classes=1000, pretrained=pretrained)\n        self.linear = nn.Linear(1000, PARAMETER.target_size)\n\n    def forward(self, input):\n        # apply network and return output\n        x = self.model(input)\n        x = self.linear(x)\n        return x\n\nclass MyModel(torch.nn.Module):\n    def __init__(self, model_name='inceptionv4', pretrained=False):\n        super().__init__()\n        # define structure of the network here\n        self.model = pretrainedmodels.__dict__[model_name](num_classes=1001, pretrained=pretrained)\n#         pretrained = torch.load(r'../input/pretrained-model-weights-pytorch/inceptionv4-8e4777a0.pth')\n#         self.model.load_state_dict(pretrained)\n        \n        \n        self.linear = nn.Linear(1001, 11)\n\n    def forward(self, input):\n        \n        x = self.model(input)\n        x = self.linear(x)\n        return x\n\n\ndef get_scheduler(optimiser):\n    global scheduler\n    if PARAMETER.scheduler == 'ReduceLROnPlateau':\n        scheduler = ReduceLROnPlateau(optimiser, mode='min', factor=PARAMETER.factor, patience=PARAMETER.patience,\n                                      threshold=0.0001, threshold_mode='rel', cooldown=0, min_lr=0,\n                                      eps=1e-08, verbose=True)\n    elif PARAMETER.scheduler == 'CosineAnnealingLR':\n        scheduler = CosineAnnealingLR(optimiser, T_max=PARAMETER.T_max, eta_min=0, last_epoch=-1)\n    elif PARAMETER.scheduler == 'CosineAnnealingWarmRestarts':\n        scheduler = CosineAnnealingWarmRestarts(optimiser, T_0=PARAMETER.T_0, T_mult=1, eta_min=0, last_epoch=-1)\n    return scheduler\n\n\nclass InputModel(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.file_names = df['StudyInstanceUID'].values\n        self.labels = df[PARAMETER.target_cols].values\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        file_name = self.file_names[idx]\n        file_path = f'{path}/train/{file_name}.jpg'\n        image = cv2.imread(file_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n        label = torch.tensor(self.labels[idx]).float()\n        return image, label\n\n\nclass OutputModel(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.file_names = df['StudyInstanceUID'].values\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        file_name = self.file_names[idx]\n        file_path = f'{path}/test/{file_name}.jpg'\n        image = cv2.imread(file_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n        return image\n\n\ndef get_score(y_true, y_pred):\n    scores = []\n    for i in range(y_true.shape[1]):\n        score = roc_auc_score(y_true[:, i], y_pred[:, i])\n        scores.append(score)\n    avg_score = np.mean(scores)\n    return avg_score, scores\n\n\ndef train_fn(train_loader, net, criterion, optimiser, epoch, device):\n    scaler = GradScaler()\n    net.train()\n    runningloss = 0\n    for i, (images, labels) in enumerate(train_loader):\n        images = images.to(device)\n        labels = labels.to(device)\n        optimiser.zero_grad()\n        with autocast():\n            outputs = net(images)\n            loss = criterion(outputs, labels)\n        scaler.scale(loss).backward()\n        scaler.step(optimiser)\n        scaler.update()\n        runningloss += loss.item()\n        if i % 32 == 31:\n            print(f'train：{(i + 1) * 100 / len(train_loader):.2f}', '% ',end='')\n            print(\"Epoch: %2d, Batch: %4d, Loss: %.3f\"\n                  % (epoch + 1, i + 1, runningloss / 32))\n            runningloss = 0\n    return\n\n\ndef valid_fn(valid_loader, net, criterion, device, valid_labels):\n    net.eval()\n    preds = []\n    runningloss = 0\n    num = 0\n    for i, (images, labels) in enumerate(valid_loader):\n        images = images.to(device)\n        labels = labels.to(device)\n        with torch.no_grad():\n            outputs = net(images)\n            loss = criterion(outputs, labels)\n        preds.append(outputs.sigmoid().to('cpu').numpy())\n        runningloss += loss.item()\n        num = num + 1\n        if i % 25 == 24:\n            print(f'正在评估：{(i + 1) * 100 / len(valid_loader):.2f}', '%')\n    predictions = np.concatenate(preds)\n    runningloss = runningloss / num\n    score, scores = get_score(valid_labels, predictions)\n    print(score)\n    print(scores)\n    PATH = str(score) + '.pth'\n    print('save model: ',PATH)\n    torch.save(net.state_dict(), PATH)\n    return runningloss\n\n\ndef train_loop(folds, fold, net):\n    print(\"Using device: {}\"\n          \"\\n\".format(str(device)))\n    trn_idx = folds[folds['fold'] != fold].index\n    val_idx = folds[folds['fold'] == fold].index\n\n    train_folds = folds.loc[trn_idx].reset_index(drop=True)\n    valid_folds = folds.loc[val_idx].reset_index(drop=True)\n    valid_labels = valid_folds[PARAMETER.target_cols].values\n\n    train_dataset = InputModel(train_folds,\n                               transform=get_transforms(data='train'))\n    valid_dataset = InputModel(valid_folds,\n                               transform=get_transforms(data='valid'))\n\n    train_loader = DataLoader(train_dataset,\n                              batch_size=PARAMETER.batch_size,\n                              shuffle=True,\n                              num_workers=PARAMETER.num_workers, pin_memory=True, drop_last=True)\n    valid_loader = DataLoader(valid_dataset,\n                              batch_size=PARAMETER.batch_size,\n                              shuffle=False,\n                              num_workers=PARAMETER.num_workers, pin_memory=True, drop_last=False)\n\n\n\n\n    optimiser = Adam(net.parameters(), lr=PARAMETER.lr, weight_decay=PARAMETER.weight_decay, amsgrad=False)\n    scheduler = get_scheduler(optimiser)\n    criterion = nn.BCEWithLogitsLoss()\n    for epoch in range(PARAMETER.epochs):\n        print(f\"\\n========== fold: {fold} training ==========\")\n        train_fn(train_loader, net, criterion, optimiser, epoch, device)\n        print('Result processing')\n        val_loss = valid_fn(valid_loader, net, criterion, device, valid_labels)\n        if PARAMETER.scheduler == 'ReduceLROnPlateau':\n            print(val_loss)\n            scheduler.step(val_loss)\n        else:\n            scheduler.step()\n\n\ndef main():\n    # GroupKFold\n    folds = train.copy()\n    GKF = GroupKFold(n_splits=PARAMETER.n_fold)\n    groups = folds['PatientID'].values\n    for n, (train_index, val_index) in enumerate(GKF.split(folds, folds[PARAMETER.target_cols], groups)):\n        folds.loc[val_index, 'fold'] = int(n)\n    folds['fold'] = folds['fold'].astype(int)\n\n    # train\n    net = MyModel().to(device)\n    net.load_state_dict(torch.load('../input/zqrgod/0.9731629988252436.pth', map_location='cuda:0'))\n    for fold in range(PARAMETER.n_fold):\n        train_loop(folds, fold, net)\n\n    # predict\n    test = pd.read_csv(os.path.join(path, \"sample_submission.csv\"))\n    test_dataset = OutputModel(test, transform=get_transforms(data='valid'))\n    test_loader = torch.utils.data.DataLoader(test_dataset,\n                                              batch_size=PARAMETER.batch_size,\n                                              shuffle=False,\n                                              num_workers=PARAMETER.num_workers, pin_memory=True, drop_last=False)\n    print('\\nPredict processing')\n    net.eval()\n    preds = []\n    for i, batch in enumerate(test_loader):\n        inputs = batch\n        inputs = inputs.to(device)\n        with torch.no_grad():\n            outputs = net(inputs)\n        preds.append(outputs.sigmoid().to('cpu').numpy())\n    predictions = np.concatenate(preds)\n    print('Save prediction')\n    test[PARAMETER.target_cols] = predictions\n    test[['StudyInstanceUID'] + PARAMETER.target_cols].to_csv('submission.csv', index=False)\n    print('Finish')\n\n\nif __name__ == '__main__':\n    main()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}