{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":14774,"databundleVersionId":875431,"sourceType":"competition"},{"sourceId":418031,"sourceType":"datasetVersion","datasetId":131128}],"dockerImageVersionId":30733,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import argparse\nimport cv2\nimport joblib\nimport math\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport os\nimport pandas as pd\nimport random\nimport time\nimport torch\nimport torch.backends.cudnn as cudnn\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport torchvision\nimport warnings\n\nfrom collections import OrderedDict\nfrom datetime import datetime\nfrom glob import glob\nfrom PIL import Image\nfrom skimage import measure\nfrom skimage.io import imread\nfrom sklearn.model_selection import StratifiedKFold, train_test_split\nfrom torch.autograd import Variable\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import DataLoader\nfrom torch.utils.data.sampler import WeightedRandomSampler\nfrom torchvision import datasets, models, transforms\nfrom tqdm import tqdm_notebook\n\nos.listdir('/kaggle/input')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Dataset","metadata":{}},{"cell_type":"code","source":"class Dataset(torch.utils.data.Dataset):\n    def __init__(self, img_paths, labels, transform=None):\n        self.img_paths = img_paths\n        self.labels = labels\n        self.transform = transform\n\n    def __getitem__(self, index):\n        img_path, label = self.img_paths[index], self.labels[index]\n\n        img = cv2.imread(img_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        # img = imread(img_path)\n        img = Image.fromarray(img)\n\n        if self.transform is not None:\n            img = self.transform(img)\n\n        return img, label\n\n\n    def __len__(self):\n        return len(self.img_paths)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### GetModels","metadata":{}},{"cell_type":"code","source":"!pip install pretrainedmodels\nimport ssl\nssl._create_default_https_context = ssl._create_unverified_context","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pretrainedmodels\n\ndef get_model(model_name='resnet18', **kwargs):\n    pretrained = 'imagenet'\n    model = pretrainedmodels.__dict__[model_name](num_classes=1000,\n                                                  pretrained=pretrained)\n\n    if 'resnet' in model_name:\n        model.avgpool = nn.AdaptiveAvgPool2d(1)\n    else:\n        model.avg_pool = nn.AdaptiveAvgPool2d(1)\n    in_features = model.last_linear.in_features\n    model.last_linear = nn.Linear(in_features, 1)\n\n    for m in model.modules():\n        if isinstance(m, nn.BatchNorm2d):\n            m.weight.requires_grad = False\n            m.bias.requires_grad = False\n\n    return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Metric","metadata":{}},{"cell_type":"code","source":"from sklearn import metrics\n\ndef quadratic_weighted_kappa(y_pred, y_true):\n    if torch.is_tensor(y_pred):\n        y_pred = y_pred.data.cpu().numpy()\n    if torch.is_tensor(y_true):\n        y_true = y_true.data.cpu().numpy()\n    if y_pred.shape[1] == 1:\n        y_pred = y_pred[:, 0]\n    else:\n        y_pred = np.argmax(y_pred, axis=1)\n    return metrics.cohen_kappa_score(y_pred, y_true, weights='quadratic')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Preprocess","metadata":{}},{"cell_type":"code","source":"def scale_radius(src, img_size, padding=False):\n    x = src[src.shape[0] // 2, ...].sum(axis=1)\n    r = (x > x.mean() / 10).sum() // 2\n    yx = src.sum(axis=2)\n    region_props = measure.regionprops((yx > yx.mean() / 10).astype('uint8'))\n    yc, xc = np.round(region_props[0].centroid).astype('int')\n    x1 = max(xc - r, 0)\n    x2 = min(xc + r, src.shape[1] - 1)\n    y1 = max(yc - r, 0)\n    y2 = min(yc + r, src.shape[0] - 1)\n    dst = src[y1:y2, x1:x2]\n    dst = cv2.resize(dst, dsize=None, fx=img_size/(2*r), fy=img_size/(2*r))\n    if padding:\n        pad_x = (img_size - dst.shape[1]) // 2\n        pad_y = (img_size - dst.shape[0]) // 2\n        dst = np.pad(dst, ((pad_y, pad_y), (pad_x, pad_x), (0, 0)), 'constant')\n    return dst\n\n\ndef normalize(src, img_size):\n    dst = cv2.addWeighted(src, 4, cv2.GaussianBlur(src, (0, 0), img_size / 30), -4, 128)\n    return dst\n\n\ndef remove_boundaries(src, img_size):\n    mask = np.zeros(src.shape)\n    cv2.circle(\n        mask,\n        center=(src.shape[1] // 2, src.shape[0] // 2),\n        radius=int(img_size / 2 * 0.9),\n        color=(1, 1, 1),\n        thickness=-1)\n    dst = src * mask + 128 * (1 - mask)\n    return dst\n\n\ndef preprocess(dataset, img_size, scale=False):\n    if dataset == 'aptos2019':\n        df = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')\n        img_paths = '../input/aptos2019-blindness-detection/train_images/' + df['id_code'].values + '.png'\n    elif dataset == 'diabetic_retinopathy':\n        df = pd.read_csv('../input/diabetic-retinopathy-resized/trainLabels.csv')\n        img_paths = '../input/diabetic-retinopathy-resized/resized_train/resized_train/' + df['image'].values + '.jpeg'\n    else:\n        NotImplementedError\n\n    dir_name = 'processed/%s/images_%d' %(dataset, img_size)\n    if scale:\n        dir_name += '_scaled'\n\n    os.makedirs(dir_name, exist_ok=True)\n    for i in tqdm_notebook(range(len(img_paths))):\n        img_path = img_paths[i]\n        if os.path.exists(os.path.join(dir_name, os.path.basename(img_path))):\n            continue\n        img = cv2.imread(img_path)\n        try:\n            if scale:\n                img = scale_radius(img, img_size=img_size, padding=False)\n        except Exception as e:\n            print(\"Preprocess \" + img_paths[i] + \" error: \" + str(e))\n        img = cv2.resize(img, (img_size, img_size))\n        cv2.imwrite(os.path.join(dir_name, os.path.basename(img_path)), img)\n\n    return dir_name","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Tools","metadata":{}},{"cell_type":"code","source":"def str2bool(v):\n    if v.lower() in ['true', 1]:\n        return True\n    elif v.lower() in ['false', 0]:\n        return False\n    else:\n        raise argparse.ArgumentTypeError('Boolean value expected.')\n\n\ndef count_params(model):\n    return sum(p.numel() for p in model.parameters() if p.requires_grad)\n\n\nclass AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n\n\ndef accuracy(output, target, topk=(1,)):\n    \"\"\"Computes the accuracy over the k top predictions for the specified values of k\"\"\"\n    with torch.no_grad():\n        maxk = max(topk)\n        batch_size = target.size(0)\n\n        _, pred = output.topk(maxk, 1, True, True)\n        pred = pred.t()\n        correct = pred.eq(target.view(1, -1).expand_as(pred))\n\n        res = []\n        for k in topk:\n            correct_k = correct[:k].view(-1).float().sum(0, keepdim=True)\n            res.append(correct_k.mul_(100.0 / batch_size))\n        return res\n\n            \ndef preds_to_classes(preds):\n    thres = [0.5, 1.5, 2.5, 3.5]\n    preds[preds < thres[0]] = 0\n    preds[(preds >= thres[0]) & (preds < thres[1])] = 1\n    preds[(preds >= thres[1]) & (preds < thres[2])] = 2\n    preds[(preds >= thres[2]) & (preds < thres[3])] = 3\n    preds[preds >= thres[3]] = 4\n    \n    return preds","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(train_loader, model, criterion, optimizer, epoch):\n    losses = AverageMeter()\n    scores = AverageMeter()\n\n    # training mode\n    model.train()\n\n    for i, (input, target) in tqdm_notebook(enumerate(train_loader), total=len(train_loader)):\n        input = input.cuda()\n        target = target.cuda()\n\n        # predicts and calculates loss\n        preds = model(input)\n        loss = criterion(preds.view(-1), target.float())\n\n        # updata weights\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n\n        # calculate score\n        preds = preds_to_classes(preds)\n        score = quadratic_weighted_kappa(preds, target)\n\n        # record training process and results\n        losses.update(loss.item(), input.size(0))\n        scores.update(score, input.size(0))\n\n    return losses.avg, scores.avg","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def validate(val_loader, model, criterion):\n    losses = AverageMeter()\n    scores = AverageMeter()\n\n    # switch to evaluate mode\n    model.eval()\n\n    with torch.no_grad():\n        for i, (input, target) in tqdm_notebook(enumerate(val_loader), total=len(val_loader)):\n            input = input.cuda()\n            target = target.cuda()\n\n            # predicts and calculates loss\n            preds = model(input)\n            loss = criterion(preds.view(-1), target.float())\n\n            # record validate process and results\n            preds = preds_to_classes(preds)\n            score = quadratic_weighted_kappa(preds, target)\n\n            losses.update(loss.item(), input.size(0))\n            scores.update(score, input.size(0))\n\n    return losses.avg, scores.avg","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_size = 288\nrotate_min = -180\nrotate_max = 180\nrescale_min = 0.8889\nrescale_max = 1.0\nshear_min = -36\nshear_max = 36\n\nmomentum = 0.9\nweight_decay = 1e-4\nlearning_rate = 1e-3\nmin_learning_rate = 1e-5\n\ninput_size = 256\n\nn_splits = 5\nepochs = 30\nbatch_size = 32","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(model_name):\n    result_dir = '%s_%s' % (model_name, datetime.now().strftime('%m%d%H'))\n\n    if not os.path.exists('./%s' % result_dir):\n        os.makedirs('./%s' % result_dir)\n\n    criterion = nn.MSELoss().cuda()\n    num_outputs = 1\n    cudnn.benchmark = True\n    \n    # Transformation\n    train_transform = transforms.Compose([\n        transforms.Resize((img_size, img_size)),\n        transforms.RandomAffine(\n            degrees=(rotate_min, rotate_max),\n            translate=None,\n            scale=(rescale_min, rescale_max),\n            shear=(shear_min, shear_max),\n        ),\n        transforms.CenterCrop(input_size),\n        transforms.RandomHorizontalFlip(p=0.5),\n        transforms.RandomVerticalFlip(p=0.5),\n        transforms.ColorJitter(\n            brightness=0,\n            contrast=True,\n            saturation=0,\n            hue=0),\n        transforms.ToTensor(),\n        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),\n    ])\n\n    val_transform = transforms.Compose([\n        transforms.Resize((img_size, input_size)),\n        transforms.ToTensor(),\n        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),\n    ])\n    \n\n    # prepare dataset\n    diabetic_retinopathy_dir = preprocess(\n        'diabetic_retinopathy',\n        img_size,\n        scale=True)\n    diabetic_retinopathy_df = pd.read_csv('../input/diabetic-retinopathy-resized/trainLabels.csv')\n    diabetic_retinopathy_img_paths = \\\n        diabetic_retinopathy_dir + '/' + diabetic_retinopathy_df['image'].values + '.jpeg'\n    diabetic_retinopathy_labels = diabetic_retinopathy_df['level'].values\n\n    aptos2019_dir = preprocess(\n        'aptos2019',\n        img_size,\n        scale=True)\n    aptos2019_df = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')\n    aptos2019_img_paths = aptos2019_dir + '/' + aptos2019_df['id_code'].values + '.png'\n    aptos2019_labels = aptos2019_df['diagnosis'].values\n\n    skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=41)\n    img_paths = []\n    labels = []\n    for fold, (train_idx, val_idx) in enumerate(skf.split(aptos2019_img_paths, aptos2019_labels)):\n        img_paths.append((np.hstack((aptos2019_img_paths[train_idx], diabetic_retinopathy_img_paths)), aptos2019_img_paths[val_idx]))\n        labels.append((np.hstack((aptos2019_labels[train_idx], diabetic_retinopathy_labels)), aptos2019_labels[val_idx]))\n        \n\n    folds = []\n    best_losses = []\n    best_scores = []\n\n    for fold, ((train_img_paths, val_img_paths), (train_labels, val_labels)) in enumerate(zip(img_paths, labels)):\n        print('Fold [%d/%d]' %(fold+1, len(img_paths)))\n\n        if os.path.exists('./%s/model_%d.pth' % (result_dir, fold+1)):\n            log = pd.read_csv('./%s/log_%d.csv' %(result_dir, fold+1))\n            best_loss, best_score = log.loc[log['val_loss'].values.argmin(), ['val_loss', 'val_score']].values\n            folds.append(str(fold + 1))\n            best_losses.append(best_loss)\n            best_scores.append(best_score)\n            continue\n\n        # train\n        train_set = Dataset(\n            train_img_paths,\n            train_labels,\n            transform=train_transform)\n\n        _, class_sample_counts = np.unique(train_labels, return_counts=True)\n        # print(class_sample_counts)\n        # weights = 1. / torch.tensor(class_sample_counts, dtype=torch.float)\n        # weights = np.array([0.2, 0.1, 0.6, 0.1, 0.1])\n        # samples_weights = weights[train_labels]\n        # sampler = WeightedRandomSampler(\n        #     weights=samples_weights,\n        #     num_samples=11000,\n        #     replacement=False)\n        train_loader = torch.utils.data.DataLoader(\n            train_set,\n            batch_size=batch_size,\n            shuffle=True,\n            num_workers=4,\n            sampler=None)\n\n        val_set = Dataset(\n            val_img_paths,\n            val_labels,\n            transform=val_transform)\n        val_loader = torch.utils.data.DataLoader(\n            val_set,\n            batch_size=batch_size,\n            shuffle=False,\n            num_workers=4)\n\n        # create model\n        model = get_model(model_name)\n        model = model.cuda()\n        # print(model)\n\n        optimizer = optim.SGD(filter(lambda p: p.requires_grad, model.parameters()), lr=learning_rate,\n                              momentum=momentum, weight_decay=weight_decay, nesterov=False)\n\n        scheduler = lr_scheduler.CosineAnnealingLR(\n                optimizer, T_max=epochs, eta_min=min_learning_rate)\n\n        log = pd.DataFrame(index=[], columns=[\n            'epoch', 'loss', 'score', 'val_loss', 'val_score'\n        ])\n        log = {\n            'epoch': [],\n            'loss': [],\n            'score': [],\n            'val_loss': [],\n            'val_score': [],\n        }\n\n        best_loss = float('inf')\n        best_score = 0\n        for epoch in range(epochs):\n            print('Epoch [%d/%d]' % (epoch + 1, epochs))\n\n            # train for one epoch\n            train_loss, train_score = train(\n                train_loader, model, criterion, optimizer, epoch)\n            # evaluate on validation set\n            val_loss, val_score = validate(val_loader, model, criterion)\n\n            scheduler.step()\n\n            print('loss %.4f - score %.4f - val_loss %.4f - val_score %.4f'\n                  % (train_loss, train_score, val_loss, val_score))\n\n            log['epoch'].append(epoch)\n            log['loss'].append(train_loss)\n            log['score'].append(train_score)\n            log['val_loss'].append(val_loss)\n            log['val_score'].append(val_score)\n\n            pd.DataFrame(log).to_csv('./%s/log_%d.csv' % (result_dir, fold+1), index=False)\n\n            if val_loss < best_loss:\n                torch.save(model.state_dict(), './%s/model_%d.pth' % (result_dir, fold+1))\n                best_loss = val_loss\n                best_score = val_score\n                print(\"=> saved best model\")\n\n        print('val_loss:  %f' % best_loss)\n        print('val_score: %f' % best_score)\n\n        folds.append(str(fold + 1))\n        best_losses.append(best_loss)\n        best_scores.append(best_score)\n\n        results = pd.DataFrame({\n            'fold': folds + ['mean'],\n            'best_loss': best_losses + [np.mean(best_losses)],\n            'best_score': best_scores + [np.mean(best_scores)],\n        })\n\n        print(results)\n        results.to_csv('./%s/results.csv' % result_dir, index=False)\n\n        torch.cuda.empty_cache()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### se_resnext50_32x4d","metadata":{}},{"cell_type":"code","source":"train_model(\"se_resnext50_32x4d\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### se_resnext101_32x4d","metadata":{}},{"cell_type":"code","source":"batch_size = 24","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_model(\"se_resnext101_32x4d\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### senet154","metadata":{}},{"cell_type":"code","source":"batch_size = 16","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_model(\"senet154\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}