{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Efficientnet baseline by Pytorch\n\n* model : efficientnet-b0(pretrained)\n* loss : CategoricalCrossEntropy(+label smoothing)\n* optimizer : Adam\n* preprocessing\n * RandomResizedCrop\n * RandomHorizontalFlip\n * RandomVerticalFlip\n * ColorJitter\n * ToTensor\n * Normalize\n*  TTA : True\n\n\n# log\n\nver2 : add \"RondomResizeCrop\" and \"ColorJitter\"  \nver3 : reduce the effect of \"ColorJitter\", add labelsmoothing  \nver4 : add TTA  \nver5 : add TTA3, increace epoch(13->25)  \nver6 : add cutmix, add fmix  \nver7 : delete fmix, change smoothing(0.2 -> 0.1)  \nver8 : delete rotate, change bbx size, change epoch(25->20)  \nver9 : imsize 512->600, change smoothing(0.1 -> 0.2), change ColorJitter rate  \nver10: using alubmentation(Fundamentally changed the augmentation)"},{"metadata":{"_uuid":"12d546ab-f82c-49ea-b282-0ebc373fbe53","_cell_guid":"2bfbde8e-6c8a-48d4-8ab0-33bfb7e12be6","trusted":true},"cell_type":"markdown","source":"## Library import"},{"metadata":{"_uuid":"f99acd7c-4ef5-4773-8bda-0f59ee783601","_cell_guid":"c8123ddc-070c-40d3-ac9b-fd0581380ffb","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport json\nfrom PIL import Image\nimport os\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nfrom sklearn import preprocessing\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import StratifiedKFold\n\nimport torch\nimport torch.nn as nn\nimport torchvision\nimport torchvision.transforms as transforms\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\n\nfrom albumentations import HorizontalFlip, VerticalFlip, ShiftScaleRotate, Transpose, HueSaturationValue, RandomResizedCrop, RandomBrightnessContrast, Cutout, Compose, Normalize, CoarseDropout\nfrom albumentations.pytorch import ToTensorV2\nimport cv2\n\nimport sys\nimport random\n\n# add efficientnet path\nefnet_path = '../input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master'\nsys.path.append(efnet_path)\nfrom efficientnet_pytorch import EfficientNet\n\n# add fmix path\nfmix_path = '../input/image-fmix/FMix-master' #'../input/efficientnet-pytorch-07/efficientnet_pytorch-0.7.0'\nsys.path.append(fmix_path)\nfrom fmix import sample_mask\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7e723b78-5a12-487f-8dc5-d85ba9be934e","_cell_guid":"fb0a4a8e-887a-4220-ab9e-f54956e4a0d5","trusted":true},"cell_type":"markdown","source":"## Directory Setting"},{"metadata":{"_uuid":"43c7d8b3-6d00-468a-9e0a-3c45531ee1ff","_cell_guid":"8dd75c2e-5275-4d6b-ad27-9b1d4bc070c8","trusted":true},"cell_type":"code","source":"DEVICE = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n\nOUTPUT_DIR = './'\nif not os.path.exists(OUTPUT_DIR):\n    os.makedirs(OUTPUT_DIR)\n\nTRAIN_DIR = '../input/cassava-leaf-disease-classification/train_images/'\nTEST_DIR = '../input/cassava-leaf-disease-classification/test_images/'","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1d008f0b-5471-4d4e-97b5-a436350fd24b","_cell_guid":"ce997916-289b-4fa4-80f5-28bb33c55ead","trusted":true},"cell_type":"markdown","source":"## Data loading"},{"metadata":{"_uuid":"ac02b3da-2a7e-4f66-8069-9dae31b13118","_cell_guid":"1d8d72cb-8d57-43ae-a59e-abb74a653f44","trusted":true},"cell_type":"code","source":"labels = json.load(open(\"../input/cassava-leaf-disease-classification/label_num_to_disease_map.json\"))\ntrain = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv')\nsample = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\n\nX, Y = train['image_id'].values, train['label'].values\nX_test = [name for name in (os.listdir(TEST_DIR))]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"292353bf-d7a8-4c01-9487-1ebebac849ef","_cell_guid":"bc3510da-a896-4690-8e1c-6438d7c511f1","trusted":true},"cell_type":"markdown","source":"## Config"},{"metadata":{"_uuid":"9a031aae-b9c7-4d18-aa42-2fdea1863114","_cell_guid":"69713d26-5161-4d88-82d5-d8f4022f2974","trusted":true},"cell_type":"code","source":"class Conf:\n    seed=777\n    \n    BATCH = 16\n    EPOCHS = 20\n\n    LR = 0.0001\n    IM_SIZE = 600\n\n    n_fold=5\n    target_col='label'\n    \n    modelname='efficientnet-b0'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Helper Functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nseed_everything(Conf.seed)\n\ndef get_img(path):\n    im_bgr = cv2.imread(path)\n    im_rgb = im_bgr[:, :, ::-1]\n    return im_rgb","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2d74feb6-c5b8-4576-babf-dcbd7cff0fec","_cell_guid":"9963df50-ee8c-4788-9b5a-b6480e2df6a7","trusted":true},"cell_type":"markdown","source":"## CV split"},{"metadata":{"_uuid":"47118400-5e44-4e88-bec7-fa8549a0d4ba","_cell_guid":"f7f05753-51bc-4287-ac84-e3ab255cb6fa","trusted":true},"cell_type":"code","source":"folds = train.copy()\nFold = StratifiedKFold(n_splits=Conf.n_fold, shuffle=True, random_state=Conf.seed)\nfor n, (train_index, val_index) in enumerate(Fold.split(folds, folds[Conf.target_col])):\n    folds.loc[val_index, 'fold'] = int(n)\nfolds['fold'] = folds['fold'].astype(int)\nprint(folds.groupby(['fold', Conf.target_col]).size())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e9f72b3a-01fc-4274-9f30-00207f8efdb8","_cell_guid":"a38d9685-b054-4fe0-b0f1-d6e5e6ba17b9","trusted":true},"cell_type":"markdown","source":"## Augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_transform():\n    return Compose([\n            RandomResizedCrop(Conf.IM_SIZE, Conf.IM_SIZE),\n            Transpose(p=0.5),\n            HorizontalFlip(0.5),\n            VerticalFlip(p=0.5),\n            ShiftScaleRotate(0.5),\n            HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n            RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1,0.1), p=0.5),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            CoarseDropout(p=0.5),\n            ToTensorV2(p=1.0)\n])\n\ndef test_transform():\n    return Compose([\n            RandomResizedCrop(Conf.IM_SIZE, Conf.IM_SIZE, (1.0,1.0), ratio=(1.0,1.0)),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], p=1.0),\n            ToTensorV2(p=1.0)\n])\n\ndef TTA1():\n    return Compose([\n            RandomResizedCrop(Conf.IM_SIZE, Conf.IM_SIZE),\n            HorizontalFlip(1.0),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], p=1.0),\n            ToTensorV2(p=1.0)\n])\n\ndef TTA2():\n    return Compose([\n            RandomResizedCrop(Conf.IM_SIZE, Conf.IM_SIZE),\n            VerticalFlip(1.0),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], p=1.0),\n            ToTensorV2(p=1.0)\n])\n\ndef TTA3():\n    return Compose([\n            RandomResizedCrop(Conf.IM_SIZE, Conf.IM_SIZE),\n            HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=1.0),\n            RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1,0.1), p=1.0),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], p=1.0),\n            ToTensorV2(p=1.0)\n])\n\nTTAs = [test_transform, TTA1, TTA2, TTA3]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"15ec2dd5-c739-4e34-86ae-a4c02a6d9055","_cell_guid":"0c6e03d6-cee4-4db9-9dc0-a8f9e562c793","trusted":true},"cell_type":"markdown","source":"## Dataset/Dataloader"},{"metadata":{"_uuid":"872ab572-903a-493a-9f13-47356e72dea7","_cell_guid":"4d5828a3-9c3e-4760-a439-68f8b1e6e5b6","trusted":true},"cell_type":"code","source":"class TrainData(Dataset):\n    def __init__(self, Dir, FNames, Labels, Transform):\n        self.dir = Dir\n        self.fnames = FNames\n        self.transform = Transform\n        self.lbs = Labels\n        \n    def __len__(self):\n        return len(self.fnames)\n\n    def __getitem__(self, index):\n        img = get_img(os.path.join(self.dir, self.fnames[index]))  \n        img = self.transform()(image=img)['image']\n        return img, self.lbs[index] \n        \nclass TestData(Dataset):\n    def __init__(self, Dir, FNames, TTAs=TTAs):\n        self.dir = Dir\n        self.fnames = FNames\n#         self.transform = Transform\n        self.ttas = TTAs\n        \n    def __len__(self):\n        return len(self.fnames)\n\n    def __getitem__(self, index):\n        img = get_img(os.path.join(self.dir, self.fnames[index]))\n        imglist = [tta()(image=img)['image'] for tta in self.ttas]\n        imglist = torch.stack(imglist)\n        return imglist, self.fnames[index]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bd1edb8c-1818-4d9a-b6d5-0976bb08c886","_cell_guid":"ce1cdc28-1823-4910-9965-672d96d685f8","trusted":true},"cell_type":"code","source":"trn_idx = folds[folds['fold'] != 0].index\nval_idx = folds[folds['fold'] == 0].index\n\nX_train, Y_train = X[trn_idx], Y[trn_idx]\nX_val, Y_val = X[val_idx], Y[val_idx]\n\ntrainset = TrainData(TRAIN_DIR, X_train, Y_train, train_transform)\ntrainloader = DataLoader(trainset,\n                         batch_size=Conf.BATCH,\n                         shuffle=True,\n                         num_workers=4)\n\nvalidset = TrainData(TRAIN_DIR, X_val, Y_val, test_transform)\nvalidloader = DataLoader(validset,\n                         batch_size=Conf.BATCH,\n                         shuffle=False,\n                         num_workers=4)\n\ntestset = TestData(TEST_DIR, X_test, TTAs)\ntestloader = DataLoader(testset,\n                        batch_size=1,\n                        shuffle=False,\n                        num_workers=4)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Cutmix and Fmix"},{"metadata":{"trusted":true},"cell_type":"code","source":"def rand_bbox(size, lam):\n    W = size[2]\n    H = size[3]\n    cut_rat = np.sqrt(1. - lam)\n    cut_w = np.int(W * cut_rat)\n    cut_h = np.int(H * cut_rat)\n\n    # uniform\n    cx = np.random.randint(W)\n    cy = np.random.randint(H)\n\n    bbx1 = np.clip(cx - cut_w // 2, 0, W)\n    bby1 = np.clip(cy - cut_h // 2, 0, H)\n    bbx2 = np.clip(cx + cut_w // 2, 0, W)\n    bby2 = np.clip(cy + cut_h // 2, 0, H)\n    return bbx1, bby1, bbx2, bby2\n\ndef cutmix(data, target, alpha):\n    indices = torch.randperm(data.size(0))\n    shuffled_data = data[indices]\n    shuffled_target = target[indices]\n\n    lam = np.clip(np.random.beta(alpha, alpha),0.5,0.6)\n    bbx1, bby1, bbx2, bby2 = rand_bbox(data.size(), lam)\n    new_data = data.clone()\n    new_data[:, :, bby1:bby2, bbx1:bbx2] = data[indices, :, bby1:bby2, bbx1:bbx2]\n    # adjust lambda to exactly match pixel ratio\n    lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (data.size()[-1] * data.size()[-2]))\n    targets = (target, shuffled_target, lam)\n\n    return new_data, targets\n\ndef fmix(data, targets, alpha, decay_power, shape, max_soft=0.0, reformulate=False):\n    lam, mask = sample_mask(alpha, decay_power, shape, max_soft, reformulate)\n    mask =torch.tensor(mask, device=DEVICE).float()\n    indices = torch.randperm(data.size(0))\n    shuffled_data = data[indices]\n    shuffled_targets = targets[indices]\n    x1 = mask.to(DEVICE)*data\n    x2 = (1-mask).to(DEVICE)*shuffled_data\n    targets=(targets, shuffled_targets, lam)\n    \n    return (x1+x2), targets","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Image Visualization"},{"metadata":{"trusted":true},"cell_type":"code","source":"class UnNormalize(object):\n    def __init__(self, mean, std):\n        self.mean = mean\n        self.std = std\n\n    def __call__(self, images):\n\n        images = images.mul_(self.std).add_(self.mean)\n            # The normalize code -> t.sub_(m).div_(s)\n        return images\n\nmean = torch.tensor([0.485, 0.456, 0.406]).view(1,3,1,1)\nstd = torch.tensor([0.229, 0.224, 0.225]).view(1,3,1,1)\ninv_normalize = UnNormalize(mean, std)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"row = 4\ncol = 4\nfig, axes = plt.subplots(row, col, figsize=(16, 16))\naxes = axes.ravel()\n\nimages,labels = next(iter(trainloader))\nimages,_ = cutmix(images, labels, 1.)\nimages = inv_normalize(images)\nimages = images.detach().numpy().transpose(0,2,3,1)\n\nfor j in range(Conf.BATCH):\n    axes[j].imshow(images[j])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3a93a5d2-56a8-4b63-b90a-d51cddba1b89","_cell_guid":"94c0f3fe-0b8d-4b68-86c5-a8ccb895adb9","trusted":true},"cell_type":"markdown","source":"## Model"},{"metadata":{"_uuid":"6331756c-e445-4790-8ab3-172f9258cea4","_cell_guid":"2c0986a4-8954-43c2-bcad-01f698c65cb3","trusted":true},"cell_type":"code","source":"# Model\nclass enetv2(nn.Module):\n    def __init__(self, out_dim=1, ModelName=\"efficientnet-b0\"):\n        super(enetv2, self).__init__()\n        self.basemodel = EfficientNet.from_name(Conf.modelname)\n        \n        self.myfc = nn.Linear(self.basemodel._fc.in_features, out_dim)\n        self.basemodel._fc = nn.Identity()        \n            \n    def extract(self, x):\n        return self.basemodel(x)\n\n    def forward(self, x):\n        x = self.basemodel(x)\n        x = self.myfc(x)\n        return x\n    \n# Loss\nclass LabelSmoothingLoss(nn.Module):\n\n    def __init__(self, classes, smoothing=0.0, dim=-1): \n        super(LabelSmoothingLoss, self).__init__() \n        self.confidence = 1.0 - smoothing \n        self.smoothing = smoothing \n        self.cls = classes \n        self.dim = dim \n    def forward(self, pred, target): \n        pred = pred.log_softmax(dim=self.dim) \n        with torch.no_grad(): \n            true_dist = torch.zeros_like(pred) \n            true_dist.fill_(self.smoothing / (self.cls - 1)) \n            true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence) \n        return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = enetv2(5, Conf.modelname)\ncheckpoint = torch.load(\"../input/efficientnet-pytorch/efficientnet-b0-08094119.pth\", map_location=DEVICE)\nmodel.load_state_dict(checkpoint, strict=False)\nmodel = model.to(DEVICE)\n\ncriterion = LabelSmoothingLoss(5, 0.2)\noptimizer = torch.optim.Adam(model.parameters(), lr=Conf.LR)\nscheduler = ReduceLROnPlateau(optimizer, mode='min', verbose=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ffad6512-da06-4032-bd80-abed87df0153","_cell_guid":"cde2ce31-1bf1-4135-ae50-9b4066744b95","trusted":true},"cell_type":"markdown","source":"# Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_one_epoch(model, train_loader):\n    model.train()\n    running_loss = 0\n    correct = 0\n    total = 0\n    \n    for batch_idx, (images, labels) in enumerate(train_loader):\n        images = images.to(DEVICE)\n        labels = labels.to(DEVICE)\n        \n        #cutmix and fmix\n#         mix_decision = np.random.rand()\n#         if mix_decision < 0.25:\n#             images, labels = cutmix(images, labels, 1.)\n#         elif mix_decision >= 0.25 and mix_decision < 0.5:\n#             images, labels = fmix(images, labels, alpha=1, decay_power=5., shape=(512,512))\n\n        mix_decision = np.random.rand()\n        if mix_decision < 0.5:\n            images, labels = cutmix(images, labels, 1.)\n        \n        optimizer.zero_grad() \n        outputs = model(images)\n        \n        if mix_decision < 0.5:\n            loss = criterion(outputs, labels[0]) * labels[2] + criterion(outputs, labels[1]) * (1. - labels[2])\n            labels = labels[0]\n        else:\n            loss = criterion(outputs, labels)\n        loss.backward()\n        optimizer.step()\n        \n        running_loss += loss.item()\n        _, predict = torch.max(outputs.data, 1)\n        correct += (predict == labels).sum().item()\n        total += labels.size(0)\n        \n    train_loss = running_loss / len(train_loader)\n    train_acc = correct / total\n    \n    return train_loss, train_acc\n\ndef valid_one_epoch(model, valid_loader):\n    model.eval()\n    running_loss = 0\n    correct = 0\n    total = 0\n    \n    with torch.no_grad():\n        \n        for batch_idx, (images, labels) in enumerate(valid_loader):\n            images = images.to(DEVICE)\n            labels = labels.to(DEVICE)\n            \n            outputs = model(images)\n            \n            loss = criterion(outputs, labels)\n            running_loss += loss.item()\n            \n            _, predict = torch.max(outputs.data, 1)\n            correct += (predict == labels).sum().item()\n            total += labels.size(0)\n            \n    val_loss = running_loss / len(valid_loader)\n    val_acc = correct / total\n    \n    return val_loss, val_acc","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### For saving quota"},{"metadata":{"trusted":true},"cell_type":"code","source":"# best_score = 0.\n\n# if len(testloader) >= 2:\n#     for epoch_idx in range(Conf.EPOCHS):\n\n#         train_loss, train_acc = train_one_epoch(model, trainloader)\n#         valid_loss, valid_acc = valid_one_epoch(model, validloader)\n\n#         # model save\n#         if valid_acc > best_score:\n#             best_score = valid_acc\n\n#             torch.save(model.state_dict(), OUTPUT_DIR+f'{Conf.modelname}_best.pth')\n\n#             print('model saved')\n\n#         # rl scheduler\n#         scheduler.step(valid_loss)\n\n#         print('Epoch: {} |train_loss: {:.3f} valid loss: {:.3f} train_acc: {:.3f} valid_acc: {:.3f}'.format(epoch_idx, train_loss, valid_loss, train_acc, valid_acc))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_score = 0.\n\nfor epoch_idx in range(Conf.EPOCHS):\n\n    train_loss, train_acc = train_one_epoch(model, trainloader)\n    valid_loss, valid_acc = valid_one_epoch(model, validloader)\n\n    # model save\n    if valid_acc > best_score:\n        best_score = valid_acc\n\n        torch.save(model.state_dict(), OUTPUT_DIR+f'{Conf.modelname}_best.pth')\n\n        print('model saved')\n\n    # rl scheduler\n    scheduler.step(valid_loss)\n\n    print('Epoch: {} |train_loss: {:.3f} valid loss: {:.3f} train_acc: {:.3f} valid_acc: {:.3f}'.format(epoch_idx, train_loss, valid_loss, train_acc, valid_acc))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Inference"},{"metadata":{"_uuid":"4a1427ef-dc5e-44b6-9842-7d2f47716d7c","_cell_guid":"e52056ff-1b34-4d1a-855c-4f9371f66253","trusted":true},"cell_type":"code","source":"if len(testloader) >= 2:\n    model.load_state_dict(torch.load(OUTPUT_DIR+f'{Conf.modelname}_best.pth'), strict=False)\ns_ls = []\n\nwith torch.no_grad():\n    model.eval()\n    for images, fname in testloader: \n        images = images.to(DEVICE)\n        batch_size, n_crops, c, h, w = images.size()\n        images = images.view(-1, c, h, w)\n        output = model(images)\n        output = output.mean(0)\n        ps = torch.exp(output)\n        _, top_class = ps.topk(1, dim=0)\n        \n        s_ls.append([fname[0], top_class.item()])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Make submission"},{"metadata":{"_uuid":"f9559f2d-dad1-4082-969d-03fd91b9c7ed","_cell_guid":"ce51405e-96f7-46ff-8b91-7849dde2ce95","trusted":true},"cell_type":"code","source":"sub = pd.DataFrame.from_records(s_ls, columns=['image_id', 'label'])\nsub.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}