{"cells":[{"metadata":{"id":"KnRsrvYbU1jy","executionInfo":{"status":"ok","timestamp":1598193780161,"user_tz":-180,"elapsed":19687,"user":{"displayName":"Olena Yukhymenko","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiaVeFIaNiIE57ytUObFjBTriX3tNj7r4-PJr96Nw=s64","userId":"12610305645255153661"}},"outputId":"1636458f-106c-4da9-91bc-578004da1614","trusted":true},"cell_type":"markdown","source":"This inference notebook is a part of work, heavily inspired by brilliant notebooks of @abhishek and @cdeotte in [SIIM-ISIC Melanoma Classification](https://www.kaggle.com/c/siim-isic-melanoma-classification). Training part is [here](https://www.kaggle.com/dunklerwald/pytorch-efficientnet-with-tta-training).\n\n**UPDATE**:\n- switched to noisy-student\n- added simple upsampling option (disabled by default)\n- added confusion matrix\n\n**UPDATE1**:\n- switched to CosineAnnealingWarmRestarts\n- upgraded to image size 512\n- switched back to B4 effnet\n\n**UPDATE(FINAL)**:\n- added weight decay\n- switched to smoothed cross entropy loss\n\n**TO DO** : looking at training progress plots for image size 512 in the training kernel, there can be some room for improvement through applying more regularization and reducing learning rate."},{"metadata":{"trusted":true},"cell_type":"code","source":"package_path = '../input/pytorch-image-models/pytorch-image-models-master'\nimport sys\nsys.path.append(package_path)    ","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","id":"Tqw3CPzjTeL9","trusted":true},"cell_type":"code","source":"import os\nimport torch\nimport albumentations\n\nimport numpy as np\nimport pandas as pd\nimport warnings\n\nimport time\nimport datetime\n\nimport matplotlib.pyplot as plt\nimport plotly.express as px\nfrom plotly.subplots import make_subplots\n\nimport torch\nimport torch.nn as nn\nfrom torch.nn import functional as F\n\nfrom sklearn import metrics\nfrom sklearn import model_selection\n\n\nfrom PIL import Image\nfrom PIL import ImageFile\n\nImageFile.LOAD_TRUNCATED_IMAGES = True\n\n\n#import efficientnet_pytorch\nimport timm\n\nwarnings.simplefilter('ignore')\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"id":"x6blyHjiJkOH","executionInfo":{"status":"ok","timestamp":1598193819013,"user_tz":-180,"elapsed":58466,"user":{"displayName":"Olena Yukhymenko","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiaVeFIaNiIE57ytUObFjBTriX3tNj7r4-PJr96Nw=s64","userId":"12610305645255153661"}},"outputId":"204ac38e-0365-4adf-c9cb-b49b648663f1","trusted":true},"cell_type":"code","source":"n_epochs = 10\nn_patience = 5\nn_folds = 3\ntrain_bsize = 24\nvalid_bsize = 48\ntest_bsize = 48\nseed = 42\n\neffnet_output = {0: 1280, 1: 1280, 2: 1408, 3: 1536, 4: 1792, 5: 2048, 6: 2304, 7: 2560}\n\nIMG_SIZE = 512\nEFFNET_MODEL = 4\n\nAUGMENTATION =[albumentations.ShiftScaleRotate(shift_limit=0.2, scale_limit=0.2, rotate_limit=15, border_mode=0, p=0.6),\n              albumentations.Flip(p=0.5),\n              albumentations.RandomRotate90(p=0.5),\n              albumentations.RandomBrightness(limit=0.2, p=0.6),\n              albumentations.RandomContrast(limit=0.2, p=0.6),\n              albumentations.HueSaturationValue(hue_shift_limit=20, sat_shift_limit=20, val_shift_limit=20, p=0.6),\n              albumentations.CoarseDropout(max_holes=8, max_height=int(IMG_SIZE*0.2), max_width=int(IMG_SIZE*0.2), p=0.6),\n              albumentations.Cutout(num_holes=1, max_h_size=int(IMG_SIZE*0.33), max_w_size=int(IMG_SIZE*0.33), p=0.6)\n              ]   \n\nIS_TTA = True\nTTA = 5\n\nUPSAMPLE = False\nN_UPSAMPLE = 1\n\nSCHEDULER_NAME = 'CosineAnnealingLR' # ReduceLROnPlateau, CosineAnnealingLR, CustomSchedulerLR\nLOSS_FN_NAME = 'CrossEntropyLoss' # WeightedFocalLoss\nSMOOTHING = 0.05\n\nDISPLAY_PLOT= True\n\ndef set_seed(seed):\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    \nset_seed(seed)","execution_count":null,"outputs":[]},{"metadata":{"id":"ocRSZoaBgcn0","trusted":true},"cell_type":"code","source":"path = '../input/cassava-leaf-disease-classification/'\ntrained_path = '../input/cassava-b4-512-final/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create folds\ndf = pd.read_csv(path + 'train.csv')\nN_CLASSES = df.label.nunique()","execution_count":null,"outputs":[]},{"metadata":{"id":"x45rWUshEWPz","trusted":true},"cell_type":"code","source":"class ClassificationDataset:\n    def __init__(self, image_paths, targets, resize, augmentations=None):\n        self.image_paths = image_paths\n        self.targets = targets\n        self.resize = resize\n        self.augmentations = augmentations\n\n    def __len__(self):\n        return len(self.image_paths)\n\n    def __getitem__(self, item):\n        image = Image.open(self.image_paths[item])\n        targets = self.targets[item]\n        if self.resize is not None:\n            image = image.resize(\n                (self.resize[1], self.resize[0]), resample=Image.BILINEAR\n            )\n        image = np.array(image)\n        if self.augmentations is not None:\n            augmented = self.augmentations(image=image)\n            image = augmented[\"image\"]\n        image = np.transpose(image, (2, 0, 1)).astype(np.float32)\n        return {\n            \"image\": torch.tensor(image),\n            \"targets\": torch.tensor(targets),\n        }\n\n\nclass ClassificationDataLoader:\n    def __init__(self, image_paths, targets, resize, augmentations=None):\n        self.image_paths = image_paths\n        self.targets = targets\n        self.resize = resize\n        self.augmentations = augmentations\n        self.dataset = ClassificationDataset(\n            image_paths=self.image_paths,\n            targets=self.targets,\n            resize=self.resize,\n            augmentations=self.augmentations\n        )\n    \n    def fetch(self, batch_size, num_workers, drop_last=False, shuffle=True, tpu=False):\n        sampler = None\n\n        data_loader = torch.utils.data.DataLoader(\n            self.dataset,\n            batch_size=batch_size,\n            sampler=sampler,\n            drop_last=drop_last,\n            shuffle=shuffle,\n            num_workers=num_workers\n        )\n        return data_loader","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Engine:\n    @staticmethod\n    def train(\n        data_loader,\n        model,\n        optimizer,\n        device,\n        scheduler=None,\n        accumulation_steps=1,\n        fp16=True,\n    ):\n\n        losses = AverageMeter()\n        accuracies = AverageMeter()\n        final_predictions = []\n        model.train()\n        if accumulation_steps > 1:\n            optimizer.zero_grad()\n\n        if fp16:\n          scaler = torch.cuda.amp.GradScaler()    \n\n        for b_idx, data in enumerate(data_loader):\n            for key, value in data.items():\n                data[key] = value.to(device)\n            if accumulation_steps == 1 and b_idx == 0:\n                optimizer.zero_grad()\n            if fp16:    \n                with torch.cuda.amp.autocast():    \n                    predictions, loss, accuracy = model(**data)\n            else:\n                predictions, loss, accuracy = model(**data)\n\n            predictions = predictions.detach().cpu().numpy()  \n            final_predictions.append(predictions) \n\n            with torch.set_grad_enabled(True):\n                if fp16:\n                    scaler.scale(loss).backward()                   \n                else:\n                    loss.backward()\n                if (b_idx + 1) % accumulation_steps == 0:\n                    if fp16:\n                        scaler.step(optimizer)\n                        scaler.update()\n                    else:     \n                        optimizer.step()\n                    if scheduler is not None:\n                         scheduler.step()\n                    if b_idx > 0:\n                        optimizer.zero_grad()\n\n            losses.update(loss.item(), data_loader.batch_size)\n            accuracies.update(accuracy.item(), data_loader.batch_size)\n\n        return final_predictions, losses.avg, accuracies.avg\n\n    @staticmethod\n    def evaluate(data_loader, model, device):\n        losses = AverageMeter()\n        accuracies = AverageMeter()\n        final_predictions = []\n        model.eval()\n        with torch.no_grad():\n            for b_idx, data in enumerate(data_loader):    \n                for key, value in data.items():\n                    data[key] = value.to(device)\n                predictions, loss, accuracy = model(**data)\n                predictions = predictions.detach().cpu().numpy()  \n                final_predictions.append(predictions) \n                \n                losses.update(loss.item(), data_loader.batch_size)    \n                accuracies.update(accuracy.item(), data_loader.batch_size)\n\n        return final_predictions, losses.avg, accuracies.avg\n\n    @staticmethod\n    def predict(data_loader, model, device):\n        model.eval()\n        final_predictions = []\n\n        with torch.no_grad():\n\n            for b_idx, data in enumerate(data_loader):    \n                for key, value in data.items():\n                    data[key] = value.to(device)\n                predictions, _, _ = model(**data)\n                predictions = predictions.detach().cpu().numpy()  \n                final_predictions.append(predictions) \n                   \n        return final_predictions","execution_count":null,"outputs":[]},{"metadata":{"id":"2KKNqpqrTeMG","trusted":true},"cell_type":"code","source":"class EfficientNet(nn.Module):\n    def __init__(self, num_classes):\n        super(EfficientNet, self).__init__()\n        #self.base_model = efficientnet_pytorch.EfficientNet.from_pretrained('efficientnet-b' + str(EFFNET_MODEL))\n        self.base_model = timm.create_model(f\"tf_efficientnet_b{str(EFFNET_MODEL)}_ns\", pretrained=False)\n        self.dropout = nn.Dropout(0.2)\n        \n        self.out = nn.Linear(\n            in_features=effnet_output[EFFNET_MODEL], \n            out_features=num_classes, \n            bias=True\n        )\n        \n    def forward(self, image, targets=None):\n        batch_size, _, _, _ = image.shape\n        \n        x = self.base_model.forward_features(image) \n        x = F.adaptive_avg_pool2d(x, 1).reshape(batch_size, -1)\n        out = self.out(self.dropout(x)) \n        \n        return out, None, None    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict(fold = 0, apply_tta = False):\n    print('=' * 20, 'Fold', fold, '=' * 20)\n    test_data_path = path + \"test_images/\"\n    df = test\n    device = \"cuda\"\n    model_path=trained_path + f\"model_fold_{fold}.bin\"\n\n    mean = (0.485, 0.456, 0.406)\n    std = (0.229, 0.224, 0.225)\n    if apply_tta:\n        aug = albumentations.Compose(AUGMENTATION + [albumentations.RandomResizedCrop(IMG_SIZE, IMG_SIZE, always_apply=True), albumentations.Normalize(mean, std, max_pixel_value=255.0, always_apply=True)])\n    else:\n        aug = albumentations.Compose([albumentations.CenterCrop(IMG_SIZE, IMG_SIZE, always_apply=True),albumentations.Normalize(mean, std, max_pixel_value=255.0, always_apply=True)])\n    \n    images = [os.path.join(test_data_path, x) for x in df.image_id.values]\n    targets = df.label.values\n\n    test_loader = ClassificationDataLoader(\n        image_paths=images,\n        targets=targets,\n        resize=None,\n        augmentations=aug,\n    ).fetch(\n        batch_size=test_bsize, \n        drop_last=False, \n        num_workers=4, \n        shuffle=False\n    )\n\n    model = EfficientNet(num_classes=N_CLASSES)\n    model.load_state_dict(torch.load(model_path))\n    model.to(device)\n\n    # PREDICT\n    print('Predicting...')\n    if apply_tta:\n        predictions = np.zeros([len(images),N_CLASSES])\n        \n        for i in range(TTA): \n            tta_predictions = Engine.predict(test_loader, model, device=device)\n            tta_predictions = np.vstack(tta_predictions)\n            predictions += tta_predictions/TTA  \n        predictions = predictions.reshape((len(images),1, N_CLASSES))    \n    else:\n        predictions = Engine.predict(test_loader, model, device=device)\n\n    return predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv(path + \"sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final_preds = None\n\nfor i in range(n_folds):\n    preds = predict(fold = i, apply_tta=IS_TTA)\n    temp_preds = None\n    for p in preds:\n        if temp_preds is None:\n            temp_preds = p\n        else:\n            temp_preds = np.vstack((temp_preds, p))\n    if final_preds is None:\n        final_preds = temp_preds\n    else:\n        final_preds += temp_preds\n\nfinal_preds /= n_folds\nfinal_preds = final_preds.argmax(axis=1)\n\ntest.label = final_preds\ntest.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}