{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nfrom collections import OrderedDict\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n# test = pd.DataFrame()\n# test['image_id'] = list(os.listdir('../input/cassava-leaf-disease-classification/test_images/'))\n# print(test.iloc[0,0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"! pip install ../input/timm-package/timm-0.1.26-py3-none-any.whl","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport cv2\nimport copy\nimport time\nimport random\nimport pickle\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n#import seaborn as sns\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nimport torchvision\nfrom torchvision import models\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.cuda import amp\n#from apex import amp\n#from torch.cuda.amp import autocast, GradScaler\nfrom tqdm import tqdm\n\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.utils import class_weight\n\nfrom collections import defaultdict\nimport albumentations as A\n\n# from albumentations import (\n#     HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n#     Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n#     IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n#     IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout,\n#     ShiftScaleRotate, CenterCrop, Resize\n# )\nfrom albumentations.pytorch import ToTensorV2\n\nimport timm\n#import pretrainedmodels\n\nclass CFG:\n    model_name = 'tf_efficientnet_b4_ns'\n    img_size = 512\n    scheduler = 'CosineAnnealingWarmRestarts'\n    T_max = 10\n    T_0 = 10\n    lr = 1e-4\n    min_lr = 1e-6\n    batch_size = 16\n    weight_decay = 1e-6\n    seed = 42\n    num_classes = 5\n    num_epochs = 10\n    n_fold = 5\n    NUM_FOLDS_TO_RUN = [2, ]\n    smoothing = 0.2\n    device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n\ndef set_seed(seed=42):\n    '''Sets the seed of the entire notebook so results are the same every time we run.\n    This is for REPRODUCIBILITY.'''\n    np.random.seed(seed)\n    random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    # When running on the CuDNN backend, two further options must be set\n    torch.backends.cudnn.deterministic = True\n    # Set a fixed value for the hash seed\n    os.environ['PYTHONHASHSEED'] = str(seed)\n\nclass CassavaLeafDataset(nn.Module):\n    def __init__(self, root_dir, df, transforms=None):\n        self.root_dir = root_dir\n        self.df = df\n        self.transforms = transforms\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, index):\n        img_path = os.path.join(self.root_dir, self.df.iloc[index, 0])\n        img = cv2.imread(img_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        #label = self.df.iloc[index, 1]\n\n        if self.transforms:\n            img = self.transforms(image=img)[\"image\"]\n\n        return img#, label\n\ndata_transforms = {\n    \"train\": A.Compose([\n        A.RandomResizedCrop(CFG.img_size, CFG.img_size),\n        A.Transpose(p=0.5),\n        A.HorizontalFlip(p=0.5),\n        A.VerticalFlip(p=0.5),\n        A.ShiftScaleRotate(p=0.5),\n        A.HueSaturationValue(\n            hue_shift_limit=0.2,\n            sat_shift_limit=0.2,\n            val_shift_limit=0.2,\n            p=0.5\n        ),\n        A.RandomBrightnessContrast(\n            brightness_limit=(-0.1, 0.1),\n            contrast_limit=(-0.1, 0.1),\n            p=0.5\n        ),\n        A.Normalize(\n            mean=[0.485, 0.456, 0.406],\n            std=[0.229, 0.224, 0.225],\n            max_pixel_value=255.0,\n            p=1.0\n        ),\n        A.CoarseDropout(p=0.5),\n        A.Cutout(p=0.5),\n        ToTensorV2()], p=1.),\n\n    \"valid\": A.Compose([\n        A.CenterCrop(CFG.img_size, CFG.img_size, p=1.),\n        A.Resize(CFG.img_size, CFG.img_size),\n        A.Normalize(\n            mean=[0.485, 0.456, 0.406],\n            std=[0.229, 0.224, 0.225],\n            max_pixel_value=255.0,\n            p=1.0\n        ),\n        ToTensorV2()], p=1.),\n    \n    \"test\":A.Compose([\n        RandomResizedCrop(CFG.img_size, CFG.img_size),\n        Transpose(p=0.5),\n        HorizontalFlip(p=0.5),\n        VerticalFlip(p=0.5),\n        HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n        RandomBrightnessContrast(brightness_limit=(-0.1, 0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n        Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n        ToTensorV2(p=1.0),\n    ], p=1.)\n    \n}\ndef get_inference_transforms():\n    return Compose([\n        RandomResizedCrop(CFG.img_size, CFG.img_size),\n        Transpose(p=0.5),\n        HorizontalFlip(p=0.5),\n        VerticalFlip(p=0.5),\n        HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n        RandomBrightnessContrast(brightness_limit=(-0.1, 0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n        Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n        ToTensorV2(p=1.0),\n    ], p=1.)\n\nclass TaylorSoftmax(nn.Module):\n\n    def __init__(self, dim=1, n=2):\n        super(TaylorSoftmax, self).__init__()\n        assert n % 2 == 0\n        self.dim = dim\n        self.n = n\n\n    def forward(self, x):\n        fn = torch.ones_like(x)\n        denor = 1.\n        for i in range(1, self.n + 1):\n            denor *= i\n            fn = fn + x.pow(i) / denor\n        out = fn / fn.sum(dim=self.dim, keepdims=True)\n        return out\n\nclass LabelSmoothingLoss(nn.Module):\n\n    def __init__(self, classes, smoothing=0.0, dim=-1):\n        super(LabelSmoothingLoss, self).__init__()\n        self.confidence = 1.0 - smoothing\n        self.smoothing = smoothing\n        self.cls = classes\n        self.dim = dim\n\n    def forward(self, pred, target):\n        \"\"\"Taylor Softmax and log are already applied on the logits\"\"\"\n        # pred = pred.log_softmax(dim=self.dim)\n        with torch.no_grad():\n            true_dist = torch.zeros_like(pred)\n            true_dist.fill_(self.smoothing / (self.cls - 1))\n            true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)\n        return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))\n\nclass TaylorCrossEntropyLoss(nn.Module):\n\n    def __init__(self, n=2, ignore_index=-1, reduction='mean', smoothing=0.2):\n        super(TaylorCrossEntropyLoss, self).__init__()\n        assert n % 2 == 0\n        self.taylor_softmax = TaylorSoftmax(dim=1, n=n)\n        self.reduction = reduction\n        self.ignore_index = ignore_index\n        self.lab_smooth = LabelSmoothingLoss(CFG.num_classes, smoothing=smoothing)\n\n    def forward(self, logits, labels):\n        log_probs = self.taylor_softmax(logits).log()\n        # loss = F.nll_loss(log_probs, labels, reduction=self.reduction,\n        #        ignore_index=self.ignore_index)\n        loss = self.lab_smooth(log_probs, labels)\n        return loss\n\ndef train_model(model, criterion, optimizer, scheduler, num_epochs, dataloaders, dataset_sizes, fold):\n    start = time.time()\n    best_model_wts = copy.deepcopy(model.state_dict())\n    best_acc = 0.0\n    history = defaultdict(list)\n    scaler = amp.GradScaler()\n\n    for epoch in range(1, num_epochs + 1):\n        print('Epoch {}/{}'.format(epoch, num_epochs))\n        print('-' * 10)\n\n        # Each epoch has a training and validation phase\n        for phase in ['train', 'valid']:\n            if (phase == 'train'):\n                model.train()  # Set model to training mode\n            else:\n                model.eval()  # Set model to evaluation mode\n\n            running_loss = 0.0\n            running_corrects = 0.0\n\n            # Iterate over data\n            for inputs, labels in tqdm(dataloaders[phase]):\n                inputs = inputs.cuda()#to(CFG.device)\n                labels = labels.cuda()#to(CFG.device)\n\n                # zero the parameter gradients\n                optimizer.zero_grad()\n\n                # forward\n                # track history if only in train\n                with torch.set_grad_enabled(phase == 'train'):\n                    with amp.autocast():\n                        outputs = model(inputs)\n                        _, preds = torch.max(outputs, 1)\n                        loss = criterion(outputs, labels)\n\n                    # backward + optimize only if in training phase\n                    if phase == 'train':\n                        scaler.scale(loss).backward()\n                        scaler.step(optimizer)\n                        scaler.update()\n\n                running_loss += loss.item() * inputs.size(0)\n                running_corrects += torch.sum(preds == labels.data).double().item()\n\n            epoch_loss = running_loss / dataset_sizes[phase]\n            epoch_acc = running_corrects / dataset_sizes[phase]\n\n            history[phase + ' loss'].append(epoch_loss)\n            history[phase + ' acc'].append(epoch_acc)\n\n            if phase == 'train' and scheduler != None:\n                scheduler.step()\n\n            print('{} Loss: {:.4f} Acc: {:.4f}'.format(\n                phase, epoch_loss, epoch_acc))\n\n            # deep copy the model\n            if phase == 'valid' and epoch_acc >= best_acc:\n                best_acc = epoch_acc\n                best_model_wts = copy.deepcopy(model.state_dict())\n                PATH = f\"Fold{fold}_{best_acc}_epoch{epoch}.bin\"\n                torch.save(model.state_dict(), PATH)\n\n        print()\n\n    end = time.time()\n    time_elapsed = end - start\n    print('Training complete in {:.0f}h {:.0f}m {:.0f}s'.format(\n        time_elapsed // 3600, (time_elapsed % 3600) // 60, (time_elapsed % 3600) % 60))\n    print(\"Best Accuracy \", best_acc)\n\n    # load best model weights\n    model.load_state_dict(best_model_wts)\n    return model, history, best_acc\n#cbf device\ndef run_fold(model, criterion, optimizer, scheduler, fold, num_epochs=10):\n    valid_df = df[df.kfold == fold]\n    train_df = df[df.kfold != fold]\n\n    train_data = CassavaLeafDataset(TRAIN_DIR, train_df, transforms=data_transforms[\"train\"])\n    valid_data = CassavaLeafDataset(TRAIN_DIR, valid_df, transforms=data_transforms[\"valid\"])\n\n    dataset_sizes = {\n        'train': len(train_data),\n        'valid': len(valid_data)\n    }\n\n    train_loader = DataLoader(dataset=train_data, batch_size=CFG.batch_size, num_workers=4, pin_memory=True,\n                              shuffle=True)\n    valid_loader = DataLoader(dataset=valid_data, batch_size=CFG.batch_size, num_workers=4, pin_memory=True,\n                              shuffle=False)\n\n    dataloaders = {\n        'train': train_loader,\n        'valid': valid_loader\n    }\n\n    model, history, best_acc = train_model(model, criterion, optimizer, scheduler, num_epochs, dataloaders,\n                                           dataset_sizes, fold)\n\n    return model, history, best_acc#\n\ndef fetch_scheduler(optimizer):\n    if CFG.scheduler == 'CosineAnnealingLR':\n        scheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=CFG.T_max, eta_min=CFG.min_lr)\n    elif CFG.scheduler == 'CosineAnnealingWarmRestarts':\n        scheduler = lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=CFG.T_0, T_mult=1, eta_min=CFG.min_lr)\n    elif CFG.scheduler == None:\n        return None\n\n    return scheduler\n\ndef inference_one_epoch(model, data_loader, device):\n    model.eval()\n\n    image_preds_all = []\n\n    pbar = tqdm(enumerate(data_loader), total=len(data_loader))\n    for step, (imgs) in pbar:\n        imgs = imgs.to(device).float()\n\n        image_preds = model(imgs)  # output = model(input)\n        image_preds_all += [torch.softmax(image_preds, 1).detach().cpu().numpy()]\n\n    image_preds_all = np.concatenate(image_preds_all, axis=0)\n    return image_preds_all\n\n    \n#     TEST_DIR = \"../input/cassava-leaf-disease-classification/test_images\"\n\n#     set_seed(CFG.seed)\n#     df = pd.read_csv('/home/cvpr/pycharmproject/leaf_disease_classification/cassava-leaf-disease-classification/csv/train_20_21397.csv')\n\n#     skf = StratifiedKFold(n_splits=CFG.n_fold)\n#     for fold, (_, val_) in enumerate(skf.split(X=df, y=df.label)):\n#         df.loc[val_, \"kfold\"] = int(fold)\n\n#     df['kfold'] = df['kfold'].astype(int)\n\n#     model = timm.create_model(CFG.model_name, pretrained=True)\n#     num_features = model.classifier.in_features\n#     model.classifier = nn.Linear(num_features, CFG.num_classes)\n#     model.cuda()#cbf to(CFG.device)\n#     #cbf\n#     if torch.cuda.device_count() > 1:\n#        model=nn.DataParallel(model)\n\n#     optimizer = optim.Adam(model.parameters(), lr=CFG.lr, weight_decay=CFG.weight_decay, amsgrad=False)\n#     criterion = TaylorCrossEntropyLoss(n=2, smoothing=0.2)\n\n#     scheduler = fetch_scheduler(optimizer)\n\n#     accs = []\n#     for fold in CFG.NUM_FOLDS_TO_RUN:\n#         print(f\"\\n\\nFOLD: {fold}\\n\\n\")\n#         model, history, ba = run_fold(model, criterion, optimizer, scheduler, fold=fold,\n#                                       num_epochs=CFG.num_epochs) #cbf device=CFG.device\n#         accs.append(ba)\n\n\n#     print(f\"MEAN_ACC - {sum(accs) / len(accs)}\")\nif __name__ == '__main__':\n    # for training only, need nightly build pytorch\n\n    set_seed(CFG.seed)\n\n    print('Inference started')\n\n    test = pd.DataFrame()\n    test['image_id'] = list(os.listdir('../input/cassava-leaf-disease-classification/test_images/'))\n    #print(test.iloc[0,0])\n    \n    test_ds = CassavaLeafDataset('../input/cassava-leaf-disease-classification/test_images/',test,transforms=data_transforms['test'])#cbf, output_label=False)\n\n\n    tst_loader = torch.utils.data.DataLoader(\n            test_ds,\n            batch_size=16,\n            num_workers=4,\n            shuffle=False,\n            pin_memory=True,\n        )\n\n    device = torch.device(CFG.device)\n    #model = CassvaImgClassifier(CFG['model_arch'], train.label.nunique()).to(device)\n    model = timm.create_model(CFG.model_name, pretrained=False)\n    num_features = model.classifier.in_features\n    model.classifier = nn.Linear(num_features, CFG.num_classes)\n    model.to(CFG.device)\n\n#         val_preds = []\n    tst_preds = []\n\n        # for epoch in range(CFG['epochs']-3):\n        #for i, epoch in enumerate(CFG['used_epochs']):\n    for i in range(1):\n            # model.load_state_dict(torch.load(\n            #     '../input/pytorch-efficientnet-baseline-train-amp-aug/{}_fold_{}_{}'.format(CFG['model_arch'], fold,\n            #                                                                                 epoch)))\n            #model.load_state_dict(torch.load('../input/models-2-3/tf_efficientnet_b4_ns_fold_4_9.pth',map_location={'cuda:1':'cuda:0'}))\n        ckpt=torch.load('../input/model-2-4-new/Fold2_0.8995092311287683_epoch7.bin',map_location={'cuda:1':'cuda:0'})\n            #ckpt_model_dict = remove_prefix(ckpt, 'module.')\n        #print(ckpt.keys())\n        ckpt_model_dict = OrderedDict([(k[7:],v) if 'module.' in k else (k,v) for k, v in ckpt.items()])\n        model.load_state_dict(ckpt_model_dict)\n\n\n        with torch.no_grad():\n                #for _ in range(CFG['tta']):\n                #                     val_preds += [\n#                         CFG['weights'][i] / sum(CFG['weights']) / CFG['tta'] * inference_one_epoch(model, val_loader,\n#                                                                                                    device)]\n                   # tst_preds += [\n                    #    CFG['weights'][i] / sum(CFG['weights']) / CFG['tta'] * inference_one_epoch(model, tst_loader,\n                                                                                                  # device)]\n            tst_preds += [inference_one_epoch(model, tst_loader,device)]\n\n#         val_preds = np.mean(val_preds, axis=0)\n        tst_preds = np.mean(tst_preds, axis=0)\n\n#         print('fold {} validation loss = {:.5f}'.format(fold, log_loss(valid_.label.values, val_preds)))\n#         print('fold {} validation accuracy = {:.5f}'.format(fold, (\n#                     valid_.label.values == np.argmax(val_preds, axis=1)).mean()))\n\n        del model\n        torch.cuda.empty_cache()\n        test['label'] = np.argmax(tst_preds, axis=1)\n        test.head()\n        test.to_csv('submission.csv', index=False)\n        print(test)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}