{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install pretrainedmodels","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nimport tqdm\nimport pathlib\nimport shutil\nimport os \nimport torch\nimport seaborn as sns\nimport torch.nn as nn\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport cv2\nimport gc\n\nfrom PIL import Image\nfrom sklearn.model_selection import StratifiedKFold\nfrom torch.utils.data import Dataset, DataLoader\nfrom collections import OrderedDict\nfrom torch import optim\nfrom torchvision import datasets, transforms, models\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\n\nimport pretrainedmodels\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nseed_everything(721)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data = \"../input/cassava-leaf-disease-classification/train_images\"\ntrain_df = pd.read_csv(\"../input/cassava-leaf-disease-classification/train.csv\")\ntrain_data_2019 = \"../input/train-images-augumented/train_2019\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## split into folds\ntrain_df_vs = train_df.copy()\ns_fold = StratifiedKFold(n_splits=5, shuffle=True)\nfor n, (train_index, val_index) in enumerate(s_fold.split(train_df_vs, train_df_vs['label'])):\n    train_df_vs.loc[val_index, 'fold'] = int(n)\ntrain_df_vs['fold'] = train_df_vs['fold'].astype(int)\nprint(train_df_vs.groupby(['fold', 'label']).size()) \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaDataset(Dataset):\n    def __init__(self, df, root, transform=None):\n        self.df = df\n        self.file_names = df['image_id'].values\n        self.labels = df['label'].values\n        self.transform = transform\n        self.root = root\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        file_name = self.file_names[index]\n        file_path = f'{self.root}/{file_name}'\n#         image = Image.open(file_path)\n        image = cv2.imread(file_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n       \n        if self.transform:\n            image = self.transform(image=image)[\"image\"]\n        label = torch.tensor(self.labels[index]).long()\n        return image, label\n        \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#transformations\n# train_transforms = transforms.Compose([\n#     transforms.Resize((512, 512)),\n#     transforms.RandomHorizontalFlip(),\n#     transforms.RandomVerticalFlip(),\n#     transforms.ToTensor(),\n#     transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])\n# ])\n# val_transforms = transforms.Compose([\n#     transforms.Resize((512, 512)),\n#     transforms.ToTensor(),\n#     transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])\n# ])\n\n\nfrom albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize\n)\n\nfrom albumentations.pytorch import ToTensorV2\n\nfrom albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize\n)\n\nfrom albumentations.pytorch import ToTensorV2\n\ndef get_train_transforms():\n    return Compose([\n            CenterCrop(512, 512, p=1.),\n            Resize(512, 512),\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            ShiftScaleRotate(p=0.5),\n            HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n            RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            CoarseDropout(p=0.5),\n            Cutout(p=0.5),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n  \n        \ndef get_valid_transforms():\n    return Compose([\n            CenterCrop(512, 512, p=1.),\n            Resize(512, 512),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n\n# def get_inference_transforms():\n#     return Compose([\n#             RandomResizedCrop(512, 512),\n#             Transpose(p=0.5),\n#             HorizontalFlip(p=0.5),\n#             VerticalFlip(p=0.5),\n#             HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n#             RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n#             Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n#             ToTensorV2(p=1.0),\n#         ], p=1.)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model = models.resnext50_32x4d(pretrained=True)\n\n# for param in model.parameters():\n#     param.required_grad = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from collections import OrderedDict\n\n# classifier = nn.Sequential(OrderedDict([\n#               ('fc1', nn.Linear(2048, 126)),\n#               ('relu1', nn.ReLU()),\n#               ('dropout1', nn.Dropout(p=0.3)),\n# #               ('fc2', nn.Linear(256, 126)),\n# #               ('relu2', nn.ReLU()),\n# #               ('dropout2', nn.Dropout(p=0.2)),\n#               ('fc3', nn.Linear(126, 5)),\n             \n              \n# ]))\n\n# model.fc = classifier","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# criterion = nn.CrossEntropyLoss()\n\n# plist = [{'params': model.parameters(), 'lr': 0.00003}]\n# optimizer = optim.Adam(model.parameters(), lr=1e-4, weight_decay=1e-6)\n\n# optimizer = optim.SGD(model.parameters(), lr = 0.001, momentum=0.9)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TaylorSoftmax(nn.Module):\n\n    def __init__(self, dim=1, n=2):\n        super(TaylorSoftmax, self).__init__()\n        assert n % 2 == 0\n        self.dim = dim\n        self.n = n\n\n    def forward(self, x):\n        \n        fn = torch.ones_like(x)\n        denor = 1.\n        for i in range(1, self.n+1):\n            denor *= i\n            fn = fn + x.pow(i) / denor\n        out = fn / fn.sum(dim=self.dim, keepdims=True)\n        return out\n\nclass LabelSmoothingLoss(nn.Module):\n\n    def __init__(self, classes, smoothing=0.0, dim=-1): \n        super(LabelSmoothingLoss, self).__init__() \n        self.confidence = 1.0 - smoothing \n        self.smoothing = smoothing \n        self.cls = classes \n        self.dim = dim \n    def forward(self, pred, target): \n        \"\"\"Taylor Softmax and log are already applied on the logits\"\"\"\n        #pred = pred.log_softmax(dim=self.dim) \n        with torch.no_grad(): \n            true_dist = torch.zeros_like(pred) \n            true_dist.fill_(self.smoothing / (self.cls - 1)) \n            true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence) \n        return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))\n    \n\nclass TaylorCrossEntropyLoss(nn.Module):\n\n    def __init__(self, n=2, ignore_index=-1, reduction='mean', smoothing=0.2):\n        super(TaylorCrossEntropyLoss, self).__init__()\n        assert n % 2 == 0\n        self.taylor_softmax = TaylorSoftmax(dim=1, n=n)\n        self.reduction = reduction\n        self.ignore_index = ignore_index\n        self.lab_smooth = LabelSmoothingLoss(5, smoothing=smoothing)\n\n    def forward(self, logits, labels):\n\n        log_probs = self.taylor_softmax(logits).log()\n        #loss = F.nll_loss(log_probs, labels, reduction=self.reduction,\n        #        ignore_index=self.ignore_index)\n        loss = self.lab_smooth(log_probs, labels)\n        return loss\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"        \ncriterion = TaylorCrossEntropyLoss(n=2, smoothing=0.3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def se_resnext50_32x4d(pretrained=False):\n    pretrained = 'imagenet' if pretrained else None\n    model = pretrainedmodels.se_resnext50_32x4d(pretrained=pretrained)\n    return model\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_2019 = pd.read_csv(\"../input/train-imagaes-aug/train_2019.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_2019 = train_2019.drop([\"Unnamed: 0\"], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs =6\nfolds = StratifiedKFold(n_splits=5, shuffle=True, ).split(np.arange(train_df.shape[0]), train_df.label.values)\nfor fold, (trn_idx, val_idx) in enumerate(folds):\n    ### initate model here so we can del the weights later\n   \n #     model = models.resnext50_32x4d(pretrained=True)\n\n#     for param in model.parameters():\n#         param.required_grad = False\n\n#     classifier = nn.Sequential(OrderedDict([\n#               ('fc1', nn.Linear(2048, 126)),\n#               ('relu1', nn.ReLU()),\n#              ('dropout1', nn.Dropout(p=0.2)),\n\n#              ('fc3', nn.Linear(126, 5)),\n\n\n#                 ]))\n\n#     model.fc = classifier\n    model = se_resnext50_32x4d(pretrained=True)\n    \n    for param in model.parameters():\n        param.required_grad = False\n    model.avg_pool = nn.AdaptiveAvgPool2d(1)\n    model.dropout= nn.Dropout(0.3)\n    num_ftrs = model.last_linear.in_features\n    \n    model.last_linear = nn.Linear(num_ftrs, 5)\n    \n    optimizer = optim.Adam(model.parameters(), lr=1e-4, weight_decay=1e-6, amsgrad=True)\n    \n    model.to(device)\n    \n    model.class_to_idx = {'0': 0, '1': 1, '2': 2, '3': 3, '4': 4}\n    \n    print(f\"========== fold: {fold} training ==========\")\n    \n    \n    train_ = train_df.loc[trn_idx,:].reset_index(drop=True)\n    valid_ = train_df.loc[val_idx,:].reset_index(drop=True)\n    \n   \n    train_dataset = CassavaDataset(train_, train_data,\n                                     transform=get_train_transforms())\n    \n    valid_dataset = CassavaDataset(valid_, train_data,\n                                     transform=get_valid_transforms())\n  \n    train_loader = DataLoader(train_dataset, \n                                  batch_size=16, \n                                  shuffle=True, \n                                  num_workers=4)\n    val_loader = DataLoader(valid_dataset, \n                                  batch_size=16, \n                                  shuffle=False, \n                                  num_workers=4)\n\n        \n        \n    for epoch in range(epochs):\n        print(f\"========== epoch: {epoch} training ==========\")\n        running_loss = 0\n        for images, labels in train_loader:\n            images, labels = images.to(device), labels.to(device)\n\n            optimizer.zero_grad()\n\n            outputs = model(images)\n            loss = criterion(outputs, labels)\n            loss.backward()\n            optimizer.step()\n\n            running_loss += loss.item()\n\n        else:\n            validation_loss = 0\n            accuracy = 0\n\n            with torch.no_grad():\n                model.eval()\n                for images, labels in val_loader:\n                    images, labels = images.to(device), labels.to(device)\n                    outputs = model(images)\n                    loss = criterion(outputs, labels)\n                    validation_loss += loss.item()\n\n\n                    ps = torch.exp(outputs)\n\n                    top_p, top_class = ps.topk(1, dim=1)\n                    equals = top_class == labels.view(*top_class.shape)\n                    accuracy += torch.mean(equals.type(torch.FloatTensor))\n\n            model.train()\n\n            print(\"Epoch: {}/{}.. \".format(epoch+1, epochs),\n                  \"Training Loss: {:.3f}.. \".format(running_loss/len(train_loader)),\n                  \"Valid Loss: {:.3f}.. \".format(validation_loss/len(val_loader)),\n              \"Valid Accuracy: {:.3f}\".format(accuracy/len(val_loader)))\n            torch.save({\"state_dict\":model.state_dict(),\n           \"class_to_idx\":model.class_to_idx}, '{}_rese_cassava_classifier_fold_{}.pth'.format(epoch+1, fold))\n    del model\n    gc.collect()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 0 = 66\n# 1 = 385\n# 2 = 392\n# 3 = 6210\n# 4 = 45","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"    \"0\":string\"Cassava Bacterial Blight (CBB)\"  - 1087\n    \"1\":string\"Cassava Brown Streak Disease (CBSD)\" - 2189\n    \"2\":string\"Cassava Green Mottle (CGM)\" - 2386\n    \"3\":string\"Cassava Mosaic Disease (CMD)\" - 13158\n    \"4\":string\"Healthy\" - 2577\n"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}