{"cells":[{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"%%time\n#!pip install '/kaggle/input/pretrainedmodels/pretrainedmodels-0.7.4/pretrainedmodels-0.7.4'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pip install torchsummary","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pip install timm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pip install cnn_finetune","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Library\n# ====================================================\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport cv2\nimport random\nimport os\nfrom tqdm import tqdm\nimport time\n\n\nimport imgaug as ia\nfrom imgaug import augmenters as iaa\n\n\nimport glob\nfrom PIL import Image\n\n\nimport torch\nimport torchvision.models as models\nimport torch.nn as nn\nimport torch.optim.lr_scheduler as lr_scheduler\nfrom torch.optim import Adam, SGD\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, CosineAnnealingLR, ReduceLROnPlateau\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom torch.cuda.amp import autocast, GradScaler\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.nn.modules.loss import _WeightedLoss\nimport torch.nn.functional as F\nfrom torchsummary import summary\nimport torchvision\nimport torchvision.transforms as transforms\nfrom albumentations.pytorch import ToTensorV2\nimport timm\n\n\nfrom cnn_finetune import make_model\n\n\nfrom sklearn.metrics import accuracy_score,f1_score,precision_score,recall_score,confusion_matrix\nfrom sklearn.model_selection import GroupKFold, StratifiedKFold\nfrom sklearn.utils.multiclass import unique_labels\n\n\nfrom albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE,\n    RandomRotate90,Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion,\n    HueSaturationValue,IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine,\n    RandomResizedCrop,IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout,\n    CoarseDropout,ShiftScaleRotate, CenterCrop, Resize\n                            )\n\n\n\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv')\ntrain.sample(5)\nsns.countplot(train.label, edgecolor = 'black',palette = sns.color_palette(\"viridis\", 5))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# -----CFG-------------------------------------------\n# ====================================================\nclass CFG:\n    seed=42\n    train_path='../input/cassava-leaf-disease-classification/train_images/'\n    test_path='../input/cassava-leaf-disease-classification/test_images/'\n    img_size=512 #256\n    target_col='label'\n    target_size=5\n    epochs=10\n    print_freq=100\n    n_fold=5\n    trn_fold=[0, 1, 2, 3, 4]\n    num_workers=4\n    batch_size=16\n    model_name='resnext50_32x4d' # #  #resnext50_32x4d #tf_efficientnet_b4_ns googlenet \n## resnet18  googlenet 1-resnext50_32x4d 2-wide_resnet50_2\n    scheduler='CosineAnnealingWarmRestarts' # ['ReduceLROnPlateau', 'CosineAnnealingLR', 'CosineAnnealingWarmRestarts']    \n    #factor=0.2 # ReduceLROnPlateau\n    #patience=4 # ReduceLROnPlateau\n    #eps=1e-6 # ReduceLROnPlateau\n    #T_max=10 # CosineAnnealingLR\n    T_0=10 # CosineAnnealingWarmRestarts\n    lr=1e-4\n    min_lr=1e-6\n    weight_decay=1e-6\n    gradient_accumulation_steps=1\n    max_grad_norm=1000\n    train=True\n    inference=False\n    accum_iter= 2\n    verbose_step=1\n    debug=False\n    apex=False\n    \nif CFG.debug:\n    CFG.epochs = 3\n    train = train.sample(n=500, random_state=CFG.seed).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = make_model('resnext50_32x4d', num_classes=CFG.target_size, pretrained=True,dropout_p=0.3)\nmodel = model.to(device)\nsummary(model, (3,  CFG.img_size,  CFG.img_size))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# -----Helper Functions-------------------------------\n# ====================================================\ndef performance_matrix(true,pred):\n    precision = precision_score(true,pred,average='macro')\n    recall = recall_score(true,pred,average='macro')\n    accuracy = accuracy_score(true,pred)\n    f1_sco = f1_score(true,pred,average='macro')\n    print('Confusion Matrix:\\n',confusion_matrix(true, pred))\n    print('Precision: {:.4f} Recall: {:.4f}, Accuracy: {:.4f}: ,f1_score: {:.4f}'.format(precision,recall,accuracy,f1_sco))\n\n\ndef seed_torch(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nseed_torch(seed=CFG.seed)\n    \ndef get_img(path):\n    im_bgr = cv2.imread(path)\n    im_rgb = im_bgr[:, :, ::-1]\n    return im_rgb\n\ndef imshow(inp, title=None):\n    inp = inp.numpy().transpose((1, 2, 0))\n    mean = np.array([0.485, 0.456, 0.406])\n    std = np.array([0.229, 0.224, 0.225])\n    inp = std * inp + mean\n    inp = np.clip(inp, 0, 1)\n    plt.imshow(inp)\n    if title is not None:\n        plt.title(title)\n    plt.pause(0.001)  # pause a bit so that plots are updated\n\nimg = get_img('../input/cassava-leaf-disease-classification/train_images/999998473.jpg')\nplt.imshow(img)\nplt.show()\n\n\nsample = train.head(4)\nplt.figure(figsize=(15, 5))\nfor ind, (image_id, label) in enumerate(zip(sample.image_id, sample.label)):\n    plt.subplot(1, 4, ind + 1)\n    img = cv2.imread(os.path.join(CFG.train_path, image_id))\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    plt.imshow(img)\n    plt.axis(\"off\")\n    \nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# -----Image Augmentations---------------------------\n# ====================================================\ndef get_transforms(*, data):\n    \n    if data == 'train':\n        return Compose([\n            #Resize(CFG.img_size, CFG.img_size),\n            RandomResizedCrop(CFG.img_size, CFG.img_size),\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            ShiftScaleRotate(p=0.5),\n            HueSaturationValue(hue_shift_limit=0.2,sat_shift_limit=0.2,val_shift_limit=0.2,p=0.5),\n            RandomBrightnessContrast(brightness_limit=(-0.1,0.1),contrast_limit=(-0.1, 0.1),p=0.5),\n            Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225],\n                max_pixel_value=255.0,\n                p=1.0\n            ),\n            CoarseDropout(p=0.5),\n            Cutout(p=0.5),\n            ToTensorV2(),\n        ])\n\n    elif data == 'valid':\n        return Compose([\n            CenterCrop(CFG.img_size, CFG.img_size, p=1.),\n            Resize(CFG.img_size, CFG.img_size),\n            Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225],\n            ),\n            ToTensorV2(),\n        ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaDataset(Dataset):\n    def __init__(self, df, data_root, transforms=None, output_label=True):\n        super().__init__()\n        self.df = df.reset_index(drop=True).copy() # data\n        self.transforms = transforms\n        self.data_root = data_root\n        self.output_label = output_label\n \n    def __len__(self):\n        return self.df.shape[0] # or len(self.df)\n    \n    def __getitem__(self, index: int):\n        \n        # get labels\n        if self.output_label:\n            target = self.df.iloc[index]['label']\n          \n        path = \"{}/{}\".format(self.data_root, self.df.iloc[index]['image_id'])\n        \n        img  = get_img(path)\n\n        if self.transforms:\n            img = self.transforms(image=img)['image']\n           \n        # do label smoothing\n        if self.output_label == True:\n            return img, target\n        else:\n            return img\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# reference: https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/173733\nclass MyCrossEntropyLoss(_WeightedLoss):\n    def __init__(self, weight=None, reduction='mean'):\n        super().__init__(weight=weight, reduction=reduction)\n        self.weight = weight\n        self.reduction = reduction\n\n    def forward(self, inputs, targets):\n        lsm = F.log_softmax(inputs, -1)\n\n        if self.weight is not None:\n            lsm = lsm * self.weight.unsqueeze(0)\n\n        loss = -(targets * lsm).sum(-1)\n\n        if  self.reduction == 'sum':\n            loss = loss.sum()\n        elif  self.reduction == 'mean':\n            loss = loss.mean()\n\n        return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassvaImgClassifierx(nn.Module): #resnext50_32x4d\n    def __init__(self, model_name, n_class, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained)\n        n_features = self.model.fc.in_features\n        self.model.fc = nn.Linear(n_features, CFG.target_size)\n\n    def forward(self, x):\n        x = self.model(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassvaImgClassifier(nn.Module): #Eff\n    def __init__(self, model_name, n_class, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained)\n        n_features = self.model.classifier.in_features\n        self.model.classifier = nn.Linear(n_features, n_class)\n    def forward(self, x):\n        x = self.model(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def Get_Dataloader(df, trn_idx, val_idx, data_root=CFG.train_path):\n    \n    from catalyst.data.sampler import BalanceClassSampler\n    \n    train_ = df.loc[trn_idx,:].reset_index(drop=True)\n    valid_ = df.loc[val_idx,:].reset_index(drop=True)\n\n    train_ds = CassavaDataset(train_, data_root, transforms=get_transforms(data='train'), output_label=True)\n    valid_ds = CassavaDataset(valid_, data_root, transforms=get_transforms(data='valid'), output_label=True)\n    \n    train_loader = torch.utils.data.DataLoader(\n        train_ds,\n        batch_size=CFG.batch_size,\n        pin_memory=False,\n        drop_last=False,\n        shuffle=True,        \n        num_workers=CFG.batch_size,\n    )\n    val_loader = torch.utils.data.DataLoader(\n        valid_ds, \n        batch_size=CFG.batch_size,\n        num_workers=CFG.batch_size,\n        shuffle=False,\n        pin_memory=False,\n    )\n    return train_loader, val_loader\n\ndef train_one_epoch(epoch, model, loss_fn, optimizer, train_loader, device, scheduler=None, schd_batch_update=False):\n    model.train()\n\n    t = time.time()\n    loss_sum = 0#1\n    sample_num = 0#2\n    image_preds_all = []#3\n    image_targets_all = []#4\n    \n    running_loss = None\n\n    pbar = tqdm(enumerate(train_loader), total=len(train_loader))\n    for step, (imgs, image_labels) in pbar:\n        imgs = imgs.to(device).float()\n        image_labels = image_labels.to(device).long()\n\n        #print(image_labels.shape, exam_label.shape)\n        with autocast():\n            image_preds = model(imgs)\n            #print(image_preds.shape, exam_pred.shape)\n            image_preds_all += [torch.argmax(image_preds, 1).detach().cpu().numpy()]\n            image_targets_all += [image_labels.detach().cpu().numpy()]\n        \n            loss = loss_fn(image_preds, image_labels)\n            \n            scaler.scale(loss).backward()\n           \n            running_loss = loss.item()\n\n            loss_sum += loss.item()*image_labels.shape[0]#1\n            sample_num += image_labels.shape[0]#2\n            loss_avg=loss_sum/sample_num#3\n            if ((step + 1) %  CFG.accum_iter == 0) or ((step + 1) == len(train_loader)):\n                # may unscale_ here if desired (e.g., to allow clipping unscaled gradients)\n\n                scaler.step(optimizer)\n                scaler.update()\n                optimizer.zero_grad() \n                \n                if scheduler is not None and schd_batch_update:\n                    scheduler.step()\n\n            if ((step + 1) % CFG.verbose_step == 0) or ((step + 1) == len(train_loader)):\n                description = f'epoch {epoch} loss: {running_loss:.4f}'\n                pbar.set_description(description)\n                \n    image_preds_all = np.concatenate(image_preds_all)#\n    image_targets_all = np.concatenate(image_targets_all)#\n    score=(image_preds_all==image_targets_all).mean() #\n    \n    if scheduler is not None and not schd_batch_update:\n        scheduler.step()\n    return score,loss_avg\n        \ndef valid_one_epoch(epoch, model, loss_fn, val_loader, device, scheduler=None, schd_loss_update=False):\n    model.eval()\n\n    t = time.time()\n    loss_sum = 0\n    sample_num = 0\n    image_preds_all = []\n    image_targets_all = []\n    \n    pbar = tqdm(enumerate(val_loader), total=len(val_loader))\n    for step, (imgs, image_labels) in pbar:\n        imgs = imgs.to(device).float()\n        image_labels = image_labels.to(device).long()\n        \n        image_preds = model(imgs)  \n        #print(image_preds.shape, exam_pred.shape)\n        image_preds_all += [torch.argmax(image_preds, 1).detach().cpu().numpy()]\n        image_targets_all += [image_labels.detach().cpu().numpy()]\n        \n        loss = loss_fn(image_preds, image_labels)\n        \n        loss_sum += loss.item()*image_labels.shape[0]\n        sample_num += image_labels.shape[0]  \n        loss_avg=loss_sum/sample_num\n        if ((step + 1) % CFG.verbose_step == 0) or ((step + 1) == len(val_loader)):\n            description = f'epoch {epoch} loss: {loss_avg:.4f}'\n            pbar.set_description(description)\n    \n    image_preds_all = np.concatenate(image_preds_all)\n    image_targets_all = np.concatenate(image_targets_all)\n    score=(image_preds_all==image_targets_all).mean()\n    performance_matrix(image_targets_all, image_preds_all)\n    if scheduler is not None:\n        if schd_loss_update:\n            scheduler.step(loss_sum/sample_num)\n        else:\n            scheduler.step()\n            \n    return score,loss_avg\n            \ndef Get_scheduler(optimizer):\n    if CFG.scheduler=='ReduceLROnPlateau':\n        scheduler = ReduceLROnPlateau(optimizer, mode='min', factor=CFG.factor, patience=CFG.patience, verbose=True, eps=CFG.eps)\n    elif CFG.scheduler=='CosineAnnealingLR':\n        scheduler = CosineAnnealingLR(optimizer, T_max=CFG.T_max, eta_min=CFG.min_lr, last_epoch=-1)\n    elif CFG.scheduler=='CosineAnnealingWarmRestarts':\n        scheduler = CosineAnnealingWarmRestarts(optimizer, T_0=CFG.T_0, T_mult=1, eta_min=CFG.min_lr, last_epoch=-1)\n    return scheduler","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if __name__ == '__main__':\n  \n    seed_torch(seed=CFG.seed)\n    # ====================================================\n    # -----Train loop-------------------------------------\n    # ====================================================\n    folds = StratifiedKFold(n_splits=CFG.n_fold,\n                            shuffle=True,\n                            random_state=CFG.seed).split(np.arange(train.shape[0]),train.label.values)\n    \n    for fold, (trn_idx, val_idx) in enumerate(folds):       \n        # we'll train fold 0 first\n        if fold > 3:\n            break \n\n        print(f\"========== fold: {fold} training ==========\")\n        print(len(trn_idx), len(val_idx))\n        \n        # ====================================================\n        # -----loader-----------------------------------------\n        # ====================================================\n        train_loader, val_loader = Get_Dataloader(train, trn_idx, val_idx)\n        \n        # ====================================================\n        # model & optimizer\n        # ====================================================\n        #------------45 pretrained---------\n        #model = make_model('resnext50_32x4d', num_classes=CFG.target_size, pretrained=True,dropout_p=0.3)\n        #model = model.to(device)\n        #------------EffNet0-7---------\n        #model = CassvaImgClassifier(CFG.model_name, train.label.nunique(), pretrained=True)\n        #model = model.to(device)\n        #-------------12 pretrained -----\n        #model = models.googlenet(pretrained=True) # resnet18  googlenet 1-resnext50_32x4d 2-wide_resnet50_2\n        #num_ftrs = model.fc.in_features\n        #model.fc = nn.Linear(num_ftrs, 5)\n        #model = model.to(device)\n        #summary(model, (3, CFG.img_size, CFG.img_size))\n        #--------------------\n        scaler = GradScaler() \n        optimizer = Adam(model.parameters(), lr=CFG.lr, weight_decay=CFG.weight_decay, amsgrad=False)\n        scheduler = Get_scheduler(optimizer)\n   \n\n        loss_tr = nn.CrossEntropyLoss().to(device) \n        loss_fn = nn.CrossEntropyLoss().to(device)\n        \n        best_score = 0\n        dresults = pd.DataFrame(columns=['Epoch','Train_Acc', 'Train_loss','Val_Acc', 'Val_loss'])\n        for epoch in range(CFG.epochs):\n            \n            train_acc,train_loss=train_one_epoch(epoch, model, loss_tr, optimizer, train_loader, device, scheduler=scheduler, schd_batch_update=False)\n            with torch.no_grad():\n                val_acc,val_loss= valid_one_epoch(epoch, model, loss_fn, val_loader, device, scheduler=None, schd_loss_update=False)\n                #----------------------------------------------\n                dresults = dresults.append({'Epoch': epoch+1,'Train_Acc': train_acc, 'Train_loss':train_loss,'Val_Acc': val_acc, 'Val_loss':val_loss},ignore_index=True)\n                #----------------------------------------------\n                print('Epoch= {}'.format(epoch+1))\n                print('Train_Acc  = {:.5f},  Train_loss={:.5f}'.format(train_acc,train_loss))\n                print('Val_Acc  = {:.5f},    Val_loss={:.5f}'.format(val_acc,val_loss))\n                #----------------------------------------------\n                if val_acc > best_score:\n                    best_score = val_acc\n                    torch.save(model.state_dict(),'{}_fold_best_{}'.format(CFG.model_name,fold))\n                #----------------------------------------------\n        filename='dresults' + str(fold) + '.csv'\n        dresults.to_csv(filename, index=False)         \n        del optimizer, train_loader, val_loader, scaler, scheduler\n        torch.cuda.empty_cache()\n\n\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}