{"cells":[{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"package_path = '../input/pytorch-image-models/pytorch-image-models-master' #'../input/efficientnet-pytorch-07/efficientnet_pytorch-0.7.0'\nimport sys; sys.path.append(package_path)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"from glob import glob\nfrom sklearn.model_selection import GroupKFold, StratifiedKFold\nimport cv2\nfrom skimage import io\nimport torch\nfrom torch import nn\nimport os\nfrom datetime import datetime\nimport time\nimport random\nimport cv2\nimport torchvision\nfrom torchvision import transforms\nimport pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\nimport seaborn as sn\n\nimport matplotlib.pyplot as plt\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom  torch.cuda.amp import autocast, GradScaler\n\nimport sklearn\nimport warnings\nimport joblib\nfrom sklearn.metrics import roc_auc_score, log_loss\nfrom sklearn import metrics\nimport warnings\nimport cv2\nimport pydicom\nimport timm #from efficientnet_pytorch import EfficientNet\nfrom scipy.ndimage.interpolation import zoom\nfrom sklearn.metrics import log_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"CFG = {\n    'fold_num': 5,\n    'seed': 719,\n    'model_arch': 'tf_efficientnet_b4_ns',\n    'img_size': 512,\n    'epochs': 10,\n    'train_bs': 32,\n    'valid_bs': 32,\n    'lr': 1e-4,\n    'num_workers': 4,\n    'accum_iter': 1, # suppoprt to do batch accumulation for backprop with effectively larger batch size\n    'verbose_step': 1,\n    'device': 'cuda:0',\n    'tta': 3,\n    'used_epochs': [6,7,8,9],\n    'weights': [1,1,1,1]\n}","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"_kg_hide-input":true,"_kg_hide-output":false},"cell_type":"code","source":"train = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv')\n# train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":false},"cell_type":"code","source":"submission = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\n# submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":false},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n    \ndef get_img(path):\n    im_bgr = cv2.imread(path)\n    im_rgb = im_bgr[:, :, ::-1]\n    #print(im_rgb)\n    return im_rgb\n\n# img = get_img('../input/cassava-leaf-disease-classification/train_images/1000015157.jpg')\n# plt.imshow(img)\n# plt.show()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"BASE_DIR = \"../input/cassava-leaf-disease-classification/\"\n\ndef visualize_batch(image_ids, labels):\n    plt.figure(figsize=(16, 12))\n    \n    for ind, (image_id, label) in enumerate(zip(image_ids, labels)):\n        plt.subplot(3, 3, ind + 1)\n        image = cv2.imread(os.path.join(BASE_DIR, \"train_images\", image_id))\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\n        plt.imshow(image)\n        plt.title(f\"Class: {label}\", fontsize=12)\n        plt.axis(\"off\")\n    \n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"class CassavaDataset(Dataset):\n    def __init__(\n        self, df, data_root, transforms=None, output_label=True\n    ):\n        \n        super().__init__()\n        self.df = df.reset_index(drop=True).copy()\n        self.transforms = transforms\n        self.data_root = data_root\n        self.output_label = output_label\n    \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, index: int):\n        \n        # get labels\n        if self.output_label:\n            target = self.df.iloc[index]['label']\n          \n        path = \"{}/{}\".format(self.data_root, self.df.iloc[index]['image_id'])\n        \n        img  = get_img(path)\n        \n        if self.transforms:\n            img = self.transforms(image=img)['image']\n            \n        # do label smoothing\n        if self.output_label == True:\n            return img, target\n        else:\n            return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"from albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize\n)\n\nfrom albumentations.pytorch import ToTensorV2\n\nfrom albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize\n)\n\nfrom albumentations.pytorch import ToTensorV2\n\ndef get_train_transforms():\n    return Compose([\n            RandomResizedCrop(CFG['img_size'], CFG['img_size']),\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            ShiftScaleRotate(p=0.5),\n            HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n            RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            CoarseDropout(p=0.5),\n            Cutout(p=0.5),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n  \n        \ndef get_valid_transforms():\n    return Compose([\n            CenterCrop(CFG['img_size'], CFG['img_size'], p=1.),\n            Resize(CFG['img_size'], CFG['img_size']),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n\ndef get_inference_transforms():\n    return Compose([\n            RandomResizedCrop(CFG['img_size'], CFG['img_size']),\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n            RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"class CassvaImgClassifier(nn.Module):\n    def __init__(self, model_arch, n_class, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_arch, pretrained=pretrained)\n        n_features = self.model.classifier.in_features\n        self.model.classifier = nn.Linear(n_features, n_class)\n        \n    def forward(self, x):\n        x = self.model(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"def inference_one_epoch(model, data_loader, device):\n    model.eval()\n\n    image_preds_all = []\n    \n    pbar = tqdm(enumerate(data_loader), total=len(data_loader))\n    for step, (imgs) in pbar:\n        imgs = imgs.to(device).float()\n        \n        image_preds = model(imgs)   #output = model(input)\n        image_preds_all += [torch.softmax(image_preds, 1).detach().cpu().numpy()]\n        \n    \n    image_preds_all = np.concatenate(image_preds_all, axis=0)\n    return image_preds_all","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"if __name__ == '__main__':\n     # for training only, need nightly build pytorch\n\n    seed_everything(CFG['seed'])\n    \n    folds = StratifiedKFold(n_splits=CFG['fold_num']).split(np.arange(train.shape[0]), train.label.values)\n    \n    for fold, (trn_idx, val_idx) in enumerate(folds):\n        # we'll train fold 0 first\n        if fold > 0:\n            break \n\n        print('Inference fold {} started'.format(fold))\n\n        valid_ = train.loc[val_idx,:].reset_index(drop=True)\n        valid_ds = CassavaDataset(valid_, '../input/cassava-leaf-disease-classification/train_images/', transforms=get_inference_transforms(), output_label=False)\n        \n        test = pd.DataFrame()\n        test['image_id'] = list(os.listdir('../input/cassava-leaf-disease-classification/test_images/'))\n        test_ds = CassavaDataset(test, '../input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms(), output_label=False)\n        \n        val_loader = torch.utils.data.DataLoader(\n            valid_ds, \n            batch_size=CFG['valid_bs'],\n            num_workers=CFG['num_workers'],\n            shuffle=False,\n            pin_memory=False,\n        )\n        \n        tst_loader = torch.utils.data.DataLoader(\n            test_ds, \n            batch_size=CFG['valid_bs'],\n            num_workers=CFG['num_workers'],\n            shuffle=False,\n            pin_memory=False,\n        )\n\n        device = torch.device(CFG['device'])\n        model = CassvaImgClassifier(CFG['model_arch'], train.label.nunique()).to(device)\n        \n        val_preds = []\n        tst_preds = []\n        \n        #for epoch in range(CFG['epochs']-3):\n        for i, epoch in enumerate(CFG['used_epochs']):    \n            model.load_state_dict(torch.load('../input/pytorch-efficientnet-baseline-train-amp-aug/{}_fold_{}_{}'.format(CFG['model_arch'], fold, epoch)))\n            \n            with torch.no_grad():\n                for _ in range(CFG['tta']):\n                    val_preds += [CFG['weights'][i]/sum(CFG['weights'])/CFG['tta']*inference_one_epoch(model, val_loader, device)]\n                    tst_preds += [CFG['weights'][i]/sum(CFG['weights'])/CFG['tta']*inference_one_epoch(model, tst_loader, device)]\n\n        val_preds = np.mean(val_preds, axis=0) \n        tst_preds = np.mean(tst_preds, axis=0) \n        \n        print('fold {} validation loss = {:.5f}'.format(fold, log_loss(valid_.label.values, val_preds)))\n        print('fold {} validation accuracy = {:.5f}'.format(fold, (valid_.label.values==np.argmax(val_preds, axis=1)).mean()))\n        \n        del model\n        torch.cuda.empty_cache()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":false},"cell_type":"code","source":"test['label'] = np.argmax(tst_preds, axis=1)\n# test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"test.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":false,"trusted":true},"cell_type":"code","source":"valid_positive = valid_.loc[(valid_.label.values==np.argmax(val_preds, axis=1))].reset_index(drop = True)\nvalid_positive['prediction'] = np.argmax(val_preds, axis=1)[(valid_.label.values==np.argmax(val_preds, axis=1))]\n# valid_positive","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Cassava Leaf Disease Classification - Results Analysis\n\n\nThis code is based on [Pytorch Efficientnet Baseline [Inference] TTA](https://www.kaggle.com/khyeh0719/pytorch-efficientnet-baseline-inference-tta) and [Cassava Leaf Disease - Exploratory Data Analysis](https://www.kaggle.com/ihelon/cassava-leaf-disease-exploratory-data-analysis)\n\nThis is results analysis of [Pytorch Efficientnet Baseline [Inference] TTA](https://www.kaggle.com/khyeh0719/pytorch-efficientnet-baseline-inference-tta)\n\nThe Goal is to understand False Prediction to adjust augmentations and hyperparameters."},{"metadata":{},"cell_type":"markdown","source":"> First, We got few data failed to predict when using Efficientnet"},{"metadata":{"trusted":true,"_kg_hide-output":false,"_kg_hide-input":true},"cell_type":"code","source":"valid_false = valid_.loc[(valid_.label.values!=np.argmax(val_preds, axis=1))].reset_index(drop = True)\nvalid_false['prediction'] = np.argmax(val_preds, axis=1)[(valid_.label.values!=np.argmax(val_preds, axis=1))]\nprint(f\"Dataframe of false prediction\")\nvalid_false","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Exploring the results\n\n* Analizing the distribution of false prediction and full Datasets.\n* Visualizing plots what classes are confused with\n* Show Images of Failed prediction and correct prediction to compare each other."},{"metadata":{},"cell_type":"markdown","source":"> The dataset has large imbalance at the class number 3 Cassava Mosaic Disease (CMD).\nBut, False prediction is quite balanced then the population."},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"plt.figure(figsize=(8, 4))\nsn.countplot(y=\"label\", data=train)\nplt.title(f'Valid data population')\nplt.show()\n\nplt.figure(figsize=(8, 4))\nsn.countplot(y=\"prediction\", data=valid_false)\nplt.title(f'Data distribution failed to predict')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> "},{"metadata":{},"cell_type":"markdown","source":"> In the false prediction of each class, Class 0(CBB) and 1(CBSD) are confused with class 4(Healthy).\nClass 2 and 3 are confused with each others. Class 4 is confused with the rest of classes well balanced."},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"for i in range(5):\n    plt.figure(figsize=(8, 4))\n    sn.countplot(y=\"prediction\", data=valid_false[valid_false['label']==i])\n    plt.title(f'Data distribution confused with class {i}')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> Then, Visualize the images that models are confused with."},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"tmp_df = valid_positive[valid_positive[\"label\"] == 4]\nprint(f\"Valid images for class 4\")\n\ntmp_df = tmp_df.sample(9)\nimage_ids = tmp_df[\"image_id\"].values\nlabels = tmp_df[\"label\"].values\n\nvisualize_batch(image_ids, labels)\n\ntmp_df = valid_false[valid_false[\"prediction\"] == 4]\nprint(f\"Images failed to predict as class 4\")\n\ntmp_df = tmp_df.sample(9)\nimage_ids = tmp_df[\"image_id\"].values\nlabels = tmp_df[\"label\"].values\n\nvisualize_batch(image_ids, labels)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"tmp_df = valid_positive[valid_positive[\"label\"] == 2]\nprint(f\"Valid images for class 2\")\n\ntmp_df = tmp_df.sample(9)\nimage_ids = tmp_df[\"image_id\"].values\nlabels = tmp_df[\"label\"].values\n\nvisualize_batch(image_ids, labels)\n\ntmp_df = valid_false[(valid_false[\"prediction\"] == 2) & (valid_false[\"label\"] == 3)]\nprint(f\"Images labeled 3 predicted as class 2\")\n\ntmp_df = tmp_df.sample(9)\nimage_ids = tmp_df[\"image_id\"].values\nlabels = tmp_df[\"label\"].values\n\nvisualize_batch(image_ids, labels)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"tmp_df = valid_positive[valid_positive[\"label\"] == 3]\nprint(f\"Valid images for class 3\")\n\ntmp_df = tmp_df.sample(9)\nimage_ids = tmp_df[\"image_id\"].values\nlabels = tmp_df[\"label\"].values\n\nvisualize_batch(image_ids, labels)\n\ntmp_df = valid_false[(valid_false[\"prediction\"] == 3) & (valid_false[\"label\"] == 2)]\nprint(f\"Images labeled 2 predicted as class 3\")\n\ntmp_df = tmp_df.sample(9)\nimage_ids = tmp_df[\"image_id\"].values\nlabels = tmp_df[\"label\"].values\n\nvisualize_batch(image_ids, labels)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}