{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install pytorch-lightning-spells geffnet","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pathlib import Path\nfrom typing import Callable, Union, Tuple, List\n\nimport torch\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nfrom matplotlib import pyplot as plt\nfrom albumentations import SmallestMaxSize, CenterCrop, Compose\nfrom albumentations.pytorch.transforms import ToTensor\nimport pytorch_lightning as pl\nfrom pytorch_lightning_spells.callbacks import CutMixCallback, MixUpCallback, SnapMixCallback\nfrom torch.utils.data import Dataset, DataLoader\n\ntorch.multiprocessing.set_sharing_strategy('file_system')\n\nOFFSET = np.asarray([0.485, 0.456, 0.406])[:, np.newaxis, np.newaxis]\nSCALE = np.asarray([0.229, 0.224, 0.225])[:, np.newaxis, np.newaxis]\nTRANSFORMATIONS = Compose([\n    SmallestMaxSize(448),\n    CenterCrop(448, 448),\n    ToTensor(normalize=dict(\n        mean=OFFSET[:,0,0], std=SCALE[:,0,0])\n    )\n])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_image(filepath: Path) -> Image.Image:\n    image = np.array(Image.open(filepath).convert('RGB'))\n    return image\n\n\ndef load_transform_image(filepath):\n    image = load_image(filepath)\n    return TRANSFORMATIONS(image=image)[\"image\"]\n\n\nclass CassavaDataset(Dataset):\n    def __init__(self, folder: Union[Path, str], df: pd.DataFrame):\n        super().__init__()\n        self._df = df\n        self._folder = Path(folder)\n\n    def __len__(self):\n        return len(self._df)\n\n    def __getitem__(self, idx: int):\n        item = self._df.iloc[idx]\n        image = load_transform_image(\n            self._folder / item.image_id)\n        return [image, torch.tensor(item.label, dtype=torch.int64)]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_loader = DataLoader(\n    CassavaDataset(\n        Path(\"/kaggle/input/cassava-leaf-disease-classification/train_images/\"),\n        pd.read_csv(\"/kaggle/input/cassava-leaf-disease-classification/train.csv\")\n    ),\n    shuffle=True,\n    batch_size=16,\n    num_workers=0,\n    drop_last=False\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## MixUp","metadata":{}},{"cell_type":"code","source":"mixup_cb = MixUpCallback(alpha=0.5, softmax_target=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch, targets = next(iter(data_loader))\nbatch_packed = [batch, targets]\nmixup_cb.on_train_batch_start(None, None, batch_packed, None, None)\nbatch_new, targets_new = batch_packed\nfig, ax = plt.subplots(nrows=4, ncols=4, figsize=(12, 12))\ncount=0\nfor row in ax:\n    for col in row:\n        col.imshow(\n            ((batch_new[count].numpy() * SCALE + OFFSET).transpose(1,2,0) * 255.).astype(np.uint8)\n        )\n        col.set_axis_off()\n        col.set_title(\n            f\"L1: {targets_new[count][0]:.0f} ({targets_new[count][2]*100:.0f}%) \"\n            f\"| L2: {targets_new[count][1]:.0f} \"\n        )\n        count += 1\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Because the nature of this dataset, the mixed up image is hard to read. We can infer that Mix Up augmentation is probablly not the best choice for this dataset.","metadata":{}},{"cell_type":"markdown","source":"## CutMix","metadata":{}},{"cell_type":"code","source":"cutmix_cb = CutMixCallback(alpha=0.9, minmax=[0.2, 0.8], softmax_target=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch, targets = next(iter(data_loader))\nbatch_packed = [batch, targets]\ncutmix_cb.on_train_batch_start(None, None, batch_packed, None, None)\nbatch_new, targets_new = batch_packed\nfig, ax = plt.subplots(nrows=4, ncols=4, figsize=(12, 12))\ncount=0\nfor row in ax:\n    for col in row:\n        col.imshow(\n            ((batch_new[count].numpy() * SCALE + OFFSET).transpose(1,2,0) * 255.).astype(np.uint8)\n        )\n        col.set_axis_off()\n        col.set_title(\n            f\"L1: {targets_new[count][0]:.0f} ({targets_new[count][2]*100:.0f}%) \"\n            f\"| L2: {targets_new[count][1]:.0f} \"\n        )\n        count += 1\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## SnapMix\n\nReference: [SnapMix: Semantically Proportional Mixing for Augmenting Fine-grained Data](https://arxiv.org/abs/2012.04846)","metadata":{}},{"cell_type":"code","source":"import geffnet\nfrom fastcore.basics import patch_to\n\n@patch_to(geffnet.gen_efficientnet.GenEfficientNet)\ndef extract_features(self, input_tensor):\n    return self.features(input_tensor)\n\n@patch_to(geffnet.gen_efficientnet.GenEfficientNet)\ndef get_fc(self):\n    return self.classifier","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load a baseline model\nmodel_dict = torch.load(\"/kaggle/input/cassava-model/full_b4_41341_3290_randaug.pth\")\nmodel = geffnet.tf_efficientnet_b4_ns(pretrained=False, drop_rate=0.4, as_sequential=False)\nmodel.classifier = torch.nn.Linear(model.classifier.in_features, 5)\nmodel.load_state_dict(model_dict[\"model_states\"])\n_ = model.cuda()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### CutMix-style Bounding Boxes","metadata":{}},{"cell_type":"code","source":"snapmix_cb = SnapMixCallback(model, image_size=(448, 448), minmax = (0.2, 0.8), alpha=0.9, cutmix_bbox=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch, targets = next(iter(data_loader))\nbatch_packed = [batch, targets]\nsnapmix_cb.on_train_batch_start(None, None, batch_packed, None, None)\nbatch_new, targets_new = batch_packed\nfig, ax = plt.subplots(nrows=4, ncols=4, figsize=(12, 12))\ncount=0\nfor row in ax:\n    for col in row:\n        col.imshow(\n            ((batch_new[count].numpy() * SCALE + OFFSET).transpose(1,2,0) * 255.).astype(np.uint8)\n        )\n        col.set_axis_off()\n        col.set_title(\n            f\"L1: {targets_new[count][0]:.0f} ({targets_new[count][2]*100:.0f}%) \"\n            f\"| L2: {targets_new[count][1]:.0f} ({targets_new[count][3]*100:.0f}%)\"\n        )\n        count += 1\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### SnapMix-style Bounding Boxes\n\n1. Randomly generate two bounding boxes (instead of one as in CutMix). \n2. The first bounding mox is used to extract a patch from the source image.\n3. The extracted patch is resized to the size of the second bounding box.\n4. The resized patch is put into the target image at the second bounding box.","metadata":{}},{"cell_type":"code","source":"snapmix_cb = SnapMixCallback(model, image_size=(448, 448), minmax = (0.2, 0.8), alpha=0.9, cutmix_bbox=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch, targets = next(iter(data_loader))\nbatch_packed = [batch, targets]\nsnapmix_cb.on_train_batch_start(None, None, batch_packed, None, None)\nbatch_new, targets_new = batch_packed\nfig, ax = plt.subplots(nrows=4, ncols=4, figsize=(12, 12))\ncount=0\nfor row in ax:\n    for col in row:\n        col.imshow(\n            ((batch_new[count].numpy() * SCALE + OFFSET).transpose(1,2,0) * 255.).astype(np.uint8)\n        )\n        col.set_axis_off()\n        col.set_title(\n            f\"L1: {targets_new[count][0]:.0f} ({targets_new[count][2]*100:.0f}%) \"\n            f\"| L2: {targets_new[count][1]:.0f} ({targets_new[count][3]*100:.0f}%)\"\n        )\n        count += 1\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}