{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":0.015453,"end_time":"2020-12-15T08:41:59.29391","exception":false,"start_time":"2020-12-15T08:41:59.278457","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# User define config"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","execution":{"iopub.execute_input":"2020-12-15T08:41:59.330313Z","iopub.status.busy":"2020-12-15T08:41:59.329471Z","iopub.status.idle":"2020-12-15T08:41:59.332696Z","shell.execute_reply":"2020-12-15T08:41:59.332141Z"},"papermill":{"duration":0.024635,"end_time":"2020-12-15T08:41:59.332796","exception":false,"start_time":"2020-12-15T08:41:59.308161","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 8\nEPOCH = 10\nWD = 1e-4\nLR = 0.0001\nVAL_RATIO = 0.2\nPHASE = ['train', 'val']\nBETA = 1.0\nCUTMIX_PROB = 1.0\nTRAINING = False\n#WEIGHT = '../input/cutmix-chacor-densenet/densenet_best_0.83.pkl'\nK_FOLD = 5","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.014225,"end_time":"2020-12-15T08:41:59.361403","exception":false,"start_time":"2020-12-15T08:41:59.347178","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Create Dataset"},{"metadata":{"execution":{"iopub.execute_input":"2020-12-15T08:41:59.401463Z","iopub.status.busy":"2020-12-15T08:41:59.400663Z","iopub.status.idle":"2020-12-15T08:42:00.939554Z","shell.execute_reply":"2020-12-15T08:42:00.938692Z"},"papermill":{"duration":1.564447,"end_time":"2020-12-15T08:42:00.939688","exception":false,"start_time":"2020-12-15T08:41:59.375241","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"from torch.utils.data.dataset import Dataset\nimport glob\nimport pandas as pd\nimport os\nfrom PIL import Image\n\nclass CLD_Dataset(Dataset):\n    def __init__(self, image_root, label_path = None, transform = None, return_name = False):\n        super(CLD_Dataset, self).__init__()\n        self.transform = transform\n        self.image_paths = glob.glob(os.path.join(image_root, \"*.jpg\"))\n        if not return_name:\n            self.label = pd.read_csv(label_path, index_col = \"image_id\")\n        self.return_name = return_name\n        \n    def set_transform(self, transform):\n        self.transform = transform\n    \n    def __getitem__(self, x):\n        img = Image.open(self.image_paths[x])\n        if self.transform is not None:\n            img = self.transform(img)\n            \n        if self.return_name:\n            return img, self.image_paths[x].split('/')[-1]\n        else:\n            label = self.label.loc[self.image_paths[x].split('/')[-1]].label\n            return img, label\n    \n    def __len__(self):\n        return len(self.image_paths)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-15T08:42:01.013054Z","iopub.status.busy":"2020-12-15T08:42:01.009656Z","iopub.status.idle":"2020-12-15T08:42:02.694787Z","shell.execute_reply":"2020-12-15T08:42:02.693887Z"},"papermill":{"duration":1.724066,"end_time":"2020-12-15T08:42:02.694905","exception":false,"start_time":"2020-12-15T08:42:00.970839","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import torchvision.transforms as transform\nfrom torch.utils.data import DataLoader\nimport torch\nfrom sklearn.model_selection import KFold\n\ntrain_transform = transform.Compose([\n    transform.Resize((448, 448)),\n    transform.RandomHorizontalFlip(),\n    transform.ToTensor(),\n    transform.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n])\n\nval_transform = transform.Compose([\n    transform.Resize((448, 448)),\n    transform.ToTensor(),\n    transform.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n])\n\nall_train_dataset = CLD_Dataset('/kaggle/input/cassava-leaf-disease-classification/train_images', \n                          '/kaggle/input/cassava-leaf-disease-classification/train.csv',\n                          train_transform)\ndataset_size = len(all_train_dataset)\n\nfold_dataloader = []\n#all_train_dataset = DataLoader(all_train_dataset, batch_size = BATCH_SIZE, shuffle = True, num_workers = 4)\nif K_FOLD != 1:\n    kf = KFold(K_FOLD, shuffle = True)\n    \n    index = 0\n    for train_idx, val_idx in kf.split(all_train_dataset):\n        train_dataset = torch.utils.data.Subset(all_train_dataset, train_idx)\n        val_dataset = torch.utils.data.Subset(all_train_dataset, val_idx)\n        fold_dataloader.append({'train': DataLoader(train_dataset, batch_size = BATCH_SIZE, shuffle = False, num_workers = 4),\n                        'val' : DataLoader(val_dataset, batch_size = BATCH_SIZE, shuffle = False, num_workers = 4)})\n        index += 1\n    print(fold_dataloader)\nelse:\n    fold_dataloader.append({'train' : DataLoader(all_train_dataset, batch_size = BATCH_SIZE, shuffle = False, num_workers = 4),\n                           'val' : None})","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.015852,"end_time":"2020-12-15T08:42:02.730186","exception":false,"start_time":"2020-12-15T08:42:02.714334","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Create Model"},{"metadata":{"execution":{"iopub.execute_input":"2020-12-15T08:42:02.765826Z","iopub.status.busy":"2020-12-15T08:42:02.764872Z","iopub.status.idle":"2020-12-15T08:42:02.768553Z","shell.execute_reply":"2020-12-15T08:42:02.769077Z"},"papermill":{"duration":0.024109,"end_time":"2020-12-15T08:42:02.7692","exception":false,"start_time":"2020-12-15T08:42:02.745091","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import re\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.utils.checkpoint as cp\nfrom collections import OrderedDict\nfrom torch import Tensor\nfrom torch.jit.annotations import List","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-15T08:42:02.81916Z","iopub.status.busy":"2020-12-15T08:42:02.813393Z","iopub.status.idle":"2020-12-15T08:42:02.889041Z","shell.execute_reply":"2020-12-15T08:42:02.889487Z"},"papermill":{"duration":0.105678,"end_time":"2020-12-15T08:42:02.889611","exception":false,"start_time":"2020-12-15T08:42:02.783933","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"__all__ = ['DenseNet', 'densenet121', 'densenet169', 'densenet201', 'densenet161']\n\nmodel_urls = {\n    'densenet121': 'https://download.pytorch.org/models/densenet121-a639ec97.pth',\n    'densenet169': 'https://download.pytorch.org/models/densenet169-b2777c0a.pth',\n    'densenet201': 'https://download.pytorch.org/models/densenet201-c1103571.pth',\n    'densenet161': 'https://download.pytorch.org/models/densenet161-8d451a50.pth',\n}\n\n\nclass _DenseLayer(nn.Module):\n    def __init__(self, num_input_features, growth_rate, bn_size, drop_rate, memory_efficient=False):\n        super(_DenseLayer, self).__init__()\n        self.add_module('norm1', nn.BatchNorm2d(num_input_features)),\n        self.add_module('relu1', nn.ReLU(inplace=True)),\n        self.add_module('conv1', nn.Conv2d(num_input_features, bn_size *\n                                           growth_rate, kernel_size=1, stride=1,\n                                           bias=False)),\n        self.add_module('norm2', nn.BatchNorm2d(bn_size * growth_rate)),\n        self.add_module('relu2', nn.ReLU(inplace=True)),\n        self.add_module('conv2', nn.Conv2d(bn_size * growth_rate, growth_rate,\n                                           kernel_size=3, stride=1, padding=1,\n                                           bias=False)),\n        self.drop_rate = float(drop_rate)\n        self.memory_efficient = memory_efficient\n\n    def bn_function(self, inputs):\n        # type: (List[Tensor]) -> Tensor\n        concated_features = torch.cat(inputs, 1)\n        bottleneck_output = self.conv1(self.relu1(self.norm1(concated_features)))  # noqa: T484\n        return bottleneck_output\n\n    # todo: rewrite when torchscript supports any\n    def any_requires_grad(self, input):\n        # type: (List[Tensor]) -> bool\n        for tensor in input:\n            if tensor.requires_grad:\n                return True\n        return False\n\n    @torch.jit.unused  # noqa: T484\n    def call_checkpoint_bottleneck(self, input):\n        # type: (List[Tensor]) -> Tensor\n        def closure(*inputs):\n            return self.bn_function(*inputs)\n\n        return cp.checkpoint(closure, input)\n\n    @torch.jit._overload_method  # noqa: F811\n    def forward(self, input):\n        # type: (List[Tensor]) -> (Tensor)\n        pass\n\n    @torch.jit._overload_method  # noqa: F811\n    def forward(self, input):\n        # type: (Tensor) -> (Tensor)\n        pass\n    \n    # torchscript does not yet support *args, so we overload method\n    # allowing it to take either a List[Tensor] or single Tensor\n    def forward(self, input):  # noqa: F811\n        if isinstance(input, Tensor):\n            prev_features = [input]\n        else:\n            prev_features = input\n\n        if self.memory_efficient and self.any_requires_grad(prev_features):\n            if torch.jit.is_scripting():\n                raise Exception(\"Memory Efficient not supported in JIT\")\n\n            bottleneck_output = self.call_checkpoint_bottleneck(prev_features)\n        else:\n            bottleneck_output = self.bn_function(prev_features)\n\n        new_features = self.conv2(self.relu2(self.norm2(bottleneck_output)))\n        if self.drop_rate > 0:\n            new_features = F.dropout(new_features, p=self.drop_rate,\n                                     training=self.training)\n        return new_features\n\n\nclass _DenseBlock(nn.ModuleDict):\n    _version = 2\n\n    def __init__(self, num_layers, num_input_features, bn_size, growth_rate, drop_rate, memory_efficient=False):\n        super(_DenseBlock, self).__init__()\n        for i in range(num_layers):\n            layer = _DenseLayer(\n                num_input_features + i * growth_rate,\n                growth_rate=growth_rate,\n                bn_size=bn_size,\n                drop_rate=drop_rate,\n                memory_efficient=memory_efficient,\n            )\n            self.add_module('denselayer%d' % (i + 1), layer)\n\n    def forward(self, init_features):\n        features = [init_features]\n        for name, layer in self.items():\n            new_features = layer(features)\n            features.append(new_features)\n        return torch.cat(features, 1)\n\n\nclass _Transition(nn.Sequential):\n    def __init__(self, num_input_features, num_output_features):\n        super(_Transition, self).__init__()\n        self.add_module('norm', nn.BatchNorm2d(num_input_features))\n        self.add_module('relu', nn.ReLU(inplace=True))\n        self.add_module('conv', nn.Conv2d(num_input_features, num_output_features,\n                                          kernel_size=1, stride=1, bias=False))\n        self.add_module('pool', nn.AvgPool2d(kernel_size=2, stride=2))\n\n\nclass DenseNet(nn.Module):\n    r\"\"\"Densenet-BC model class, based on\n    `\"Densely Connected Convolutional Networks\" <https://arxiv.org/pdf/1608.06993.pdf>`_\n\n    Args:\n        growth_rate (int) - how many filters to add each layer (`k` in paper)\n        block_config (list of 4 ints) - how many layers in each pooling block\n        num_init_features (int) - the number of filters to learn in the first convolution layer\n        bn_size (int) - multiplicative factor for number of bottle neck layers\n          (i.e. bn_size * k features in the bottleneck layer)\n        drop_rate (float) - dropout rate after each dense layer\n        num_classes (int) - number of classification classes\n        memory_efficient (bool) - If True, uses checkpointing. Much more memory efficient,\n          but slower. Default: *False*. See `\"paper\" <https://arxiv.org/pdf/1707.06990.pdf>`_\n    \"\"\"\n\n    def __init__(self, growth_rate=32, block_config=(6, 12, 24, 16),\n                 num_init_features=64, bn_size=4, drop_rate=0, num_classes=1000, memory_efficient=False):\n\n        super(DenseNet, self).__init__()\n\n        # First convolution\n        self.features = nn.Sequential(OrderedDict([\n            ('conv0', nn.Conv2d(3, num_init_features, kernel_size=7, stride=2,\n                                padding=3, bias=False)),\n            ('norm0', nn.BatchNorm2d(num_init_features)),\n            ('relu0', nn.ReLU(inplace=True)),\n            ('pool0', nn.MaxPool2d(kernel_size=3, stride=2, padding=1)),\n        ]))\n\n        # Each denseblock\n        num_features = num_init_features\n        for i, num_layers in enumerate(block_config):\n            block = _DenseBlock(\n                num_layers=num_layers,\n                num_input_features=num_features,\n                bn_size=bn_size,\n                growth_rate=growth_rate,\n                drop_rate=drop_rate,\n                memory_efficient=memory_efficient\n            )\n            self.features.add_module('denseblock%d' % (i + 1), block)\n            num_features = num_features + num_layers * growth_rate\n            if i != len(block_config) - 1:\n                trans = _Transition(num_input_features=num_features,\n                                    num_output_features=num_features // 2)\n                self.features.add_module('transition%d' % (i + 1), trans)\n                num_features = num_features // 2\n\n        # Final batch norm\n        self.features.add_module('norm5', nn.BatchNorm2d(num_features))\n        \n        # Linear layer\n        self.classifier = nn.Linear(num_features, num_classes)\n\n        # Official init from torch repo.\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                nn.init.kaiming_normal_(m.weight)\n            elif isinstance(m, nn.BatchNorm2d):\n                nn.init.constant_(m.weight, 1)\n                nn.init.constant_(m.bias, 0)\n            elif isinstance(m, nn.Linear):\n                nn.init.constant_(m.bias, 0)\n                \n    def _construct_fc_layer(self, fc_dims, input_dim, dropout_p=None):\n        \"\"\"\n        Construct fully connected layer\n\n        - fc_dims (list or tuple): dimensions of fc layers, if None,\n                                   no fc layers are constructed\n        - input_dim (int): input dimension\n        - dropout_p (float): dropout probability, if None, dropout is unused\n        \"\"\"\n        if fc_dims is None:\n            self.feature_dim = input_dim\n            return None\n        \n        assert isinstance(fc_dims, (list, tuple)), \"fc_dims must be either list or tuple, but got {}\".format(type(fc_dims))\n        \n        layers = []\n        for dim in fc_dims:\n            layers.append(nn.Linear(input_dim, dim))\n            layers.append(nn.BatchNorm1d(dim))\n            layers.append(nn.ReLU(inplace=True))\n            if dropout_p is not None:\n                layers.append(nn.Dropout(p=dropout_p))\n            input_dim = dim\n        \n        self.feature_dim = fc_dims[-1]\n        \n        return nn.Sequential(*layers)\n\n    def feature_extract(self, x):\n        x = self.features.conv0(x)\n        x = self.features.norm0(x)\n        x = self.features.relu0(x)\n        x = self.features.pool0(x)\n        x = self.features.denseblock1(x)\n        x = self.features.transition1(x)\n        x = self.features.denseblock2(x)\n        x = self.features.transition2(x)\n        x = self.features.denseblock3(x)\n        x = self.features.transition3(x)\n        x = self.features.denseblock4(x)\n        x = self.features.norm5(x)\n        return x\n    \n    def forward(self, x):\n        features = self.feature_extract(x)\n        out = F.relu(features, inplace=True)\n        out = F.adaptive_avg_pool2d(out, (1, 1))\n        out = torch.flatten(out, 1)\n        \n        out = self.classifier(out)\n        return out\n\n\ndef _load_state_dict(model, arch, progress):\n    # '.'s are no longer allowed in module names, but previous _DenseLayer\n    # has keys 'norm.1', 'relu.1', 'conv.1', 'norm.2', 'relu.2', 'conv.2'.\n    # They are also in the checkpoints in model_urls. This pattern is used\n    # to find such keys.\n    pattern = re.compile(\n        r'^(.*denselayer\\d+\\.(?:norm|relu|conv))\\.((?:[12])\\.(?:weight|bias|running_mean|running_var))$')\n\n    state_dict = torch.load('../../pretrain/{}.pth'.format(arch))\n    for key in list(state_dict.keys()):\n        res = pattern.match(key)\n        if res:\n            new_key = res.group(1) + res.group(2)\n            state_dict[new_key] = state_dict[key]\n            del state_dict[key]\n    load = []\n    not_load = []\n    for name, param in state_dict.items():\n        if name in model.state_dict():\n            try:\n                model.state_dict()[name].copy_(param)\n                load.append(name)\n            except:\n                not_load.append(name)\n    \n    print(\"Load pretrain : \")\n    print(\"Load : {} layers\".format(len(load)))\n    print(\"Miss : {} layers\".format(len(not_load)))\n    #model.load_state_dict(state_dict)\n\n\ndef _densenet(arch, growth_rate, block_config, num_init_features, pretrained, progress,\n              **kwargs):\n    model = DenseNet(growth_rate, block_config, num_init_features, **kwargs)\n    if pretrained:\n        _load_state_dict(model, arch, progress)\n    return model\n\n\ndef densenet121(pretrained=False, progress=True, **kwargs):\n    r\"\"\"Densenet-121 model from\n    `\"Densely Connected Convolutional Networks\" <https://arxiv.org/pdf/1608.06993.pdf>`_\n\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n        progress (bool): If True, displays a progress bar of the download to stderr\n        memory_efficient (bool) - If True, uses checkpointing. Much more memory efficient,\n          but slower. Default: *False*. See `\"paper\" <https://arxiv.org/pdf/1707.06990.pdf>`_\n    \"\"\"\n    return _densenet('densenet121', 32, (6, 12, 24, 16), 64, pretrained, progress,\n                     **kwargs)\n\n\ndef densenet161(pretrained=False, progress=True, **kwargs):\n    r\"\"\"Densenet-161 model from\n    `\"Densely Connected Convolutional Networks\" <https://arxiv.org/pdf/1608.06993.pdf>`_\n\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n        progress (bool): If True, displays a progress bar of the download to stderr\n        memory_efficient (bool) - If True, uses checkpointing. Much more memory efficient,\n          but slower. Default: *False*. See `\"paper\" <https://arxiv.org/pdf/1707.06990.pdf>`_\n    \"\"\"\n    return _densenet('densenet161', 48, (6, 12, 36, 24), 96, pretrained, progress,\n                     **kwargs)\n\n\ndef densenet169(pretrained=False, progress=True, **kwargs):\n    r\"\"\"Densenet-169 model from\n    `\"Densely Connected Convolutional Networks\" <https://arxiv.org/pdf/1608.06993.pdf>`_\n\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n        progress (bool): If True, displays a progress bar of the download to stderr\n        memory_efficient (bool) - If True, uses checkpointing. Much more memory efficient,\n          but slower. Default: *False*. See `\"paper\" <https://arxiv.org/pdf/1707.06990.pdf>`_\n    \"\"\"\n    return _densenet('densenet169', 32, (6, 12, 32, 32), 64, pretrained, progress,\n                     **kwargs)\n\n\ndef densenet201(pretrained=False, progress=True, **kwargs):\n    r\"\"\"Densenet-201 model from\n    `\"Densely Connected Convolutional Networks\" <https://arxiv.org/pdf/1608.06993.pdf>`_\n\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n        progress (bool): If True, displays a progress bar of the download to stderr\n        memory_efficient (bool) - If True, uses checkpointing. Much more memory efficient,\n          but slower. Default: *False*. See `\"paper\" <https://arxiv.org/pdf/1707.06990.pdf>`_\n    \"\"\"\n    return _densenet('densenet201', 32, (6, 12, 48, 32), 64, pretrained, progress,\n                     **kwargs)\n","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-15T08:42:03.28172Z","iopub.status.busy":"2020-12-15T08:42:03.281031Z","iopub.status.idle":"2020-12-15T08:42:12.878295Z","shell.execute_reply":"2020-12-15T08:42:12.877722Z"},"papermill":{"duration":9.973997,"end_time":"2020-12-15T08:42:12.878434","exception":false,"start_time":"2020-12-15T08:42:02.904437","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"if torch.cuda.is_available():\n    device = 'cuda:0'\nelse:\n    device = 'cpu'\nprint(device)\n\ndef create_new_model(pretrained = True):\n    return densenet121(num_classes = 5, pretrained = pretrained).to(device)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.02251,"end_time":"2020-12-15T08:42:12.925998","exception":false,"start_time":"2020-12-15T08:42:12.903488","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Define Loss function and Optimizer"},{"metadata":{"execution":{"iopub.execute_input":"2020-12-15T08:42:12.984513Z","iopub.status.busy":"2020-12-15T08:42:12.979484Z","iopub.status.idle":"2020-12-15T08:42:12.987443Z","shell.execute_reply":"2020-12-15T08:42:12.986967Z"},"papermill":{"duration":0.038973,"end_time":"2020-12-15T08:42:12.987554","exception":false,"start_time":"2020-12-15T08:42:12.948581","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def create_loss_opti():\n    criterion = torch.nn.CrossEntropyLoss()\n    optimizer = torch.optim.Adam(model.parameters(), lr = LR, weight_decay = WD)\n    #lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 4, gamma=0.5, last_epoch=-1)\n    lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0 = 10, T_mult = 1, eta_min = 1e-6, last_epoch = -1)\n    return criterion, optimizer, lr_scheduler","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.023967,"end_time":"2020-12-15T08:42:13.03594","exception":false,"start_time":"2020-12-15T08:42:13.011973","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# CutMix"},{"metadata":{"execution":{"iopub.execute_input":"2020-12-15T08:42:13.095706Z","iopub.status.busy":"2020-12-15T08:42:13.094887Z","iopub.status.idle":"2020-12-15T08:42:13.098121Z","shell.execute_reply":"2020-12-15T08:42:13.097574Z"},"papermill":{"duration":0.037094,"end_time":"2020-12-15T08:42:13.098228","exception":false,"start_time":"2020-12-15T08:42:13.061134","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def rand_bbox(size, lab):\n    W = size[2]\n    H = size[3]\n    cut_rat = np.sqrt(1. - lam)\n    cut_w = np.int(W * cut_rat)\n    cut_h = np.int(H * cut_rat)\n    \n    # uniform\n    cx = np.random.randint(W)\n    cy = np.random.randint(H)\n    \n    bbx1 = np.clip(cx - cut_w // 2, 0, W)\n    bby1 = np.clip(cy - cut_h // 2, 0, H)\n    bbx2 = np.clip(cx + cut_w // 2, 0, W)\n    bby2 = np.clip(cy + cut_h // 2, 0, H)\n    \n    return bbx1, bby1, bbx2, bby2","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.02595,"end_time":"2020-12-15T08:42:13.150487","exception":false,"start_time":"2020-12-15T08:42:13.124537","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Record"},{"metadata":{"execution":{"iopub.execute_input":"2020-12-15T08:42:13.209936Z","iopub.status.busy":"2020-12-15T08:42:13.209039Z","iopub.status.idle":"2020-12-15T08:42:13.211327Z","shell.execute_reply":"2020-12-15T08:42:13.211894Z"},"papermill":{"duration":0.035482,"end_time":"2020-12-15T08:42:13.212012","exception":false,"start_time":"2020-12-15T08:42:13.17653","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"class AverageMeter():\n    \"\"\"Computes and stores the average and current value\"\"\"\n\n    def __init__(self, acc):\n        self.reset()\n        self.acc = acc\n    def reset(self):\n        self.value = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, value, batch):\n        self.value = value\n        if self.acc:\n            self.sum += value\n        else:       \n            self.sum += value * batch\n        self.count += batch\n        self.avg = self.sum / self.count\n","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.057449,"end_time":"2020-12-15T08:42:13.294228","exception":false,"start_time":"2020-12-15T08:42:13.236779","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_step(model, criterion, optimizer, image, label, phase):\n    b_image = image.to(device)\n    b_label = label.to(device)\n\n    # CUTMIX -------------------------------------------\n    r = np.random.rand(1)\n    if BETA > 0 and r < CUTMIX_PROB:\n        lam = np.random.beta(BETA, BETA)\n        rand_index = torch.randperm(b_image.size()[0]).to(device)\n        target_a = b_label\n        target_b = b_label[rand_index]\n        bbx1, bby1, bbx2, bby2 = rand_bbox(b_image.size(), lam)\n        b_image[:, :, bbx1:bbx2, bby1:bby2] = b_image[rand_index, :, bbx1:bbx2, bby1:bby2]\n        lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (b_image.size()[-1] * b_image.size()[-2]))\n\n        output = model(b_image)\n        loss = criterion(output, target_a) * lam + criterion(output, target_b) * (1. - lam)\n    else:\n        output = model(b_image)\n        loss = criterion(output, b_label)\n    # --------------------------------------------------\n\n    _, predicted = torch.max(output.data, dim = 1)\n    correct = (predicted.cpu() == label).sum().item()\n    if phase == 'train':\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        \n    return correct, loss.item()\n    ","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-15T08:42:13.367623Z","iopub.status.busy":"2020-12-15T08:42:13.366802Z","iopub.status.idle":"2020-12-15T08:42:13.369939Z","shell.execute_reply":"2020-12-15T08:42:13.369464Z"},"papermill":{"duration":0.049858,"end_time":"2020-12-15T08:42:13.370036","exception":false,"start_time":"2020-12-15T08:42:13.320178","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import numpy as np\nfrom tqdm import tqdm\n\nmax_acc = 0.0\nACCMeter = []\nLOSSMeter = []\nfor i in range(K_FOLD):\n    ACCMeter.append(AverageMeter(True))\n    LOSSMeter.append(AverageMeter(False))\n    \nif TRAINING:   \n    for index, dataloader in enumerate(fold_dataloader):\n        model = create_new_model()\n        criterion, optimizer, lr_scheduler = create_loss_opti()\n        Best_ACC = 0.0\n        for epoch in range(1, EPOCH + 1):\n            tmp_ACCMeter = AverageMeter(True)\n            tmp_LOSSMeter = AverageMeter(False)\n            correct_t = 0\n            total = 0\n            loss_t = 0.0\n            for phase in PHASE:\n                if phase == 'train':\n                    model.train(True)\n                    all_train_dataset.set_transform(train_transform)\n                else:\n                    model.train(False)\n                    all_train_dataset.set_transform(val_transform)\n                    \n                for image, label in tqdm(dataloader[phase], total=len(dataloader[phase]), position=0, leave=True):\n                    correct, loss = train_step(model, criterion, optimizer, image, label, phase)\n                    \n                    if phase == 'val':\n                        tmp_ACCMeter.update(correct, label.size(0))\n                        tmp_LOSSMeter.update(loss, label.size(0))\n                        total += label.size(0)\n                        loss_t += loss * label.size(0)\n                        correct_t += correct\n                \n                if phase == 'val' and Best_ACC < tmp_ACCMeter.avg:\n                    Best_ACC = tmp_ACCMeter.avg\n                    ACCMeter[index] = tmp_ACCMeter\n                    LOSSMeter[index] = tmp_LOSSMeter\n                    torch.save(model.state_dict(), './resnet50_kfold_{}_{}_{:.2f}.pkl'.format(index + 1, epoch, tmp_ACCMeter.avg))\n            print('Fold : {}/ {} Epoch : {} / {} loss : {:.6f} ACC : {:.6f}'.format(index + 1, K_FOLD, epoch, EPOCH, loss_t / total, correct_t / total))\n            lr_scheduler.step()\n            \n            ","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.022903,"end_time":"2020-12-15T08:42:13.416869","exception":false,"start_time":"2020-12-15T08:42:13.393966","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Analyze K-fold result"},{"metadata":{"execution":{"iopub.execute_input":"2020-12-15T08:42:13.473679Z","iopub.status.busy":"2020-12-15T08:42:13.472985Z","iopub.status.idle":"2020-12-15T08:42:13.476196Z","shell.execute_reply":"2020-12-15T08:42:13.477063Z"},"papermill":{"duration":0.035166,"end_time":"2020-12-15T08:42:13.477214","exception":false,"start_time":"2020-12-15T08:42:13.442048","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"acc_sum = 0;\nloss_sum = 0;\nfor i in range(K_FOLD):\n    acc_sum += ACCMeter[i].avg\n    loss_sum += LOSSMeter[i].avg\n    \nprint(\"K-fold {} ACC : {:.6f}\".format(K_FOLD, acc_sum / K_FOLD))\nprint(\"K-fold {} ACC : {:.6f}\".format(K_FOLD, loss_sum / K_FOLD))","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.023777,"end_time":"2020-12-15T08:42:13.525752","exception":false,"start_time":"2020-12-15T08:42:13.501975","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Testing"},{"metadata":{"execution":{"iopub.execute_input":"2020-12-15T08:42:22.571682Z","iopub.status.busy":"2020-12-15T08:42:22.570784Z","iopub.status.idle":"2020-12-15T08:42:23.822713Z","shell.execute_reply":"2020-12-15T08:42:23.821177Z"},"papermill":{"duration":1.29375,"end_time":"2020-12-15T08:42:23.822844","exception":false,"start_time":"2020-12-15T08:42:22.529094","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import pandas as pd\n\ntest_dataset = CLD_Dataset('/kaggle/input/cassava-leaf-disease-classification/test_images', \n                          transform = val_transform,\n                          return_name = True)\ntest_dataloader = DataLoader(test_dataset, batch_size = 1, shuffle = False, num_workers = 4)\n\nimage_names = []\nimage_probs = []\n\n\nfor params_path in glob.glob('../input/densenet121-softtarget/*.pkl'):\n    model = create_new_model(pretrained = False)\n    params = torch.load(params_path)\n    load = []\n    not_load = []\n    image_name = []\n    image_prob = []\n    for name, param in params.items():\n        if name in model.state_dict():\n            try:\n                model.state_dict()[name].copy_(param)\n                load.append(name)\n            except:\n                not_load.append(name)\n    print(\"Trained weight load : {}\".format(len(load)))\n    print(\"Trained weight not load : {}\".format(len(not_load)))\n    print(not_load)\n    model.train(False)\n    for step, (img, img_name) in enumerate(test_dataloader):\n        b_img = img.to(device)\n\n        output = model(b_img)\n        _, predicted = torch.max(output, dim = 1)\n\n        image_name.append(img_name[0])\n        image_prob.append(np.array(output[0].cpu().detach()))\n\n    image_names.append(image_name)\n    image_probs.append(image_prob)\n    \nimage_names = np.array(image_names)\nimage_probs = np.array(image_probs)\nimage_labels = []\nfor img_idx in range(image_probs.shape[1]):\n    probs = 0.0\n    for fold in range(image_names.shape[0]):\n        prob = image_probs[fold][img_idx]\n        prob_e = np.exp(prob) \n        prob_e_sum = prob_e / sum(prob_e + 1e-4)\n        #probs += image_probs[fold][img_idx]\n        probs += prob_e_sum\n    label = np.argmax(probs)\n    image_labels.append(label)\n    \n\nimage_labels = np.array(image_labels)\nimage_names = np.array(image_names)\ndf = pd.DataFrame({'image_id' : image_names[0], 'label' : image_labels})\nprint(df)\ndf.to_csv('/kaggle/working/submission.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}