{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# # This Python 3 environment comes with many helpful analytics libraries installed\n# # It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# # For example, here's several helpful packages to load\n\n# import numpy as np # linear algebra\n# import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# # Input data files are available in the read-only \"../input/\" directory\n# # For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# # You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# # You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os\nimport torch\nimport pandas as pd\nfrom skimage import io, transform\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport math\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms, utils\nfrom copy import deepcopy","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"is_submission = True\ndata_path = '/kaggle/input/cassava-leaf-disease-classification'\ncsv_file_name = 'train.csv' if not is_submission else 'sample_submission.csv'\ntrain_batch_size, val_batch_size, sub_batch_size = 4, 8, 8\ntrain_rate = 0.7 # rate for train dataset\nnum_workers = 2\nmean, std = 0.5, 0.5\n\nuse_balanced_sample = False\nuse_class_weight = not use_balanced_sample\nmul_weight = torch.tensor([3.0, 1.5, 1.0, 1.0, 4.5])\nlearning_rate = 0.001\nweight_decay = 0.00002\nefficient_net_version = 3 # (1.4, 1.2)\n\ntrain_epoch = (0, 20)\ndebug = not is_submission\n\nuse_pre_trained_weight = True\npre_trained_weight_path = '../input/cassava-leaf-disease-classification-weight/weight.pth'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaLeafDiseaseDataset(Dataset):\n    \"Cassava Leaf Disease\"\n\n    def __init__(self, csv_file, root_dir, transform=None, train=True):\n        \"\"\"\n        Args:\n            csv_file (string): csv file path\n            root_dir (string): directory path of exist all image\n            transform (callable, optional): Optional transform for sample\n        \"\"\"\n        self.cassava_leaf_disease = pd.read_csv(csv_file)\n        self.root_dir = root_dir\n        self.transform = transform\n        self.folder = 'train_images' if train else 'test_images'\n\n    def __len__(self):\n        return len(self.cassava_leaf_disease)\n\n    def __getitem__(self, idx):\n        if torch.is_tensor(idx):\n            idx = idx.tolist()\n\n        img_name = '/'.join([self.root_dir,\n                             self.folder,\n                             self.cassava_leaf_disease.image_id[idx]])\n        img = io.imread(img_name)\n        if self.transform:\n            img = self.transform(img)\n        label = self.cassava_leaf_disease.label[idx]\n        sample = (img, label)\n\n        return sample\n    \ndef split_dataset(dataset, rate):\n    train_length = int(len(dataset) * rate)\n    val_length = len(dataset) - train_length\n    train_set, val_set = torch.utils.data.random_split(\n        dataset, [train_length, val_length])\n    return train_set, val_set","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if is_submission:\n    transform = transforms.Compose([transforms.ToTensor(),\n                                    transforms.Resize((512, 512)),\n                                    transforms.Normalize((mean, mean, mean), (std, std, std))])\nelse:\n    transform = transforms.Compose([transforms.ToTensor(),\n                                    transforms.Resize((512, 512)),\n                                    transforms.RandomRotation(30),\n                                    transforms.RandomHorizontalFlip(),\n                                    transforms.RandomVerticalFlip(),\n                                    transforms.Normalize((mean, mean, mean), (std, std, std))])\n\ndataset = CassavaLeafDiseaseDataset(csv_file='/'.join([data_path, csv_file_name]),\n                                    root_dir=data_path,\n                                    transform=transform,\n                                    train=not is_submission)\n\nif is_submission:\n    sub_loader = DataLoader(dataset,\n                            batch_size=sub_batch_size,\n                            shuffle=False,\n                            num_workers=num_workers)\nelse:\n    if use_balanced_sample:\n        indices_for_each_label = [[] for i in range(5)]\n\n        for i in range(len(dataset)):\n            label = dataset.cassava_leaf_disease.label[i]\n            indices_for_each_label[label].append(i)\n\n        dataset_for_each_label = [torch.utils.data.dataset.Subset(dataset, indices) for indices in indices_for_each_label]\n        dataset_for_each_label[3], _ = torch.utils.data.random_split(dataset_for_each_label[3], [2500, 13158 - 2500])\n\n        train_set_for_each_label = []\n        val_set_for_each_label = []\n        for sub_dataset in dataset_for_each_label:\n            sub_train_set, sub_val_set = split_dataset(sub_dataset, train_rate)\n            train_set_for_each_label.append(sub_train_set)\n            val_set_for_each_label.append(sub_val_set)\n\n        train_set_for_each_label[0] = torch.utils.data.ConcatDataset([train_set_for_each_label[0],\n                                                                      train_set_for_each_label[0]])\n        val_set_for_each_label[0] = torch.utils.data.ConcatDataset([val_set_for_each_label[0],\n                                                                    val_set_for_each_label[0]])\n\n        train_set = torch.utils.data.ConcatDataset(train_set_for_each_label)\n        val_set = torch.utils.data.ConcatDataset(val_set_for_each_label)\n    else:\n        train_set, val_set = split_dataset(dataset, train_rate)\n    \n    train_loader = DataLoader(train_set,\n                          batch_size=train_batch_size,\n                          shuffle=True,\n                          num_workers=num_workers)\n\n    val_loader = DataLoader(val_set,\n                            batch_size=val_batch_size,\n                            shuffle=False,\n                            num_workers=num_workers)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if use_class_weight:\n    def class_weight(label_count):\n        total = sum(label_count)\n        weight = [total / (2 * count) for count in label_count]\n        return torch.tensor(weight)\n\n    label_count = [0 for i in range(5)]\n\n    for i in range(len(dataset)):\n        label = dataset.cassava_leaf_disease.label[i]\n        label_count[label] += 1\n    print(label_count)\n\n    loss_weight = class_weight(label_count)\n    loss_weight *= mul_weight\n    if torch.cuda.is_available:\n        loss_weight = loss_weight.cuda()\n        print(loss_weight)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def progress_print(comment, current, total, num_of_print=50, slow=5):\n#     end = '\\x1b[1K\\r' if current < total else '\\n'\n    end = '\\r' if current < total else '\\n'\n    symbol = ['\\\\', '/', '-']\n    done = int((current - 1) / total * num_of_print)\n    doing = symbol[(current % (len(symbol) * slow) // slow)] if current < total else ''\n    yet = max(0, num_of_print - 1 - done)\n    print(comment + ': ' + '#' * done + doing + '*' * yet, end=end)\n    \ndef num_of_correct(outputs, labels):\n    _, predicted = torch.max(outputs, 1)\n    c = (predicted == labels).squeeze()\n    return c.sum().item()\n\n\ndef train(model=None, criterion=None,\n          optimizer=None, data_loader=None,\n          debug=True, epoch=0, debug_rate=0.01):\n    model.train()\n    data_loader_length = len(data_loader)\n    correct, total = 0, 0\n    running_loss = 0.0\n    running_debug_rate = debug_rate\n    running_debug_step = int(data_loader_length * running_debug_rate)\n    for i, data in enumerate(data_loader):\n        inputs, labels = data\n        if torch.cuda.is_available():\n            inputs, labels = inputs.cuda(), labels.cuda()\n        optimizer.zero_grad()\n\n        outputs = model(inputs)\n        loss = criterion(outputs, labels)\n        loss.backward()\n        optimizer.step()\n\n        correct += num_of_correct(outputs, labels)\n        total += len(inputs)\n\n        running_loss += loss.item() * len(inputs)\n        if debug:\n            progress_print('train', i, running_debug_step, slow=3)\n        if debug and i == running_debug_step - 1:\n            print('epoch: %d (%d/%d),\\tloss: %.3f' % (epoch + 1,\n                                                     i + 1,\n                                                     data_loader_length,\n                                                     running_loss / total))\n            running_debug_rate += debug_rate\n            running_debug_step = min(data_loader_length,\n                                     int(data_loader_length * running_debug_rate))\n    return correct / total, running_loss / total\n\ndef valid(model=None, data_loader=None, debug=True):\n    model.eval()\n    correct, total = 0, 0\n    with torch.no_grad():\n        for i, data in enumerate(data_loader):\n            inputs, labels = data\n            if torch.cuda.is_available():\n                inputs, labels = inputs.cuda(), labels.cuda()\n            outputs = model(inputs)\n\n            correct += num_of_correct(outputs, labels)\n            total += len(inputs)\n            if debug:\n                progress_print('validation', i, len(data_loader), slow=3)\n    return correct / total\n\ndef get_confusion_matrix(model, data_loader, num_classes):\n    model.eval()\n    confusion_matrix = np.zeros((num_classes, num_classes), dtype=np.int32)\n    with torch.no_grad():\n        for i, data in enumerate(data_loader):\n            inputs, labels = data\n            if torch.cuda.is_available():\n                inputs = inputs.cuda()\n            outputs = model(inputs)\n            _, predicted = torch.max(outputs, 1)\n            predicts = predicted.squeeze()\n            for p, l in zip(predicts, labels):\n                confusion_matrix[p.item()][l] += 1\n            \n            progress_print('confusion_matrix', i, len(data_loader), slow=3)\n    return confusion_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ConvUnit(torch.nn.Module):\n    def __init__(self,\n                 in_channels:int,\n                 out_channels:int,\n                 kernel_size:int,\n                 padding:int=0,\n                 stride:int=1,\n                 groups:int=1,\n                 batch_norm=True,\n                 activation=True):\n        super().__init__()\n        modules = [torch.nn.Conv2d(in_channels,\n                                   out_channels,\n                                   kernel_size=kernel_size,\n                                   padding=padding,\n                                   stride=stride,\n                                   groups=groups,\n                                   bias=not batch_norm)]\n        if batch_norm:\n            modules.append(torch.nn.BatchNorm2d(out_channels))\n        if activation:\n            modules.append(torch.nn.LeakyReLU())\n        self.sequence = torch.nn.Sequential(*modules)\n        self.out_channels = out_channels\n    \n    def forward(self, x):\n        x = self.sequence(x)\n        return x\n\n# Depthwise Convolution\ndef DConvUnit(in_channels:int, kernel_size:int, stride:int=1) -> torch.nn.Module:\n    padding = kernel_size // 2\n    return ConvUnit(in_channels, in_channels, kernel_size, padding, stride, in_channels)\n\n# Expansion Convolution\ndef EConvUnit(in_channels:int, factor:int=6) -> torch.nn.Module:\n    return ConvUnit(in_channels, factor * in_channels, 1)\n\n# Pointwise Convolution\ndef PConvUnit(in_channels, out_channels, activation=False) -> torch.nn.Module:\n    return ConvUnit(in_channels, out_channels, 1, activation=activation)\n\n# Depthwise Separable Convolution\nclass DSConvUnit(torch.nn.Module):\n    def __init__(self,\n                 in_channels:int,\n                 out_channels:int,\n                 kernel_size:int=3,\n                 stride:int=1,\n                 activation=True):\n        super().__init__()\n        modules = [\n            DConvUnit(in_channels, kernel_size, stride),\n            PConvUnit(in_channels, out_channels, activation)\n        ]\n        self.sequence = torch.nn.Sequential(*modules)\n        self.out_channels = out_channels\n    \n    def forward(self, x):\n        x = self.sequence(x)\n        return x\n\n# Squeeze and Exication \nclass SEUnit(torch.nn.Module):\n    def __init__(self,\n                 in_channels:int,\n                 se_ratio:float):\n        super().__init__()\n        se_channels = max(1, int(in_channels * se_ratio))\n        self.sequence = torch.nn.Sequential(\n            torch.nn.AdaptiveAvgPool2d((1, 1)),\n            ConvUnit(in_channels, se_channels, 1, batch_norm=False),\n            ConvUnit(se_channels, in_channels, 1, batch_norm=False, activation=False),\n            torch.nn.Sigmoid()\n        )\n        self.out_channels = in_channels\n    \n    def forward(self, x):\n        y = self.sequence(x)\n        z = x * y\n        return z\n        \n\nclass BottleneckUnit(torch.nn.Module):\n    def __init__(self,\n                 in_channels:int,\n                 out_channels:int,\n                 factor:int=6,\n                 kernel_size:int=3,\n                 stride:int=1,\n                 has_se:bool=True,\n                 se_ratio:float=0.25):\n        super().__init__()\n        modules = []\n        self.residual = stride==1 and in_channels == out_channels\n        hid_channels = factor * in_channels\n        if factor > 1:\n            modules.append(EConvUnit(in_channels, factor))\n        modules.append(DConvUnit(hid_channels, kernel_size, stride))\n        if has_se:\n            calced_se_ratio = se_ratio / factor\n            modules.append(SEUnit(hid_channels, calced_se_ratio))\n        modules.append(PConvUnit(hid_channels, out_channels, False))\n        self.sequence = torch.nn.Sequential(*modules)\n        self.activation = torch.nn.LeakyReLU()\n        self.out_channels = in_channels\n    \n    def forward(self, x):\n        y = self.sequence(x)\n        z = self.activation(x + y if self.residual else y)\n        return z","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Calculate floor of var * coef\ndef round_filters(filters, coef:float, divisor:int) -> int:\n    filters *= coef\n    filters = max(divisor, int(filters + divisor / 2) // divisor * divisor)\n    return filters\n\ndef round_repeats(repeats, coef):\n    repeats = int(math.ceil(repeats * coef)) if repeats > 0 else -repeats\n    return repeats\n\nclass BlockArgs():\n    def __init__(self,\n                 module:torch.nn.Module,\n                 filters:int,\n                 num_repeats:int=1,\n                 **kwargs):\n        self.module = module\n        self.filters = filters\n        self.num_repeats = num_repeats\n        self.kwargs = kwargs\n    \n    def __call__(self,\n                 in_channels:int,\n                 width_coef:float=1.0,\n                 depth_coef:float=1.0,\n                 divisor:int=8) -> torch.nn.Module:\n        out_channels = round_filters(self.filters, width_coef, divisor)\n        num_repeats = round_repeats(self.num_repeats, depth_coef)\n        modules = []\n        for _ in range(num_repeats):\n            modules.append(self.module(in_channels, out_channels, **self.kwargs))\n            if 'stride' in self.kwargs:\n                del self.kwargs['stride']\n            in_channels = out_channels\n        return torch.nn.Sequential(*modules), in_channels\n\nEFFICIENT_NET_BLOCK_ARGS = [\n#     BlockArgs(ConvUnit, 32, -1, kernel_size=3, stride=2, padding=1),\n    BlockArgs(ConvUnit, 32, -1, kernel_size=3, stride=2, padding=1),\n    BlockArgs(BottleneckUnit, 16, 1, factor=1),\n    BlockArgs(BottleneckUnit, 24, 2, stride=2),\n    BlockArgs(BottleneckUnit, 40, 2, kernel_size=5, stride=2),\n    BlockArgs(BottleneckUnit, 80, 3, stride=2),\n    BlockArgs(BottleneckUnit, 112, 3, kernel_size=5),\n    BlockArgs(BottleneckUnit, 192, 4, kernel_size=5, stride=2),\n    BlockArgs(BottleneckUnit, 320, 1)\n]\n\n# (depth_coef, width_coef)\nEFFICIENT_NET_COMPOUND_COEF = [\n    (1.0, 1.0),\n    (1.1, 1.0),\n    (1.2, 1.1),\n    (1.4, 1.2),\n    (1.8, 1.4),\n    (2.2, 1.6),\n    (2.6, 1.8),\n    (3.1, 2.0)\n]\n\nclass EfficientNet(torch.nn.Module):\n    def __init__(self,\n                 in_channels:int=3,\n                 num_of_class:int=1000,\n                 depth_coef:float=1.0,\n                 width_coef:float=1.0,\n                 hidden_channels:int=1280,\n                 block_args=EFFICIENT_NET_BLOCK_ARGS):\n        super().__init__()\n        modules = []\n        self.hidden_channels = round_filters(hidden_channels, width_coef, 8)\n        for block_arg in block_args:\n            module, in_channels = block_arg(in_channels, width_coef, depth_coef)\n            modules.append(module)\n        modules.append(PConvUnit(in_channels, self.hidden_channels, True))\n        modules.append(torch.nn.AdaptiveAvgPool2d((1, 1)))\n        self.sequence = torch.nn.Sequential(*modules)\n        self.linear = torch.nn.Linear(self.hidden_channels, num_of_class)\n    \n    def forward(self, x):\n        x = self.sequence(x)\n        x = x.view(-1, self.hidden_channels)\n        x = self.linear(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = EfficientNet(3, 5, *EFFICIENT_NET_COMPOUND_COEF[efficient_net_version])\nif use_pre_trained_weight:\n    pre_trained_weight = torch.load(pre_trained_weight_path)\n    model.load_state_dict(pre_trained_weight)\nif torch.cuda.is_available():\n    model = model.cuda()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train"},{"metadata":{"trusted":true},"cell_type":"code","source":"if not is_submission:\n    if use_class_weight:\n        criterion = torch.nn.CrossEntropyLoss(weight=loss_weight)\n    elif mul_weight is not None:\n        if torch.cuda.is_available():\n            mul_weight = mul_weight.cuda()\n        criterion = torch.nn.CrossEntropyLoss(weight=mul_weight)\n    else:\n        criterion = torch.nn.CrossEntropyLoss()\n    optimizer = torch.optim.Adam(model.parameters(),\n                                 lr=learning_rate,\n                                 weight_decay=weight_decay)\n    train_accuracy = []\n    train_losses = []\n    val_accuracy= []\n    best_accuracy = 0.0 if len(val_accuracy) == 0 else min(val_accuracy)\n    best_model = deepcopy(model.state_dict())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if not is_submission:\n    for epoch in range(*train_epoch):\n        train_acc, train_loss = train(model, criterion, optimizer, train_loader, epoch=epoch, debug=debug, debug_rate=0.1)\n        val_acc = valid(model, val_loader, debug=debug)\n        train_accuracy.append(train_acc)\n        train_losses.append(train_loss)\n        val_accuracy.append(val_acc)\n        if best_accuracy < val_acc:\n            best_model = deepcopy(model.state_dict())\n            best_accuracy = val_acc\n            print('Best model is updated.')\n        print(train_acc, val_acc)\n    torch.save(best_model, 'weight.pth')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if not is_submission:\n    print(train_accuracy)\n    print(train_losses)\n    print(val_accuracy)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if not is_submission:\n    plt.plot(train_accuracy, label='train')\n    plt.plot(train_losses, label='loss')\n    plt.plot(val_accuracy, label='validation')\n    plt.legend(loc='upper left')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Confusion Matrix"},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_confusion_matrix(confusion_matrix):\n    row_sums = confusion_matrix.sum(axis=1)\n    normalized_confusion_matrix = confusion_matrix / row_sums[:, np.newaxis]\n    diagonal_confusion_matrix = deepcopy(confusion_matrix)\n    np.fill_diagonal(diagonal_confusion_matrix, 0)\n    row_sums = diagonal_confusion_matrix.sum(axis=1)\n    normalized_diagonal_confusion_matrix = diagonal_confusion_matrix / row_sums[:, np.newaxis]\n    plt.figure(figsize=(2,1))\n    plt.matshow(normalized_confusion_matrix, cmap='gray')\n    plt.matshow(normalized_diagonal_confusion_matrix, cmap='gray')\n    plt.show()\n    print(confusion_matrix)\n    print(normalized_confusion_matrix)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if not is_submission:\n    confusion_matrix = get_confusion_matrix(model, val_loader, 5)\n#     confusion_matrix = np.array([[  597,   181,    39,    70,   288],\n#                                  [   89,  1504,    77,   193,   102],\n#                                  [   32,    49,  1685,   554,   153],\n#                                  [   40,   172,   317, 12053,   235],\n#                                  [  329,   283,   268,   288,  1799]])\n    show_confusion_matrix(confusion_matrix)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"if is_submission:\n    model.eval()\n    submission = dataset.cassava_leaf_disease.copy()\n    predicts = []\n    with torch.no_grad():\n        for i, data in enumerate(sub_loader):\n            inputs, labels = data\n            if torch.cuda.is_available():\n                inputs = inputs.cuda()\n            outputs = model(inputs)\n            _, predicted = torch.max(outputs, 1)\n            predicts.extend(predicted.cpu().numpy())\n            \n    submission['label'] = predicts\n    submission.to_csv('submission.csv', index=False)\n    ","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}