{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os.path as osp\n\n# computation\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\n\n# data pipeline\nimport imageio\n#from imgaug import augmenters as iaa\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision\nfrom torchvision import transforms\nimport torchvision.transforms as transforms\n\n# utils\nfrom tqdm.notebook import tqdm\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import train_test_split\n\nfrom torch.utils.data.sampler import SubsetRandomSampler, WeightedRandomSampler","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Some constants\nROOT = '/kaggle/input/cassava-leaf-disease-classification'\nTRAIN_DIR = f'{ROOT}/train_images/'\nTRAIN_CSV = f'{ROOT}/train.csv'\nTEST_DIR = f'{ROOT}/test_images/'\nTEST_CSV = f'{ROOT}/sample_submission.csv'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaDataset(Dataset):\n    def __init__(self, split, transform=None):\n        assert split in ('train', 'val', 'test')\n        self.split = split\n        self.transform = transform\n        if split in ('train', 'val'):\n            csv = pd.read_csv(TRAIN_CSV)\n            \n            #csv = csv[:5000]  # 데이터 개수 조절\n            \n            self.df = train_test_split(\n                csv, test_size=0.1, random_state=0,stratify=csv[['label']]\n            )[0 if split == 'train' else 1].reset_index()\n        else:\n            self.df = pd.read_csv(TEST_CSV)\n        \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, i: int):\n        base_dir = TRAIN_DIR if self.split in ('train', 'val') else TEST_DIR\n        x = imageio.imread(osp.join(base_dir, self.df['image_id'][i]))\n        y = self.df['label'][i] if self.split in ('train', 'val') else -1\n        if self.transform:\n            x = self.transform(x)\n        return (x, y)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Implement your data augmentation pipelines"},{"metadata":{"trusted":true},"cell_type":"code","source":"# TODO: You'll need some heavy augmentations to get a high score. \n#\n# See \n#     https://github.com/aleju/imgaug \n#     https://imageio.readthedocs.io/en/stable/\n#\n# for detailed `imgaug` references.\n#\n# NOTE: If you choose to normalize the training images, you should normalize the test images as well.\n#\nINPUT_SIZE = 224\nRESIZE = 500\njitter_param = 0.05\n\nTRANSFORMS = {\n    'train': transforms.Compose([\n        \n#         iaa.Sequential([\n#             iaa.Resize((INPUT_SIZE, INPUT_SIZE)),\n#             iaa.Fliplr(0.5),\n#         ]).augment_image,  \n        transforms.ToPILImage(),\n#         transforms.Resize((RESIZE,RESIZE)),\n#         transforms.RandomCrop((INPUT_SIZE,INPUT_SIZE)),\n#        transforms.RandomResizedCrop((INPUT_SIZE,INPUT_SIZE)),\n        transforms.CenterCrop((INPUT_SIZE,INPUT_SIZE)),\n        transforms.ColorJitter(\n            brightness=jitter_param,\n            contrast=jitter_param,\n            saturation=jitter_param),\n        transforms.RandomVerticalFlip(),\n        transforms.RandomHorizontalFlip(),        \n        transforms.ToTensor(),\n        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))\n    ]),\n    'val': transforms.Compose([\n        transforms.ToPILImage(),\n        #transforms.RandomResizedCrop((INPUT_SIZE,INPUT_SIZE)),\n        transforms.CenterCrop((INPUT_SIZE,INPUT_SIZE)),\n        transforms.ToTensor(),\n        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))\n    ]),\n    'test': transforms.Compose([\n#         transforms.ToPILImage(),\n#         transforms.Resize((RESIZE,RESIZE)),\n#         transforms.RandomCrop((INPUT_SIZE,INPUT_SIZE)),\n#         transforms.ToTensor(),\n#         transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))\n        transforms.ToPILImage(),\n        transforms.CenterCrop((INPUT_SIZE,INPUT_SIZE)),\n        #transforms.RandomResizedCrop((INPUT_SIZE,INPUT_SIZE)),\n        transforms.ToTensor(),\n        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))\n    ]),\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 128\n\ntrain_dataset = CassavaDataset('train', transform=TRANSFORMS['train'])\n\nval_dataset = CassavaDataset('val', transform=TRANSFORMS['val'])\n\ntest_dataset = CassavaDataset('test', transform=TRANSFORMS['test'])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# y_train = []\n# for i in range(len(train_dataset)):\n#     y_train.append(train_dataset[i][1])\n#     y_train = np.array(y_train)\ny_train = np.array(train_dataset.df['label'])\n\nclass_sample_count = np.array([len(np.where(y_train==t)[0]) for t in np.unique(y_train)])\n\nweight = 1. / class_sample_count\n\nsamples_weight = np.array([weight[t] for t in y_train])\n\nsamples_weight = torch.from_numpy(samples_weight)\n\ntrain_sampler = WeightedRandomSampler(samples_weight.type('torch.DoubleTensor'), len(samples_weight))\n          ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"counts = np.bincount(y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"counts","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":""},{"metadata":{"trusted":true},"cell_type":"code","source":"train_loader = DataLoader(train_dataset, batch_size=batch_size,sampler=train_sampler)\nval_loader = DataLoader(val_dataset, batch_size=batch_size)\ntest_loader = DataLoader(test_dataset, batch_size=batch_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# hist = np.array([0, 0, 0, 0, 0])\n# for x,y in train_loader:\n#     y = np.array(y)\n#     for i in range(len(y)):\n#         #print(y[i])\n#         hist[y[i]] += 1\n# print(hist)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"training data:\",len(train_loader.dataset))\nprint(\"validation data:\",len(val_loader.dataset))\nprint(\"test data:\",len(test_loader.dataset))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Build your Cassava leaf classifier"},{"metadata":{"trusted":true},"cell_type":"code","source":"# TODO: Implement your classifier.\n# class CassavaClassifier(nn.Module):\n#     def __init__(self):\n\n#         super().__init__()\n#         CH_LAST = 128\n#         NUM_CLASSES = 5\n#         # TODO: Change this part to improve your score\n#         self.conv1 = nn.Conv2d(3, 64, kernel_size=3, padding=1)\n#         self.conv2 = nn.Conv2d(64, 128, kernel_size=3, padding=1)\n#         self.conv3 = nn.Conv2d(128, CH_LAST, kernel_size=3, padding=1)\n#         # NOTE: input channel should match CH_LAST * (IMAGE_INPUT_SIZE / 2 ** NUM_DOWNSAMPLE) ** 2 \n#         self.fc = nn.Linear(CH_LAST *  (INPUT_SIZE // 2 ** 3) ** 2, NUM_CLASSES)\n#         self.pool = nn.MaxPool2d(kernel_size=(3, 3), stride=2, padding=1)\n        \n#     def forward(self, x):\n#         h = self.pool(F.relu(self.conv1(x)))\n#         h = self.pool(F.relu(self.conv2(h)))\n#         h = self.pool(F.relu(self.conv3(h)))\n#         return self.fc(h.flatten(1))\n# model = CassavaClassifier().cuda()\n#model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#from torchvision.models.resnet import ResNet, BasicBlock, Bottleneck\n\nmodel_ft = torchvision.models.resnet18(pretrained=False)\n#model_ft = torchvision.models.resnet34(pretrained=True)\n#model_ft = torchvision.models.mobilenet(pretrained=False)\nnum_features = model_ft.fc.in_features # 마지막 레이어 아웃풋 숫자\n\n# Here the size of each output sample is set to 10.\n# Alternatively, it can be generalized to nn.Linear(num_ftrs, len(class_names)).\n\nmodel_ft.fc = nn.Linear(num_features, 5)\nmodel = model_ft.cuda()\n\n#print(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import torchvision.models as models\n# class Cassavaclassifier(nn.Module):\n#     def __init__(self):\n#         super().__init__()\n#         self.mobilenet = models.mobilenet_v2()\n#         self.mobilenet.fc = nn.Linear(self.mobilenet.fc.in_features, 5)\n    \n#     def forward(self, x):\n#         return self.mobilenet(x)\n    \n# model = Cassavaclassifier().cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # TODO: Implement your classifier.\n# class CassavaClassifier(nn.Module):\n#     def __init__(self):\n\n#         super().__init__()\n#         CH_LAST = 128\n#         NUM_CLASSES = 5\n\n#         # TODO: Change this part to improve your score\n#         self.conv1 = nn.Conv2d(3, 32, kernel_size=3, padding=1)\n\n#         self.conv2 = nn.Conv2d(32, 64, kernel_size=3, padding=1)\n\n#         self.conv3 = nn.Conv2d(64, CH_LAST, kernel_size=3, padding=1)\n\n#         # NOTE: input channel should match CH_LAST * (IMAGE_INPUT_SIZE / 2 ** NUM_DOWNSAMPLE) ** 2 \n#         self.fc1 = nn.Linear(CH_LAST *  (INPUT_SIZE // 2 ** 3) ** 2, 512)\n#         self.fc2 = nn.Linear(512,1024)\n#         self.fc3 = nn.Linear(1024,256)\n#         self.fc4 = nn.Linear(256, NUM_CLASSES)\n#         self.pool = nn.MaxPool2d(kernel_size=(3, 3), stride=2, padding=1)\n#         self.bn1 = nn.BatchNorm2d(32)\n#         self.bn2 = nn.BatchNorm2d(64)\n#         self.bn3 = nn.BatchNorm2d(CH_LAST)\n#         self.dropout = nn.Dropout(p=0.5)\n#     def forward(self, x):\n#         h = self.pool(self.bn1(F.relu(self.conv1(x))))\n#         h = self.pool(self.bn2(F.relu(self.conv2(h))))\n#         h = self.pool(self.bn3(F.relu(self.conv3(h))))\n#         h = self.fc1(h.flatten(1))\n#         h = self.dropout(h)\n#         h = self.fc2(h)\n#         h = self.dropout(h)\n#         h = self.fc3(h)\n#         h = self.dropout(h)\n#         h = self.fc4(h)\n#         return h\n# model = CassavaClassifier().cuda()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Implement the training loop"},{"metadata":{"trusted":true},"cell_type":"code","source":"# TODO: Prepare some components for training your network.\n#\n# See\n#\n#     Optimizer:    https://pytorch.org/docs/stable/optim.html#how-to-use-an-optimizer\n#     LR Scheduler: https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\n#     DataLoader:   https://pytorch.org/docs/stable/data.html?highlight=dataloader#torch.utils.data.DataLoader\n#\n# for their actual APIs and detailed usages.\n#model = NotImplemented\n#optimizer = NotImplemented\n#train_dataset = NotImplemented\n#val_dataset = NotImplemented","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.autograd import Variable\n\nclass FocalLoss(nn.Module):\n    def __init__(self, gamma=0, alpha=None, size_average=True):\n        super(FocalLoss, self).__init__()\n        self.gamma = gamma\n        self.alpha = alpha\n        if isinstance(alpha,(float,int)): self.alpha = torch.Tensor([alpha,1-alpha])\n        #if isinstance(alpha,(float,int,long)): self.alpha = torch.Tensor([alpha,1-alpha])    \n        if isinstance(alpha,list): self.alpha = torch.Tensor(alpha)\n        self.size_average = size_average\n\n    def forward(self, input, target):\n        if input.dim()>2:\n            input = input.view(input.size(0),input.size(1),-1)  # N,C,H,W => N,C,H*W\n            input = input.transpose(1,2)    # N,C,H*W => N,H*W,C\n            input = input.contiguous().view(-1,input.size(2))   # N,H*W,C => N*H*W,C\n        target = target.view(-1,1)\n\n        logpt = F.log_softmax(input)\n        logpt = logpt.gather(1,target)\n        logpt = logpt.view(-1)\n        pt = Variable(logpt.data.exp())\n\n        if self.alpha is not None:\n            if self.alpha.type()!=input.data.type():\n                self.alpha = self.alpha.type_as(input.data)\n            at = self.alpha.gather(0,target.data.view(-1))\n            logpt = logpt * Variable(at)\n\n        loss = -1 * (1-pt)**self.gamma * logpt\n        if self.size_average: return loss.mean()\n        else: return loss.sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#loss_fn = nn.CrossEntropyLoss()\n#loss_fn = FocalLoss(gamma=0.1)()\noptimizer = torch.optim.SGD(model.parameters(), lr=0.01,momentum=0.9,weight_decay=0.0001)\nscheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[5,10,15], gamma=0.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# TODO: Implement your training loop here.\nn_epochs = 100\n\ntrain_losses = []\nval_losses = []\navg_train_losses = []\navg_val_losses = []\n\nbest_score = -100\nfor epoch in range(n_epochs + 1):\n    \n    print(\"current learning rate\")\n    for param_group in optimizer.param_groups:\n            print(\"lr:\",param_group['lr'])\n    # training loop\n    model.train()\n    train_correct = 0\n    train_total = 0\n    \n    for batch_idx, samples in enumerate(train_loader):\n        x_train, y_train = samples\n        \n        x_train = x_train.cuda()\n        y_train = y_train.cuda()\n\n        y_pred = model(x_train)\n        loss = FocalLoss(gamma=1)(y_pred, y_train)\n        #loss = loss_fn(y_pred, y_train)\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        \n        train_losses.append(loss.item())\n        train_correct += (torch.max(model(x_train),1)[1] == y_train).sum()\n        train_total += x_train.size(0)\n        \n        if batch_idx % int(10) == 0:\n            print('Epoch {:4d}/{} Batch {}/{} Loss: {:.6f}'.format(\n                epoch, n_epochs, batch_idx+1, len(train_loader),\n                loss.item()\n                ))\n        \n    accuracy = 100 * int(train_correct) / train_total\n    print(\"Train Accuracy ={}\".format(accuracy))\n    \n    # validatoin loop\n    model.eval()\n    val_correct = 0\n    val_total = 0\n    \n    for batch_idx, samples in enumerate(val_loader):\n        x_train, y_train = samples\n        \n        x_train = x_train.cuda()\n        y_train = y_train.cuda()\n\n        y_pred = model(x_train)\n        loss = FocalLoss(gamma=1)(y_pred, y_train)\n        #loss = loss_fn(y_pred, y_train)\n        \n        val_losses.append(loss.item())\n        val_correct += (torch.max(model(x_train),1)[1] == y_train).sum()\n        val_total += x_train.size(0)\n        \n    accuracy = 100 *int(val_correct) / val_total\n    print(\"Validation Accuracy ={}\".format(accuracy))\n    \n    \n    \n    train_loss = np.average(train_losses)\n    val_loss = np.average(val_losses)\n    avg_train_losses.append(train_loss)\n    avg_val_losses.append(val_loss)\n\n    score = -val_loss\n    \n    if score > best_score:\n        best_score = score\n        torch.save(model, 'model.pt')\n        print(\"saved\")\n    \n    print(\"best_score:\",best_score)    \n    \n    epoch_len = len(str(n_epochs))\n\n    print_msg = (f'[{epoch:>{epoch_len}}/{n_epochs:>{epoch_len}}] ' +\n                 f'train_loss: {train_loss:.5f} ' +\n                 f'valid_loss: {val_loss:.5f}')\n\n    print(print_msg)\n    \n    train_losses = []\n    val_losses = []\n    scheduler.step()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\nprint(avg_train_losses)\nprint(avg_val_losses)\n# 훈련이 진행되는 과정에 따라 loss를 시각화\nfig = plt.figure(figsize=(10,8))\nplt.plot(range(1,len(avg_train_losses)+1),avg_train_losses, label='Training Loss')\nplt.plot(range(1,len(avg_val_losses)+1),avg_val_losses,label='Validation Loss')\n\n# validation loss의 최저값 지점을 찾기\n#minposs = valid_loss.index(min(valid_loss))+1\n#plt.axvline(minposs, linestyle='--', color='r',label='Early Stopping Checkpoint')\n\nplt.xlabel('epochs')\nplt.ylabel('loss')\n#plt.ylim(0, 0.5) # 일정한 scale\nplt.xlim(0, len(avg_train_losses)+1) # 일정한 scale\nplt.grid(True)\nplt.legend()\nplt.tight_layout()\nplt.show()\nfig.savefig('loss_plot.png', bbox_inches = 'tight')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# test loss 및 accuracy을 모니터링하기 위해 list 초기화\n\n# https://stackoverflow.com/questions/31421413/how-to-compute-precision-recall-accuracy-and-f1-score-for-the-multiclass-case\n# https://www.sciencedirect.com/science/article/pii/S1110866520301110\n\nfrom sklearn.metrics import precision_recall_fscore_support as score #!!\n\ntest_loss = 0.0\nclass_correct = list(0. for i in range(5))\nclass_total = list(0. for i in range(5))\nmodel = torch.load('/kaggle/working/model.pt')\nmodel.eval() # prep model for evaluation\ntotal = 0\n\ny_pred = []  #!!\ny_test = []  #!!\n\nfor data, target in val_loader :\n    y_test.extend(target.tolist())  #!!\n    data = data.cuda()\n    target = target.cuda()\n#     if len(target.data) != batch_size:\n#         break\n\n    # forward pass: 입력을 모델로 전달하여 예측된 출력 계산\n    output = model(data)\n    \n    # calculate the loss\n    loss =loss_fn(output, target)\n    # update test loss\n    test_loss += loss.item()*data.size(0)\n    # 출력된 확률을 예측된 클래스로 변환\n    _, pred = torch.max(output, 1)\n    y_pred.extend(pred.tolist())  #!!\n    # 예측과 실제 라벨과 비교\n    correct = np.squeeze(pred.eq(target.data.view_as(pred)))\n    # 각 object class에 대해 test accuracy 계산\n    \n    #for i in range(batch_size):\n    for i in range(len(target.data)):\n        label = target.data[i]\n        class_correct[label] += correct[i].item()\n        class_total[label] +=1\n    total += data.size(0)\n#     print(total)\n#     print(test_loss)\n\nprecision, recall, fscore, support = score(y_test, y_pred) #!!\n\n# print('precision : {}'.format(precision)) #!!\n# print('recall    : {}'.format(recall)) #!!\n# print('fscore    : {}'.format(fscore)) #!!\n# print('support   : {}'.format(support)) #!!\n\nprint(\"           precision recall    fscore    support   \")\nfor ii in range(5):\n    print('class %1d |  %.3f     %.3f     %.3f     %.3f' %(ii,precision[ii],recall[ii],fscore[ii],support[ii]))\n    \n# calculate and print avg test loss\ntest_loss = test_loss/total\n\nprint('Test Loss: {:.6f}\\n'.format(test_loss))\n\nfor i in range(5):\n    if class_total[i] > 0:\n        print('Test Accuracy of %5s: %2d%% (%2d/%2d)' % (\n            str(i), 100 * class_correct[i] / class_total[i],\n            np.sum(class_correct[i]), np.sum(class_total[i])))\n    else:\n        print('Test Accuracy of %5s: N/A (no training examples)' % (classes[i]))\n\nprint('\\nTest Accuracy (Overall): %.3f%% (%2d/%2d)' % (\n    100. * np.sum(class_correct) / np.sum(class_total),\n    np.sum(class_correct), np.sum(class_total)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Verify that your trained model can be loaded"},{"metadata":{"trusted":true},"cell_type":"code","source":"model = torch.load('/kaggle/working/model.pt')\nmodel.eval()\ntest_dataset = CassavaDataset('test', transform=TRANSFORMS['test'])\ntest_loader = DataLoader(test_dataset, batch_size=64)\ntest_csv = pd.read_csv(TEST_CSV)\n\ny_hats = []\nfor x, _ in test_loader:\n    y_hat = model(x.cuda())\n    y_hat = torch.argmax(y_hat,dim=1)\n    y_hats.extend(y_hat.cpu().detach().numpy().tolist())\n    \n\ntest_csv['label'] = y_hats\ntest_csv[['image_id','label']].to_csv(\"submission.csv\", index=False)\ntest_csv.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}