{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os.path as osp\n\n# computation\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\n\n# data pipeline\nimport imageio\n#from imgaug import augmenters as iaa\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nimport torchvision.transforms as transforms\n\n# utils\nfrom tqdm.notebook import tqdm\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import train_test_split\n\nfrom torch.utils.data.sampler import SubsetRandomSampler, WeightedRandomSampler","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Some constants\nROOT = '/kaggle/input/cassava-leaf-disease-classification'\nTRAIN_DIR = f'{ROOT}/train_images/'\nTRAIN_CSV = f'{ROOT}/train.csv'\nTEST_DIR = f'{ROOT}/test_images/'\nTEST_CSV = f'{ROOT}/sample_submission.csv'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaDataset(Dataset):\n    def __init__(self, split, transform=None):\n        assert split in ('train', 'val', 'test')\n        self.split = split\n        self.transform = transform\n        if split in ('train', 'val'):\n            csv = pd.read_csv(TRAIN_CSV)\n            \n            #csv = csv[:10000]  # 데이터 개수 조절\n            \n            self.df = train_test_split(\n                csv, test_size=0.1, random_state=0,stratify=csv[['label']]\n            )[0 if split == 'train' else 1].reset_index()\n        else:\n            self.df = pd.read_csv(TEST_CSV)\n        \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, i: int):\n        base_dir = TRAIN_DIR if self.split in ('train', 'val') else TEST_DIR\n        x = imageio.imread(osp.join(base_dir, self.df['image_id'][i]))\n        y = self.df['label'][i] if self.split in ('train', 'val') else -1\n        if self.transform:\n            x = self.transform(x)\n        return (x, y)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Implement your data augmentation pipelines"},{"metadata":{"trusted":true},"cell_type":"code","source":"# TODO: You'll need some heavy augmentations to get a high score. \n#\n# See \n#     https://github.com/aleju/imgaug \n#     https://imageio.readthedocs.io/en/stable/\n#\n# for detailed `imgaug` references.\n#\n# NOTE: If you choose to normalize the training images, you should normalize the test images as well.\n#\nINPUT_SIZE = 224\nRESIZE = 500\nTRANSFORMS = {\n    'train': transforms.Compose([\n        \n#         iaa.Sequential([\n#             iaa.Resize((INPUT_SIZE, INPUT_SIZE)),\n#             iaa.Fliplr(0.5),\n#         ]).augment_image,  \n        transforms.ToPILImage(),\n        transforms.Resize((RESIZE,RESIZE)),\n        transforms.RandomCrop((INPUT_SIZE,INPUT_SIZE)),\n        #transforms.RandomResizedCrop(224),\n        transforms.RandomVerticalFlip(p=0.2),\n        transforms.RandomHorizontalFlip(p=0.2),        \n        transforms.ToTensor(),\n         transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))\n    ]),\n    'val': transforms.Compose([\n#         iaa.Sequential([\n#             iaa.Resize((INPUT_SIZE, INPUT_SIZE)),\n#         ]).augment_image,\n        transforms.ToPILImage(),\n        transforms.Resize((RESIZE,RESIZE)),\n        transforms.RandomCrop((INPUT_SIZE,INPUT_SIZE)),\n        #transforms.Resize((INPUT_SIZE,INPUT_SIZE)),\n        transforms.ToTensor(),\n        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))\n    ]),\n    'test': transforms.Compose([\n#         iaa.Sequential([\n#             iaa.Resize((INPUT_SIZE, INPUT_SIZE)),\n#         ]).augment_image,\n        transforms.ToPILImage(),\n        transforms.Resize((RESIZE,RESIZE)),\n        transforms.RandomCrop((INPUT_SIZE,INPUT_SIZE)),\n        #transforms.Resize((INPUT_SIZE,INPUT_SIZE)),\n        transforms.ToTensor(),\n        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))\n    ]),\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 128\n\ntrain_dataset = CassavaDataset('train', transform=TRANSFORMS['train'])\n\nval_dataset = CassavaDataset('val', transform=TRANSFORMS['val'])\n\ntest_dataset = CassavaDataset('test', transform=TRANSFORMS['test'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# y_train = []\n# for i in range(len(train_dataset)):\n#     y_train.append(train_dataset[i][1])\n# y_train = np.array(y_train)\ny_train=np.array(train_dataset.df['label'])\n\nclass_sample_count = np.array([len(np.where(y_train==t)[0]) for t in np.unique(y_train)])\nweight = 1. / class_sample_count\nsamples_weight = np.array([weight[t] for t in y_train])\n\nsamples_weight = torch.from_numpy(samples_weight)\n\ntrain_sampler = WeightedRandomSampler(samples_weight.type('torch.DoubleTensor'), len(samples_weight))\n          ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# hist = np.array([0, 0, 0, 0, 0])\n# for x,y in train_loader:\n#     y = np.array(y)\n#     for i in range(len(y)):\n#         #print(y[i])\n#         hist[y[i]] += 1\n# print(hist)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_loader = DataLoader(train_dataset, batch_size=batch_size, sampler=train_sampler)\nval_loader = DataLoader(val_dataset, batch_size=batch_size)\ntest_loader = DataLoader(test_dataset, batch_size=batch_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 64\n\ntrain_dataset = CassavaDataset('train', transform=TRANSFORMS['train'])\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size)\n\nval_dataset = CassavaDataset('val', transform=TRANSFORMS['val'])\nval_loader = DataLoader(val_dataset, batch_size=batch_size)\n\ntest_dataset = CassavaDataset('test', transform=TRANSFORMS['test'])\ntest_loader = DataLoader(test_dataset, batch_size=batch_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"training data:\",len(train_loader.dataset))\nprint(\"validation data:\",len(val_loader.dataset))\nprint(\"test data:\",len(test_loader.dataset))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Build your Cassava leaf classifier"},{"metadata":{"trusted":true},"cell_type":"code","source":"# import torchvision.models as models\n# class Cassavaclassfier(nn.Module):\n#     def __init__(self):\n#         super().__init__()\n#         self.resnet = models.resnet50()\n#         self.resnet.fc = nn.Linear(sle.fresnet.fc.in_features, 5)\n    \n#     def forward(self, x):\n#         return self.resnet(x)\n    \n# model = CassavaClassifier().cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# # TODO: Implement your classifier.\n# class CassavaClassifier(nn.Module):\n#     def __init__(self):\n\n#         super().__init__()\n#         CH_LAST = 128\n#         NUM_CLASSES = 5\n#         # TODO: Change this part to improve your score\n#         self.conv1 = nn.Conv2d(3, 64, kernel_size=3, padding=1)\n#         self.conv2 = nn.Conv2d(64, 128, kernel_size=3, padding=1)\n#         self.conv3 = nn.Conv2d(128, CH_LAST, kernel_size=3, padding=1)\n#         # NOTE: input channel should match CH_LAST * (IMAGE_INPUT_SIZE / 2 ** NUM_DOWNSAMPLE) ** 2 \n#         self.fc = nn.Linear(CH_LAST *  (INPUT_SIZE // 2 ** 3) ** 2, NUM_CLASSES)\n#         self.pool = nn.MaxPool2d(kernel_size=(3, 3), stride=2, padding=1)\n        \n#     def forward(self, x):\n#         h = self.pool(F.relu(self.conv1(x)))\n#         h = self.pool(F.relu(self.conv2(h)))\n#         h = self.pool(F.relu(self.conv3(h)))\n#         return self.fc(h.flatten(1))\n\n# model = CassavaClassifier().cuda()\n# #model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torchvision.models as models\nclass Cassavaclassfier(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.mobilenet = models.mobilenet_v2()\n        self.mobilenet.fc = nn.Linear(sle.fresnet.fc.in_features, 5)\n    \n    def forward(self, x):\n        return self.mobilenet(x)\n    \nmodel = CassavaClassifier().cuda()\n\n#import torchvision.models as models\n#model = models.mobilenet_v2().cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # TODO: Implement your classifier.\n# class CassavaClassifier(nn.Module):\n#     def __init__(self):\n\n#         super().__init__()\n#         CH_LAST = 128\n#         NUM_CLASSES = 5\n#         # TODO: Change this part to improve your score\n#         self.conv1 = nn.Conv2d(3, 64, kernel_size=3, padding=1)\n#         self.conv2 = nn.Conv2d(64, 128, kernel_size=3, padding=1)\n#         self.conv3 = nn.Conv2d(128, CH_LAST, kernel_size=3, padding=1)\n#         # NOTE: input channel should match CH_LAST * (IMAGE_INPUT_SIZE / 2 ** NUM_DOWNSAMPLE) ** 2 \n#         self.fc = nn.Linear(CH_LAST *  (INPUT_SIZE // 2 ** 3) ** 2, NUM_CLASSES)\n#         self.pool = nn.MaxPool2d(kernel_size=(3, 3), stride=2, padding=1)\n        \n#     def forward(self, x):\n#         h = self.pool(F.relu(self.conv1(x)))\n#         h = self.pool(F.relu(self.conv2(h)))\n#         h = self.pool(F.relu(self.conv3(h)))\n#         return self.fc(h.flatten(1))\n    \n# model = CassavaClassifier().cuda()\n# #model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Implement the training loop"},{"metadata":{"trusted":true},"cell_type":"code","source":"loss_fn = nn.CrossEntropyLoss()\noptimizer = torch.optim.SGD(model.parameters(), lr=0.01,momentum=0.9,weight_decay=0.001)\nscheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[5,10], gamma=0.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# TODO: Define some hyperparameters here.\n# loss_fn = nn.CrossEntropyLoss()\n# optimizer = torch.optim.Adam(model.parameters(), lr=1e-4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# TODO: Implement your training loop here.\nn_epochs = 30\n\ntrain_losses = []\nval_losses = []\navg_train_losses = []\navg_val_losses = []\n\nbest_score = -100\nfor epoch in range(n_epochs + 1):\n    \n    print(\"current learning rate\")\n    for param_group in optimizer.param_groups:\n            print(\"lr:\",param_group['lr'])\n    # training loop\n    model.train()\n    train_correct = 0\n    train_total = 0\n    \n    for batch_idx, samples in enumerate(train_loader):\n        x_train, y_train = samples\n        \n        x_train = x_train.cuda()\n        y_train = y_train.cuda()\n\n        y_pred = model(x_train)\n        loss = loss_fn(y_pred, y_train)\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        \n        train_losses.append(loss.item())\n        train_correct += (torch.max(model(x_train),1)[1] == y_train).sum()\n        train_total += x_train.size(0)\n        \n        if batch_idx % int(10) == 0:\n            print('Epoch {:4d}/{} Batch {}/{} Loss: {:.6f}'.format(\n                epoch, n_epochs, batch_idx+1, len(train_loader),\n                loss.item()\n                ))\n        \n    accuracy = 100 * int(train_correct) / train_total\n    print(\"Train Accuracy ={}\".format(accuracy))\n    \n    # validatoin loop\n    model.eval()\n    val_correct = 0\n    val_total = 0\n    \n    for batch_idx, samples in enumerate(val_loader):\n        x_train, y_train = samples\n        \n        x_train = x_train.cuda()\n        y_train = y_train.cuda()\n\n        y_pred = model(x_train)\n        loss = loss_fn(y_pred, y_train)\n        \n        val_losses.append(loss.item())\n        val_correct += (torch.max(model(x_train),1)[1] == y_train).sum()\n        val_total += x_train.size(0)\n        \n    accuracy = 100 *int(val_correct) / val_total\n    print(\"Validation Accuracy ={}\".format(accuracy))\n    \n    \n    \n    train_loss = np.average(train_losses)\n    val_loss = np.average(val_losses)\n    avg_train_losses.append(train_loss)\n    avg_val_losses.append(val_loss)\n\n    score = -val_loss\n    \n    if score > best_score:\n        best_score = score\n        torch.save(model, 'model.pt')\n    \n    print(\"best_score:\",best_score)    \n    \n    epoch_len = len(str(n_epochs))\n\n    print_msg = (f'[{epoch:>{epoch_len}}/{n_epochs:>{epoch_len}}] ' +\n                 f'train_loss: {train_loss:.5f} ' +\n                 f'valid_loss: {val_loss:.5f}')\n\n    print(print_msg)\n    \n    train_losses = []\n    val_losses = []\n    scheduler.step()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\n\n# 훈련이 진행되는 과정에 따라 loss를 시각화\nfig = plt.figure(figsize=(10,8))\nplt.plot(range(1,len(avg_train_losses)+1),avg_train_losses, label='Training Loss')\nplt.plot(range(1,len(avg_val_losses)+1),avg_val_losses,label='Validation Loss')\n\n# validation loss의 최저값 지점을 찾기\n#minposs = valid_loss.index(min(valid_loss))+1\n#plt.axvline(minposs, linestyle='--', color='r',label='Early Stopping Checkpoint')\n\nplt.xlabel('epochs')\nplt.ylabel('loss')\n#plt.ylim(0, 0.5) # 일정한 scale\nplt.xlim(0, len(avg_train_losses)+1) # 일정한 scale\nplt.grid(True)\nplt.legend()\nplt.tight_layout()\nplt.show()\nfig.savefig('loss_plot.png', bbox_inches = 'tight')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# test loss 및 accuracy을 모니터링하기 위해 list 초기화\ntest_loss = 0.0\nclass_correct = list(0. for i in range(5))\nclass_total = list(0. for i in range(5))\n\nmodel.eval() # prep model for evaluation\ntotal = 0\n\nfor data, target in val_loader :\n\n    data = data.cuda()\n    target = target.cuda()\n#     if len(target.data) != batch_size:\n#         break\n\n    # forward pass: 입력을 모델로 전달하여 예측된 출력 계산\n    output = model(data)\n    \n    # calculate the loss\n    loss =loss_fn(output, target)\n    # update test loss\n    test_loss += loss.item()*data.size(0)\n    # 출력된 확률을 예측된 클래스로 변환\n    _, pred = torch.max(output, 1)\n    # 예측과 실제 라벨과 비교\n    correct = np.squeeze(pred.eq(target.data.view_as(pred)))\n    # 각 object class에 대해 test accuracy 계산\n    \n    #for i in range(batch_size):\n    for i in range(len(target.data)):\n        label = target.data[i]\n        class_correct[label] += correct[i].item()\n        class_total[label] +=1\n    total += data.size(0)\n#     print(total)\n#     print(test_loss)\n    \n# calculate and print avg test loss\ntest_loss = test_loss/total\n\nprint('Test Loss: {:.6f}\\n'.format(test_loss))\n\nfor i in range(5):\n    if class_total[i] > 0:\n        print('Test Accuracy of %5s: %2d%% (%2d/%2d)' % (\n            str(i), 100 * class_correct[i] / class_total[i],\n            np.sum(class_correct[i]), np.sum(class_total[i])))\n    else:\n        print('Test Accuracy of %5s: N/A (no training examples)' % (classes[i]))\n\nprint('\\nTest Accuracy (Overall): %.3f%% (%2d/%2d)' % (\n    100. * np.sum(class_correct) / np.sum(class_total),\n    np.sum(class_correct), np.sum(class_total)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\n\n# 훈련이 진행되는 과정에 따라 loss를 시각화\nfig = plt.figure(figsize=(10,8))\nplt.plot(range(1,len(avg_train_losses)+1),avg_train_losses, label='Training Loss')\nplt.plot(range(1,len(avg_val_losses)+1),avg_val_losses,label='Validation Loss')\n\n# validation loss의 최저값 지점을 찾기\n#minposs = valid_loss.index(min(valid_loss))+1\n#plt.axvline(minposs, linestyle='--', color='r',label='Early Stopping Checkpoint')\n\nplt.xlabel('epochs')\nplt.ylabel('loss')\n#plt.ylim(0, 0.5) # 일정한 scale\nplt.xlim(0, len(avg_train_losses)+1) # 일정한 scale\nplt.grid(True)\nplt.legend()\nplt.tight_layout()\nplt.show()\nfig.savefig('loss_plot.png', bbox_inches = 'tight')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# TODO: Implement your training loop here.\n# for x, y in data_loader:\n#     x, y = x.cuda(), y.cuda()\n#     logits = model(x)\n#     loss = creterion(logits,)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train(model, epochs, train_loader, test_loader, criterion, \n          optimizer, RESULTS_PATH, scheduler=None, MODEL_PATH=None):\n    # Run on GPU if available\n    device = torch.device(\"cuda:0\")\n    print(device)\n    model.to(device)\n    \n    # Training loop\n    # -------------------------------\n    cols       = ['epoch', 'train_loss', 'train_err', 'test_err']\n    results_df = pd.DataFrame(columns=cols).set_index('epoch')\n    print('Epoch \\tBatch \\tNLLLoss_Train')\n    \n    for epoch in range(epochs):  # loop over the dataset multiple times\n        \n        model.train()\n        running_loss  = 0.0\n        best_test_err = 1.0\n        for i, data in enumerate(train_loader, 0):   # Do a batch iteration\n            \n            # get the inputs\n            inputs, labels = data\n            inputs, labels = inputs.to(device), labels.to(device)\n            \n            # zero the parameter gradients\n            optimizer.zero_grad()\n            \n            # forward + backward + optimize\n            outputs = model(inputs)\n            loss = criterion(outputs, labels)\n            loss.backward()\n            optimizer.step()\n            \n            # print average loss for last 50 mini-batches\n            running_loss += loss.item()\n            if i % 50 == 49:\n                print('%d \\t%d \\t%.3f' %\n                      (epoch + 1, i + 1, running_loss / 50))\n                running_loss = 0.0\n        \n        if scheduler:\n            scheduler.step()\n        \n        # Record metrics\n        model.eval()\n        train_loss = loss.item()\n        train_err = evaluate(model, train_loader, device)\n        test_err = evaluate(model, test_loader, device)\n        results_df.loc[epoch] = [train_loss, train_err, test_err]\n        results_df.to_csv(RESULTS_PATH)\n        print(f'train_err: {train_err} test_err: {test_err}')\n        \n        # Save best model\n        if MODEL_PATH and (test_err < best_test_err):\n            torch.save(model.state_dict(), MODEL_PATH)\n            best_test_err = test_err\n        \n        \n    \n    print('Finished Training')\n    model.eval()\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Save your trained model. Note that it is recommended to save the 'best' checkpoint, rather than just saving the 'last' checkpoint.\ntorch.save(model, 'model.pt')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Verify that your trained model can be loaded"},{"metadata":{"trusted":true},"cell_type":"code","source":"model = torch.load('/kaggle/working/model.pt')\nmodel.eval()\ntest_dataset = CassavaDataset('test', transform=TRANSFORMS['test'])\ntest_loader = DataLoader(test_dataset, batch_size=32)\ntest_csv = pd.read_csv(TEST_CSV)\n\ny_hats = []\nfor x, _ in test_loader:\n    y_hat = model(x.cuda())\n    y_hat = torch.argmax(y_hat,dim=1)\n    y_hats.extend(y_hat.cpu().detach().numpy().tolist())\n    \n\ntest_csv['label'] = y_hats\ntest_csv[['image_id','label']].to_csv(\"submission.csv\", index=False)\ntest_csv.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}