{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"package_path = '../input/timm-pytorch-image-models/pytorch-image-models-master' #'../input/efficientnet-pytorch-07/efficientnet_pytorch-0.7.0'\nimport sys; sys.path.append(package_path)","execution_count":null,"outputs":[]},{"metadata":{"id":"-V4i71Un9v-0","trusted":true},"cell_type":"code","source":"import os\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torchvision.datasets import ImageFolder\nfrom torchvision.transforms import ToTensor , RandomResizedCrop , Compose\nfrom torch.utils.data import random_split\nfrom torchvision.utils import make_grid\nfrom torch.utils.data.dataloader import DataLoader , Dataset\nimport pandas as pd\nfrom PIL import Image\nfrom tqdm import tqdm\nfrom torchvision import transforms\nfrom sklearn import model_selection, metrics\nimport time\nfrom torch.cuda.amp import autocast, GradScaler\nfrom torch.nn.modules.loss import _WeightedLoss\n%matplotlib inline\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CFG = {\n    'fold_num': 5,\n    'seed': 719,\n    #'model_arch': \"vit_base_patch16_224\",\n    'model_arch': 'tf_efficientnet_b4_ns',\n    #'model_arch': 'vit_base_patch32_384',\n    'img_size': 512,\n    'epochs': 4,\n    'train_bs': 16,\n    'valid_bs': 32,\n    'T_0': 10,\n    'lr': 1e-4,\n    'min_lr': 1e-6,\n    'weight_decay':1e-6,\n    'num_workers': 4,\n    'accum_iter': 2, # suppoprt to do batch accumulation for backprop with effectively larger batch size\n    'verbose_step': 1,\n    'device': 'cuda:0'\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_default_device():\n    \"\"\"Pick GPU if available, else CPU\"\"\"\n    if torch.cuda.is_available():\n        return torch.device('cuda')\n    else:\n        return torch.device('cpu')\ndevice = get_default_device()\ndevice","execution_count":null,"outputs":[]},{"metadata":{"id":"4WZkgY1ouj9O","outputId":"dbdfef63-76fa-442d-a094-2a457bf1a5ab","trusted":true},"cell_type":"code","source":"df = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv')\nlen(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_PATH = \"../input/cassava-leaf-disease-classification\"\nTRAIN_PATH = \"../input/cassava-leaf-disease-classification/train_images/\"\nTEST_PATH = \"../input/cassava-leaf-disease-classification/test_images/\"\nMODEL_PATH = (\n    \"../input/vit-base/jx_vit_base_p16_224-80ecf9dd.pth\"\n)\n\n# model specific global variables\nIMG_SIZE = 224\nBATCH_SIZE = 16\nLR = 2e-05\nGAMMA = 0.7\nN_EPOCHS = 3","execution_count":null,"outputs":[]},{"metadata":{"id":"6tk6hG9XAOjS","trusted":true},"cell_type":"code","source":"import cv2\n\ndef get_img(path):\n    im_bgr = cv2.imread(path)\n    im_rgb = im_bgr[:, :, ::-1]\n    #print(im_rgb)\n    return im_rgb\n    \n# create image augmentations\ntransforms_train = transforms.Compose(\n    [\n        transforms.Resize((IMG_SIZE, IMG_SIZE)),\n        transforms.RandomHorizontalFlip(p=0.3),\n        transforms.RandomVerticalFlip(p=0.3),\n        transforms.RandomResizedCrop(IMG_SIZE),\n        transforms.ToTensor(),\n        transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),\n    ]\n)\n\nfrom albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize\n)\n\nfrom albumentations.pytorch import ToTensorV2\n\ntransforms_train= Compose([\n            RandomResizedCrop(CFG['img_size'], CFG['img_size']),\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            ShiftScaleRotate(p=0.5),\n            HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n            RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            CoarseDropout(p=0.5),\n            Cutout(p=0.5),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n  \n        \ntransforms_valid = Compose([\n            CenterCrop(CFG['img_size'], CFG['img_size'], p=1.),\n            Resize(CFG['img_size'], CFG['img_size']),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n\ntransforms_test =  Compose([\n            RandomResizedCrop(CFG['img_size'], CFG['img_size']),\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n            RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n\nclass CassavaDataset(Dataset):\n    def __init__(\n        self, df, data_root, transforms=None, output_label=True\n    ):\n        \n        super().__init__()\n        self.df = df.reset_index(drop=True).copy()\n        self.transforms = transforms\n        self.data_root = data_root\n        self.output_label = output_label\n    \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, index: int):\n        \n        # get labels\n        if self.output_label:\n            target = self.df.iloc[index]['label']\n          \n        path = \"{}/{}\".format(self.data_root, self.df.iloc[index]['image_id'])\n        img = get_img(path)\n        #img = Image.open(path).convert(\"RGB\")\n        if self.transforms is not None:\n            img = self.transforms(image=img)['image']\n            #img = data_transforms(Image.fromarray(img))\n            #img = self.transforms(img)\n\n        # do label smoothing\n        if self.output_label == True:\n            return img, target\n        else:\n            return img\n\n\ndatapath = '../input/cassava-leaf-disease-classification/train_images/'\ndf_ds = CassavaDataset(df, datapath, transforms=transforms_train, output_label=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#!pip install timm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import timm\nprint(\"Available Vision Transformer Models: \")\ntimm.list_models(\"vit*\")\n#timm.list_models(\"vo*\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ViTBase16(nn.Module):\n    def __init__(self, n_classes, pretrained=False):\n\n        super(ViTBase16, self).__init__()\n\n        self.model = timm.create_model(CFG['model_arch'], pretrained=pretrained)\n        #if pretrained:\n        #   self.model = timm.create_model(CFG['model_arch'], pretrained=True)\n            #self.model.load_state_dict(torch.load(MODEL_PATH))\n        #self.model.head = nn.Linear(self.model.head.in_features, n_classes)\n        self.model.classifier = nn.Linear(self.model.classifier.in_features, n_classes)\n\n    def forward(self, x):\n        x = self.model(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#model = ViTBase16(n_classes=5, pretrained=True).to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.label.value_counts().plot(kind=\"bar\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df, valid_df = model_selection.train_test_split(\n    df, test_size=0.1, random_state=42, stratify=df.label.values\n)\ntrain_dataset = CassavaDataset(train_df, datapath, transforms=transforms_train, output_label=True)\nvalid_dataset = CassavaDataset(valid_df, datapath, transforms=transforms_valid, output_label=True)","execution_count":null,"outputs":[]},{"metadata":{"id":"jRUismUhARZ3","outputId":"fef082b5-9255-429c-8a8e-890631dfb56e","trusted":true},"cell_type":"code","source":"test = pd.DataFrame()\ntest['image_id'] = list(os.listdir('../input/cassava-leaf-disease-classification/test_images/'))\ntest_dataset = CassavaDataset(test, '../input/cassava-leaf-disease-classification/test_images/', transforms=transforms_test, output_label=False)\nlen(test_dataset)","execution_count":null,"outputs":[]},{"metadata":{"id":"fMKpSW7OyVFP","outputId":"c3d412f4-5959-4d82-a600-2b310f833719","trusted":true},"cell_type":"code","source":"img , label = train_dataset[56]\nprint(tuple(img.shape))","execution_count":null,"outputs":[]},{"metadata":{"id":"Q6grgy9jASNp","outputId":"90d01a82-80e4-454d-c767-10a1573d1bc1","trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport numpy as np\nindex = 7989\nplt.imshow(np.transpose(train_dataset[index][0].numpy(), (1, 2, 0)))\nprint(\"label : \",train_dataset[index][1])","execution_count":null,"outputs":[]},{"metadata":{"id":"UnRqwWQJAYx4","outputId":"b3401f5c-521e-4288-c4d3-7d86deed2610","trusted":true},"cell_type":"code","source":"#dataset = train_ds\n#val_size = int(0.7*len(dataset))\n#train_size = len(dataset) - val_size\n\n#train_ds, val_ds = random_split(dataset, [train_size, val_size])\n#len(train_ds), len(val_ds)\n\n\n#dataset = train_ds\n#val_size = int(0.5*len(dataset))\n#train_size = len(dataset) - val_size\n\n#train_ds, val_ds = random_split(dataset, [train_size, val_size])\n#len(train_ds), len(val_ds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def prepare_dataloader(df, trn_idx, val_idx, data_root='../input/cassava-leaf-disease-classification/train_images/'):\n    \n    from catalyst.data.sampler import BalanceClassSampler\n    \n    train_ = df.loc[trn_idx,:].reset_index(drop=True)\n    valid_ = df.loc[val_idx,:].reset_index(drop=True)\n        \n    train_ds = CassavaDataset(train_, data_root, transforms=get_train_transforms(), output_label=True, one_hot_label=False, do_fmix=False, do_cutmix=False)\n    valid_ds = CassavaDataset(valid_, data_root, transforms=get_valid_transforms(), output_label=True)\n    \n    train_loader = torch.utils.data.DataLoader(\n        train_ds,\n        batch_size=CFG['train_bs'],\n        pin_memory=False,\n        drop_last=False,\n        shuffle=True,        \n        num_workers=CFG['num_workers'],\n        #sampler=BalanceClassSampler(labels=train_['label'].values, mode=\"downsampling\")\n    )\n    val_loader = torch.utils.data.DataLoader(\n        valid_ds, \n        batch_size=CFG['valid_bs'],\n        num_workers=CFG['num_workers'],\n        shuffle=False,\n        pin_memory=False,\n    )\n    return train_loader, val_loader\n\ndef train_one_epoch(epoch, model, loss_fn, optimizer, train_loader, device, scheduler=None, schd_batch_update=False):\n    model.train()\n\n    t = time.time()\n    running_loss = None\n\n    pbar = tqdm(enumerate(train_loader), total=len(train_loader))\n    for step, (imgs, image_labels) in pbar:\n        imgs = imgs.to(device).float()\n        image_labels = image_labels.to(device).long()\n\n        #print(image_labels.shape, exam_label.shape)\n        with autocast():\n            image_preds = model(imgs)   #output = model(input)\n            #print(image_preds.shape, exam_pred.shape)\n\n            loss = loss_fn(image_preds, image_labels)\n            \n            scaler.scale(loss).backward()\n\n            if running_loss is None:\n                running_loss = loss.item()\n            else:\n                running_loss = running_loss * .99 + loss.item() * .01\n\n            if ((step + 1) %  CFG['accum_iter'] == 0) or ((step + 1) == len(train_loader)):\n                # may unscale_ here if desired (e.g., to allow clipping unscaled gradients)\n\n                scaler.step(optimizer)\n                scaler.update()\n                optimizer.zero_grad() \n                \n                if scheduler is not None and schd_batch_update:\n                    scheduler.step()\n\n            if ((step + 1) % CFG['verbose_step'] == 0) or ((step + 1) == len(train_loader)):\n                description = f'epoch {epoch} loss: {running_loss:.4f}'\n                \n                pbar.set_description(description)\n                \n    if scheduler is not None and not schd_batch_update:\n        scheduler.step()\n        \ndef valid_one_epoch(epoch, model, loss_fn, val_loader, device, scheduler=None, schd_loss_update=False):\n    model.eval()\n\n    t = time.time()\n    loss_sum = 0\n    sample_num = 0\n    image_preds_all = []\n    image_targets_all = []\n    \n    pbar = tqdm(enumerate(val_loader), total=len(val_loader))\n    for step, (imgs, image_labels) in pbar:\n        imgs = imgs.to(device).float()\n        image_labels = image_labels.to(device).long()\n        \n        image_preds = model(imgs)   #output = model(input)\n        #print(image_preds.shape, exam_pred.shape)\n        image_preds_all += [torch.argmax(image_preds, 1).detach().cpu().numpy()]\n        image_targets_all += [image_labels.detach().cpu().numpy()]\n        \n        loss = loss_fn(image_preds, image_labels)\n        \n        loss_sum += loss.item()*image_labels.shape[0]\n        sample_num += image_labels.shape[0]  \n\n        if ((step + 1) % CFG['verbose_step'] == 0) or ((step + 1) == len(val_loader)):\n            description = f'epoch {epoch} loss: {loss_sum/sample_num:.4f}'\n            pbar.set_description(description)\n    \n    image_preds_all = np.concatenate(image_preds_all)\n    image_targets_all = np.concatenate(image_targets_all)\n    print('validation multi-class accuracy = {:.4f}'.format((image_preds_all==image_targets_all).mean()))\n    \n    if scheduler is not None:\n        if schd_loss_update:\n            scheduler.step(loss_sum/sample_num)\n        else:\n            scheduler.step()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MyCrossEntropyLoss(_WeightedLoss):\n    def __init__(self, weight=None, reduction='mean'):\n        super().__init__(weight=weight, reduction=reduction)\n        self.weight = weight\n        self.reduction = reduction\n\n    def forward(self, inputs, targets):\n        lsm = F.log_softmax(inputs, -1)\n\n        if self.weight is not None:\n            lsm = lsm * self.weight.unsqueeze(0)\n\n        loss = -(targets * lsm).sum(-1)\n\n        if  self.reduction == 'sum':\n            loss = loss.sum()\n        elif  self.reduction == 'mean':\n            loss = loss.mean()\n\n        return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train_loader, val_loader = prepare_dataloader(train, trn_idx, val_idx, data_root='../input/cassava-leaf-disease-classification/train_images/')\n#train_dataset = CassavaDataset(df, datapath, transforms=transforms_train, output_label=True)\n\ntrain_loader = torch.utils.data.DataLoader(\n        train_dataset,\n        batch_size=CFG['train_bs'],\n        pin_memory=False,\n        drop_last=False,\n        shuffle=True,        \n        num_workers=CFG['num_workers'],\n        #sampler=BalanceClassSampler(labels=train_['label'].values, mode=\"downsampling\")\n    )\nval_loader = torch.utils.data.DataLoader(\n        valid_dataset, \n        batch_size=CFG['valid_bs'],\n        num_workers=CFG['num_workers'],\n        shuffle=False,\n        pin_memory=False,\n    )\n#optimizer = torch.optim.Adam(model.parameters(), lr=LR)\n\n#optimizer = torch.optim.Adam(model.parameters(), lr=CFG['lr'], weight_decay=CFG['weight_decay'])\n#scheduler = torch.optim.lr_scheduler.StepLR(optimizer, gamma=GAMMA,step_size=N_EPOCHS-1)\n\n#scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=CFG['T_0'], T_mult=1, eta_min=CFG['min_lr'], last_epoch=-1)\n#scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=CFG['T_0'], T_mult=1, eta_min=CFG['min_lr'], last_epoch=-1)\n\n#loss_tr = nn.CrossEntropyLoss().to(device) #MyCrossEntropyLoss().to(device)\n#loss_fn = nn.CrossEntropyLoss().to(device)\nscaler = GradScaler()   \n#for epoch in range(N_EPOCHS):\n#            train_one_epoch(epoch, model, loss_tr, optimizer, train_loader, device, scheduler=scheduler, schd_batch_update=False)\n\n            #with torch.no_grad():\n            #    valid_one_epoch(epoch, model, loss_fn, val_loader, device, scheduler=None, schd_loss_update=False)\n\n#            torch.save(model.state_dict(),'{}_epoch_{}'.format(CFG['model_arch'], epoch))\n\n            \n\n#del model, optimizer, train_loader, val_loader, scaler, scheduler\n#torch.cuda.empty_cache()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = ViTBase16(n_classes=5, pretrained=False)\nepoch = 2\nmodel.load_state_dict(torch.load('../input/vit-base/{}_epoch_{}'.format(CFG['model_arch'], epoch)))\nmodel = model.to(device)\n\ntest_loader = torch.utils.data.DataLoader(\n        test_dataset, \n        batch_size=CFG['valid_bs'],\n        num_workers=CFG['num_workers'],\n        shuffle=False,\n        pin_memory=False,\n    )\n\nwith torch.no_grad():\n    model.eval()\n\n    t = time.time()\n    loss_sum = 0\n    sample_num = 0\n    image_preds_all = []\n    image_targets_all = []\n    \n    pbar = tqdm(enumerate(test_loader), total=len(test_loader))\n    for step, imgs in pbar:\n        imgs = imgs.to(device).float()\n        \n        image_preds = model(imgs)   #output = model(input)\n        #print(image_preds.shape, exam_pred.shape)\n        image_preds_all += [torch.argmax(image_preds, 1).detach().cpu().numpy()]\n\n    image_preds_all = np.concatenate(image_preds_all)    \n\ntest['label'] = image_preds_all\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#vit_*_patch16_224\n#0.8621 large (epoch 2) (0.8467, 0.8379 , 0.8621 , 0.8579 , 0.8509 ,0.8561)\n#base (0.8318,0.8449,0.8519,0.8537,0.8430,0.8547,0.8561,0.8551,0.8421,0.8542)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#'lr': 1e-4   vit_base_patch32_384\n'''\n\nepoch 0 loss: 0.5096: 100%|██████████| 1204/1204 [06:01<00:00,  3.33it/s]\nepoch 0 loss: 0.4842: 100%|██████████| 67/67 [00:24<00:00,  2.69it/s]\n\nvalidation multi-class accuracy = 0.8355\n\n\nepoch 1 loss: 0.4736: 100%|██████████| 1204/1204 [06:00<00:00,  3.34it/s]\nepoch 1 loss: 0.4327: 100%|██████████| 67/67 [00:24<00:00,  2.78it/s]\n\nvalidation multi-class accuracy = 0.8575\n\n\nepoch 2 loss: 0.4539: 100%|██████████| 1204/1204 [06:00<00:00,  3.34it/s]\nepoch 2 loss: 0.4418: 100%|██████████| 67/67 [00:24<00:00,  2.77it/s]\n\nvalidation multi-class accuracy = 0.8463\n\n\nepoch 3 loss: 0.4201: 100%|██████████| 1204/1204 [06:01<00:00,  3.33it/s]\nepoch 3 loss: 0.4239: 100%|██████████| 67/67 [00:24<00:00,  2.76it/s]\n\nvalidation multi-class accuracy = 0.8505\n\n\nepoch 4 loss: 0.4262: 100%|██████████| 1204/1204 [06:01<00:00,  3.33it/s]\nepoch 4 loss: 0.4455: 100%|██████████| 67/67 [00:24<00:00,  2.76it/s]\n\nvalidation multi-class accuracy = 0.8453\n\n\nepoch 5 loss: 0.4044: 100%|██████████| 1204/1204 [06:00<00:00,  3.34it/s]\nepoch 5 loss: 0.4322: 100%|██████████| 67/67 [00:24<00:00,  2.77it/s]\n\nvalidation multi-class accuracy = 0.8514\n\n\nepoch 6 loss: 0.3910: 100%|██████████| 1204/1204 [06:01<00:00,  3.33it/s]\nepoch 6 loss: 0.4282: 100%|██████████| 67/67 [00:24<00:00,  2.79it/s]\n\nvalidation multi-class accuracy = 0.8598\n\n\nepoch 7 loss: 0.3573: 100%|██████████| 1204/1204 [06:00<00:00,  3.34it/s]\nepoch 7 loss: 0.4278: 100%|██████████| 67/67 [00:23<00:00,  2.87it/s]\n\nvalidation multi-class accuracy = 0.8612\n\n\nepoch 8 loss: 0.3483: 100%|██████████| 1204/1204 [06:00<00:00,  3.34it/s]\nepoch 8 loss: 0.4324: 100%|██████████| 67/67 [00:24<00:00,  2.76it/s]\n\nvalidation multi-class accuracy = 0.8519\n\n\nepoch 9 loss: 0.3072: 100%|██████████| 1204/1204 [06:01<00:00,  3.33it/s]\nepoch 9 loss: 0.4389: 100%|██████████| 67/67 [00:24<00:00,  2.79it/s]\n\nvalidation multi-class accuracy = 0.8621\n\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#'lr': 1e-4  \n\n'''\nepoch 0 loss: 0.4371: 100%|██████████| 1204/1204 [14:30<00:00,  1.38it/s]\nepoch 0 loss: 0.3726: 100%|██████████| 67/67 [00:36<00:00,  1.84it/s]\n\nvalidation multi-class accuracy = 0.8757\n\n\nepoch 1 loss: 0.4025: 100%|██████████| 1204/1204 [14:30<00:00,  1.38it/s]\nepoch 1 loss: 0.3566: 100%|██████████| 67/67 [00:35<00:00,  1.90it/s]\n\nvalidation multi-class accuracy = 0.8780\n\n\nepoch 2 loss: 0.3661: 100%|██████████| 1204/1204 [14:29<00:00,  1.38it/s]\nepoch 2 loss: 0.3388: 100%|██████████| 67/67 [00:34<00:00,  1.92it/s]\n\nvalidation multi-class accuracy = 0.8911\n\n\nepoch 3 loss: 0.3390: 100%|██████████| 1204/1204 [14:29<00:00,  1.38it/s]\nepoch 3 loss: 0.3506: 100%|██████████| 67/67 [00:35<00:00,  1.91it/s]\n\nvalidation multi-class accuracy = 0.8771\n\n\nepoch 4 loss: 0.3354: 100%|██████████| 1204/1204 [14:30<00:00,  1.38it/s]\nepoch 4 loss: 0.3348: 100%|██████████| 67/67 [00:35<00:00,  1.87it/s]\n\nvalidation multi-class accuracy = 0.8883\n\n\nepoch 5 loss: 0.3473: 100%|██████████| 1204/1204 [14:30<00:00,  1.38it/s]\nepoch 5 loss: 0.3494: 100%|██████████| 67/67 [00:35<00:00,  1.90it/s]\n\nvalidation multi-class accuracy = 0.8799\n\n\nepoch 6 loss: 0.3234: 100%|██████████| 1204/1204 [14:29<00:00,  1.38it/s]\nepoch 6 loss: 0.3438: 100%|██████████| 67/67 [00:35<00:00,  1.90it/s]\n\nvalidation multi-class accuracy = 0.8841\n\n\nepoch 7 loss: 0.2839: 100%|██████████| 1204/1204 [14:30<00:00,  1.38it/s]\nepoch 7 loss: 0.3568: 100%|██████████| 67/67 [00:35<00:00,  1.90it/s]\n\nvalidation multi-class accuracy = 0.8748\n\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#'lr': 2e-5\n\n'''\nepoch 0 loss: 0.5750: 100%|██████████| 1204/1204 [14:33<00:00,  1.38it/s]\nepoch 0 loss: 0.5037: 100%|██████████| 67/67 [00:36<00:00,  1.84it/s]\n\nvalidation multi-class accuracy = 0.8201\n\n\nepoch 1 loss: 0.4907: 100%|██████████| 1204/1204 [14:27<00:00,  1.39it/s]\nepoch 1 loss: 0.4051: 100%|██████████| 67/67 [00:35<00:00,  1.88it/s]\n\nvalidation multi-class accuracy = 0.8621\n\n\nepoch 2 loss: 0.4327: 100%|██████████| 1204/1204 [14:28<00:00,  1.39it/s]\nepoch 2 loss: 0.3753: 100%|██████████| 67/67 [00:35<00:00,  1.89it/s]\n\nvalidation multi-class accuracy = 0.8762\n\n\nepoch 3 loss: 0.4160: 100%|██████████| 1204/1204 [14:27<00:00,  1.39it/s]\nepoch 3 loss: 0.3637: 100%|██████████| 67/67 [00:35<00:00,  1.90it/s]\n\nvalidation multi-class accuracy = 0.8813\n\n\nepoch 4 loss: 0.3774: 100%|██████████| 1204/1204 [14:27<00:00,  1.39it/s]\nepoch 4 loss: 0.3495: 100%|██████████| 67/67 [00:35<00:00,  1.90it/s]\n\nvalidation multi-class accuracy = 0.8883\n\n\nepoch 5 loss: 0.3772: 100%|██████████| 1204/1204 [14:27<00:00,  1.39it/s]\nepoch 5 loss: 0.3554: 100%|██████████| 67/67 [00:34<00:00,  1.91it/s]\n\nvalidation multi-class accuracy = 0.8822\n\n\nepoch 6 loss: 0.3493: 100%|██████████| 1204/1204 [14:28<00:00,  1.39it/s]\nepoch 6 loss: 0.3519: 100%|██████████| 67/67 [00:36<00:00,  1.84it/s]\n\nvalidation multi-class accuracy = 0.8808\n\n\nepoch 7 loss: 0.3564: 100%|██████████| 1204/1204 [14:28<00:00,  1.39it/s]\nepoch 7 loss: 0.3431: 100%|██████████| 67/67 [00:35<00:00,  1.89it/s]\n\nvalidation multi-class accuracy = 0.8879\n\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}