{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# !pip install torch==1.10.1+cu111 torchvision==0.11.2+cu111 torchaudio==0.10.1 -f https://download.pytorch.org/whl/torch_stable.html\n!pip install timm # install pytorch image models\n!pip install torchmetrics","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-25T02:23:23.589968Z","iopub.execute_input":"2022-05-25T02:23:23.590408Z","iopub.status.idle":"2022-05-25T02:23:44.92454Z","shell.execute_reply.started":"2022-05-25T02:23:23.590319Z","shell.execute_reply":"2022-05-25T02:23:44.923723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport os\nimport pandas as pd\nimport numpy as np\nimport random \n\nimport albumentations as A #图像增强库\nimport cv2\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nfrom sklearn import preprocessing\nfrom sklearn.model_selection import StratifiedKFold\nimport timm\n\nimport torchvision\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nimport torchvision.models as models\nimport torch.nn.functional as F\nfrom torch import nn\nimport torchmetrics \nfrom torch.nn.modules.loss import _Loss","metadata":{"execution":{"iopub.status.busy":"2022-05-25T02:23:44.926712Z","iopub.execute_input":"2022-05-25T02:23:44.927031Z","iopub.status.idle":"2022-05-25T02:23:53.544308Z","shell.execute_reply.started":"2022-05-25T02:23:44.92698Z","shell.execute_reply":"2022-05-25T02:23:53.54349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!nvidia-smi","metadata":{"execution":{"iopub.status.busy":"2022-05-25T02:23:53.545946Z","iopub.execute_input":"2022-05-25T02:23:53.546204Z","iopub.status.idle":"2022-05-25T02:23:54.263015Z","shell.execute_reply.started":"2022-05-25T02:23:53.546168Z","shell.execute_reply":"2022-05-25T02:23:54.262133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class GlobalConstantsConfigure():\n    def __init__(self):\n        self.continue_training = False\n        self.last_model = '../input/sorghum-cultivar-identification-models/eff_v2_sgd_50_silu.pt' \n        self.num_epochs_done = 0\n        self.seed = 107\n        self.fold = 1\n        self.num_folds = 4\n        self.num_classes = 100 #一百分类问题\n        self.biggest_loss = 999\n        self.training_size_rate = 0.8\n        self.training_dir = '../input/sorghum-id-fgvc-9/train_images'\n        self.model_name = 'tf_efficientnetv2_m_in21k'#模型名称\n        self.model_path = './efficientnetv2_b5_sgd_50.pt'\n        self.image_size = 512\n        self.batch_size = 8\n        self.val_batch_size = 32\n        self.lr = 3e-5 # 3e-5\n        self.num_epochs = 25\n        self.steps_per_decay = 5\n        self.device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n        self.num_workers = 2  # if torch.cuda.is_available() else 4\ngcc = GlobalConstantsConfigure()","metadata":{"execution":{"iopub.status.busy":"2022-05-25T02:23:54.264461Z","iopub.execute_input":"2022-05-25T02:23:54.264708Z","iopub.status.idle":"2022-05-25T02:23:54.327927Z","shell.execute_reply.started":"2022-05-25T02:23:54.264683Z","shell.execute_reply":"2022-05-25T02:23:54.327056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed) : #设置随机数种子，方便训练周期的复现，确保每一个epoch的卷积的输入输出都是一致的\n    os.environ['PYTHONHASHSEED'] = str(seed) #禁止hash随机化，使得实验可复现\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)#CPU设置生成固定随机数的种子\n    torch.cuda.manual_seed(seed)#GPU设置固定生成随机数的种子\n    torch.backends.cudnn.deterministic = True#表示返回的卷积算法是确定的，配合上面的随机数种子的设置，可以确定每次运行网络的时候相同输入的输出是固定的\n    torch.backends.cudnn.benchmark = True #让内置的 cuDNN 的 auto-tuner 自动寻找最适合当前配置的高效算法，来达到优化运行效率的问题。\nset_seed(gcc.seed)","metadata":{"execution":{"iopub.status.busy":"2022-05-25T02:23:54.331383Z","iopub.execute_input":"2022-05-25T02:23:54.331686Z","iopub.status.idle":"2022-05-25T02:23:54.340371Z","shell.execute_reply.started":"2022-05-25T02:23:54.331648Z","shell.execute_reply":"2022-05-25T02:23:54.339521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all = pd.read_csv('../input/sorghum-id-fgvc-9/train_cultivar_mapping.csv')#数据读取\nprint(len(df_all))\ndf_all.dropna(inplace=True)#，数据预处理：删除数据中空值的数据\nprint(len(df_all))\ndf_all.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-25T02:23:54.342007Z","iopub.execute_input":"2022-05-25T02:23:54.34231Z","iopub.status.idle":"2022-05-25T02:23:54.401637Z","shell.execute_reply.started":"2022-05-25T02:23:54.342273Z","shell.execute_reply":"2022-05-25T02:23:54.400962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_cultivars = list(df_all[\"cultivar\"].unique())","metadata":{"execution":{"iopub.status.busy":"2022-05-25T02:23:54.403117Z","iopub.execute_input":"2022-05-25T02:23:54.403584Z","iopub.status.idle":"2022-05-25T02:23:54.419158Z","shell.execute_reply.started":"2022-05-25T02:23:54.403547Z","shell.execute_reply":"2022-05-25T02:23:54.418441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#数据加载\ndf_all[\"file_path\"] = df_all[\"image\"].apply(lambda image: '../input/sorghum-id-fgvc-9/train_images/' + image)\ndf_all[\"cultivar_index\"] = df_all[\"cultivar\"].map(lambda item: unique_cultivars.index(item))#根据标签从小到大排序\ndf_all[\"is_exist\"] = df_all[\"file_path\"].apply(lambda file_path: os.path.exists(file_path))\ndf_all = df_all[df_all.is_exist==True]#拿到所有file_path为True字段的数据\ndf_all.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-25T02:23:54.420787Z","iopub.execute_input":"2022-05-25T02:23:54.421262Z","iopub.status.idle":"2022-05-25T02:24:31.782369Z","shell.execute_reply.started":"2022-05-25T02:23:54.421226Z","shell.execute_reply":"2022-05-25T02:24:31.78164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=gcc.num_folds, shuffle=True, random_state=gcc.seed)\n#分层随机抽样，验证集中不同类别占比与原始样本的比例保持一致\n\n\ntrain_folds = []\nval_folds = []\n\nfor train_idx, valid_idx in skf.split(df_all['image'], df_all[\"cultivar_index\"]):#训练集与验证集的划分\n    train_folds.append(train_idx)\n    val_folds.append(valid_idx)\n#     df_train = df_all.iloc[train_idx]\n#     df_valid = df_all.iloc[valid_idx]\n\n# print(train_folds)\n# print(val_folds)\ndf_train = df_all.iloc[train_folds[gcc.fold]]\ndf_valid = df_all.iloc[val_folds[gcc.fold]]\n\n\n\n\nprint(f\"train size: {len(df_train)}\")#train size: 16645\nprint(f\"valid size: {len(df_valid)}\")#valid size: 5548\n\nprint(df_train.cultivar.value_counts())#打印训练集各类标签的总数\nprint(df_valid.cultivar.value_counts())","metadata":{"execution":{"iopub.status.busy":"2022-05-25T02:24:31.783502Z","iopub.execute_input":"2022-05-25T02:24:31.78377Z","iopub.status.idle":"2022-05-25T02:24:31.815606Z","shell.execute_reply.started":"2022-05-25T02:24:31.783734Z","shell.execute_reply":"2022-05-25T02:24:31.814783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SorghumDataset(Dataset):#高粱数据类\n    def __init__(self, dirs, labels, transformation=None):\n        super(SorghumDataset,self).__init__()\n        self.dirs = dirs\n        self.labels = labels\n        self.transformation = transformation\n    def __len__(self):\n        return len(self.dirs)\n\n    def __getitem__(self, index):\n        image = cv2.imread(self.dirs[index])\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)#图像二值化，方便后续的预处理操作\n        label = self.labels[index] # need to one hot encoding here独热编码\n        \n        image = np.array(image)\n\n        if self.transformation:\n            aug_image = self.transformation(image=image) #做一些图像增强的操作\n            image = aug_image['image']\n            \n        image = image / 255. #归一化\n        image = image.transpose((2, 0, 1))#索引置换，矩阵置换\n        \n        image = torch.from_numpy(image).type(torch.float32)#把数组转换成张量tensor，且二者共享内存，对张量进行修改比如重新赋值，那么原始数组也会相应发生改变。\n        image = transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))(image) #将这些范围内的数据，转化为-1~1之间\n        \n        labels = torch.from_numpy(np.array(self.labels[index])).type(torch.float32)#label数组转为张量tensor\n\n\n        return image, labels","metadata":{"execution":{"iopub.status.busy":"2022-05-25T02:24:31.817197Z","iopub.execute_input":"2022-05-25T02:24:31.817472Z","iopub.status.idle":"2022-05-25T02:24:31.829834Z","shell.execute_reply.started":"2022-05-25T02:24:31.817436Z","shell.execute_reply":"2022-05-25T02:24:31.829039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_transformation = A.Compose([ #做图像增强\n    A.Resize(width=gcc.image_size, height=gcc.image_size, p=1.0),#图像重定义大小size\n    A.Flip(p=0.5),\n    A.RandomRotate90(p=0.5),#图像随机旋转90°\n    A.ShiftScaleRotate(p=0.5),#改变旋转尺度\n    A.HueSaturationValue(p=0.5),#调整图片色相饱和度\n#     A.OneOf([\n#         A.RandomBrightnessContrast(p=0.5),\n#         A.RandomGamma(p=0.5),\n#     ], p=0.5),\n#     A.OneOf([\n#         A.Blur(p=0.1),\n#         A.GaussianBlur(p=0.1),\n#         A.MotionBlur(p=0.1),\n#     ], p=0.1),\n#     A.OneOf([\n#         A.GaussNoise(p=0.1),\n#         A.ISONoise(p=0.1),\n#         A.GridDropout(ratio=0.5, p=0.2),\n#         A.CoarseDropout(max_holes=16, min_holes=8, max_height=16, max_width=16, min_height=8, min_width=8, p=0.2)\n#     ], p=0.2),\n\n])\nvalidation_transformation = A.Compose([\n    A.Resize(width=gcc.image_size, height=gcc.image_size, p=1.0)\n])","metadata":{"execution":{"iopub.status.busy":"2022-05-25T02:24:31.831342Z","iopub.execute_input":"2022-05-25T02:24:31.831771Z","iopub.status.idle":"2022-05-25T02:24:31.841066Z","shell.execute_reply.started":"2022-05-25T02:24:31.831735Z","shell.execute_reply":"2022-05-25T02:24:31.84024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# training_set = SorghumDataset(training_dirs, training_labels, training_transformation)\n# validation_set = SorghumDataset(validation_dirs, validation_labels, validation_transformation)\n\n#数据划分并初始化\n\ntraining_set = SorghumDataset(df_train.file_path.values, df_train.cultivar_index.values, training_transformation)\nvalidation_set = SorghumDataset(df_valid.file_path.values, df_valid.cultivar_index.values, validation_transformation)\n\n\ntraining_dataloader = DataLoader(\n    training_set,\n    batch_size = gcc.batch_size,\n    shuffle = True,\n    num_workers = gcc.num_workers,\n    pin_memory = True, \n    drop_last = True\n)\nvalidation_dataloader = DataLoader(\n    validation_set,\n    batch_size = gcc.val_batch_size,\n    shuffle = True,\n    num_workers = gcc.num_workers,\n    pin_memory = True,\n    drop_last = True\n)","metadata":{"execution":{"iopub.status.busy":"2022-05-25T02:24:31.842456Z","iopub.execute_input":"2022-05-25T02:24:31.843305Z","iopub.status.idle":"2022-05-25T02:24:31.851542Z","shell.execute_reply.started":"2022-05-25T02:24:31.84327Z","shell.execute_reply":"2022-05-25T02:24:31.850855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomModel(torch.nn.Module):#定义模型 \n    def __init__(self, model_backbone):\n        super(CustomModel,self).__init__()\n        self.model = model_backbone#主干网络，用来做特征提取的网络，代表网络的一部分，一般用于前端提取图片信息，生成特征图feature map，供后面的网络使用。\n        \n        self.model.classifier = nn.Sequential(#网络定义\n            nn.BatchNorm1d(1280),#特征归一化，取1280个特征\n            nn.Linear(1280, 512),#定义全连接层，输出为512\n            nn.Dropout(0.5),#dropout层，比例为0.5，防止过拟合\n            nn.ReLU(inplace=True),#激活函数\n            nn.Linear(512, gcc.num_classes),\n            \n#             nn.BatchNorm1d(1280),\n#             nn.Linear(1280, 512),\n#             nn.Dropout(0.5),\n#             nn.SiLU(inplace=True),\n\n#             nn.Linear(512, 256),\n#             nn.Dropout(0.5),\n#             nn.SiLU(inplace=True),\n#             nn.Linear(256, gcc.num_classes)\n        )\n    def forward(self,x):\n        x = self.model(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-05-25T02:24:31.852833Z","iopub.execute_input":"2022-05-25T02:24:31.853568Z","iopub.status.idle":"2022-05-25T02:24:31.863881Z","shell.execute_reply.started":"2022-05-25T02:24:31.853533Z","shell.execute_reply":"2022-05-25T02:24:31.863158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def to_one_hot(labels, num_classes, dtype=torch.float, dim=1):\n    if labels.ndim < dim + 1:\n        shape = list(labels.shape) + [1] * (dim + 1 - len(labels.shape))\n        labels = torch.reshape(labels, shape)\n    sh = list(labels.shape)\n    sh[dim] = num_classes\n    o = torch.zeros(size=sh, dtype=dtype, device=labels.device)\n    labels = o.scatter_(dim=dim, index=labels.long(), value=1)\n    return labels\n\n\nclass PolyLoss(_Loss):\n    def __init__(self, softmax, ce_weight=None, reduction='mean', epsilon=1.0):\n        super().__init__()\n        self.softmax = softmax\n        self.reduction = reduction\n        self.epsilon = epsilon\n        self.cross_entropy = nn.CrossEntropyLoss(weight=ce_weight, reduction='none')\n\n    def forward(self, input, target):\n\n        if len(input.shape) - len(target.shape) == 1:\n            target = target.unsqueeze(1).long()\n        n_pred_ch, n_target_ch = input.shape[1], target.shape[1]\n        if n_pred_ch != n_target_ch:\n            self.ce_loss = self.cross_entropy(input, torch.squeeze(target, dim=1).long())\n            target = to_one_hot(target, num_classes=n_pred_ch)\n        else:\n            self.ce_loss = self.cross_entropy(input, torch.argmax(target, dim=1))\n\n        if self.softmax:\n            input = torch.softmax(input, 1)\n\n        pt = (input * target).sum(dim=1) \n        \n        poly_loss = self.ce_loss + self.epsilon * (1 - pt)\n\n        polyl = torch.mean(poly_loss)  # the batch and channel average\n        # polyl = torch.sum(poly_loss)  # sum over the batch and channel dims\n        return (polyl)","metadata":{"execution":{"iopub.status.busy":"2022-05-25T02:24:31.867867Z","iopub.execute_input":"2022-05-25T02:24:31.868271Z","iopub.status.idle":"2022-05-25T02:24:31.880337Z","shell.execute_reply.started":"2022-05-25T02:24:31.86824Z","shell.execute_reply":"2022-05-25T02:24:31.879412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# backbone = models.efficientnet_b5(pretrained=True) \nbackbone = timm.create_model(gcc.model_name,pretrained=True)#主干网络\n\n\n# print(index)\nmodel = CustomModel(backbone)\nce = torch.nn.CrossEntropyLoss()\nloss_func = PolyLoss(ce) # torch.nn.CrossEntropyLoss()\nmetrics_acc = torchmetrics.Accuracy(threshold=0.0, num_classes = gcc.num_classes)\nprint(model)\n\n\n# for index, child in enumerate(backbone.children()):\n#     print(index)\n#     if index <= 7:\n#         for param in child.parameters():\n#             param.requires_grad = False\n\n\ntrainable_parameters = [param for param in model.parameters() if param.requires_grad == True]\noptimizer = torch.optim.Adam(trainable_parameters, lr = gcc.lr)\n# optimizer = torch.optim.SGD(trainable_parameters, lr = gcc.lr, momentum = 0.9)\n# optimizer = torch.optim.SGD(trainable_parameters, lr = gcc.lr, momentum=0.9)\n# lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, gcc.steps_per_decay)\nlr_scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=5, T_mult=2)\n\nmodel.to(gcc.device)\n\nif gcc.continue_training == True:\n    # model.load_state_dict(torch.load(gcc.last_model))\n    checkpoint = torch.load(gcc.last_model)\n    \n    model.load_state_dict(checkpoint['model_state_dict'])\n    \n    \n    # optimizer = torch.optim.Adam(trainable_parameters, lr = gcc.lr)\n    # optimizer.load_state_dict(checkpoint['optimizer_state_dict'])\n    \n    # lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, gcc.steps_per_decay)\n    # lr_scheduler.load_state_dict(checkpoint['scheduler_state_dict'])\n    \n    # print(lr_scheduler.state_dict())\n\nprint('load model done')","metadata":{"execution":{"iopub.status.busy":"2022-05-25T02:24:31.88188Z","iopub.execute_input":"2022-05-25T02:24:31.882412Z","iopub.status.idle":"2022-05-25T02:24:51.864953Z","shell.execute_reply.started":"2022-05-25T02:24:31.882375Z","shell.execute_reply":"2022-05-25T02:24:51.864154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def calc_accuracy(pred, true):\n#     # print(pred, true)\n#     true = true.type(torch.int64) # label\n#     pred = F.softmax(pred, dim = 1)\n#     true = torch.zeros(pred.shape[0], pred.shape[1]).scatter_(1, true.unsqueeze(1), 1.)\n#     acc = (true.argmax(-1) == pred.argmax(-1)).float().detach().numpy()\n#     acc = float(acc.sum() / len(acc))\n#     return round(acc, 4)","metadata":{"execution":{"iopub.status.busy":"2022-05-25T02:24:51.866701Z","iopub.execute_input":"2022-05-25T02:24:51.867031Z","iopub.status.idle":"2022-05-25T02:24:51.871189Z","shell.execute_reply.started":"2022-05-25T02:24:51.866976Z","shell.execute_reply":"2022-05-25T02:24:51.870118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def training_progress(training_dataloader, loss_func, scheduler):\n    model.train()\n    training_loss = 0\n    training_acc = 0\n    cnt = 0 \n    print('Learning rate: ',scheduler.get_last_lr())\n    print(scheduler.state_dict())\n    training_loader = tqdm(training_dataloader, desc='Iterating through the training set')\n    for image, label in training_loader:\n        image = image.to(gcc.device)\n        label = label.to(gcc.device)\n        \n        output = model(image)\n        # output.to(gcc.device)\n\n        acc = metrics_acc(output.cpu().argmax(1), label.cpu().int())\n        loss = loss_func(output, label.long())\n        # calculate accuracy here\n\n        training_loss += loss.item()\n        training_acc += acc\n        cnt +=1 \n        \n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        scheduler.step()\n    \n    mean_training_loss = training_loss / cnt\n    mean_training_acc = training_acc / cnt\n    \n    return mean_training_loss, mean_training_acc","metadata":{"execution":{"iopub.status.busy":"2022-05-25T02:24:51.872731Z","iopub.execute_input":"2022-05-25T02:24:51.873251Z","iopub.status.idle":"2022-05-25T02:24:51.883646Z","shell.execute_reply.started":"2022-05-25T02:24:51.873211Z","shell.execute_reply":"2022-05-25T02:24:51.882891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def validation_progress(validation_dataloader, loss_func):\n    model.eval()\n    validation_loss = 0\n    validation_acc = 0\n    cnt = 0 \n    validation_loader = tqdm(validation_dataloader, desc='Iterating through the validation set')\n    with torch.no_grad():\n        for image, label in validation_loader:\n            image = image.to(gcc.device)\n            label = label.to(gcc.device)\n\n            output = model(image)\n            loss = loss_func(output, label.long())\n            # acc = calc_accuracy(output.cpu(), label.cpu())\n            # output.to(gcc.device)\n            acc = metrics_acc(output.cpu().argmax(1), label.cpu().int())\n            # calculate accuracy here\n            validation_loss += loss.item()\n            validation_acc += acc\n            \n            cnt += 1\n\n    mean_validation_loss = validation_loss / cnt\n    mean_validation_acc = validation_acc / cnt\n    return mean_validation_loss, mean_validation_acc","metadata":{"execution":{"iopub.status.busy":"2022-05-25T02:24:51.88527Z","iopub.execute_input":"2022-05-25T02:24:51.885857Z","iopub.status.idle":"2022-05-25T02:24:51.895409Z","shell.execute_reply.started":"2022-05-25T02:24:51.88582Z","shell.execute_reply":"2022-05-25T02:24:51.894506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def training_model(model, training_dataloader, validation_dataloader, loss_func, scheduler):\n    training_losses_history, validation_losses_history = [], []\n    training_acc_history, validation_acc_history = [], []\n    best_loss = gcc.biggest_loss\n    for epoch in range(gcc.num_epochs):\n        \n        training_loss, training_acc = training_progress(training_dataloader, loss_func, scheduler)\n        training_losses_history.append(training_loss)\n        training_acc_history.append(training_acc)\n        \n        validation_loss, validation_acc = validation_progress(validation_dataloader, loss_func)\n        validation_losses_history.append(validation_loss)\n        validation_acc_history.append(validation_acc)\n        \n        if validation_loss <= best_loss: # sussy baka\n            best_loss = validation_loss\n            torch.save({\n                'model_state_dict': model.state_dict(),\n                'optimizer_state_dict': optimizer.state_dict(),\n                'scheduler_state_dict': scheduler.state_dict()\n            }, gcc.model_name + '_best.pt')\n            # torch.save(model.state_dict(), gcc.model_name + '_best.pt')\n        \n        if epoch == gcc.num_epochs - 1: # i believe my timing capability\n            torch.save({\n                'model_state_dict': model.state_dict(),\n                'optimizer_state_dict': optimizer.state_dict(),\n                'scheduler_state_dict': scheduler.state_dict()\n            }, gcc.model_name + '_' + str(gcc.num_epochs_done + gcc.num_epochs) + '_last.pt')\n\n        print(f'Epoch {epoch + 1}/{gcc.num_epochs} | Training_loss : {training_loss:.3f} | Validation_loss : {validation_loss:.3f}' \n             + f' Training_acc : {training_acc:.3f} | Validation_acc : {validation_acc:.3f}'\n             )\n    return training_losses_history, validation_losses_history, training_acc_history, validation_acc_history","metadata":{"execution":{"iopub.status.busy":"2022-05-25T02:24:51.896915Z","iopub.execute_input":"2022-05-25T02:24:51.897855Z","iopub.status.idle":"2022-05-25T02:24:51.90956Z","shell.execute_reply.started":"2022-05-25T02:24:51.897816Z","shell.execute_reply":"2022-05-25T02:24:51.908685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_losses_history, validation_losses_history, training_acc_history, validation_acc_history = training_model(model, training_dataloader, validation_dataloader, loss_func, lr_scheduler)","metadata":{"execution":{"iopub.status.busy":"2022-05-25T02:24:51.912275Z","iopub.execute_input":"2022-05-25T02:24:51.912462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# testing_progress(testing_dataloader, loss_func)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_loss_history(model_name, train_loss_history, val_loss_history, num_epochs):\n    \n    x = np.arange(num_epochs)\n    fig = plt.figure(figsize=(10, 6))\n    plt.plot(x, train_loss_history, label='Train Loss', lw=3)\n    plt.plot(x, val_loss_history, label='Validation Loss', lw=3)\n\n    plt.title(f\"{model_name}\", fontsize=20)\n    plt.legend(fontsize=12)\n    plt.xlabel(\"Epoch\", fontsize=15)\n    plt.ylabel(\"Loss\", fontsize=15)\n\n    plt.show()\n    \nplot_loss_history(gcc.model_name, training_losses_history, validation_losses_history, gcc.num_epochs)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_acc_history(model_name, train_acc_history, val_acc_history, num_epochs):\n    \n    x = np.arange(num_epochs)\n    fig = plt.figure(figsize=(10, 6))\n    plt.plot(x, train_acc_history, label='Training Accuracy', lw=3)\n    plt.plot(x, val_acc_history, label='Validation Accuracy', lw=3)\n\n    plt.title(f\"{model_name}\", fontsize=20)\n    plt.legend(fontsize=12)\n    plt.xlabel(\"Epoch\", fontsize=15)\n    plt.ylabel(\"Accuracy\", fontsize=15)\n\n    plt.show()\n    \nplot_acc_history(gcc.model_name, training_acc_history, validation_acc_history, gcc.num_epochs)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"checkpoint = torch.load(gcc.model_name + '_best.pt')\nmodel.load_state_dict(checkpoint['model_state_dict'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv('../input/sorghum-id-fgvc-9/sample_submission.csv')\nsub.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub[\"filename\"] = sub[\"filename\"].apply(lambda image: '../input/sorghum-id-fgvc-9/test/' + image)\nsub[\"cultivar\"] = 0\nsub.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testing_dataset = SorghumDataset(sub['filename'], sub['cultivar'], validation_transformation)\ntesting_dataloader = DataLoader(testing_dataset, \n                                batch_size=gcc.val_batch_size, \n                                shuffle=False, \n                                num_workers=gcc.num_workers)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# predictions = np.zeros(len(testing_dataloader))\npredictions = []\ncnt = 0 \nwith torch.no_grad():\n    for image, label in tqdm(testing_dataloader):\n        image = image.to(gcc.device)\n        outputs = model(image)\n        # print(outputs)\n        preds = outputs.detach().cpu()\n        predictions.append(preds.argmax(1)) # need optimize here\n        # print(predictions)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tmp = predictions[0]\nfor i in range(len(predictions) - 1):\n    tmp = torch.cat((tmp, predictions[i+1]))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# predictions = label_encoder.inverse_transform(tmp)\npredictions = [unique_cultivars[pred] for pred in tmp]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv('../input/sorghum-id-fgvc-9/sample_submission.csv')\nsub['cultivar'] = predictions\nsub.to_csv('../output/kaggle/working/submission.csv', index=False)\nsub.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}